pmg-log-tracker.c

   1 /*
   2
   3   (C) 2007-2017 Proxmox Server Solutions GmbH, All Rights Reserved
   4
   5   Proxmox Mail Tracker
   6
   7   This program is free software: you can redistribute it and/or modify
   8   it under the terms of the GNU Affero General Public License as published by
   9   the Free Software Foundation, either version 3 of the License, or
  10   (at your option) any later version.
  11
  12   This program is distributed in the hope that it will be useful,
  13   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15   GNU Affero General Public License for more details.
  16
  17   You should have received a copy of the GNU Affero General Public License
  18   along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20   Author: Dietmar Maurer <dietmar@proxmox.com>
  21
  22   See http://www.proxmox.com for more information
  23
  24 */
  25
  26 #define _GNU_SOURCE
  27
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30 #include <glib.h>
  31 #include <string.h>
  32 #include <ctype.h>
  33 #include <sys/time.h>
  34 #include <time.h>
  35 #include <unistd.h>
  36 #include <sys/types.h>
  37 #include <stdint.h>
  38 #include <zlib.h>
  39 #include <fnmatch.h>
  40
  41 /*
  42   We assume the syslog files belong to one host, i.e. we do not
  43   consider the hostname at all
  44 */
  45
  46 /*
  47   REGEX: we do not use posix regular expressions (libc implementation
  48   is too slow). fnmatch() is also slow.
  49   Future versions may use our own implementation of a DFA. But currently we
  50   just use strstr (no regex support at all)
  51 */
  52
  53 //#define STRMATCH(pattern, string) (fnmatch (pattern, string, FNM_CASEFOLD) == 0)
  54 #define STRMATCH(pattern, string) (strcasestr (string, pattern) != NULL)
  55
  56
  57 //#define EPOOL_BLOCK_SIZE 512
  58 //#define EPOOL_MAX_SIZE 128
  59 #define EPOOL_BLOCK_SIZE 2048
  60 #define EPOOL_MAX_SIZE 128
  61 #define MAX_LOGFILES 32
  62 //#define EPOOL_DEBUG
  63 //#define DEBUG
  64
  65 typedef struct _SList SList;
  66 struct _SList {
  67   gpointer data;
  68   SList *next;
  69 };
  70
  71 typedef struct _NQList NQList;
  72 struct _NQList {
  73   char *from;
  74   char *to;
  75   char dstatus;
  76   time_t ltime;
  77   NQList *next;
  78 };
  79
  80 typedef struct _TOList TOList;
  81 struct _TOList {
  82   char *to;
  83   char *relay;
  84   char dstatus;
  85   time_t ltime;
  86   TOList *next;
  87 };
  88
  89 #define MatchTypeQID 1
  90 #define MatchTypeRelLineNr 2
  91
  92 typedef struct _MatchList MatchList;
  93 struct _MatchList {
  94   unsigned int mtype;
  95   char *id;
  96   time_t ltime;
  97   unsigned long rel_line_nr;
  98   MatchList *next;
  99 };
 100
 101 #ifdef DEBUG
 102   GHashTable *smtpd_debug_alloc;
 103   GHashTable *qmgr_debug_alloc;
 104   GHashTable *filter_debug_alloc;
 105   GHashTable *smtpd_debug_free;
 106   GHashTable *qmgr_debug_free;
 107   GHashTable *filter_debug_free;
 108 #endif
 109
 110 // EPool: Entry related memory pools
 111
 112
 113 #ifdef EPOOL_DEBUG
 114 int ep_allocated;
 115 int ep_maxalloc;
 116 #endif
 117
 118 typedef struct _EPool EPool;
 119 struct _EPool {
 120   SList *blocks;     // allocated usiing g_slice_alloc(EPOOL_BLOCK_SIZE)
 121   SList *mblocks;    // allocated use g_malloc
 122   int cpos;
 123 #ifdef EPOOL_DEBUG
 124   int allocated;
 125 #endif
 126 };
 127
 128 typedef struct {
 129   EPool ep;
 130   GHashTable *smtpd_h;
 131   GHashTable *qmgr_h;
 132   GHashTable *filter_h;
 133   //GHashTable *track_h;
 134   gzFile stream[MAX_LOGFILES];
 135   char *from;
 136   char *to;
 137   time_t year[MAX_LOGFILES];
 138   time_t start;
 139   time_t end;
 140   time_t ctime;
 141   MatchList *match_list;
 142   char *server;
 143   char *msgid;
 144   char *strmatch;
 145   unsigned long limit;
 146   unsigned long count;
 147   int verbose;
 148   unsigned int exclude_greylist:1;
 149   unsigned int exclude_ndrs:1;
 150 } LParser;
 151
 152 typedef struct _SEntry SEntry;
 153 typedef struct _QEntry QEntry;
 154 typedef struct _FEntry FEntry;
 155
 156 typedef struct _LogEntry LogEntry;
 157 typedef struct _LogList LogList;
 158
 159 struct _LogEntry {
 160   const char *text;
 161   unsigned long linenr;
 162   LogEntry *next;
 163 };
 164
 165 struct _LogList {
 166   LogEntry *log;
 167   LogEntry *logs_last; // pointer to last log (speedup add log)
 168 };
 169
 170 // SEntry: Store SMTPD related logs
 171
 172 struct _SEntry {
 173
 174   EPool ep;
 175   LogList loglist;
 176
 177   int pid;
 178
 179   SList *refs;
 180
 181   NQList *nqlist;
 182
 183   char *connect;
 184
 185  // time,rel_line_nr is used as cursor/ID
 186   time_t ltime;
 187   unsigned long rel_line_nr;
 188
 189   //unsigned int external:1;        // not from local host
 190   unsigned int disconnect:1;
 191   unsigned int strmatch:1;
 192
 193 };
 194
 195 // QEntry: Store Queue (qmgr, smtp, lmtp) related logs
 196
 197 struct _QEntry {
 198
 199   EPool ep;
 200   LogList loglist;
 201
 202   char *qid;
 203
 204   SEntry *smtpd;
 205   FEntry *filter;
 206
 207   TOList *tolist;
 208
 209   unsigned int size;
 210   char *from;
 211   char *client;
 212   char *msgid;
 213
 214   unsigned int cleanup:1;
 215   unsigned int removed:1;
 216   unsigned int filtered:1; // set when passed via lmtp to filter
 217   unsigned int strmatch:1;
 218 };
 219
 220 // FEntry: Store filter (proxprox) related logs
 221
 222 struct _FEntry {
 223
 224   EPool ep;
 225   LogList loglist;
 226
 227   char *logid; // proxprox log id
 228
 229   TOList *tolist;
 230
 231   float ptime;
 232
 233   unsigned int finished:1;
 234   unsigned int strmatch:1;
 235 };
 236
 237 // Prototypes
 238 void      debug_error (char *msg, const char *line);
 239
 240 EPool    *epool_init (EPool *ep);
 241 void      epool_free (EPool *ep);
 242 gpointer  epool_alloc (EPool *ep, int size);
 243 char     *epool_strndup (EPool *ep, const char *s, int len);
 244 char     *epool_strndup0 (EPool *ep, const char *s, int len);
 245 char     *epool_strdup (EPool *ep, const char *s);
 246
 247 void      loglist_print (LogList *loglist);
 248 void      loglist_add (EPool *ep, LogList *loglist, const char *text, int len, unsigned long linenr);
 249
 250 SEntry   *sentry_new (int pid, time_t ltime, unsigned long rel_line_nr);
 251 SEntry   *sentry_get (LParser *parser, int pid, time_t ltime, unsigned long rel_line_nr);
 252 void      sentry_ref_add (SEntry *sentry, QEntry *qentry);
 253 int       sentry_ref_del (SEntry *sentry, QEntry *qentry);
 254 void      sentry_ref_finalize (LParser *parser, SEntry *sentry);
 255 int       sentry_ref_rem_unneeded (LParser *parser, SEntry *sentry);
 256 void      sentry_nqlist_add (SEntry *sentry, time_t ltime, const char *from, int from_len,
 257                              const char *to, int to_len, char dstatus);
 258 void      sentry_print (LParser *parser, SEntry *sentry);
 259 void      sentry_set_connect (SEntry *sentry, const char *connect, int len);
 260 void      sentry_free_noremove (SEntry *sentry);
 261 void      sentry_free (LParser *parser, SEntry *sentry);
 262 void      sentry_cleanup_hash (gpointer key, gpointer value, gpointer user_data);
 263
 264
 265 QEntry   *qentry_new (const char *qid);
 266 QEntry   *qentry_get (LParser *parser, const char *qid);
 267 void      qentry_tolist_add (QEntry *qentry, time_t ltime, char dstatus, const char *to,
 268                              int to_len, const char *relay, int relay_len);
 269 void      qentry_set_from (QEntry *qentry, const char *from, int len);
 270 void      qentry_set_msgid (QEntry *qentry, const char *msgid, int len);
 271 void      qentry_set_client (QEntry *qentry, const char *client, int len);
 272 void      qentry_print (LParser *parser, QEntry *qentry);
 273 void      qentry_finalize (LParser *parser, QEntry *qentry);
 274 void      qentry_free_noremove (LParser *parser, QEntry *qentry);
 275 void      qentry_free (LParser *parser, QEntry *qentry);
 276 void      qentry_cleanup_hash (gpointer key, gpointer value, gpointer user_data);
 277
 278
 279 FEntry   *fentry_new (const char *logid);
 280 FEntry   *fentry_get (LParser *parser, const char *logid);
 281 void      fentry_tolist_add (FEntry *fentry, char dstatus, const char *to,
 282                              int to_len, const char *qid, int qid_len);
 283 void      fentry_free_noremove (LParser *parser, FEntry *fentry);
 284 void      fentry_free (LParser *parser, FEntry *fentry);
 285 void      fentry_cleanup_hash (gpointer key, gpointer value, gpointer user_data);
 286
 287 LParser  *parser_new ();
 288 void      parser_free (LParser *parser);
 289
 290 //char     *parser_track (LParser *parser, const char *qid, gboolean insert);
 291
 292 // Implementations
 293
 294 // Checksum Macros
 295 #define PROXPROX                0xE0E4DEF0
 296 #define PMG_SMTP_FILTER         0x0A85A6B7
 297 #define POSTFIX_POSTSCREEN      0xD17E2019
 298 #define POSTFIX_QMGR            0x48465316
 299 #define POSTFIX_SMTP            0x4A466014
 300 #define POSTFIX_LMTP            0x43466014
 301 #define POSTFIX_LOCAL           0x484F05AF
 302 #define POSTFIX_ERROR           0x4B5E13AE
 303 #define POSTFIX_SMTPD           0x466014AE
 304 #define POSTFIX_CLEANUP         0x05A8BAC1
 305
 306 //#define LOGPATH "./log/"
 307 #define LOGPATH "/var/log/"
 308 //#define LOGPATH "/root/testlog/"
 309
 310 static const char *logfiles[] = {
 311   LOGPATH "syslog",
 312   LOGPATH "syslog.1",
 313   LOGPATH "syslog.2.gz",
 314   LOGPATH "syslog.3.gz",
 315   LOGPATH "syslog.4.gz",
 316   LOGPATH "syslog.5.gz",
 317   LOGPATH "syslog.6.gz",
 318   LOGPATH "syslog.7.gz",
 319   LOGPATH "syslog.8.gz",
 320   LOGPATH "syslog.9.gz",
 321   LOGPATH "syslog.10.gz",
 322   LOGPATH "syslog.11.gz",
 323   LOGPATH "syslog.12.gz",
 324   LOGPATH "syslog.13.gz",
 325   LOGPATH "syslog.14.gz",
 326   LOGPATH "syslog.15.gz",
 327   LOGPATH "syslog.16.gz",
 328   LOGPATH "syslog.17.gz",
 329   LOGPATH "syslog.18.gz",
 330   LOGPATH "syslog.19.gz",
 331   LOGPATH "syslog.20.gz",
 332   LOGPATH "syslog.21.gz",
 333   LOGPATH "syslog.22.gz",
 334   LOGPATH "syslog.23.gz",
 335   LOGPATH "syslog.24.gz",
 336   LOGPATH "syslog.25.gz",
 337   LOGPATH "syslog.26.gz",
 338   LOGPATH "syslog.27.gz",
 339   LOGPATH "syslog.28.gz",
 340   LOGPATH "syslog.29.gz",
 341   LOGPATH "syslog.30.gz",
 342   LOGPATH "syslog.31.gz",
 343 };
 344
 345 void
 346 debug_error (char *msg, const char *line)
 347 {
 348 #ifdef DEBUG
 349   fprintf (stderr, "ERROR: %s\n", msg);
 350   if (line) fprintf (stderr, "LINE: %s\n", line);
 351
 352   G_BREAKPOINT();
 353
 354   exit (-1);
 355 #endif
 356 }
 357
 358 EPool *
 359 epool_init (EPool *ep)
 360 {
 361   gpointer data;
 362   SList *blocks;
 363
 364   data = g_slice_alloc0(EPOOL_BLOCK_SIZE);
 365   blocks = (SList *)data;
 366
 367 #ifdef EPOOL_DEBUG
 368   ep->allocated += EPOOL_BLOCK_SIZE;
 369   ep_allocated += EPOOL_BLOCK_SIZE;
 370   ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
 371 #endif
 372
 373
 374   blocks->data = data;
 375   blocks->next = NULL;
 376
 377   ep->blocks = blocks;
 378   ep->cpos = sizeof (SList);
 379
 380   return ep;
 381 }
 382
 383 void
 384 epool_free (EPool *ep)
 385 {
 386   SList *l;
 387   gpointer data;
 388
 389 #ifdef EPOOL_DEBUG
 390   ep_allocated -= ep->allocated;
 391   printf ("MEM: %d\n", ep_allocated);
 392 #endif
 393
 394 #ifdef DEBUG
 395   return;
 396 #endif
 397
 398   l = ep->mblocks;
 399   while (l) {
 400     data = l->data;
 401     l = l->next;
 402     g_free (data);
 403   }
 404
 405   l = ep->blocks;
 406   while (l) {
 407     data = l->data;
 408     l = l->next;
 409     g_slice_free1(EPOOL_BLOCK_SIZE, data);
 410   }
 411 }
 412
 413 gpointer
 414 epool_alloc (EPool *ep, int size)
 415 {
 416   int rs = (size + 3) & ~3;
 417   int space = EPOOL_BLOCK_SIZE - sizeof (SList) - ep->cpos;
 418   gpointer data;
 419
 420   if (size > EPOOL_MAX_SIZE) {
 421     SList *blocks;
 422     if (space >= sizeof (SList)) {
 423       blocks = (SList *)((char *)ep->blocks->data + ep->cpos);
 424       ep->cpos += sizeof (SList);
 425     } else {
 426       blocks = (SList *)epool_alloc (ep, sizeof (SList));
 427     }
 428
 429     data = g_malloc (size);
 430 #ifdef EPOOL_DEBUG
 431     ep->allocated += size;
 432     ep_allocated += size;
 433     ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
 434 #endif
 435     blocks->data = data;
 436     blocks->next = ep->mblocks;
 437
 438     ep->mblocks = blocks;
 439
 440     return data;
 441
 442   } else if (space >= rs) {
 443     data = (char *)ep->blocks->data + ep->cpos;
 444     ep->cpos += rs;
 445
 446     return data;
 447
 448   } else {
 449     SList *blocks = (SList *)((char *)ep->blocks->data + ep->cpos);
 450
 451     data = g_slice_alloc0 (EPOOL_BLOCK_SIZE);
 452     blocks->data = data;
 453     blocks->next = ep->blocks;
 454
 455     ep->blocks = blocks;
 456     ep->cpos = rs;
 457
 458 #ifdef EPOOL_DEBUG
 459     ep->allocated += EPOOL_BLOCK_SIZE;
 460     ep_allocated += EPOOL_BLOCK_SIZE;
 461     ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
 462 #endif
 463
 464     return data;
 465   }
 466 }
 467
 468 char *
 469 epool_strndup (EPool *ep, const char *s, int len)
 470 {
 471   int l = len + 1;
 472   char *res = epool_alloc (ep, l);
 473   strncpy (res, s, l);
 474   return res;
 475 }
 476
 477 char *
 478 epool_strndup0 (EPool *ep, const char *s, int len)
 479 {
 480   char *res = epool_alloc (ep, len + 1);
 481   strncpy (res, s, len);
 482   res[len] = 0;
 483
 484   return res;
 485 }
 486
 487 char *
 488 epool_strdup (EPool *ep, const char *s)
 489 {
 490   int l = strlen (s) + 1;
 491   char *res = epool_alloc (ep, l);
 492   strncpy (res, s, l);
 493   return res;
 494 }
 495
 496 void
 497 loglist_print (LogList *loglist)
 498 {
 499   LogEntry *log = loglist->log;
 500   while (log) {
 501     printf ("L%08X %s", log->linenr, log->text);
 502     log = log->next;
 503   }
 504 }
 505
 506
 507 void
 508 loglist_add (EPool *ep, LogList *loglist, const char *text, int len, unsigned long linenr)
 509 {
 510   LogEntry *log;
 511
 512 #ifdef DEBUG
 513   if (len != strlen (text)) {
 514     debug_error ("string with wrong len", NULL);
 515   }
 516 #endif
 517   if (text[len] != 0) {
 518     debug_error ("string is not null terminated", NULL);
 519     return;
 520   }
 521
 522   log = epool_alloc (ep, sizeof (LogEntry));
 523
 524   log->text = epool_strndup (ep, text, len);
 525   log->linenr = linenr;
 526   log->next = NULL;
 527
 528   if (loglist->logs_last) {
 529     loglist->logs_last->next = log;
 530     loglist->logs_last = log;
 531   } else {
 532     loglist->log = log;
 533     loglist->logs_last = log;
 534   }
 535
 536   return;
 537 }
 538
 539 SEntry*
 540 sentry_new (int pid, time_t ltime, unsigned long rel_line_nr)
 541 {
 542   SEntry *sentry;
 543   SList *blocks;
 544   int cpos;
 545
 546   sentry = (SEntry *)g_slice_alloc0(EPOOL_BLOCK_SIZE);
 547   sentry->pid = pid;
 548   sentry->ltime = ltime;
 549   sentry->rel_line_nr = rel_line_nr;
 550
 551 #ifdef EPOOL_DEBUG
 552   sentry->ep.allocated += EPOOL_BLOCK_SIZE;
 553   ep_allocated += EPOOL_BLOCK_SIZE;
 554   ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
 555 #endif
 556
 557 #ifdef DEBUG
 558   {
 559     SEntry *se;
 560     if ((se = g_hash_table_lookup (smtpd_debug_alloc, sentry))) {
 561       debug_error ("SEntry already alloced", NULL);
 562     } else {
 563       g_hash_table_insert (smtpd_debug_alloc, sentry, sentry);
 564     }
 565   }
 566 #endif
 567
 568   cpos = sizeof (SEntry);
 569
 570   blocks = (SList *)((char *)sentry + cpos);
 571
 572   cpos += sizeof (SList);
 573
 574   blocks->data = sentry;
 575   blocks->next = NULL;
 576
 577   sentry->ep.blocks = blocks;
 578   sentry->ep.cpos = cpos;
 579
 580   return sentry;
 581 }
 582
 583 SEntry *
 584 sentry_get (LParser *parser, int pid, time_t ltime, unsigned long rel_line_nr)
 585 {
 586   SEntry *sentry;
 587
 588   if ((sentry = g_hash_table_lookup (parser->smtpd_h, &pid))) {
 589     return sentry;
 590   } else {
 591
 592     if ((sentry = sentry_new (pid, ltime, rel_line_nr))) {
 593       g_hash_table_insert (parser->smtpd_h, &sentry->pid, sentry);
 594     }
 595
 596     return sentry;
 597   }
 598 }
 599
 600 void
 601 sentry_ref_add (SEntry *sentry, QEntry *qentry)
 602 {
 603   SList *l;
 604
 605   if (qentry->smtpd) {
 606     if (qentry->smtpd != sentry) {
 607       debug_error ("qentry ref already set", NULL);
 608     }
 609     return;
 610   }
 611
 612   l = sentry->refs;
 613   while (l) {
 614     if (l->data == qentry) return;
 615     l = l->next;
 616   }
 617
 618   l = epool_alloc (&sentry->ep, sizeof (SList));
 619
 620   qentry->smtpd = sentry;
 621
 622   l->data = qentry;
 623   l->next = sentry->refs;
 624
 625   sentry->refs = l;
 626 }
 627
 628 int
 629 sentry_ref_del (SEntry *sentry, QEntry *qentry)
 630 {
 631   SList *l = sentry->refs;
 632   int count = 0;
 633
 634   if (!qentry->smtpd) {
 635     debug_error ("qentry does not hav a qentry ref", NULL);
 636     return 0;
 637   }
 638
 639   l = sentry->refs;
 640
 641   while (l) {
 642     QEntry *qe = (QEntry *)l->data;
 643     if (qe == qentry) {
 644       l->data = NULL;
 645     } else {
 646       if (qe && qe->cleanup) count++;
 647     }
 648     l = l->next;
 649   }
 650
 651   return count;
 652 }
 653
 654 void
 655 sentry_ref_finalize (LParser *parser, SEntry *sentry)
 656 {
 657   SList *l = sentry->refs;
 658
 659   int count = 0;
 660
 661   while (l) {
 662     SList *cl = l;
 663     QEntry *qe = l->data;
 664
 665     l = l->next;
 666
 667     if (!qe) continue;
 668
 669     count++;
 670
 671     if (!qe->removed) continue;
 672
 673     FEntry *fe = qe->filter;
 674
 675     if (fe && !fe->finished) continue;
 676
 677     count--;
 678
 679     qentry_print (parser, qe);
 680
 681     cl->data = NULL;
 682     qe->smtpd = NULL;
 683
 684     qentry_free (parser, qe);
 685
 686     if (fe) fentry_free (parser, fe);
 687
 688   }
 689
 690   if (!count) sentry_free_noremove (sentry);
 691 }
 692
 693 int
 694 sentry_ref_rem_unneeded (LParser *parser, SEntry *sentry)
 695 {
 696   SList *l = sentry->refs;
 697   int count = 0;
 698
 699   while (l) {
 700     QEntry *qe = (QEntry *)l->data;
 701
 702     if (qe) {
 703       if (!qe->cleanup) {
 704         qe->smtpd = NULL;
 705         qentry_free (parser, qe);
 706         l->data = NULL;
 707       } else {
 708         count++;
 709       }
 710     }
 711
 712     l = l->next;
 713   }
 714
 715   return count;
 716 }
 717
 718 void
 719 sentry_nqlist_add (SEntry *sentry, time_t ltime, const char *from, int from_len,
 720                    const char *to, int to_len, char dstatus)
 721 {
 722   NQList *nq = (NQList *)epool_alloc (&sentry->ep, sizeof (NQList));
 723
 724   nq->from = epool_strndup0 (&sentry->ep, from, from_len);
 725   nq->to = epool_strndup0 (&sentry->ep, to, to_len);
 726   nq->dstatus = dstatus;
 727
 728   nq->next = sentry->nqlist;
 729   nq->ltime = ltime;
 730   sentry->nqlist = nq;
 731 }
 732
 733 void
 734 sentry_print (LParser *parser, SEntry *sentry)
 735 {
 736   NQList *nq;
 737
 738   if (parser->msgid) return;
 739
 740   if (parser->server) {
 741     if (!sentry->connect) return;
 742     if (!strcasestr (sentry->connect, parser->server)) return;
 743   }
 744
 745   MatchList *match = parser->match_list;
 746   if (match) {
 747     int found = 0;
 748     while(match) {
 749       if (match->mtype == MatchTypeQID) {
 750         return;
 751       } else if (match->mtype == MatchTypeRelLineNr) {
 752         if (match->ltime == sentry->ltime && match->rel_line_nr == sentry->rel_line_nr) {
 753           found = 1;
 754           break;
 755         }
 756       } else {
 757         g_error("implement me");
 758       }
 759       match = match->next;
 760     }
 761     if (!found) return;
 762   }
 763
 764   if (parser->from || parser->to ||
 765       parser->exclude_greylist || parser->exclude_ndrs) {
 766     nq = sentry->nqlist;
 767     int found = 0;
 768     while (nq) {
 769       if (parser->from) {
 770         if (!*(parser->from)) {
 771           if (*(nq->from)) nq->dstatus = 0;
 772         } else if (!STRMATCH(parser->from, nq->from)) {
 773           nq->dstatus = 0;
 774         }
 775       }
 776
 777       if (parser->exclude_greylist && nq->dstatus == 'G') nq->dstatus = 0;
 778
 779       if (parser->exclude_ndrs && nq->from && !*nq->from) nq->dstatus = 0;
 780
 781       if (parser->to && nq->to && !STRMATCH(parser->to, nq->to)) {
 782         nq->dstatus = 0;
 783       }
 784
 785       if (nq->dstatus != 0) found = 1;
 786
 787       nq = nq->next;
 788     }
 789     if (!found) return;
 790   }
 791
 792   if (parser->strmatch && !sentry->strmatch) return;
 793
 794   if (parser->verbose) {
 795
 796     printf ("SMTPD: T%08lXL%08lX\n", sentry->ltime, sentry->rel_line_nr);
 797
 798     printf ("CTIME: %08lX\n", parser->ctime);
 799
 800     if (sentry->connect) { printf ("CLIENT: %s\n", sentry->connect); }
 801     //printf ("EXTERNAL: %d\n", sentry->external);
 802
 803   }
 804
 805   nq = sentry->nqlist;
 806   while (nq) {
 807     if (nq->from && nq->to && nq->dstatus) {
 808       printf ("TO:%08lX:T%08lXL%08lX:%c: from <%s> to <%s>\n", nq->ltime,
 809               sentry->ltime, sentry->rel_line_nr, nq->dstatus,
 810               nq->from, nq->to);
 811       parser->count++;
 812     }
 813     nq = nq->next;
 814   }
 815
 816   if (!parser->verbose)  { fflush (stdout); return; }
 817
 818   if (parser->verbose > 1) {
 819     printf ("LOGS:\n");
 820     loglist_print (&sentry->loglist);
 821   }
 822
 823   printf ("\n");
 824
 825   fflush (stdout);
 826 }
 827
 828 void
 829 sentry_set_connect (SEntry *sentry, const char *connect, int len)
 830 {
 831   if (sentry->connect) {
 832 #ifdef DEBUG
 833     if (strncmp (sentry->connect, connect, len)) {
 834       debug_error ("duplicate connect", NULL);
 835     }
 836 #endif
 837   } else {
 838     sentry->connect = epool_strndup0 (&sentry->ep, connect, len);
 839   }
 840 }
 841
 842 void
 843 sentry_free_noremove (SEntry *sentry)
 844 {
 845   SList *l;
 846   gpointer data;
 847
 848 #ifdef EPOOL_DEBUG
 849   ep_allocated -= sentry->ep.allocated;
 850   printf ("MEM: %d\n", ep_allocated);
 851 #endif
 852
 853 #ifdef DEBUG
 854   {
 855     SEntry *se;
 856     if ((se = g_hash_table_lookup (smtpd_debug_free, sentry))) {
 857       debug_error ("SEntry already freed", NULL);
 858     } else {
 859       g_hash_table_insert (smtpd_debug_free, sentry, sentry);
 860     }
 861   }
 862   return;
 863 #endif
 864
 865   l = sentry->ep.mblocks;
 866   while (l) {
 867     data = l->data;
 868     l = l->next;
 869     g_free (data);
 870   }
 871
 872   l = sentry->ep.blocks;
 873   while (l) {
 874     data = l->data;
 875     l = l->next;
 876     g_slice_free1(EPOOL_BLOCK_SIZE, data);
 877   }
 878 }
 879
 880 void
 881 sentry_free (LParser *parser, SEntry *sentry)
 882 {
 883   g_hash_table_remove (parser->smtpd_h, &sentry->pid);
 884
 885   sentry_free_noremove (sentry);
 886 }
 887
 888 void
 889 sentry_cleanup_hash (gpointer key,
 890                      gpointer value,
 891                      gpointer user_data)
 892 {
 893   SEntry *se = value;
 894   LParser *parser = (LParser *)user_data;
 895
 896   sentry_print (parser, se);
 897   sentry_free_noremove (se);
 898 }
 899
 900 #ifdef DEBUG
 901 void
 902 sentry_debug_alloc (gpointer key,
 903                     gpointer value,
 904                     gpointer user_data)
 905 {
 906   LParser *parser = (LParser *)user_data;
 907   SEntry *se = value;
 908   SEntry *fe;
 909
 910   if ((fe = g_hash_table_lookup (smtpd_debug_free, se))) {
 911     return;
 912   }
 913
 914   printf ("FOUND ALLOCATED SENTRY:\n");
 915   sentry_print (parser, se);
 916
 917   exit (-1);
 918 }
 919 #endif
 920
 921 // QEntry
 922
 923 QEntry*
 924 qentry_new (const char *qid)
 925 {
 926   QEntry *qentry;
 927   SList *blocks;
 928   int cpos;
 929   char *qid_cp;
 930
 931   qentry = (QEntry *)g_slice_alloc0(EPOOL_BLOCK_SIZE);
 932
 933 #ifdef EPOOL_DEBUG
 934   qentry->ep.allocated += EPOOL_BLOCK_SIZE;
 935   ep_allocated += EPOOL_BLOCK_SIZE;
 936   ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
 937 #endif
 938
 939 #ifdef DEBUG
 940   {
 941     QEntry *qe;
 942     if ((qe = g_hash_table_lookup (qmgr_debug_alloc, qentry))) {
 943       debug_error ("QEntry already alloced", NULL);
 944     } else {
 945       g_hash_table_insert (qmgr_debug_alloc, qentry, qentry);
 946     }
 947   }
 948 #endif
 949
 950   cpos = sizeof (QEntry);
 951
 952   blocks = (SList *)((char *)qentry + cpos);
 953
 954   cpos += sizeof (SList);
 955
 956   blocks->data = qentry;
 957   blocks->next = NULL;
 958
 959   qentry->qid = qid_cp = (char *)qentry + cpos;
 960   while ((*qid_cp++ = *qid++)) cpos++;
 961
 962   cpos = (cpos + 4) & ~3;
 963
 964   qentry->ep.blocks = blocks;
 965   qentry->ep.cpos = cpos;;
 966
 967   return qentry;
 968 }
 969
 970 void
 971 qentry_tolist_add (QEntry *qentry, time_t ltime, char dstatus, const char *to, int to_len,
 972                    const char *relay, int relay_len)
 973 {
 974   TOList *tl = (TOList *)epool_alloc (&qentry->ep, sizeof (TOList));
 975
 976   tl->to = epool_strndup0 (&qentry->ep, to, to_len);
 977   tl->relay = epool_strndup0 (&qentry->ep, relay, relay_len);
 978   tl->dstatus = dstatus;
 979   tl->ltime = ltime;
 980   tl->next = qentry->tolist;
 981
 982   qentry->tolist = tl;
 983 }
 984
 985 void
 986 qentry_set_from (QEntry *qentry, const char *from, int len)
 987 {
 988   if (qentry->from) {
 989 #ifdef DEBUG
 990     if (strncmp (qentry->from, from, len)) {
 991       debug_error ("duplicate from", NULL);
 992     }
 993 #endif
 994   } else {
 995     qentry->from = epool_strndup0 (&qentry->ep, from, len);
 996   }
 997 }
 998
 999 void
1000 qentry_set_msgid (QEntry *qentry, const char *msgid, int len)
1001 {
1002   if (qentry->msgid) {
1003 #ifdef DEBUG
1004     if (strncmp (qentry->msgid, msgid, len)) {
1005       debug_error ("duplicate msgid", NULL);
1006     }
1007 #endif
1008   } else {
1009     qentry->msgid = epool_strndup0 (&qentry->ep, msgid, len);
1010   }
1011 }
1012
1013 void
1014 qentry_set_client (QEntry *qentry, const char *client, int len)
1015 {
1016   if (qentry->client) {
1017 #ifdef DEBUG
1018     if (strncmp (qentry->client, client, len)) {
1019       debug_error ("duplicate client", NULL);
1020     }
1021 #endif
1022   } else {
1023     qentry->client = epool_strndup0 (&qentry->ep, client, len);
1024   }
1025 }
1026
1027 void
1028 qentry_print (LParser *parser, QEntry *qentry)
1029 {
1030   TOList *tl, *fl;
1031   SEntry *se = qentry->smtpd;
1032   FEntry *fe = qentry->filter;
1033
1034   if (parser->msgid) {
1035     if (!qentry->msgid) return;
1036     if (strcasecmp (parser->msgid, qentry->msgid)) return;
1037   }
1038
1039   MatchList *match = parser->match_list;
1040   if (match) {
1041     int found = 0;
1042     while(match) {
1043       if (match->mtype == MatchTypeQID) {
1044         if ((fe && !strcmp (fe->logid, match->id)) ||
1045             (!strcmp (qentry->qid, match->id))) {
1046           found = 1;
1047           break;
1048         }
1049       } else if (match->mtype == MatchTypeRelLineNr) {
1050         if (se && match->ltime == se->ltime && match->rel_line_nr == se->rel_line_nr) {
1051           found = 1;
1052           break;
1053         }
1054       } else {
1055         g_error("implement me");
1056       }
1057       match = match->next;
1058     }
1059     if (!found) return;
1060   }
1061
1062   if (parser->server) {
1063     int found = 0;
1064     if (se && se->connect && strcasestr (se->connect, parser->server)) found = 1;
1065     if (qentry->client && strcasestr (qentry->client, parser->server)) found = 1;
1066
1067     if (!found) return;
1068   }
1069
1070   if (parser->from) {
1071     if (!qentry->from) return;
1072     if (!*(parser->from)) {
1073       if (*(qentry->from)) return;
1074     } else if (!STRMATCH(parser->from, qentry->from)) {
1075       return;
1076     }
1077   } else {
1078     if (parser->exclude_ndrs && qentry->from && !*qentry->from) return;
1079   }
1080
1081   if (parser->to) {
1082     tl = qentry->tolist;
1083     int found = 0;
1084     while (tl) {
1085       if (parser->to && !STRMATCH(parser->to, tl->to)) {
1086         tl->to = NULL;
1087       } else {
1088         found = 1;
1089       }
1090       tl = tl->next;
1091     }
1092     if (!found) return;
1093   }
1094
1095   if (parser->strmatch &&
1096       !(qentry->strmatch || (se && se->strmatch) || (fe && fe->strmatch)))
1097     return;
1098
1099
1100   if (parser->verbose) {
1101
1102     printf ("QENTRY: %s\n", qentry->qid);
1103
1104     printf ("CTIME: %08lX\n", parser->ctime);
1105     printf ("SIZE: %u\n", qentry->size);
1106
1107     if (qentry->client) {
1108       printf ("CLIENT: %s\n", qentry->client);
1109     } else if (se && se->connect) {
1110       printf ("CLIENT: %s\n", se->connect);
1111     }
1112
1113     if (qentry->msgid) { printf ("MSGID: %s\n", qentry->msgid); }
1114
1115   }
1116
1117   tl = qentry->tolist;
1118   while (tl) {
1119     if (tl->to) {
1120       fl = NULL;
1121       if (fe && tl->dstatus == '2') {
1122         fl = fe->tolist;
1123         while (fl) {
1124           if (fl->to && !strcmp (tl->to, fl->to)) {
1125             break;
1126           }
1127           fl = fl->next;
1128         }
1129       }
1130       char *to;
1131       char dstatus;
1132       char *relay;
1133
1134       if (fl) {
1135         to = fl->to;
1136         dstatus = fl->dstatus;
1137         relay = fl->relay;
1138       } else {
1139         to = tl->to;
1140         dstatus = tl->dstatus;
1141         relay = tl->relay;
1142       }
1143
1144       printf ("TO:%08lX:%s:%c: from <%s> to <%s> (%s)\n", tl->ltime, qentry->qid, dstatus, qentry->from ? qentry->from : "", to ? to : "", relay ? relay : "none");
1145
1146       parser->count++;
1147     }
1148     tl = tl->next;
1149   }
1150
1151
1152   if (!parser->verbose)  { fflush (stdout); return; }
1153
1154   if (parser->verbose > 1) {
1155
1156     if (se && se->loglist.log) {
1157       printf ("SMTP:\n");
1158       loglist_print (&se->loglist);
1159     }
1160
1161     if (fe && fe->loglist.log) {
1162       printf ("FILTER: %s\n", fe->logid);
1163       loglist_print (&fe->loglist);
1164     }
1165
1166     if (qentry->loglist.log) {
1167       printf ("QMGR:\n");
1168       loglist_print (&qentry->loglist);
1169     }
1170   }
1171
1172   printf ("\n");
1173
1174   fflush (stdout);
1175
1176   //sleep (1);
1177 }
1178
1179 QEntry *
1180 qentry_get (LParser *parser, const char *qid)
1181 {
1182   QEntry *qentry;
1183
1184   if ((qentry = g_hash_table_lookup (parser->qmgr_h, qid))) {
1185     return qentry;
1186   } else {
1187     if ((qentry = qentry_new (qid))) {
1188       g_hash_table_insert (parser->qmgr_h, qentry->qid, qentry);
1189     }
1190
1191     return qentry;
1192   }
1193 }
1194
1195 void
1196 qentry_free_noremove (LParser *parser, QEntry *qentry)
1197 {
1198   SList *l;
1199   gpointer data;
1200   SEntry *se;
1201
1202   if ((se = qentry->smtpd)) {
1203     if (sentry_ref_del (se, qentry) == 0) {
1204       if (se->disconnect) {
1205         sentry_free_noremove (se);
1206       }
1207     }
1208   }
1209
1210 #ifdef EPOOL_DEBUG
1211   ep_allocated -= qentry->ep.allocated;
1212   printf ("MEM: %d\n", ep_allocated);
1213 #endif
1214
1215 #ifdef DEBUG
1216   {
1217     QEntry *qe;
1218     if ((qe = g_hash_table_lookup (qmgr_debug_free, qentry))) {
1219       debug_error ("QEntry already freed", NULL);
1220     } else {
1221       g_hash_table_insert (qmgr_debug_free, qentry, qentry);
1222     }
1223   }
1224   return;
1225 #endif
1226
1227   l = qentry->ep.mblocks;
1228   while (l) {
1229     data = l->data;
1230     l = l->next;
1231     g_free (data);
1232   }
1233
1234   l = qentry->ep.blocks;
1235   while (l) {
1236     data = l->data;
1237     l = l->next;
1238     g_slice_free1(EPOOL_BLOCK_SIZE, data);
1239   }
1240 }
1241
1242 void
1243 qentry_free (LParser *parser, QEntry *qentry)
1244 {
1245   g_hash_table_remove (parser->qmgr_h, qentry->qid);
1246
1247   qentry_free_noremove (parser, qentry);
1248 }
1249
1250 void
1251 qentry_cleanup_hash (gpointer key,
1252                      gpointer value,
1253                      gpointer user_data)
1254 {
1255   QEntry *qe = value;
1256   LParser *parser = (LParser *)user_data;
1257
1258   qentry_print (parser, qe);
1259   qentry_free_noremove (parser, qe);
1260 }
1261
1262 void
1263 qentry_finalize (LParser *parser, QEntry *qentry)
1264 {
1265   if (qentry && qentry->removed) {
1266     SEntry *se = qentry->smtpd;
1267
1268     if (se && !se->disconnect) return;
1269
1270     FEntry *fe = qentry->filter;
1271
1272     if (fe && !fe->finished) return;
1273
1274     qentry_print (parser, qentry);
1275     qentry_free (parser, qentry);
1276
1277     if (fe) fentry_free (parser, fe);
1278   }
1279 }
1280
1281 // FEntry
1282
1283 FEntry*
1284 fentry_new (const char *logid)
1285 {
1286   FEntry *fentry;
1287   SList *blocks;
1288   int cpos;
1289   char *logid_cp;
1290
1291   fentry = (FEntry *)g_slice_alloc0(EPOOL_BLOCK_SIZE);
1292
1293 #ifdef EPOOL_DEBUG
1294   fentry->ep.allocated += EPOOL_BLOCK_SIZE;
1295   ep_allocated += EPOOL_BLOCK_SIZE;
1296   ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
1297 #endif
1298
1299 #ifdef DEBUG
1300   {
1301     FEntry *fe;
1302     if ((fe = g_hash_table_lookup (filter_debug_alloc, fentry))) {
1303       debug_error ("FEntry already alloced", NULL);
1304     } else {
1305       g_hash_table_insert (filter_debug_alloc, fentry, fentry);
1306     }
1307   }
1308 #endif
1309
1310   cpos = sizeof (FEntry);
1311
1312   blocks = (SList *)((char *)fentry + cpos);
1313
1314   cpos += sizeof (SList);
1315
1316   blocks->data = fentry;
1317   blocks->next = NULL;
1318
1319   fentry->logid = logid_cp = (char *)fentry + cpos;
1320   while ((*logid_cp++ = *logid++)) cpos++;
1321   cpos = (cpos + 4) & ~3;
1322
1323   fentry->ep.blocks = blocks;
1324   fentry->ep.cpos = cpos;;
1325
1326   return fentry;
1327 }
1328
1329 FEntry *
1330 fentry_get (LParser *parser, const char *logid)
1331 {
1332   FEntry *fentry;
1333
1334   if ((fentry = g_hash_table_lookup (parser->filter_h, logid))) {
1335     return fentry;
1336   } else {
1337     if ((fentry = fentry_new (logid))) {
1338       g_hash_table_insert (parser->filter_h, fentry->logid, fentry);
1339     }
1340
1341     return fentry;
1342   }
1343 }
1344
1345 void
1346 fentry_tolist_add (FEntry *fentry, char dstatus, const char *to, int to_len,
1347                    const char *qid, int qid_len)
1348 {
1349   TOList *tl = (TOList *)epool_alloc (&fentry->ep, sizeof (TOList));
1350
1351   tl->to = epool_strndup0 (&fentry->ep, to, to_len);
1352
1353   if (qid) {
1354     tl->relay = epool_strndup0 (&fentry->ep, qid, qid_len);
1355   } else {
1356     tl->relay = NULL;
1357   }
1358   tl->dstatus = dstatus;
1359   tl->next = fentry->tolist;
1360
1361   fentry->tolist = tl;
1362 }
1363
1364 void
1365 fentry_free_noremove (LParser *parser, FEntry *fentry)
1366 {
1367   SList *l;
1368   gpointer data;
1369
1370 #ifdef EPOOL_DEBUG
1371   ep_allocated -= fentry->ep.allocated;
1372   printf ("MEM: %d\n", ep_allocated);
1373 #endif
1374
1375 #ifdef DEBUG
1376   {
1377     FEntry *fe;
1378     if ((fe = g_hash_table_lookup (filter_debug_free, fentry))) {
1379       debug_error ("FEntry already freed", NULL);
1380     } else {
1381       g_hash_table_insert (filter_debug_free, fentry, fentry);
1382     }
1383   }
1384   return;
1385 #endif
1386
1387   l = fentry->ep.mblocks;
1388   while (l) {
1389     data = l->data;
1390     l = l->next;
1391     g_free (data);
1392   }
1393
1394   l = fentry->ep.blocks;
1395   while (l) {
1396     data = l->data;
1397     l = l->next;
1398     g_slice_free1(EPOOL_BLOCK_SIZE, data);
1399   }
1400 }
1401
1402 void
1403 fentry_free (LParser *parser, FEntry *fentry)
1404 {
1405   g_hash_table_remove (parser->filter_h, fentry->logid);
1406
1407   fentry_free_noremove (parser, fentry);
1408 }
1409
1410 void
1411 fentry_cleanup_hash (gpointer key,
1412                      gpointer value,
1413                      gpointer user_data)
1414 {
1415   FEntry *fe = value;
1416   LParser *parser = (LParser *)user_data;
1417
1418   fentry_free_noremove (parser, fe);
1419 }
1420
1421 // Parser
1422
1423 LParser*
1424 parser_new ()
1425 {
1426   LParser *parser = g_malloc0 (sizeof (LParser));
1427   struct timeval tv;
1428   struct tm *ltime;
1429   int i;
1430
1431   epool_init (&parser->ep);
1432
1433   if (!(parser->smtpd_h = g_hash_table_new (g_int_hash, g_int_equal))) {
1434     return NULL;
1435   }
1436
1437   if (!(parser->qmgr_h = g_hash_table_new (g_str_hash, g_str_equal))) {
1438     return NULL;
1439   }
1440
1441   if (!(parser->filter_h = g_hash_table_new (g_str_hash, g_str_equal))) {
1442     return NULL;
1443   }
1444
1445   //if (!(parser->track_h = g_hash_table_new (g_str_hash, g_str_equal))) {
1446   //return NULL;
1447   //}
1448
1449   for (i = 0; i < MAX_LOGFILES; i++) {
1450     gettimeofday(&tv, NULL);
1451     tv.tv_sec -= 3600*24*i;
1452     ltime = localtime (&tv.tv_sec);
1453     parser->year[i] = ltime->tm_year + 1900;
1454   }
1455
1456   return parser;
1457 }
1458
1459 void
1460 parser_free (LParser *parser)
1461 {
1462   int i;
1463
1464   for (i = 0; i < MAX_LOGFILES; i++) {
1465     if (parser->stream[i]) gzclose (parser->stream[i]);
1466   }
1467
1468   epool_free (&parser->ep);
1469
1470   g_hash_table_destroy (parser->smtpd_h);
1471   g_hash_table_destroy (parser->qmgr_h);
1472   g_hash_table_destroy (parser->filter_h);
1473   //g_hash_table_destroy (parser->track_h);
1474
1475   g_free (parser);
1476 }
1477
1478 #if 0
1479 char *
1480 parser_track (LParser *parser, const char *qid, gboolean insert)
1481 {
1482   char *res;
1483
1484   if ((res = g_hash_table_lookup (parser->track_h, qid))) {
1485     return res;
1486   } else {
1487     if (insert && (res = epool_strdup (&parser->ep, qid))) {
1488       g_hash_table_insert (parser->track_h, res, res);
1489       return res;
1490     }
1491   }
1492   return NULL;
1493 }
1494 #endif
1495
1496 static const int linebufsize = 8192;
1497 static int cur_year;
1498 static int cur_month = 0;
1499 static int cal_mtod[12] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334};
1500
1501 static time_t
1502 mkgmtime (struct tm *tm)
1503 {
1504   time_t res;
1505
1506   int year = tm->tm_year + 1900;
1507   int mon = tm->tm_mon;
1508
1509   res = (year - 1970) * 365 + cal_mtod[mon];
1510
1511   // leap year corrections (gregorian calendar)
1512   if (mon <= 1) year -= 1;
1513   res += (year - 1968) / 4;
1514   res -= (year - 1900) / 100;
1515   res += (year - 1600) / 400;
1516
1517   res += tm->tm_mday - 1;
1518   res = res*24 + tm->tm_hour;
1519   res = res*60 + tm->tm_min;
1520   res = res*60 + tm->tm_sec;
1521
1522   return res;
1523 }
1524
1525 time_t
1526 parse_time (const char **text, int len)
1527 {
1528   time_t ltime = 0;
1529
1530   int year = cur_year;
1531
1532   int mon = 0;
1533   int mday = 0;
1534   int hour = 0;
1535   int min = 0;
1536   int sec = 0;
1537   int found;
1538
1539   const char *line = *text;
1540
1541   if (len == (linebufsize - 1)) {
1542     debug_error ("skipping long line data", line);
1543     return 0;
1544   }
1545
1546   if (len < 15) {
1547     debug_error ("skipping short line data", line);
1548     return 0;
1549   }
1550
1551   // parse month
1552   int csum = (line[0]<<16) + (line[1]<<8) + line[2];
1553
1554   switch (csum) {
1555   case 4874606: mon = 0; break;
1556   case 4613474: mon = 1; break;
1557   case 5071218: mon = 2; break;
1558   case 4288626: mon = 3; break;
1559   case 5071225: mon = 4; break;
1560   case 4879726: mon = 5; break;
1561   case 4879724: mon = 6; break;
1562   case 4289895: mon = 7; break;
1563   case 5465456: mon = 8; break;
1564   case 5202804: mon = 9; break;
1565   case 5140342: mon = 10; break;
1566   case 4482403: mon = 11; break;
1567   default:
1568     debug_error ("unable to parse month", line);
1569     return 0;
1570   }
1571
1572   // year change heuristik
1573   if (cur_month == 11 && mon == 0) {
1574     year++;
1575   }
1576   if (mon > cur_month) cur_month = mon;
1577
1578   ltime = (year - 1970)*365 + cal_mtod[mon];
1579
1580   // leap year corrections (gregorian calendar)
1581   if (mon <= 1) year -= 1;
1582   ltime += (year - 1968) / 4;
1583   ltime -= (year - 1900) / 100;
1584   ltime += (year - 1600) / 400;
1585
1586   const char *cpos = line + 3;
1587
1588   found = 0; while (isspace (*cpos)) { cpos++; found = 1; }
1589   if (!found) {
1590     debug_error ("missing spaces after month", line);
1591     return 0;
1592   }
1593
1594   found = 0; while (isdigit (*cpos)) { mday = mday*10 + *cpos - 48; cpos++; found++; }
1595   if (found < 1 || found > 2) {
1596     debug_error ("unable to parse day of month", line);
1597     return 0;
1598   }
1599
1600   ltime += mday - 1;
1601
1602   found = 0; while (isspace (*cpos)) { cpos++; found++; }
1603   if (!found) {
1604     debug_error ("missing spaces after day of month", line);
1605     return 0;
1606   }
1607
1608   found = 0; while (isdigit (*cpos)) { hour = hour*10 + *cpos - 48; cpos++; found++; }
1609   if (found < 1 || found > 2) {
1610     debug_error ("unable to parse hour", line);
1611     return 0;
1612   }
1613
1614   ltime *= 24;
1615   ltime += hour;
1616
1617   if (*cpos != ':') {
1618     debug_error ("missing collon after hour", line);
1619     return 0;
1620   }
1621   cpos++;
1622
1623   found = 0; while (isdigit (*cpos)) { min = min*10 + *cpos - 48; cpos++; found++; }
1624   if (found < 1 || found > 2) {
1625     debug_error ("unable to parse minute", line);
1626     return 0;
1627   }
1628
1629   ltime *= 60;
1630   ltime += min;
1631
1632   if (*cpos != ':') {
1633     debug_error ("missing collon after minute", line);
1634     return 0;
1635   }
1636   cpos++;
1637
1638   found = 0; while (isdigit (*cpos)) { sec = sec*10 + *cpos - 48; cpos++; found++; }
1639   if (found < 1 || found > 2) {
1640     debug_error ("unable to parse second", line);
1641     return 0;
1642   }
1643
1644   ltime *= 60;
1645   ltime += sec;
1646
1647   found = 0; while (isspace (*cpos)) { cpos++; found = 1; }
1648   if (!found) {
1649     debug_error ("missing spaces after time", line);
1650     return 0;
1651   }
1652
1653   *text = cpos;
1654
1655   return ltime;
1656 }
1657
1658
1659 int
1660 parser_count_files (LParser *parser)
1661 {
1662   int i;
1663   time_t start = parser->start;
1664   char linebuf[linebufsize];
1665   const char *line;
1666   gzFile stream;
1667
1668   for (i = 0; i < (MAX_LOGFILES - 1); i++) {
1669     cur_year = parser->year[i];
1670     cur_month = 0;
1671
1672     if ((stream = gzopen (logfiles[i], "r"))) {
1673       if ((line = gzgets (stream, linebuf, linebufsize))) {
1674         if (parse_time (&line, strlen (line)) < start) {
1675           break;
1676         }
1677       } else {
1678         return i;
1679       }
1680       gzclose (stream);
1681     } else {
1682       return i;
1683     }
1684   }
1685
1686   return i + 1;
1687 }
1688
1689 static char *
1690 parse_qid (const char **text, char *out, char delim, int maxlen)
1691 {
1692   const char *idx;
1693   char *copy = out;
1694
1695   int found = 0;
1696   idx = *text;
1697   while (isxdigit (*idx)) { *copy++ = *idx++; found++; if (found > maxlen) break; }
1698
1699   if (found > 1 && found < maxlen &&
1700       ((delim && (*idx == delim)) || (!delim && isspace (*idx)))) {
1701     *copy = 0;
1702     idx++;
1703     while (isspace (*idx)) idx++;
1704     *text = idx;
1705     return out;
1706   }
1707   return NULL;
1708 }
1709
1710 void
1711 print_usage (const char *name)
1712 {
1713   fprintf (stderr, "usage: %s [OPTIONS] [OUTPUTFILENAME]\n", name);
1714   fprintf (stderr, "\t-f SENDER      mails from SENDER\n");
1715   fprintf (stderr, "\t-t RECIPIENT   mails to RECIPIENT\n");
1716   fprintf (stderr, "\t-h Server      Server IP or Hostname\n");
1717   fprintf (stderr, "\t-s START       start time (YYYY-MM-DD HH:MM:SS)\n");
1718   fprintf (stderr, "\t               or seconds since epoch\n");
1719   fprintf (stderr, "\t-e END         end time (YYYY-MM-DD HH:MM:SS)\n");
1720   fprintf (stderr, "\t               or seconds since epoch\n");
1721   fprintf (stderr, "\t-m MSGID       message ID (exact match)\n");
1722   fprintf (stderr, "\t-q QID         queue ID (exact match), can be\n");
1723   fprintf (stderr, "\t               specified multiple times.\n");
1724   fprintf (stderr, "\t-x STRING      search for strings\n");
1725   fprintf (stderr, "\t-l LIMIT       print max limit entries\n");
1726   fprintf (stderr, "\t-g             exclude greylist entries\n");
1727   fprintf (stderr, "\t-n             exclude NDR entries\n");
1728   fprintf (stderr, "\t-v             verbose output (no logs)\n");
1729   fprintf (stderr, "\t-vv            verbose output with logs\n");
1730 }
1731
1732
1733 // gzgets is ways too slow, so we do it our own way
1734
1735 static int
1736 mygzgetc (gzFile stream)
1737 {
1738   int br;
1739   static char readbuf[16384];
1740   static char *readend = readbuf + sizeof (readbuf);
1741   static char *readpos = readbuf + sizeof (readbuf);
1742
1743   if (readpos < readend) return *readpos++;
1744
1745   if ((br = gzread (stream, readbuf, sizeof (readbuf))) <= 0) {
1746     return -1;
1747   } else {
1748     readpos = readbuf;
1749     readend = readbuf + br;
1750
1751     return *readpos++;
1752   }
1753 }
1754
1755 static char *
1756 mygzgets (gzFile stream, char *line, int bufsize)
1757 {
1758   int c=0;
1759   char *cpos;
1760
1761   cpos = line;
1762   while (--bufsize > 0 && (c = mygzgetc(stream)) != -1) {
1763     *cpos++ = c;
1764     if (c == '\n')
1765       break;
1766   }
1767   if (c == -1 && cpos == line)
1768     return NULL;
1769   *cpos++ = '\0';
1770   return line;
1771 }
1772
1773
1774 extern char *optarg;
1775 extern int optind, opterr, optopt;
1776
1777 int
1778 main (int argc, char * const argv[])
1779 {
1780   char linebuf[linebufsize];
1781   char *line;
1782   char *inputfile = NULL;
1783
1784   const char *text;
1785   const char *idx1;
1786   const char *idx2;
1787   const char *cpos;
1788   int found = 0;
1789   uint32_t csum_prog;
1790   unsigned long lines = 0;
1791   unsigned long rel_line_nr = 0;
1792   char qidbuf[30];
1793   int i;
1794
1795   struct tm *ltime;
1796   struct timeval tv;
1797   time_t ctime, next_ctime, start, end;
1798
1799   LParser *parser;
1800   int opt;
1801
1802 #ifdef DEBUG
1803   smtpd_debug_alloc = g_hash_table_new (g_direct_hash, g_direct_equal);
1804   qmgr_debug_alloc = g_hash_table_new (g_direct_hash, g_direct_equal);
1805   filter_debug_alloc = g_hash_table_new (g_direct_hash, g_direct_equal);
1806   smtpd_debug_free = g_hash_table_new (g_direct_hash, g_direct_equal);
1807   qmgr_debug_free = g_hash_table_new (g_direct_hash, g_direct_equal);
1808   filter_debug_free = g_hash_table_new (g_direct_hash, g_direct_equal);
1809 #endif
1810
1811   if (!(parser = parser_new ())) {
1812     fprintf (stderr, "unable to alloc parser structure\n");
1813     exit (-1);
1814   }
1815
1816   while ((opt = getopt (argc, argv, "f:t:s:e:h:m:q:x:i:l:vgn")) != -1) {
1817     if (opt == 'f') {
1818       parser->from = epool_strdup (&parser->ep, optarg);
1819     } else if (opt == 't') {
1820       parser->to = epool_strdup (&parser->ep, optarg);
1821     } else if (opt == 'v') {
1822       parser->verbose += 1;
1823     } else if (opt == 'g') {
1824       parser->exclude_greylist = 1;
1825     } else if (opt == 'n') {
1826       parser->exclude_ndrs = 1;
1827     } else if (opt == 'h') {
1828       parser->server = epool_strdup (&parser->ep, optarg);
1829     } else if (opt == 'm') {
1830       parser->msgid = epool_strdup (&parser->ep, optarg);
1831     } else if (opt == 'q') {
1832       time_t ltime;
1833       unsigned long rel_line_nr;
1834       MatchList *match = (MatchList *)epool_alloc(&parser->ep, sizeof(MatchList));
1835       if (sscanf(optarg, "T%08lXL%08lX", &ltime, &rel_line_nr) == 2) {
1836         match->mtype = MatchTypeRelLineNr;
1837         match->ltime = ltime;
1838         match->rel_line_nr = rel_line_nr;
1839         match->next = parser->match_list;
1840         parser->match_list = match;
1841       } else {
1842         match->mtype = MatchTypeQID;
1843         match->id = epool_strdup(&parser->ep, optarg);
1844         match->next = parser->match_list;
1845         parser->match_list = match;
1846       }
1847     } else if (opt == 'x') {
1848       parser->strmatch = epool_strdup (&parser->ep, optarg);
1849     } else if (opt == 'i') {
1850       inputfile = optarg;
1851     } else if (opt == 'l') {
1852       char *l;
1853       parser->limit = strtoul (optarg, &l, 0);
1854       if (!*optarg || *l) {
1855         fprintf (stderr, "unable to parse limit '%s'\n", optarg);
1856         exit (-1);
1857       }
1858     } else if (opt == 's') {
1859       // use strptime to convert time
1860       struct tm tm;
1861       char *res;
1862       if ((!(res = strptime (optarg, "%F %T", &tm)) &&
1863            !(res = strptime (optarg, "%s", &tm))) || *res) {
1864         fprintf (stderr, "unable to parse start time\n");
1865         exit (-1);
1866       } else {
1867         parser->start = mkgmtime (&tm);
1868       }
1869     } else if (opt == 'e') {
1870       struct tm tm;
1871       char *res;
1872       if ((!(res = strptime (optarg, "%F %T", &tm)) &&
1873            !(res = strptime (optarg, "%s", &tm))) || *res) {
1874         fprintf (stderr, "unable to parse end time\n");
1875         exit (-1);
1876       } else {
1877         parser->end = mkgmtime (&tm);
1878       }
1879     } else {
1880       print_usage (argv[0]);
1881       exit (-1);
1882     }
1883   }
1884
1885   if (optind < argc) {
1886
1887     if ((argc - optind) > 1) {
1888       print_usage (argv[0]);
1889       exit (-1);
1890     }
1891
1892     char *tmpfn = g_strdup_printf ("/tmp/.proxtrack-%08X.txt", getpid ());
1893
1894     if ((stdout = freopen (tmpfn, "w", stdout)) == NULL) {
1895       perror ("unable to open output file");
1896       exit (-1);
1897     }
1898     if (rename (tmpfn, argv[optind]) != 0) {
1899       perror ("unable to open output file");
1900       unlink (tmpfn);
1901       exit (-1);
1902     }
1903   }
1904
1905   // we use gmtime exerywhere to speedup things, this can cause problems
1906   // when daylight saving time changes
1907
1908   gettimeofday(&tv, NULL);
1909   ltime = localtime (&tv.tv_sec);
1910
1911   if (!parser->start) {
1912     ltime->tm_sec = 0;
1913     ltime->tm_min = 0;
1914     ltime->tm_hour = 0;
1915     parser->start = mkgmtime (ltime);
1916   }
1917
1918   ltime = localtime (&tv.tv_sec);
1919
1920   if (!parser->end) {
1921     parser->end = mkgmtime (ltime);
1922   }
1923
1924   if (parser->end < parser->start) {
1925     fprintf (stderr, "end time before start time\n");
1926     exit (-1);
1927   }
1928
1929   int filecount;
1930   if (inputfile) {
1931       filecount = 1;
1932   } else if ((filecount = parser_count_files (parser)) <= 0) {
1933     fprintf (stderr, "unable to access log files\n");
1934     exit (-1);
1935   }
1936
1937   printf ("# LogReader: %d\n", getpid());
1938
1939   printf ("# Query options\n");
1940   if (parser->from) printf ("# Sender:    %s\n", parser->from);
1941   if (parser->to) printf ("# Recipient: %s\n", parser->to);
1942   if (parser->server) printf ("# Server:    %s\n", parser->server);
1943   if (parser->msgid) printf ("# MsgID:     %s\n", parser->msgid);
1944
1945   MatchList *match = parser->match_list;
1946   while (match) {
1947     if (match->mtype == MatchTypeQID) {
1948       printf ("# QID:       %s\n", match->id);
1949     } else if (match->mtype == MatchTypeRelLineNr) {
1950       printf ("# QID:       T%08lXL%08lX\n", match->ltime, match->rel_line_nr);
1951     } else {
1952       g_error("internal error - unknown match type %d\n", match->mtype);
1953     }
1954     match = match->next;
1955   }
1956
1957   if (parser->strmatch) printf ("# Match:     %s\n", parser->strmatch);
1958
1959   strftime (linebuf, 256, "%F %T", gmtime (&parser->start));
1960   printf ("# Start:     %s (%lu)\n", linebuf, parser->start);
1961   strftime (linebuf, 256, "%F %T", gmtime (&parser->end));
1962   printf ("# END:       %s (%lu)\n", linebuf, parser->end);
1963   printf ("# End Query Options\n\n");
1964
1965   fflush (stdout);
1966
1967   start = parser->start;
1968   end = parser->end;
1969   ctime = 0;
1970
1971   for (i = filecount - 1; i >= 0; i--) {
1972     gpointer stream;
1973
1974     // syslog files does not conain years, so we must compute then
1975     // cur_year is the assumed start year
1976
1977     cur_month = 0;
1978     cur_year = parser->year[i];
1979
1980     if (i <= 1) {
1981       if (inputfile && strlen(inputfile) == 1 && *inputfile == '-') {
1982         stream = (gpointer) stdin;
1983       } else if (inputfile) {
1984         if (!(stream = (gpointer) fopen (inputfile, "r"))) {
1985           fprintf(stderr, "unable to open log file\n");
1986           exit (-1);
1987         }
1988       } else if (!(stream = (gpointer) fopen (logfiles[i], "r"))) continue;
1989     } else {
1990       if (!(stream = (gpointer) gzopen (logfiles[i], "r"))) continue;
1991     }
1992
1993     while (1) {
1994
1995       if (parser->limit && (parser->count >= parser->limit)) {
1996         printf ("STATUS: aborted by limit (too many hits)\n");
1997         exit (0);
1998       }
1999
2000       if (i <= 1) {
2001         line = fgets (linebuf, linebufsize, (FILE *)stream);
2002       } else {
2003         line = mygzgets ((gzFile)stream, linebuf, linebufsize);
2004       }
2005
2006       if (!line) break;
2007
2008       int len = strlen (line);
2009       int pid = 0;
2010
2011       cpos = line;
2012
2013       next_ctime = parse_time (&cpos, len);
2014
2015       if (!next_ctime) {
2016         continue;
2017       }
2018
2019       if (next_ctime != ctime) {
2020         rel_line_nr = 0;
2021       } else {
2022         rel_line_nr++;
2023       }
2024
2025       ctime = next_ctime;
2026
2027       if (ctime < start) continue;
2028       if (ctime > end) break;
2029
2030       lines++;
2031
2032       found = 0; while (!isspace (*cpos)) { cpos++; found = 1; }
2033       if (!found) {
2034         debug_error ("missing hostname", line);
2035         continue;
2036       }
2037
2038       found = 0; while (isspace (*cpos)) { cpos++; found = 1; }
2039       if (!found) {
2040         debug_error ("missing spaces after host", line);
2041         continue;
2042       }
2043
2044       if ((*cpos == 'l') && !strncmp (cpos, "last message repeated", 21)) {
2045         continue;
2046       }
2047
2048       //printf ("LINE: %s\n", line);
2049       //const char *prog = cpos;
2050
2051       csum_prog = 0;
2052       found = 0; while (*cpos && (*cpos != ':') && (*cpos != '[')) {
2053         csum_prog = ((csum_prog << 8)|(csum_prog >> 24)) + *cpos;
2054         cpos++;
2055         found++;
2056       }
2057
2058       //idx1 = g_strndup (prog, found);
2059       //printf ("TEST:%s:%08X\n", idx1, csum_prog);
2060       //free (idx1);
2061
2062       if (*cpos == '[') {
2063         cpos++;
2064         found = 0; while (isdigit (*cpos)) {
2065           pid = pid*10 + *cpos - 48;
2066           cpos++;
2067           found++;
2068         }
2069         if (found < 1 || found > 15 || *cpos != ']') {
2070           debug_error ("unable to parse pid", line);
2071           continue;
2072         }
2073         cpos++;
2074       }
2075
2076       if (*cpos++ != ':') {
2077         debug_error ("missing collon", line);
2078         continue;
2079       }
2080
2081
2082       if (!isspace (*cpos++)) {
2083         debug_error ("missing space after collon", line);
2084         continue;
2085       }
2086
2087       text = cpos;
2088
2089       parser->ctime = ctime;
2090
2091       int strmatch = 0;
2092
2093       if (parser->strmatch && STRMATCH(parser->strmatch, text)) {
2094         strmatch = 1;
2095       }
2096
2097       if ((csum_prog == PROXPROX) ||
2098           (csum_prog == PMG_SMTP_FILTER)) {
2099
2100         if ((idx1 = parse_qid (&cpos, qidbuf, ':', 25))) {
2101
2102           FEntry *fe;
2103
2104           if (!(fe = fentry_get (parser, idx1))) {
2105             continue;
2106           }
2107
2108           loglist_add (&fe->ep, &fe->loglist, line, len, lines);
2109
2110           if (strmatch) fe->strmatch = 1;
2111
2112           //fixme: BCC, Notify?
2113           //fixme: parse virus info
2114           //fixme: parse spam score
2115
2116           if ((*cpos == 'a') && !strncmp (cpos, "accept mail to <", 16)) {
2117
2118             const char *to_s, *to_e;
2119
2120             to_s = cpos = cpos + 16;
2121
2122             while (*cpos && (*cpos != '>')) { cpos++; }
2123
2124             if (*cpos != '>') continue;
2125
2126             to_e = cpos;
2127
2128             cpos++;
2129
2130             if ((*cpos++ != ' ') || (*cpos++ != '(')) continue;
2131
2132             if (!(idx1 = parse_qid (&cpos, qidbuf, ')', 15))) continue;
2133
2134             // parser_track (parser, idx1, 1);
2135
2136             fentry_tolist_add (fe, 'A', to_s, to_e - to_s, idx1, strlen (idx1));
2137
2138           } else if ((*cpos == 'm') && !strncmp (cpos, "moved mail for <", 16)) {
2139             const char *to_s, *to_e;
2140
2141             to_s = cpos = cpos + 16;
2142
2143             while (*cpos && (*cpos != '>')) { cpos++; }
2144
2145             to_e = cpos;
2146
2147             if (strncmp (cpos, "> to ", 5)) continue;
2148             cpos += 5;
2149
2150             if (!strncmp (cpos, "spam", 4)) {
2151               cpos += 4;
2152             } else if (!strncmp (cpos, "virus", 5)) {
2153               cpos += 5;
2154             } else {
2155               continue;
2156             }
2157
2158             if (strncmp (cpos, " quarantine - ", 14)) continue;
2159             cpos += 14;
2160
2161             if (!(idx1 = parse_qid (&cpos, qidbuf, 0, 25))) continue;
2162
2163             fentry_tolist_add (fe, 'Q', to_s, to_e - to_s, idx1, strlen (idx1));
2164
2165           } else if ((*cpos == 'b') && !strncmp (cpos, "block mail to <", 15)) {
2166
2167             const char *to_s;
2168
2169             to_s = cpos = cpos + 15;
2170
2171             while (*cpos && (*cpos != '>')) { cpos++; }
2172
2173             if (*cpos != '>') continue;
2174
2175             fentry_tolist_add (fe, 'B', to_s, cpos - to_s, NULL, 0);
2176
2177           } else if ((*cpos == 'p') && !strncmp (cpos, "processing time: ", 17)) {
2178             cpos += 17;
2179
2180             sscanf (cpos, "%f", &fe->ptime);
2181
2182             fe->finished = 1;
2183           }
2184
2185         }
2186
2187       } else if (csum_prog == POSTFIX_POSTSCREEN) {
2188
2189               SEntry *se;
2190
2191               if (!pid) {
2192                       debug_error ("no pid for postscreen", line);
2193                       continue;
2194               }
2195
2196
2197               if ((*text == 'N') && !strncmp (text, "NOQUEUE: reject: RCPT from ", 27)) {
2198
2199                       cpos = text + 27;
2200
2201                       if (!(idx1 = strstr (cpos, "; client ["))) continue;
2202
2203                       const char *client = cpos = idx1 + 10;
2204
2205                       while (*cpos && (*cpos != ']')) { cpos++; }
2206
2207                       const char *client_end = cpos;
2208
2209                       if (!(idx1 = strstr (cpos, "; from=<"))) continue;
2210
2211                       const char *from = cpos = idx1 + 8;
2212
2213                       while (*cpos && (*cpos != '>')) { cpos++; }
2214                       idx1 = cpos;
2215
2216                       if ((*cpos == '>') && strncmp (cpos, ">, to=<", 7)) continue;
2217
2218                       const char *to = cpos = cpos + 7;
2219
2220                       while (*cpos && (*cpos != '>')) { cpos++; }
2221
2222                       if (*cpos != '>') continue;
2223
2224                       if (!(se = sentry_get (parser, pid, ctime, rel_line_nr))) {
2225                               continue;
2226                       }
2227
2228                       if (strmatch) se->strmatch = 1;
2229
2230                       loglist_add (&se->ep, &se->loglist, line, len, lines);
2231
2232                       sentry_nqlist_add (se, ctime, from, idx1 - from, to, cpos - to, 'N');
2233
2234                       sentry_set_connect (se, client, client_end - client);
2235
2236                       g_hash_table_remove (parser->smtpd_h, &se->pid);
2237
2238                       se->disconnect = 1;
2239
2240                       sentry_print (parser, se);
2241                       sentry_free (parser, se);
2242               }
2243
2244       } else if (csum_prog == POSTFIX_QMGR) {
2245
2246         if ((idx2 = text) && (idx1 = parse_qid (&idx2, qidbuf, ':', 15))) {
2247
2248           QEntry *qe;
2249
2250           if (!(qe = qentry_get (parser, idx1))) {
2251             continue;
2252           }
2253
2254           if (strmatch) qe->strmatch = 1;
2255
2256           qe->cleanup = 1;
2257
2258           loglist_add (&qe->ep, &qe->loglist, line, len, lines);
2259
2260           if ((*idx2 == 'f') && !strncmp (idx2, "from=<", 6)) {
2261
2262             cpos = idx2 = idx2 + 6;
2263             while (*cpos && (*cpos != '>')) { cpos++; }
2264
2265             if (*cpos != '>') {
2266               debug_error ("unable to parse 'from' address", line);
2267               continue;
2268             }
2269
2270             qentry_set_from (qe, idx2, cpos - idx2);
2271
2272             cpos++;
2273
2274             if ((*cpos == ',') && !strncmp (cpos, ", size=", 7)) {
2275               int size = 0;
2276               cpos += 7;
2277
2278               while (isdigit (*cpos)) { size = size*10 + *cpos++ - 48; }
2279
2280               qe->size = size;
2281             }
2282
2283           } else if ((*idx2 == 'r') && !strncmp (idx2, "removed\n", 8)) {
2284
2285             qe->removed = 1;
2286
2287             qentry_finalize (parser, qe);
2288           }
2289         }
2290
2291       } else if ((csum_prog == POSTFIX_SMTP) ||
2292                  (csum_prog == POSTFIX_LMTP) ||
2293                  (csum_prog == POSTFIX_LOCAL) ||
2294                  (csum_prog == POSTFIX_ERROR)) {
2295
2296         int lmtp = (csum_prog == POSTFIX_LMTP);
2297
2298         if ((cpos = text) && (idx1 = parse_qid (&cpos, qidbuf, ':', 15))) {
2299
2300           QEntry *qe;
2301
2302           if (!(qe = qentry_get (parser, idx1))) {
2303             continue;
2304           }
2305
2306           if (strmatch) qe->strmatch = 1;
2307
2308           qe->cleanup = 1;
2309
2310           loglist_add (&qe->ep, &qe->loglist, line, len, lines);
2311
2312           if (strncmp (cpos, "to=<", 4)) continue;
2313           cpos += 4;
2314
2315           const char *to_s, *to_e;
2316
2317           to_s = cpos;
2318
2319           while (*cpos && (*cpos != '>')) { cpos++; }
2320
2321           if (*cpos != '>') continue;
2322
2323           to_e = cpos;
2324
2325           cpos ++;
2326
2327           if (!(cpos = strstr (cpos, ", relay="))) continue;
2328           cpos += 8;
2329
2330           const char *relay_s, *relay_e;
2331
2332           relay_s = cpos;
2333
2334           while (*cpos && (*cpos != ',')) { cpos++; }
2335
2336           if (*cpos != ',') continue;
2337
2338           relay_e = cpos;
2339
2340           if (!(idx1 = strstr (cpos + 1, ", dsn="))) continue;
2341
2342           cpos = idx1 + 6;
2343
2344           if (!isdigit (*cpos)) continue;
2345
2346           char dstatus = *cpos;
2347
2348           qentry_tolist_add (qe, ctime, dstatus, to_s, to_e - to_s,
2349                              relay_s, relay_e - relay_s);
2350
2351           if (!lmtp) continue; // filter always uses lmtp
2352
2353           if (!(idx1 = strstr (cpos + 1, "status=sent (250 2.")))
2354             continue;
2355
2356           cpos = idx1 = idx1 + 19;
2357
2358           if (*cpos == '5' && (idx1 = strstr (cpos, "5.0 OK ("))) {
2359             cpos = idx1 = idx1 + 8;
2360           } else if (*cpos == '7' && (idx1 = strstr (cpos, "7.0 BLOCKED ("))) {
2361             cpos = idx1 = idx1 + 13;
2362           } else {
2363             continue;
2364           }
2365
2366           if (!(idx1 = parse_qid (&cpos, qidbuf, ')', 25)))
2367             continue;
2368
2369           FEntry *fe;
2370
2371           qe->filtered = 1;
2372
2373           if ((fe = g_hash_table_lookup (parser->filter_h, idx1))) {
2374             qe->filter = fe;
2375           }
2376         }
2377
2378       } else if (csum_prog == POSTFIX_SMTPD) {
2379         SEntry *se;
2380
2381         if (!pid) {
2382           debug_error ("no pid for smtpd", line);
2383           continue;
2384         }
2385
2386         if (!(se = sentry_get (parser, pid, ctime, rel_line_nr))) {
2387           continue;
2388         }
2389
2390         if (strmatch) se->strmatch = 1;
2391
2392         loglist_add (&se->ep, &se->loglist, line, len, lines);
2393
2394         if ((*text == 'c') && !strncmp (text, "connect from ", 13)) {
2395
2396           cpos = idx1 = text + 13;
2397
2398           while (*idx1 && !isspace (*idx1)) { idx1++; }
2399
2400           sentry_set_connect (se, cpos, idx1 - cpos);
2401
2402           // fixme: do we need this?
2403           //if (strcmp (se->connect, "localhost[127.0.0.1]")) {
2404           //  se->external = 1;
2405           //}
2406
2407         } else if ((*text == 'd') && !strncmp (text, "disconnect from", 15)) {
2408
2409           // unlink
2410           g_hash_table_remove (parser->smtpd_h, &se->pid);
2411
2412           se->disconnect = 1;
2413
2414           if (sentry_ref_rem_unneeded (parser, se) == 0) {
2415             sentry_print (parser, se);
2416             sentry_free (parser, se);
2417           } else {
2418             sentry_ref_finalize (parser, se);
2419           }
2420
2421         } else if ((*text == 'N') && !strncmp (text, "NOQUEUE:", 8)) {
2422
2423           cpos = text + 8;
2424
2425           // parse 'whatsup' (reject:)
2426           while (*cpos && (*cpos != ':')) { cpos++; }
2427           if (*cpos != ':') continue;
2428           cpos++;
2429
2430           // parse '%s from %s:'
2431           while (*cpos && (*cpos != ':')) { cpos++; }
2432           if (*cpos != ':') continue;
2433           idx1 = cpos++;
2434
2435           // parse '%s;'
2436           while (*cpos && (*cpos != ';')) { cpos++; }
2437           if (*cpos != ';') continue;
2438
2439           // greylisting test
2440           char dstatus = 'N';
2441           *(char *)cpos = 0; // dangerous hack: delimit string
2442           if (strstr (idx1, ": Recipient address rejected: Service is unavailable (try later)")) {
2443             dstatus = 'G';
2444           }
2445           *(char *)cpos = ';'; // dangerous hack: restore line
2446
2447           if (!(idx1 = strstr (cpos, "; from=<"))) continue;
2448
2449           const char *from = cpos = idx1 + 8;
2450
2451           while (*cpos && (*cpos != '>')) { cpos++; }
2452           idx1 = cpos;
2453
2454           if ((*cpos == '>') && strncmp (cpos, "> to=<", 6)) continue;
2455
2456           const char *to = cpos = cpos + 6;
2457
2458           while (*cpos && (*cpos != '>')) { cpos++; }
2459
2460           if (*cpos != '>') continue;
2461
2462           sentry_nqlist_add (se, ctime, from, idx1 - from, to, cpos - to, dstatus);
2463
2464         } else if ((idx2 = text) && (idx1 = parse_qid (&idx2, qidbuf, ':', 15))) {
2465
2466           QEntry *qe;
2467
2468           if ((qe = qentry_get (parser, idx1))) {
2469
2470             if (strmatch) qe->strmatch = 1;
2471
2472             sentry_ref_add (se, qe);
2473
2474             if ((*idx2 == 'c') && !strncmp (idx2, "client=", 7)) {
2475               cpos = idx2 = idx2 + 7;
2476
2477               while (*cpos && !isspace (*cpos)) { cpos++; }
2478
2479               qentry_set_client (qe, idx2, cpos - idx2);
2480             }
2481           }
2482
2483         }
2484
2485       } else if (csum_prog == POSTFIX_CLEANUP) {
2486
2487         QEntry *qe;
2488
2489         idx2 = text;
2490         if ((idx1 = parse_qid (&idx2, qidbuf, ':', 15))) {
2491           if ((qe = qentry_get (parser, idx1))) {
2492
2493             if (strmatch) qe->strmatch = 1;
2494
2495             loglist_add (&qe->ep, &qe->loglist, line, len, lines);
2496
2497             if ((*idx2 == 'm') && !strncmp (idx2, "message-id=", 11)) {
2498
2499               cpos = idx2 = idx2 + 11;
2500
2501               while (*cpos && !isspace(*cpos)) { cpos++; }
2502
2503               qentry_set_msgid (qe, idx2, cpos - idx2);
2504
2505               qe->cleanup = 1;
2506             }
2507           }
2508         }
2509       }
2510     }
2511
2512     if (i <= 1) {
2513       fclose ((FILE *)stream);
2514     } else {
2515       gzclose ((gzFile)stream);
2516     }
2517
2518     if (ctime > end) break;
2519   }
2520
2521
2522 #ifdef DEBUG
2523   printf ("LINES: %d\n", lines);
2524   printf ("MEM SMTPD  entries: %d\n", g_hash_table_size (parser->smtpd_h));
2525   printf ("MEM QMGR   entries: %d\n", g_hash_table_size (parser->qmgr_h));
2526   printf ("MEM FILTER entries: %d\n", g_hash_table_size (parser->filter_h));
2527
2528   printf ("MEMDEB SMTPD entries: %d %d\n",
2529           g_hash_table_size (smtpd_debug_alloc),
2530           g_hash_table_size (smtpd_debug_free));
2531   printf ("MEMDEB QMGR  entries: %d %d\n",
2532           g_hash_table_size (qmgr_debug_alloc),
2533           g_hash_table_size (qmgr_debug_free));
2534   printf ("MEMDEB FILTER  entries: %d %d\n",
2535           g_hash_table_size (filter_debug_alloc),
2536           g_hash_table_size (filter_debug_free));
2537 #endif
2538
2539   g_hash_table_foreach (parser->qmgr_h, qentry_cleanup_hash, parser);
2540   g_hash_table_foreach (parser->smtpd_h, sentry_cleanup_hash, parser);
2541   g_hash_table_foreach (parser->filter_h, fentry_cleanup_hash, parser);
2542
2543 #ifdef DEBUG
2544   printf ("MEMDEB SMTPD entries: %d %d\n",
2545           g_hash_table_size (smtpd_debug_alloc),
2546           g_hash_table_size (smtpd_debug_free));
2547   printf ("MEMDEB QMGR  entries: %d %d\n",
2548           g_hash_table_size (qmgr_debug_alloc),
2549           g_hash_table_size (qmgr_debug_free));
2550   printf ("MEMDEB FILTER  entries: %d %d\n",
2551           g_hash_table_size (filter_debug_alloc),
2552           g_hash_table_size (filter_debug_free));
2553
2554   g_hash_table_foreach (smtpd_debug_alloc, sentry_debug_alloc, parser);
2555
2556 #endif
2557
2558
2559 #ifdef EPOOL_DEBUG
2560   printf ("MEMMAX %d\n", ep_maxalloc);
2561 #endif
2562
2563   parser_free (parser);
2564
2565   fclose (stdout);
2566
2567   exit (0);
2568 }