pmg-log-tracker.c

   1 /*
   2
   3   (C) 2007-2017 Proxmox Server Solutions GmbH, All Rights Reserved
   4
   5   Proxmox Mail Tracker
   6
   7   This program is free software: you can redistribute it and/or modify
   8   it under the terms of the GNU Affero General Public License as published by
   9   the Free Software Foundation, either version 3 of the License, or
  10   (at your option) any later version.
  11
  12   This program is distributed in the hope that it will be useful,
  13   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15   GNU Affero General Public License for more details.
  16
  17   You should have received a copy of the GNU Affero General Public License
  18   along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20   Author: Dietmar Maurer <dietmar@proxmox.com>
  21
  22   See http://www.proxmox.com for more information
  23
  24 */
  25
  26 #define _GNU_SOURCE
  27
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30 #include <glib.h>
  31 #include <string.h>
  32 #include <ctype.h>
  33 #include <sys/time.h>
  34 #include <time.h>
  35 #include <unistd.h>
  36 #include <sys/types.h>
  37 #include <zlib.h>
  38 #include <fnmatch.h>
  39
  40 /*
  41   We assume the syslog files belong to one host, i.e. we do not
  42   consider the hostname at all
  43 */
  44
  45 /*
  46   REGEX: we do not use posix regular expressions (libc implementation
  47   is too slow). fnmatch() is also slow.
  48   Future versions may use our own implementation of a DFA. But currently we
  49   just use strstr (no regex support at all)
  50 */
  51
  52 //#define STRMATCH(pattern, string) (fnmatch (pattern, string, FNM_CASEFOLD) == 0)
  53 #define STRMATCH(pattern, string) (strcasestr (string, pattern) != NULL)
  54
  55
  56 //#define EPOOL_BLOCK_SIZE 512
  57 //#define EPOOL_MAX_SIZE 128
  58 #define EPOOL_BLOCK_SIZE 2048
  59 #define EPOOL_MAX_SIZE 128
  60 #define MAX_LOGFILES 32
  61 //#define EPOOL_DEBUG
  62 //#define DEBUG
  63
  64 typedef struct _SList SList;
  65 struct _SList {
  66   gpointer data;
  67   SList *next;
  68 };
  69
  70 typedef struct _NQList NQList;
  71 struct _NQList {
  72   char *from;
  73   char *to;
  74   char dstatus;
  75   time_t ltime;
  76   NQList *next;
  77 };
  78
  79 typedef struct _TOList TOList;
  80 struct _TOList {
  81   char *to;
  82   char *relay;
  83   char dstatus;
  84   time_t ltime;
  85   TOList *next;
  86 };
  87
  88 #define MatchTypeQID 1
  89 #define MatchTypeRelLineNr 2
  90
  91 typedef struct _MatchList MatchList;
  92 struct _MatchList {
  93   unsigned int mtype;
  94   char *id;
  95   time_t ltime;
  96   unsigned long rel_line_nr;
  97   MatchList *next;
  98 };
  99
 100 #ifdef DEBUG
 101   GHashTable *smtpd_debug_alloc;
 102   GHashTable *qmgr_debug_alloc;
 103   GHashTable *filter_debug_alloc;
 104   GHashTable *smtpd_debug_free;
 105   GHashTable *qmgr_debug_free;
 106   GHashTable *filter_debug_free;
 107 #endif
 108
 109 // EPool: Entry related memory pools
 110
 111
 112 #ifdef EPOOL_DEBUG
 113 int ep_allocated;
 114 int ep_maxalloc;
 115 #endif
 116
 117 typedef struct _EPool EPool;
 118 struct _EPool {
 119   SList *blocks;     // allocated usiing g_slice_alloc(EPOOL_BLOCK_SIZE)
 120   SList *mblocks;    // allocated use g_malloc
 121   int cpos;
 122 #ifdef EPOOL_DEBUG
 123   int allocated;
 124 #endif
 125 };
 126
 127 typedef struct {
 128   EPool ep;
 129   GHashTable *smtpd_h;
 130   GHashTable *qmgr_h;
 131   GHashTable *filter_h;
 132   //GHashTable *track_h;
 133   gzFile stream[MAX_LOGFILES];
 134   char *from;
 135   char *to;
 136   time_t year[MAX_LOGFILES];
 137   time_t start;
 138   time_t end;
 139   time_t ctime;
 140   MatchList *match_list;
 141   char *server;
 142   char *msgid;
 143   char *strmatch;
 144   unsigned long limit;
 145   unsigned long count;
 146   int verbose;
 147   unsigned int exclude_greylist:1;
 148   unsigned int exclude_ndrs:1;
 149 } LParser;
 150
 151 typedef struct _SEntry SEntry;
 152 typedef struct _QEntry QEntry;
 153 typedef struct _FEntry FEntry;
 154
 155 typedef struct _LogEntry LogEntry;
 156 typedef struct _LogList LogList;
 157
 158 struct _LogEntry {
 159   const char *text;
 160   unsigned long linenr;
 161   LogEntry *next;
 162 };
 163
 164 struct _LogList {
 165   LogEntry *log;
 166   LogEntry *logs_last; // pointer to last log (speedup add log)
 167 };
 168
 169 // SEntry: Store SMTPD related logs
 170
 171 struct _SEntry {
 172
 173   EPool ep;
 174   LogList loglist;
 175
 176   int pid;
 177
 178   SList *refs;
 179
 180   NQList *nqlist;
 181
 182   char *connect;
 183
 184  // time,rel_line_nr is used as cursor/ID
 185   time_t ltime;
 186   unsigned long rel_line_nr;
 187
 188   //unsigned int external:1;        // not from local host
 189   unsigned int disconnect:1;
 190   unsigned int strmatch:1;
 191
 192 };
 193
 194 // QEntry: Store Queue (qmgr, smtp, lmtp) related logs
 195
 196 struct _QEntry {
 197
 198   EPool ep;
 199   LogList loglist;
 200
 201   char *qid;
 202
 203   SEntry *smtpd;
 204   FEntry *filter;
 205
 206   TOList *tolist;
 207
 208   unsigned int size;
 209   char *from;
 210   char *client;
 211   char *msgid;
 212
 213   unsigned int cleanup:1;
 214   unsigned int removed:1;
 215   unsigned int filtered:1; // set when passed via lmtp to filter
 216   unsigned int strmatch:1;
 217 };
 218
 219 // FEntry: Store filter (proxprox) related logs
 220
 221 struct _FEntry {
 222
 223   EPool ep;
 224   LogList loglist;
 225
 226   char *logid; // proxprox log id
 227
 228   TOList *tolist;
 229
 230   float ptime;
 231
 232   unsigned int finished:1;
 233   unsigned int strmatch:1;
 234 };
 235
 236 // Prototypes
 237 void      debug_error (char *msg, const char *line);
 238
 239 EPool    *epool_init (EPool *ep);
 240 void      epool_free (EPool *ep);
 241 gpointer  epool_alloc (EPool *ep, int size);
 242 char     *epool_strndup (EPool *ep, const char *s, int len);
 243 char     *epool_strndup0 (EPool *ep, const char *s, int len);
 244 char     *epool_strdup (EPool *ep, const char *s);
 245
 246 void      loglist_print (LogList *loglist);
 247 void      loglist_add (EPool *ep, LogList *loglist, const char *text, int len, unsigned long linenr);
 248
 249 SEntry   *sentry_new (int pid, time_t ltime, unsigned long rel_line_nr);
 250 SEntry   *sentry_get (LParser *parser, int pid, time_t ltime, unsigned long rel_line_nr);
 251 void      sentry_ref_add (SEntry *sentry, QEntry *qentry);
 252 int       sentry_ref_del (SEntry *sentry, QEntry *qentry);
 253 void      sentry_ref_finalize (LParser *parser, SEntry *sentry);
 254 int       sentry_ref_rem_unneeded (LParser *parser, SEntry *sentry);
 255 void      sentry_nqlist_add (SEntry *sentry, time_t ltime, const char *from, int from_len,
 256                              const char *to, int to_len, char dstatus);
 257 void      sentry_print (LParser *parser, SEntry *sentry);
 258 void      sentry_set_connect (SEntry *sentry, const char *connect, int len);
 259 void      sentry_free_noremove (SEntry *sentry);
 260 void      sentry_free (LParser *parser, SEntry *sentry);
 261 void      sentry_cleanup_hash (gpointer key, gpointer value, gpointer user_data);
 262
 263
 264 QEntry   *qentry_new (const char *qid);
 265 QEntry   *qentry_get (LParser *parser, const char *qid);
 266 void      qentry_tolist_add (QEntry *qentry, time_t ltime, char dstatus, const char *to,
 267                              int to_len, const char *relay, int relay_len);
 268 void      qentry_set_from (QEntry *qentry, const char *from, int len);
 269 void      qentry_set_msgid (QEntry *qentry, const char *msgid, int len);
 270 void      qentry_set_client (QEntry *qentry, const char *client, int len);
 271 void      qentry_print (LParser *parser, QEntry *qentry);
 272 void      qentry_finalize (LParser *parser, QEntry *qentry);
 273 void      qentry_free_noremove (LParser *parser, QEntry *qentry);
 274 void      qentry_free (LParser *parser, QEntry *qentry);
 275 void      qentry_cleanup_hash (gpointer key, gpointer value, gpointer user_data);
 276
 277
 278 FEntry   *fentry_new (const char *logid);
 279 FEntry   *fentry_get (LParser *parser, const char *logid);
 280 void      fentry_tolist_add (FEntry *fentry, char dstatus, const char *to,
 281                              int to_len, const char *qid, int qid_len);
 282 void      fentry_free_noremove (LParser *parser, FEntry *fentry);
 283 void      fentry_free (LParser *parser, FEntry *fentry);
 284 void      fentry_cleanup_hash (gpointer key, gpointer value, gpointer user_data);
 285
 286 LParser  *parser_new ();
 287 void      parser_free (LParser *parser);
 288
 289 //char     *parser_track (LParser *parser, const char *qid, gboolean insert);
 290
 291 // Implementations
 292
 293 //#define LOGPATH "./log/"
 294 #define LOGPATH "/var/log/"
 295 //#define LOGPATH "/root/testlog/"
 296
 297 static const char *logfiles[] = {
 298   LOGPATH "syslog",
 299   LOGPATH "syslog.1",
 300   LOGPATH "syslog.2.gz",
 301   LOGPATH "syslog.3.gz",
 302   LOGPATH "syslog.4.gz",
 303   LOGPATH "syslog.5.gz",
 304   LOGPATH "syslog.6.gz",
 305   LOGPATH "syslog.7.gz",
 306   LOGPATH "syslog.8.gz",
 307   LOGPATH "syslog.9.gz",
 308   LOGPATH "syslog.10.gz",
 309   LOGPATH "syslog.11.gz",
 310   LOGPATH "syslog.12.gz",
 311   LOGPATH "syslog.13.gz",
 312   LOGPATH "syslog.14.gz",
 313   LOGPATH "syslog.15.gz",
 314   LOGPATH "syslog.16.gz",
 315   LOGPATH "syslog.17.gz",
 316   LOGPATH "syslog.18.gz",
 317   LOGPATH "syslog.19.gz",
 318   LOGPATH "syslog.20.gz",
 319   LOGPATH "syslog.21.gz",
 320   LOGPATH "syslog.22.gz",
 321   LOGPATH "syslog.23.gz",
 322   LOGPATH "syslog.24.gz",
 323   LOGPATH "syslog.25.gz",
 324   LOGPATH "syslog.26.gz",
 325   LOGPATH "syslog.27.gz",
 326   LOGPATH "syslog.28.gz",
 327   LOGPATH "syslog.29.gz",
 328   LOGPATH "syslog.30.gz",
 329   LOGPATH "syslog.31.gz",
 330 };
 331
 332 void
 333 debug_error (char *msg, const char *line)
 334 {
 335 #ifdef DEBUG
 336   fprintf (stderr, "ERROR: %s\n", msg);
 337   if (line) fprintf (stderr, "LINE: %s\n", line);
 338
 339   G_BREAKPOINT();
 340
 341   exit (-1);
 342 #endif
 343 }
 344
 345 EPool *
 346 epool_init (EPool *ep)
 347 {
 348   gpointer data;
 349   SList *blocks;
 350
 351   data = g_slice_alloc0(EPOOL_BLOCK_SIZE);
 352   blocks = (SList *)data;
 353
 354 #ifdef EPOOL_DEBUG
 355   ep->allocated += EPOOL_BLOCK_SIZE;
 356   ep_allocated += EPOOL_BLOCK_SIZE;
 357   ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
 358 #endif
 359
 360
 361   blocks->data = data;
 362   blocks->next = NULL;
 363
 364   ep->blocks = blocks;
 365   ep->cpos = sizeof (SList);
 366
 367   return ep;
 368 }
 369
 370 void
 371 epool_free (EPool *ep)
 372 {
 373   SList *l;
 374   gpointer data;
 375
 376 #ifdef EPOOL_DEBUG
 377   ep_allocated -= ep->allocated;
 378   printf ("MEM: %d\n", ep_allocated);
 379 #endif
 380
 381 #ifdef DEBUG
 382   return;
 383 #endif
 384
 385   l = ep->mblocks;
 386   while (l) {
 387     data = l->data;
 388     l = l->next;
 389     g_free (data);
 390   }
 391
 392   l = ep->blocks;
 393   while (l) {
 394     data = l->data;
 395     l = l->next;
 396     g_slice_free1(EPOOL_BLOCK_SIZE, data);
 397   }
 398 }
 399
 400 gpointer
 401 epool_alloc (EPool *ep, int size)
 402 {
 403   int rs = (size + 3) & ~3;
 404   int space = EPOOL_BLOCK_SIZE - sizeof (SList) - ep->cpos;
 405   gpointer data;
 406
 407   if (size > EPOOL_MAX_SIZE) {
 408     SList *blocks;
 409     if (space >= sizeof (SList)) {
 410       blocks = (SList *)((char *)ep->blocks->data + ep->cpos);
 411       ep->cpos += sizeof (SList);
 412     } else {
 413       blocks = (SList *)epool_alloc (ep, sizeof (SList));
 414     }
 415
 416     data = g_malloc (size);
 417 #ifdef EPOOL_DEBUG
 418     ep->allocated += size;
 419     ep_allocated += size;
 420     ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
 421 #endif
 422     blocks->data = data;
 423     blocks->next = ep->mblocks;
 424
 425     ep->mblocks = blocks;
 426
 427     return data;
 428
 429   } else if (space >= rs) {
 430     data = (char *)ep->blocks->data + ep->cpos;
 431     ep->cpos += rs;
 432
 433     return data;
 434
 435   } else {
 436     SList *blocks = (SList *)((char *)ep->blocks->data + ep->cpos);
 437
 438     data = g_slice_alloc0 (EPOOL_BLOCK_SIZE);
 439     blocks->data = data;
 440     blocks->next = ep->blocks;
 441
 442     ep->blocks = blocks;
 443     ep->cpos = rs;
 444
 445 #ifdef EPOOL_DEBUG
 446     ep->allocated += EPOOL_BLOCK_SIZE;
 447     ep_allocated += EPOOL_BLOCK_SIZE;
 448     ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
 449 #endif
 450
 451     return data;
 452   }
 453 }
 454
 455 char *
 456 epool_strndup (EPool *ep, const char *s, int len)
 457 {
 458   int l = len + 1;
 459   char *res = epool_alloc (ep, l);
 460   strncpy (res, s, l);
 461   return res;
 462 }
 463
 464 char *
 465 epool_strndup0 (EPool *ep, const char *s, int len)
 466 {
 467   char *res = epool_alloc (ep, len + 1);
 468   strncpy (res, s, len);
 469   res[len] = 0;
 470
 471   return res;
 472 }
 473
 474 char *
 475 epool_strdup (EPool *ep, const char *s)
 476 {
 477   int l = strlen (s) + 1;
 478   char *res = epool_alloc (ep, l);
 479   strncpy (res, s, l);
 480   return res;
 481 }
 482
 483 void
 484 loglist_print (LogList *loglist)
 485 {
 486   LogEntry *log = loglist->log;
 487   while (log) {
 488     printf ("L%08X %s", log->linenr, log->text);
 489     log = log->next;
 490   }
 491 }
 492
 493
 494 void
 495 loglist_add (EPool *ep, LogList *loglist, const char *text, int len, unsigned long linenr)
 496 {
 497   LogEntry *log;
 498
 499 #ifdef DEBUG
 500   if (len != strlen (text)) {
 501     debug_error ("string with wrong len", NULL);
 502   }
 503 #endif
 504   if (text[len] != 0) {
 505     debug_error ("string is not null terminated", NULL);
 506     return;
 507   }
 508
 509   log = epool_alloc (ep, sizeof (LogEntry));
 510
 511   log->text = epool_strndup (ep, text, len);
 512   log->linenr = linenr;
 513   log->next = NULL;
 514
 515   if (loglist->logs_last) {
 516     loglist->logs_last->next = log;
 517     loglist->logs_last = log;
 518   } else {
 519     loglist->log = log;
 520     loglist->logs_last = log;
 521   }
 522
 523   return;
 524 }
 525
 526 SEntry*
 527 sentry_new (int pid, time_t ltime, unsigned long rel_line_nr)
 528 {
 529   SEntry *sentry;
 530   SList *blocks;
 531   int cpos;
 532
 533   sentry = (SEntry *)g_slice_alloc0(EPOOL_BLOCK_SIZE);
 534   sentry->pid = pid;
 535   sentry->ltime = ltime;
 536   sentry->rel_line_nr = rel_line_nr;
 537
 538 #ifdef EPOOL_DEBUG
 539   sentry->ep.allocated += EPOOL_BLOCK_SIZE;
 540   ep_allocated += EPOOL_BLOCK_SIZE;
 541   ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
 542 #endif
 543
 544 #ifdef DEBUG
 545   {
 546     SEntry *se;
 547     if ((se = g_hash_table_lookup (smtpd_debug_alloc, sentry))) {
 548       debug_error ("SEntry already alloced", NULL);
 549     } else {
 550       g_hash_table_insert (smtpd_debug_alloc, sentry, sentry);
 551     }
 552   }
 553 #endif
 554
 555   cpos = sizeof (SEntry);
 556
 557   blocks = (SList *)((char *)sentry + cpos);
 558
 559   cpos += sizeof (SList);
 560
 561   blocks->data = sentry;
 562   blocks->next = NULL;
 563
 564   sentry->ep.blocks = blocks;
 565   sentry->ep.cpos = cpos;
 566
 567   return sentry;
 568 }
 569
 570 SEntry *
 571 sentry_get (LParser *parser, int pid, time_t ltime, unsigned long rel_line_nr)
 572 {
 573   SEntry *sentry;
 574
 575   if ((sentry = g_hash_table_lookup (parser->smtpd_h, &pid))) {
 576     return sentry;
 577   } else {
 578
 579     if ((sentry = sentry_new (pid, ltime, rel_line_nr))) {
 580       g_hash_table_insert (parser->smtpd_h, &sentry->pid, sentry);
 581     }
 582
 583     return sentry;
 584   }
 585 }
 586
 587 void
 588 sentry_ref_add (SEntry *sentry, QEntry *qentry)
 589 {
 590   SList *l;
 591
 592   if (qentry->smtpd) {
 593     if (qentry->smtpd != sentry) {
 594       debug_error ("qentry ref already set", NULL);
 595     }
 596     return;
 597   }
 598
 599   l = sentry->refs;
 600   while (l) {
 601     if (l->data == qentry) return;
 602     l = l->next;
 603   }
 604
 605   l = epool_alloc (&sentry->ep, sizeof (SList));
 606
 607   qentry->smtpd = sentry;
 608
 609   l->data = qentry;
 610   l->next = sentry->refs;
 611
 612   sentry->refs = l;
 613 }
 614
 615 int
 616 sentry_ref_del (SEntry *sentry, QEntry *qentry)
 617 {
 618   SList *l = sentry->refs;
 619   int count = 0;
 620
 621   if (!qentry->smtpd) {
 622     debug_error ("qentry does not hav a qentry ref", NULL);
 623     return 0;
 624   }
 625
 626   l = sentry->refs;
 627
 628   while (l) {
 629     QEntry *qe = (QEntry *)l->data;
 630     if (qe == qentry) {
 631       l->data = NULL;
 632     } else {
 633       if (qe && qe->cleanup) count++;
 634     }
 635     l = l->next;
 636   }
 637
 638   return count;
 639 }
 640
 641 void
 642 sentry_ref_finalize (LParser *parser, SEntry *sentry)
 643 {
 644   SList *l = sentry->refs;
 645
 646   int count = 0;
 647
 648   while (l) {
 649     SList *cl = l;
 650     QEntry *qe = l->data;
 651
 652     l = l->next;
 653
 654     if (!qe) continue;
 655
 656     count++;
 657
 658     if (!qe->removed) continue;
 659
 660     FEntry *fe = qe->filter;
 661
 662     if (fe && !fe->finished) continue;
 663
 664     count--;
 665
 666     qentry_print (parser, qe);
 667
 668     cl->data = NULL;
 669     qe->smtpd = NULL;
 670
 671     qentry_free (parser, qe);
 672
 673     if (fe) fentry_free (parser, fe);
 674
 675   }
 676
 677   if (!count) sentry_free_noremove (sentry);
 678 }
 679
 680 int
 681 sentry_ref_rem_unneeded (LParser *parser, SEntry *sentry)
 682 {
 683   SList *l = sentry->refs;
 684   int count = 0;
 685
 686   while (l) {
 687     QEntry *qe = (QEntry *)l->data;
 688
 689     if (qe) {
 690       if (!qe->cleanup) {
 691         qe->smtpd = NULL;
 692         qentry_free (parser, qe);
 693         l->data = NULL;
 694       } else {
 695         count++;
 696       }
 697     }
 698
 699     l = l->next;
 700   }
 701
 702   return count;
 703 }
 704
 705 void
 706 sentry_nqlist_add (SEntry *sentry, time_t ltime, const char *from, int from_len,
 707                    const char *to, int to_len, char dstatus)
 708 {
 709   NQList *nq = (NQList *)epool_alloc (&sentry->ep, sizeof (NQList));
 710
 711   nq->from = epool_strndup0 (&sentry->ep, from, from_len);
 712   nq->to = epool_strndup0 (&sentry->ep, to, to_len);
 713   nq->dstatus = dstatus;
 714
 715   nq->next = sentry->nqlist;
 716   nq->ltime = ltime;
 717   sentry->nqlist = nq;
 718 }
 719
 720 void
 721 sentry_print (LParser *parser, SEntry *sentry)
 722 {
 723   NQList *nq;
 724
 725   if (parser->msgid) return;
 726
 727   if (parser->server) {
 728     if (!sentry->connect) return;
 729     if (!strcasestr (sentry->connect, parser->server)) return;
 730   }
 731
 732   MatchList *match = parser->match_list;
 733   if (match) {
 734     int found = 0;
 735     while(match) {
 736       if (match->mtype == MatchTypeQID) {
 737         return;
 738       } else if (match->mtype == MatchTypeRelLineNr) {
 739         if (match->ltime == sentry->ltime && match->rel_line_nr == sentry->rel_line_nr) {
 740           found = 1;
 741           break;
 742         }
 743       } else {
 744         g_error("implement me");
 745       }
 746       match = match->next;
 747     }
 748     if (!found) return;
 749   }
 750
 751   if (parser->from || parser->to ||
 752       parser->exclude_greylist || parser->exclude_ndrs) {
 753     nq = sentry->nqlist;
 754     int found = 0;
 755     while (nq) {
 756       if (parser->from) {
 757         if (!*(parser->from)) {
 758           if (*(nq->from)) nq->dstatus = 0;
 759         } else if (!STRMATCH(parser->from, nq->from)) {
 760           nq->dstatus = 0;
 761         }
 762       }
 763
 764       if (parser->exclude_greylist && nq->dstatus == 'G') nq->dstatus = 0;
 765
 766       if (parser->exclude_ndrs && nq->from && !*nq->from) nq->dstatus = 0;
 767
 768       if (parser->to && nq->to && !STRMATCH(parser->to, nq->to)) {
 769         nq->dstatus = 0;
 770       }
 771
 772       if (nq->dstatus != 0) found = 1;
 773
 774       nq = nq->next;
 775     }
 776     if (!found) return;
 777   }
 778
 779   if (parser->strmatch && !sentry->strmatch) return;
 780
 781   if (parser->verbose) {
 782
 783     printf ("SMTPD: T%08lXL%08lX\n", sentry->ltime, sentry->rel_line_nr);
 784
 785     printf ("CTIME: %08lX\n", parser->ctime);
 786
 787     if (sentry->connect) { printf ("CLIENT: %s\n", sentry->connect); }
 788     //printf ("EXTERNAL: %d\n", sentry->external);
 789
 790   }
 791
 792   nq = sentry->nqlist;
 793   while (nq) {
 794     if (nq->from && nq->to && nq->dstatus) {
 795       printf ("TO:%08lX:T%08lXL%08lX:%c: from <%s> to <%s>\n", nq->ltime,
 796               sentry->ltime, sentry->rel_line_nr, nq->dstatus,
 797               nq->from, nq->to);
 798       parser->count++;
 799     }
 800     nq = nq->next;
 801   }
 802
 803   if (!parser->verbose)  { fflush (stdout); return; }
 804
 805   if (parser->verbose > 1) {
 806     printf ("LOGS:\n");
 807     loglist_print (&sentry->loglist);
 808   }
 809
 810   printf ("\n");
 811
 812   fflush (stdout);
 813 }
 814
 815 void
 816 sentry_set_connect (SEntry *sentry, const char *connect, int len)
 817 {
 818   if (sentry->connect) {
 819 #ifdef DEBUG
 820     if (strncmp (sentry->connect, connect, len)) {
 821       debug_error ("duplicate connect", NULL);
 822     }
 823 #endif
 824   } else {
 825     sentry->connect = epool_strndup0 (&sentry->ep, connect, len);
 826   }
 827 }
 828
 829 void
 830 sentry_free_noremove (SEntry *sentry)
 831 {
 832   SList *l;
 833   gpointer data;
 834
 835 #ifdef EPOOL_DEBUG
 836   ep_allocated -= sentry->ep.allocated;
 837   printf ("MEM: %d\n", ep_allocated);
 838 #endif
 839
 840 #ifdef DEBUG
 841   {
 842     SEntry *se;
 843     if ((se = g_hash_table_lookup (smtpd_debug_free, sentry))) {
 844       debug_error ("SEntry already freed", NULL);
 845     } else {
 846       g_hash_table_insert (smtpd_debug_free, sentry, sentry);
 847     }
 848   }
 849   return;
 850 #endif
 851
 852   l = sentry->ep.mblocks;
 853   while (l) {
 854     data = l->data;
 855     l = l->next;
 856     g_free (data);
 857   }
 858
 859   l = sentry->ep.blocks;
 860   while (l) {
 861     data = l->data;
 862     l = l->next;
 863     g_slice_free1(EPOOL_BLOCK_SIZE, data);
 864   }
 865 }
 866
 867 void
 868 sentry_free (LParser *parser, SEntry *sentry)
 869 {
 870   g_hash_table_remove (parser->smtpd_h, &sentry->pid);
 871
 872   sentry_free_noremove (sentry);
 873 }
 874
 875 void
 876 sentry_cleanup_hash (gpointer key,
 877                      gpointer value,
 878                      gpointer user_data)
 879 {
 880   SEntry *se = value;
 881   LParser *parser = (LParser *)user_data;
 882
 883   sentry_print (parser, se);
 884   sentry_free_noremove (se);
 885 }
 886
 887 #ifdef DEBUG
 888 void
 889 sentry_debug_alloc (gpointer key,
 890                     gpointer value,
 891                     gpointer user_data)
 892 {
 893   LParser *parser = (LParser *)user_data;
 894   SEntry *se = value;
 895   SEntry *fe;
 896
 897   if ((fe = g_hash_table_lookup (smtpd_debug_free, se))) {
 898     return;
 899   }
 900
 901   printf ("FOUND ALLOCATED SENTRY:\n");
 902   sentry_print (parser, se);
 903
 904   exit (-1);
 905 }
 906 #endif
 907
 908 // QEntry
 909
 910 QEntry*
 911 qentry_new (const char *qid)
 912 {
 913   QEntry *qentry;
 914   SList *blocks;
 915   int cpos;
 916   char *qid_cp;
 917
 918   qentry = (QEntry *)g_slice_alloc0(EPOOL_BLOCK_SIZE);
 919
 920 #ifdef EPOOL_DEBUG
 921   qentry->ep.allocated += EPOOL_BLOCK_SIZE;
 922   ep_allocated += EPOOL_BLOCK_SIZE;
 923   ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
 924 #endif
 925
 926 #ifdef DEBUG
 927   {
 928     QEntry *qe;
 929     if ((qe = g_hash_table_lookup (qmgr_debug_alloc, qentry))) {
 930       debug_error ("QEntry already alloced", NULL);
 931     } else {
 932       g_hash_table_insert (qmgr_debug_alloc, qentry, qentry);
 933     }
 934   }
 935 #endif
 936
 937   cpos = sizeof (QEntry);
 938
 939   blocks = (SList *)((char *)qentry + cpos);
 940
 941   cpos += sizeof (SList);
 942
 943   blocks->data = qentry;
 944   blocks->next = NULL;
 945
 946   qentry->qid = qid_cp = (char *)qentry + cpos;
 947   while ((*qid_cp++ = *qid++)) cpos++;
 948
 949   cpos = (cpos + 4) & ~3;
 950
 951   qentry->ep.blocks = blocks;
 952   qentry->ep.cpos = cpos;;
 953
 954   return qentry;
 955 }
 956
 957 void
 958 qentry_tolist_add (QEntry *qentry, time_t ltime, char dstatus, const char *to, int to_len,
 959                    const char *relay, int relay_len)
 960 {
 961   TOList *tl = (TOList *)epool_alloc (&qentry->ep, sizeof (TOList));
 962
 963   tl->to = epool_strndup0 (&qentry->ep, to, to_len);
 964   tl->relay = epool_strndup0 (&qentry->ep, relay, relay_len);
 965   tl->dstatus = dstatus;
 966   tl->ltime = ltime;
 967   tl->next = qentry->tolist;
 968
 969   qentry->tolist = tl;
 970 }
 971
 972 void
 973 qentry_set_from (QEntry *qentry, const char *from, int len)
 974 {
 975   if (qentry->from) {
 976 #ifdef DEBUG
 977     if (strncmp (qentry->from, from, len)) {
 978       debug_error ("duplicate from", NULL);
 979     }
 980 #endif
 981   } else {
 982     qentry->from = epool_strndup0 (&qentry->ep, from, len);
 983   }
 984 }
 985
 986 void
 987 qentry_set_msgid (QEntry *qentry, const char *msgid, int len)
 988 {
 989   if (qentry->msgid) {
 990 #ifdef DEBUG
 991     if (strncmp (qentry->msgid, msgid, len)) {
 992       debug_error ("duplicate msgid", NULL);
 993     }
 994 #endif
 995   } else {
 996     qentry->msgid = epool_strndup0 (&qentry->ep, msgid, len);
 997   }
 998 }
 999
1000 void
1001 qentry_set_client (QEntry *qentry, const char *client, int len)
1002 {
1003   if (qentry->client) {
1004 #ifdef DEBUG
1005     if (strncmp (qentry->client, client, len)) {
1006       debug_error ("duplicate client", NULL);
1007     }
1008 #endif
1009   } else {
1010     qentry->client = epool_strndup0 (&qentry->ep, client, len);
1011   }
1012 }
1013
1014 void
1015 qentry_print (LParser *parser, QEntry *qentry)
1016 {
1017   TOList *tl, *fl;
1018   SEntry *se = qentry->smtpd;
1019   FEntry *fe = qentry->filter;
1020
1021   if (parser->msgid) {
1022     if (!qentry->msgid) return;
1023     if (strcasecmp (parser->msgid, qentry->msgid)) return;
1024   }
1025
1026   MatchList *match = parser->match_list;
1027   if (match) {
1028     int found = 0;
1029     while(match) {
1030       if (match->mtype == MatchTypeQID) {
1031         if ((fe && !strcmp (fe->logid, match->id)) ||
1032             (!strcmp (qentry->qid, match->id))) {
1033           found = 1;
1034           break;
1035         }
1036       } else if (match->mtype == MatchTypeRelLineNr) {
1037         if (se && match->ltime == se->ltime && match->rel_line_nr == se->rel_line_nr) {
1038           found = 1;
1039           break;
1040         }
1041       } else {
1042         g_error("implement me");
1043       }
1044       match = match->next;
1045     }
1046     if (!found) return;
1047   }
1048
1049   if (parser->server) {
1050     int found = 0;
1051     if (se && se->connect && strcasestr (se->connect, parser->server)) found = 1;
1052     if (qentry->client && strcasestr (qentry->client, parser->server)) found = 1;
1053
1054     if (!found) return;
1055   }
1056
1057   if (parser->from) {
1058     if (!qentry->from) return;
1059     if (!*(parser->from)) {
1060       if (*(qentry->from)) return;
1061     } else if (!STRMATCH(parser->from, qentry->from)) {
1062       return;
1063     }
1064   } else {
1065     if (parser->exclude_ndrs && qentry->from && !*qentry->from) return;
1066   }
1067
1068   if (parser->to) {
1069     tl = qentry->tolist;
1070     int found = 0;
1071     while (tl) {
1072       if (parser->to && !STRMATCH(parser->to, tl->to)) {
1073         tl->to = NULL;
1074       } else {
1075         found = 1;
1076       }
1077       tl = tl->next;
1078     }
1079     if (!found) return;
1080   }
1081
1082   if (parser->strmatch &&
1083       !(qentry->strmatch || (se && se->strmatch) || (fe && fe->strmatch)))
1084     return;
1085
1086
1087   if (parser->verbose) {
1088
1089     printf ("QENTRY: %s\n", qentry->qid);
1090
1091     printf ("CTIME: %08lX\n", parser->ctime);
1092     printf ("SIZE: %u\n", qentry->size);
1093
1094     if (qentry->client) {
1095       printf ("CLIENT: %s\n", qentry->client);
1096     } else if (se && se->connect) {
1097       printf ("CLIENT: %s\n", se->connect);
1098     }
1099
1100     if (qentry->msgid) { printf ("MSGID: %s\n", qentry->msgid); }
1101
1102   }
1103
1104   tl = qentry->tolist;
1105   while (tl) {
1106     if (tl->to) {
1107       fl = NULL;
1108       if (fe && tl->dstatus == '2') {
1109         fl = fe->tolist;
1110         while (fl) {
1111           if (fl->to && !strcmp (tl->to, fl->to)) {
1112             break;
1113           }
1114           fl = fl->next;
1115         }
1116       }
1117       char *to;
1118       char dstatus;
1119       char *relay;
1120
1121       if (fl) {
1122         to = fl->to;
1123         dstatus = fl->dstatus;
1124         relay = fl->relay;
1125       } else {
1126         to = tl->to;
1127         dstatus = tl->dstatus;
1128         relay = tl->relay;
1129       }
1130
1131       printf ("TO:%08lX:%s:%c: from <%s> to <%s> (%s)\n", tl->ltime, qentry->qid, dstatus, qentry->from ? qentry->from : "", to ? to : "", relay ? relay : "none");
1132
1133       parser->count++;
1134     }
1135     tl = tl->next;
1136   }
1137
1138
1139   if (!parser->verbose)  { fflush (stdout); return; }
1140
1141   if (parser->verbose > 1) {
1142
1143     if (se && se->loglist.log) {
1144       printf ("SMTP:\n");
1145       loglist_print (&se->loglist);
1146     }
1147
1148     if (fe && fe->loglist.log) {
1149       printf ("FILTER: %s\n", fe->logid);
1150       loglist_print (&fe->loglist);
1151     }
1152
1153     if (qentry->loglist.log) {
1154       printf ("QMGR:\n");
1155       loglist_print (&qentry->loglist);
1156     }
1157   }
1158
1159   printf ("\n");
1160
1161   fflush (stdout);
1162
1163   //sleep (1);
1164 }
1165
1166 QEntry *
1167 qentry_get (LParser *parser, const char *qid)
1168 {
1169   QEntry *qentry;
1170
1171   if ((qentry = g_hash_table_lookup (parser->qmgr_h, qid))) {
1172     return qentry;
1173   } else {
1174     if ((qentry = qentry_new (qid))) {
1175       g_hash_table_insert (parser->qmgr_h, qentry->qid, qentry);
1176     }
1177
1178     return qentry;
1179   }
1180 }
1181
1182 void
1183 qentry_free_noremove (LParser *parser, QEntry *qentry)
1184 {
1185   SList *l;
1186   gpointer data;
1187   SEntry *se;
1188
1189   if ((se = qentry->smtpd)) {
1190     if (sentry_ref_del (se, qentry) == 0) {
1191       if (se->disconnect) {
1192         sentry_free_noremove (se);
1193       }
1194     }
1195   }
1196
1197 #ifdef EPOOL_DEBUG
1198   ep_allocated -= qentry->ep.allocated;
1199   printf ("MEM: %d\n", ep_allocated);
1200 #endif
1201
1202 #ifdef DEBUG
1203   {
1204     QEntry *qe;
1205     if ((qe = g_hash_table_lookup (qmgr_debug_free, qentry))) {
1206       debug_error ("QEntry already freed", NULL);
1207     } else {
1208       g_hash_table_insert (qmgr_debug_free, qentry, qentry);
1209     }
1210   }
1211   return;
1212 #endif
1213
1214   l = qentry->ep.mblocks;
1215   while (l) {
1216     data = l->data;
1217     l = l->next;
1218     g_free (data);
1219   }
1220
1221   l = qentry->ep.blocks;
1222   while (l) {
1223     data = l->data;
1224     l = l->next;
1225     g_slice_free1(EPOOL_BLOCK_SIZE, data);
1226   }
1227 }
1228
1229 void
1230 qentry_free (LParser *parser, QEntry *qentry)
1231 {
1232   g_hash_table_remove (parser->qmgr_h, qentry->qid);
1233
1234   qentry_free_noremove (parser, qentry);
1235 }
1236
1237 void
1238 qentry_cleanup_hash (gpointer key,
1239                      gpointer value,
1240                      gpointer user_data)
1241 {
1242   QEntry *qe = value;
1243   LParser *parser = (LParser *)user_data;
1244
1245   qentry_print (parser, qe);
1246   qentry_free_noremove (parser, qe);
1247 }
1248
1249 void
1250 qentry_finalize (LParser *parser, QEntry *qentry)
1251 {
1252   if (qentry && qentry->removed) {
1253     SEntry *se = qentry->smtpd;
1254
1255     if (se && !se->disconnect) return;
1256
1257     FEntry *fe = qentry->filter;
1258
1259     if (fe && !fe->finished) return;
1260
1261     qentry_print (parser, qentry);
1262     qentry_free (parser, qentry);
1263
1264     if (fe) fentry_free (parser, fe);
1265   }
1266 }
1267
1268 // FEntry
1269
1270 FEntry*
1271 fentry_new (const char *logid)
1272 {
1273   FEntry *fentry;
1274   SList *blocks;
1275   int cpos;
1276   char *logid_cp;
1277
1278   fentry = (FEntry *)g_slice_alloc0(EPOOL_BLOCK_SIZE);
1279
1280 #ifdef EPOOL_DEBUG
1281   fentry->ep.allocated += EPOOL_BLOCK_SIZE;
1282   ep_allocated += EPOOL_BLOCK_SIZE;
1283   ep_maxalloc = (ep_allocated > ep_maxalloc) ? ep_allocated : ep_maxalloc;
1284 #endif
1285
1286 #ifdef DEBUG
1287   {
1288     FEntry *fe;
1289     if ((fe = g_hash_table_lookup (filter_debug_alloc, fentry))) {
1290       debug_error ("FEntry already alloced", NULL);
1291     } else {
1292       g_hash_table_insert (filter_debug_alloc, fentry, fentry);
1293     }
1294   }
1295 #endif
1296
1297   cpos = sizeof (FEntry);
1298
1299   blocks = (SList *)((char *)fentry + cpos);
1300
1301   cpos += sizeof (SList);
1302
1303   blocks->data = fentry;
1304   blocks->next = NULL;
1305
1306   fentry->logid = logid_cp = (char *)fentry + cpos;
1307   while ((*logid_cp++ = *logid++)) cpos++;
1308   cpos = (cpos + 4) & ~3;
1309
1310   fentry->ep.blocks = blocks;
1311   fentry->ep.cpos = cpos;;
1312
1313   return fentry;
1314 }
1315
1316 FEntry *
1317 fentry_get (LParser *parser, const char *logid)
1318 {
1319   FEntry *fentry;
1320
1321   if ((fentry = g_hash_table_lookup (parser->filter_h, logid))) {
1322     return fentry;
1323   } else {
1324     if ((fentry = fentry_new (logid))) {
1325       g_hash_table_insert (parser->filter_h, fentry->logid, fentry);
1326     }
1327
1328     return fentry;
1329   }
1330 }
1331
1332 void
1333 fentry_tolist_add (FEntry *fentry, char dstatus, const char *to, int to_len,
1334                    const char *qid, int qid_len)
1335 {
1336   TOList *tl = (TOList *)epool_alloc (&fentry->ep, sizeof (TOList));
1337
1338   tl->to = epool_strndup0 (&fentry->ep, to, to_len);
1339
1340   if (qid) {
1341     tl->relay = epool_strndup0 (&fentry->ep, qid, qid_len);
1342   } else {
1343     tl->relay = NULL;
1344   }
1345   tl->dstatus = dstatus;
1346   tl->next = fentry->tolist;
1347
1348   fentry->tolist = tl;
1349 }
1350
1351 void
1352 fentry_free_noremove (LParser *parser, FEntry *fentry)
1353 {
1354   SList *l;
1355   gpointer data;
1356
1357 #ifdef EPOOL_DEBUG
1358   ep_allocated -= fentry->ep.allocated;
1359   printf ("MEM: %d\n", ep_allocated);
1360 #endif
1361
1362 #ifdef DEBUG
1363   {
1364     FEntry *fe;
1365     if ((fe = g_hash_table_lookup (filter_debug_free, fentry))) {
1366       debug_error ("FEntry already freed", NULL);
1367     } else {
1368       g_hash_table_insert (filter_debug_free, fentry, fentry);
1369     }
1370   }
1371   return;
1372 #endif
1373
1374   l = fentry->ep.mblocks;
1375   while (l) {
1376     data = l->data;
1377     l = l->next;
1378     g_free (data);
1379   }
1380
1381   l = fentry->ep.blocks;
1382   while (l) {
1383     data = l->data;
1384     l = l->next;
1385     g_slice_free1(EPOOL_BLOCK_SIZE, data);
1386   }
1387 }
1388
1389 void
1390 fentry_free (LParser *parser, FEntry *fentry)
1391 {
1392   g_hash_table_remove (parser->filter_h, fentry->logid);
1393
1394   fentry_free_noremove (parser, fentry);
1395 }
1396
1397 void
1398 fentry_cleanup_hash (gpointer key,
1399                      gpointer value,
1400                      gpointer user_data)
1401 {
1402   FEntry *fe = value;
1403   LParser *parser = (LParser *)user_data;
1404
1405   fentry_free_noremove (parser, fe);
1406 }
1407
1408 // Parser
1409
1410 LParser*
1411 parser_new ()
1412 {
1413   LParser *parser = g_malloc0 (sizeof (LParser));
1414   struct timeval tv;
1415   struct tm *ltime;
1416   int i;
1417
1418   epool_init (&parser->ep);
1419
1420   if (!(parser->smtpd_h = g_hash_table_new (g_int_hash, g_int_equal))) {
1421     return NULL;
1422   }
1423
1424   if (!(parser->qmgr_h = g_hash_table_new (g_str_hash, g_str_equal))) {
1425     return NULL;
1426   }
1427
1428   if (!(parser->filter_h = g_hash_table_new (g_str_hash, g_str_equal))) {
1429     return NULL;
1430   }
1431
1432   //if (!(parser->track_h = g_hash_table_new (g_str_hash, g_str_equal))) {
1433   //return NULL;
1434   //}
1435
1436   for (i = 0; i < MAX_LOGFILES; i++) {
1437     gettimeofday(&tv, NULL);
1438     tv.tv_sec -= 3600*24*i;
1439     ltime = localtime (&tv.tv_sec);
1440     parser->year[i] = ltime->tm_year + 1900;
1441   }
1442
1443   return parser;
1444 }
1445
1446 void
1447 parser_free (LParser *parser)
1448 {
1449   int i;
1450
1451   for (i = 0; i < MAX_LOGFILES; i++) {
1452     if (parser->stream[i]) gzclose (parser->stream[i]);
1453   }
1454
1455   epool_free (&parser->ep);
1456
1457   g_hash_table_destroy (parser->smtpd_h);
1458   g_hash_table_destroy (parser->qmgr_h);
1459   g_hash_table_destroy (parser->filter_h);
1460   //g_hash_table_destroy (parser->track_h);
1461
1462   g_free (parser);
1463 }
1464
1465 #if 0
1466 char *
1467 parser_track (LParser *parser, const char *qid, gboolean insert)
1468 {
1469   char *res;
1470
1471   if ((res = g_hash_table_lookup (parser->track_h, qid))) {
1472     return res;
1473   } else {
1474     if (insert && (res = epool_strdup (&parser->ep, qid))) {
1475       g_hash_table_insert (parser->track_h, res, res);
1476       return res;
1477     }
1478   }
1479   return NULL;
1480 }
1481 #endif
1482
1483 static const int linebufsize = 8192;
1484 static int cur_year;
1485 static int cur_month = 0;
1486 static int cal_mtod[12] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334};
1487
1488 static time_t
1489 mkgmtime (struct tm *tm)
1490 {
1491   time_t res;
1492
1493   int year = tm->tm_year + 1900;
1494   int mon = tm->tm_mon;
1495
1496   res = (year - 1970) * 365 + cal_mtod[mon];
1497
1498   // leap year corrections (gregorian calendar)
1499   if (mon <= 1) year -= 1;
1500   res += (year - 1968) / 4;
1501   res -= (year - 1900) / 100;
1502   res += (year - 1600) / 400;
1503
1504   res += tm->tm_mday - 1;
1505   res = res*24 + tm->tm_hour;
1506   res = res*60 + tm->tm_min;
1507   res = res*60 + tm->tm_sec;
1508
1509   return res;
1510 }
1511
1512 time_t
1513 parse_time (const char **text, int len)
1514 {
1515   time_t ltime = 0;
1516
1517   int year = cur_year;
1518
1519   int mon = 0;
1520   int mday = 0;
1521   int hour = 0;
1522   int min = 0;
1523   int sec = 0;
1524   int found;
1525
1526   const char *line = *text;
1527
1528   if (len == (linebufsize - 1)) {
1529     debug_error ("skipping long line data", line);
1530     return 0;
1531   }
1532
1533   if (len < 15) {
1534     debug_error ("skipping short line data", line);
1535     return 0;
1536   }
1537
1538   // parse month
1539   int csum = (line[0]<<16) + (line[1]<<8) + line[2];
1540
1541   switch (csum) {
1542   case 4874606: mon = 0; break;
1543   case 4613474: mon = 1; break;
1544   case 5071218: mon = 2; break;
1545   case 4288626: mon = 3; break;
1546   case 5071225: mon = 4; break;
1547   case 4879726: mon = 5; break;
1548   case 4879724: mon = 6; break;
1549   case 4289895: mon = 7; break;
1550   case 5465456: mon = 8; break;
1551   case 5202804: mon = 9; break;
1552   case 5140342: mon = 10; break;
1553   case 4482403: mon = 11; break;
1554   default:
1555     debug_error ("unable to parse month", line);
1556     return 0;
1557   }
1558
1559   // year change heuristik
1560   if (cur_month == 11 && mon == 0) {
1561     year++;
1562   }
1563   if (mon > cur_month) cur_month = mon;
1564
1565   ltime = (year - 1970)*365 + cal_mtod[mon];
1566
1567   // leap year corrections (gregorian calendar)
1568   if (mon <= 1) year -= 1;
1569   ltime += (year - 1968) / 4;
1570   ltime -= (year - 1900) / 100;
1571   ltime += (year - 1600) / 400;
1572
1573   const char *cpos = line + 3;
1574
1575   found = 0; while (isspace (*cpos)) { cpos++; found = 1; }
1576   if (!found) {
1577     debug_error ("missing spaces after month", line);
1578     return 0;
1579   }
1580
1581   found = 0; while (isdigit (*cpos)) { mday = mday*10 + *cpos - 48; cpos++; found++; }
1582   if (found < 1 || found > 2) {
1583     debug_error ("unable to parse day of month", line);
1584     return 0;
1585   }
1586
1587   ltime += mday - 1;
1588
1589   found = 0; while (isspace (*cpos)) { cpos++; found++; }
1590   if (!found) {
1591     debug_error ("missing spaces after day of month", line);
1592     return 0;
1593   }
1594
1595   found = 0; while (isdigit (*cpos)) { hour = hour*10 + *cpos - 48; cpos++; found++; }
1596   if (found < 1 || found > 2) {
1597     debug_error ("unable to parse hour", line);
1598     return 0;
1599   }
1600
1601   ltime *= 24;
1602   ltime += hour;
1603
1604   if (*cpos != ':') {
1605     debug_error ("missing collon after hour", line);
1606     return 0;
1607   }
1608   cpos++;
1609
1610   found = 0; while (isdigit (*cpos)) { min = min*10 + *cpos - 48; cpos++; found++; }
1611   if (found < 1 || found > 2) {
1612     debug_error ("unable to parse minute", line);
1613     return 0;
1614   }
1615
1616   ltime *= 60;
1617   ltime += min;
1618
1619   if (*cpos != ':') {
1620     debug_error ("missing collon after minute", line);
1621     return 0;
1622   }
1623   cpos++;
1624
1625   found = 0; while (isdigit (*cpos)) { sec = sec*10 + *cpos - 48; cpos++; found++; }
1626   if (found < 1 || found > 2) {
1627     debug_error ("unable to parse second", line);
1628     return 0;
1629   }
1630
1631   ltime *= 60;
1632   ltime += sec;
1633
1634   found = 0; while (isspace (*cpos)) { cpos++; found = 1; }
1635   if (!found) {
1636     debug_error ("missing spaces after time", line);
1637     return 0;
1638   }
1639
1640   *text = cpos;
1641
1642   return ltime;
1643 }
1644
1645
1646 int
1647 parser_count_files (LParser *parser)
1648 {
1649   int i;
1650   time_t start = parser->start;
1651   char linebuf[linebufsize];
1652   const char *line;
1653   gzFile stream;
1654
1655   for (i = 0; i < (MAX_LOGFILES - 1); i++) {
1656     cur_year = parser->year[i];
1657     cur_month = 0;
1658
1659     if ((stream = gzopen (logfiles[i], "r"))) {
1660       if ((line = gzgets (stream, linebuf, linebufsize))) {
1661         if (parse_time (&line, strlen (line)) < start) {
1662           break;
1663         }
1664       } else {
1665         return i;
1666       }
1667       gzclose (stream);
1668     } else {
1669       return i;
1670     }
1671   }
1672
1673   return i + 1;
1674 }
1675
1676 static char *
1677 parse_qid (const char **text, char *out, char delim, int maxlen)
1678 {
1679   const char *idx;
1680   char *copy = out;
1681
1682   int found = 0;
1683   idx = *text;
1684   while (isxdigit (*idx)) { *copy++ = *idx++; found++; if (found > maxlen) break; }
1685
1686   if (found > 1 && found < maxlen &&
1687       ((delim && (*idx == delim)) || (!delim && isspace (*idx)))) {
1688     *copy = 0;
1689     idx++;
1690     while (isspace (*idx)) idx++;
1691     *text = idx;
1692     return out;
1693   }
1694   return NULL;
1695 }
1696
1697 void
1698 print_usage (const char *name)
1699 {
1700   fprintf (stderr, "usage: %s [OPTIONS] [OUTPUTFILENAME]\n", name);
1701   fprintf (stderr, "\t-f SENDER      mails from SENDER\n");
1702   fprintf (stderr, "\t-t RECIPIENT   mails to RECIPIENT\n");
1703   fprintf (stderr, "\t-h Server      Server IP or Hostname\n");
1704   fprintf (stderr, "\t-s START       start time (YYYY-MM-DD HH:MM:SS)\n");
1705   fprintf (stderr, "\t               or seconds since epoch\n");
1706   fprintf (stderr, "\t-e END         end time (YYYY-MM-DD HH:MM:SS)\n");
1707   fprintf (stderr, "\t               or seconds since epoch\n");
1708   fprintf (stderr, "\t-m MSGID       message ID (exact match)\n");
1709   fprintf (stderr, "\t-q QID         queue ID (exact match), can be\n");
1710   fprintf (stderr, "\t               specified multiple times.\n");
1711   fprintf (stderr, "\t-x STRING      search for strings\n");
1712   fprintf (stderr, "\t-l LIMIT       print max limit entries\n");
1713   fprintf (stderr, "\t-g             exclude greylist entries\n");
1714   fprintf (stderr, "\t-n             exclude NDR entries\n");
1715   fprintf (stderr, "\t-v             verbose output (no logs)\n");
1716   fprintf (stderr, "\t-vv            verbose output with logs\n");
1717 }
1718
1719
1720 // gzgets is ways too slow, so we do it our own way
1721
1722 static int
1723 mygzgetc (gzFile stream)
1724 {
1725   int br;
1726   static char readbuf[16384];
1727   static char *readend = readbuf + sizeof (readbuf);
1728   static char *readpos = readbuf + sizeof (readbuf);
1729
1730   if (readpos < readend) return *readpos++;
1731
1732   if ((br = gzread (stream, readbuf, sizeof (readbuf))) <= 0) {
1733     return -1;
1734   } else {
1735     readpos = readbuf;
1736     readend = readbuf + br;
1737
1738     return *readpos++;
1739   }
1740 }
1741
1742 static char *
1743 mygzgets (gzFile stream, char *line, int bufsize)
1744 {
1745   int c=0;
1746   char *cpos;
1747
1748   cpos = line;
1749   while (--bufsize > 0 && (c = mygzgetc(stream)) != -1) {
1750     *cpos++ = c;
1751     if (c == '\n')
1752       break;
1753   }
1754   if (c == -1 && cpos == line)
1755     return NULL;
1756   *cpos++ = '\0';
1757   return line;
1758 }
1759
1760
1761 extern char *optarg;
1762 extern int optind, opterr, optopt;
1763
1764 int
1765 main (int argc, char * const argv[])
1766 {
1767   char linebuf[linebufsize];
1768   char *line;
1769
1770   const char *text;
1771   const char *idx1;
1772   const char *idx2;
1773   const char *cpos;
1774   int found = 0;
1775   int csum_prog;
1776   unsigned long lines = 0;
1777   unsigned long rel_line_nr = 0;
1778   char qidbuf[30];
1779   int i;
1780
1781   struct tm *ltime;
1782   struct timeval tv;
1783   time_t ctime, next_ctime, start, end;
1784
1785   LParser *parser;
1786   int opt;
1787
1788 #ifdef DEBUG
1789   smtpd_debug_alloc = g_hash_table_new (g_direct_hash, g_direct_equal);
1790   qmgr_debug_alloc = g_hash_table_new (g_direct_hash, g_direct_equal);
1791   filter_debug_alloc = g_hash_table_new (g_direct_hash, g_direct_equal);
1792   smtpd_debug_free = g_hash_table_new (g_direct_hash, g_direct_equal);
1793   qmgr_debug_free = g_hash_table_new (g_direct_hash, g_direct_equal);
1794   filter_debug_free = g_hash_table_new (g_direct_hash, g_direct_equal);
1795 #endif
1796
1797   if (!(parser = parser_new ())) {
1798     fprintf (stderr, "unable to alloc parser structure\n");
1799     exit (-1);
1800   }
1801
1802   while ((opt = getopt (argc, argv, "f:t:s:e:h:m:q:x:l:vgn")) != -1) {
1803     if (opt == 'f') {
1804       parser->from = epool_strdup (&parser->ep, optarg);
1805     } else if (opt == 't') {
1806       parser->to = epool_strdup (&parser->ep, optarg);
1807     } else if (opt == 'v') {
1808       parser->verbose += 1;
1809     } else if (opt == 'g') {
1810       parser->exclude_greylist = 1;
1811     } else if (opt == 'n') {
1812       parser->exclude_ndrs = 1;
1813     } else if (opt == 'h') {
1814       parser->server = epool_strdup (&parser->ep, optarg);
1815     } else if (opt == 'm') {
1816       parser->msgid = epool_strdup (&parser->ep, optarg);
1817     } else if (opt == 'q') {
1818       time_t ltime;
1819       unsigned long rel_line_nr;
1820       MatchList *match = (MatchList *)epool_alloc(&parser->ep, sizeof(MatchList));
1821       if (sscanf(optarg, "T%08lXL%08lX", &ltime, &rel_line_nr) == 2) {
1822         match->mtype = MatchTypeRelLineNr;
1823         match->ltime = ltime;
1824         match->rel_line_nr = rel_line_nr;
1825         match->next = parser->match_list;
1826         parser->match_list = match;
1827       } else {
1828         match->mtype = MatchTypeQID;
1829         match->id = epool_strdup(&parser->ep, optarg);
1830         match->next = parser->match_list;
1831         parser->match_list = match;
1832       }
1833     } else if (opt == 'x') {
1834       parser->strmatch = epool_strdup (&parser->ep, optarg);
1835     } else if (opt == 'l') {
1836       char *l;
1837       parser->limit = strtoul (optarg, &l, 0);
1838       if (!*optarg || *l) {
1839         fprintf (stderr, "unable to parse limit '%s'\n", optarg);
1840         exit (-1);
1841       }
1842     } else if (opt == 's') {
1843       // use strptime to convert time
1844       struct tm tm;
1845       char *res;
1846       if ((!(res = strptime (optarg, "%F %T", &tm)) &&
1847            !(res = strptime (optarg, "%s", &tm))) || *res) {
1848         fprintf (stderr, "unable to parse start time\n");
1849         exit (-1);
1850       } else {
1851         parser->start = mkgmtime (&tm);
1852       }
1853     } else if (opt == 'e') {
1854       struct tm tm;
1855       char *res;
1856       if ((!(res = strptime (optarg, "%F %T", &tm)) &&
1857            !(res = strptime (optarg, "%s", &tm))) || *res) {
1858         fprintf (stderr, "unable to parse end time\n");
1859         exit (-1);
1860       } else {
1861         parser->end = mkgmtime (&tm);
1862       }
1863     } else {
1864       print_usage (argv[0]);
1865       exit (-1);
1866     }
1867   }
1868
1869   if (optind < argc) {
1870
1871     if ((argc - optind) > 1) {
1872       print_usage (argv[0]);
1873       exit (-1);
1874     }
1875
1876     char *tmpfn = g_strdup_printf ("/tmp/.proxtrack-%08X.txt", getpid ());
1877
1878     if ((stdout = freopen (tmpfn, "w", stdout)) == NULL) {
1879       perror ("unable to open output file");
1880       exit (-1);
1881     }
1882     if (rename (tmpfn, argv[optind]) != 0) {
1883       perror ("unable to open output file");
1884       unlink (tmpfn);
1885       exit (-1);
1886     }
1887   }
1888
1889   // we use gmtime exerywhere to speedup things, this can cause problems
1890   // when daylight saving time changes
1891
1892   gettimeofday(&tv, NULL);
1893   ltime = localtime (&tv.tv_sec);
1894
1895   if (!parser->start) {
1896     ltime->tm_sec = 0;
1897     ltime->tm_min = 0;
1898     ltime->tm_hour = 0;
1899     parser->start = mkgmtime (ltime);
1900   }
1901
1902   ltime = localtime (&tv.tv_sec);
1903
1904   if (!parser->end) {
1905     parser->end = mkgmtime (ltime);
1906   }
1907
1908   if (parser->end < parser->start) {
1909     fprintf (stderr, "end time before start time\n");
1910     exit (-1);
1911   }
1912
1913   int filecount;
1914   if ((filecount = parser_count_files (parser)) <= 0) {
1915     fprintf (stderr, "unable to access log files\n");
1916     exit (-1);
1917   }
1918
1919   printf ("# LogReader: %d\n", getpid());
1920
1921   printf ("# Query options\n");
1922   if (parser->from) printf ("# Sender:    %s\n", parser->from);
1923   if (parser->to) printf ("# Recipient: %s\n", parser->to);
1924   if (parser->server) printf ("# Server:    %s\n", parser->server);
1925   if (parser->msgid) printf ("# MsgID:     %s\n", parser->msgid);
1926
1927   MatchList *match = parser->match_list;
1928   while (match) {
1929     if (match->mtype == MatchTypeQID) {
1930       printf ("# QID:       %s\n", match->id);
1931     } else if (match->mtype == MatchTypeRelLineNr) {
1932       printf ("# QID:       T%08lXL%08lX\n", match->ltime, match->rel_line_nr);
1933     } else {
1934       g_error("internal error - unknown match type %d\n", match->mtype);
1935     }
1936     match = match->next;
1937   }
1938
1939   if (parser->strmatch) printf ("# Match:     %s\n", parser->strmatch);
1940
1941   strftime (linebuf, 256, "%F %T", gmtime (&parser->start));
1942   printf ("# Start:     %s (%lu)\n", linebuf, parser->start);
1943   strftime (linebuf, 256, "%F %T", gmtime (&parser->end));
1944   printf ("# END:       %s (%lu)\n", linebuf, parser->end);
1945   printf ("# End Query Options\n\n");
1946
1947   fflush (stdout);
1948
1949   start = parser->start;
1950   end = parser->end;
1951   ctime = 0;
1952
1953   for (i = filecount - 1; i >= 0; i--) {
1954     gpointer stream;
1955
1956     // syslog files does not conain years, so we must compute then
1957     // cur_year is the assumed start year
1958
1959     cur_month = 0;
1960     cur_year = parser->year[i];
1961
1962     if (i <= 1) {
1963       if (!(stream = (gpointer) fopen (logfiles[i], "r"))) continue;
1964     } else {
1965       if (!(stream = (gpointer) gzopen (logfiles[i], "r"))) continue;
1966     }
1967
1968     while (1) {
1969
1970       if (parser->limit && (parser->count >= parser->limit)) {
1971         printf ("STATUS: aborted by limit (too many hits)\n");
1972         exit (0);
1973       }
1974
1975       if (i <= 1) {
1976         line = fgets (linebuf, linebufsize, (FILE *)stream);
1977       } else {
1978         line = mygzgets ((gzFile)stream, linebuf, linebufsize);
1979       }
1980
1981       if (!line) break;
1982
1983       int len = strlen (line);
1984       int pid = 0;
1985
1986       cpos = line;
1987
1988       next_ctime = parse_time (&cpos, len);
1989
1990       if (!next_ctime) {
1991         continue;
1992       }
1993
1994       if (next_ctime != ctime) {
1995         rel_line_nr = 0;
1996       } else {
1997         rel_line_nr++;
1998       }
1999
2000       ctime = next_ctime;
2001
2002       if (ctime < start) continue;
2003       if (ctime > end) break;
2004
2005       lines++;
2006
2007       found = 0; while (!isspace (*cpos)) { cpos++; found = 1; }
2008       if (!found) {
2009         debug_error ("missing hostname", line);
2010         continue;
2011       }
2012
2013       found = 0; while (isspace (*cpos)) { cpos++; found = 1; }
2014       if (!found) {
2015         debug_error ("missing spaces after host", line);
2016         continue;
2017       }
2018
2019       if ((*cpos == 'l') && !strncmp (cpos, "last message repeated", 21)) {
2020         continue;
2021       }
2022
2023       //printf ("LINE: %s\n", line);
2024       //const char *prog = cpos;
2025
2026       csum_prog = 0;
2027       found = 0; while (*cpos && (*cpos != ':') && (*cpos != '[')) {
2028         csum_prog = (csum_prog <<8) + *cpos;
2029         cpos++;
2030         found++;
2031       }
2032
2033       //idx1 = g_strndup (prog, found);
2034       //printf ("TEST:%s:%08X\n", idx1, csum_prog);
2035       //free (idx1);
2036
2037       if (*cpos == '[') {
2038         cpos++;
2039         found = 0; while (isdigit (*cpos)) {
2040           pid = pid*10 + *cpos - 48;
2041           cpos++;
2042           found++;
2043         }
2044         if (found < 1 || found > 15 || *cpos != ']') {
2045           debug_error ("unable to parse pid", line);
2046           continue;
2047         }
2048         cpos++;
2049       }
2050
2051       if (*cpos++ != ':') {
2052         debug_error ("missing collon", line);
2053         continue;
2054       }
2055
2056
2057       if (!isspace (*cpos++)) {
2058         debug_error ("missing space after collon", line);
2059         continue;
2060       }
2061
2062       text = cpos;
2063
2064       parser->ctime = ctime;
2065
2066       int strmatch = 0;
2067
2068       if (parser->strmatch && STRMATCH(parser->strmatch, text)) {
2069         strmatch = 1;
2070       }
2071
2072       if ((csum_prog == 0x70726F78) ||// proxprox
2073           (csum_prog == 0x6C746572)) { // pmg-smtp-filter
2074
2075         if ((idx1 = parse_qid (&cpos, qidbuf, ':', 25))) {
2076
2077           FEntry *fe;
2078
2079           if (!(fe = fentry_get (parser, idx1))) {
2080             continue;
2081           }
2082
2083           loglist_add (&fe->ep, &fe->loglist, line, len, lines);
2084
2085           if (strmatch) fe->strmatch = 1;
2086
2087           //fixme: BCC, Notify?
2088           //fixme: parse virus info
2089           //fixme: parse spam score
2090
2091           if ((*cpos == 'a') && !strncmp (cpos, "accept mail to <", 16)) {
2092
2093             const char *to_s, *to_e;
2094
2095             to_s = cpos = cpos + 16;
2096
2097             while (*cpos && (*cpos != '>')) { cpos++; }
2098
2099             if (*cpos != '>') continue;
2100
2101             to_e = cpos;
2102
2103             cpos++;
2104
2105             if ((*cpos++ != ' ') || (*cpos++ != '(')) continue;
2106
2107             if (!(idx1 = parse_qid (&cpos, qidbuf, ')', 15))) continue;
2108
2109             // parser_track (parser, idx1, 1);
2110
2111             fentry_tolist_add (fe, 'A', to_s, to_e - to_s, idx1, strlen (idx1));
2112
2113           } else if ((*cpos == 'm') && !strncmp (cpos, "moved mail for <", 16)) {
2114             const char *to_s, *to_e;
2115
2116             to_s = cpos = cpos + 16;
2117
2118             while (*cpos && (*cpos != '>')) { cpos++; }
2119
2120             to_e = cpos;
2121
2122             if (strncmp (cpos, "> to ", 5)) continue;
2123             cpos += 5;
2124
2125             if (!strncmp (cpos, "spam", 4)) {
2126               cpos += 4;
2127             } else if (!strncmp (cpos, "virus", 5)) {
2128               cpos += 5;
2129             } else {
2130               continue;
2131             }
2132
2133             if (strncmp (cpos, " quarantine - ", 14)) continue;
2134             cpos += 14;
2135
2136             if (!(idx1 = parse_qid (&cpos, qidbuf, 0, 25))) continue;
2137
2138             fentry_tolist_add (fe, 'Q', to_s, to_e - to_s, idx1, strlen (idx1));
2139
2140           } else if ((*cpos == 'b') && !strncmp (cpos, "block mail to <", 15)) {
2141
2142             const char *to_s;
2143
2144             to_s = cpos = cpos + 15;
2145
2146             while (*cpos && (*cpos != '>')) { cpos++; }
2147
2148             if (*cpos != '>') continue;
2149
2150             fentry_tolist_add (fe, 'B', to_s, cpos - to_s, NULL, 0);
2151
2152           } else if ((*cpos == 'p') && !strncmp (cpos, "processing time: ", 17)) {
2153             cpos += 17;
2154
2155             sscanf (cpos, "%f", &fe->ptime);
2156
2157             fe->finished = 1;
2158           }
2159
2160         }
2161
2162       } else if (csum_prog == 0x7265656E) { // postfix/postscreen
2163
2164               SEntry *se;
2165
2166               if (!pid) {
2167                       debug_error ("no pid for postscreen", line);
2168                       continue;
2169               }
2170
2171
2172               if ((*text == 'N') && !strncmp (text, "NOQUEUE: reject: RCPT from ", 27)) {
2173
2174                       cpos = text + 27;
2175
2176                       if (!(idx1 = strstr (cpos, "; client ["))) continue;
2177
2178                       const char *client = cpos = idx1 + 10;
2179
2180                       while (*cpos && (*cpos != ']')) { cpos++; }
2181
2182                       const char *client_end = cpos;
2183
2184                       if (!(idx1 = strstr (cpos, "; from=<"))) continue;
2185
2186                       const char *from = cpos = idx1 + 8;
2187
2188                       while (*cpos && (*cpos != '>')) { cpos++; }
2189                       idx1 = cpos;
2190
2191                       if ((*cpos == '>') && strncmp (cpos, ">, to=<", 7)) continue;
2192
2193                       const char *to = cpos = cpos + 7;
2194
2195                       while (*cpos && (*cpos != '>')) { cpos++; }
2196
2197                       if (*cpos != '>') continue;
2198
2199                       if (!(se = sentry_get (parser, pid, ctime, rel_line_nr))) {
2200                               continue;
2201                       }
2202
2203                       if (strmatch) se->strmatch = 1;
2204
2205                       loglist_add (&se->ep, &se->loglist, line, len, lines);
2206
2207                       sentry_nqlist_add (se, ctime, from, idx1 - from, to, cpos - to, 'N');
2208
2209                       sentry_set_connect (se, client, client_end - client);
2210
2211                       g_hash_table_remove (parser->smtpd_h, &se->pid);
2212
2213                       se->disconnect = 1;
2214
2215                       sentry_print (parser, se);
2216                       sentry_free (parser, se);
2217               }
2218
2219       } else if (csum_prog == 0x716D6772) { // postfix/qmgr
2220
2221         if ((idx2 = text) && (idx1 = parse_qid (&idx2, qidbuf, ':', 15))) {
2222
2223           QEntry *qe;
2224
2225           if (!(qe = qentry_get (parser, idx1))) {
2226             continue;
2227           }
2228
2229           if (strmatch) qe->strmatch = 1;
2230
2231           qe->cleanup = 1;
2232
2233           loglist_add (&qe->ep, &qe->loglist, line, len, lines);
2234
2235           if ((*idx2 == 'f') && !strncmp (idx2, "from=<", 6)) {
2236
2237             cpos = idx2 = idx2 + 6;
2238             while (*cpos && (*cpos != '>')) { cpos++; }
2239
2240             if (*cpos != '>') {
2241               debug_error ("unable to parse 'from' address", line);
2242               continue;
2243             }
2244
2245             qentry_set_from (qe, idx2, cpos - idx2);
2246
2247             cpos++;
2248
2249             if ((*cpos == ',') && !strncmp (cpos, ", size=", 7)) {
2250               int size = 0;
2251               cpos += 7;
2252
2253               while (isdigit (*cpos)) { size = size*10 + *cpos++ - 48; }
2254
2255               qe->size = size;
2256             }
2257
2258           } else if ((*idx2 == 'r') && !strncmp (idx2, "removed\n", 8)) {
2259
2260             qe->removed = 1;
2261
2262             qentry_finalize (parser, qe);
2263           }
2264         }
2265
2266       } else if ((csum_prog == 0x736D7470) || //postfix/smtp
2267                  (csum_prog == 0x6C6D7470) || //postfix/lmtp
2268                  (csum_prog == 0x6F63616C) || //postfix/local
2269                  (csum_prog == 0x72726F72)) { //postfix/error
2270
2271         int lmtp = (csum_prog == 0x6C6D7470);
2272
2273         if ((cpos = text) && (idx1 = parse_qid (&cpos, qidbuf, ':', 15))) {
2274
2275           QEntry *qe;
2276
2277           if (!(qe = qentry_get (parser, idx1))) {
2278             continue;
2279           }
2280
2281           if (strmatch) qe->strmatch = 1;
2282
2283           qe->cleanup = 1;
2284
2285           loglist_add (&qe->ep, &qe->loglist, line, len, lines);
2286
2287           if (strncmp (cpos, "to=<", 4)) continue;
2288           cpos += 4;
2289
2290           const char *to_s, *to_e;
2291
2292           to_s = cpos;
2293
2294           while (*cpos && (*cpos != '>')) { cpos++; }
2295
2296           if (*cpos != '>') continue;
2297
2298           to_e = cpos;
2299
2300           cpos ++;
2301
2302           if (!(cpos = strstr (cpos, ", relay="))) continue;
2303           cpos += 8;
2304
2305           const char *relay_s, *relay_e;
2306
2307           relay_s = cpos;
2308
2309           while (*cpos && (*cpos != ',')) { cpos++; }
2310
2311           if (*cpos != ',') continue;
2312
2313           relay_e = cpos;
2314
2315           if (!(idx1 = strstr (cpos + 1, ", dsn="))) continue;
2316
2317           cpos = idx1 + 6;
2318
2319           if (!isdigit (*cpos)) continue;
2320
2321           char dstatus = *cpos;
2322
2323           qentry_tolist_add (qe, ctime, dstatus, to_s, to_e - to_s,
2324                              relay_s, relay_e - relay_s);
2325
2326           if (!lmtp) continue; // filter always uses lmtp
2327
2328           if (!(idx1 = strstr (cpos + 1, "status=sent (250 2.")))
2329             continue;
2330
2331           cpos = idx1 = idx1 + 19;
2332
2333           if (*cpos == '5' && (idx1 = strstr (cpos, "5.0 OK ("))) {
2334             cpos = idx1 = idx1 + 8;
2335           } else if (*cpos == '7' && (idx1 = strstr (cpos, "7.0 BLOCKED ("))) {
2336             cpos = idx1 = idx1 + 13;
2337           } else {
2338             continue;
2339           }
2340
2341           if (!(idx1 = parse_qid (&cpos, qidbuf, ')', 25)))
2342             continue;
2343
2344           FEntry *fe;
2345
2346           qe->filtered = 1;
2347
2348           if ((fe = g_hash_table_lookup (parser->filter_h, idx1))) {
2349             qe->filter = fe;
2350           }
2351         }
2352
2353       } else if (csum_prog == 0x6D747064) { // postfix/smtpd
2354         SEntry *se;
2355
2356         if (!pid) {
2357           debug_error ("no pid for smtpd", line);
2358           continue;
2359         }
2360
2361         if (!(se = sentry_get (parser, pid, ctime, rel_line_nr))) {
2362           continue;
2363         }
2364
2365         if (strmatch) se->strmatch = 1;
2366
2367         loglist_add (&se->ep, &se->loglist, line, len, lines);
2368
2369         if ((*text == 'c') && !strncmp (text, "connect from ", 13)) {
2370
2371           cpos = idx1 = text + 13;
2372
2373           while (*idx1 && !isspace (*idx1)) { idx1++; }
2374
2375           sentry_set_connect (se, cpos, idx1 - cpos);
2376
2377           // fixme: do we need this?
2378           //if (strcmp (se->connect, "localhost[127.0.0.1]")) {
2379           //  se->external = 1;
2380           //}
2381
2382         } else if ((*text == 'd') && !strncmp (text, "disconnect from", 15)) {
2383
2384           // unlink
2385           g_hash_table_remove (parser->smtpd_h, &se->pid);
2386
2387           se->disconnect = 1;
2388
2389           if (sentry_ref_rem_unneeded (parser, se) == 0) {
2390             sentry_print (parser, se);
2391             sentry_free (parser, se);
2392           } else {
2393             sentry_ref_finalize (parser, se);
2394           }
2395
2396         } else if ((*text == 'N') && !strncmp (text, "NOQUEUE:", 8)) {
2397
2398           cpos = text + 8;
2399
2400           // parse 'whatsup' (reject:)
2401           while (*cpos && (*cpos != ':')) { cpos++; }
2402           if (*cpos != ':') continue;
2403           cpos++;
2404
2405           // parse '%s from %s:'
2406           while (*cpos && (*cpos != ':')) { cpos++; }
2407           if (*cpos != ':') continue;
2408           idx1 = cpos++;
2409
2410           // parse '%s;'
2411           while (*cpos && (*cpos != ';')) { cpos++; }
2412           if (*cpos != ';') continue;
2413
2414           // greylisting test
2415           char dstatus = 'N';
2416           *(char *)cpos = 0; // dangerous hack: delimit string
2417           if (strstr (idx1, ": Recipient address rejected: Service is unavailable (try later)")) {
2418             dstatus = 'G';
2419           }
2420           *(char *)cpos = ';'; // dangerous hack: restore line
2421
2422           if (!(idx1 = strstr (cpos, "; from=<"))) continue;
2423
2424           const char *from = cpos = idx1 + 8;
2425
2426           while (*cpos && (*cpos != '>')) { cpos++; }
2427           idx1 = cpos;
2428
2429           if ((*cpos == '>') && strncmp (cpos, "> to=<", 6)) continue;
2430
2431           const char *to = cpos = cpos + 6;
2432
2433           while (*cpos && (*cpos != '>')) { cpos++; }
2434
2435           if (*cpos != '>') continue;
2436
2437           sentry_nqlist_add (se, ctime, from, idx1 - from, to, cpos - to, dstatus);
2438
2439         } else if ((idx2 = text) && (idx1 = parse_qid (&idx2, qidbuf, ':', 15))) {
2440
2441           QEntry *qe;
2442
2443           if ((qe = qentry_get (parser, idx1))) {
2444
2445             if (strmatch) qe->strmatch = 1;
2446
2447             sentry_ref_add (se, qe);
2448
2449             if ((*idx2 == 'c') && !strncmp (idx2, "client=", 7)) {
2450               cpos = idx2 = idx2 + 7;
2451
2452               while (*cpos && !isspace (*cpos)) { cpos++; }
2453
2454               qentry_set_client (qe, idx2, cpos - idx2);
2455             }
2456           }
2457
2458         }
2459
2460       } else if (csum_prog == 0x616E7570) { // postfix/cleanup
2461
2462         QEntry *qe;
2463
2464         idx2 = text;
2465         if ((idx1 = parse_qid (&idx2, qidbuf, ':', 15))) {
2466           if ((qe = qentry_get (parser, idx1))) {
2467
2468             if (strmatch) qe->strmatch = 1;
2469
2470             loglist_add (&qe->ep, &qe->loglist, line, len, lines);
2471
2472             if ((*idx2 == 'm') && !strncmp (idx2, "message-id=", 11)) {
2473
2474               cpos = idx2 = idx2 + 11;
2475
2476               while (*cpos && !isspace(*cpos)) { cpos++; }
2477
2478               qentry_set_msgid (qe, idx2, cpos - idx2);
2479
2480               qe->cleanup = 1;
2481             }
2482           }
2483         }
2484       }
2485     }
2486
2487     if (i <= 1) {
2488       fclose ((FILE *)stream);
2489     } else {
2490       gzclose ((gzFile)stream);
2491     }
2492
2493     if (ctime > end) break;
2494   }
2495
2496
2497 #ifdef DEBUG
2498   printf ("LINES: %d\n", lines);
2499   printf ("MEM SMTPD  entries: %d\n", g_hash_table_size (parser->smtpd_h));
2500   printf ("MEM QMGR   entries: %d\n", g_hash_table_size (parser->qmgr_h));
2501   printf ("MEM FILTER entries: %d\n", g_hash_table_size (parser->filter_h));
2502
2503   printf ("MEMDEB SMTPD entries: %d %d\n",
2504           g_hash_table_size (smtpd_debug_alloc),
2505           g_hash_table_size (smtpd_debug_free));
2506   printf ("MEMDEB QMGR  entries: %d %d\n",
2507           g_hash_table_size (qmgr_debug_alloc),
2508           g_hash_table_size (qmgr_debug_free));
2509   printf ("MEMDEB FILTER  entries: %d %d\n",
2510           g_hash_table_size (filter_debug_alloc),
2511           g_hash_table_size (filter_debug_free));
2512 #endif
2513
2514   g_hash_table_foreach (parser->qmgr_h, qentry_cleanup_hash, parser);
2515   g_hash_table_foreach (parser->smtpd_h, sentry_cleanup_hash, parser);
2516   g_hash_table_foreach (parser->filter_h, fentry_cleanup_hash, parser);
2517
2518 #ifdef DEBUG
2519   printf ("MEMDEB SMTPD entries: %d %d\n",
2520           g_hash_table_size (smtpd_debug_alloc),
2521           g_hash_table_size (smtpd_debug_free));
2522   printf ("MEMDEB QMGR  entries: %d %d\n",
2523           g_hash_table_size (qmgr_debug_alloc),
2524           g_hash_table_size (qmgr_debug_free));
2525   printf ("MEMDEB FILTER  entries: %d %d\n",
2526           g_hash_table_size (filter_debug_alloc),
2527           g_hash_table_size (filter_debug_free));
2528
2529   g_hash_table_foreach (smtpd_debug_alloc, sentry_debug_alloc, parser);
2530
2531 #endif
2532
2533
2534 #ifdef EPOOL_DEBUG
2535   printf ("MEMMAX %d\n", ep_maxalloc);
2536 #endif
2537
2538   parser_free (parser);
2539
2540   fclose (stdout);
2541
2542   exit (0);
2543 }