
##########################################################################
# $Id: amavis,v 1.32 2006/07/06 00:25:00 bjorn Exp $
##########################################################################
# $Log: amavis,v $
# Revision 1.32  2006/07/06 00:25:00  bjorn
# Additional filtering and counting, by Geert Janssens
#
# Revision 1.31  2006/06/29 16:02:05  bjorn
# Corrected two previous log entries.
#
# Revision 1.30  2006/06/27 15:17:53  bjorn
# Improved parsing and counting from Geert Janssens.
#
# Revision 1.29  2006/06/24 16:06:12  bjorn
# Major rewrite from Mike Cappella.
#
# Revision 1.28  2006/05/26 18:32:50  bjorn
# Additional regexp adjustment, by 'Who Knows'.
#
# Revision 1.27  2006/05/21 18:17:41  bjorn
# Complies with recent amavis releases, by Who Knows.
#
# Revision 1.26  2006/04/02 17:26:52  kirk
# fixed spelling error
#
# Revision 1.25  2006/01/29 23:52:53  bjorn
# Print out virus names and sender, by Felix Schwarz.
#
# Revision 1.24  2005/12/07 19:15:56  bjorn
# Detect and count timeouts, by 'Who Knows'.
#
# Revision 1.23  2005/11/30 05:34:10  bjorn
# Corrected regexp with space, by Markus Lude.
#
# Revision 1.22  2005/11/22 18:34:32  bjorn
# Recognize 'Passed' bad headers, by "Who Knows".
#
# Revision 1.21  2005/10/26 05:40:43  bjorn
# Additional patches for amavisd-new, by Mike Cappella
#
##########################################################################
# This was originally written by:
#    Jim O'Halloran <jim@kendle.com.au>
#
# Please send all comments, suggestions, bug reports,
#    etc, to logwatch-devel@logwatch.org and jim@kendle.com.au.
##########################################################################

$Detail = $ENV{'LOGWATCH_DETAIL_LEVEL'};
my $re_IP = '(?:\d{1,3}\.){3}(?:\d{1,3})';

# Parse logfile
while (defined($ThisLine = <STDIN>)) {
   $Action = "";
   $From = "";
   $FromIP = "";
   $Towards = "";
   $Key = "";
   $Item = "";
   $Reason = "";

   $ThisLine =~ s/^\([A-z\d-]+\) //;
   while ($ThisLine =~ s/\.\.\.$//) {
      chomp($ThisLine);
      $NextLine = <STDIN>;
      $NextLine =~ s/^\([\d-]+\) \.\.\.//;
      $ThisLine .= $NextLine;
   }

   
   if ( ($ThisLine =~ /^do_ascii/) 
       # We don't care about these
        or ($ThisLine =~ /^SPAM,.*score=(?:[\d.+])+ tag=/)
        or ($ThisLine =~ /^Found av scanner/) 
        or ($ThisLine =~ /^Found myself/)
        or ($ThisLine =~ /^Module/)
        or ($ThisLine =~ /^TIMING/)
        or ($ThisLine =~ /^Checking/)
        or ($ThisLine =~ /^(ESMTP|FWD|SEND) via/)
        or ($ThisLine =~ /^spam_scan/)
        or ($ThisLine =~ /^Not-Delivered/)
        or ($ThisLine =~ /^SpamControl/)
        or ($ThisLine =~ /^SPAM-TAG/)
        or ($ThisLine =~ /SPAM\.TAG2/)
        or ($ThisLine =~ /BAD-HEADER\.TAG2/)
        or ($ThisLine =~ /^Net/)
        or ($ThisLine =~ /^Perl/)
        or ($ThisLine =~ /^ESMTP/)
        or ($ThisLine =~ /^LMTP/)
        or ($ThisLine =~ /^tempdir being removed/)
        or ($ThisLine =~ /^Found \$[\S]+[\s]+at/)  
        or ($ThisLine =~ /^No \$[\S]+,[\s]+not using it/) 
        or ($ThisLine =~ /^mail_via_smtp/)
        or ($ThisLine =~ /^local delivery: /)
        or ($ThisLine =~ /^cached [a-zA-Z0-9]+ /)
        or ($ThisLine =~ /^loaded policy bank/)
        or ($ThisLine =~ /^wbl: soft-(?:white|black)listed/)
        or ($ThisLine =~ /^p\d+ \d+(\/\d+)* Content-Type: /)
        or ($ThisLine =~ /^Requesting (a |)process rundown after [0-9]+ tasks/)
        or ($ThisLine =~ /^NOTICE: Not sending DSN, spam level [0-9.]+ exceeds DSN cutoff level/)
        or ($ThisLine =~ /^INFO: unfolded \d+ illegal all-whitespace continuation line/)
        or ($ThisLine =~ /^Cached (virus|spam) check expired/)
        or ($ThisLine =~ /^p\.path BANNED/)
        or ($ThisLine =~ /^virus_scan: /)
        or ($ThisLine =~ /^providing full original message to scanners as/)
        or ($ThisLine =~ /^WARN: MIME::Parser error: /)
        or ($ThisLine =~ /^Actual message size [0-9]+ B(,| greater than the) declared [0-9]+ B/)
        or ($ThisLine =~ /^disabling DSN/)
        or ($ThisLine =~ /^(?:run|ask)_av /)
        or ($ThisLine =~ /^Virus [^,]+ matches [^,]+, sender addr ignored/)
        or ($ThisLine =~ /^Not calling virus scanners, no files to scan in/)
        or ($ThisLine =~ /^lookup_ip_acl /)
         #or ($ThisLine =~ /^extra modules loaded/)
         #or ($ThisLine =~ /^BAD HEADER from [(<][^>)]+[)>]: [^:]*:/ )
        ) {

   } elsif ($ThisLine =~ /^Passed( CLEAN)?, /) {
      $Counts{'CleanMsgsPassed'}++;

   } elsif ($ThisLine =~ /^Blocked( CLEAN)?, /) {
      $Counts{'CleanMsgsBlocked'}++;

   } elsif (($Action, $FromIP, $From, $Towards) = ( $ThisLine =~ /^(Passed |Blocked )?SPAM(?:MY)?,(?: LOCAL)?(?: \[($re_IP)\])?(?: \[$re_IP\])* [<(]([^>)]*)[>)] -\> [(<]([^>)]*)[)>]/o )) {
      # Blocked SPAM, [192.168.0.1] [01.0.0.2] <bogus@example.com> -> <someuser@sample.net>...
      # Blocked SPAM, LOCAL [192.168.0.1] [01.0.0.2] <bogus@example.com> -> <someuser@sample.net>...
      # XXX can null IPs occur? they shouldn't...
      # print "XXX FromIP: \"$FromIP\", From: \"$From\", Towards: \"$Towards\"\n";

      if ($Action eq "Passed ") {
         $Counts{'SpamPassed'}++;
      } else {
         $Counts{'SpamBlocked'}++;
      }
      if ($Detail >= 5) {
         $From = '<>' if ($From =~ /^$/);
         $Spams{$Towards}{$FromIP}{$From}++;
      }

   } elsif (($Action, $Key, $FromIP, $From) = ( $ThisLine =~ /^(?:Virus found - quarantined|(Passed |Blocked )?INFECTED) \(([^\)]+)\),(?: LOCAL)?(?: \[($re_IP)\])?(?: \[$re_IP\])* [<(]([^>)]*)[>)]/o )) {
      # Blocked INFECTED (HTML.Phishing.Pay-43), [192.168.0.1] [10.0.0.2] <bogus@example.com> -> <someuser@sample.net>...
      # Blocked INFECTED (HTML.Phishing.Pay-43), LOCAL [192.168.0.1] [10.0.0.2] <bogus@example.com> -> <someuser@sample.net>...
      # print "XXX Key: \"$Key\", FromIP: \"$FromIP\", From: \"$From\"\n";

      if ($Action eq "Passed ") {
         $Counts{'MalwarePassed'}++;
      } else {
         $Counts{'MalwareBlocked'}++;
      }
      if ($Detail >= 5) {
        $Malware{$Key}{$FromIP}{$From}++;
      }

   } elsif (($Action, $Item, $FromIP, $From, $Towards) = ( $ThisLine =~ /^(Blocked |Passed )?BANNED (?:name\/type )?\(([^\)]+)\),(?: LOCAL)?(?: \[($re_IP)\])?(?: \[$re_IP\])* [<(]([^>)]*)[>)] -> [(<]([^(<]+)[(>]/o)) {
      # the first IP is the envelope sender.
      # Blocked BANNED (multipart/report | message/partial,.txt), [192.168.0.1] [10.0.0.2] <> -> <someuser@sample.net>...
      # Blocked BANNED (multipart/report | message/partial,.txt), LOCAL [192.168.0.1] [10.0.0.2] <> -> <someuser@sample.net>...
      # print "XXX Item: \"$Item\", FromIP: \"$FromIP\", From: \"$From\", Towards: \"$Towards\"\n";

      if ($Action eq "Passed ") {
         $Counts{'BannedNamesPassed'}++;
      } else {
         $Counts{'BannedNamesBlocked'}++;
      }
      if ($Detail >= 5) {
         $From = '<>' if ($From =~ /^$/);
         $Banned{$Towards}{$Item}{$FromIP}{$From}++;
      }

   } elsif (($Action, $FromIP, $From, $Towards) = ( $ThisLine =~ /^(Blocked |Passed )?BAD-HEADER,(?: LOCAL)?(?: \[([^\]]*)\])* [(<]([^>)]*)[)>](?: -\> [(<]([^>)]+)[)>])[^:]*/ )) {
      # Passed BAD-HEADER, [192.168.0.1] [10.0.0.2] <bogus@example.com> -> <someuser@sample.net>...
      # Passed BAD-HEADER, LOCAL [192.168.0.1] [10.0.0.2] <bogus@example.com> -> <someuser@sample.net>...
      # Passed BAD-HEADER, [192.168.0.1] [10.0.0.2] <bogus@example.com> -> <someuser@sample.net>...
      # print "XXX Bad Header: FromIP: \"$FromIP\", From: \"$From\", Towards: \"$Towards\"\n";

      if ($Action eq "Passed ") {
         $Counts{'BadHeadersPassed'}++;
      } else {
         $Counts{'BadHeadersBlocked'}++;
      }
      $From = '<>' if ($From =~ /^$/);
      $BadHeaders{$Towards}{$FromIP}{$From}++;

   } elsif (($From, $Reason) = ( $ThisLine =~ /^BAD HEADER from (?:[(](?:bulk|list|junk) *[)] )?[(<]([^>)]+)[)>]: ([^:]*):.*/ )) {
       # When log_level > 1, provide additional header or MIME violations

       # BAD HEADER from <bogus@example.com>: Non-encoded 8-bit data (char F1 hex) in message header 'Subject': ...
       # BAD HEADER from <bogus@example.com>: Improper use of control character (char 0D hex) in message header 'Received': ...
       # BAD HEADER from <bogus@example.com>: MIME error: error: part did not end with expected boundary
       # BAD HEADER from (list) <bogus@bounces@lists.example.com>: Non-encoded 8-bit data (char E9 hex) in message header 'From'
       # BAD HEADER from (bulk ) <bogus@bounces@lists.example.com>: Non-encoded 8-bit data (char E9 hex) in message header 'From'

      $Counts{'BadHeadersSupp'}++;
      if ($Detail >= 5) {
        $BadHeadersSupp{$Reason}{$From}++;
      };

   } elsif ( $ThisLine =~ /: spam level exceeds quarantine cutoff level/ ) {
      $Counts{'NoQuarantine'}++;

   } elsif ( $ThisLine =~ /^NOTICE: Not sending DSN, spam level exceeds DSN cutoff level(?: for all recips)?, mail intentionally dropped/ ) {
      $Counts{'NoDSNSentCutoff'}++;

   } elsif ( $ThisLine =~ /^NOTICE: Not sending DSN to believed-to-be-faked sender/ ) {
      $Counts{'NoDSNSentFaked'}++;

   } elsif ( $ThisLine =~ /^NOTICE: DSN contains [^;]+; bounce is not bounc[ai]ble, mail intentionally dropped/ ) {
      $Counts{'NoDSNSentBad'}++;

   } elsif ( $ThisLine =~ /^INFO: no existing header field 'Subject', inserting it/ ) {
      $Counts{'NoSubject'}++;

   } elsif ( $ThisLine =~ /zero length members, archive retained/ ) {
      $Counts{'ZipEmptyMembers'}++;

   } elsif ( $ThisLine =~ /^(?:\(\!\) )?SA TIMED OUT,/ ) {
      $Counts{'SATimeout'}++;

   } elsif ( $ThisLine =~ /^missing message body; fatal error/ ) {
      $Counts{'DCCErrors'}++;

   } elsif (($ThisLine =~ /^white_black_list: whitelisted sender/ )
	 		or ( $ThisLine =~ /.* WHITELISTED/) ) {
	 		$Counts{'Whitelisted'}++;

   } elsif (($ThisLine =~ /^white_black_list: blacklisted sender/ )
	 		or ( $ThisLine =~ /.* BLACKLISTED/) ) {
	 		$Counts{'Blacklisted'}++;

   # Extra Modules loaded at runtime
   # extra modules loaded: unicore/lib/gc_sc/Digit.pl, unicore/lib/gc_sc/SpacePer.pl
   } elsif (($Item) = ( $ThisLine =~ /^extra modules loaded: (.+)$/ )) {
      foreach my $code (split /, /, $Item) {
         $ExtraModules{$code}++;
      }

   # Decoders
   } elsif (($Suffix, $Info) = ( $ThisLine =~ /^Internal decoder for (\.\S*)\s*(?:\(([^)]*)\))?$/ )) {
      $StartInfo{'Decoders'}{'Internal'}{$Suffix} = $Info;

   } elsif (($Suffix, $Attempted) = ( $ThisLine =~ /^No decoder for\s+(\.\S*)\s+tried:\s+(.*)$/ )) {
      $StartInfo{'Decoders'}{'None'}{$Suffix} = "tried: $Attempted";

   } elsif (($Suffix, $Decoder) = ( $ThisLine =~ /^Found decoder for\s+(\.\S*)\s+at\s+(.*)$/ )) {
      $StartInfo{'Decoders'}{'External'}{$Suffix} = $Decoder;

   # AV Scanners
   } elsif (($Type, $Item) = ( $ThisLine =~ /^Found (primary|secondary) av scanner (.+)$/ )) {
      $StartInfo{'AVScanner'}{"\u$Type"}{$Item}++;

   } elsif (($Type, $Item) = ( $ThisLine =~ /^Using internal av scanner code for \(([^)]+)\) (.+)$/ )) {
      $StartInfo{'AVScanner'}{"Internal \u$Type"}{$Item}++;

   # (Un)Loaded code, protocols, etc.
   } elsif (($Item, $Loaded) = ( $ThisLine =~ /^(\S+)\s+(?:proto? |base )?\s*(?:code)?\s+((?:NOT )?loaded)$/ )) {
      $StartInfo{'Code'}{$Loaded}{$Item}++;

   } elsif (($Item) = ( $ThisLine =~ /^INFO: no optional modules: (.+)$/ )) {
      foreach my $code (split / /, $Item) {
         $StartInfo{'Code'}{'NOT loaded'}{$code}++;
      }

   } elsif (($Item) = ( $ThisLine =~ /^starting\.\s+ (.+)$/ )) {
      $StartInfo{'Start'} = $Item;

   } elsif (($Item) = ( $ThisLine =~ /^Creating db in (.+)$/ )) {
      $StartInfo{'DB'} = $Item;

   } elsif (( $ThisLine =~ /^user=([^,]*), EUID: (\d+) [(](\d+)[)];\s+group=([^,]*), EGID: ([\d ]+)[(]([\d ]+)[)]/ )) {
      $StartInfo{'IDs'}{'User'} = $1;
      $StartInfo{'IDs'}{'EUID'} = $2;
      $StartInfo{'IDs'}{'UID'} = $3;
      $StartInfo{'IDs'}{'Group'} = $4;
      $StartInfo{'IDs'}{'EGID'} = $5;
      $StartInfo{'IDs'}{'GID'} = $6;

   } else {
      # Report any unmatched entries...
      chomp($ThisLine);
      $OtherList{$ThisLine}++;        
   }
}


#######################################################
# Output report


my $TotalMsgs = $Counts{'CleanMsgsPassed'} + $Counts{'CleanMsgsBlocked'} +
                $Counts{'SpamPassed'} + $Counts{'SpamBlocked'} +
                $Counts{'MalwarePassed'} + $Counts{'MalwareBlocked'} +
                $Counts{'BannedNamesPassed'} + $Counts{'BannedNamesBlocked'} +
                $Counts{'BadHeadersPassed'} + $Counts{'BadHeadersBlocked'};

#
# Counts key, print format, optional percentage divisor
#
my @Formats = (
   [ 'CleanMsgsPassed',    "%8d (%5.2f%%)   Clean passed",                   $TotalMsgs ],
   [ 'CleanMsgsBlocked',   "%8d (%5.2f%%)   Clean blocked",                  $TotalMsgs ],
   [ 'MalwarePassed',      "%8d (%5.2f%%)   Malware passed",                 $TotalMsgs ],
   [ 'MalwareBlocked',     "%8d (%5.2f%%)   Malware blocked",                $TotalMsgs ],
   [ 'SpamPassed',         "%8d (%5.2f%%)   Spam passed",                    $TotalMsgs ],
   [ 'SpamBlocked',        "%8d (%5.2f%%)   Spam blocked",                   $TotalMsgs ],
   [ 'NoQuarantine',       "%8d (%5.2f%%)   Spam discarded, not quarantined (spam score > quarantine cutoff)", $Counts{'SpamBlocked'} ],
   [ 'BannedNamesPassed',  "%8d (%5.2f%%)   Banned file names passed",       $TotalMsgs ],
   [ 'BannedNamesBlocked', "%8d (%5.2f%%)   Banned file names blocked",      $TotalMsgs ],
   [ 'BadHeadersPassed',   "%8d (%5.2f%%)   Bad headers passed",             $TotalMsgs ],
   [ 'BadHeadersBlocked',  "%8d (%5.2f%%)   Bad headers blocked",            $TotalMsgs ],
   [ 'BadHeadersSupp',     "%8d            Bad headers (debug supplemental)" ],
   [ 'ZipEmptyMembers',    "%8d            Compressed attachments with empty members" ],
   [ 'NoDSNSentBad',       "%8d            DSNs not sent: bad DSN" ],
   [ 'NoDSNSentCutoff',    "%8d            DSNs not sent: spam score > DSN cutoff" ],
   [ 'NoDSNSentFaked',     "%8d            DSNs not sent: presumed bogus sender" ],
   [ 'NoSubject',          "%8d            Headers without subject line" ],
   [ 'Whitelisted',        "%8d            Whitelisted" ],
   [ 'Blacklisted',        "%8d            Blacklisted" ],
   [ 'SATimeout',          "%8d            SpamAssassin timeouts" ],
   [ 'DCCErrors',          "%8d            DCC errors" ],
);

if ($TotalMsgs > 0) {
   printf "\n%8d            Scanned\n", $TotalMsgs;
}

for $row ( @Formats ) {
   if ($Counts{@$row[0]} > 0) {
      #print "Row2 is @$row[2], Row0 is @$row[0], Counts is $Counts{@$row[0]}\n";
      if (@$row[2] > 0) {
         printf "@$row[1]\n", $Counts{@$row[0]}, $Counts{@$row[0]} * 100 / @$row[2];
      }
      else {
         printf "@$row[1]\n", $Counts{@$row[0]};
      }
   }
}

print;

sub doReport(\% $ $) {
   local $theHash = shift;
   local $title = shift;
   local $level = shift;
   local @tmpList;

   local $item;

   local $count = 0;

   @tmpList = ();

   foreach $item (sort keys %$theHash) {
      if (ref($theHash->{$item}) eq "HASH") {
         #print " " x ($level * 4), "LEVEL $level: Item: $item, type: \"", ref($theHash->{$item}), "\"\n";

         ($retval, @childList) = doReport ($theHash->{$item}, '', $level + 1);

         # me + children
         push (@tmpList, sprintf "%s %8d  $item\n", '     ' x $level, $retval);
         push (@tmpList, @childList) if ($Detail > ($level + 5));
         #print "Pushed me and Children: tmpList: <<<", "@tmpList", ">>>\n";

         $count += $retval;

         #print " " x ($level * 4), "LEVEL $level: Found $retval, running total: $count\n";
      }
      else {
         push (@tmpList, sprintf "%s %8d  $item\n", '             ' x $level, $theHash->{$item})  if ($Detail > ($level + 4));
         $count += $theHash->{$item};

         #print " " x ($level * 4), "LEVEL $level: Item: $item (n = $theHash->{$item}), Total: $count\n";
         #print " " x ($level * 4), "LEVEL $level: tmpList3 is <<<@tmpList>>>\n";
      }
   }

   #print " " x ($level * 4), "LEVEL $level: Returning from level $level\n";

   if ($level == 0 && ($count > 0)) {
      printf "\n%8d $title:\n @tmpList", $count;
   }

   return ($count, @tmpList);
}

if ($Detail >= 5) {

   doReport (%Malware,        "Malware Messages", 0);
   doReport (%Spams,          "Spam Messages", 0);
   doReport (%BadHeaders,     "Bad Headers", 0);
   doReport (%BadHeadersSupp, "Bad Headers (supplemental)", 0);
   doReport (%Banned,         "Banned File Names", 0);

   if (keys %ExtraModules) {

      print "\n\nExtra Code Modules loaded at runtime:\n";
      foreach $code (sort keys %ExtraModules) {
         printf "%8d : %s\n", $ExtraModules{$code}, $code;
      }
   }

   if (keys %StartInfo) {

      print "\n\nAmavis Startup:\n";

      print "    Amavis:   $StartInfo{'Start'}\n"                   if ($StartInfo{'Start'});
      print "    Database: $StartInfo{'DB'}\n"                      if ($StartInfo{'DB'});

      if (keys %{$StartInfo{'IDs'}}) {
         print "    Process User/Group:\n";
         print "        User:  $StartInfo{'IDs'}{'User'}, EUID: $StartInfo{'IDs'}{'EUID'}, UID: $StartInfo{'IDs'}{'UID'}\n";
         print "        Group: $StartInfo{'IDs'}{'Group'}, EGID: $StartInfo{'IDs'}{'EGID'}, GID: $StartInfo{'IDs'}{'GID'}\n";
      }

      foreach $loaded (keys %{$StartInfo{'Code'}}) {
         if ($Detail < 10) {
            local $, = ' ';
            print "    Code/Module - \u$loaded:  ", sort keys %{$StartInfo{'Code'}{$loaded}};
         }
         else {
            print "    Code/Module - \u$loaded:\n";
            foreach $code (sort keys %{$StartInfo{'Code'}{$loaded}}) {
               print "        $code\n";
            }
         }
         printf "\n";
      }

      foreach $type (keys %{$StartInfo{'AVScanner'}}) {
         # primary, secondary, internal

         if ($Detail < 10) {
            local $, = ' ';
            print "    AV Scanner - $type:  ", sort keys %{$StartInfo{'AVScanner'}{$type}};
         }
         else {
            print "    AV Scanner - $type:\n";
            foreach $scanner (keys %{$StartInfo{'AVScanner'}{$type}}) {
               print "        $scanner\n";
            }
         }
         print "\n";
      }

      foreach $type (sort keys %{$StartInfo{'Decoders'}}) {
         # external, internal, none

         if ($Detail < 10) {
            local $, = ' ';
            print "    Decoder - $type:  ", sort keys %{$StartInfo{'Decoders'}{$type}};
         }
         else {
            print "    Decoder - $type:\n";
            foreach $suffix (sort keys %{$StartInfo{'Decoders'}{$type}}) {
               printf "          %6s : %s\n", $suffix, $StartInfo{'Decoders'}{$type}{$suffix};
            }
         }
         print "\n";
      }
   }
}

if (keys %OtherList) {
   print "\n\n**Unmatched Entries**\n";
   foreach $line (sort {$OtherList{$b}<=>$OtherList{$a} } keys %OtherList) {
      print "   $line: $OtherList{$line} Time(s)\n";
   }
}


exit(0);

# vi: shiftwidth=3 tabstop=3 syntax=perl et
