Mercurial > hg > maltfilter

changeset 16:87c0cdc048f5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Many changes and cleanups. Works again.
author Matti Hamalainen <ccr@tnsp.org>
date Sat, 15 Aug 2009 19:41:03 +0300
parents b05d0f0ff106
children fe220b5a975a
files maltfilter
diffstat 1 files changed, 201 insertions(+), 168 deletions(-) [+]
line wrap: on
line diff
--- a/maltfilter	Fri Aug 14 19:12:14 2009 +0300
+++ b/maltfilter	Sat Aug 15 19:41:03 2009 +0300
@@ -61,14 +61,19 @@
 my $pid_file = "";       # Name of Maltfilter daemon pid file
 my $LOGFILE;             # Maltfilter logfile handle
 
-my %blocklist = ();      # IPs currently blocked in Netfilter $blocklist{$ip} = date_blocked
+# IPs currently blocked in Netfilter $blocklist{$ip} = date
+my %blocklist = ();      
 
 # Gathered information about hosts
 # $statlist{$ip}->
-# "date"     = latest change
-# "hits"     = number of hits
-# "reason"   = latest reason why added
-# "reasons"  = array of reasons (only set when $reportmode == 1)
+# "date1"    = timestamp of first hit
+# "date2"    = timestamp of latest hit
+# "hits"     = number of hits to this IP
+# $statlist{$ip}{"reason"}{$class}->
+#   "msg"    = reason message (array if $reportmode)
+#   "hits"   = hits to this class
+#   "date1"  = timestamp of first hit
+#   "date2"  = timestamp of latest hit
 my %statlist = ();
 
 # Gathered information about ignored hits (e.g. hits for tests that are not enabled)
@@ -86,13 +91,13 @@
     
     # (1.1) Generic login scan attempts
     if ($merr =~ /^Failed password for invalid user \S+ from (\d+\.\d+\.\d+\.\d+)/) {
-      check_add_entry($1, $mdate, "SSH login scan", "", $settings{"CHK_SSHD"});
+      check_add_hit($1, $mdate, "SSH login scan", "", $settings{"CHK_SSHD"});
     }
     # (1.2) Root SSH login password bruteforcing attempts
     # NOTICE! Do not enable this setting, if you allow SSH root logins via
     # password authentication! Mistyping password may get you blocked then. :)
     elsif (/^Failed password for root from (\d+\.\d+\.\d+\.\d+)/) {
-      check_add_entry($1, $mdate, "Root SSH password bruteforce", "", $settings{"CHK_ROOT_SSH_PWD"});
+      check_add_hit($1, $mdate, "Root SSH password bruteforce", "", $settings{"CHK_ROOT_SSH_PWD"});
     }
   }
   # (2) Common/known exploitable CGI/PHP software scans (like phpMyAdmin)
@@ -106,7 +111,7 @@
     if ($merr =~ /^File does not exist: (.+)$/) {
       my $tmp = $1;
       if ($tmp =~ /\/mss2|\/pma|admin|sql|\/roundcube|\/webmail|\/bin|\/mail|xampp|zen|mailto:|appserv|cube|round|_vti_bin|wiki/i) {
-        check_add_entry($mip, $mdate, "CGI vuln scan", $tmp, $settings{"CHK_KNOWN_CGI"});
+        check_add_hit($mip, $mdate, "CGI vuln scan", $tmp, $settings{"CHK_KNOWN_CGI"});
       }
     }
   }
@@ -122,13 +127,13 @@
     # used in the URIs.
     if ($merr =~ /\.php\?\S*?=http:\/\/([^\/]+)/) {
       if (!check_hosts($settings{"CHK_GOOD_HOSTS"}, $1)) {
-        check_add_entry($mip, $mdate, "PHP XSS", $merr, $settings{"CHK_PHP_XSS"});
+        check_add_hit($mip, $mdate, "PHP XSS", $merr, $settings{"CHK_PHP_XSS"});
       }
     }
     # (3.2) Try to match proxy scanning attempts
     elsif ($merr =~ /^http:\/\/([^\/]+)/) {
       if (!check_hosts($settings{"CHK_GOOD_HOSTS"}, $1)) {
-        check_add_entry($mip, $mdate, "Proxy scan", $merr, $settings{"CHK_PROXY_SCAN"});
+        check_add_hit($mip, $mdate, "Proxy scan", $merr, $settings{"CHK_PROXY_SCAN"});
       }
     }
   }
@@ -138,24 +143,6 @@
 #############################################################################
 ### Status output functionality
 #############################################################################
-sub cmp_ips($$)
-{
-  my @ipa = split(/\./, $_[0]);
-  my @ipb = split(/\./, $_[1]);
-  for (my $i = 0; $i < 4; $i++) {
-    return -1 if ($ipa[$i] > $ipb[$i]);
-    return  1 if ($ipa[$i] < $ipb[$i]);
-  }
-  return 0;
-}
-
-sub cmp_ip_hits($$$$)
-{
-  return -1 if ($_[2] > $_[3]);
-  return  1 if ($_[2] < $_[3]);
-  return cmp_ips($_[0], $_[1]);
-}
-
 sub printH($$$$)
 {
   my $fh = $_[1];
@@ -213,11 +200,105 @@
   return $_[0] ? "<$_[1]>" : "";
 }
 
-sub getIP($$)
+sub getLink($$)
 {
   return $_[0] ? "<a href=\"http://whois.domaintools.com/$_[1]\">$_[1]</a>" : $_[1];
 }
 
+sub printTable1($$$$$)
+{
+  my ($m, $f, $table, $keys, $func) = @_;
+  my $ntotal = 0;
+
+  printElem($m, $f,
+  "<table class=\"detailed\">\n<tr>".
+  "<th>Hits</th><th>IP-address</th><th>First hit</th><th>Latest hit</th><th>Reason(s)</th>".
+  "</tr>\n");
+  
+  foreach my $mip (sort { $func->($table, $a, $b) } keys %{$keys}) {
+    printElem($m, $f, " <tr>");
+    printTD($m, $f, sprintf("%-10d", $table->{$mip}{"hits"}));
+    printTD($m, $f, sprintf("%-15s", getLink($m, $mip)));
+    printElem(!$m, $f, " : ");
+    printTD($m, $f, scalar localtime($table->{$mip}{"date1"}));
+    printElem(!$m, $f, " : ");
+    printTD($m, $f, scalar localtime($table->{$mip}{"date2"}));
+    printElem(!$m, $f, " : ");
+    my @reasons = ();
+    foreach my $class (sort keys %{$table->{$mip}{"reason"}}) {
+      my $msgs;
+      if ($reportmode) {
+        my @tmp = @{$table->{$mip}{"reason"}{$class}{"msg"}};
+        if ($#tmp > 5) { $#tmp = 5; }
+        $msgs = join(" ".bb($m)."|".eb($m)." ", @tmp);
+      } else {
+        $msgs = $table->{$mip}{"reason"}{$class}{"msg"};
+      }
+      push(@reasons, bb($m).$class.eb($m)." #".$table->{$mip}{"reason"}{$class}{"hits"}." ( ".$msgs." )");
+    }
+    printTD($m, $f, join(", ", @reasons));
+    printElem($m, $f, "</tr>", "\n");
+    $ntotal++;
+  }
+  printElem($m, $f, "</table>\n");
+  printP($m, $f, bb($m).$ntotal.eb($m)." entries total.\n");
+}
+
+sub printTable2($$$$$)
+{
+  my ($m, $f, $table, $keys, $func) = @_;
+  my $nhits = 0;
+  my $str = "<th>IP-address</th><th>Hits</th><th>Latest hit</th><th>Class(es)</th>";
+
+  printElem($m, $f, "<table class=\"summary\">\n<tr>". $str."<th> </th>".$str ."</tr>\n");
+  
+  my $printEntry = sub {
+    printTD($m, $f, sprintf("%-15s", getLink($m, $_[0])));
+    printElem(!$m, $f, " : ");
+    printTD($m, $f, sprintf("%-8d ", $table->{$_[0]}{"hits"}));
+    printElem(!$m, $f, " : ");
+    printTD($m, $f, scalar localtime($table->{$_[0]}{"date2"}));
+    printElem(!$m, $f, " : ");
+    my $tmp = join(", ", sort keys %{$table->{$_[0]}{"reason"}});
+    printTD($m, $f, sprintf("%-30s", $tmp));
+    $nhits += $table->{$_[0]}{"hits"};
+  };
+
+  my @mkeys = sort { $func->($table, $a, $b) } keys %{$keys};
+  my $nkeys = scalar @mkeys;
+  my $kmax = $nkeys / 2;
+  for (my $i = 0; $i <= $kmax; $i++) {
+    printElem($m, $f, " <tr>");
+    if ($i < $kmax) { $printEntry->($mkeys[$i]); }
+    printElem($m, $f, "<th> </th>", " || ");
+    if ($i + $kmax + 1 < $nkeys) { $printEntry->($mkeys[$i + $kmax + 1]); }
+    printElem($m, $f, "</tr>\n", "\n");
+  }
+
+  printElem($m, $f, "</table>\n");
+  printP($m, $f, bb($m).$nkeys.eb($m)." entries total, ".bb($m).$nhits.eb($m)." hits total.\n");
+}
+
+sub cmp_ips($$$)
+{
+  my @ipa = split(/\./, $_[1]);
+  my @ipb = split(/\./, $_[2]);
+  for (my $i = 0; $i < 4; $i++) {
+    return -1 if ($ipa[$i] > $ipb[$i]);
+    return  1 if ($ipa[$i] < $ipb[$i]);
+  }
+  return 0;
+}
+
+sub cmp_hits($$$)
+{
+  return $_[0]->{$_[2]}{"hits"} <=> $_[0]->{$_[1]}{"hits"};
+}
+
+
+###
+###
+###
 sub generate_status($$)
 {
   my $filename = shift;
@@ -243,7 +324,6 @@
 <body>
 ");
 
-
   printH($m, $f, 1, "Maltfilter v$progversion status report");
   my $val = $settings{"WEEDPERIOD"};
   my $period;
@@ -262,94 +342,18 @@
   "Generated ".bb($m).$mtime.eb($m).". Data computed from ".
   ($reportmode ? "complete logfile scan" : "a period of last $period").".\n");
 
-  printH($m, $f, 2, $reportmode ? "Detailed report" : "Blocked entries");
-  printElem($m, $f, "<table>\n<tr>". "<th>Hits</th><th>IP-address</th><th>Date of last hit</th><th>Reason(s)</th>"."</tr>\n");
-  my $nexcluded = 0;
-  my $ntotal = 0;
-  foreach my $mip (sort { $hitcount{$b} <=> $hitcount{$a} } keys %iplist) {
-    $nexcluded++ if check_hosts_array(\@noblock_ips, $mip);
-
-    printElem($m, $f, " <tr>");
-    printTD($m, $f, sprintf("%-10d", $hitcount{$mip}));
-    printTD($m, $f, sprintf("%-15s", getIP($m, $mip)));
-    printElem(!$m, $f, " : ");
-    printTD($m, $f, scalar localtime($iplist{$mip}));
-    my @s = ();
-    foreach my $cond (sort keys %{$reason{$mip}}) {
-      my $str;
-      if ($reportmode) {
-        my @tmp =  reverse(@{$reason{$mip}{$cond}});
-        $#tmp = 5 if ($#tmp > 5);
-        $str = join(" | ", @tmp);
-      } else {
-        $str = $reason{$mip}{$cond};
-      }
-      push(@s, bb($m).$cond.eb($m)." [".$reason_n{$mip}{$cond}." hits] (".$str.")");
-    }
-    printTD($m, $f, join(", ".($m ? "<br />" : ""), @s));
-    printElem($m, $f, "</tr>\n", "\n");
-    $ntotal++;
-  }
-  printElem($m, $f, "</table>\n");
-  printP($m, $f, bb($m).$ntotal.eb($m)." entries listed, ".
-  bb($m).($ntotal - $nexcluded).eb($m)." blocked, ".bb($m).$nexcluded.eb($m).
-  " excluded (defined in NOBLOCK_IPS).\n");
-
-
-  printH($m, $f, 2, "Overview of hits in general");
-  printP($m, $f, "List of 'hits' of suspicious activity noticed by Maltfilter, but not necessarily acted upon.\n");
-  
-  my $tmp = "<th>IP-address</th><th># of hits</th><th>Reasons</th>";
-  printElem($m, $f, "<table>\n<tr>". $tmp."<th> </th>".$tmp ."</tr>\n");
-  my $hits = 0;
-  my @keys = sort { cmp_ips($a, $b) } keys %hitcount;
-  my $nkeys = scalar @keys;
+  printH($m, $f, 2, "Currently blocked entries");
+  printP($m, $f, "List of IPs that are currently blocked (or would be, if this is a report-only mode).");
+  printTable1($m, $f, \%statlist, \%blocklist, \&cmp_hits);
 
-  my $printEntry = sub {
-    printTD($m, $f, sprintf("%-15s", getIP($m, $_[0])));
-    printElem(!$m, $f, " : ");
-    printTD($m, $f, sprintf("%-8d ", $hitcount{$_[0]}));
-    printElem(!$m, $f, " : ");
-    my $tmp = join(", ", sort keys %{$reason{$_[0]}});
-    printTD($m, $f, sprintf("%-30s", $tmp));
-    $hits += $hitcount{$_[0]};
-  };
-
-  my $kmax = $nkeys / 2;
-  for (my $i = 0; $i <= $kmax; $i++) {
-    printElem($m, $f, " <tr>");
-    if ($i < $kmax) {
-      &$printEntry($keys[$i]);
-    }
-    printElem($m, $f, "<th> </th>", " || ");
-    if ($i + $kmax + 1 < $nkeys) {
-      &$printEntry($keys[$i + $kmax + 1]);
-    }
-    printElem($m, $f, "</tr>\n", "\n");
-  }
+  printH($m, $f, 2, "Summary of non-ignored entries");
+  printP($m, $f, "List of 'hits' of suspicious activity noticed by Maltfilter, but not necessarily acted upon.\n");
+  printTable2($m, $f, \%statlist, \%statlist, \&cmp_ips);
 
-  printElem($m, $f, "</table>\n");
-
-  printP($m, $f, bb($m).(scalar keys %hitcount).eb($m)." IPs total, ".bb($m).$hits.eb($m)." hits total.\n");
-
-  
-  printH($m, $f, 2, "Ignored hit types");
+  printH($m, $f, 2, "Ignored entries");
   printP($m, $f, "List of hits that were ignored (not acted upon), because the test was disabled.\n");
+  printTable1($m, $f, \%ignorelist, \%ignorelist, \&cmp_hits);
 
-  printElem($m, $f, "<table>\n<tr><th>IP-address</th><th>Type (hits, last time of note)</th></tr>\n");
-  foreach my $mip (sort { cmp_ips($a, $b) } keys %ignored) {
-    printElem($m, $f, "<tr>");
-    printTD($m, $f, sprintf("%-15s", $mip));
-    printElem($m, $f, "<td>", " : ");
-    foreach my $mcond (sort keys %{$ignored{$mip}}) {
-      my $s = $mcond." (".$hitcount{$mip}." hits, last ".scalar localtime($ignored_d{$mip}{$mcond}).")";
-      unless ($ignored{$mip}{$mcond} eq "") { $s .= " for '".$ignored{$mip}{$mcond}."'"; }
-      print $f $s;
-    }
-    printElem($m, $f, "</td></tr>", "\n");
-  }
-  printElem($m, $f, "</table>\n");
-  
   printElem($m, $f, "</body>\n</html>\n");
   close(STATUS);
 }
@@ -358,7 +362,7 @@
 #############################################################################
 ### Entry management / handling functions
 #############################################################################
-### Host and IP matching functions
+### Check if given IP or host exists in array
 sub check_hosts_array($$)
 {
   my $chk_host = $_[1];
@@ -377,6 +381,7 @@
   return 0;
 }
 
+### Check IP/host against | separated list of IPs/hosts
 sub check_hosts($$)
 {
   my @tmp = split(/\s*\|\s*/, $_[0]);
@@ -394,45 +399,45 @@
   }
 }
 
-### Get current Netfilter INPUT table entries we manage
-sub update_iplist($)
+### Get current Netfilter INPUT table entries that match
+### entry types we manage, e.g. blocklist
+sub update_blocklist($)
 {
   my $mdate = $_[0];
   open(STATUS, $settings{"IPTABLES"}." -v -n -L INPUT |") or
     die("Could not execute ".$settings{"IPTABLES"}."\n");
+  %blocklist = ();
+  undef(%blocklist);
   while (<STATUS>) {
     chomp;
     if (/^\s*(\d+)\s+\d+\s+$settings{"ACTION"}\s+all\s+--\s+\*\s+\*\s+(\d+\.\d+\.\d+\.\d+)\s+0\.0\.0\.0\/0\s*$/) {
-      my $mcount = $1;
       my $mip = $2;
-      if (!defined($blocklist{$mip}) && $mdate >= 0) {
-        mlog(2, "* $mip appeared in iptables, adding.\n");
-        $hitcount{$mip} = $settings{"THRESHOLD"};
-        check_add_entry($mip, $mdate, "?", "added from iptables", 1);
+      if (!defined($blocklist{$mip}) && $mdate > 0) {
+        mlog(2, "* $mip appeared in iptables.");
       }
+      $blocklist{$2} = $mdate;
+      update_entry(\%statlist, $mip, $mdate, "?", "From iptables.");
     }
   }
   close(STATUS);
 }
 
-### Weed out old entries
+### Check if given timestamp is _newer_ than weedperiod threshold.
+### Returns false if timestamp is over weed period, e.g. needs weeding.
 sub check_time($)
 {
   return ($_[0] >= time() - ($settings{"WEEDPERIOD"} * 60 * 60));
 }
 
+### Weed out old entries
 sub weed_do($)
 {
-  if (defined($blocklist{$_[0]})) {
-    mlog(2, "* Weeding $_[0] ($iplist{$_[0]})\n");
-    exec_iptables("-D", "INPUT", "-s", $_[0], "-d", "0.0.0.0/0", "-j", $settings{"ACTION"});
-  }
-  undef($reason{$_[0]});
-  undef($reason_n{$_[0]});
-  undef($ignored{$_[0]});
-  undef($ignored_d{$_[0]});
-  undef($iplist{$_[0]});
-  undef($blocklist{$_[0]});
+  my $mtime = $blocklist{$_[0]};
+  mlog(2, "* Weeding $_[0] (".($mtime >= 0 ? scalar localtime($mtime) : $mtime)."\n");
+  exec_iptables("-D", "INPUT", "-s", $_[0], "-d", "0.0.0.0/0", "-j", $settings{"ACTION"});
+  delete($blocklist{$_[0]});
+  delete($statlist{$_[0]});
+  delete($ignorelist{$_[0]});
 }
 
 sub weed_entries()
@@ -440,55 +445,79 @@
   # Don't weed in report mode.
 # return if ($reportmode);
 
-  foreach my $mip (keys %iplist) {
-    if (defined($iplist{$mip})) {
-      if ($iplist{$mip} >= 0) {
-        if (!check_time($iplist{$mip})) { weed_do($mip); }
+  my @mips = keys %blocklist;
+  foreach my $mip (@mips) {
+    if (defined($blocklist{$mip})) {
+      if ($blocklist{$mip} >= 0) {
+        weed_do($mip) unless check_time($blocklist{$mip});
       } else {
         weed_do($mip);
       }
     }
   }
-  mlog(-1, "hmm\n");
+}
+
+### Update one entry of
+sub update_entry($$$$$)
+{
+  my ($struct, $mip, $mdate, $mclass, $mreason) = @_;
+
+  my $cnt = $struct->{$mip}{"hits"}++;
+  $struct->{$mip}{"reason"}{$mclass}{"hits"}++;
+
+  if ($reportmode) {
+    push(@{$struct->{$mip}{"reason"}{$mclass}{"msg"}}, $mreason);
+  } else {
+    $struct->{$mip}{"reason"}{$mclass}{"msg"} = $mreason;
+  }
+
+  if (!defined($struct->{$mip}{"date1"})) {
+    $struct->{$mip}{"date1"} = $mdate;
+  }
+  $struct->{$mip}{"date2"} = $mdate;
+    
+  if (!defined($struct->{$mip}{"reason"}{$mclass}{"date2"})) {
+    $struct->{$mip}{"reason"}{$mclass}{"date2"} = $mdate;
+  }
+  $struct->{$mip}{"reason"}{$mclass}{"date2"} = $mdate;
+
+  return $cnt;
 }
 
 ### Check if given "try count" exceeds treshold and if entry
 ### is NOT in Netfilter already, then add it if so.
-sub check_add_entry($$$$$)
+sub check_add_hit($$$$$)
 {
   my $mip = $_[0];
   my $mdate = str2time($_[1]);
   my $mclass = $_[2];
   my $mreason = $_[3];
   my $mcond = $_[4];
-  
-  my $cnt = $hitcount{$mip}++;
-  $reason_n{$mip}{$mclass}++;
-  if ($reportmode) {
-    push(@{$reason{$mip}{$mclass}}, $mreason);
-  } else {
-    $reason{$mip}{$mclass} = $mreason;
+  my $cnt;
+
+  if (check_hosts_array(\@noblock_ips, $mip)) {
+    mlog(3, "Hit to NOBLOCK_IPS($mip): [$mclass] $mreason\n");
+    return;
   }
-  if ($reportmode || ($cnt >= $settings{"TRESHOLD"} && check_time($mdate))) {
-    my $pat;
-    if (!$mcond) {
-      $ignored{$mip}{$mclass} = $mreason;
-      $ignored_d{$mip}{$mclass} = $mdate;
-      return;
+  
+  # If condition is true, we add to regular statlist  
+  if ($mcond) {
+    $cnt = update_entry(\%statlist, $mip, $mdate, $mclass, $mreason);
+  } else {
+    # This is an ignored hit (for disabled test), add to ignorelist
+    update_entry(\%ignorelist, $mip, $mdate, $mclass, $mreason);
+    return;
+  }
+
+  # Check if we have exceeded treshold etc.
+  if ($cnt >= $settings{"TRESHOLD"} && check_time($mdate)) {
+    # Add to blocklist, unless already there.
+    if (!defined($blocklist{$mip})) {
+      mlog(1, "* Adding $mip ($mdate): [$mclass] $mreason\n");
+      exec_iptables("-I", "INPUT", "1", "-s", $mip, "-j", $settings{"ACTION"});
     }
-    if (!defined($iplist{$mip})) {
-      if (!check_hosts_array(\@noblock_ips, $mip)) {
-        # Add entry that has >= treshold hits and is not added yet
-        mlog(1, "* Adding $mip ($mdate): $mreason\n");
-        exec_iptables("-I", "INPUT", "1", "-s", $mip, "-j", $settings{"ACTION"});
-      }
-      $iplist{$mip} = $mdate;
-    } else {
-      # Over treshold, but is added, check if we can update the timedate
-      if ($mdate > $iplist{$mip}) {
-        $iplist{$mip} = $mdate;
-      }
-    }
+    # Update date of last hit
+    $blocklist{$mip} = $mdate;
   }
 }
 
@@ -511,7 +540,7 @@
 ### Initialize
 sub malt_init {
   mlog(0, "Updating initial blocklist from netfilter.\n");
-  update_iplist(-1);
+  update_blocklist(-1);
 
   foreach my $filename (@scanfiles) {
     local *INFILE;
@@ -545,6 +574,7 @@
   undef($LOGFILE);
 }
 
+### Signal handlers
 sub malt_int {
   mlog(-1, "\nCaught Interrupt (^C), aborting.\n");
   malt_cleanup();
@@ -583,7 +613,7 @@
       # (in case entries have appeared there from "outside")
       # and perform weeding of old entries.
       $counter = 0;
-      update_iplist(time());
+      update_blocklist(time());
       weed_entries();
       generate_status($settings{"STATUS_FILE_PLAIN"}, 0);
       generate_status($settings{"STATUS_FILE_HTML"}, 1);
@@ -595,6 +625,7 @@
   }
 }
 
+### Read one configuration file
 sub malt_read_config($)
 {
   my $filename = $_[0];
@@ -673,6 +704,7 @@
 # Read configuration files
 if (defined(my $filename = shift)) {
   # Let user define his/her own logfiles to scan
+  @scanfiles_def = ();
   undef(@scanfiles_def);
   die("Errors in configuration file '$filename', bailing out.\n")
     unless (malt_read_config($filename) == 0);
@@ -688,8 +720,9 @@
 my %saw = ();
 @scanfiles = grep(!$saw{$_}++, @scanfiles_def);
 
+%saw = ();
+@noblock_ips = grep(!$saw{$_}++, @noblock_ips_def);
 undef(%saw);
-@noblock_ips = grep(!$saw{$_}++, @noblock_ips_def);
 
 # Open logfile
 if ($settings{"DRY_RUN"}) {