Mercurial > hg > maltfilter

changeset 65:d2e2b82dd2f2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Work on DroneBL support.
author Matti Hamalainen <ccr@tnsp.org>
date Tue, 18 Aug 2009 00:43:10 +0300
parents 213e5204abea
children 42889eed0ce8
files README example.conf maltfilter
diffstat 3 files changed, 189 insertions(+), 82 deletions(-) [+]
line wrap: on
line diff
--- a/README	Mon Aug 17 17:46:15 2009 +0300
+++ b/README	Tue Aug 18 00:43:10 2009 +0300
@@ -1,4 +1,4 @@
-Malicious Attack Livid Termination Filter daemon (maltfilter) v0.14.0
+Malicious Attack Livid Termination Filter daemon (maltfilter) v0.15.0
 =====================================================================
 Programmed by Matti 'ccr' Hämäläinen <ccr@tnsp.org>
 (C) Copyright 2009 Tecnic Software productions (TNSP)
@@ -14,7 +14,8 @@
 these connections are then blocked via Netfilter (iptables).
 
 Additionally Maltfilter can generate status reports (either continuously
-in daemon mode, or as once-run report), in plaintext and HTML formats.
+in daemon mode, or as once-run report), in plaintext and HTML formats
+and submit data to DroneBL DNSBL service.
 
 Since v0.14, there is also option for gathering "evidence" about certain
 PHP XSS exploit attempts into specified directory. These evidence files
--- a/example.conf	Mon Aug 17 17:46:15 2009 +0300
+++ b/example.conf	Tue Aug 18 00:43:10 2009 +0300
@@ -34,19 +34,20 @@
 #############################################################################
 ### Actions, etc. settings
 #############################################################################
-## Weeding treshold in hours. Entries older than this will be removed
-## off from current netfilter settings (e.g. they become unblocked again.)
-#WEED_BLOCK = 168
+## Weeding threshold in hours. Entries older than this will be removed
+## off from current netfilter settings. Also, entries older than this
+## will not be added to netfilter to begin with.
+#WEED_FILTER = 168
 
 ## For how many hours to keep general information about IP. Affects from
-## how long period statistics dump shows data. Also hitcount tresholds
-## take the old data into account, meaning that if WEED_BLOCK < WEED_GLOBAL
-## hit data older than WEED_BLOCK will be counted towards THRESHOLD.
+## how long period statistics dump shows data. Also hitcount thresholds
+## take the old data into account, meaning that if WEED_FILTER < WEED_GLOBAL
+## hit data older than WEED_FILTER will be counted towards THRESHOLD.
 #WEED_GLOBAL = 336
 
 ## How many "hits" the IP needs until it is eligible to be blocked.
 ## (the "hits" can be from any "source", e.g. sshd crack, httpd, etc.)
-#TRESHOLD = 3
+#THRESHOLD = 3
 
 ## Target iptables action for added entries, default is DROP, but you
 ## can use whatever rule chain name you want to here.
@@ -174,3 +175,30 @@
 
 #EVIDENCE            = 0
 #EVIDENCE_DIR        = "/var/run/malt-evidence"
+
+
+#############################################################################
+### DroneBL submissions
+#############################################################################
+## Maltfilter can automatically submit entries to DroneBL DNSBL service.
+## See <http://www.dronebl.org/> for more information.
+
+## 0 = submission disabled, otherwise enabled
+DRONEBL              = 0
+
+## Number of hits required before host IP goes to submission queue.
+## This setting is independent of the general THRESHOLD value and
+## only affects DroneBL submissions.
+DRONEBL_THRESHOLD    = 5
+
+## Maximum age of hits counted towards DroneBL submission threshold.
+## There is currently no weeding of submissions.
+DRONEBL_MAX_AGE      = 30
+
+## Your personal RPC key. This _MUST_ be set to a valid value, if you
+## have enabled submissions. To get a personal key, go to:
+## http://www.dronebl.org/rpckey_signup
+DRONEBL_RPC_KEY      = ""
+
+## RPC2 submission URI, usually you do not need to change this.
+#DRONEBL_RPC_URI      = "http://dronebl.org/RPC2"
--- a/maltfilter	Mon Aug 17 17:46:15 2009 +0300
+++ b/maltfilter	Tue Aug 18 00:43:10 2009 +0300
@@ -12,7 +12,7 @@
 use Net::DNS;
 use LWP::UserAgent;
 
-my $progversion = "0.14.0";
+my $progversion = "0.15.0";
 my $progbanner =
 "Malicious Attack Livid Termination Filter daemon (maltfilter) v$progversion\n".
 "Programmed by Matti 'ccr' Hamalainen <ccr\@tnsp.org>\n".
@@ -24,13 +24,17 @@
 my %settings = (
   "VERBOSITY" => 3,
   "DRY_RUN" => 1,
-  "WEED_BLOCK" => 168,
-  "WEED_GLOBAL" => 336,
-  "TRESHOLD" => 3,
+  "WEED_FILTER" => 168,  # in hours
+  "WEED_GLOBAL" => 336,  # in hours
+  "THRESHOLD" => 3,
   "ACTION" => "DROP",
   "LOGFILE" => "",
   "IPTABLES" => "/sbin/iptables",
 
+  "PASSWD"              => "/etc/passwd",
+  "SYSACCT_MIN_UID"     => 1,
+  "SYSACCT_MAX_UID"     => 100,
+
   "FULL_TIME"           => 1,
   "STATUS_FILE_PLAIN"   => "",
   "STATUS_FILE_HTML"    => "",
@@ -45,12 +49,14 @@
   "CHK_SYSACCT_SSH_PWD" => 0,
   "CHK_GOOD_HOSTS"      => "",
 
-  "PASSWD"              => "/etc/passwd",
-  "SYSACCT_MIN_UID"     => 1,
-  "SYSACCT_MAX_UID"     => 100,
-
   "EVIDENCE"            => 0,
   "EVIDENCE_DIR"        => "",
+
+  "DRONEBL"             => 0,
+  "DRONEBL_THRESHOLD"   => 5,
+  "DRONEBL_MAX_AGE"     => 30, # in minutes
+  "DRONEBL_RPC_URI"     => "http://dronebl.org/RPC2",
+  "DRONEBL_RPC_KEY"     => "",
 );
 
 my @noblock_ips_def = (
@@ -58,6 +64,8 @@
 );
 
 my %systemacct = ();
+sub check_add_hit($$$$$$);
+
 
 #############################################################################
 ### Check given logfile line for matches
@@ -107,7 +115,7 @@
     if ($merr =~ /\.php\?\S*?=http:\/\/([^\/]+)/) {
       if (!check_hosts($settings{"CHK_GOOD_HOSTS"}, $1)) {
         if ($merr =~ /\.php\?\S*?=(http:\/\/[^\&\?]+\??)/) {
-          check_add_evidence($mip, $1);
+          check_add_evidence($mip, $1, $merr);
         }
         check_add_hit($mip, $mdate, "PHP XSS", $merr, $settings{"CHK_PHP_XSS"});
       }
@@ -133,6 +141,7 @@
 my $pid_file = "";       # Name of Maltfilter daemon pid file
 my @configfiles = ();    # Array of configuration file names
 my $LOGFILE;             # Maltfilter logfile handle
+my %dronebl = ();
 
 # IPs currently blocked in Netfilter $blocklist{$ip} = date
 my %blocklist = ();      
@@ -477,9 +486,9 @@
   printP($m, $f, "The hit classes marked as 'IPTABLES' are a pseudo-class meaning an\n".
   "blocked IP that was in Netfilter before Maltfilter was started.\n");
 
-  printH($m, $f, 2, "Currently blocked entries");
-  $period = get_period($settings{"WEED_BLOCK"});
-  printP($m, $f, "List of IPs that are currently blocked (or would be, if this is\n".
+  printH($m, $f, 2, "Currently filtered entries");
+  $period = get_period($settings{"WEED_FILTER"});
+  printP($m, $f, "List of IPs that are currently filtered (or would be, if this is\n".
   "a report-only mode). Data from period of $period.\n");
   print_table1($m, $f, \%statlist, \%blocklist, \&cmp_hits, "blocked");
 
@@ -499,13 +508,63 @@
 
 
 #############################################################################
+### DroneBL submission support
+#############################################################################
+sub dronebl_process
+{
+#  return if ($reportmode);
+  return unless ($settings{"DRONEBL"} > 0);
+
+  # Create submission data
+  my $xml = "<?xml version=\"1.0\"?>\n<request key=\"".$settings{"DRONEBL_RPC_KEY"}."\">\n";
+  my $entries = 0;
+  while (my ($ip, $entry) = each(%dronebl)) {
+    if ($entry->{"sent"} == 0) {
+      $xml .= "<add ip=\"".$ip."\" type=\"".$entry->{"type"}."\" />\n";
+      $entries++;
+    }
+  }
+  $xml .= "</request>\n";
+
+  mlog(1, "Trying to submit $entries entries to DroneBL.\n");
+  print STDERR $xml;
+  return;
+  
+  return unless ($entries > 0);
+    
+  # Submit via HTTP XML-RPC
+  my $tmp = LWP::UserAgent->new;
+  $tmp->agent("Maltfilter/".$progversion);
+  $tmp->timeout(10);
+  my $req = HTTP::Request->new(POST => $settings{"DRONEBL_RPC_URI"});
+  $req->content_type("text/xml");
+  $req->content($xml);
+  $req->user_agent("Maltfilter/".$progversion);
+  my $res = $tmp->request($req);
+
+  if ($res->is_success) {
+    while (my ($ip, $entry) = each(%dronebl)) {
+      $entry->{"sent"} = 1;
+    }
+  } else {
+    mlog(-1, "DroneBL submission failed: [".$res->code."] ".$res->message."\n");
+  }
+  
+  # Remove submitted expired entries
+  while (my ($ip, $entry) = each(%dronebl)) {
+    print "$ip: ".$entry->{"sent"}."\n" unless check_time3($entry->{"date"});
+  }
+}
+
+
+#############################################################################
 ### Evidence gathering
 #############################################################################
 my %evidence = ();
 
-sub check_add_evidence($$)
+sub check_add_evidence($$$)
 {
-  my ($mip, $mdata) = @_;
+  my ($mip, $mdata, $mfull) = @_;
 
   return unless ($settings{"EVIDENCE"});
 
@@ -516,6 +575,7 @@
 
   $evidence{$mdata}{"coll"} = $tmp;
   $evidence{$mdata}{"hosts"}{$mip} = 1;
+  $evidence{$mdata}{"full"}{$mfull} = 1;
 }
 
 sub http_fetch($$)
@@ -538,64 +598,69 @@
   mdie("Evidence directory '$base' has disappeared.\n") unless (-e $base);
   
   foreach my $url (keys %evidence) {
-    if (!$evidence{$url}{"done"}) {
-      my $filename = $base."/".$evidence{$url}{"coll"}.".data";
-      my $filename2 = $base."/".$evidence{$url}{"coll"}.".hosts";
-      my $code = 0;
-      my $message = "";
-
-      # Get data contents only once
-      if (! -e $filename) {
-        mlog(1, "Fetching evidence for $url\n");
-        my $res = http_fetch($url, "");
-        $code = $res->code;
-        $message = $res->message;
-        open(FILE, ">:raw", $filename) or mdie("Could not open '$filename' for writing.\n");
-        binmode(FILE, ":raw");
-        if ($res->code >= 200 && $res->code <= 201) {
-          print FILE $res->content;
-        } else {
-          print FILE "[$code] $message\n";
-        }
-        close(FILE);
-      }
+    my $did_fetch = 0;
+    my $filename = $base."/".$evidence{$url}{"coll"}.".data";
+    my $filename2 = $base."/".$evidence{$url}{"coll"}.".hosts";
+    my $filename3 = $base."/".$evidence{$url}{"coll"}.".info";
 
-      # Check if we are appending hosts to existing data
-      if (-e $filename2) {
-        open(FILE, "<", $filename2) or mdie("Could not open '$filename2' for reading.\n");
-        while (<FILE>) {
-          if (/^(\d+\.\d+\.\d+\.\d+) *\|/) {
-            if (defined($evidence{$url}{"hosts"}{$1})) {
-              delete($evidence{$url}{"hosts"}{$1});
-            }
-          }
-        }
-        close(FILE);
-        open(FILE, ">>", $filename2) or mdie("Could not open '$filename2' for appending.\n");
-      } else {
-        open(FILE, ">", $filename2) or mdie("Could not open '$filename2' for writing.\n");
-        print FILE "# HTTP request result: [$code] $message\n";
-        print FILE "# Hosts which requested $url\n\n";
-      }
-
-      foreach my $host (sort keys %{$evidence{$url}{"hosts"}}) {
-        print "lol: $host\n";
-        my $query = $dns->search($host);
-        my @names = ();
-        undef(@names);
-        if ($query) {
-          foreach my $rr ($query->answer) {
-            push(@names, $rr->{"ptrdname"}) if defined($rr->{"ptrdname"});
-          }
-        }
-        printf FILE "%-15s | %s\n", $host, join(" | ", @names);
+    # Get data contents only once
+    if (! -e $filename) {
+      $did_fetch = 1;
+      mlog(1, "Fetching evidence for $url\n");
+      my $res = http_fetch($url, "");
+      open(FILE, ">:raw", $filename) or mdie("Could not open '$filename' for writing.\n");
+      binmode(FILE, ":raw");
+      if ($res->is_success && $res->code >= 200 && $res->code <= 201) {
+        print FILE $res->content;
       }
       close(FILE);
 
-      delete($evidence{$url});
+      open(FILE, ">:raw", $filename3) or mdie("Could not open '$filename3' for writing.\n");
+      binmode(FILE, ":raw");
+      print FILE "XSS URI           : $url\n";
+      print FILE "Time of retrieval : ".get_time_str(time())."\n";
+      print FILE "HTTP return code  : [".$res->code."] ".$res->message."\n";
+      print FILE "Content-Type      : ".($res->content_type ? $res->content_type : "?")."\n";
+      print FILE "Last modified     : ".($res->last_modified ? $res->last_modified : "?")."\n";
+      print FILE "------ HTTP Headers ------\n".$res->headers_as_string."\n";
+      print FILE "------ Requests ------\n";
+      print FILE $_."\n" foreach (keys %{$evidence{$url}{"full"}});
+      close(FILE);
+    }
 
-      return unless $reportmode;
+    # Check if we are appending hosts to existing data
+    if (-e $filename2) {
+      open(FILE, "<", $filename2) or mdie("Could not open '$filename2' for reading.\n");
+      while (<FILE>) {
+        if (/^(\d+\.\d+\.\d+\.\d+) *\|/) {
+          if (defined($evidence{$url}{"hosts"}{$1})) {
+            delete($evidence{$url}{"hosts"}{$1});
+          }
+        }
+      }
+      close(FILE);
+      open(FILE, ">>", $filename2) or mdie("Could not open '$filename2' for appending.\n");
+    } else {
+      open(FILE, ">", $filename2) or mdie("Could not open '$filename2' for writing.\n");
     }
+    foreach my $host (sort keys %{$evidence{$url}{"hosts"}}) {
+      my $query = $dns->search($host);
+      my @names = ();
+      undef(@names);
+      if ($query) {
+        foreach my $rr ($query->answer) {
+          push(@names, $rr->{"ptrdname"}) if defined($rr->{"ptrdname"});
+        }
+      }
+      printf FILE "%-15s | %s\n", $host, join(" | ", @names);
+    }
+    close(FILE);
+
+    # This entry has been handled, delete it
+    delete($evidence{$url});
+
+    # If not in report mode, handle only one fetched entry
+    return unless ($reportmode || !$did_fetch);
   }
 }
 
@@ -679,12 +744,17 @@
 ### Returns false if timestamp is over weed period, e.g. needs weeding.
 sub check_time1($)
 {
-  return ($_[0] >= time() - ($settings{"WEED_BLOCK"} * 60 * 60));
+  return ($_[0] > time() - ($settings{"WEED_FILTER"} * 60 * 60));
 }
 
 sub check_time2($)
 {
-  return ($_[0] >= time() - ($settings{"WEED_GLOBAL"} * 60 * 60));
+  return ($_[0] > time() - ($settings{"WEED_GLOBAL"} * 60 * 60));
+}
+
+sub check_time3($)
+{
+  return ($_[0] > time() - ($settings{"DRONEBL_MAX_AGE"} * 60));
 }
 
 ### Weed out old entries
@@ -780,15 +850,16 @@
   return $entry->{"hits"};
 }
 
-### Check if given "try count" exceeds treshold and if entry
+### Check if given "try count" exceeds threshold and if entry
 ### is NOT in Netfilter already, then add it if so.
-sub check_add_hit($$$$$)
+sub check_add_hit($$$$$$)
 {
   my $mip = $_[0];
   my $mdate = str2time($_[1]);
   my $mclass = $_[2];
   my $mreason = $_[3];
-  my $mcond = $_[4];
+  my $mtype = $_[4];
+  my $mcond = $_[5];
   my $cnt;
 
   if (check_hosts_array(\@noblock_ips, $mip)) {
@@ -805,8 +876,8 @@
     return;
   }
 
-  # Check if we have exceeded treshold etc.
-  if ($cnt >= $settings{"TRESHOLD"} && check_time1($mdate)) {
+  # Check if we have exceeded threshold etc.
+  if ($cnt >= $settings{"THRESHOLD"} && check_time1($mdate)) {
     # Add to blocklist, unless already there.
     if (!defined($blocklist{$mip})) {
       mlog(1, "* Adding $mip ($mdate): [$mclass] $mreason\n");
@@ -815,6 +886,13 @@
     # Update date of last hit
     $blocklist{$mip} = $mdate;
   }
+
+  # Separate check for DroneBL
+  if ($settings{"DRONEBL"} > 0 && $mtype > 0 && $cnt >= $settings{"DRONEBL_THRESHOLD"} && check_time3($mdate)) {
+    $dronebl{$mip}{"type"} = $mtype;
+    $dronebl{$mip}{"date"} = $mdate;
+    $dronebl{$mip}{"sent"} = 0 unless defined($dronebl{$mip}{"sent"});
+  }
 }