# HG changeset patch
# User Matti Hamalainen <ccr@tnsp.org>
# Date 1251321462 -10800
# Node ID adb4795f451e1406321c4359985e2acdea8c6aa9
# Parent  b090ddfccdab1d8a8983dd977db2c3db59042494
Finished DroneBL support (hopefully).

diff -r b090ddfccdab -r adb4795f451e README
--- a/README	Wed Aug 26 15:19:08 2009 +0300
+++ b/README	Thu Aug 27 00:17:42 2009 +0300
@@ -1,4 +1,4 @@
-Malicious Attack Livid Termination Filter daemon (maltfilter) v0.16.2
+Malicious Attack Livid Termination Filter daemon (maltfilter) v0.17.0
 =====================================================================
 Programmed by Matti 'ccr' Hämäläinen <ccr@tnsp.org>
 (C) Copyright 2009 Tecnic Software productions (TNSP)
diff -r b090ddfccdab -r adb4795f451e maltfilter
--- a/maltfilter	Wed Aug 26 15:19:08 2009 +0300
+++ b/maltfilter	Thu Aug 27 00:17:42 2009 +0300
@@ -12,7 +12,7 @@
 use Net::DNS;
 use LWP::UserAgent;
 
-my $progversion = "0.16.2";
+my $progversion = "0.17.0";
 my $progbanner =
 "Malicious Attack Livid Termination Filter daemon (maltfilter) v$progversion\n".
 "Programmed by Matti 'ccr' Hamalainen <ccr\@tnsp.org>\n".
@@ -160,7 +160,7 @@
 # "date1"    = timestamp of first hit
 # "date2"    = timestamp of latest hit
 # "hits"     = number of hits to this IP
-# "dronebl"  = 1 == queued for submission, 2 == submitted
+# "dronebl"  = 0 == n/a, 1 == queued for submission, 2 == submitted
 # $statlist{$ip}{"reason"}{$class}->
 #   "msg"    = reason message (array if $reportmode)
 #   "hits"   = hits to this class
@@ -382,8 +382,8 @@
 {
   my ($m, $f, $table, $keys, $func, $class) = @_;
   my $nhits = 0;
-  my $str = "<th>IP-address</th><th>Hits</th><th>First hit</th><th>Latest hit</th><th>Class</th>";
-  my $str2 = "IP-address      | Hits      | First hit                | Latest hit               | Class                         ";
+  my $str = "<th>IP-address</th><th>Hits</th><th>DroneBL?</th><th>First hit</th><th>Latest hit</th><th>Class</th>";
+  my $str2 = "IP-address      | Hits      | DroneBL | First hit                | Latest hit               | Class                         ";
 
   printElem($m, $f,
   "<table class=\"".$class."\">\n<tr>". $str."<th> </th>".$str ."</tr>\n",
@@ -404,6 +404,8 @@
     printElem(!$m, $f, " | ");
     printTD($m, $f, sprintf("%-8d ", $table->{$_[0]}{"hits"}), $blocked);
     printElem(!$m, $f, " | ");
+    printTD($m, $f, sprintf("%-6s ", $table->{$_[0]}{"dronebl"}), $blocked);
+    printElem(!$m, $f, " | ");
     printTD($m, $f, get_ago_str($table->{$_[0]}{"date1"}), $blocked);
     printElem(!$m, $f, " | ");
     printTD($m, $f, get_ago_str($table->{$_[0]}{"date2"}), $blocked);
@@ -531,8 +533,8 @@
   my $entries = 0;
   while (my ($ip, $entry) = each(%dronebl)) {
     if ($entry->{"sent"} == 0 && $entry->{"tries"} < 3) {
-#      $xml .= "<add ip=\"".$ip."\" type=\"".$entry->{"type"}."\" />\n";
-      $xml .= "<add ip=\"".$ip."\" type=\"1\" />\n";
+      $xml .= "<add ip=\"".$ip."\" type=\"".$entry->{"type"}."\" />\n";
+#      $xml .= "<add ip=\"".$ip."\" type=\"1\" />\n";
       $entries++;
     }
   }
@@ -541,13 +543,12 @@
   # Bait out if no entries to submit
   return unless ($entries > 0);
   if ($settings{"DRY_RUN"}) {
-    mlog(1, "[DroneBL] Would submit $entries entries.\n");
-#    return;
+    mlog(2, "[DroneBL] Would submit $entries entries.\n");
+    return;
   } else {
-    mlog(1, "[DroneBL] Trying to submit $entries entries.\n");
+    mlog(2, "[DroneBL] Trying to submit $entries entries.\n");
   }
 
-if (0) {
   # Submit via HTTP XML-RPC
   my $tmp = LWP::UserAgent->new;
   $tmp->agent("Maltfilter/".$progversion);
@@ -559,20 +560,40 @@
   my $res = $tmp->request($req);
 
   if ($res->is_success) {
-    mlog(3, "[DroneBL] [".$res->code."] ".$res->message."\n");
-    print $res->content."\n";
-    
+    mlog(3, "[DroneBL] HTTP response [".$res->code."] ".$res->message."\n");
+    my $str = $res->content;
+    my ($type, $msg);
+    $str =~ tr/\n/ /;
 
-#    while (my ($ip, $entry) = each(%dronebl)) {
-#      $entry->{"sent"} = 1;
-#      $statlist{$ip}{"dronebl"} = 2 if (defined($statlist{$ip}));
-#    }
+    if ($str =~ /<response\s*type=.(success|error).>(.*?)<\/response>/gm) {
+      $type = $1; $msg = $2;
+    } elsif ($str =~ /<response\s*type=.(success|error). *\/>/gm) {
+      $type = $1; $msg = "";
+    }
+    
+    if ($type eq "success") {
+        mlog(1, "[DroneBL] Succesfully submitted $entries entries.\n$msg\n");
+        while (my ($ip, $entry) = each(%dronebl)) {
+          $entry->{"sent"} = 1;
+          $statlist{$ip}{"dronebl"} = 2 if defined($statlist{$ip});
+        }
+    } elsif ($type eq "error") {
+      # If we don't have a valid key, disable further submissions.
+      if ($msg =~ /<code>403<\/code>/) {
+        mlog(-1, "Disabling DroneBL submission due to invalid key.\n");
+        $settings{"DRONEBL"} = 0;
+      }
+      # Log error message mangled
+      $msg =~ s{\s*</?[^>]+>}{ }g;
+      mlog(-1, "[DroneBL] Error in submission: $msg\n");
+    } else {
+      mlog(-1, "[DroneBL] Unsupported response message ".$str."\n");
+    }
   } else {
-    mlog(-1, "[DroneBL] Submission failed: [".$res->code."] ".$res->message."\n");
+    mlog(-1, "[DroneBL] HTTP request failed: [".$res->code."] ".$res->message."\n");
   }
-}
   
-  # Clean up expired entries, warning about unsubmitted ones.
+  # Clean up expired entries, warn/note about unsubmitted ones.
   while (my ($ip, $entry) = each(%dronebl)) {
     if (!check_time3($entry->{"date"})) {
       mlog(1, "[DroneBL] $ip submission expired.\n") unless ($entry->{"sent"} > 0);
@@ -594,7 +615,7 @@
     $dronebl{$mip}{"date"} = $mdate;
     $dronebl{$mip}{"sent"} = 0;
     $dronebl{$mip}{"tries"} = 0;
-    $statlist{$mip}{"dronebl"} = 1 if (defined($statlist{$mip}));
+    $statlist{$mip}{"dronebl"} = 1 if defined($statlist{$mip});
   }
 }
 
@@ -629,6 +650,7 @@
   return $tmp->request($req);
 }
 
+my $evidence_dir = 0;
 sub evidence_gather
 {
   my $dns = Net::DNS::Resolver->new;
@@ -636,17 +658,23 @@
 
   return unless ($settings{"EVIDENCE"} > 0);
 
-  mdie("Evidence directory '$base' has disappeared.\n") unless (-e $base);
+  if (! -e $base) {
+    mlog(-1, "Evidence directory '$base' has disappeared.\n") unless ($evidence_dir > 0);
+    mdie("Evidence directory '$base' has been absent for $evidence_dir cycles, dying.\n") if ($evidence_dir++ > 10);
+    return;
+  } else {
+    $evidence_dir = 0;
+  }
   
+  my $fetched = 0;
   foreach my $url (keys %evidence) {
-    my $did_fetch = 0;
     my $filename = $base."/".$evidence{$url}{"coll"}.".data";
     my $filename2 = $base."/".$evidence{$url}{"coll"}.".hosts";
     my $filename3 = $base."/".$evidence{$url}{"coll"}.".info";
 
     # Get data contents only once
     if (! -e $filename) {
-      $did_fetch = 1;
+      $fetched++;
       mlog(1, "Fetching evidence for $url\n");
       my $res = evidence_fetch($url, "");
       open(FILE, ">:raw", $filename) or mdie("Could not open '$filename' for writing.\n");
@@ -700,8 +728,8 @@
     # This entry has been handled, delete it
     delete($evidence{$url});
 
-    # If not in report mode, handle only one fetched entry
-    return unless ($reportmode || !$did_fetch);
+    # If not in report mode, handle only 5 fetched entries at time
+    return unless ($reportmode || $fetched < 5);
   }
 }
 
@@ -873,6 +901,8 @@
   $entry->{"reason"}{$mclass} = {} unless defined($entry->{"reason"}{$mclass});
   my $reason = $entry->{"reason"}{$mclass};
 
+  $entry->{"dronebl"} = 0 unless defined($entry->{"dronebl"});
+  
   # Add hits only when requested  
   if ($addhits) {
     $entry->{"hits"}++;
@@ -1074,7 +1104,7 @@
       $counter = 0;
       malt_maintenance();
     }
-    sleep(2);
+    sleep(1);
     foreach my $filename (keys %filehandles) {
       seek($filehandles{$filename}, $filepos{$filename}, 0);
     }