Mercurial > hg > maltfilter

comparison maltfilter @ 70:adb4795f451e maltfilter-0.17.0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Finished DroneBL support (hopefully).
author Matti Hamalainen <ccr@tnsp.org>
date Thu, 27 Aug 2009 00:17:42 +0300
parents b090ddfccdab
children e8fbe7cd65a7
comparison
equal deleted inserted replaced
69:b090ddfccdab 70:adb4795f451e
10 use Date::Parse; 10 use Date::Parse;
11 use Net::IP; 11 use Net::IP;
12 use Net::DNS; 12 use Net::DNS;
13 use LWP::UserAgent; 13 use LWP::UserAgent;
14 14
15 my $progversion = "0.16.2"; 15 my $progversion = "0.17.0";
16 my $progbanner = 16 my $progbanner =
17 "Malicious Attack Livid Termination Filter daemon (maltfilter) v$progversion\n". 17 "Malicious Attack Livid Termination Filter daemon (maltfilter) v$progversion\n".
18 "Programmed by Matti 'ccr' Hamalainen <ccr\@tnsp.org>\n". 18 "Programmed by Matti 'ccr' Hamalainen <ccr\@tnsp.org>\n".
19 "(C) Copyright 2009 Tecnic Software productions (TNSP)\n"; 19 "(C) Copyright 2009 Tecnic Software productions (TNSP)\n";
20 20
158 # Gathered information about hosts 158 # Gathered information about hosts
159 # $statlist{$ip}-> 159 # $statlist{$ip}->
160 # "date1" = timestamp of first hit 160 # "date1" = timestamp of first hit
161 # "date2" = timestamp of latest hit 161 # "date2" = timestamp of latest hit
162 # "hits" = number of hits to this IP 162 # "hits" = number of hits to this IP
163 # "dronebl" = 1 == queued for submission, 2 == submitted 163 # "dronebl" = 0 == n/a, 1 == queued for submission, 2 == submitted
164 # $statlist{$ip}{"reason"}{$class}-> 164 # $statlist{$ip}{"reason"}{$class}->
165 # "msg" = reason message (array if $reportmode) 165 # "msg" = reason message (array if $reportmode)
166 # "hits" = hits to this class 166 # "hits" = hits to this class
167 # "date1" = timestamp of first hit 167 # "date1" = timestamp of first hit
168 # "date2" = timestamp of latest hit 168 # "date2" = timestamp of latest hit
380 380
381 sub print_table2($$$$$$) 381 sub print_table2($$$$$$)
382 { 382 {
383 my ($m, $f, $table, $keys, $func, $class) = @_; 383 my ($m, $f, $table, $keys, $func, $class) = @_;
384 my $nhits = 0; 384 my $nhits = 0;
385 my $str = "<th>IP-address</th><th>Hits</th><th>First hit</th><th>Latest hit</th><th>Class</th>"; 385 my $str = "<th>IP-address</th><th>Hits</th><th>DroneBL?</th><th>First hit</th><th>Latest hit</th><th>Class</th>";
386 my $str2 = "IP-address | Hits | First hit | Latest hit | Class "; 386 my $str2 = "IP-address | Hits | DroneBL | First hit | Latest hit | Class ";
387 387
388 printElem($m, $f, 388 printElem($m, $f,
389 "<table class=\"".$class."\">\n<tr>". $str."<th> </th>".$str ."</tr>\n", 389 "<table class=\"".$class."\">\n<tr>". $str."<th> </th>".$str ."</tr>\n",
390 $str2." || ".$str2."\n"); 390 $str2." || ".$str2."\n");
391 391
401 my $str = "style=\"background: ".$ipcolors[$ncolor[$_[1]] % scalar @ipcolors].";\""; 401 my $str = "style=\"background: ".$ipcolors[$ncolor[$_[1]] % scalar @ipcolors].";\"";
402 402
403 printTD($m, $f, sprintf("%-15s", get_link($m, $_[0])), $str); 403 printTD($m, $f, sprintf("%-15s", get_link($m, $_[0])), $str);
404 printElem(!$m, $f, " | "); 404 printElem(!$m, $f, " | ");
405 printTD($m, $f, sprintf("%-8d ", $table->{$_[0]}{"hits"}), $blocked); 405 printTD($m, $f, sprintf("%-8d ", $table->{$_[0]}{"hits"}), $blocked);
406 printElem(!$m, $f, " | ");
407 printTD($m, $f, sprintf("%-6s ", $table->{$_[0]}{"dronebl"}), $blocked);
406 printElem(!$m, $f, " | "); 408 printElem(!$m, $f, " | ");
407 printTD($m, $f, get_ago_str($table->{$_[0]}{"date1"}), $blocked); 409 printTD($m, $f, get_ago_str($table->{$_[0]}{"date1"}), $blocked);
408 printElem(!$m, $f, " | "); 410 printElem(!$m, $f, " | ");
409 printTD($m, $f, get_ago_str($table->{$_[0]}{"date2"}), $blocked); 411 printTD($m, $f, get_ago_str($table->{$_[0]}{"date2"}), $blocked);
410 printElem(!$m, $f, " | "); 412 printElem(!$m, $f, " | ");
529 # Create submission data 531 # Create submission data
530 my $xml = "<?xml version=\"1.0\"?>\n<request key=\"".$settings{"DRONEBL_RPC_KEY"}."\">\n"; 532 my $xml = "<?xml version=\"1.0\"?>\n<request key=\"".$settings{"DRONEBL_RPC_KEY"}."\">\n";
531 my $entries = 0; 533 my $entries = 0;
532 while (my ($ip, $entry) = each(%dronebl)) { 534 while (my ($ip, $entry) = each(%dronebl)) {
533 if ($entry->{"sent"} == 0 && $entry->{"tries"} < 3) { 535 if ($entry->{"sent"} == 0 && $entry->{"tries"} < 3) {
534 # $xml .= "<add ip=\"".$ip."\" type=\"".$entry->{"type"}."\" />\n"; 536 $xml .= "<add ip=\"".$ip."\" type=\"".$entry->{"type"}."\" />\n";
535 $xml .= "<add ip=\"".$ip."\" type=\"1\" />\n"; 537 # $xml .= "<add ip=\"".$ip."\" type=\"1\" />\n";
536 $entries++; 538 $entries++;
537 } 539 }
538 } 540 }
539 $xml .= "</request>\n"; 541 $xml .= "</request>\n";
540 542
541 # Bait out if no entries to submit 543 # Bait out if no entries to submit
542 return unless ($entries > 0); 544 return unless ($entries > 0);
543 if ($settings{"DRY_RUN"}) { 545 if ($settings{"DRY_RUN"}) {
544 mlog(1, "[DroneBL] Would submit $entries entries.\n"); 546 mlog(2, "[DroneBL] Would submit $entries entries.\n");
545 # return; 547 return;
546 } else { 548 } else {
547 mlog(1, "[DroneBL] Trying to submit $entries entries.\n"); 549 mlog(2, "[DroneBL] Trying to submit $entries entries.\n");
548 } 550 }
549 551
550 if (0) {
551 # Submit via HTTP XML-RPC 552 # Submit via HTTP XML-RPC
552 my $tmp = LWP::UserAgent->new; 553 my $tmp = LWP::UserAgent->new;
553 $tmp->agent("Maltfilter/".$progversion); 554 $tmp->agent("Maltfilter/".$progversion);
554 $tmp->timeout(10); 555 $tmp->timeout(10);
555 my $req = HTTP::Request->new(POST => $settings{"DRONEBL_RPC_URI"}); 556 my $req = HTTP::Request->new(POST => $settings{"DRONEBL_RPC_URI"});
557 $req->content($xml); 558 $req->content($xml);
558 $req->user_agent("Maltfilter/".$progversion); 559 $req->user_agent("Maltfilter/".$progversion);
559 my $res = $tmp->request($req); 560 my $res = $tmp->request($req);
560 561
561 if ($res->is_success) { 562 if ($res->is_success) {
562 mlog(3, "[DroneBL] [".$res->code."] ".$res->message."\n"); 563 mlog(3, "[DroneBL] HTTP response [".$res->code."] ".$res->message."\n");
563 print $res->content."\n"; 564 my $str = $res->content;
565 my ($type, $msg);
566 $str =~ tr/\n/ /;
567
568 if ($str =~ /<response\s*type=.(success|error).>(.*?)<\/response>/gm) {
569 $type = $1; $msg = $2;
570 } elsif ($str =~ /<response\s*type=.(success|error). *\/>/gm) {
571 $type = $1; $msg = "";
572 }
564 573
565 574 if ($type eq "success") {
566 # while (my ($ip, $entry) = each(%dronebl)) { 575 mlog(1, "[DroneBL] Succesfully submitted $entries entries.\n$msg\n");
567 # $entry->{"sent"} = 1; 576 while (my ($ip, $entry) = each(%dronebl)) {
568 # $statlist{$ip}{"dronebl"} = 2 if (defined($statlist{$ip})); 577 $entry->{"sent"} = 1;
569 # } 578 $statlist{$ip}{"dronebl"} = 2 if defined($statlist{$ip});
570 } else { 579 }
571 mlog(-1, "[DroneBL] Submission failed: [".$res->code."] ".$res->message."\n"); 580 } elsif ($type eq "error") {
572 } 581 # If we don't have a valid key, disable further submissions.
573 } 582 if ($msg =~ /<code>403<\/code>/) {
574 583 mlog(-1, "Disabling DroneBL submission due to invalid key.\n");
575 # Clean up expired entries, warning about unsubmitted ones. 584 $settings{"DRONEBL"} = 0;
585 }
586 # Log error message mangled
587 $msg =~ s{\s*</?[^>]+>}{ }g;
588 mlog(-1, "[DroneBL] Error in submission: $msg\n");
589 } else {
590 mlog(-1, "[DroneBL] Unsupported response message ".$str."\n");
591 }
592 } else {
593 mlog(-1, "[DroneBL] HTTP request failed: [".$res->code."] ".$res->message."\n");
594 }
595
596 # Clean up expired entries, warn/note about unsubmitted ones.
576 while (my ($ip, $entry) = each(%dronebl)) { 597 while (my ($ip, $entry) = each(%dronebl)) {
577 if (!check_time3($entry->{"date"})) { 598 if (!check_time3($entry->{"date"})) {
578 mlog(1, "[DroneBL] $ip submission expired.\n") unless ($entry->{"sent"} > 0); 599 mlog(1, "[DroneBL] $ip submission expired.\n") unless ($entry->{"sent"} > 0);
579 delete($dronebl{$ip}); 600 delete($dronebl{$ip});
580 } 601 }
592 mlog(3, "[DroneBL] Queueing $mip \@ $mdate ($mtype)\n"); 613 mlog(3, "[DroneBL] Queueing $mip \@ $mdate ($mtype)\n");
593 $dronebl{$mip}{"type"} = $mtype; 614 $dronebl{$mip}{"type"} = $mtype;
594 $dronebl{$mip}{"date"} = $mdate; 615 $dronebl{$mip}{"date"} = $mdate;
595 $dronebl{$mip}{"sent"} = 0; 616 $dronebl{$mip}{"sent"} = 0;
596 $dronebl{$mip}{"tries"} = 0; 617 $dronebl{$mip}{"tries"} = 0;
597 $statlist{$mip}{"dronebl"} = 1 if (defined($statlist{$mip})); 618 $statlist{$mip}{"dronebl"} = 1 if defined($statlist{$mip});
598 } 619 }
599 } 620 }
600 621
601 ############################################################################# 622 #############################################################################
602 ### Evidence gathering 623 ### Evidence gathering
627 $tmp->default_headers->referer($_[1]); 648 $tmp->default_headers->referer($_[1]);
628 my $req = HTTP::Request->new(GET => $_[0]); 649 my $req = HTTP::Request->new(GET => $_[0]);
629 return $tmp->request($req); 650 return $tmp->request($req);
630 } 651 }
631 652
653 my $evidence_dir = 0;
632 sub evidence_gather 654 sub evidence_gather
633 { 655 {
634 my $dns = Net::DNS::Resolver->new; 656 my $dns = Net::DNS::Resolver->new;
635 my $base = $settings{"EVIDENCE_DIR"}; 657 my $base = $settings{"EVIDENCE_DIR"};
636 658
637 return unless ($settings{"EVIDENCE"} > 0); 659 return unless ($settings{"EVIDENCE"} > 0);
638 660
639 mdie("Evidence directory '$base' has disappeared.\n") unless (-e $base); 661 if (! -e $base) {
640 662 mlog(-1, "Evidence directory '$base' has disappeared.\n") unless ($evidence_dir > 0);
663 mdie("Evidence directory '$base' has been absent for $evidence_dir cycles, dying.\n") if ($evidence_dir++ > 10);
664 return;
665 } else {
666 $evidence_dir = 0;
667 }
668
669 my $fetched = 0;
641 foreach my $url (keys %evidence) { 670 foreach my $url (keys %evidence) {
642 my $did_fetch = 0;
643 my $filename = $base."/".$evidence{$url}{"coll"}.".data"; 671 my $filename = $base."/".$evidence{$url}{"coll"}.".data";
644 my $filename2 = $base."/".$evidence{$url}{"coll"}.".hosts"; 672 my $filename2 = $base."/".$evidence{$url}{"coll"}.".hosts";
645 my $filename3 = $base."/".$evidence{$url}{"coll"}.".info"; 673 my $filename3 = $base."/".$evidence{$url}{"coll"}.".info";
646 674
647 # Get data contents only once 675 # Get data contents only once
648 if (! -e $filename) { 676 if (! -e $filename) {
649 $did_fetch = 1; 677 $fetched++;
650 mlog(1, "Fetching evidence for $url\n"); 678 mlog(1, "Fetching evidence for $url\n");
651 my $res = evidence_fetch($url, ""); 679 my $res = evidence_fetch($url, "");
652 open(FILE, ">:raw", $filename) or mdie("Could not open '$filename' for writing.\n"); 680 open(FILE, ">:raw", $filename) or mdie("Could not open '$filename' for writing.\n");
653 binmode(FILE, ":raw"); 681 binmode(FILE, ":raw");
654 if ($res->is_success && $res->code >= 200 && $res->code <= 201) { 682 if ($res->is_success && $res->code >= 200 && $res->code <= 201) {
698 close(FILE); 726 close(FILE);
699 727
700 # This entry has been handled, delete it 728 # This entry has been handled, delete it
701 delete($evidence{$url}); 729 delete($evidence{$url});
702 730
703 # If not in report mode, handle only one fetched entry 731 # If not in report mode, handle only 5 fetched entries at time
704 return unless ($reportmode || !$did_fetch); 732 return unless ($reportmode || $fetched < 5);
705 } 733 }
706 } 734 }
707 735
708 736
709 ############################################################################# 737 #############################################################################
871 $struct->{$mip} = {} unless defined($struct->{$mip}); 899 $struct->{$mip} = {} unless defined($struct->{$mip});
872 my $entry = $struct->{$mip}; 900 my $entry = $struct->{$mip};
873 $entry->{"reason"}{$mclass} = {} unless defined($entry->{"reason"}{$mclass}); 901 $entry->{"reason"}{$mclass} = {} unless defined($entry->{"reason"}{$mclass});
874 my $reason = $entry->{"reason"}{$mclass}; 902 my $reason = $entry->{"reason"}{$mclass};
875 903
904 $entry->{"dronebl"} = 0 unless defined($entry->{"dronebl"});
905
876 # Add hits only when requested 906 # Add hits only when requested
877 if ($addhits) { 907 if ($addhits) {
878 $entry->{"hits"}++; 908 $entry->{"hits"}++;
879 $reason->{"hits"}++; 909 $reason->{"hits"}++;
880 } else { 910 } else {
1072 if ($counter < 0 || $counter++ >= 30) { 1102 if ($counter < 0 || $counter++ >= 30) {
1073 # Every once in a while, execute maintenance functions 1103 # Every once in a while, execute maintenance functions
1074 $counter = 0; 1104 $counter = 0;
1075 malt_maintenance(); 1105 malt_maintenance();
1076 } 1106 }
1077 sleep(2); 1107 sleep(1);
1078 foreach my $filename (keys %filehandles) { 1108 foreach my $filename (keys %filehandles) {
1079 seek($filehandles{$filename}, $filepos{$filename}, 0); 1109 seek($filehandles{$filename}, $filepos{$filename}, 0);
1080 } 1110 }
1081 } 1111 }
1082 } 1112 }