Mercurial > hg > maltfilter

comparison maltfilter @ 65:d2e2b82dd2f2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Work on DroneBL support.
author Matti Hamalainen <ccr@tnsp.org>
date Tue, 18 Aug 2009 00:43:10 +0300
parents 6917de5b91be
children 42889eed0ce8
comparison
equal deleted inserted replaced
64:213e5204abea 65:d2e2b82dd2f2
10 use Date::Parse; 10 use Date::Parse;
11 use Net::IP; 11 use Net::IP;
12 use Net::DNS; 12 use Net::DNS;
13 use LWP::UserAgent; 13 use LWP::UserAgent;
14 14
15 my $progversion = "0.14.0"; 15 my $progversion = "0.15.0";
16 my $progbanner = 16 my $progbanner =
17 "Malicious Attack Livid Termination Filter daemon (maltfilter) v$progversion\n". 17 "Malicious Attack Livid Termination Filter daemon (maltfilter) v$progversion\n".
18 "Programmed by Matti 'ccr' Hamalainen <ccr\@tnsp.org>\n". 18 "Programmed by Matti 'ccr' Hamalainen <ccr\@tnsp.org>\n".
19 "(C) Copyright 2009 Tecnic Software productions (TNSP)\n"; 19 "(C) Copyright 2009 Tecnic Software productions (TNSP)\n";
20 20
22 ### Default settings and configuration 22 ### Default settings and configuration
23 ############################################################################# 23 #############################################################################
24 my %settings = ( 24 my %settings = (
25 "VERBOSITY" => 3, 25 "VERBOSITY" => 3,
26 "DRY_RUN" => 1, 26 "DRY_RUN" => 1,
27 "WEED_BLOCK" => 168, 27 "WEED_FILTER" => 168, # in hours
28 "WEED_GLOBAL" => 336, 28 "WEED_GLOBAL" => 336, # in hours
29 "TRESHOLD" => 3, 29 "THRESHOLD" => 3,
30 "ACTION" => "DROP", 30 "ACTION" => "DROP",
31 "LOGFILE" => "", 31 "LOGFILE" => "",
32 "IPTABLES" => "/sbin/iptables", 32 "IPTABLES" => "/sbin/iptables",
33
34 "PASSWD" => "/etc/passwd",
35 "SYSACCT_MIN_UID" => 1,
36 "SYSACCT_MAX_UID" => 100,
33 37
34 "FULL_TIME" => 1, 38 "FULL_TIME" => 1,
35 "STATUS_FILE_PLAIN" => "", 39 "STATUS_FILE_PLAIN" => "",
36 "STATUS_FILE_HTML" => "", 40 "STATUS_FILE_HTML" => "",
37 "STATUS_FILE_CSS" => "", 41 "STATUS_FILE_CSS" => "",
43 "CHK_PROXY_SCAN" => 1, 47 "CHK_PROXY_SCAN" => 1,
44 "CHK_ROOT_SSH_PWD" => 0, 48 "CHK_ROOT_SSH_PWD" => 0,
45 "CHK_SYSACCT_SSH_PWD" => 0, 49 "CHK_SYSACCT_SSH_PWD" => 0,
46 "CHK_GOOD_HOSTS" => "", 50 "CHK_GOOD_HOSTS" => "",
47 51
48 "PASSWD" => "/etc/passwd",
49 "SYSACCT_MIN_UID" => 1,
50 "SYSACCT_MAX_UID" => 100,
51
52 "EVIDENCE" => 0, 52 "EVIDENCE" => 0,
53 "EVIDENCE_DIR" => "", 53 "EVIDENCE_DIR" => "",
54
55 "DRONEBL" => 0,
56 "DRONEBL_THRESHOLD" => 5,
57 "DRONEBL_MAX_AGE" => 30, # in minutes
58 "DRONEBL_RPC_URI" => "http://dronebl.org/RPC2",
59 "DRONEBL_RPC_KEY" => "",
54 ); 60 );
55 61
56 my @noblock_ips_def = ( 62 my @noblock_ips_def = (
57 "127.0.0.1", 63 "127.0.0.1",
58 ); 64 );
59 65
60 my %systemacct = (); 66 my %systemacct = ();
67 sub check_add_hit($$$$$$);
68
61 69
62 ############################################################################# 70 #############################################################################
63 ### Check given logfile line for matches 71 ### Check given logfile line for matches
64 ############################################################################# 72 #############################################################################
65 sub check_log_line($) 73 sub check_log_line($)
105 113
106 # (3.1) Simple match for generic PHP XSS vulnerability scans 114 # (3.1) Simple match for generic PHP XSS vulnerability scans
107 if ($merr =~ /\.php\?\S*?=http:\/\/([^\/]+)/) { 115 if ($merr =~ /\.php\?\S*?=http:\/\/([^\/]+)/) {
108 if (!check_hosts($settings{"CHK_GOOD_HOSTS"}, $1)) { 116 if (!check_hosts($settings{"CHK_GOOD_HOSTS"}, $1)) {
109 if ($merr =~ /\.php\?\S*?=(http:\/\/[^\&\?]+\??)/) { 117 if ($merr =~ /\.php\?\S*?=(http:\/\/[^\&\?]+\??)/) {
110 check_add_evidence($mip, $1); 118 check_add_evidence($mip, $1, $merr);
111 } 119 }
112 check_add_hit($mip, $mdate, "PHP XSS", $merr, $settings{"CHK_PHP_XSS"}); 120 check_add_hit($mip, $mdate, "PHP XSS", $merr, $settings{"CHK_PHP_XSS"});
113 } 121 }
114 } 122 }
115 # (3.2) Try to match proxy scanning attempts 123 # (3.2) Try to match proxy scanning attempts
131 my @noblock_ips = (); # IPs not to block 139 my @noblock_ips = (); # IPs not to block
132 my %filehandles = (); # Global hash holding opened scanned log filehandles 140 my %filehandles = (); # Global hash holding opened scanned log filehandles
133 my $pid_file = ""; # Name of Maltfilter daemon pid file 141 my $pid_file = ""; # Name of Maltfilter daemon pid file
134 my @configfiles = (); # Array of configuration file names 142 my @configfiles = (); # Array of configuration file names
135 my $LOGFILE; # Maltfilter logfile handle 143 my $LOGFILE; # Maltfilter logfile handle
144 my %dronebl = ();
136 145
137 # IPs currently blocked in Netfilter $blocklist{$ip} = date 146 # IPs currently blocked in Netfilter $blocklist{$ip} = date
138 my %blocklist = (); 147 my %blocklist = ();
139 148
140 # Gathered information about hosts 149 # Gathered information about hosts
475 ($reportmode ? "complete logfile scan" : "a period of last $period").".\n"); 484 ($reportmode ? "complete logfile scan" : "a period of last $period").".\n");
476 485
477 printP($m, $f, "The hit classes marked as 'IPTABLES' are a pseudo-class meaning an\n". 486 printP($m, $f, "The hit classes marked as 'IPTABLES' are a pseudo-class meaning an\n".
478 "blocked IP that was in Netfilter before Maltfilter was started.\n"); 487 "blocked IP that was in Netfilter before Maltfilter was started.\n");
479 488
480 printH($m, $f, 2, "Currently blocked entries"); 489 printH($m, $f, 2, "Currently filtered entries");
481 $period = get_period($settings{"WEED_BLOCK"}); 490 $period = get_period($settings{"WEED_FILTER"});
482 printP($m, $f, "List of IPs that are currently blocked (or would be, if this is\n". 491 printP($m, $f, "List of IPs that are currently filtered (or would be, if this is\n".
483 "a report-only mode). Data from period of $period.\n"); 492 "a report-only mode). Data from period of $period.\n");
484 print_table1($m, $f, \%statlist, \%blocklist, \&cmp_hits, "blocked"); 493 print_table1($m, $f, \%statlist, \%blocklist, \&cmp_hits, "blocked");
485 494
486 printH($m, $f, 2, "Summary of non-ignored entries"); 495 printH($m, $f, 2, "Summary of non-ignored entries");
487 printP($m, $f, "List of 'hits' of suspicious activity noticed by Maltfilter, but not\n". 496 printP($m, $f, "List of 'hits' of suspicious activity noticed by Maltfilter, but not\n".
497 close(STATUS); 506 close(STATUS);
498 } 507 }
499 508
500 509
501 ############################################################################# 510 #############################################################################
511 ### DroneBL submission support
512 #############################################################################
513 sub dronebl_process
514 {
515 # return if ($reportmode);
516 return unless ($settings{"DRONEBL"} > 0);
517
518 # Create submission data
519 my $xml = "<?xml version=\"1.0\"?>\n<request key=\"".$settings{"DRONEBL_RPC_KEY"}."\">\n";
520 my $entries = 0;
521 while (my ($ip, $entry) = each(%dronebl)) {
522 if ($entry->{"sent"} == 0) {
523 $xml .= "<add ip=\"".$ip."\" type=\"".$entry->{"type"}."\" />\n";
524 $entries++;
525 }
526 }
527 $xml .= "</request>\n";
528
529 mlog(1, "Trying to submit $entries entries to DroneBL.\n");
530 print STDERR $xml;
531 return;
532
533 return unless ($entries > 0);
534
535 # Submit via HTTP XML-RPC
536 my $tmp = LWP::UserAgent->new;
537 $tmp->agent("Maltfilter/".$progversion);
538 $tmp->timeout(10);
539 my $req = HTTP::Request->new(POST => $settings{"DRONEBL_RPC_URI"});
540 $req->content_type("text/xml");
541 $req->content($xml);
542 $req->user_agent("Maltfilter/".$progversion);
543 my $res = $tmp->request($req);
544
545 if ($res->is_success) {
546 while (my ($ip, $entry) = each(%dronebl)) {
547 $entry->{"sent"} = 1;
548 }
549 } else {
550 mlog(-1, "DroneBL submission failed: [".$res->code."] ".$res->message."\n");
551 }
552
553 # Remove submitted expired entries
554 while (my ($ip, $entry) = each(%dronebl)) {
555 print "$ip: ".$entry->{"sent"}."\n" unless check_time3($entry->{"date"});
556 }
557 }
558
559
560 #############################################################################
502 ### Evidence gathering 561 ### Evidence gathering
503 ############################################################################# 562 #############################################################################
504 my %evidence = (); 563 my %evidence = ();
505 564
506 sub check_add_evidence($$) 565 sub check_add_evidence($$$)
507 { 566 {
508 my ($mip, $mdata) = @_; 567 my ($mip, $mdata, $mfull) = @_;
509 568
510 return unless ($settings{"EVIDENCE"}); 569 return unless ($settings{"EVIDENCE"});
511 570
512 my $tmp = $mdata; 571 my $tmp = $mdata;
513 $tmp =~ s/http:\/\///; 572 $tmp =~ s/http:\/\///;
514 $tmp =~ s/^\.+/_/; 573 $tmp =~ s/^\.+/_/;
515 $tmp =~ s/[^A-Za-z0-9:\.]/_/g; 574 $tmp =~ s/[^A-Za-z0-9:\.]/_/g;
516 575
517 $evidence{$mdata}{"coll"} = $tmp; 576 $evidence{$mdata}{"coll"} = $tmp;
518 $evidence{$mdata}{"hosts"}{$mip} = 1; 577 $evidence{$mdata}{"hosts"}{$mip} = 1;
578 $evidence{$mdata}{"full"}{$mfull} = 1;
519 } 579 }
520 580
521 sub http_fetch($$) 581 sub http_fetch($$)
522 { 582 {
523 my $tmp = LWP::UserAgent->new; 583 my $tmp = LWP::UserAgent->new;
536 return unless ($settings{"EVIDENCE"}); 596 return unless ($settings{"EVIDENCE"});
537 597
538 mdie("Evidence directory '$base' has disappeared.\n") unless (-e $base); 598 mdie("Evidence directory '$base' has disappeared.\n") unless (-e $base);
539 599
540 foreach my $url (keys %evidence) { 600 foreach my $url (keys %evidence) {
541 if (!$evidence{$url}{"done"}) { 601 my $did_fetch = 0;
542 my $filename = $base."/".$evidence{$url}{"coll"}.".data"; 602 my $filename = $base."/".$evidence{$url}{"coll"}.".data";
543 my $filename2 = $base."/".$evidence{$url}{"coll"}.".hosts"; 603 my $filename2 = $base."/".$evidence{$url}{"coll"}.".hosts";
544 my $code = 0; 604 my $filename3 = $base."/".$evidence{$url}{"coll"}.".info";
545 my $message = ""; 605
546 606 # Get data contents only once
547 # Get data contents only once 607 if (! -e $filename) {
548 if (! -e $filename) { 608 $did_fetch = 1;
549 mlog(1, "Fetching evidence for $url\n"); 609 mlog(1, "Fetching evidence for $url\n");
550 my $res = http_fetch($url, ""); 610 my $res = http_fetch($url, "");
551 $code = $res->code; 611 open(FILE, ">:raw", $filename) or mdie("Could not open '$filename' for writing.\n");
552 $message = $res->message; 612 binmode(FILE, ":raw");
553 open(FILE, ">:raw", $filename) or mdie("Could not open '$filename' for writing.\n"); 613 if ($res->is_success && $res->code >= 200 && $res->code <= 201) {
554 binmode(FILE, ":raw"); 614 print FILE $res->content;
555 if ($res->code >= 200 && $res->code <= 201) { 615 }
556 print FILE $res->content; 616 close(FILE);
557 } else { 617
558 print FILE "[$code] $message\n"; 618 open(FILE, ">:raw", $filename3) or mdie("Could not open '$filename3' for writing.\n");
559 } 619 binmode(FILE, ":raw");
560 close(FILE); 620 print FILE "XSS URI : $url\n";
561 } 621 print FILE "Time of retrieval : ".get_time_str(time())."\n";
562 622 print FILE "HTTP return code : [".$res->code."] ".$res->message."\n";
563 # Check if we are appending hosts to existing data 623 print FILE "Content-Type : ".($res->content_type ? $res->content_type : "?")."\n";
564 if (-e $filename2) { 624 print FILE "Last modified : ".($res->last_modified ? $res->last_modified : "?")."\n";
565 open(FILE, "<", $filename2) or mdie("Could not open '$filename2' for reading.\n"); 625 print FILE "------ HTTP Headers ------\n".$res->headers_as_string."\n";
566 while (<FILE>) { 626 print FILE "------ Requests ------\n";
567 if (/^(\d+\.\d+\.\d+\.\d+) *\|/) { 627 print FILE $_."\n" foreach (keys %{$evidence{$url}{"full"}});
568 if (defined($evidence{$url}{"hosts"}{$1})) { 628 close(FILE);
569 delete($evidence{$url}{"hosts"}{$1}); 629 }
570 } 630
631 # Check if we are appending hosts to existing data
632 if (-e $filename2) {
633 open(FILE, "<", $filename2) or mdie("Could not open '$filename2' for reading.\n");
634 while (<FILE>) {
635 if (/^(\d+\.\d+\.\d+\.\d+) *\|/) {
636 if (defined($evidence{$url}{"hosts"}{$1})) {
637 delete($evidence{$url}{"hosts"}{$1});
571 } 638 }
572 } 639 }
573 close(FILE); 640 }
574 open(FILE, ">>", $filename2) or mdie("Could not open '$filename2' for appending.\n"); 641 close(FILE);
575 } else { 642 open(FILE, ">>", $filename2) or mdie("Could not open '$filename2' for appending.\n");
576 open(FILE, ">", $filename2) or mdie("Could not open '$filename2' for writing.\n"); 643 } else {
577 print FILE "# HTTP request result: [$code] $message\n"; 644 open(FILE, ">", $filename2) or mdie("Could not open '$filename2' for writing.\n");
578 print FILE "# Hosts which requested $url\n\n"; 645 }
579 } 646 foreach my $host (sort keys %{$evidence{$url}{"hosts"}}) {
580 647 my $query = $dns->search($host);
581 foreach my $host (sort keys %{$evidence{$url}{"hosts"}}) { 648 my @names = ();
582 print "lol: $host\n"; 649 undef(@names);
583 my $query = $dns->search($host); 650 if ($query) {
584 my @names = (); 651 foreach my $rr ($query->answer) {
585 undef(@names); 652 push(@names, $rr->{"ptrdname"}) if defined($rr->{"ptrdname"});
586 if ($query) {
587 foreach my $rr ($query->answer) {
588 push(@names, $rr->{"ptrdname"}) if defined($rr->{"ptrdname"});
589 }
590 } 653 }
591 printf FILE "%-15s | %s\n", $host, join(" | ", @names); 654 }
592 } 655 printf FILE "%-15s | %s\n", $host, join(" | ", @names);
593 close(FILE); 656 }
594 657 close(FILE);
595 delete($evidence{$url}); 658
596 659 # This entry has been handled, delete it
597 return unless $reportmode; 660 delete($evidence{$url});
598 } 661
662 # If not in report mode, handle only one fetched entry
663 return unless ($reportmode || !$did_fetch);
599 } 664 }
600 } 665 }
601 666
602 667
603 ############################################################################# 668 #############################################################################
677 742
678 ### Check if given timestamp is _newer_ than weedperiod threshold. 743 ### Check if given timestamp is _newer_ than weedperiod threshold.
679 ### Returns false if timestamp is over weed period, e.g. needs weeding. 744 ### Returns false if timestamp is over weed period, e.g. needs weeding.
680 sub check_time1($) 745 sub check_time1($)
681 { 746 {
682 return ($_[0] >= time() - ($settings{"WEED_BLOCK"} * 60 * 60)); 747 return ($_[0] > time() - ($settings{"WEED_FILTER"} * 60 * 60));
683 } 748 }
684 749
685 sub check_time2($) 750 sub check_time2($)
686 { 751 {
687 return ($_[0] >= time() - ($settings{"WEED_GLOBAL"} * 60 * 60)); 752 return ($_[0] > time() - ($settings{"WEED_GLOBAL"} * 60 * 60));
753 }
754
755 sub check_time3($)
756 {
757 return ($_[0] > time() - ($settings{"DRONEBL_MAX_AGE"} * 60));
688 } 758 }
689 759
690 ### Weed out old entries 760 ### Weed out old entries
691 sub weed_do($) 761 sub weed_do($)
692 { 762 {
778 update_date($reason, $mdate); 848 update_date($reason, $mdate);
779 849
780 return $entry->{"hits"}; 850 return $entry->{"hits"};
781 } 851 }
782 852
783 ### Check if given "try count" exceeds treshold and if entry 853 ### Check if given "try count" exceeds threshold and if entry
784 ### is NOT in Netfilter already, then add it if so. 854 ### is NOT in Netfilter already, then add it if so.
785 sub check_add_hit($$$$$) 855 sub check_add_hit($$$$$$)
786 { 856 {
787 my $mip = $_[0]; 857 my $mip = $_[0];
788 my $mdate = str2time($_[1]); 858 my $mdate = str2time($_[1]);
789 my $mclass = $_[2]; 859 my $mclass = $_[2];
790 my $mreason = $_[3]; 860 my $mreason = $_[3];
791 my $mcond = $_[4]; 861 my $mtype = $_[4];
862 my $mcond = $_[5];
792 my $cnt; 863 my $cnt;
793 864
794 if (check_hosts_array(\@noblock_ips, $mip)) { 865 if (check_hosts_array(\@noblock_ips, $mip)) {
795 mlog(3, "Hit to NOBLOCK_IPS($mip): [$mclass] $mreason\n"); 866 mlog(3, "Hit to NOBLOCK_IPS($mip): [$mclass] $mreason\n");
796 return; 867 return;
803 # This is an ignored hit (for disabled test), add to ignorelist 874 # This is an ignored hit (for disabled test), add to ignorelist
804 update_entry(\%ignorelist, $mip, $mdate, $mclass, $mreason, 1); 875 update_entry(\%ignorelist, $mip, $mdate, $mclass, $mreason, 1);
805 return; 876 return;
806 } 877 }
807 878
808 # Check if we have exceeded treshold etc. 879 # Check if we have exceeded threshold etc.
809 if ($cnt >= $settings{"TRESHOLD"} && check_time1($mdate)) { 880 if ($cnt >= $settings{"THRESHOLD"} && check_time1($mdate)) {
810 # Add to blocklist, unless already there. 881 # Add to blocklist, unless already there.
811 if (!defined($blocklist{$mip})) { 882 if (!defined($blocklist{$mip})) {
812 mlog(1, "* Adding $mip ($mdate): [$mclass] $mreason\n"); 883 mlog(1, "* Adding $mip ($mdate): [$mclass] $mreason\n");
813 exec_iptables("-I", "INPUT", "1", "-s", $mip, "-j", $settings{"ACTION"}); 884 exec_iptables("-I", "INPUT", "1", "-s", $mip, "-j", $settings{"ACTION"});
814 } 885 }
815 # Update date of last hit 886 # Update date of last hit
816 $blocklist{$mip} = $mdate; 887 $blocklist{$mip} = $mdate;
888 }
889
890 # Separate check for DroneBL
891 if ($settings{"DRONEBL"} > 0 && $mtype > 0 && $cnt >= $settings{"DRONEBL_THRESHOLD"} && check_time3($mdate)) {
892 $dronebl{$mip}{"type"} = $mtype;
893 $dronebl{$mip}{"date"} = $mdate;
894 $dronebl{$mip}{"sent"} = 0 unless defined($dronebl{$mip}{"sent"});
817 } 895 }
818 } 896 }
819 897
820 898
821 ############################################################################# 899 #############################################################################