# HG changeset patch # User Matti Hamalainen # Date 1250354463 -10800 # Node ID 87c0cdc048f5c479fa0a96e01b419de4fd80faa4 # Parent b05d0f0ff10633d2afc77655f8605c466c1cd4b7 Many changes and cleanups. Works again. diff -r b05d0f0ff106 -r 87c0cdc048f5 maltfilter --- a/maltfilter Fri Aug 14 19:12:14 2009 +0300 +++ b/maltfilter Sat Aug 15 19:41:03 2009 +0300 @@ -61,14 +61,19 @@ my $pid_file = ""; # Name of Maltfilter daemon pid file my $LOGFILE; # Maltfilter logfile handle -my %blocklist = (); # IPs currently blocked in Netfilter $blocklist{$ip} = date_blocked +# IPs currently blocked in Netfilter $blocklist{$ip} = date +my %blocklist = (); # Gathered information about hosts # $statlist{$ip}-> -# "date" = latest change -# "hits" = number of hits -# "reason" = latest reason why added -# "reasons" = array of reasons (only set when $reportmode == 1) +# "date1" = timestamp of first hit +# "date2" = timestamp of latest hit +# "hits" = number of hits to this IP +# $statlist{$ip}{"reason"}{$class}-> +# "msg" = reason message (array if $reportmode) +# "hits" = hits to this class +# "date1" = timestamp of first hit +# "date2" = timestamp of latest hit my %statlist = (); # Gathered information about ignored hits (e.g. hits for tests that are not enabled) @@ -86,13 +91,13 @@ # (1.1) Generic login scan attempts if ($merr =~ /^Failed password for invalid user \S+ from (\d+\.\d+\.\d+\.\d+)/) { - check_add_entry($1, $mdate, "SSH login scan", "", $settings{"CHK_SSHD"}); + check_add_hit($1, $mdate, "SSH login scan", "", $settings{"CHK_SSHD"}); } # (1.2) Root SSH login password bruteforcing attempts # NOTICE! Do not enable this setting, if you allow SSH root logins via # password authentication! Mistyping password may get you blocked then. :) elsif (/^Failed password for root from (\d+\.\d+\.\d+\.\d+)/) { - check_add_entry($1, $mdate, "Root SSH password bruteforce", "", $settings{"CHK_ROOT_SSH_PWD"}); + check_add_hit($1, $mdate, "Root SSH password bruteforce", "", $settings{"CHK_ROOT_SSH_PWD"}); } } # (2) Common/known exploitable CGI/PHP software scans (like phpMyAdmin) @@ -106,7 +111,7 @@ if ($merr =~ /^File does not exist: (.+)$/) { my $tmp = $1; if ($tmp =~ /\/mss2|\/pma|admin|sql|\/roundcube|\/webmail|\/bin|\/mail|xampp|zen|mailto:|appserv|cube|round|_vti_bin|wiki/i) { - check_add_entry($mip, $mdate, "CGI vuln scan", $tmp, $settings{"CHK_KNOWN_CGI"}); + check_add_hit($mip, $mdate, "CGI vuln scan", $tmp, $settings{"CHK_KNOWN_CGI"}); } } } @@ -122,13 +127,13 @@ # used in the URIs. if ($merr =~ /\.php\?\S*?=http:\/\/([^\/]+)/) { if (!check_hosts($settings{"CHK_GOOD_HOSTS"}, $1)) { - check_add_entry($mip, $mdate, "PHP XSS", $merr, $settings{"CHK_PHP_XSS"}); + check_add_hit($mip, $mdate, "PHP XSS", $merr, $settings{"CHK_PHP_XSS"}); } } # (3.2) Try to match proxy scanning attempts elsif ($merr =~ /^http:\/\/([^\/]+)/) { if (!check_hosts($settings{"CHK_GOOD_HOSTS"}, $1)) { - check_add_entry($mip, $mdate, "Proxy scan", $merr, $settings{"CHK_PROXY_SCAN"}); + check_add_hit($mip, $mdate, "Proxy scan", $merr, $settings{"CHK_PROXY_SCAN"}); } } } @@ -138,24 +143,6 @@ ############################################################################# ### Status output functionality ############################################################################# -sub cmp_ips($$) -{ - my @ipa = split(/\./, $_[0]); - my @ipb = split(/\./, $_[1]); - for (my $i = 0; $i < 4; $i++) { - return -1 if ($ipa[$i] > $ipb[$i]); - return 1 if ($ipa[$i] < $ipb[$i]); - } - return 0; -} - -sub cmp_ip_hits($$$$) -{ - return -1 if ($_[2] > $_[3]); - return 1 if ($_[2] < $_[3]); - return cmp_ips($_[0], $_[1]); -} - sub printH($$$$) { my $fh = $_[1]; @@ -213,11 +200,105 @@ return $_[0] ? "<$_[1]>" : ""; } -sub getIP($$) +sub getLink($$) { return $_[0] ? "$_[1]" : $_[1]; } +sub printTable1($$$$$) +{ + my ($m, $f, $table, $keys, $func) = @_; + my $ntotal = 0; + + printElem($m, $f, + "\n". + "". + "\n"); + + foreach my $mip (sort { $func->($table, $a, $b) } keys %{$keys}) { + printElem($m, $f, " "); + printTD($m, $f, sprintf("%-10d", $table->{$mip}{"hits"})); + printTD($m, $f, sprintf("%-15s", getLink($m, $mip))); + printElem(!$m, $f, " : "); + printTD($m, $f, scalar localtime($table->{$mip}{"date1"})); + printElem(!$m, $f, " : "); + printTD($m, $f, scalar localtime($table->{$mip}{"date2"})); + printElem(!$m, $f, " : "); + my @reasons = (); + foreach my $class (sort keys %{$table->{$mip}{"reason"}}) { + my $msgs; + if ($reportmode) { + my @tmp = @{$table->{$mip}{"reason"}{$class}{"msg"}}; + if ($#tmp > 5) { $#tmp = 5; } + $msgs = join(" ".bb($m)."|".eb($m)." ", @tmp); + } else { + $msgs = $table->{$mip}{"reason"}{$class}{"msg"}; + } + push(@reasons, bb($m).$class.eb($m)." #".$table->{$mip}{"reason"}{$class}{"hits"}." ( ".$msgs." )"); + } + printTD($m, $f, join(", ", @reasons)); + printElem($m, $f, "", "\n"); + $ntotal++; + } + printElem($m, $f, "
HitsIP-addressFirst hitLatest hitReason(s)
\n"); + printP($m, $f, bb($m).$ntotal.eb($m)." entries total.\n"); +} + +sub printTable2($$$$$) +{ + my ($m, $f, $table, $keys, $func) = @_; + my $nhits = 0; + my $str = "IP-addressHitsLatest hitClass(es)"; + + printElem($m, $f, "\n". $str."".$str ."\n"); + + my $printEntry = sub { + printTD($m, $f, sprintf("%-15s", getLink($m, $_[0]))); + printElem(!$m, $f, " : "); + printTD($m, $f, sprintf("%-8d ", $table->{$_[0]}{"hits"})); + printElem(!$m, $f, " : "); + printTD($m, $f, scalar localtime($table->{$_[0]}{"date2"})); + printElem(!$m, $f, " : "); + my $tmp = join(", ", sort keys %{$table->{$_[0]}{"reason"}}); + printTD($m, $f, sprintf("%-30s", $tmp)); + $nhits += $table->{$_[0]}{"hits"}; + }; + + my @mkeys = sort { $func->($table, $a, $b) } keys %{$keys}; + my $nkeys = scalar @mkeys; + my $kmax = $nkeys / 2; + for (my $i = 0; $i <= $kmax; $i++) { + printElem($m, $f, " "); + if ($i < $kmax) { $printEntry->($mkeys[$i]); } + printElem($m, $f, "", " || "); + if ($i + $kmax + 1 < $nkeys) { $printEntry->($mkeys[$i + $kmax + 1]); } + printElem($m, $f, "\n", "\n"); + } + + printElem($m, $f, "
\n"); + printP($m, $f, bb($m).$nkeys.eb($m)." entries total, ".bb($m).$nhits.eb($m)." hits total.\n"); +} + +sub cmp_ips($$$) +{ + my @ipa = split(/\./, $_[1]); + my @ipb = split(/\./, $_[2]); + for (my $i = 0; $i < 4; $i++) { + return -1 if ($ipa[$i] > $ipb[$i]); + return 1 if ($ipa[$i] < $ipb[$i]); + } + return 0; +} + +sub cmp_hits($$$) +{ + return $_[0]->{$_[2]}{"hits"} <=> $_[0]->{$_[1]}{"hits"}; +} + + +### +### +### sub generate_status($$) { my $filename = shift; @@ -243,7 +324,6 @@ "); - printH($m, $f, 1, "Maltfilter v$progversion status report"); my $val = $settings{"WEEDPERIOD"}; my $period; @@ -262,94 +342,18 @@ "Generated ".bb($m).$mtime.eb($m).". Data computed from ". ($reportmode ? "complete logfile scan" : "a period of last $period").".\n"); - printH($m, $f, 2, $reportmode ? "Detailed report" : "Blocked entries"); - printElem($m, $f, "\n". ""."\n"); - my $nexcluded = 0; - my $ntotal = 0; - foreach my $mip (sort { $hitcount{$b} <=> $hitcount{$a} } keys %iplist) { - $nexcluded++ if check_hosts_array(\@noblock_ips, $mip); - - printElem($m, $f, " "); - printTD($m, $f, sprintf("%-10d", $hitcount{$mip})); - printTD($m, $f, sprintf("%-15s", getIP($m, $mip))); - printElem(!$m, $f, " : "); - printTD($m, $f, scalar localtime($iplist{$mip})); - my @s = (); - foreach my $cond (sort keys %{$reason{$mip}}) { - my $str; - if ($reportmode) { - my @tmp = reverse(@{$reason{$mip}{$cond}}); - $#tmp = 5 if ($#tmp > 5); - $str = join(" | ", @tmp); - } else { - $str = $reason{$mip}{$cond}; - } - push(@s, bb($m).$cond.eb($m)." [".$reason_n{$mip}{$cond}." hits] (".$str.")"); - } - printTD($m, $f, join(", ".($m ? "
" : ""), @s)); - printElem($m, $f, "\n", "\n"); - $ntotal++; - } - printElem($m, $f, "
HitsIP-addressDate of last hitReason(s)
\n"); - printP($m, $f, bb($m).$ntotal.eb($m)." entries listed, ". - bb($m).($ntotal - $nexcluded).eb($m)." blocked, ".bb($m).$nexcluded.eb($m). - " excluded (defined in NOBLOCK_IPS).\n"); - - - printH($m, $f, 2, "Overview of hits in general"); - printP($m, $f, "List of 'hits' of suspicious activity noticed by Maltfilter, but not necessarily acted upon.\n"); - - my $tmp = "IP-address# of hitsReasons"; - printElem($m, $f, "\n". $tmp."".$tmp ."\n"); - my $hits = 0; - my @keys = sort { cmp_ips($a, $b) } keys %hitcount; - my $nkeys = scalar @keys; + printH($m, $f, 2, "Currently blocked entries"); + printP($m, $f, "List of IPs that are currently blocked (or would be, if this is a report-only mode)."); + printTable1($m, $f, \%statlist, \%blocklist, \&cmp_hits); - my $printEntry = sub { - printTD($m, $f, sprintf("%-15s", getIP($m, $_[0]))); - printElem(!$m, $f, " : "); - printTD($m, $f, sprintf("%-8d ", $hitcount{$_[0]})); - printElem(!$m, $f, " : "); - my $tmp = join(", ", sort keys %{$reason{$_[0]}}); - printTD($m, $f, sprintf("%-30s", $tmp)); - $hits += $hitcount{$_[0]}; - }; - - my $kmax = $nkeys / 2; - for (my $i = 0; $i <= $kmax; $i++) { - printElem($m, $f, " "); - if ($i < $kmax) { - &$printEntry($keys[$i]); - } - printElem($m, $f, "", " || "); - if ($i + $kmax + 1 < $nkeys) { - &$printEntry($keys[$i + $kmax + 1]); - } - printElem($m, $f, "\n", "\n"); - } + printH($m, $f, 2, "Summary of non-ignored entries"); + printP($m, $f, "List of 'hits' of suspicious activity noticed by Maltfilter, but not necessarily acted upon.\n"); + printTable2($m, $f, \%statlist, \%statlist, \&cmp_ips); - printElem($m, $f, "
\n"); - - printP($m, $f, bb($m).(scalar keys %hitcount).eb($m)." IPs total, ".bb($m).$hits.eb($m)." hits total.\n"); - - - printH($m, $f, 2, "Ignored hit types"); + printH($m, $f, 2, "Ignored entries"); printP($m, $f, "List of hits that were ignored (not acted upon), because the test was disabled.\n"); + printTable1($m, $f, \%ignorelist, \%ignorelist, \&cmp_hits); - printElem($m, $f, "\n\n"); - foreach my $mip (sort { cmp_ips($a, $b) } keys %ignored) { - printElem($m, $f, ""); - printTD($m, $f, sprintf("%-15s", $mip)); - printElem($m, $f, "", "\n"); - } - printElem($m, $f, "
IP-addressType (hits, last time of note)
", " : "); - foreach my $mcond (sort keys %{$ignored{$mip}}) { - my $s = $mcond." (".$hitcount{$mip}." hits, last ".scalar localtime($ignored_d{$mip}{$mcond}).")"; - unless ($ignored{$mip}{$mcond} eq "") { $s .= " for '".$ignored{$mip}{$mcond}."'"; } - print $f $s; - } - printElem($m, $f, "
\n"); - printElem($m, $f, "\n\n"); close(STATUS); } @@ -358,7 +362,7 @@ ############################################################################# ### Entry management / handling functions ############################################################################# -### Host and IP matching functions +### Check if given IP or host exists in array sub check_hosts_array($$) { my $chk_host = $_[1]; @@ -377,6 +381,7 @@ return 0; } +### Check IP/host against | separated list of IPs/hosts sub check_hosts($$) { my @tmp = split(/\s*\|\s*/, $_[0]); @@ -394,45 +399,45 @@ } } -### Get current Netfilter INPUT table entries we manage -sub update_iplist($) +### Get current Netfilter INPUT table entries that match +### entry types we manage, e.g. blocklist +sub update_blocklist($) { my $mdate = $_[0]; open(STATUS, $settings{"IPTABLES"}." -v -n -L INPUT |") or die("Could not execute ".$settings{"IPTABLES"}."\n"); + %blocklist = (); + undef(%blocklist); while () { chomp; if (/^\s*(\d+)\s+\d+\s+$settings{"ACTION"}\s+all\s+--\s+\*\s+\*\s+(\d+\.\d+\.\d+\.\d+)\s+0\.0\.0\.0\/0\s*$/) { - my $mcount = $1; my $mip = $2; - if (!defined($blocklist{$mip}) && $mdate >= 0) { - mlog(2, "* $mip appeared in iptables, adding.\n"); - $hitcount{$mip} = $settings{"THRESHOLD"}; - check_add_entry($mip, $mdate, "?", "added from iptables", 1); + if (!defined($blocklist{$mip}) && $mdate > 0) { + mlog(2, "* $mip appeared in iptables."); } + $blocklist{$2} = $mdate; + update_entry(\%statlist, $mip, $mdate, "?", "From iptables."); } } close(STATUS); } -### Weed out old entries +### Check if given timestamp is _newer_ than weedperiod threshold. +### Returns false if timestamp is over weed period, e.g. needs weeding. sub check_time($) { return ($_[0] >= time() - ($settings{"WEEDPERIOD"} * 60 * 60)); } +### Weed out old entries sub weed_do($) { - if (defined($blocklist{$_[0]})) { - mlog(2, "* Weeding $_[0] ($iplist{$_[0]})\n"); - exec_iptables("-D", "INPUT", "-s", $_[0], "-d", "0.0.0.0/0", "-j", $settings{"ACTION"}); - } - undef($reason{$_[0]}); - undef($reason_n{$_[0]}); - undef($ignored{$_[0]}); - undef($ignored_d{$_[0]}); - undef($iplist{$_[0]}); - undef($blocklist{$_[0]}); + my $mtime = $blocklist{$_[0]}; + mlog(2, "* Weeding $_[0] (".($mtime >= 0 ? scalar localtime($mtime) : $mtime)."\n"); + exec_iptables("-D", "INPUT", "-s", $_[0], "-d", "0.0.0.0/0", "-j", $settings{"ACTION"}); + delete($blocklist{$_[0]}); + delete($statlist{$_[0]}); + delete($ignorelist{$_[0]}); } sub weed_entries() @@ -440,55 +445,79 @@ # Don't weed in report mode. # return if ($reportmode); - foreach my $mip (keys %iplist) { - if (defined($iplist{$mip})) { - if ($iplist{$mip} >= 0) { - if (!check_time($iplist{$mip})) { weed_do($mip); } + my @mips = keys %blocklist; + foreach my $mip (@mips) { + if (defined($blocklist{$mip})) { + if ($blocklist{$mip} >= 0) { + weed_do($mip) unless check_time($blocklist{$mip}); } else { weed_do($mip); } } } - mlog(-1, "hmm\n"); +} + +### Update one entry of +sub update_entry($$$$$) +{ + my ($struct, $mip, $mdate, $mclass, $mreason) = @_; + + my $cnt = $struct->{$mip}{"hits"}++; + $struct->{$mip}{"reason"}{$mclass}{"hits"}++; + + if ($reportmode) { + push(@{$struct->{$mip}{"reason"}{$mclass}{"msg"}}, $mreason); + } else { + $struct->{$mip}{"reason"}{$mclass}{"msg"} = $mreason; + } + + if (!defined($struct->{$mip}{"date1"})) { + $struct->{$mip}{"date1"} = $mdate; + } + $struct->{$mip}{"date2"} = $mdate; + + if (!defined($struct->{$mip}{"reason"}{$mclass}{"date2"})) { + $struct->{$mip}{"reason"}{$mclass}{"date2"} = $mdate; + } + $struct->{$mip}{"reason"}{$mclass}{"date2"} = $mdate; + + return $cnt; } ### Check if given "try count" exceeds treshold and if entry ### is NOT in Netfilter already, then add it if so. -sub check_add_entry($$$$$) +sub check_add_hit($$$$$) { my $mip = $_[0]; my $mdate = str2time($_[1]); my $mclass = $_[2]; my $mreason = $_[3]; my $mcond = $_[4]; - - my $cnt = $hitcount{$mip}++; - $reason_n{$mip}{$mclass}++; - if ($reportmode) { - push(@{$reason{$mip}{$mclass}}, $mreason); - } else { - $reason{$mip}{$mclass} = $mreason; + my $cnt; + + if (check_hosts_array(\@noblock_ips, $mip)) { + mlog(3, "Hit to NOBLOCK_IPS($mip): [$mclass] $mreason\n"); + return; } - if ($reportmode || ($cnt >= $settings{"TRESHOLD"} && check_time($mdate))) { - my $pat; - if (!$mcond) { - $ignored{$mip}{$mclass} = $mreason; - $ignored_d{$mip}{$mclass} = $mdate; - return; + + # If condition is true, we add to regular statlist + if ($mcond) { + $cnt = update_entry(\%statlist, $mip, $mdate, $mclass, $mreason); + } else { + # This is an ignored hit (for disabled test), add to ignorelist + update_entry(\%ignorelist, $mip, $mdate, $mclass, $mreason); + return; + } + + # Check if we have exceeded treshold etc. + if ($cnt >= $settings{"TRESHOLD"} && check_time($mdate)) { + # Add to blocklist, unless already there. + if (!defined($blocklist{$mip})) { + mlog(1, "* Adding $mip ($mdate): [$mclass] $mreason\n"); + exec_iptables("-I", "INPUT", "1", "-s", $mip, "-j", $settings{"ACTION"}); } - if (!defined($iplist{$mip})) { - if (!check_hosts_array(\@noblock_ips, $mip)) { - # Add entry that has >= treshold hits and is not added yet - mlog(1, "* Adding $mip ($mdate): $mreason\n"); - exec_iptables("-I", "INPUT", "1", "-s", $mip, "-j", $settings{"ACTION"}); - } - $iplist{$mip} = $mdate; - } else { - # Over treshold, but is added, check if we can update the timedate - if ($mdate > $iplist{$mip}) { - $iplist{$mip} = $mdate; - } - } + # Update date of last hit + $blocklist{$mip} = $mdate; } } @@ -511,7 +540,7 @@ ### Initialize sub malt_init { mlog(0, "Updating initial blocklist from netfilter.\n"); - update_iplist(-1); + update_blocklist(-1); foreach my $filename (@scanfiles) { local *INFILE; @@ -545,6 +574,7 @@ undef($LOGFILE); } +### Signal handlers sub malt_int { mlog(-1, "\nCaught Interrupt (^C), aborting.\n"); malt_cleanup(); @@ -583,7 +613,7 @@ # (in case entries have appeared there from "outside") # and perform weeding of old entries. $counter = 0; - update_iplist(time()); + update_blocklist(time()); weed_entries(); generate_status($settings{"STATUS_FILE_PLAIN"}, 0); generate_status($settings{"STATUS_FILE_HTML"}, 1); @@ -595,6 +625,7 @@ } } +### Read one configuration file sub malt_read_config($) { my $filename = $_[0]; @@ -673,6 +704,7 @@ # Read configuration files if (defined(my $filename = shift)) { # Let user define his/her own logfiles to scan + @scanfiles_def = (); undef(@scanfiles_def); die("Errors in configuration file '$filename', bailing out.\n") unless (malt_read_config($filename) == 0); @@ -688,8 +720,9 @@ my %saw = (); @scanfiles = grep(!$saw{$_}++, @scanfiles_def); +%saw = (); +@noblock_ips = grep(!$saw{$_}++, @noblock_ips_def); undef(%saw); -@noblock_ips = grep(!$saw{$_}++, @noblock_ips_def); # Open logfile if ($settings{"DRY_RUN"}) {