Mercurial > hg > egg-tcls

changeset 164:d5a0e4248f3e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
weather: Implement configuration file support for weather fetcher backend.
author Matti Hamalainen <ccr@tnsp.org>
date Mon, 02 Jun 2014 20:18:59 +0300
parents bee5cd89d41c
children ff64ef21c20d
files fetch_weather.pl
diffstat 1 files changed, 170 insertions(+), 57 deletions(-) [+]
line wrap: on
line diff
--- a/fetch_weather.pl	Mon Jun 02 19:29:33 2014 +0300
+++ b/fetch_weather.pl	Mon Jun 02 20:18:59 2014 +0300
@@ -8,13 +8,24 @@
 use Data::Dumper;
 
 ###
-### Settings
+### Globals
 ###
-my $opt_outfile = "/home/niinuska/bot/weather.data";
+my %settings = (
+  "opt_fmi" => 0,
+  "opt_tiehallinto" => 0,
+  "fmi_api_key" => "",
+  "outfile" => "",
+);
 
 ###
 ### Helper functions
 ###
+sub mlog($)
+{
+  print STDERR $_[0];
+}
+
+
 sub str_trim($)
 {
   my $str = $_[0];
@@ -183,80 +194,182 @@
 
 
 ###
+### Configuration
+###
+sub opt_chk_bool($)
+{
+  if (defined($settings{$_[0]}))
+  {
+    my $val = $settings{$_[0]};
+    return ($val == 1 || $val eq "true" || $val eq "on" || $val eq "1");
+  }
+  else
+  {
+    return 0;
+  }
+}
+
+
+sub opt_chk_valid($$)
+{
+  if (defined($settings{$_[0]}))
+  {
+    my $val = $settings{$_[0]};
+    return length($val) >= $_[1];
+  }
+  else
+  {
+    return 0;
+  }
+}
+
+
+sub opt_get($)
+{
+  if (defined($settings{$_[0]}))
+  {
+    return $settings{$_[0]};
+  }
+  else
+  {
+    return undef;
+  }
+}
+
+
+sub opt_read_config($)
+{
+  my $filename = $_[0];
+  my $errors = 0;
+  my $line = 0;
+
+  open(CONFFILE, "<", $filename) or die("Could not open configuration '".$filename."'!\n");
+  while (<CONFFILE>)
+  {
+    $line++;
+    chomp;
+    if (/(^\s*#|^\s*$)/) {
+      # Ignore comments and empty lines
+    } elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*(\d+),?\s*$/) {
+      my $key = lc($1);
+      my $value = $2;
+      if (defined($settings{$key})) {
+        $settings{$key} = $value;
+      } else {
+        mlog("[$filename:$line] Unknown setting '$key' = $value\n");
+        $errors = 1;
+      }
+    } elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*\"(.*?)\",?\s*$/) {
+      my $key = lc($1);
+      my $value = $2;
+      if (defined($settings{$key})) {
+        $settings{$key} = $value;
+      } else {
+        mlog("[$filename:$line] Unknown setting '$key' = '$value'\n");
+        $errors = 1;
+      }
+    } else {
+      mlog("[$filename:$line] Syntax error: $_\n");
+      $errors = 1;
+    }
+  }
+  close(CONFFILE);
+  return $errors;
+}
+
+
+###
 ### Main program begins
 ###
 my $weatherdata = {};
 
+die(
+"Weather Fetch v0.1 by ccr/TNSP <ccr\@tnsp.org>\n".
+"Usage: $0 <config file>\n"
+) unless scalar(@ARGV) >= 1;
+
+opt_read_config(shift) == 0 or exit;
+
+
 ### Fetch tiehallinto data
-for (my $i = 1; $i <= 22; $i++)
+if (opt_chk_bool("opt_tiehallinto"))
 {
-  my $res = fetch_http("http://alk.tiehallinto.fi/alk/tiesaa/tiesaa_maak_".$i.".html");
-  if ($res->code >= 200 && $res->code <= 201)
+  for (my $i = 1; $i <= 22; $i++)
   {
-    my $data = $res->decoded_content;
-    
-    # Filter out crap tags we don't want or need
-    $data =~ s/\n/§/g;
-    $data =~ s/<!--.*?-->//ig;
-    $data =~ s/<map[^>]*>.*?<\/map>//ig;
-    $data =~ s/<form[^>]*>.*?<\/form>//ig;
-    $data =~ s/<script[^>]*>.*?<\/script>//ig;
-    $data =~ s/<meta[^>]*>//ig;
-    $data =~ s/<font[^>]*>//ig;
-    $data =~ s/<\/font>//ig;
-    $data =~ s/<span[^>]*>//ig;
-    $data =~ s/<\/span>//ig;
-    $data =~ s/<\/?b>//ig;
-
-    $data =~ s/<br>//ig;
-    $data =~ s/&nbsp;/ /ig;
-    $data =~ s/§/\n/g;
+    my $res = fetch_http("http://alk.tiehallinto.fi/alk/tiesaa/tiesaa_maak_".$i.".html");
+    if ($res->code >= 200 && $res->code <= 201)
+    {
+      my $data = $res->decoded_content;
+      
+      # Filter out crap tags we don't want or need
+      $data =~ s/\n/§/g;
+      $data =~ s/<!--.*?-->//ig;
+      $data =~ s/<map[^>]*>.*?<\/map>//ig;
+      $data =~ s/<form[^>]*>.*?<\/form>//ig;
+      $data =~ s/<script[^>]*>.*?<\/script>//ig;
+      $data =~ s/<meta[^>]*>//ig;
+      $data =~ s/<font[^>]*>//ig;
+      $data =~ s/<\/font>//ig;
+      $data =~ s/<span[^>]*>//ig;
+      $data =~ s/<\/span>//ig;
+      $data =~ s/<\/?b>//ig;
 
-    # Parse the HTML mess
-    my $otree = parse_html($data);
+      $data =~ s/<br>//ig;
+      $data =~ s/&nbsp;/ /ig;
+      $data =~ s/§/\n/g;
+
+      # Parse the HTML mess
+      my $otree = parse_html($data);
 
-    # Find our desired element nodes
-    my $odata = fnodea(fnode($otree, "body"), "div", "class=elementc");
-    my $oupdate = fnode($odata, "p");
-    my $time_base = str2time("00:00");
-    if ($oupdate) {
-      my $tmp = $oupdate->{"nodes"}[0]{"text"};
-      if ($tmp =~ /:\s+(\d\d\.\d\d\.\d\d\d\d)\s+(\d\d:\d\d)/) {
-        $time_base = str2time($1);
+      # Find our desired element nodes
+      my $odata = fnodea(fnode($otree, "body"), "div", "class=elementc");
+      my $oupdate = fnode($odata, "p");
+      my $time_base = str2time("00:00");
+      if ($oupdate) {
+        my $tmp = $oupdate->{"nodes"}[0]{"text"};
+        if ($tmp =~ /:\s+(\d\d\.\d\d\.\d\d\d\d)\s+(\d\d:\d\d)/) {
+          $time_base = str2time($1);
+        }
       }
-    }
-    
-    my $oelems = fnode($odata, "table")->{"nodes"};
-    for (my $n = 1; $n < scalar(@$oelems); $n++)
-    {
-      my $fdata = @$oelems[$n]->{"nodes"};
-      $weatherdata->{$fdata->[0]{"nodes"}[0]{"text"}} =
-      [
-        parse_timestamp(get_node($fdata, "text", 1), $time_base),
-        get_node($fdata, "text", 2),
-        get_node($fdata, "text", 3),
-        get_node($fdata, "text", 4),
-        get_node($fdata, "text", 5),
-      ];
+      
+      my $oelems = fnode($odata, "table")->{"nodes"};
+      for (my $n = 1; $n < scalar(@$oelems); $n++)
+      {
+        my $fdata = @$oelems[$n]->{"nodes"};
+        $weatherdata->{$fdata->[0]{"nodes"}[0]{"text"}} =
+        [
+          parse_timestamp(get_node($fdata, "text", 1), $time_base),
+          get_node($fdata, "text", 2),
+          get_node($fdata, "text", 3),
+          get_node($fdata, "text", 4),
+          get_node($fdata, "text", 5),
+        ];
+      }
     }
   }
 }
 
 ### Fetch FMI data
-my $res = fetch_http("http://data.fmi.fi/fmi-apikey/".$opt_api_key.
-  "/wfs?request=getFeature&storedquery_id=fmi::observations::weather::cities::multipointcoverage".
-  "&parameters=temperature,humidity");
+if (opt_chk_bool("opt_fmi"))
+{
+  die("FMI data scrape enabled, but no API key set.\n") unless opt_chk_valid("fmi_api_key", 10);
 
-if ($res->code >= 200 && $res->code <= 201)
-{
-  my $xml = XMLin($res->decoded_content,
-    KeyAttr => { server => 'name' },
-    ForceArray => [ 'server', 'address' ]);
+  my $res = fetch_http("http://data.fmi.fi/fmi-apikey/".opt_get("fmi_api_key").
+    "/wfs?request=getFeature&storedquery_id=fmi::observations::weather::cities::multipointcoverage".
+    "&parameters=temperature,humidity");
+
+  if ($res->code >= 200 && $res->code <= 201)
+  {
+    my $xml = XMLin($res->decoded_content,
+      KeyAttr => { server => 'name' },
+      ForceArray => [ 'server', 'address' ]);
+  }
 }
 
+
 ### Output
-if (defined($opt_outfile)) {
-  open(STDOUT, '>', $opt_outfile) or die("Could not open output file '$opt_outfile'.\n");
+if (opt_chk_valid("outfile", 1)) {
+  open(STDOUT, '>', opt_get("outfile")) or die("Could not open output file '".opt_get("outfile")."'.\n");
 }
 
 binmode STDOUT, ':encoding(utf-8)';