Mercurial > hg > egg-tcls

changeset 412:e0d86ccef873

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
author Matti Hamalainen <ccr@tnsp.org>
date Sat, 07 Jan 2017 22:34:52 +0200
parents fe47617e7251
children b63525aed190
files fetch_weather.pl
diffstat 1 files changed, 19 insertions(+), 17 deletions(-) [+]
line wrap: on
line diff
--- a/fetch_weather.pl	Sat Jan 07 22:30:32 2017 +0200
+++ b/fetch_weather.pl	Sat Jan 07 22:34:52 2017 +0200
@@ -27,8 +27,8 @@
 use Date::Format;
 use Date::Parse;
 use Data::Dumper;
-use File::Slurper qw(read_text write_text);
-use Text::CSV;
+use File::Slurper qw(read_text write_binary);
+use JSON;
 
 
 ###
@@ -355,14 +355,13 @@
       my $meta_str;
       if ($fetch_meta || $force_update)
       {
-        my $uri = "https://raw.githubusercontent.com/finnishtransportagency/metadata/master/csv/meta_rws_stations.csv";
+        my $uri = "http://tie.digitraffic.fi/api/v1/metadata/weather-stations";
         print STDERR "Fetching Tiehallinto static meta data from $uri\n" if (opt_get_int("debug") > 1);
         my $res = fetch_http($uri);
         die("Failed to fetch $uri data.\n") unless ($res->code <= 200 && $res->code <= 201);
 
-        print STDERR "Storing to cache '$meta_file'.\n" if (opt_get_int("debug") > 0);
         $meta_str = $res->decoded_content;
-        write_text($meta_file, $meta_str, "utf8");
+        $fetch_meta = 1;
       }
       else
       {
@@ -372,19 +371,22 @@
 
       # Parse the data ..
       my $meta_data = {};
-#      my $csv = Text::CSV->new({blank_is_undef => 1, decode_utf8 => 1, sep_char => ";"});
-      my $csv = Text::CSV->new({blank_is_undef => 1, sep_char => ";"});
-      die("Failed to instantiate Text::CSV object?\n") unless defined($csv);
+      my $json = JSON->new->decode($meta_str);
 
-      foreach my $line (split(/\s*\n\s*/, $meta_str))
+      if ($fetch_meta)
       {
-        if (defined($line) && $csv->parse($line))
+        # Save new cache, in more optimal form, if needed.
+        print STDERR "Storing to cache '$meta_file'.\n" if (opt_get_int("debug") > 0);
+        write_binary($meta_file, JSON->new->encode($json));
+      }
+
+      foreach my $ms (@{$json->{"features"}})
+      {
+        if (defined($ms->{"properties"}) &&
+            defined($ms->{"geometry"}{"coordinates"}) &&
+            defined($ms->{"properties"}{"names"}{"fi"}))
         {
-          my @fields = $csv->fields();
-          if (scalar(@fields) > 1)
-          {
-            $$meta_data{$fields[0]} = \@fields;
-          }
+          $meta_data->{$ms->{"id"}} = $ms;
         }
       }
 
@@ -395,10 +397,10 @@
         foreach my $wdata (@{$data->{"roadweatherdata"}{"roadweather"}})
         {
           my $wid = $wdata->{"stationid"};
-          if (defined($meta_data->{$wid}) && defined($meta_data->{$wid}[2]) && $meta_data->{$wid}[2] ne "")
+          if (defined($meta_data->{$wid}))
           {
             $nrecords++;
-            $weatherdata->{$meta_data->{$wid}[2]} =
+            $weatherdata->{$meta_data->{$wid}{"properties"}{"names"}{"fi"}} =
             [
               1,
               str2time(plonk_data($wdata->{"measurementtime"}{"utc"})),