Mercurial > hg > egg-tcls

changeset 690:8a6cca58e853

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fetch_weather, weather: Update to match with Digitraffic API changes.
author Matti Hamalainen <ccr@tnsp.org>
date Thu, 22 Jun 2023 13:32:07 +0300
parents 299a17b98f84
children e1aba44b8c7b
files fetch_weather.pl weather.tcl
diffstat 2 files changed, 81 insertions(+), 85 deletions(-) [+]
line wrap: on
line diff
--- a/fetch_weather.pl	Thu Jun 22 13:28:34 2023 +0300
+++ b/fetch_weather.pl	Thu Jun 22 13:32:07 2023 +0300
@@ -1,8 +1,8 @@
 #!/usr/bin/perl -w
 ##########################################################################
 #
-# Fetch Weather v1.2.0 by Matti 'ccr' Hamalainen <ccr@tnsp.org>
-# (C) Copyright 2014-2021 Tecnic Software productions (TNSP)
+# Fetch Weather v1.3.0 by Matti 'ccr' Hamalainen <ccr@tnsp.org>
+# (C) Copyright 2014-2023 Tecnic Software productions (TNSP)
 # This script is freely distributable under GNU GPL (version 2) license.
 #
 # Should be ran as a cronjob, and configured properly.
@@ -47,8 +47,8 @@
   "http_user_agent" => "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 6.0) Opera 10.63  [en]",
   "tiehallinto_meta" => "tiehallinto.meta",
   "tiehallinto_meta_period" => 7,
-  "tiehallinto_rw_url" => "https://tie.digitraffic.fi/api/v1/data/weather-data",
-  "tiehallinto_meta_url" => "https://tie.digitraffic.fi/api/v1/metadata/weather-stations",
+  "tiehallinto_rw_url" => "https://tie.digitraffic.fi/api/weather/v1/stations/data",
+  "tiehallinto_meta_url" => "https://tie.digitraffic.fi/api/weather/v1/stations",
   "fmi_weather_base_url" => "https://opendata.fmi.fi/wfs",
   "fmi_weather_extra_params" => "&maxlocations=300&bbox=19,59,32,75",
 );
@@ -101,20 +101,6 @@
 }
 
 
-sub parse_timestamp($$)
-{
-  my ($str, $offs) = @_;
-  if ($str =~ /^(\d+):(\d+)$/)
-  {
-    return $offs + (60 * 60 * $1) + ($2 * 60);
-  }
-  else
-  {
-    return $offs;
-  }
-}
-
-
 sub format_time_gmt($)
 {
   # 2012-02-27T00:00:00Z
@@ -135,17 +121,22 @@
 }
 
 
-### Return either data or if not defined, empty string
-sub plonk_data
+sub plonk_data_var($$$)
 {
-  return defined($_[0]) ? $_[0] : (defined($_[1]) ? $_[1] : "");
+  return defined($_[0]->{$_[1]}) ? $_[0]->{$_[1]} : $_[2];
 }
 
 
-### Same as plonk_data() but also lowercase the data string
-sub plonk_data_lc($)
+sub plonk_data_value($$)
 {
-  return defined($_[0]) ? lc($_[0]) : (defined($_[1]) ? $_[1] : "");
+  if (defined($_[0]) && defined($_[0]{$_[1]}) && defined($_[0]{$_[1]}{"value"}))
+  {
+    return $_[0]{$_[1]}{"value"};
+  }
+  else
+  {
+    return "";
+  }
 }
 
 
@@ -291,7 +282,7 @@
 if (!defined($opt_cfgfile))
 {
   die(
-    "Weather Fetch v1.0 by ccr/TNSP <ccr\@tnsp.org>\n".
+    "Weather Fetch v1.3.0 by ccr/TNSP <ccr\@tnsp.org>\n".
     "Usage: $0 <config file> [options]\n".
     "\n".
     " -force      : Force updating of all data\n".
@@ -345,7 +336,7 @@
 
     print $json_str if opt_chk_bool("dump");
 
-    if (!defined($data->{"dataUpdatedTime"}) || !defined($data->{"weatherStations"}))
+    if (!defined($data->{"dataUpdatedTime"}) || !defined($data->{"stations"}))
     {
       print STDERR "ERROR: REST/JSON call result did not contain required data.\n";
       print STDERR $json_str."\n\n";
@@ -381,6 +372,8 @@
       my $meta_data = {};
       my $json = JSON->new->decode($meta_str);
 
+      print $meta_str if opt_chk_bool("dump");
+
       if ($fetch_meta)
       {
         # Save new cache, in more optimal form, if needed.
@@ -390,48 +383,56 @@
 
       foreach my $ms (@{$json->{"features"}})
       {
+        # Filter functional stations?
         if (defined($ms->{"properties"}) &&
             defined($ms->{"geometry"}{"coordinates"}) &&
-            defined($ms->{"properties"}{"names"}{"fi"}))
+            $ms->{"properties"}{"collectionStatus"} eq "GATHERING" &&
+            $ms->{"properties"}{"name"} !~ /^TEST_/
+            )
         {
           $meta_data->{$ms->{"id"}} = $ms;
         }
       }
 
       my $nrecords = 0;
-      foreach my $wdata (@{$data->{"weatherStations"}})
+      foreach my $wdata (@{$data->{"stations"}})
       {
         my $wid = $wdata->{"id"};
-        if (defined($meta_data->{$wid}) && defined($wdata->{"sensorValues"}))
+        if (defined($meta_data->{$wid}) &&
+            defined($wdata->{"sensorValues"}))
         {
-          $wdata->{"sensors"} = {};
+          my $sensors = {};
           foreach my $sensor (@{$wdata->{"sensorValues"}})
           {
-            $wdata->{"sensors"}->{$sensor->{"oldName"}} = $sensor;
+            $sensors->{$sensor->{"name"}} = $sensor;
           }
 
+          my $wqname = $meta_data->{$wid}{"properties"}{"name"};
+          $wqname =~ s#_# #g;
+          $wqname =~ s/(^|\s)(vt|yt|st|kt)(\d+)/"Tie ".$3/ge;
+          $wqname .= " (TH)";
+
           $nrecords++;
-          $weatherdata->{$meta_data->{$wid}{"properties"}{"names"}{"fi"}} =
+          $weatherdata->{$wqname} =
           [
-            # Measurement source type
-            1,
-
             # Basic data
-            plonk_data($meta_data->{$wid}{"geometry"}{"coordinates"}[1], 0),
-            plonk_data($meta_data->{$wid}{"geometry"}{"coordinates"}[0], 0),
-            plonk_data($meta_data->{$wid}{"geometry"}{"coordinates"}[2], 0),
-            str2time(plonk_data($wdata->{"measuredTime"})),
-            plonk_data($wdata->{"sensors"}->{"airtemperature1"}->{"sensorValue"}),
-            plonk_data($wdata->{"sensors"}->{"humidity"}->{"sensorValue"}),
-            plonk_data($wdata->{"sensors"}->{"averagewindspeed"}->{"sensorValue"}),
-            plonk_data($wdata->{"sensors"}->{"winddirection"}->{"sensorValue"}),
+            $meta_data->{$wid}{"geometry"}{"coordinates"}[1],
+            $meta_data->{$wid}{"geometry"}{"coordinates"}[0],
+
+            # This is kind of wrong, because the actual measuredTime exists
+            # for each sensor separately ... but .. whatever.
+            str2time(plonk_data_var($wdata, "dataUpdatedTime", "1970-01-01T14:15:16")),
 
-            # Station type dependant data
-            "", # unused for station type 1
-            plonk_data($wdata->{"sensors"}->{"roadsurfacetemperature1"}->{"sensorValue"}),
-            plonk_data($wdata->{"sensors"}->{"precipitation"}->{"sensorValueDescriptionFi"}),
-            plonk_data($wdata->{"sensors"}->{"visibility"}->{"sensorValue"}),
-            plonk_data($wdata->{"sensors"}->{"precipitation"}->{"sensorValue"}),
+            plonk_data_value($sensors, "ILMA"),
+            plonk_data_value($sensors, "ILMAN_KOSTEUS"),
+            plonk_data_value($sensors, "KESKITUULI"),
+            plonk_data_value($sensors, "TUULENSUUNTA"),
+
+            "", # total cloud cover
+            plonk_data_value($sensors, "TIE_1"),
+            plonk_data_value($sensors, "SADE"),
+            plonk_data_value($sensors, "NÄKYVYYS_KM"),
+            plonk_data_value($sensors, "SADE_INTENSITEETTI"),
           ];
         }
         else
@@ -542,28 +543,25 @@
             # if timestamp is newer, store to location data
             if ($frec->{"lat"} == $flat && $frec->{"long"} == $flong &&
                  (!defined($weatherdata->{$floc->{"gml:name"}}) || 
-                 $frec->{"time"} >= $weatherdata->{$floc->{"gml:name"}}[4])
+                 $frec->{"time"} >= $weatherdata->{$floc->{"gml:name"}}[2])
                )
             {
               $nrecords++ unless defined($weatherdata->{$floc->{"gml:name"}});
 
               $weatherdata->{$floc->{"gml:name"}} =
               [
-                # Measurement source type
-                2,
+                # Basic data
+                plonk_data_var($frec, "lat", 0),
+                plonk_data_var($frec, "long", 0),
 
-                # Basic data
-                plonk_data($frec->{"lat"}, 0),
-                plonk_data($frec->{"long"}, 0),
-                0,
-                plonk_data($frec->{"time"}),
-                plonk_data($frec->{"temperature"}),
-                plonk_data($frec->{"humidity"}),
-                plonk_data($frec->{"windspeedms"}),
-                plonk_data($frec->{"winddirection"}),
+                plonk_data_var($frec, "time", ""),
+                plonk_data_var($frec, "temperature", ""),
+                plonk_data_var($frec, "humidity", ""),
+                plonk_data_var($frec, "windspeedms", ""),
+                plonk_data_var($frec, "winddirection", ""),
 
-                # Station type dependant data
-                plonk_data($frec->{"totalcloudcover"}),
+                plonk_data_var($frec, "totalcloudcover", ""),
+                
               ];
             }
           }
@@ -602,7 +600,7 @@
 
     foreach my $key (keys %$weatherdata)
     {
-      if ($wqtime - $weatherdata->{$key}[4] > (60 * $purge))
+      if ($wqtime - $weatherdata->{$key}[2] > (60 * $purge))
       {
         delete $$weatherdata{$key};
       }
--- a/weather.tcl	Thu Jun 22 13:28:34 2023 +0300
+++ b/weather.tcl	Thu Jun 22 13:32:07 2023 +0300
@@ -1,7 +1,7 @@
 ##########################################################################
 #
-# Weather v2.1.0 by Matti 'ccr' Hamalainen <ccr@tnsp.org>
-# (C) Copyright 2014-2021 Tecnic Software productions (TNSP)
+# Weather v2.2.0 by Matti 'ccr' Hamalainen <ccr@tnsp.org>
+# (C) Copyright 2014-2023 Tecnic Software productions (TNSP)
 #
 # Requires data fetcher to be run as a cronjob, see fetch_weather.pl
 # for more information.
@@ -25,7 +25,7 @@
 package require textutil::split
 
 set weather_name "Weather"
-set weather_message "$weather_name v2.1.0 (C) 2014-2021 ccr/TNSP"
+set weather_message "$weather_name v2.2.0 (C) 2014-2023 ccr/TNSP"
 
 
 #-------------------------------------------------------------------------
@@ -130,7 +130,7 @@
       gets $ufile uline
       set udata [split $uline "|"]
       if {[llength $udata] > 0} {
-        set utemp [lindex $udata 6]
+        set utemp [lindex $udata 4]
         set ukey [lindex $udata 0]
 
         set weather_data($ukey) $udata
@@ -231,23 +231,21 @@
 
   array unset uvals
   set uvals(station) [lindex $udata 0]
-  set uvals(type) [lindex $udata 1]
-  set uvals(c_lat) [lindex $udata 2]
-  set uvals(c_lng) [lindex $udata 3]
-  set uvals(c_height) [lindex $udata 4]
-  set uvals(vtime) [lindex $udata 5]
-  set uvals(temp) [lindex $udata 6]
-  set uvals(humidity) [lindex $udata 7]
-  set uvals(wind_speed) [lindex $udata 8]
-  set uvals(wind_direction) [weather_get_wind_direction [lindex $udata 9]]
-  set uvals(wind_direction_deg) [lindex $udata 9]
-  set uvals(cloudiness) [weather_get_table_value $weather_msg_cloudiness [lindex $udata 10]]
-  set uvals(cloudiness_val) [weather_get_raw_table_value $weather_msg_cloudiness [lindex $udata 10]]
-  set uvals(road_surface_temp) [lindex $udata 11]
-  set uvals(precipitation) [lindex $udata 12]
-  set uvals(visibility) [lindex $udata 13]
-  set uvals(precipitation2) [weather_get_table_value $weather_msg_precipitation [lindex $udata 14]]
-  set uvals(precipitation_val) [weather_get_raw_table_value $weather_msg_precipitation [lindex $udata 14]]
+  set uvals(c_lat) [lindex $udata 1]
+  set uvals(c_lng) [lindex $udata 2]
+  set uvals(vtime) [lindex $udata 3]
+  set uvals(temp) [lindex $udata 4]
+  set uvals(humidity) [lindex $udata 5]
+  set uvals(wind_speed) [lindex $udata 6]
+  set uvals(wind_direction) [weather_get_wind_direction [lindex $udata 7]]
+  set uvals(wind_direction_deg) [lindex $udata 7]
+  set uvals(cloudiness) [weather_get_table_value $weather_msg_cloudiness [lindex $udata 8]]
+  set uvals(cloudiness_val) [weather_get_raw_table_value $weather_msg_cloudiness [lindex $udata 8]]
+  set uvals(road_surface_temp) [lindex $udata 9]
+  set uvals(precipitation) [lindex $udata 10]
+  set uvals(visibility) [lindex $udata 11]
+  set uvals(precipitation2) [weather_get_table_value $weather_msg_precipitation [lindex $udata 12]]
+  set uvals(precipitation_val) [weather_get_raw_table_value $weather_msg_precipitation [lindex $udata 13]]
 
   if {[expr [clock seconds] - $uvals(vtime)] < 3600} {
     set uvals(ctime) [clock format $uvals(vtime) -format "%H:%M"]