Mercurial > hg > egg-tcls

comparison fetch_weather.pl @ 412:e0d86ccef873

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
author Matti Hamalainen <ccr@tnsp.org>
date Sat, 07 Jan 2017 22:34:52 +0200
parents fe47617e7251
children b63525aed190
comparison
equal deleted inserted replaced
411:fe47617e7251 412:e0d86ccef873
25 use Compress::Zlib; 25 use Compress::Zlib;
26 use XML::Simple; 26 use XML::Simple;
27 use Date::Format; 27 use Date::Format;
28 use Date::Parse; 28 use Date::Parse;
29 use Data::Dumper; 29 use Data::Dumper;
30 use File::Slurper qw(read_text write_text); 30 use File::Slurper qw(read_text write_binary);
31 use Text::CSV; 31 use JSON;
32 32
33 33
34 ### 34 ###
35 ### Configuration settings 35 ### Configuration settings
36 ### 36 ###
353 353
354 # Fetch or read the cache 354 # Fetch or read the cache
355 my $meta_str; 355 my $meta_str;
356 if ($fetch_meta || $force_update) 356 if ($fetch_meta || $force_update)
357 { 357 {
358 my $uri = "https://raw.githubusercontent.com/finnishtransportagency/metadata/master/csv/meta_rws_stations.csv"; 358 my $uri = "http://tie.digitraffic.fi/api/v1/metadata/weather-stations";
359 print STDERR "Fetching Tiehallinto static meta data from $uri\n" if (opt_get_int("debug") > 1); 359 print STDERR "Fetching Tiehallinto static meta data from $uri\n" if (opt_get_int("debug") > 1);
360 my $res = fetch_http($uri); 360 my $res = fetch_http($uri);
361 die("Failed to fetch $uri data.\n") unless ($res->code <= 200 && $res->code <= 201); 361 die("Failed to fetch $uri data.\n") unless ($res->code <= 200 && $res->code <= 201);
362 362
363 print STDERR "Storing to cache '$meta_file'.\n" if (opt_get_int("debug") > 0);
364 $meta_str = $res->decoded_content; 363 $meta_str = $res->decoded_content;
365 write_text($meta_file, $meta_str, "utf8"); 364 $fetch_meta = 1;
366 } 365 }
367 else 366 else
368 { 367 {
369 print STDERR "Using CACHED Tiehallinto static meta data from '$meta_file'.\n" if (opt_get_int("debug") > 0); 368 print STDERR "Using CACHED Tiehallinto static meta data from '$meta_file'.\n" if (opt_get_int("debug") > 0);
370 $meta_str = read_text($meta_file, "utf-8"); 369 $meta_str = read_text($meta_file, "utf-8");
371 } 370 }
372 371
373 # Parse the data .. 372 # Parse the data ..
374 my $meta_data = {}; 373 my $meta_data = {};
375 # my $csv = Text::CSV->new({blank_is_undef => 1, decode_utf8 => 1, sep_char => ";"}); 374 my $json = JSON->new->decode($meta_str);
376 my $csv = Text::CSV->new({blank_is_undef => 1, sep_char => ";"}); 375
377 die("Failed to instantiate Text::CSV object?\n") unless defined($csv); 376 if ($fetch_meta)
378 377 {
379 foreach my $line (split(/\s*\n\s*/, $meta_str)) 378 # Save new cache, in more optimal form, if needed.
380 { 379 print STDERR "Storing to cache '$meta_file'.\n" if (opt_get_int("debug") > 0);
381 if (defined($line) && $csv->parse($line)) 380 write_binary($meta_file, JSON->new->encode($json));
381 }
382
383 foreach my $ms (@{$json->{"features"}})
384 {
385 if (defined($ms->{"properties"}) &&
386 defined($ms->{"geometry"}{"coordinates"}) &&
387 defined($ms->{"properties"}{"names"}{"fi"}))
382 { 388 {
383 my @fields = $csv->fields(); 389 $meta_data->{$ms->{"id"}} = $ms;
384 if (scalar(@fields) > 1)
385 {
386 $$meta_data{$fields[0]} = \@fields;
387 }
388 } 390 }
389 } 391 }
390 392
391 # Parse XML and combine with the station meta data 393 # Parse XML and combine with the station meta data
392 if (defined($data->{"roadweatherdata"})) 394 if (defined($data->{"roadweatherdata"}))
393 { 395 {
394 my $nrecords = 0; 396 my $nrecords = 0;
395 foreach my $wdata (@{$data->{"roadweatherdata"}{"roadweather"}}) 397 foreach my $wdata (@{$data->{"roadweatherdata"}{"roadweather"}})
396 { 398 {
397 my $wid = $wdata->{"stationid"}; 399 my $wid = $wdata->{"stationid"};
398 if (defined($meta_data->{$wid}) && defined($meta_data->{$wid}[2]) && $meta_data->{$wid}[2] ne "") 400 if (defined($meta_data->{$wid}))
399 { 401 {
400 $nrecords++; 402 $nrecords++;
401 $weatherdata->{$meta_data->{$wid}[2]} = 403 $weatherdata->{$meta_data->{$wid}{"properties"}{"names"}{"fi"}} =
402 [ 404 [
403 1, 405 1,
404 str2time(plonk_data($wdata->{"measurementtime"}{"utc"})), 406 str2time(plonk_data($wdata->{"measurementtime"}{"utc"})),
405 plonk_data($wdata->{"airtemperature1"}), 407 plonk_data($wdata->{"airtemperature1"}),
406 408