Mercurial > hg > egg-tcls

comparison fetch_weather.pl @ 448:ac3b79eca0ca

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data. Sidenote: How can JSON format data be designed so idiotically?
author Matti Hamalainen <ccr@tnsp.org>
date Thu, 28 Sep 2017 15:22:03 +0300
parents 11124f9bf994
children aa8cc5b67585
comparison
equal deleted inserted replaced
447:11124f9bf994 448:ac3b79eca0ca
41 "opt_tiehallinto" => 0, 41 "opt_tiehallinto" => 0,
42 "purge_threshold" => 60, 42 "purge_threshold" => 60,
43 "fmi_api_key" => "", 43 "fmi_api_key" => "",
44 "outfile" => "", 44 "outfile" => "",
45 "http_user_agent" => "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 6.0) Opera 10.63 [en]", 45 "http_user_agent" => "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 6.0) Opera 10.63 [en]",
46 "tiehallinto_rw_url" => "http://tie.digitraffic.fi/sujuvuus/ws/roadWeather",
47 "tiehallinto_meta" => "tiehallinto.meta", 46 "tiehallinto_meta" => "tiehallinto.meta",
48 "tiehallinto_meta_period" => 7, 47 "tiehallinto_meta_period" => 7,
48 "tiehallinto_rw_url" => "http://tie.digitraffic.fi/api/v1/data/weather-data",
49 "tiehallinto_meta_url" => "http://tie.digitraffic.fi/api/v1/metadata/weather-stations", 49 "tiehallinto_meta_url" => "http://tie.digitraffic.fi/api/v1/metadata/weather-stations",
50 ); 50 );
51 51
52 52
53 ### 53 ###
358 my $uri = opt_get("tiehallinto_rw_url"); 358 my $uri = opt_get("tiehallinto_rw_url");
359 print STDERR "Fetching Tiehallinto road weather data from ".$uri."\n" if (opt_get_int("debug") > 0); 359 print STDERR "Fetching Tiehallinto road weather data from ".$uri."\n" if (opt_get_int("debug") > 0);
360 my $res = fetch_http($uri); 360 my $res = fetch_http($uri);
361 if ($res->code >= 200 && $res->code <= 201) 361 if ($res->code >= 200 && $res->code <= 201)
362 { 362 {
363 my $xml = XMLin($res->decoded_content); 363 my $json_str = $res->decoded_content;
364 364 my $data = JSON->new->decode($json_str);
365 if (!defined($xml->{"soap:Body"}) || !defined($xml->{"soap:Body"}{"RoadWeatherResponse"})) 365
366 { 366 if (!defined($data->{"dataUpdatedTime"}) || !defined($data->{"weatherStations"}))
367 print STDERR "ERROR: SOAP call result did not contain required data.\n"; 367 {
368 print STDERR $res->decoded_content."\n\n"; 368 print STDERR "ERROR: REST/JSON call result did not contain required data.\n";
369 print STDERR $json_str."\n\n";
369 } 370 }
370 else 371 else
371 { 372 {
372 # Parse the XML
373 my $data = $xml->{"soap:Body"}{"RoadWeatherResponse"};
374
375 # Check if we need to update the static meta data 373 # Check if we need to update the static meta data
376 my $meta_file = opt_get("tiehallinto_meta"); 374 my $meta_file = opt_get("tiehallinto_meta");
377 my $meta_stamp = (-e $meta_file) ? (stat($meta_file))[9] : -1; 375 my $meta_stamp = (-e $meta_file) ? (stat($meta_file))[9] : -1;
378 my $fetch_meta = ($meta_stamp + 60*60*24 * opt_get_int("tiehallinto_meta_period")) < time(); 376 my $fetch_meta = ($meta_stamp + 60*60*24 * opt_get_int("tiehallinto_meta_period")) < time();
379 377
424 { 422 {
425 $meta_data->{$ms->{"id"}} = $ms; 423 $meta_data->{$ms->{"id"}} = $ms;
426 } 424 }
427 } 425 }
428 426
429 # Parse XML and combine with the station meta data 427 my $nrecords = 0;
430 if (defined($data->{"roadweatherdata"})) 428 foreach my $wdata (@{$data->{"weatherStations"}})
431 { 429 {
432 my $nrecords = 0; 430 my $wid = $wdata->{"id"};
433 foreach my $wdata (@{$data->{"roadweatherdata"}{"roadweather"}}) 431 if (defined($meta_data->{$wid}) && defined($wdata->{"sensorValues"}))
434 { 432 {
435 my $wid = $wdata->{"stationid"}; 433 $wdata->{"sensors"} = {};
436 if (defined($meta_data->{$wid})) 434 foreach my $sensor (@{$wdata->{"sensorValues"}})
437 { 435 {
438 $nrecords++; 436 $wdata->{"sensors"}->{$sensor->{"oldName"}} = $sensor;
439 $weatherdata->{$meta_data->{$wid}{"properties"}{"names"}{"fi"}} =
440 [
441 1,
442
443 $meta_data->{$wid}{"geometry"}{"coordinates"}[1],
444 $meta_data->{$wid}{"geometry"}{"coordinates"}[0],
445 $meta_data->{$wid}{"geometry"}{"coordinates"}[2],
446
447 str2time(plonk_data($wdata->{"measurementtime"}{"utc"})),
448 plonk_data($wdata->{"airtemperature1"}),
449
450 plonk_data($wdata->{"humidity"}),
451 plonk_data($wdata->{"averagewindspeed"}),
452 ];
453 } 437 }
454 else 438
455 { 439 $nrecords++;
456 print STDERR "Station ID #".$wid." not defined?\n" if (opt_get_int("debug") > 0); 440 $weatherdata->{$meta_data->{$wid}{"properties"}{"names"}{"fi"}} =
457 #.Dumper($meta_data->{$wid}); 441 [
458 } 442 # Measurement source type
443 1,
444
445 # Basic data
446 $meta_data->{$wid}{"geometry"}{"coordinates"}[1],
447 $meta_data->{$wid}{"geometry"}{"coordinates"}[0],
448 $meta_data->{$wid}{"geometry"}{"coordinates"}[2],
449 str2time(plonk_data($wdata->{"measuredTime"})),
450 plonk_data($wdata->{"sensors"}->{"airtemperature1"}->{"sensorValue"}),
451 plonk_data($wdata->{"sensors"}->{"humidity"}->{"sensorValue"}),
452 plonk_data($wdata->{"sensors"}->{"averagewindspeed"}->{"sensorValue"}),
453
454 # Station type dependant data
455 "",
456 plonk_data($wdata->{"sensors"}->{"roadsurfacetemperature1"}->{"sensorValue"}),
457 plonk_data($wdata->{"sensors"}->{"precipitation"}->{"sensorValueDescriptionFi"}),
458 plonk_data($wdata->{"sensors"}->{"visibility"}->{"sensorValue"}),
459 ];
459 } 460 }
460 print STDERR $nrecords." records from Tiehallinto.\n" if (opt_get_int("debug") > 0); 461 else
461 } 462 {
462 else 463 print STDERR "Station ID #".$wid." not defined?\n" if (opt_get_int("debug") > 0);
463 { 464 #.Dumper($meta_data->{$wid});
464 print STDERR "ERROR: Invalid (or unsupported) road weather data blob.\n"; 465 }
465 print STDERR $res->decoded_content."\n\n"; 466 }
466 } 467 print STDERR $nrecords." records from Tiehallinto.\n" if (opt_get_int("debug") > 0);
467 } 468 }
468 } 469 }
469 } 470 }
470 471
471 472
558 $floc->{"gml:name"} ne "") 559 $floc->{"gml:name"} ne "")
559 { 560 {
560 $nrecords++; 561 $nrecords++;
561 $weatherdata->{$floc->{"gml:name"}} = 562 $weatherdata->{$floc->{"gml:name"}} =
562 [ 563 [
563 1, 564 # Measurement source type
564 565 2,
566
567 # Basic data
565 $frec->{"lat"}, 568 $frec->{"lat"},
566 $frec->{"long"}, 569 $frec->{"long"},
567 0, 570 0,
568
569 plonk_data($frec->{"time"}), 571 plonk_data($frec->{"time"}),
570 plonk_data($frec->{"temperature"}), 572 plonk_data($frec->{"temperature"}),
571
572 plonk_data($frec->{"humidity"}), 573 plonk_data($frec->{"humidity"}),
573 plonk_data($frec->{"windspeedms"}), 574 plonk_data($frec->{"windspeedms"}),
575
576 # Station type dependant data
574 translate_clouds(plonk_data($frec->{"totalcloudcover"})), 577 translate_clouds(plonk_data($frec->{"totalcloudcover"})),
575 ]; 578 ];
576 } 579 }
577 } 580 }
578 } 581 }