Mercurial > hg > egg-tcls

comparison fetch_weather.pl @ 463:fe478b7bd80e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
author Matti Hamalainen <ccr@tnsp.org>
date Mon, 18 Dec 2017 14:54:18 +0200
parents 9b4b1e4ce313
children 4075cf3e094c
comparison
equal deleted inserted replaced
462:9b4b1e4ce313 463:fe478b7bd80e
17 ########################################################################## 17 ##########################################################################
18 use 5.18.0; 18 use 5.18.0;
19 use strict; 19 use strict;
20 use warnings; 20 use warnings;
21 use utf8; 21 use utf8;
22 use Encode;
22 use LWP::UserAgent; 23 use LWP::UserAgent;
23 use HTTP::Message; 24 use HTTP::Message;
24 use HTML::Entities; 25 use HTML::Entities;
25 use Compress::Zlib; 26 use Compress::Zlib;
26 use XML::Simple; 27 use XML::Simple;
159 } 160 }
160 return $tmp; 161 return $tmp;
161 } 162 }
162 163
163 164
165 sub force_decode_utf8($)
166 {
167 if (!utf8::is_utf8($_[0]))
168 {
169 return decode("utf8", $_[0]);
170 }
171 else
172 {
173 return $_[0];
174 }
175 }
176
177
164 ### Return either data or if not defined, empty string 178 ### Return either data or if not defined, empty string
165 sub plonk_data($) 179 sub plonk_data($)
166 { 180 {
167 return defined($_[0]) ? $_[0] : ""; 181 return defined($_[0]) ? $_[0] : "";
168 } 182 }
332 if (opt_chk_valid("outfile", 1) && !opt_chk_bool("force_update")) 346 if (opt_chk_valid("outfile", 1) && !opt_chk_bool("force_update"))
333 { 347 {
334 my $filename = opt_get("outfile"); 348 my $filename = opt_get("outfile");
335 if (-e "$filename") 349 if (-e "$filename")
336 { 350 {
337 my $str = read_text($filename); 351 my $str = force_decode_utf8(read_text($filename));
338 if (defined($str)) 352 if (defined($str))
339 { 353 {
340 foreach my $line (split(/\s*\n\s*/, $str)) 354 foreach my $line (split(/\s*\n\s*/, $str))
341 { 355 {
342 my @mtmp = split(/\|/, $line, -1); 356 my @mtmp = split(/\|/, $line, -1);
359 my $uri = opt_get("tiehallinto_rw_url"); 373 my $uri = opt_get("tiehallinto_rw_url");
360 print STDERR "Fetching Tiehallinto road weather data from ".$uri."\n" if (opt_get_int("debug") > 0); 374 print STDERR "Fetching Tiehallinto road weather data from ".$uri."\n" if (opt_get_int("debug") > 0);
361 my $res = fetch_http($uri); 375 my $res = fetch_http($uri);
362 if ($res->code >= 200 && $res->code <= 201) 376 if ($res->code >= 200 && $res->code <= 201)
363 { 377 {
364 my $json_str = $res->decoded_content; 378 my $json_str = force_decode_utf8($res->decoded_content);
365 my $data = JSON->new->decode($json_str); 379 my $data = JSON->new->decode($json_str);
366 380
367 if (!defined($data->{"dataUpdatedTime"}) || !defined($data->{"weatherStations"})) 381 if (!defined($data->{"dataUpdatedTime"}) || !defined($data->{"weatherStations"}))
368 { 382 {
369 print STDERR "ERROR: REST/JSON call result did not contain required data.\n"; 383 print STDERR "ERROR: REST/JSON call result did not contain required data.\n";
383 my $uri = opt_get("tiehallinto_meta_url"); 397 my $uri = opt_get("tiehallinto_meta_url");
384 print STDERR "Fetching Tiehallinto static meta data from ".$uri."\n" if (opt_get_int("debug") > 1); 398 print STDERR "Fetching Tiehallinto static meta data from ".$uri."\n" if (opt_get_int("debug") > 1);
385 my $res = fetch_http($uri); 399 my $res = fetch_http($uri);
386 die("Failed to fetch $uri data.\n") unless ($res->code <= 200 && $res->code <= 201); 400 die("Failed to fetch $uri data.\n") unless ($res->code <= 200 && $res->code <= 201);
387 401
388 $meta_str = $res->decoded_content; 402 $meta_str = force_decode_utf8($res->decoded_content);
389
390 # XXX: This is a hack. For some reason the data does not get utf8 flagged internally.
391 if (!utf8::is_utf8($meta_str))
392 {
393 printf STDERR "Upgrading meta_str to UTF-8.\n" if (opt_get_int("debug") > 0);
394 utf8::upgrade($meta_str);
395 }
396
397 $fetch_meta = 1; 403 $fetch_meta = 1;
398 } 404 }
399 else 405 else
400 { 406 {
401 print STDERR "Using CACHED Tiehallinto static meta data from '".$meta_file."'.\n" if (opt_get_int("debug") > 0); 407 print STDERR "Using CACHED Tiehallinto static meta data from '".$meta_file."'.\n" if (opt_get_int("debug") > 0);
402 $meta_str = read_text($meta_file); 408 $meta_str = force_decode_utf8(read_text($meta_file));
403 } 409 }
404 410
405 print STDERR "Is meta_str UTF8? ".(utf8::is_utf8($meta_str) ? "yes" : "NO!")."\n" if (opt_get_int("debug") > 0); 411 print STDERR "Is meta_str UTF8? ".(utf8::is_utf8($meta_str) ? "yes" : "NO!")."\n" if (opt_get_int("debug") > 0);
406 412
407 # Parse the data .. 413 # Parse the data ..
410 416
411 if ($fetch_meta) 417 if ($fetch_meta)
412 { 418 {
413 # Save new cache, in more optimal form, if needed. 419 # Save new cache, in more optimal form, if needed.
414 print STDERR "Storing to cache '".$meta_file."'.\n" if (opt_get_int("debug") > 0); 420 print STDERR "Storing to cache '".$meta_file."'.\n" if (opt_get_int("debug") > 0);
415 write_binary($meta_file, JSON->new->encode($json)); 421 write_binary($meta_file, JSON->new->utf8->encode($json));
416 } 422 }
417 423
418 foreach my $ms (@{$json->{"features"}}) 424 foreach my $ms (@{$json->{"features"}})
419 { 425 {
420 if (defined($ms->{"properties"}) && 426 if (defined($ms->{"properties"}) &&
489 print STDERR "FMI URI: ".$uri."\n" if (opt_get_int("debug") > 0); 495 print STDERR "FMI URI: ".$uri."\n" if (opt_get_int("debug") > 0);
490 496
491 my $res = fetch_http($uri); 497 my $res = fetch_http($uri);
492 if ($res->code >= 200 && $res->code <= 201) 498 if ($res->code >= 200 && $res->code <= 201)
493 { 499 {
494 my $xml = XMLin($res->decoded_content); 500 my $xml = XMLin(force_decode_utf8($res->decoded_content));
495 my $time_base = time(); 501 my $time_base = time();
496 502
497 if (defined($xml->{"wfs:member"}{"omso:GridSeriesObservation"})) 503 if (defined($xml->{"wfs:member"}{"omso:GridSeriesObservation"}))
498 { 504 {
499 my $fdata = $xml->{"wfs:member"}{"omso:GridSeriesObservation"}; 505 my $fdata = $xml->{"wfs:member"}{"omso:GridSeriesObservation"};