# HG changeset patch
# User Matti Hamalainen <ccr@tnsp.org>
# Date 1513601658 -7200
# Node ID fe478b7bd80ee030f4d1d507e2380d59a7fc31ee
# Parent  9b4b1e4ce313781d21507514d445f2c1748226e4
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.

diff -r 9b4b1e4ce313 -r fe478b7bd80e fetch_weather.pl
--- a/fetch_weather.pl	Mon Dec 18 14:42:15 2017 +0200
+++ b/fetch_weather.pl	Mon Dec 18 14:54:18 2017 +0200
@@ -19,6 +19,7 @@
 use strict;
 use warnings;
 use utf8;
+use Encode;
 use LWP::UserAgent;
 use HTTP::Message;
 use HTML::Entities;
@@ -161,6 +162,19 @@
 }
 
 
+sub force_decode_utf8($)
+{
+  if (!utf8::is_utf8($_[0]))
+  {
+    return decode("utf8", $_[0]);
+  }
+  else
+  {
+    return $_[0];
+  }
+}
+
+
 ### Return either data or if not defined, empty string
 sub plonk_data($)
 {
@@ -334,7 +348,7 @@
   my $filename = opt_get("outfile");
   if (-e "$filename")
   {
-    my $str = read_text($filename);
+    my $str = force_decode_utf8(read_text($filename));
     if (defined($str))
     {
       foreach my $line (split(/\s*\n\s*/, $str))
@@ -361,7 +375,7 @@
   my $res = fetch_http($uri);
   if ($res->code >= 200 && $res->code <= 201)
   {
-    my $json_str = $res->decoded_content;
+    my $json_str = force_decode_utf8($res->decoded_content);
     my $data = JSON->new->decode($json_str);
 
     if (!defined($data->{"dataUpdatedTime"}) || !defined($data->{"weatherStations"}))
@@ -385,21 +399,13 @@
         my $res = fetch_http($uri);
         die("Failed to fetch $uri data.\n") unless ($res->code <= 200 && $res->code <= 201);
 
-        $meta_str = $res->decoded_content;
-
-        # XXX: This is a hack. For some reason the data does not get utf8 flagged internally.
-        if (!utf8::is_utf8($meta_str))
-        {
-          printf STDERR "Upgrading meta_str to UTF-8.\n" if (opt_get_int("debug") > 0);
-          utf8::upgrade($meta_str);
-        }
-
+        $meta_str = force_decode_utf8($res->decoded_content);
         $fetch_meta = 1;
       }
       else
       {
         print STDERR "Using CACHED Tiehallinto static meta data from '".$meta_file."'.\n" if (opt_get_int("debug") > 0);
-        $meta_str = read_text($meta_file);
+        $meta_str = force_decode_utf8(read_text($meta_file));
       }
 
       print STDERR "Is meta_str UTF8? ".(utf8::is_utf8($meta_str) ? "yes" : "NO!")."\n" if (opt_get_int("debug") > 0);
@@ -412,7 +418,7 @@
       {
         # Save new cache, in more optimal form, if needed.
         print STDERR "Storing to cache '".$meta_file."'.\n" if (opt_get_int("debug") > 0);
-        write_binary($meta_file, JSON->new->encode($json));
+        write_binary($meta_file, JSON->new->utf8->encode($json));
       }
 
       foreach my $ms (@{$json->{"features"}})
@@ -491,7 +497,7 @@
   my $res = fetch_http($uri);
   if ($res->code >= 200 && $res->code <= 201)
   {
-    my $xml = XMLin($res->decoded_content);
+    my $xml = XMLin(force_decode_utf8($res->decoded_content));
     my $time_base = time();
 
     if (defined($xml->{"wfs:member"}{"omso:GridSeriesObservation"}))