Mercurial > hg > egg-tcls
annotate fetch_weather.pl @ 475:4075cf3e094c
fetch_weather: Update 'documentation'.
author | Matti Hamalainen <ccr@tnsp.org> |
---|---|
date | Sun, 30 Dec 2018 04:16:09 +0200 |
parents | fe478b7bd80e |
children | ba50359871b7 |
rev | line source |
---|---|
381 | 1 #!/usr/bin/perl -w |
182 | 2 ########################################################################## |
3 # | |
450
aa8cc5b67585
fetch_weather: Bump version.
Matti Hamalainen <ccr@tnsp.org>
parents:
448
diff
changeset
|
4 # Fetch Weather v1.0 by Matti 'ccr' Hamalainen <ccr@tnsp.org> |
407
b05da2077e44
fetch_weather: Bump copyright and version.
Matti Hamalainen <ccr@tnsp.org>
parents:
406
diff
changeset
|
5 # (C) Copyright 2014-2017 Tecnic Software productions (TNSP) |
206 | 6 # This script is freely distributable under GNU GPL (version 2) license. |
182 | 7 # |
8 # Should be ran as a cronjob, and configured properly. | |
206 | 9 # */10 * * * * perl -w /absolute/path/to/fetch_weather.pl /path/to/configfile |
182 | 10 # |
475
4075cf3e094c
fetch_weather: Update 'documentation'.
Matti Hamalainen <ccr@tnsp.org>
parents:
463
diff
changeset
|
11 # Configuration file example is in "config.fetch_weather.example" |
182 | 12 # |
206 | 13 # Requires various Perl modules, in Debian the packages should be: |
475
4075cf3e094c
fetch_weather: Update 'documentation'.
Matti Hamalainen <ccr@tnsp.org>
parents:
463
diff
changeset
|
14 # libwww-perl libxml-simple-perl libtimedate-perl libjson-perl |
4075cf3e094c
fetch_weather: Update 'documentation'.
Matti Hamalainen <ccr@tnsp.org>
parents:
463
diff
changeset
|
15 # libfile-slurper-perl |
182 | 16 # |
17 ########################################################################## | |
417 | 18 use 5.18.0; |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
19 use strict; |
380
729eb88d1c63
fetch_weather: "use warnings".
Matti Hamalainen <ccr@tnsp.org>
parents:
379
diff
changeset
|
20 use warnings; |
243 | 21 use utf8; |
463
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
22 use Encode; |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
23 use LWP::UserAgent; |
399
a80de0130e45
fetch_weather: Support gzip encoded content in HTTP requests.
Matti Hamalainen <ccr@tnsp.org>
parents:
398
diff
changeset
|
24 use HTTP::Message; |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
25 use HTML::Entities; |
399
a80de0130e45
fetch_weather: Support gzip encoded content in HTTP requests.
Matti Hamalainen <ccr@tnsp.org>
parents:
398
diff
changeset
|
26 use Compress::Zlib; |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
27 use XML::Simple; |
189 | 28 use Date::Format; |
199
95012f8f687a
Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents:
198
diff
changeset
|
29 use Date::Parse; |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
30 use Data::Dumper; |
412
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
31 use File::Slurper qw(read_text write_binary); |
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
32 use JSON; |
166 | 33 |
400 | 34 |
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
35 ### |
182 | 36 ### Configuration settings |
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
37 ### |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
38 my %settings = ( |
442
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
39 "force_update" => 0, |
166 | 40 "debug" => 0, |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
41 "opt_fmi" => 0, |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
42 "opt_tiehallinto" => 0, |
374
26b7963e7fbd
fetch_weather: Oops, forgot to define the configuration setting
Matti Hamalainen <ccr@tnsp.org>
parents:
373
diff
changeset
|
43 "purge_threshold" => 60, |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
44 "fmi_api_key" => "", |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
45 "outfile" => "", |
185
64e06ee172c7
Add HTTP user agent setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
182
diff
changeset
|
46 "http_user_agent" => "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 6.0) Opera 10.63 [en]", |
446
ba559d5d44db
fetch_weather: Rename the Tiehallinto static metadata cache filename setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
445
diff
changeset
|
47 "tiehallinto_meta" => "tiehallinto.meta", |
447
11124f9bf994
fetch_weather: Instead of checking the latest static metadata update timestamp against the weather data XML, have a setting of days until the metadata is considered stale.
Matti Hamalainen <ccr@tnsp.org>
parents:
446
diff
changeset
|
48 "tiehallinto_meta_period" => 7, |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
49 "tiehallinto_rw_url" => "http://tie.digitraffic.fi/api/v1/data/weather-data", |
443
1f16d27c571d
fetch_weather: Make tiehallinto API urls configurable.
Matti Hamalainen <ccr@tnsp.org>
parents:
442
diff
changeset
|
50 "tiehallinto_meta_url" => "http://tie.digitraffic.fi/api/v1/metadata/weather-stations", |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
51 ); |
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
52 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
53 |
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
54 ### |
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
55 ### Helper functions |
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
56 ### |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
57 sub mlog($) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
58 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
59 print STDERR $_[0]; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
60 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
61 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
62 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
63 sub fetch_http($) |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
64 { |
198 | 65 my $agent = LWP::UserAgent->new; |
461 | 66 $agent->agent(opt_get("http_user_agent")); |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
67 $agent->timeout(20); |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
68 |
198 | 69 my $req = HTTP::Request->new(GET => $_[0]); |
399
a80de0130e45
fetch_weather: Support gzip encoded content in HTTP requests.
Matti Hamalainen <ccr@tnsp.org>
parents:
398
diff
changeset
|
70 $req->header('Accept-Encoding' => scalar HTTP::Message::decodable()); |
339 | 71 |
411
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
72 print STDERR "# FETCHING URL: ".$_[0]."\n" if (opt_get_int("debug") > 0); |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
73 |
406
4c3abf052d01
fetch_weather: "use 5.018;" and some cleanups.
Matti Hamalainen <ccr@tnsp.org>
parents:
405
diff
changeset
|
74 my $res = $agent->request($req); |
411
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
75 |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
76 if (opt_get_int("debug") > 0) |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
77 { |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
78 print STDERR "# Response: ".$res->code.": ".$res->message."\n"; |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
79 if ($res->code >= 200 && $res->code <= 201) |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
80 { |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
81 print STDERR |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
82 "# Content-charset: ".$res->content_charset."\n". |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
83 "# Content-encoding: ".$res->content_encoding."\n". |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
84 "# Is decoded_content UTF8? ".(utf8::is_utf8($res->decoded_content) ? "yes" : "NO!")."\n"; |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
85 } |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
86 } |
fe47617e7251
fetch_weather: Add some debugging output to fetch_http().
Matti Hamalainen <ccr@tnsp.org>
parents:
410
diff
changeset
|
87 |
406
4c3abf052d01
fetch_weather: "use 5.018;" and some cleanups.
Matti Hamalainen <ccr@tnsp.org>
parents:
405
diff
changeset
|
88 return $res; |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
89 } |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
90 |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
91 |
382
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
92 sub str_trim($) |
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
93 { |
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
94 my $tmp = $_[0]; |
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
95 $tmp =~ s/^\s*//; |
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
96 $tmp =~ s/\s*$//; |
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
97 return $tmp; |
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
98 } |
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
99 |
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
100 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
101 sub parse_timestamp($$) |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
102 { |
196
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
103 my ($str, $offs) = @_; |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
104 if ($str =~ /^(\d+):(\d+)$/) |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
105 { |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
106 return $offs + (60 * 60 * $1) + ($2 * 60); |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
107 } |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
108 else |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
109 { |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
110 return $offs; |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
111 } |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
112 } |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
113 |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
114 |
194 | 115 sub format_time_gmt($) |
189 | 116 { |
117 # 2012-02-27T00:00:00Z | |
199
95012f8f687a
Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents:
198
diff
changeset
|
118 return time2str("%Y-%m-%dT%TZ", $_[0], "UTC"); |
189 | 119 } |
120 | |
193 | 121 |
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
122 my %th_rain_states = |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
123 ( |
237 | 124 "Pouta" => "poutaa", |
125 "Heikko" => "heikkoa sadetta", | |
126 "Kohtalainen" => "kohtalaista sadetta", | |
127 "Voimakas" => "voimakasta sadetta", | |
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
128 ); |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
129 |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
130 my $th_rain_states_k = join("|", map {quotemeta} sort { length($b)<=>length($a) } keys %th_rain_states); |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
131 |
242 | 132 sub translate_rain($) |
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
133 { |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
134 my $tmp = $_[0]; |
237 | 135 $tmp =~ s/($th_rain_states_k)/$th_rain_states{$1}/igo; |
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
136 return $tmp; |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
137 } |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
138 |
331
b8ae5e99341b
fetc_weather: Remove support for old HTML format parsing of Tiehallinto data, it's not provided in this format anymore.
Matti Hamalainen <ccr@tnsp.org>
parents:
325
diff
changeset
|
139 |
244 | 140 my %th_cloud_states = |
141 ( | |
142 0 => "selkeää", | |
247
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
143 1 => "melkein selkeää", |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
144 2 => "verrattain selkeää", |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
145 3 => "verrattain selkeää", |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
146 4 => "puolipilvistä", |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
147 5 => "verrattain pilvistä", |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
148 6 => "verrattain pilvistä", |
244 | 149 7 => "melkein pilvistä", |
150 8 => "pilvistä", | |
151 ); | |
152 | |
153 sub translate_clouds($) | |
154 { | |
247
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
155 return "" if ($_[0] eq "NaN" || $_[0] eq ""); |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
156 my $tmp = int($_[0]); |
244 | 157 foreach my $n (sort { $a <=> $b } keys %th_cloud_states) |
158 { | |
247
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
159 return $th_cloud_states{$n}." (".$n."/8)" if ($tmp == $n); |
244 | 160 } |
161 return $tmp; | |
162 } | |
163 | |
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
164 |
463
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
165 sub force_decode_utf8($) |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
166 { |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
167 if (!utf8::is_utf8($_[0])) |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
168 { |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
169 return decode("utf8", $_[0]); |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
170 } |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
171 else |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
172 { |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
173 return $_[0]; |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
174 } |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
175 } |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
176 |
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
177 |
410 | 178 ### Return either data or if not defined, empty string |
228 | 179 sub plonk_data($) |
180 { | |
181 return defined($_[0]) ? $_[0] : ""; | |
182 } | |
183 | |
184 | |
410 | 185 ### Same as plonk_data() but also lowercase the data string |
237 | 186 sub plonk_data_lc($) |
187 { | |
188 return defined($_[0]) ? lc($_[0]) : ""; | |
189 } | |
190 | |
191 | |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
192 ### |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
193 ### Configuration handling |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
194 ### |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
195 sub opt_chk_bool($) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
196 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
197 if (defined($settings{$_[0]})) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
198 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
199 my $val = $settings{$_[0]}; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
200 return ($val == 1 || $val eq "true" || $val eq "on" || $val eq "1"); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
201 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
202 else |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
203 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
204 return 0; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
205 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
206 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
207 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
208 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
209 sub opt_chk_valid($$) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
210 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
211 if (defined($settings{$_[0]})) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
212 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
213 my $val = $settings{$_[0]}; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
214 return length($val) >= $_[1]; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
215 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
216 else |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
217 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
218 return 0; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
219 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
220 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
221 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
222 |
324
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
223 sub opt_get_int($) |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
224 { |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
225 if (defined($settings{$_[0]})) |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
226 { |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
227 return int($settings{$_[0]}); |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
228 } |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
229 else |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
230 { |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
231 return -1; |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
232 } |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
233 } |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
234 |
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
235 |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
236 sub opt_get($) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
237 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
238 if (defined($settings{$_[0]})) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
239 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
240 return $settings{$_[0]}; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
241 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
242 else |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
243 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
244 return undef; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
245 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
246 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
247 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
248 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
249 sub opt_read_config($) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
250 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
251 my $filename = $_[0]; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
252 my $errors = 0; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
253 my $line = 0; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
254 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
255 open(CONFFILE, "<", $filename) or die("Could not open configuration '".$filename."'!\n"); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
256 while (<CONFFILE>) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
257 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
258 $line++; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
259 chomp; |
353 | 260 if (/(^\s*#|^\s*$)/) |
261 { | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
262 # Ignore comments and empty lines |
353 | 263 } |
386 | 264 elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*(\d+),?\s*$/) |
265 { | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
266 my $key = lc($1); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
267 my $value = $2; |
461 | 268 if (defined($settings{$key})) |
269 { | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
270 $settings{$key} = $value; |
353 | 271 } |
272 else | |
273 { | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
274 mlog("[$filename:$line] Unknown setting '$key' = $value\n"); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
275 $errors = 1; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
276 } |
353 | 277 } |
386 | 278 elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*\"(.*?)\",?\s*$/) |
279 { | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
280 my $key = lc($1); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
281 my $value = $2; |
353 | 282 if (defined($settings{$key})) |
283 { | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
284 $settings{$key} = $value; |
353 | 285 } |
286 else | |
287 { | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
288 mlog("[$filename:$line] Unknown setting '$key' = '$value'\n"); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
289 $errors = 1; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
290 } |
353 | 291 } |
292 else | |
293 { | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
294 mlog("[$filename:$line] Syntax error: $_\n"); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
295 $errors = 1; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
296 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
297 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
298 close(CONFFILE); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
299 return $errors; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
300 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
301 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
302 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
303 ### |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
304 ### Main program begins |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
305 ### |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
306 my $weatherdata = {}; |
442
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
307 my $opt_cfgfile; |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
308 |
442
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
309 while (scalar(@ARGV) > 0) |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
310 { |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
311 my $arg = shift; |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
312 if ($arg eq "-force") |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
313 { |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
314 $settings{"force_update"} = 1; |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
315 } |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
316 elsif ($arg eq "-debug") |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
317 { |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
318 $settings{"debug"} = 1; |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
319 } |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
320 else |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
321 { |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
322 die("Configuration file already specified!\n") if defined($opt_cfgfile); |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
323 $opt_cfgfile = $arg; |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
324 } |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
325 } |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
326 |
442
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
327 if (!defined($opt_cfgfile)) |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
328 { |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
329 die( |
450
aa8cc5b67585
fetch_weather: Bump version.
Matti Hamalainen <ccr@tnsp.org>
parents:
448
diff
changeset
|
330 "Weather Fetch v1.0 by ccr/TNSP <ccr\@tnsp.org>\n". |
442
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
331 "Usage: $0 <config file> [options]\n". |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
332 "\n". |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
333 " -force : Force updating of all data\n". |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
334 " -debug : Enable debug\n". |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
335 "\n" |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
336 ); |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
337 } |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
338 |
19fe8ef0a902
fetch_weather: Improve commandline handling, add -force and -debug options.
Matti Hamalainen <ccr@tnsp.org>
parents:
438
diff
changeset
|
339 opt_read_config($opt_cfgfile) == 0 or die("Errors while parsing configuration file '".$opt_cfgfile."'.\n"); |
461 | 340 print STDERR "Forcing update of all data.\n" if opt_chk_bool("force_update"); |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
341 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
342 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
343 ### |
354
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
344 ### Load already cached data |
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
345 ### |
462
9b4b1e4ce313
fetch_weather: When force_update is set, do not load the old data cache.
Matti Hamalainen <ccr@tnsp.org>
parents:
461
diff
changeset
|
346 if (opt_chk_valid("outfile", 1) && !opt_chk_bool("force_update")) |
354
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
347 { |
438
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
348 my $filename = opt_get("outfile"); |
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
349 if (-e "$filename") |
354
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
350 { |
463
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
351 my $str = force_decode_utf8(read_text($filename)); |
438
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
352 if (defined($str)) |
354
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
353 { |
438
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
354 foreach my $line (split(/\s*\n\s*/, $str)) |
354
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
355 { |
438
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
356 my @mtmp = split(/\|/, $line, -1); |
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
357 if (scalar(\@mtmp) >= 3) |
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
358 { |
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
359 $weatherdata->{shift @mtmp} = \@mtmp; |
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
360 } |
354
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
361 } |
438
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
362 print STDERR scalar(keys %$weatherdata)." old records reloaded.\n" if (opt_get_int("debug") > 0); |
354
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
363 } |
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
364 } |
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
365 } |
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
366 |
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
367 |
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
368 ### |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
369 ### Fetch Tiehallinto data |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
370 ### |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
371 if (opt_chk_bool("opt_tiehallinto")) |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
372 { |
443
1f16d27c571d
fetch_weather: Make tiehallinto API urls configurable.
Matti Hamalainen <ccr@tnsp.org>
parents:
442
diff
changeset
|
373 my $uri = opt_get("tiehallinto_rw_url"); |
406
4c3abf052d01
fetch_weather: "use 5.018;" and some cleanups.
Matti Hamalainen <ccr@tnsp.org>
parents:
405
diff
changeset
|
374 print STDERR "Fetching Tiehallinto road weather data from ".$uri."\n" if (opt_get_int("debug") > 0); |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
375 my $res = fetch_http($uri); |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
376 if ($res->code >= 200 && $res->code <= 201) |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
377 { |
463
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
378 my $json_str = force_decode_utf8($res->decoded_content); |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
379 my $data = JSON->new->decode($json_str); |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
380 |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
381 if (!defined($data->{"dataUpdatedTime"}) || !defined($data->{"weatherStations"})) |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
382 { |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
383 print STDERR "ERROR: REST/JSON call result did not contain required data.\n"; |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
384 print STDERR $json_str."\n\n"; |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
385 } |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
386 else |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
387 { |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
388 # Check if we need to update the static meta data |
446
ba559d5d44db
fetch_weather: Rename the Tiehallinto static metadata cache filename setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
445
diff
changeset
|
389 my $meta_file = opt_get("tiehallinto_meta"); |
447
11124f9bf994
fetch_weather: Instead of checking the latest static metadata update timestamp against the weather data XML, have a setting of days until the metadata is considered stale.
Matti Hamalainen <ccr@tnsp.org>
parents:
446
diff
changeset
|
390 my $meta_stamp = (-e $meta_file) ? (stat($meta_file))[9] : -1; |
11124f9bf994
fetch_weather: Instead of checking the latest static metadata update timestamp against the weather data XML, have a setting of days until the metadata is considered stale.
Matti Hamalainen <ccr@tnsp.org>
parents:
446
diff
changeset
|
391 my $fetch_meta = ($meta_stamp + 60*60*24 * opt_get_int("tiehallinto_meta_period")) < time(); |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
392 |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
393 # Fetch or read the cache |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
394 my $meta_str; |
444 | 395 if ($fetch_meta || opt_chk_bool("force_update")) |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
396 { |
443
1f16d27c571d
fetch_weather: Make tiehallinto API urls configurable.
Matti Hamalainen <ccr@tnsp.org>
parents:
442
diff
changeset
|
397 my $uri = opt_get("tiehallinto_meta_url"); |
445 | 398 print STDERR "Fetching Tiehallinto static meta data from ".$uri."\n" if (opt_get_int("debug") > 1); |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
399 my $res = fetch_http($uri); |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
400 die("Failed to fetch $uri data.\n") unless ($res->code <= 200 && $res->code <= 201); |
339 | 401 |
463
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
402 $meta_str = force_decode_utf8($res->decoded_content); |
412
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
403 $fetch_meta = 1; |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
404 } |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
405 else |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
406 { |
445 | 407 print STDERR "Using CACHED Tiehallinto static meta data from '".$meta_file."'.\n" if (opt_get_int("debug") > 0); |
463
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
408 $meta_str = force_decode_utf8(read_text($meta_file)); |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
409 } |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
410 |
416
c88f7449b526
fetch_weather: Possibly another UTF8 fix :S
Matti Hamalainen <ccr@tnsp.org>
parents:
414
diff
changeset
|
411 print STDERR "Is meta_str UTF8? ".(utf8::is_utf8($meta_str) ? "yes" : "NO!")."\n" if (opt_get_int("debug") > 0); |
c88f7449b526
fetch_weather: Possibly another UTF8 fix :S
Matti Hamalainen <ccr@tnsp.org>
parents:
414
diff
changeset
|
412 |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
413 # Parse the data .. |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
414 my $meta_data = {}; |
412
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
415 my $json = JSON->new->decode($meta_str); |
398 | 416 |
412
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
417 if ($fetch_meta) |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
418 { |
412
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
419 # Save new cache, in more optimal form, if needed. |
445 | 420 print STDERR "Storing to cache '".$meta_file."'.\n" if (opt_get_int("debug") > 0); |
463
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
421 write_binary($meta_file, JSON->new->utf8->encode($json)); |
412
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
422 } |
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
423 |
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
424 foreach my $ms (@{$json->{"features"}}) |
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
425 { |
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
426 if (defined($ms->{"properties"}) && |
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
427 defined($ms->{"geometry"}{"coordinates"}) && |
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
428 defined($ms->{"properties"}{"names"}{"fi"})) |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
429 { |
412
e0d86ccef873
fetch_weather: Change to using JSON static weather station metadata from digitraffic instead of the "old" API's CSV format data.
Matti Hamalainen <ccr@tnsp.org>
parents:
411
diff
changeset
|
430 $meta_data->{$ms->{"id"}} = $ms; |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
431 } |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
432 } |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
433 |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
434 my $nrecords = 0; |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
435 foreach my $wdata (@{$data->{"weatherStations"}}) |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
436 { |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
437 my $wid = $wdata->{"id"}; |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
438 if (defined($meta_data->{$wid}) && defined($wdata->{"sensorValues"})) |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
439 { |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
440 $wdata->{"sensors"} = {}; |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
441 foreach my $sensor (@{$wdata->{"sensorValues"}}) |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
442 { |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
443 $wdata->{"sensors"}->{$sensor->{"oldName"}} = $sensor; |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
444 } |
414
d623652df6b5
fetch_weather / weather: Add weather measurement station location data (GPS WGS-84 lat/long/height) to weather cache. Not used in weather.tcl yet, tho.
Matti Hamalainen <ccr@tnsp.org>
parents:
413
diff
changeset
|
445 |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
446 $nrecords++; |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
447 $weatherdata->{$meta_data->{$wid}{"properties"}{"names"}{"fi"}} = |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
448 [ |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
449 # Measurement source type |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
450 1, |
414
d623652df6b5
fetch_weather / weather: Add weather measurement station location data (GPS WGS-84 lat/long/height) to weather cache. Not used in weather.tcl yet, tho.
Matti Hamalainen <ccr@tnsp.org>
parents:
413
diff
changeset
|
451 |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
452 # Basic data |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
453 $meta_data->{$wid}{"geometry"}{"coordinates"}[1], |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
454 $meta_data->{$wid}{"geometry"}{"coordinates"}[0], |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
455 $meta_data->{$wid}{"geometry"}{"coordinates"}[2], |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
456 str2time(plonk_data($wdata->{"measuredTime"})), |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
457 plonk_data($wdata->{"sensors"}->{"airtemperature1"}->{"sensorValue"}), |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
458 plonk_data($wdata->{"sensors"}->{"humidity"}->{"sensorValue"}), |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
459 plonk_data($wdata->{"sensors"}->{"averagewindspeed"}->{"sensorValue"}), |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
460 |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
461 # Station type dependant data |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
462 "", |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
463 plonk_data($wdata->{"sensors"}->{"roadsurfacetemperature1"}->{"sensorValue"}), |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
464 plonk_data($wdata->{"sensors"}->{"precipitation"}->{"sensorValueDescriptionFi"}), |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
465 plonk_data($wdata->{"sensors"}->{"visibility"}->{"sensorValue"}), |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
466 ]; |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
467 } |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
468 else |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
469 { |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
470 print STDERR "Station ID #".$wid." not defined?\n" if (opt_get_int("debug") > 0); |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
471 #.Dumper($meta_data->{$wid}); |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
472 } |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
473 } |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
474 print STDERR $nrecords." records from Tiehallinto.\n" if (opt_get_int("debug") > 0); |
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
475 } |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
476 } |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
477 } |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
478 |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
479 |
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
480 ### |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
481 ### Fetch FMI data |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
482 ### |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
483 if (opt_chk_bool("opt_fmi")) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
484 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
485 die("FMI data scrape enabled, but no API key set.\n") unless opt_chk_valid("fmi_api_key", 10); |
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
486 my @fmitems = ("temperature", "humidity", "windspeedms", "totalcloudcover"); |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
487 |
189 | 488 my $uri = "http://data.fmi.fi/fmi-apikey/".opt_get("fmi_api_key"). |
489 "/wfs?request=getFeature&storedquery_id=fmi::observations::weather::". | |
490 "multipointcoverage". | |
491 # "timevaluepair". | |
194 | 492 "&starttime=".format_time_gmt(time() - 10*60)."&endtime=".format_time_gmt(time()). |
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
493 "¶meters=".join(",", @fmitems)."&maxlocations=100&bbox=19,59,32,75"; |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
494 |
324
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
495 print STDERR "FMI URI: ".$uri."\n" if (opt_get_int("debug") > 0); |
189 | 496 |
497 my $res = fetch_http($uri); | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
498 if ($res->code >= 200 && $res->code <= 201) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
499 { |
463
fe478b7bd80e
fetch_weather: Perhaps, possibly, fix the UTF8 issues. Maybe.
Matti Hamalainen <ccr@tnsp.org>
parents:
462
diff
changeset
|
500 my $xml = XMLin(force_decode_utf8($res->decoded_content)); |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
501 my $time_base = time(); |
339 | 502 |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
503 if (defined($xml->{"wfs:member"}{"omso:GridSeriesObservation"})) |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
504 { |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
505 my $fdata = $xml->{"wfs:member"}{"omso:GridSeriesObservation"}; |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
506 my $fshit = $fdata->{"om:result"}{"gmlcov:MultiPointCoverage"}; |
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
507 |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
508 my @position_lines = split(/\n/, $fshit->{"gml:domainSet"}{"gmlcov:SimpleMultiPoint"}{"gmlcov:positions"}); |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
509 my @data_lines = split(/\n/, $fshit->{"gml:rangeSet"}{"gml:DataBlock"}{"gml:doubleOrNilReasonTupleList"}); |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
510 my @farray = (); |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
511 |
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
512 if (scalar(@position_lines) == scalar(@data_lines)) |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
513 { |
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
514 for (my $nline = 0; $nline < scalar(@position_lines); $nline++) |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
515 { |
382
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
516 my $dline = str_trim($data_lines[$nline]); |
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
517 my $pline = str_trim($position_lines[$nline]); |
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
518 |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
519 my @fmatches = ($dline =~ /\s*([\+\-]?\d+\.\d*|NaN)\s*/ig); |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
520 if (scalar(@fmatches) != scalar(@fmitems)) |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
521 { |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
522 print STDERR "Not enough items in scalar line #".$nline." (". |
452
ec97c29768e7
fetch_weather: Improve some debug messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
450
diff
changeset
|
523 scalar(@fmatches). " vs ".scalar(@fmitems)."): ".$dline."\n" |
ec97c29768e7
fetch_weather: Improve some debug messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
450
diff
changeset
|
524 if (opt_get_int("debug") > 0); |
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
525 } |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
526 else |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
527 { |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
528 my $vtmp = {}; |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
529 for (my $fni = 0; $fni < scalar(@fmitems); $fni++) |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
530 { |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
531 $$vtmp{$fmitems[$fni]} = $fmatches[$fni] if (lc($fmatches[$fni]) ne "nan"); |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
532 } |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
533 if ($pline =~ /^\s*([\+\-]?\d+\.\d*)\s+([\+\-]?\d+\.\d*)\s+(\d+)\s*$/) |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
534 { |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
535 $$vtmp{"lat"} = $1; |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
536 $$vtmp{"long"} = $2; |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
537 $$vtmp{"time"} = $3; |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
538 push(@farray, $vtmp); |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
539 } |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
540 else |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
541 { |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
542 print STDERR "Data mismatch #".$nline.": ".$pline."\n"; |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
543 } |
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
544 } |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
545 } |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
546 } |
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
547 else |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
548 { |
452
ec97c29768e7
fetch_weather: Improve some debug messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
450
diff
changeset
|
549 print STDERR "Position and data line counts do not match ". |
ec97c29768e7
fetch_weather: Improve some debug messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
450
diff
changeset
|
550 scalar(@position_lines)." <> ".scalar(@data_lines)."\n"; |
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
551 goto skip_it; |
189 | 552 } |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
553 # XXX Hashify the array into lat/long keys |
339 | 554 |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
555 # This is horrible :S |
352
19b363586b40
fetch_weather: Fix debug prints.
Matti Hamalainen <ccr@tnsp.org>
parents:
349
diff
changeset
|
556 my $nrecords = 0; |
410 | 557 foreach my $xnode (@{$fdata->{"om:featureOfInterest"}{"sams:SF_SpatialSamplingFeature"}{"sams:shape"}{"gml:MultiPoint"}{"gml:pointMember"}}) |
189 | 558 { |
228 | 559 my $floc = $xnode->{"gml:Point"}; |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
560 if ($floc->{"gml:pos"} =~ /^\s*([\+\-]?\d+\.\d*)\s+([\+\-]?\d+\.\d*)\s*$/) |
189 | 561 { |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
562 my ($flat, $flong) = ($1, $2); |
349
f7793ef7c925
fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
343
diff
changeset
|
563 |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
564 # Should use a hash - |
410 | 565 foreach my $frec (@farray) |
189 | 566 { |
410 | 567 if ($frec->{"lat"} == $flat && $frec->{"long"} == $flong && |
383
1ba1191da15d
fetch_weather: Check for empty station name.
Matti Hamalainen <ccr@tnsp.org>
parents:
382
diff
changeset
|
568 $floc->{"gml:name"} ne "") |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
569 { |
349
f7793ef7c925
fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
343
diff
changeset
|
570 $nrecords++; |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
571 $weatherdata->{$floc->{"gml:name"}} = |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
572 [ |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
573 # Measurement source type |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
574 2, |
410 | 575 |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
576 # Basic data |
414
d623652df6b5
fetch_weather / weather: Add weather measurement station location data (GPS WGS-84 lat/long/height) to weather cache. Not used in weather.tcl yet, tho.
Matti Hamalainen <ccr@tnsp.org>
parents:
413
diff
changeset
|
577 $frec->{"lat"}, |
d623652df6b5
fetch_weather / weather: Add weather measurement station location data (GPS WGS-84 lat/long/height) to weather cache. Not used in weather.tcl yet, tho.
Matti Hamalainen <ccr@tnsp.org>
parents:
413
diff
changeset
|
578 $frec->{"long"}, |
d623652df6b5
fetch_weather / weather: Add weather measurement station location data (GPS WGS-84 lat/long/height) to weather cache. Not used in weather.tcl yet, tho.
Matti Hamalainen <ccr@tnsp.org>
parents:
413
diff
changeset
|
579 0, |
410 | 580 plonk_data($frec->{"time"}), |
581 plonk_data($frec->{"temperature"}), | |
582 plonk_data($frec->{"humidity"}), | |
583 plonk_data($frec->{"windspeedms"}), | |
448
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
584 |
ac3b79eca0ca
fetch_weather: Change to use the new REST/JSON API for fetching Tiehallinto digitraffic weather data.
Matti Hamalainen <ccr@tnsp.org>
parents:
447
diff
changeset
|
585 # Station type dependant data |
410 | 586 translate_clouds(plonk_data($frec->{"totalcloudcover"})), |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
587 ]; |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
588 } |
189 | 589 } |
590 } | |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
591 } |
352
19b363586b40
fetch_weather: Fix debug prints.
Matti Hamalainen <ccr@tnsp.org>
parents:
349
diff
changeset
|
592 print STDERR $nrecords." records from FMI.\n" if (opt_get_int("debug") > 0); |
353 | 593 } |
594 else | |
595 { | |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
596 # defined |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
597 print STDERR "Invalid XML received:\n"; |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
598 print STDERR $res->decoded_content."\n\n"; |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
599 } |
353 | 600 } |
601 else | |
602 { | |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
603 print STDERR "Error fetching FMI XML: ".$res->status_line."\n"; |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
604 } |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
605 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
606 |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
607 |
375
c303ddc0a683
fetch_weather: Don't die() in case of bad XML from FMI, just skip parsing
Matti Hamalainen <ccr@tnsp.org>
parents:
374
diff
changeset
|
608 ### Skip here if the FMI shit fails due to broken data |
c303ddc0a683
fetch_weather: Don't die() in case of bad XML from FMI, just skip parsing
Matti Hamalainen <ccr@tnsp.org>
parents:
374
diff
changeset
|
609 skip_it: |
c303ddc0a683
fetch_weather: Don't die() in case of bad XML from FMI, just skip parsing
Matti Hamalainen <ccr@tnsp.org>
parents:
374
diff
changeset
|
610 |
c303ddc0a683
fetch_weather: Don't die() in case of bad XML from FMI, just skip parsing
Matti Hamalainen <ccr@tnsp.org>
parents:
374
diff
changeset
|
611 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
612 ### |
394
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
613 ### Purge too old entries |
371
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
614 ### |
394
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
615 if (opt_chk_valid("purge_threshold", 1)) |
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
616 { |
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
617 my $purge = opt_get_int("purge_threshold"); |
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
618 if ($purge > 0) |
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
619 { |
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
620 my $wqtime = time(); |
395
33b5e7adc30c
fetch_weather: Make purge_threshold output some information when debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
394
diff
changeset
|
621 my $nold = scalar(keys %$weatherdata); |
373
bc7746ca622e
fetch_weather: Implement purge_threshold setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
371
diff
changeset
|
622 |
394
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
623 foreach my $key (keys %$weatherdata) |
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
624 { |
414
d623652df6b5
fetch_weather / weather: Add weather measurement station location data (GPS WGS-84 lat/long/height) to weather cache. Not used in weather.tcl yet, tho.
Matti Hamalainen <ccr@tnsp.org>
parents:
413
diff
changeset
|
625 if ($wqtime - $weatherdata->{$key}[4] > (60 * $purge)) |
394
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
626 { |
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
627 delete $$weatherdata{$key}; |
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
628 } |
315f31e4f964
fetch_weather: Make purge_threshold option entirely optional, e.g. not set or 0 will disable purging.
Matti Hamalainen <ccr@tnsp.org>
parents:
392
diff
changeset
|
629 } |
395
33b5e7adc30c
fetch_weather: Make purge_threshold output some information when debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
394
diff
changeset
|
630 |
33b5e7adc30c
fetch_weather: Make purge_threshold output some information when debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
394
diff
changeset
|
631 my $nnew = scalar(keys %$weatherdata); |
33b5e7adc30c
fetch_weather: Make purge_threshold output some information when debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
394
diff
changeset
|
632 print STDERR "Purged data older than ".$purge." minutes, ".$nold." -> ".$nnew." = ".($nold - $nnew)." removed.\n" if (opt_get_int("debug") > 0); |
371
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
633 } |
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
634 } |
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
635 |
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
636 |
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
637 ### |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
638 ### Output |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
639 ### |
353 | 640 if (opt_chk_valid("outfile", 1)) |
641 { | |
438
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
642 my $filename = opt_get("outfile"); |
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
643 print STDERR "Dumping data to output file '".$filename."'\n" if (opt_get_int("debug") > 0); |
368
ed1c7f2d81df
fetch_weather: Close STDOUT filehandle before reopening it, to make sure we
Matti Hamalainen <ccr@tnsp.org>
parents:
367
diff
changeset
|
644 close(STDOUT); |
438
a17b7fdd03c0
fetch_weather: Simplify and check for existence of output file before trying to read the old one.
Matti Hamalainen <ccr@tnsp.org>
parents:
417
diff
changeset
|
645 open(STDOUT, '>', $filename) or die("Could not open output file '".$filename."': $!\n"); |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
646 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
647 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
648 binmode STDOUT, ':encoding(utf-8)'; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
649 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
650 foreach my $key (sort { $a cmp $b } keys %$weatherdata) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
651 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
652 print STDOUT $key."|".join("|", @{$weatherdata->{$key}})."\n"; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
653 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
654 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
655 close(STDOUT); |