Mercurial > hg > egg-tcls
annotate fetch_weather.pl @ 383:1ba1191da15d
fetch_weather: Check for empty station name.
| author | Matti Hamalainen <ccr@tnsp.org> |
|---|---|
| date | Sun, 07 Feb 2016 18:49:36 +0200 |
| parents | 5573c1681da9 |
| children | a375c276ec4d |
| rev | line source |
|---|---|
| 381 | 1 #!/usr/bin/perl -w |
| 182 | 2 ########################################################################## |
| 3 # | |
|
359
3b8ffc6eca6d
fetch_weather: Bump version.
Matti Hamalainen <ccr@tnsp.org>
parents:
356
diff
changeset
|
4 # Fetch Weather v0.6 by Matti 'ccr' Hamalainen <ccr@tnsp.org> |
|
233
dc1afa011883
weather: Bump version, copyright.
Matti Hamalainen <ccr@tnsp.org>
parents:
232
diff
changeset
|
5 # (C) Copyright 2014-2015 Tecnic Software productions (TNSP) |
| 206 | 6 # This script is freely distributable under GNU GPL (version 2) license. |
| 182 | 7 # |
| 8 # Should be ran as a cronjob, and configured properly. | |
| 206 | 9 # */10 * * * * perl -w /absolute/path/to/fetch_weather.pl /path/to/configfile |
| 182 | 10 # |
|
186
b41c525bdaf7
Add example configuration file.
Matti Hamalainen <ccr@tnsp.org>
parents:
185
diff
changeset
|
11 # Configuration file example is in fetch_weather.config |
| 182 | 12 # |
| 206 | 13 # Requires various Perl modules, in Debian the packages should be: |
| 14 # libwww-perl libxml-simple-perl libtimedate-perl | |
| 15 # | |
| 182 | 16 # |
| 17 ########################################################################## | |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
18 use strict; |
|
380
729eb88d1c63
fetch_weather: "use warnings".
Matti Hamalainen <ccr@tnsp.org>
parents:
379
diff
changeset
|
19 use warnings; |
| 243 | 20 use utf8; |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
21 use LWP::UserAgent; |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
22 use HTML::Entities; |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
23 use XML::Simple; |
| 189 | 24 use Date::Format; |
|
199
95012f8f687a
Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents:
198
diff
changeset
|
25 use Date::Parse; |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
26 use Data::Dumper; |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
27 use File::Slurp; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
28 use Text::CSV; |
| 166 | 29 |
|
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
30 ### |
| 182 | 31 ### Configuration settings |
|
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
32 ### |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
33 my %settings = ( |
| 166 | 34 "debug" => 0, |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
35 "opt_fmi" => 0, |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
36 "opt_tiehallinto" => 0, |
|
374
26b7963e7fbd
fetch_weather: Oops, forgot to define the configuration setting
Matti Hamalainen <ccr@tnsp.org>
parents:
373
diff
changeset
|
37 "purge_threshold" => 60, |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
38 "fmi_api_key" => "", |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
39 "outfile" => "", |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
40 "tiehallinto_static_meta" => "tiehallinto.meta", |
|
185
64e06ee172c7
Add HTTP user agent setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
182
diff
changeset
|
41 "http_user_agent" => "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 6.0) Opera 10.63 [en]", |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
42 ); |
|
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
43 |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
44 |
|
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
45 ### |
|
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
46 ### Helper functions |
|
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
47 ### |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
48 sub mlog($) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
49 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
50 print STDERR $_[0]; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
51 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
52 |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
53 |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
54 sub fetch_http($) |
|
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
55 { |
| 198 | 56 my $agent = LWP::UserAgent->new; |
| 57 $agent->agent($settings{"http_user_agent"}); | |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
58 $agent->timeout(20); |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
59 |
| 198 | 60 my $req = HTTP::Request->new(GET => $_[0]); |
| 339 | 61 |
| 198 | 62 return $agent->request($req); |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
63 } |
|
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
64 |
|
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
65 |
|
382
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
66 sub str_trim($) |
|
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
67 { |
|
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
68 my $tmp = $_[0]; |
|
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
69 $tmp =~ s/^\s*//; |
|
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
70 $tmp =~ s/\s*$//; |
|
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
71 return $tmp; |
|
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
72 } |
|
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
73 |
|
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
74 |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
75 sub parse_timestamp($$) |
|
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
76 { |
|
196
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
77 my ($str, $offs) = @_; |
|
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
78 if ($str =~ /^(\d+):(\d+)$/) |
|
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
79 { |
|
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
80 return $offs + (60 * 60 * $1) + ($2 * 60); |
|
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
81 } |
|
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
82 else |
|
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
83 { |
|
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
84 return $offs; |
|
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
85 } |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
86 } |
|
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
87 |
|
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
88 |
| 194 | 89 sub format_time_gmt($) |
| 189 | 90 { |
| 91 # 2012-02-27T00:00:00Z | |
|
199
95012f8f687a
Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents:
198
diff
changeset
|
92 return time2str("%Y-%m-%dT%TZ", $_[0], "UTC"); |
| 189 | 93 } |
| 94 | |
| 193 | 95 |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
96 ### |
|
331
b8ae5e99341b
fetc_weather: Remove support for old HTML format parsing of Tiehallinto data, it's not provided in this format anymore.
Matti Hamalainen <ccr@tnsp.org>
parents:
325
diff
changeset
|
97 ### Helper functions |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
98 ### |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
99 |
|
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
100 my %th_rain_states = |
|
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
101 ( |
| 237 | 102 "Pouta" => "poutaa", |
| 103 "Heikko" => "heikkoa sadetta", | |
| 104 "Kohtalainen" => "kohtalaista sadetta", | |
| 105 "Voimakas" => "voimakasta sadetta", | |
|
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
106 ); |
|
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
107 |
|
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
108 my $th_rain_states_k = join("|", map {quotemeta} sort { length($b)<=>length($a) } keys %th_rain_states); |
|
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
109 |
| 242 | 110 sub translate_rain($) |
|
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
111 { |
|
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
112 my $tmp = $_[0]; |
| 237 | 113 $tmp =~ s/($th_rain_states_k)/$th_rain_states{$1}/igo; |
|
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
114 return $tmp; |
|
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
115 } |
|
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
116 |
|
331
b8ae5e99341b
fetc_weather: Remove support for old HTML format parsing of Tiehallinto data, it's not provided in this format anymore.
Matti Hamalainen <ccr@tnsp.org>
parents:
325
diff
changeset
|
117 |
| 244 | 118 my %th_cloud_states = |
| 119 ( | |
| 120 0 => "selkeää", | |
|
247
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
121 1 => "melkein selkeää", |
|
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
122 2 => "verrattain selkeää", |
|
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
123 3 => "verrattain selkeää", |
|
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
124 4 => "puolipilvistä", |
|
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
125 5 => "verrattain pilvistä", |
|
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
126 6 => "verrattain pilvistä", |
| 244 | 127 7 => "melkein pilvistä", |
| 128 8 => "pilvistä", | |
| 129 ); | |
| 130 | |
| 131 sub translate_clouds($) | |
| 132 { | |
|
247
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
133 return "" if ($_[0] eq "NaN" || $_[0] eq ""); |
|
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
134 my $tmp = int($_[0]); |
| 244 | 135 foreach my $n (sort { $a <=> $b } keys %th_cloud_states) |
| 136 { | |
|
247
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
137 return $th_cloud_states{$n}." (".$n."/8)" if ($tmp == $n); |
| 244 | 138 } |
| 139 return $tmp; | |
| 140 } | |
| 141 | |
|
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
142 |
| 228 | 143 sub plonk_data($) |
| 144 { | |
| 145 return defined($_[0]) ? $_[0] : ""; | |
| 146 } | |
| 147 | |
| 148 | |
| 237 | 149 sub plonk_data_lc($) |
| 150 { | |
| 151 return defined($_[0]) ? lc($_[0]) : ""; | |
| 152 } | |
| 153 | |
| 154 | |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
155 ### |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
156 ### Configuration handling |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
157 ### |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
158 sub opt_chk_bool($) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
159 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
160 if (defined($settings{$_[0]})) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
161 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
162 my $val = $settings{$_[0]}; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
163 return ($val == 1 || $val eq "true" || $val eq "on" || $val eq "1"); |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
164 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
165 else |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
166 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
167 return 0; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
168 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
169 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
170 |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
171 |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
172 sub opt_chk_valid($$) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
173 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
174 if (defined($settings{$_[0]})) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
175 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
176 my $val = $settings{$_[0]}; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
177 return length($val) >= $_[1]; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
178 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
179 else |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
180 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
181 return 0; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
182 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
183 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
184 |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
185 |
|
324
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
186 sub opt_get_int($) |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
187 { |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
188 if (defined($settings{$_[0]})) |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
189 { |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
190 return int($settings{$_[0]}); |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
191 } |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
192 else |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
193 { |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
194 return -1; |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
195 } |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
196 } |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
197 |
|
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
198 |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
199 sub opt_get($) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
200 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
201 if (defined($settings{$_[0]})) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
202 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
203 return $settings{$_[0]}; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
204 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
205 else |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
206 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
207 return undef; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
208 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
209 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
210 |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
211 |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
212 sub opt_read_config($) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
213 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
214 my $filename = $_[0]; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
215 my $errors = 0; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
216 my $line = 0; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
217 |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
218 open(CONFFILE, "<", $filename) or die("Could not open configuration '".$filename."'!\n"); |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
219 while (<CONFFILE>) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
220 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
221 $line++; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
222 chomp; |
| 353 | 223 if (/(^\s*#|^\s*$)/) |
| 224 { | |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
225 # Ignore comments and empty lines |
| 353 | 226 } |
| 227 elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*(\d+),?\s*$/) { | |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
228 my $key = lc($1); |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
229 my $value = $2; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
230 if (defined($settings{$key})) { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
231 $settings{$key} = $value; |
| 353 | 232 } |
| 233 else | |
| 234 { | |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
235 mlog("[$filename:$line] Unknown setting '$key' = $value\n"); |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
236 $errors = 1; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
237 } |
| 353 | 238 } |
| 239 elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*\"(.*?)\",?\s*$/) { | |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
240 my $key = lc($1); |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
241 my $value = $2; |
| 353 | 242 if (defined($settings{$key})) |
| 243 { | |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
244 $settings{$key} = $value; |
| 353 | 245 } |
| 246 else | |
| 247 { | |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
248 mlog("[$filename:$line] Unknown setting '$key' = '$value'\n"); |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
249 $errors = 1; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
250 } |
| 353 | 251 } |
| 252 else | |
| 253 { | |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
254 mlog("[$filename:$line] Syntax error: $_\n"); |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
255 $errors = 1; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
256 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
257 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
258 close(CONFFILE); |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
259 return $errors; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
260 } |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
261 |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
262 |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
263 ### |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
264 ### Main program begins |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
265 ### |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
266 my $weatherdata = {}; |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
267 |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
268 die( |
|
359
3b8ffc6eca6d
fetch_weather: Bump version.
Matti Hamalainen <ccr@tnsp.org>
parents:
356
diff
changeset
|
269 "Weather Fetch v0.6 by ccr/TNSP <ccr\@tnsp.org>\n". |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
270 "Usage: $0 <config file>\n" |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
271 ) unless scalar(@ARGV) >= 1; |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
272 |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
273 my $cfgfile = shift; |
|
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
274 opt_read_config($cfgfile) == 0 or die("Errors while parsing configuration file '".$cfgfile."'.\n"); |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
275 |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
276 |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
277 ### |
|
354
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
278 ### Load already cached data |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
279 ### |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
280 if (opt_chk_valid("outfile", 1)) |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
281 { |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
282 my $str = read_file(opt_get("outfile"), binmode => ':utf8', err_mode => 'quiet'); |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
283 if (defined($str)) |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
284 { |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
285 foreach my $line (split(/\s*\n\s*/, $str)) |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
286 { |
|
367
1ca8ae195077
fetch_weather: Fix parsing of old/cached datafile.
Matti Hamalainen <ccr@tnsp.org>
parents:
363
diff
changeset
|
287 my @mtmp = split(/\|/, $line, -1); |
|
356
0de6e0c2c89c
fetch_weather: Adjust minimum acceptable recached entry array size.
Matti Hamalainen <ccr@tnsp.org>
parents:
355
diff
changeset
|
288 if (scalar(\@mtmp) >= 3) |
|
354
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
289 { |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
290 $weatherdata->{shift @mtmp} = \@mtmp; |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
291 } |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
292 } |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
293 print STDERR scalar(keys %$weatherdata)." old records reloaded.\n" if (opt_get_int("debug") > 0); |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
294 } |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
295 } |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
296 |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
297 |
|
de411e7834aa
fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents:
353
diff
changeset
|
298 ### |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
299 ### Fetch Tiehallinto data |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
300 ### |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
301 if (opt_chk_bool("opt_tiehallinto")) |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
302 { |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
303 my $uri = "http://tie.digitraffic.fi/sujuvuus/ws/roadWeather"; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
304 my $res = fetch_http($uri); |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
305 if ($res->code >= 200 && $res->code <= 201) |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
306 { |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
307 my $xml = XMLin($res->decoded_content); |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
308 |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
309 if (!defined($xml->{"soap:Body"}) || !defined($xml->{"soap:Body"}{"RoadWeatherResponse"})) |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
310 { |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
311 print STDERR "ERROR: SOAP call result did not contain required data.\n"; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
312 print STDERR $res->decoded_content."\n\n"; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
313 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
314 else |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
315 { |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
316 my $data = $xml->{"soap:Body"}{"RoadWeatherResponse"}; |
|
349
f7793ef7c925
fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
343
diff
changeset
|
317 print STDERR "Checking Tiehallinto response from ".$uri."\n" if (opt_get_int("debug") > 0); |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
318 |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
319 # Check if we need to update the static meta data |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
320 my $meta_file = opt_get("tiehallinto_static_meta"); |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
321 my $fetch_meta = (-e $meta_file) ? 0 : 1; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
322 |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
323 if (defined($data->{"laststaticdataupdate"})) |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
324 { |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
325 # Compare metadata cache file modification timestamp to info in XML |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
326 my $tmp1 = str2time($data->{"laststaticdataupdate"}); |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
327 my $tmp2 = (-e $meta_file) ? (stat($meta_file))[9] : -1; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
328 $fetch_meta = 1 unless ($tmp1 < $tmp2); |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
329 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
330 |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
331 # Fetch or read the cache |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
332 my $meta_str; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
333 if ($fetch_meta) |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
334 { |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
335 print STDERR "Fetching Tiehallinto static meta data.\n" if (opt_get_int("debug") > 0); |
|
377
0a6d9f0f0a57
fetch_weather: Fix Tiehallinto metadata handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
375
diff
changeset
|
336 my $uri = "https://raw.githubusercontent.com/finnishtransportagency/metadata/master/csv/meta_rws_stations.csv"; |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
337 my $res = fetch_http($uri); |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
338 die("Failed to fetch $uri data.\n") unless ($res->code <= 200 && $res->code <= 201); |
| 339 | 339 |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
340 print STDERR "Storing to cache '$meta_file'.\n" if (opt_get_int("debug") > 0); |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
341 $meta_str = $res->decoded_content; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
342 write_file($meta_file, {binmode => ':utf8'}, $meta_str); |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
343 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
344 else |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
345 { |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
346 print STDERR "Using CACHED Tiehallinto static meta data from '$meta_file'.\n" if (opt_get_int("debug") > 0); |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
347 $meta_str = read_file($meta_file, binmode => ':utf8'); |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
348 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
349 |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
350 # Parse the data .. |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
351 my $meta_data = {}; |
|
377
0a6d9f0f0a57
fetch_weather: Fix Tiehallinto metadata handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
375
diff
changeset
|
352 my $csv = Text::CSV->new({blank_is_undef => 1, decode_utf8 => 1, sep_char => ";"}); |
|
0a6d9f0f0a57
fetch_weather: Fix Tiehallinto metadata handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
375
diff
changeset
|
353 # my $csv = Text::CSV->new({blank_is_undef => 1}); |
|
343
3fff571a904b
fetch_weather: Some fixes to make this actually work.
Matti Hamalainen <ccr@tnsp.org>
parents:
340
diff
changeset
|
354 die("Failed to instantiate Text::CSV object?\n") unless defined($csv); |
| 355 | 355 |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
356 foreach my $line (split(/\s*\n\s*/, $meta_str)) |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
357 { |
|
343
3fff571a904b
fetch_weather: Some fixes to make this actually work.
Matti Hamalainen <ccr@tnsp.org>
parents:
340
diff
changeset
|
358 if (defined($line) && $csv->parse($line)) |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
359 { |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
360 my @fields = $csv->fields(); |
|
363
3917a1515e31
fetch_weather: Fix parsing of "empty" CSV data.
Matti Hamalainen <ccr@tnsp.org>
parents:
359
diff
changeset
|
361 if (scalar(@fields) > 1) |
|
3917a1515e31
fetch_weather: Fix parsing of "empty" CSV data.
Matti Hamalainen <ccr@tnsp.org>
parents:
359
diff
changeset
|
362 { |
|
377
0a6d9f0f0a57
fetch_weather: Fix Tiehallinto metadata handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
375
diff
changeset
|
363 $$meta_data{$fields[0]} = \@fields; |
|
363
3917a1515e31
fetch_weather: Fix parsing of "empty" CSV data.
Matti Hamalainen <ccr@tnsp.org>
parents:
359
diff
changeset
|
364 } |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
365 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
366 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
367 |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
368 # Parse XML and combine with the station meta data |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
369 if (defined($data->{"roadweatherdata"})) |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
370 { |
|
349
f7793ef7c925
fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
343
diff
changeset
|
371 my $nrecords = 0; |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
372 foreach my $wdata (@{$data->{"roadweatherdata"}{"roadweather"}}) |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
373 { |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
374 my $wid = $wdata->{"stationid"}; |
|
383
1ba1191da15d
fetch_weather: Check for empty station name.
Matti Hamalainen <ccr@tnsp.org>
parents:
382
diff
changeset
|
375 if (defined($meta_data->{$wid}) && $meta_data->{$wid}[2] ne "") |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
376 { |
|
349
f7793ef7c925
fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
343
diff
changeset
|
377 $nrecords++; |
|
379
3715df1cd9aa
fetch_weather: Use Finnish location names instead of Swedish ones from static Tiehallinto station metadata.
Matti Hamalainen <ccr@tnsp.org>
parents:
378
diff
changeset
|
378 $weatherdata->{$meta_data->{$wid}[2]} = |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
379 [ |
|
343
3fff571a904b
fetch_weather: Some fixes to make this actually work.
Matti Hamalainen <ccr@tnsp.org>
parents:
340
diff
changeset
|
380 1, |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
381 str2time(plonk_data($wdata->{"measurementtime"}{"utc"})), |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
382 plonk_data($wdata->{"airtemperature1"}), |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
383 |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
384 plonk_data($wdata->{"humidity"}), |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
385 plonk_data($wdata->{"averagewindspeed"}), |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
386 ]; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
387 } |
|
377
0a6d9f0f0a57
fetch_weather: Fix Tiehallinto metadata handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
375
diff
changeset
|
388 else |
|
0a6d9f0f0a57
fetch_weather: Fix Tiehallinto metadata handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
375
diff
changeset
|
389 { |
|
0a6d9f0f0a57
fetch_weather: Fix Tiehallinto metadata handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
375
diff
changeset
|
390 print STDERR "Station ID #".$wid." not defined?\n"; |
|
0a6d9f0f0a57
fetch_weather: Fix Tiehallinto metadata handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
375
diff
changeset
|
391 } |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
392 } |
|
349
f7793ef7c925
fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
343
diff
changeset
|
393 print STDERR $nrecords." records from Tiehallinto.\n" if (opt_get_int("debug") > 0); |
|
338
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
394 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
395 else |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
396 { |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
397 print STDERR "ERROR: Invalid (or unsupported) road weather data blob.\n"; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
398 print STDERR $res->decoded_content."\n\n"; |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
399 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
400 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
401 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
402 } |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
403 |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
404 |
|
afb8c0e2d513
fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents:
332
diff
changeset
|
405 ### |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
406 ### Fetch FMI data |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
407 ### |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
408 if (opt_chk_bool("opt_fmi")) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
409 { |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
410 die("FMI data scrape enabled, but no API key set.\n") unless opt_chk_valid("fmi_api_key", 10); |
|
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
411 my @fmitems = ("temperature", "humidity", "windspeedms", "totalcloudcover"); |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
412 |
| 189 | 413 my $uri = "http://data.fmi.fi/fmi-apikey/".opt_get("fmi_api_key"). |
| 414 "/wfs?request=getFeature&storedquery_id=fmi::observations::weather::". | |
| 415 "multipointcoverage". | |
| 416 # "timevaluepair". | |
| 194 | 417 "&starttime=".format_time_gmt(time() - 10*60)."&endtime=".format_time_gmt(time()). |
|
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
418 "¶meters=".join(",", @fmitems)."&maxlocations=100&bbox=19,59,32,75"; |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
419 |
|
324
19c64798de91
weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
247
diff
changeset
|
420 print STDERR "FMI URI: ".$uri."\n" if (opt_get_int("debug") > 0); |
| 189 | 421 |
| 422 my $res = fetch_http($uri); | |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
423 if ($res->code >= 200 && $res->code <= 201) |
|
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
424 { |
|
170
e4faa3746869
Remove useless parameters from XMLin().
Matti Hamalainen <ccr@tnsp.org>
parents:
169
diff
changeset
|
425 my $xml = XMLin($res->decoded_content); |
|
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
426 my $time_base = time(); |
| 339 | 427 |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
428 if (defined($xml->{"wfs:member"}{"omso:GridSeriesObservation"})) |
|
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
429 { |
|
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
430 my $fdata = $xml->{"wfs:member"}{"omso:GridSeriesObservation"}; |
|
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
431 my $fshit = $fdata->{"om:result"}{"gmlcov:MultiPointCoverage"}; |
|
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
432 |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
433 my @position_lines = split(/\n/, $fshit->{"gml:domainSet"}{"gmlcov:SimpleMultiPoint"}{"gmlcov:positions"}); |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
434 my @data_lines = split(/\n/, $fshit->{"gml:rangeSet"}{"gml:DataBlock"}{"gml:doubleOrNilReasonTupleList"}); |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
435 my @farray = (); |
|
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
436 |
|
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
437 if (scalar(@position_lines) == scalar(@data_lines)) |
|
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
438 { |
|
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
439 for (my $nline = 0; $nline < scalar(@position_lines); $nline++) |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
440 { |
|
382
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
441 my $dline = str_trim($data_lines[$nline]); |
|
5573c1681da9
fetch_weather: Trim split input data lines in FMI XML.
Matti Hamalainen <ccr@tnsp.org>
parents:
381
diff
changeset
|
442 my $pline = str_trim($position_lines[$nline]); |
|
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
443 |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
444 my @fmatches = ($dline =~ /\s*([\+\-]?\d+\.\d*|NaN)\s*/ig); |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
445 if (scalar(@fmatches) != scalar(@fmitems)) |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
446 { |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
447 print STDERR "Not enough items in scalar line #".$nline." (". |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
448 scalar(@fmatches). " vs ".scalar(@fmitems)."): ".$dline."\n"; |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
449 } |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
450 else |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
451 { |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
452 my $vtmp = {}; |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
453 for (my $fni = 0; $fni < scalar(@fmitems); $fni++) |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
454 { |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
455 $$vtmp{$fmitems[$fni]} = $fmatches[$fni] if (lc($fmatches[$fni]) ne "nan"); |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
456 } |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
457 if ($pline =~ /^\s*([\+\-]?\d+\.\d*)\s+([\+\-]?\d+\.\d*)\s+(\d+)\s*$/) |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
458 { |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
459 $$vtmp{"lat"} = $1; |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
460 $$vtmp{"long"} = $2; |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
461 $$vtmp{"time"} = $3; |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
462 push(@farray, $vtmp); |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
463 } |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
464 else |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
465 { |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
466 print STDERR "Data mismatch #".$nline.": ".$pline."\n"; |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
467 } |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
468 } |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
469 } |
|
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
470 } |
|
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
471 else |
|
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
472 { |
|
378
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
473 print STDERR "Position and data lines do not match.\n"; |
|
b706e8ebecf8
fetch_weather: Possibly improve FMI XML mess parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
377
diff
changeset
|
474 goto skip_it; |
| 189 | 475 } |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
476 # XXX Hashify the array into lat/long keys |
| 339 | 477 |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
478 # This is horrible :S |
|
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
479 my $fcrap = $fdata->{"om:featureOfInterest"}{"sams:SF_SpatialSamplingFeature"}{"sams:shape"}{"gml:MultiPoint"}{"gml:pointMember"}; |
|
352
19b363586b40
fetch_weather: Fix debug prints.
Matti Hamalainen <ccr@tnsp.org>
parents:
349
diff
changeset
|
480 my $nrecords = 0; |
| 228 | 481 foreach my $xnode (@{$fcrap}) |
| 189 | 482 { |
| 228 | 483 my $floc = $xnode->{"gml:Point"}; |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
484 if ($floc->{"gml:pos"} =~ /^\s*([\+\-]?\d+\.\d*)\s+([\+\-]?\d+\.\d*)\s*$/) |
| 189 | 485 { |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
486 my ($flat, $flong) = ($1, $2); |
|
349
f7793ef7c925
fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
343
diff
changeset
|
487 |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
488 # Should use a hash - |
|
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
489 foreach my $flol (@farray) |
| 189 | 490 { |
|
383
1ba1191da15d
fetch_weather: Check for empty station name.
Matti Hamalainen <ccr@tnsp.org>
parents:
382
diff
changeset
|
491 if ($flol->{"lat"} == $flat && $flol->{"long"} == $flong && |
|
1ba1191da15d
fetch_weather: Check for empty station name.
Matti Hamalainen <ccr@tnsp.org>
parents:
382
diff
changeset
|
492 $floc->{"gml:name"} ne "") |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
493 { |
|
349
f7793ef7c925
fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
343
diff
changeset
|
494 $nrecords++; |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
495 $weatherdata->{$floc->{"gml:name"}} = |
|
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
496 [ |
|
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
497 1, |
| 228 | 498 plonk_data($flol->{"time"}), |
| 237 | 499 plonk_data($flol->{"temperature"}), |
|
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
500 |
| 228 | 501 plonk_data($flol->{"humidity"}), |
| 237 | 502 plonk_data($flol->{"windspeedms"}), |
| 244 | 503 translate_clouds(plonk_data($flol->{"totalcloudcover"})), |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
504 ]; |
|
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
505 } |
| 189 | 506 } |
| 507 } | |
|
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
508 } |
|
352
19b363586b40
fetch_weather: Fix debug prints.
Matti Hamalainen <ccr@tnsp.org>
parents:
349
diff
changeset
|
509 print STDERR $nrecords." records from FMI.\n" if (opt_get_int("debug") > 0); |
| 353 | 510 } |
| 511 else | |
| 512 { | |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
513 # defined |
|
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
514 print STDERR "Invalid XML received:\n"; |
|
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
515 print STDERR $res->decoded_content."\n\n"; |
|
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
516 } |
| 353 | 517 } |
| 518 else | |
| 519 { | |
|
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
520 print STDERR "Error fetching FMI XML: ".$res->status_line."\n"; |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
521 } |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
522 } |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
523 |
|
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
524 |
|
375
c303ddc0a683
fetch_weather: Don't die() in case of bad XML from FMI, just skip parsing
Matti Hamalainen <ccr@tnsp.org>
parents:
374
diff
changeset
|
525 ### Skip here if the FMI shit fails due to broken data |
|
c303ddc0a683
fetch_weather: Don't die() in case of bad XML from FMI, just skip parsing
Matti Hamalainen <ccr@tnsp.org>
parents:
374
diff
changeset
|
526 skip_it: |
|
c303ddc0a683
fetch_weather: Don't die() in case of bad XML from FMI, just skip parsing
Matti Hamalainen <ccr@tnsp.org>
parents:
374
diff
changeset
|
527 |
|
c303ddc0a683
fetch_weather: Don't die() in case of bad XML from FMI, just skip parsing
Matti Hamalainen <ccr@tnsp.org>
parents:
374
diff
changeset
|
528 |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
529 ### |
|
373
bc7746ca622e
fetch_weather: Implement purge_threshold setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
371
diff
changeset
|
530 ### Purge very old entries, default to 60 minutes |
|
371
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
531 ### |
|
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
532 my $wqtime = time(); |
|
373
bc7746ca622e
fetch_weather: Implement purge_threshold setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
371
diff
changeset
|
533 my $purge = opt_get_int("purge_threshold"); |
|
bc7746ca622e
fetch_weather: Implement purge_threshold setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
371
diff
changeset
|
534 $purge = 60 if ($purge <= 0); |
|
bc7746ca622e
fetch_weather: Implement purge_threshold setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
371
diff
changeset
|
535 |
|
371
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
536 foreach my $key (keys %$weatherdata) |
|
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
537 { |
|
373
bc7746ca622e
fetch_weather: Implement purge_threshold setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
371
diff
changeset
|
538 if ($wqtime - $weatherdata->{$key}[1] > (60 * $purge)) |
|
371
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
539 { |
|
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
540 delete $$weatherdata{$key}; |
|
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
541 } |
|
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
542 } |
|
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
543 |
|
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
544 |
|
36869897f21d
fetch_weather: Purge sufficiently old data.
Matti Hamalainen <ccr@tnsp.org>
parents:
370
diff
changeset
|
545 ### |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
546 ### Output |
|
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
547 ### |
| 353 | 548 if (opt_chk_valid("outfile", 1)) |
| 549 { | |
|
369
86adad909681
fetch_weather: Improve debugging and one error message.
Matti Hamalainen <ccr@tnsp.org>
parents:
368
diff
changeset
|
550 print STDERR "Dumping data to ".opt_get("outfile")."\n" if (opt_get_int("debug") > 0); |
|
368
ed1c7f2d81df
fetch_weather: Close STDOUT filehandle before reopening it, to make sure we
Matti Hamalainen <ccr@tnsp.org>
parents:
367
diff
changeset
|
551 close(STDOUT); |
|
369
86adad909681
fetch_weather: Improve debugging and one error message.
Matti Hamalainen <ccr@tnsp.org>
parents:
368
diff
changeset
|
552 open(STDOUT, '>', opt_get("outfile")) or die("Could not open output file '".opt_get("outfile")."': $!\n"); |
|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
553 } |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
554 |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
555 binmode STDOUT, ':encoding(utf-8)'; |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
556 |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
557 foreach my $key (sort { $a cmp $b } keys %$weatherdata) |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
558 { |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
559 print STDOUT $key."|".join("|", @{$weatherdata->{$key}})."\n"; |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
560 } |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
561 |
|
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
562 close(STDOUT); |