Mercurial > hg > egg-tcls

annotate fetch_weather.pl @ 368:ed1c7f2d81df

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fetch_weather: Close STDOUT filehandle before reopening it, to make sure we are actually writing data to the intended output.
author Matti Hamalainen <ccr@tnsp.org>
date Fri, 18 Dec 2015 10:09:32 +0200
parents 1ca8ae195077
children 86adad909681
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
1 #!/usr/bin/perl -w
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
2 ##########################################################################
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
3 #
359
3b8ffc6eca6d fetch_weather: Bump version.
Matti Hamalainen <ccr@tnsp.org>
parents: 356
diff changeset
4 # Fetch Weather v0.6 by Matti 'ccr' Hamalainen <ccr@tnsp.org>
233
dc1afa011883 weather: Bump version, copyright.
Matti Hamalainen <ccr@tnsp.org>
parents: 232
diff changeset
5 # (C) Copyright 2014-2015 Tecnic Software productions (TNSP)
206
2ac661d551b3 Update documentation.
Matti Hamalainen <ccr@tnsp.org>
parents: 201
diff changeset
6 # This script is freely distributable under GNU GPL (version 2) license.
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
7 #
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
8 # Should be ran as a cronjob, and configured properly.
206
2ac661d551b3 Update documentation.
Matti Hamalainen <ccr@tnsp.org>
parents: 201
diff changeset
9 # */10 * * * * perl -w /absolute/path/to/fetch_weather.pl /path/to/configfile
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
10 #
186
b41c525bdaf7 Add example configuration file.
Matti Hamalainen <ccr@tnsp.org>
parents: 185
diff changeset
11 # Configuration file example is in fetch_weather.config
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
12 #
206
2ac661d551b3 Update documentation.
Matti Hamalainen <ccr@tnsp.org>
parents: 201
diff changeset
13 # Requires various Perl modules, in Debian the packages should be:
2ac661d551b3 Update documentation.
Matti Hamalainen <ccr@tnsp.org>
parents: 201
diff changeset
14 # libwww-perl libxml-simple-perl libtimedate-perl
2ac661d551b3 Update documentation.
Matti Hamalainen <ccr@tnsp.org>
parents: 201
diff changeset
15 #
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
16 #
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
17 ##########################################################################
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
18 use strict;
243
aef2cf1dc3be Use UTF8 source.
Matti Hamalainen <ccr@tnsp.org>
parents: 242
diff changeset
19 use utf8;
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
20 use LWP::UserAgent;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
21 use HTML::Entities;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
22 use XML::Simple;
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
23 use Date::Format;
199
95012f8f687a Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents: 198
diff changeset
24 use Date::Parse;
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
25 use Data::Dumper;
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
26 use File::Slurp;
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
27 use Text::CSV;
166
c8d5f3a7c4b7 Add debug option.
Matti Hamalainen <ccr@tnsp.org>
parents: 164
diff changeset
28
162
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
29 ###
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
30 ### Configuration settings
162
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
31 ###
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
32 my %settings = (
166
c8d5f3a7c4b7 Add debug option.
Matti Hamalainen <ccr@tnsp.org>
parents: 164
diff changeset
33 "debug" => 0,
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
34 "opt_fmi" => 0,
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
35 "opt_tiehallinto" => 0,
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
36 "fmi_api_key" => "",
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
37 "outfile" => "",
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
38 "tiehallinto_static_meta" => "tiehallinto.meta",
185
64e06ee172c7 Add HTTP user agent setting.
Matti Hamalainen <ccr@tnsp.org>
parents: 182
diff changeset
39 "http_user_agent" => "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 6.0) Opera 10.63 [en]",
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
40 );
162
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
41
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
42
162
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
43 ###
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
44 ### Helper functions
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
45 ###
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
46 sub mlog($)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
47 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
48 print STDERR $_[0];
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
49 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
50
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
51
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
52 sub fetch_http($)
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
53 {
198
60a4c69c7f25 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 197
diff changeset
54 my $agent = LWP::UserAgent->new;
60a4c69c7f25 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 197
diff changeset
55 $agent->agent($settings{"http_user_agent"});
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
56 $agent->timeout(20);
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
57
198
60a4c69c7f25 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 197
diff changeset
58 my $req = HTTP::Request->new(GET => $_[0]);
339
6ea410acbe74 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 338
diff changeset
59
198
60a4c69c7f25 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 197
diff changeset
60 return $agent->request($req);
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
61 }
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
62
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
63
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
64 sub parse_timestamp($$)
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
65 {
196
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
66 my ($str, $offs) = @_;
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
67 if ($str =~ /^(\d+):(\d+)$/)
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
68 {
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
69 return $offs + (60 * 60 * $1) + ($2 * 60);
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
70 }
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
71 else
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
72 {
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
73 return $offs;
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
74 }
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
75 }
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
76
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
77
194
55ed3db51ee7 Rename a function.
Matti Hamalainen <ccr@tnsp.org>
parents: 193
diff changeset
78 sub format_time_gmt($)
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
79 {
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
80 # 2012-02-27T00:00:00Z
199
95012f8f687a Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents: 198
diff changeset
81 return time2str("%Y-%m-%dT%TZ", $_[0], "UTC");
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
82 }
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
83
193
93ec73deebc5 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 192
diff changeset
84
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
85 sub str_trim($)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
86 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
87 my $str = $_[0];
193
93ec73deebc5 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 192
diff changeset
88 if (defined($str))
93ec73deebc5 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 192
diff changeset
89 {
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
90 $str =~ s/^\s*//;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
91 $str =~ s/\s*$//;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
92 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
93 return $str;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
94 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
95
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
96
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
97 ###
331
b8ae5e99341b fetc_weather: Remove support for old HTML format parsing of Tiehallinto data, it's not provided in this format anymore.
Matti Hamalainen <ccr@tnsp.org>
parents: 325
diff changeset
98 ### Helper functions
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
99 ###
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
100
212
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
101 my %th_rain_states =
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
102 (
237
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
103 "Pouta" => "poutaa",
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
104 "Heikko" => "heikkoa sadetta",
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
105 "Kohtalainen" => "kohtalaista sadetta",
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
106 "Voimakas" => "voimakasta sadetta",
212
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
107 );
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
108
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
109 my $th_rain_states_k = join("|", map {quotemeta} sort { length($b)<=>length($a) } keys %th_rain_states);
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
110
242
ffd49b583e34 Rename function.
Matti Hamalainen <ccr@tnsp.org>
parents: 238
diff changeset
111 sub translate_rain($)
212
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
112 {
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
113 my $tmp = $_[0];
237
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
114 $tmp =~ s/($th_rain_states_k)/$th_rain_states{$1}/igo;
212
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
115 return $tmp;
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
116 }
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
117
331
b8ae5e99341b fetc_weather: Remove support for old HTML format parsing of Tiehallinto data, it's not provided in this format anymore.
Matti Hamalainen <ccr@tnsp.org>
parents: 325
diff changeset
118
244
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
119 my %th_cloud_states =
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
120 (
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
121 0 => "selkeää",
247
85ad2912e720 Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents: 245
diff changeset
122 1 => "melkein selkeää",
85ad2912e720 Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents: 245
diff changeset
123 2 => "verrattain selkeää",
85ad2912e720 Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents: 245
diff changeset
124 3 => "verrattain selkeää",
85ad2912e720 Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents: 245
diff changeset
125 4 => "puolipilvistä",
85ad2912e720 Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents: 245
diff changeset
126 5 => "verrattain pilvistä",
85ad2912e720 Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents: 245
diff changeset
127 6 => "verrattain pilvistä",
244
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
128 7 => "melkein pilvistä",
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
129 8 => "pilvistä",
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
130 );
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
131
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
132 sub translate_clouds($)
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
133 {
247
85ad2912e720 Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents: 245
diff changeset
134 return "" if ($_[0] eq "NaN" || $_[0] eq "");
85ad2912e720 Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents: 245
diff changeset
135 my $tmp = int($_[0]);
244
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
136 foreach my $n (sort { $a <=> $b } keys %th_cloud_states)
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
137 {
247
85ad2912e720 Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents: 245
diff changeset
138 return $th_cloud_states{$n}." (".$n."/8)" if ($tmp == $n);
244
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
139 }
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
140 return $tmp;
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
141 }
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
142
212
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
143
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
144 sub plonk_data($)
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
145 {
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
146 return defined($_[0]) ? $_[0] : "";
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
147 }
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
148
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
149
237
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
150 sub plonk_data_lc($)
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
151 {
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
152 return defined($_[0]) ? lc($_[0]) : "";
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
153 }
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
154
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
155
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
156 ###
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
157 ### Configuration handling
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
158 ###
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
159 sub opt_chk_bool($)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
160 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
161 if (defined($settings{$_[0]}))
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
162 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
163 my $val = $settings{$_[0]};
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
164 return ($val == 1 || $val eq "true" || $val eq "on" || $val eq "1");
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
165 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
166 else
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
167 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
168 return 0;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
169 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
170 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
171
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
172
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
173 sub opt_chk_valid($$)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
174 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
175 if (defined($settings{$_[0]}))
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
176 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
177 my $val = $settings{$_[0]};
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
178 return length($val) >= $_[1];
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
179 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
180 else
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
181 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
182 return 0;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
183 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
184 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
185
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
186
324
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
187 sub opt_get_int($)
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
188 {
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
189 if (defined($settings{$_[0]}))
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
190 {
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
191 return int($settings{$_[0]});
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
192 }
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
193 else
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
194 {
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
195 return -1;
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
196 }
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
197 }
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
198
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
199
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
200 sub opt_get($)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
201 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
202 if (defined($settings{$_[0]}))
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
203 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
204 return $settings{$_[0]};
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
205 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
206 else
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
207 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
208 return undef;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
209 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
210 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
211
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
212
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
213 sub opt_read_config($)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
214 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
215 my $filename = $_[0];
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
216 my $errors = 0;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
217 my $line = 0;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
218
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
219 open(CONFFILE, "<", $filename) or die("Could not open configuration '".$filename."'!\n");
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
220 while (<CONFFILE>)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
221 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
222 $line++;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
223 chomp;
353
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
224 if (/(^\s*#|^\s*$)/)
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
225 {
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
226 # Ignore comments and empty lines
353
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
227 }
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
228 elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*(\d+),?\s*$/) {
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
229 my $key = lc($1);
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
230 my $value = $2;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
231 if (defined($settings{$key})) {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
232 $settings{$key} = $value;
353
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
233 }
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
234 else
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
235 {
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
236 mlog("[$filename:$line] Unknown setting '$key' = $value\n");
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
237 $errors = 1;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
238 }
353
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
239 }
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
240 elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*\"(.*?)\",?\s*$/) {
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
241 my $key = lc($1);
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
242 my $value = $2;
353
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
243 if (defined($settings{$key}))
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
244 {
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
245 $settings{$key} = $value;
353
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
246 }
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
247 else
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
248 {
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
249 mlog("[$filename:$line] Unknown setting '$key' = '$value'\n");
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
250 $errors = 1;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
251 }
353
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
252 }
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
253 else
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
254 {
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
255 mlog("[$filename:$line] Syntax error: $_\n");
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
256 $errors = 1;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
257 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
258 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
259 close(CONFFILE);
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
260 return $errors;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
261 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
262
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
263
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
264 ###
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
265 ### Main program begins
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
266 ###
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
267 my $weatherdata = {};
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
268
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
269 die(
359
3b8ffc6eca6d fetch_weather: Bump version.
Matti Hamalainen <ccr@tnsp.org>
parents: 356
diff changeset
270 "Weather Fetch v0.6 by ccr/TNSP <ccr\@tnsp.org>\n".
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
271 "Usage: $0 <config file>\n"
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
272 ) unless scalar(@ARGV) >= 1;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
273
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
274 my $cfgfile = shift;
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
275 opt_read_config($cfgfile) == 0 or die("Errors while parsing configuration file '".$cfgfile."'.\n");
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
276
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
277
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
278 ###
354
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
279 ### Load already cached data
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
280 ###
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
281 if (opt_chk_valid("outfile", 1))
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
282 {
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
283 my $str = read_file(opt_get("outfile"), binmode => ':utf8', err_mode => 'quiet');
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
284 if (defined($str))
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
285 {
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
286 foreach my $line (split(/\s*\n\s*/, $str))
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
287 {
367
1ca8ae195077 fetch_weather: Fix parsing of old/cached datafile.
Matti Hamalainen <ccr@tnsp.org>
parents: 363
diff changeset
288 my @mtmp = split(/\|/, $line, -1);
356
0de6e0c2c89c fetch_weather: Adjust minimum acceptable recached entry array size.
Matti Hamalainen <ccr@tnsp.org>
parents: 355
diff changeset
289 if (scalar(\@mtmp) >= 3)
354
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
290 {
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
291 $weatherdata->{shift @mtmp} = \@mtmp;
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
292 }
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
293 }
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
294 print STDERR scalar(keys %$weatherdata)." old records reloaded.\n" if (opt_get_int("debug") > 0);
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
295 }
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
296 }
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
297
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
298
de411e7834aa fetch_weather: Reload old data from stored data.
Matti Hamalainen <ccr@tnsp.org>
parents: 353
diff changeset
299 ###
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
300 ### Fetch Tiehallinto data
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
301 ###
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
302 if (opt_chk_bool("opt_tiehallinto"))
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
303 {
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
304 my $uri = "http://tie.digitraffic.fi/sujuvuus/ws/roadWeather";
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
305 my $res = fetch_http($uri);
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
306 if ($res->code >= 200 && $res->code <= 201)
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
307 {
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
308 my $xml = XMLin($res->decoded_content);
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
309
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
310 if (!defined($xml->{"soap:Body"}) || !defined($xml->{"soap:Body"}{"RoadWeatherResponse"}))
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
311 {
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
312 print STDERR "ERROR: SOAP call result did not contain required data.\n";
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
313 print STDERR $res->decoded_content."\n\n";
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
314 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
315 else
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
316 {
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
317 my $data = $xml->{"soap:Body"}{"RoadWeatherResponse"};
349
f7793ef7c925 fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 343
diff changeset
318 print STDERR "Checking Tiehallinto response from ".$uri."\n" if (opt_get_int("debug") > 0);
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
319
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
320 # Check if we need to update the static meta data
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
321 my $meta_file = opt_get("tiehallinto_static_meta");
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
322 my $fetch_meta = (-e $meta_file) ? 0 : 1;
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
323
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
324 if (defined($data->{"laststaticdataupdate"}))
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
325 {
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
326 # Compare metadata cache file modification timestamp to info in XML
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
327 my $tmp1 = str2time($data->{"laststaticdataupdate"});
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
328 my $tmp2 = (-e $meta_file) ? (stat($meta_file))[9] : -1;
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
329 $fetch_meta = 1 unless ($tmp1 < $tmp2);
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
330 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
331
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
332 # Fetch or read the cache
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
333 my $meta_str;
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
334 if ($fetch_meta)
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
335 {
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
336 print STDERR "Fetching Tiehallinto static meta data.\n" if (opt_get_int("debug") > 0);
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
337 my $uri = "https://raw.githubusercontent.com/finnishtransportagency/metadata/master/csv/meta_traffic_stations.csv";
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
338 my $res = fetch_http($uri);
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
339 die("Failed to fetch $uri data.\n") unless ($res->code <= 200 && $res->code <= 201);
339
6ea410acbe74 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 338
diff changeset
340
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
341 print STDERR "Storing to cache '$meta_file'.\n" if (opt_get_int("debug") > 0);
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
342 $meta_str = $res->decoded_content;
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
343 write_file($meta_file, {binmode => ':utf8'}, $meta_str);
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
344 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
345 else
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
346 {
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
347 print STDERR "Using CACHED Tiehallinto static meta data from '$meta_file'.\n" if (opt_get_int("debug") > 0);
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
348 $meta_str = read_file($meta_file, binmode => ':utf8');
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
349 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
350
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
351 # Parse the data ..
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
352 my $meta_data = {};
355
a74cb8a1abf5 Add in commented out code.
Matti Hamalainen <ccr@tnsp.org>
parents: 354
diff changeset
353 # my $csv = Text::CSV->new({blank_is_undef => 1, decode_utf8 => 1});
343
3fff571a904b fetch_weather: Some fixes to make this actually work.
Matti Hamalainen <ccr@tnsp.org>
parents: 340
diff changeset
354 my $csv = Text::CSV->new({blank_is_undef => 1});
3fff571a904b fetch_weather: Some fixes to make this actually work.
Matti Hamalainen <ccr@tnsp.org>
parents: 340
diff changeset
355 die("Failed to instantiate Text::CSV object?\n") unless defined($csv);
355
a74cb8a1abf5 Add in commented out code.
Matti Hamalainen <ccr@tnsp.org>
parents: 354
diff changeset
356
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
357 foreach my $line (split(/\s*\n\s*/, $meta_str))
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
358 {
343
3fff571a904b fetch_weather: Some fixes to make this actually work.
Matti Hamalainen <ccr@tnsp.org>
parents: 340
diff changeset
359 if (defined($line) && $csv->parse($line))
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
360 {
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
361 my @fields = $csv->fields();
363
3917a1515e31 fetch_weather: Fix parsing of "empty" CSV data.
Matti Hamalainen <ccr@tnsp.org>
parents: 359
diff changeset
362 if (scalar(@fields) > 1)
3917a1515e31 fetch_weather: Fix parsing of "empty" CSV data.
Matti Hamalainen <ccr@tnsp.org>
parents: 359
diff changeset
363 {
3917a1515e31 fetch_weather: Fix parsing of "empty" CSV data.
Matti Hamalainen <ccr@tnsp.org>
parents: 359
diff changeset
364 $$meta_data{$fields[1]} = \@fields;
3917a1515e31 fetch_weather: Fix parsing of "empty" CSV data.
Matti Hamalainen <ccr@tnsp.org>
parents: 359
diff changeset
365 }
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
366 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
367 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
368
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
369 # Parse XML and combine with the station meta data
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
370 if (defined($data->{"roadweatherdata"}))
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
371 {
349
f7793ef7c925 fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 343
diff changeset
372 my $nrecords = 0;
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
373 foreach my $wdata (@{$data->{"roadweatherdata"}{"roadweather"}})
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
374 {
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
375 my $wid = $wdata->{"stationid"};
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
376 if (defined($meta_data->{$wid}))
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
377 {
349
f7793ef7c925 fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 343
diff changeset
378 $nrecords++;
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
379 $weatherdata->{$meta_data->{$wid}[3]} =
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
380 [
343
3fff571a904b fetch_weather: Some fixes to make this actually work.
Matti Hamalainen <ccr@tnsp.org>
parents: 340
diff changeset
381 1,
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
382 str2time(plonk_data($wdata->{"measurementtime"}{"utc"})),
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
383 plonk_data($wdata->{"airtemperature1"}),
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
384
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
385 plonk_data($wdata->{"humidity"}),
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
386 plonk_data($wdata->{"averagewindspeed"}),
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
387 ];
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
388 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
389 }
349
f7793ef7c925 fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 343
diff changeset
390 print STDERR $nrecords." records from Tiehallinto.\n" if (opt_get_int("debug") > 0);
338
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
391 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
392 else
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
393 {
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
394 print STDERR "ERROR: Invalid (or unsupported) road weather data blob.\n";
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
395 print STDERR $res->decoded_content."\n\n";
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
396 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
397 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
398 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
399 }
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
400
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
401
afb8c0e2d513 fetch_weather: Implement support for new Tiehallinto data API.
Matti Hamalainen <ccr@tnsp.org>
parents: 332
diff changeset
402 ###
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
403 ### Fetch FMI data
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
404 ###
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
405 if (opt_chk_bool("opt_fmi"))
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
406 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
407 die("FMI data scrape enabled, but no API key set.\n") unless opt_chk_valid("fmi_api_key", 10);
238
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
408 my @fmitems = ("temperature", "humidity", "windspeedms", "totalcloudcover");
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
409
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
410 my $uri = "http://data.fmi.fi/fmi-apikey/".opt_get("fmi_api_key").
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
411 "/wfs?request=getFeature&storedquery_id=fmi::observations::weather::".
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
412 "multipointcoverage".
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
413 # "timevaluepair".
194
55ed3db51ee7 Rename a function.
Matti Hamalainen <ccr@tnsp.org>
parents: 193
diff changeset
414 "&starttime=".format_time_gmt(time() - 10*60)."&endtime=".format_time_gmt(time()).
238
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
415 "&parameters=".join(",", @fmitems)."&maxlocations=100&bbox=19,59,32,75";
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
416
324
19c64798de91 weather: Add some better optional debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 247
diff changeset
417 print STDERR "FMI URI: ".$uri."\n" if (opt_get_int("debug") > 0);
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
418
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
419 my $res = fetch_http($uri);
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
420 if ($res->code >= 200 && $res->code <= 201)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
421 {
170
e4faa3746869 Remove useless parameters from XMLin().
Matti Hamalainen <ccr@tnsp.org>
parents: 169
diff changeset
422 my $xml = XMLin($res->decoded_content);
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
423 my $time_base = time();
339
6ea410acbe74 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 338
diff changeset
424
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
425 if (defined($xml->{"wfs:member"}{"omso:GridSeriesObservation"}))
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
426 {
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
427 my $fdata = $xml->{"wfs:member"}{"omso:GridSeriesObservation"};
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
428 my $fshit = $fdata->{"om:result"}{"gmlcov:MultiPointCoverage"};
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
429 my @farray = ();
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
430
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
431 foreach my $fline (split(/\n/, $fshit->{"gml:domainSet"}{"gmlcov:SimpleMultiPoint"}{"gmlcov:positions"}))
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
432 {
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
433 if ($fline =~ /^\s*([\+\-]?\d+\.\d*)\s+([\+\-]?\d+\.\d*)\s+(\d+)\s*$/)
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
434 {
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
435 push(@farray, {"lat" => $1, "long" => $2, "time" => $3});
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
436 }
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
437 }
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
438
339
6ea410acbe74 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 338
diff changeset
439 my $findex = 0;
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
440 foreach my $fline (split(/\n/, $fshit->{"gml:rangeSet"}{"gml:DataBlock"}{"gml:doubleOrNilReasonTupleList"}))
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
441 {
238
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
442 my @fmatches = ($fline =~ /\s*([\+\-]?\d+\.\d*|NaN)\s*/ig);
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
443 if (scalar(@fmatches) > 0)
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
444 {
238
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
445 die("Not enough items in scalar line (".scalar(@fmatches). " vs ".scalar(@fmitems).
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
446 "): ".$fline."\n") if (scalar(@fmatches) != scalar(@fmitems));
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
447 for (my $fni = 0; $fni < scalar(@fmitems); $fni++)
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
448 {
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
449 $farray[$findex]{$fmitems[$fni]} = $fmatches[$fni] if (lc($fmatches[$fni]) ne "nan");
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
450 }
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
451 $findex++;
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
452 }
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
453 }
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
454 # XXX Hashify the array into lat/long keys
339
6ea410acbe74 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 338
diff changeset
455
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
456 # This is horrible :S
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
457 my $fcrap = $fdata->{"om:featureOfInterest"}{"sams:SF_SpatialSamplingFeature"}{"sams:shape"}{"gml:MultiPoint"}{"gml:pointMember"};
352
19b363586b40 fetch_weather: Fix debug prints.
Matti Hamalainen <ccr@tnsp.org>
parents: 349
diff changeset
458 my $nrecords = 0;
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
459 foreach my $xnode (@{$fcrap})
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
460 {
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
461 my $floc = $xnode->{"gml:Point"};
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
462 if ($floc->{"gml:pos"} =~ /^\s*([\+\-]?\d+\.\d*)\s+([\+\-]?\d+\.\d*)\s*$/)
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
463 {
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
464 my ($flat, $flong) = ($1, $2);
349
f7793ef7c925 fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 343
diff changeset
465
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
466 # Should use a hash -
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
467 foreach my $flol (@farray)
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
468 {
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
469 if ($flol->{"lat"} == $flat && $flol->{"long"} == $flong)
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
470 {
349
f7793ef7c925 fetch_weather: Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 343
diff changeset
471 $nrecords++;
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
472 $weatherdata->{$floc->{"gml:name"}} =
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
473 [
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
474 1,
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
475 plonk_data($flol->{"time"}),
237
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
476 plonk_data($flol->{"temperature"}),
238
a1d6e2d8789e Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents: 237
diff changeset
477
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
478 plonk_data($flol->{"humidity"}),
237
56af82fb7a6d Use lower case.
Matti Hamalainen <ccr@tnsp.org>
parents: 236
diff changeset
479 plonk_data($flol->{"windspeedms"}),
244
7dea85b9a431 Translate cloudiness data.
Matti Hamalainen <ccr@tnsp.org>
parents: 243
diff changeset
480 translate_clouds(plonk_data($flol->{"totalcloudcover"})),
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
481 ];
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
482 }
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
483 }
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
484 }
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
485 }
352
19b363586b40 fetch_weather: Fix debug prints.
Matti Hamalainen <ccr@tnsp.org>
parents: 349
diff changeset
486 print STDERR $nrecords." records from FMI.\n" if (opt_get_int("debug") > 0);
353
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
487 }
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
488 else
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
489 {
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
490 # defined
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
491 print STDERR "Invalid XML received:\n";
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
492 print STDERR $res->decoded_content."\n\n";
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
493 }
353
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
494 }
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
495 else
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
496 {
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
497 print STDERR "Error fetching FMI XML: ".$res->status_line."\n";
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
498 }
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
499 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
500
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
501
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
502 ###
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
503 ### Output
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
504 ###
353
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
505 if (opt_chk_valid("outfile", 1))
a52a983c0c14 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 352
diff changeset
506 {
368
ed1c7f2d81df fetch_weather: Close STDOUT filehandle before reopening it, to make sure we
Matti Hamalainen <ccr@tnsp.org>
parents: 367
diff changeset
507 close(STDOUT);
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
508 open(STDOUT, '>', opt_get("outfile")) or die("Could not open output file '".opt_get("outfile")."'.\n");
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
509 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
510
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
511 binmode STDOUT, ':encoding(utf-8)';
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
512
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
513 foreach my $key (sort { $a cmp $b } keys %$weatherdata)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
514 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
515 print STDOUT $key."|".join("|", @{$weatherdata->{$key}})."\n";
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
516 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
517
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
518 close(STDOUT);