Mercurial > hg > egg-tcls
annotate fetch_weather.pl @ 247:85ad2912e720
Possibly handle cloudyness translation better.
author | Matti Hamalainen <ccr@tnsp.org> |
---|---|
date | Mon, 19 Jan 2015 20:43:06 +0200 |
parents | 9435745b0dbb |
children | 19c64798de91 |
rev | line source |
---|---|
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
1 #!/usr/bin/perl -w |
182 | 2 ########################################################################## |
3 # | |
245 | 4 # Fetch Weather v0.3 by Matti 'ccr' Hamalainen <ccr@tnsp.org> |
233
dc1afa011883
weather: Bump version, copyright.
Matti Hamalainen <ccr@tnsp.org>
parents:
232
diff
changeset
|
5 # (C) Copyright 2014-2015 Tecnic Software productions (TNSP) |
206 | 6 # This script is freely distributable under GNU GPL (version 2) license. |
182 | 7 # |
8 # Should be ran as a cronjob, and configured properly. | |
206 | 9 # */10 * * * * perl -w /absolute/path/to/fetch_weather.pl /path/to/configfile |
182 | 10 # |
186
b41c525bdaf7
Add example configuration file.
Matti Hamalainen <ccr@tnsp.org>
parents:
185
diff
changeset
|
11 # Configuration file example is in fetch_weather.config |
182 | 12 # |
206 | 13 # Requires various Perl modules, in Debian the packages should be: |
14 # libwww-perl libxml-simple-perl libtimedate-perl | |
15 # | |
182 | 16 # |
17 ########################################################################## | |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
18 use strict; |
243 | 19 use utf8; |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
20 use LWP::UserAgent; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
21 use HTML::Entities; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
22 use XML::Simple; |
189 | 23 use Date::Format; |
199
95012f8f687a
Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents:
198
diff
changeset
|
24 use Date::Parse; |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
25 use Data::Dumper; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
26 |
166 | 27 |
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
28 ### |
182 | 29 ### Configuration settings |
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
30 ### |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
31 my %settings = ( |
166 | 32 "debug" => 0, |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
33 "opt_fmi" => 0, |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
34 "opt_tiehallinto" => 0, |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
35 "fmi_api_key" => "", |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
36 "outfile" => "", |
185
64e06ee172c7
Add HTTP user agent setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
182
diff
changeset
|
37 "http_user_agent" => "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 6.0) Opera 10.63 [en]", |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
38 ); |
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
39 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
40 |
162
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
41 ### |
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
42 ### Helper functions |
4f8a163b2bc1
Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents:
158
diff
changeset
|
43 ### |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
44 sub mlog($) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
45 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
46 print STDERR $_[0]; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
47 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
48 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
49 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
50 sub fetch_http($) |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
51 { |
198 | 52 my $agent = LWP::UserAgent->new; |
53 $agent->agent($settings{"http_user_agent"}); | |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
54 $agent->timeout(20); |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
55 |
198 | 56 my $req = HTTP::Request->new(GET => $_[0]); |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
57 |
198 | 58 return $agent->request($req); |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
59 } |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
60 |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
61 |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
62 sub parse_timestamp($$) |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
63 { |
196
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
64 my ($str, $offs) = @_; |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
65 if ($str =~ /^(\d+):(\d+)$/) |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
66 { |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
67 return $offs + (60 * 60 * $1) + ($2 * 60); |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
68 } |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
69 else |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
70 { |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
71 return $offs; |
a38c123921a9
Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents:
195
diff
changeset
|
72 } |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
73 } |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
74 |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
75 |
194 | 76 sub format_time_gmt($) |
189 | 77 { |
78 # 2012-02-27T00:00:00Z | |
199
95012f8f687a
Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents:
198
diff
changeset
|
79 return time2str("%Y-%m-%dT%TZ", $_[0], "UTC"); |
189 | 80 } |
81 | |
193 | 82 |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
83 sub str_trim($) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
84 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
85 my $str = $_[0]; |
193 | 86 if (defined($str)) |
87 { | |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
88 $str =~ s/^\s*//; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
89 $str =~ s/\s*$//; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
90 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
91 return $str; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
92 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
93 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
94 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
95 ### |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
96 ### Loose HTML parser |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
97 ### |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
98 sub pop_token_a($) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
99 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
100 my $tokens = $_[0]; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
101 return shift(@$tokens); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
102 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
103 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
104 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
105 sub pop_token($) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
106 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
107 return str_trim(pop_token_a($_[0])); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
108 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
109 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
110 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
111 sub parse_html_str($) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
112 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
113 my $tokens = $_[0]; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
114 my $token = pop_token($tokens); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
115 my $str = ""; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
116 $token =~ s/^\s*//; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
117 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
118 return undef unless (substr($token, 0, 1) eq '"'); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
119 $token = substr($token, 1); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
120 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
121 while (defined($token)) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
122 my $tmp = $token; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
123 $tmp =~ s/\s*$//; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
124 if (substr($tmp, -1) eq '"') { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
125 $str .= substr($tmp, 0, -1); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
126 return $str; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
127 } else { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
128 $str .= $token; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
129 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
130 $token = shift(@$tokens); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
131 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
132 return undef; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
133 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
134 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
135 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
136 sub parse_html_tree($$); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
137 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
138 sub parse_html_tree($$) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
139 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
140 my ($tokens, $tree) = @_; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
141 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
142 while (my $token = pop_token($tokens)) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
143 if ($token =~ /^<[!\/]?[a-zA-Z]+/) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
144 $token = lc($token); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
145 if ($token =~ /^<\!.*>$/) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
146 # Ignore comments etc. |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
147 } elsif ($token =~ /^<([a-z]+[1-9]*)(.*)>$/) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
148 my ($name, $args) = ($1, $2); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
149 if ($name eq "meta" || $name eq "img") { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
150 my $tmp = {}; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
151 parse_html_tree($tokens, $tree); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
152 return $tree; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
153 } else { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
154 my $tmp = { "name" => $name, "args" => str_trim($args) }; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
155 parse_html_tree($tokens, $tmp); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
156 push(@{$$tree{"nodes"}}, $tmp); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
157 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
158 } elsif ($token =~ /^<\/([a-z]+[1-9]*)>$/) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
159 return $tree; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
160 } else { |
232 | 161 print STDERR "ERROR: Failed to parse '$token'\n"; |
162 return undef; | |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
163 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
164 } else { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
165 $token = str_trim(decode_entities($token)); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
166 push(@{$$tree{"nodes"}}, { "name" => "text", "args" => "", "text" => $token }) if length($token) > 0; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
167 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
168 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
169 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
170 return $tree; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
171 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
172 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
173 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
174 sub parse_html($) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
175 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
176 return undef unless defined($_[0]); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
177 my $str = $_[0]; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
178 my $res = { "name" => "", "args" => "" }; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
179 $str =~ tr/\r/ /; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
180 $str =~ tr/\n/ /; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
181 my @tokens = grep { !($_ =~ /^\s*$/) } split(/(<\/?[a-zA-Z]+.*?>)/, $str); |
232 | 182 if (defined(parse_html_tree(\@tokens, $res))) |
183 { | |
184 return $res; | |
185 } | |
186 else | |
187 { | |
188 return undef; | |
189 } | |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
190 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
191 |
197 | 192 |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
193 sub html_find_node($$$); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
194 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
195 sub html_find_node($$$) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
196 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
197 my ($node, $name, $args) = @_; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
198 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
199 if (defined($node)) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
200 if (ref($node) eq "ARRAY") { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
201 foreach my $n (@$node) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
202 my $tmp = html_find_node($n, $name, $args); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
203 # Must do it like this, in order not to break the loop |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
204 return $tmp if defined($tmp); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
205 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
206 } elsif (ref($node) eq "HASH") { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
207 if (defined($$node{"name"})) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
208 if ($$node{"name"} eq $name) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
209 if ($args ne "") { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
210 if (defined($$node{"args"}) && $$node{"args"} =~ /$args/) { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
211 } else { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
212 return html_find_node($$node{"nodes"}, $name, $args); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
213 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
214 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
215 return $node; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
216 } else { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
217 return html_find_node($$node{"nodes"}, $name, $args); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
218 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
219 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
220 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
221 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
222 return undef; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
223 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
224 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
225 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
226 ### |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
227 ### Helper functions for locating/traversing nodes |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
228 ### in the parsed data tree structure. |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
229 ### |
228 | 230 |
231 sub fnodep(@) | |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
232 { |
228 | 233 my $res = html_find_node(shift, shift, ""); |
234 if (defined(my $tmp = shift)) | |
235 { | |
236 return defined($res) ? $res->{$tmp} : undef; | |
237 } | |
238 else | |
239 { | |
240 return $res; | |
241 } | |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
242 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
243 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
244 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
245 sub fnodea($$$) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
246 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
247 return html_find_node($_[0], $_[1], $_[2]); |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
248 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
249 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
250 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
251 sub get_node($$$) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
252 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
253 return defined($_[0]->[$_[2]]{"nodes"}[0]{$_[1]}) ? $_[0]->[$_[2]]{"nodes"}[0]{$_[1]} : ""; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
254 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
255 |
237 | 256 sub get_node_lc($$$) |
257 { | |
258 return lc(get_node($_[0], $_[1], $_[2])); | |
259 } | |
260 | |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
261 |
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
262 my %th_rain_states = |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
263 ( |
237 | 264 "Pouta" => "poutaa", |
265 "Heikko" => "heikkoa sadetta", | |
266 "Kohtalainen" => "kohtalaista sadetta", | |
267 "Voimakas" => "voimakasta sadetta", | |
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
268 ); |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
269 |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
270 my $th_rain_states_k = join("|", map {quotemeta} sort { length($b)<=>length($a) } keys %th_rain_states); |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
271 |
242 | 272 sub translate_rain($) |
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
273 { |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
274 my $tmp = $_[0]; |
237 | 275 $tmp =~ s/($th_rain_states_k)/$th_rain_states{$1}/igo; |
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
276 return $tmp; |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
277 } |
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
278 |
244 | 279 my %th_cloud_states = |
280 ( | |
281 0 => "selkeää", | |
247
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
282 1 => "melkein selkeää", |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
283 2 => "verrattain selkeää", |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
284 3 => "verrattain selkeää", |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
285 4 => "puolipilvistä", |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
286 5 => "verrattain pilvistä", |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
287 6 => "verrattain pilvistä", |
244 | 288 7 => "melkein pilvistä", |
289 8 => "pilvistä", | |
290 ); | |
291 | |
292 sub translate_clouds($) | |
293 { | |
247
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
294 return "" if ($_[0] eq "NaN" || $_[0] eq ""); |
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
295 my $tmp = int($_[0]); |
244 | 296 foreach my $n (sort { $a <=> $b } keys %th_cloud_states) |
297 { | |
247
85ad2912e720
Possibly handle cloudyness translation better.
Matti Hamalainen <ccr@tnsp.org>
parents:
245
diff
changeset
|
298 return $th_cloud_states{$n}." (".$n."/8)" if ($tmp == $n); |
244 | 299 } |
300 return $tmp; | |
301 } | |
302 | |
212
5f04ec60e8d9
weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents:
208
diff
changeset
|
303 |
228 | 304 sub plonk_data($) |
305 { | |
306 return defined($_[0]) ? $_[0] : ""; | |
307 } | |
308 | |
309 | |
237 | 310 sub plonk_data_lc($) |
311 { | |
312 return defined($_[0]) ? lc($_[0]) : ""; | |
313 } | |
314 | |
315 | |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
316 ### |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
317 ### Configuration handling |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
318 ### |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
319 sub opt_chk_bool($) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
320 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
321 if (defined($settings{$_[0]})) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
322 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
323 my $val = $settings{$_[0]}; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
324 return ($val == 1 || $val eq "true" || $val eq "on" || $val eq "1"); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
325 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
326 else |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
327 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
328 return 0; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
329 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
330 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
331 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
332 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
333 sub opt_chk_valid($$) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
334 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
335 if (defined($settings{$_[0]})) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
336 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
337 my $val = $settings{$_[0]}; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
338 return length($val) >= $_[1]; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
339 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
340 else |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
341 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
342 return 0; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
343 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
344 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
345 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
346 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
347 sub opt_get($) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
348 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
349 if (defined($settings{$_[0]})) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
350 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
351 return $settings{$_[0]}; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
352 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
353 else |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
354 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
355 return undef; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
356 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
357 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
358 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
359 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
360 sub opt_read_config($) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
361 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
362 my $filename = $_[0]; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
363 my $errors = 0; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
364 my $line = 0; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
365 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
366 open(CONFFILE, "<", $filename) or die("Could not open configuration '".$filename."'!\n"); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
367 while (<CONFFILE>) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
368 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
369 $line++; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
370 chomp; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
371 if (/(^\s*#|^\s*$)/) { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
372 # Ignore comments and empty lines |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
373 } elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*(\d+),?\s*$/) { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
374 my $key = lc($1); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
375 my $value = $2; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
376 if (defined($settings{$key})) { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
377 $settings{$key} = $value; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
378 } else { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
379 mlog("[$filename:$line] Unknown setting '$key' = $value\n"); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
380 $errors = 1; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
381 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
382 } elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*\"(.*?)\",?\s*$/) { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
383 my $key = lc($1); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
384 my $value = $2; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
385 if (defined($settings{$key})) { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
386 $settings{$key} = $value; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
387 } else { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
388 mlog("[$filename:$line] Unknown setting '$key' = '$value'\n"); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
389 $errors = 1; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
390 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
391 } else { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
392 mlog("[$filename:$line] Syntax error: $_\n"); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
393 $errors = 1; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
394 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
395 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
396 close(CONFFILE); |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
397 return $errors; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
398 } |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
399 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
400 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
401 ### |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
402 ### Main program begins |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
403 ### |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
404 my $weatherdata = {}; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
405 |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
406 die( |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
407 "Weather Fetch v0.1 by ccr/TNSP <ccr\@tnsp.org>\n". |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
408 "Usage: $0 <config file>\n" |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
409 ) unless scalar(@ARGV) >= 1; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
410 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
411 my $cfgfile = shift; |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
412 opt_read_config($cfgfile) == 0 or die("Errors while parsing configuration file '".$cfgfile."'.\n"); |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
413 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
414 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
415 ### |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
416 ### Fetch tiehallinto road weather measurement data |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
417 ### |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
418 if (opt_chk_bool("opt_tiehallinto")) |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
419 { |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
420 for (my $i = 1; $i <= 22; $i++) |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
421 { |
232 | 422 my $qurl = "http://alk.tiehallinto.fi/alk/tiesaa/tiesaa_maak_".$i.".html"; |
423 my $res = fetch_http($qurl); | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
424 if ($res->code >= 200 && $res->code <= 201) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
425 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
426 my $data = $res->decoded_content; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
427 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
428 # Filter out crap tags we don't want or need |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
429 $data =~ s/\n/§/g; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
430 $data =~ s/<!--.*?-->//ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
431 $data =~ s/<map[^>]*>.*?<\/map>//ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
432 $data =~ s/<form[^>]*>.*?<\/form>//ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
433 $data =~ s/<script[^>]*>.*?<\/script>//ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
434 $data =~ s/<meta[^>]*>//ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
435 $data =~ s/<font[^>]*>//ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
436 $data =~ s/<\/font>//ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
437 $data =~ s/<span[^>]*>//ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
438 $data =~ s/<\/span>//ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
439 $data =~ s/<\/?b>//ig; |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
440 |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
441 $data =~ s/<br>//ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
442 $data =~ s/ / /ig; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
443 $data =~ s/§/\n/g; |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
444 |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
445 # Parse the HTML mess |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
446 my $otree = parse_html($data); |
232 | 447 if (!defined($otree)) |
448 { | |
449 print STDERR "ERROR: Failed to parse file '".$qurl."'.\n"; | |
450 next; | |
451 } | |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
452 |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
453 # Find our desired element nodes |
228 | 454 my $odata = fnodea(fnodep($otree, "body"), "div", "class=elementc"); |
455 my $oupdate = fnodep($odata, "p"); | |
199
95012f8f687a
Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents:
198
diff
changeset
|
456 my $time_base = str2time("00:00"); |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
457 if ($oupdate) { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
458 my $tmp = $oupdate->{"nodes"}[0]{"text"}; |
199
95012f8f687a
Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents:
198
diff
changeset
|
459 if ($tmp =~ /:\s+(\d\d)\.(\d\d)\.(\d\d\d\d)\s+(\d\d:\d\d)/) { |
95012f8f687a
Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents:
198
diff
changeset
|
460 $time_base = str2time($3."-".$2."-".$1); |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
461 } |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
462 } |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
463 |
228 | 464 my $oelems = fnodep($odata, "table", "nodes"); |
465 if (defined($oelems)) | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
466 { |
228 | 467 for (my $n = 1; $n < scalar(@$oelems); $n++) |
468 { | |
469 my $fdata = @$oelems[$n]->{"nodes"}; | |
470 $weatherdata->{get_node($fdata, "text", 0)} = | |
471 [ | |
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
472 # type, timestamp, temperature |
228 | 473 0, |
474 parse_timestamp(get_node($fdata, "text", 1), $time_base), | |
237 | 475 get_node_lc($fdata, "text", 2), |
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
476 # and the rest |
237 | 477 get_node_lc($fdata, "text", 3), |
242 | 478 translate_rain(get_node($fdata, "text", 4)), |
237 | 479 get_node_lc($fdata, "text", 5), |
228 | 480 ]; |
481 } | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
482 } |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
483 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
484 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
485 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
486 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
487 |
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
488 ### |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
489 ### Fetch FMI data |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
490 ### |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
491 if (opt_chk_bool("opt_fmi")) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
492 { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
493 die("FMI data scrape enabled, but no API key set.\n") unless opt_chk_valid("fmi_api_key", 10); |
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
494 my @fmitems = ("temperature", "humidity", "windspeedms", "totalcloudcover"); |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
495 |
189 | 496 my $uri = "http://data.fmi.fi/fmi-apikey/".opt_get("fmi_api_key"). |
497 "/wfs?request=getFeature&storedquery_id=fmi::observations::weather::". | |
498 "multipointcoverage". | |
499 # "timevaluepair". | |
194 | 500 "&starttime=".format_time_gmt(time() - 10*60)."&endtime=".format_time_gmt(time()). |
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
501 "¶meters=".join(",", @fmitems)."&maxlocations=100&bbox=19,59,32,75"; |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
502 |
189 | 503 print STDERR "FMI URI: ".$uri."\n" if opt_chk_bool("debug"); |
504 | |
505 my $res = fetch_http($uri); | |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
506 if ($res->code >= 200 && $res->code <= 201) |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
507 { |
170
e4faa3746869
Remove useless parameters from XMLin().
Matti Hamalainen <ccr@tnsp.org>
parents:
169
diff
changeset
|
508 my $xml = XMLin($res->decoded_content); |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
509 my $time_base = time(); |
189 | 510 |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
511 if (defined($xml->{"wfs:member"}{"omso:GridSeriesObservation"})) |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
512 { |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
513 my $fdata = $xml->{"wfs:member"}{"omso:GridSeriesObservation"}; |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
514 my $fshit = $fdata->{"om:result"}{"gmlcov:MultiPointCoverage"}; |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
515 my @farray = (); |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
516 |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
517 foreach my $fline (split(/\n/, $fshit->{"gml:domainSet"}{"gmlcov:SimpleMultiPoint"}{"gmlcov:positions"})) |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
518 { |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
519 if ($fline =~ /^\s*([\+\-]?\d+\.\d*)\s+([\+\-]?\d+\.\d*)\s+(\d+)\s*$/) |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
520 { |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
521 push(@farray, {"lat" => $1, "long" => $2, "time" => $3}); |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
522 } |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
523 } |
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
524 |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
525 my $findex = 0; |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
526 foreach my $fline (split(/\n/, $fshit->{"gml:rangeSet"}{"gml:DataBlock"}{"gml:doubleOrNilReasonTupleList"})) |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
527 { |
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
528 my @fmatches = ($fline =~ /\s*([\+\-]?\d+\.\d*|NaN)\s*/ig); |
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
529 if (scalar(@fmatches) > 0) |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
530 { |
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
531 die("Not enough items in scalar line (".scalar(@fmatches). " vs ".scalar(@fmitems). |
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
532 "): ".$fline."\n") if (scalar(@fmatches) != scalar(@fmitems)); |
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
533 for (my $fni = 0; $fni < scalar(@fmitems); $fni++) |
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
534 { |
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
535 $farray[$findex]{$fmitems[$fni]} = $fmatches[$fni] if (lc($fmatches[$fni]) ne "nan"); |
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
536 } |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
537 $findex++; |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
538 } |
189 | 539 } |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
540 # XXX Hashify the array into lat/long keys |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
541 |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
542 # This is horrible :S |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
543 my $fcrap = $fdata->{"om:featureOfInterest"}{"sams:SF_SpatialSamplingFeature"}{"sams:shape"}{"gml:MultiPoint"}{"gml:pointMember"}; |
228 | 544 foreach my $xnode (@{$fcrap}) |
189 | 545 { |
228 | 546 my $floc = $xnode->{"gml:Point"}; |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
547 if ($floc->{"gml:pos"} =~ /^\s*([\+\-]?\d+\.\d*)\s+([\+\-]?\d+\.\d*)\s*$/) |
189 | 548 { |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
549 my ($flat, $flong) = ($1, $2); |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
550 # Should use a hash - |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
551 foreach my $flol (@farray) |
189 | 552 { |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
553 if ($flol->{"lat"} == $flat && $flol->{"long"} == $flong) |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
554 { |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
555 $weatherdata->{$floc->{"gml:name"}} = |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
556 [ |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
557 1, |
228 | 558 plonk_data($flol->{"time"}), |
237 | 559 plonk_data($flol->{"temperature"}), |
238
a1d6e2d8789e
Add new data from FMI service and change datafile format.
Matti Hamalainen <ccr@tnsp.org>
parents:
237
diff
changeset
|
560 |
228 | 561 plonk_data($flol->{"humidity"}), |
237 | 562 plonk_data($flol->{"windspeedms"}), |
244 | 563 translate_clouds(plonk_data($flol->{"totalcloudcover"})), |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
564 ]; |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
565 } |
189 | 566 } |
567 } | |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
568 } |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
569 } else { |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
570 # defined |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
571 print STDERR "Invalid XML received:\n"; |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
572 print STDERR $res->decoded_content."\n\n"; |
169
c12e422a543a
FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents:
168
diff
changeset
|
573 } |
208
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
574 } else { |
c6bbf69afc57
Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents:
206
diff
changeset
|
575 print STDERR "Error fetching FMI XML: ".$res->status_line."\n"; |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
576 } |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
577 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
578 |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
579 |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
580 ### |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
581 ### Output |
178
9b8ec700ede4
Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
170
diff
changeset
|
582 ### |
164
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
583 if (opt_chk_valid("outfile", 1)) { |
d5a0e4248f3e
weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
162
diff
changeset
|
584 open(STDOUT, '>', opt_get("outfile")) or die("Could not open output file '".opt_get("outfile")."'.\n"); |
158
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
585 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
586 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
587 binmode STDOUT, ':encoding(utf-8)'; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
588 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
589 foreach my $key (sort { $a cmp $b } keys %$weatherdata) |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
590 { |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
591 print STDOUT $key."|".join("|", @{$weatherdata->{$key}})."\n"; |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
592 } |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
593 |
db2b1f74d994
Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
594 close(STDOUT); |