Mercurial > hg > egg-tcls

annotate fetch_weather.pl @ 236:f36c26b3c38b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Adjust FMI query.
author Matti Hamalainen <ccr@tnsp.org>
date Mon, 12 Jan 2015 16:17:41 +0200
parents dc1afa011883
children 56af82fb7a6d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
1 #!/usr/bin/perl -w
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
2 ##########################################################################
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
3 #
233
dc1afa011883 weather: Bump version, copyright.
Matti Hamalainen <ccr@tnsp.org>
parents: 232
diff changeset
4 # Fetch Weather v0.2 by Matti 'ccr' Hamalainen <ccr@tnsp.org>
dc1afa011883 weather: Bump version, copyright.
Matti Hamalainen <ccr@tnsp.org>
parents: 232
diff changeset
5 # (C) Copyright 2014-2015 Tecnic Software productions (TNSP)
206
2ac661d551b3 Update documentation.
Matti Hamalainen <ccr@tnsp.org>
parents: 201
diff changeset
6 # This script is freely distributable under GNU GPL (version 2) license.
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
7 #
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
8 # Should be ran as a cronjob, and configured properly.
206
2ac661d551b3 Update documentation.
Matti Hamalainen <ccr@tnsp.org>
parents: 201
diff changeset
9 # */10 * * * * perl -w /absolute/path/to/fetch_weather.pl /path/to/configfile
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
10 #
186
b41c525bdaf7 Add example configuration file.
Matti Hamalainen <ccr@tnsp.org>
parents: 185
diff changeset
11 # Configuration file example is in fetch_weather.config
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
12 #
206
2ac661d551b3 Update documentation.
Matti Hamalainen <ccr@tnsp.org>
parents: 201
diff changeset
13 # Requires various Perl modules, in Debian the packages should be:
2ac661d551b3 Update documentation.
Matti Hamalainen <ccr@tnsp.org>
parents: 201
diff changeset
14 # libwww-perl libxml-simple-perl libtimedate-perl
2ac661d551b3 Update documentation.
Matti Hamalainen <ccr@tnsp.org>
parents: 201
diff changeset
15 #
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
16 #
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
17 ##########################################################################
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
18 use strict;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
19 use LWP::UserAgent;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
20 use HTML::Entities;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
21 use XML::Simple;
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
22 use Date::Format;
199
95012f8f687a Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents: 198
diff changeset
23 use Date::Parse;
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
24 use Data::Dumper;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
25
166
c8d5f3a7c4b7 Add debug option.
Matti Hamalainen <ccr@tnsp.org>
parents: 164
diff changeset
26
162
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
27 ###
182
24f9f38210fe Add comments.
Matti Hamalainen <ccr@tnsp.org>
parents: 178
diff changeset
28 ### Configuration settings
162
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
29 ###
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
30 my %settings = (
166
c8d5f3a7c4b7 Add debug option.
Matti Hamalainen <ccr@tnsp.org>
parents: 164
diff changeset
31 "debug" => 0,
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
32 "opt_fmi" => 0,
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
33 "opt_tiehallinto" => 0,
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
34 "fmi_api_key" => "",
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
35 "outfile" => "",
185
64e06ee172c7 Add HTTP user agent setting.
Matti Hamalainen <ccr@tnsp.org>
parents: 182
diff changeset
36 "http_user_agent" => "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 6.0) Opera 10.63 [en]",
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
37 );
162
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
38
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
39
162
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
40 ###
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
41 ### Helper functions
4f8a163b2bc1 Work on the backend weather data fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 158
diff changeset
42 ###
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
43 sub mlog($)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
44 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
45 print STDERR $_[0];
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
46 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
47
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
48
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
49 sub fetch_http($)
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
50 {
198
60a4c69c7f25 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 197
diff changeset
51 my $agent = LWP::UserAgent->new;
60a4c69c7f25 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 197
diff changeset
52 $agent->agent($settings{"http_user_agent"});
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
53 $agent->timeout(20);
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
54
198
60a4c69c7f25 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 197
diff changeset
55 my $req = HTTP::Request->new(GET => $_[0]);
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
56
198
60a4c69c7f25 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 197
diff changeset
57 return $agent->request($req);
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
58 }
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
59
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
60
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
61 sub parse_timestamp($$)
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
62 {
196
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
63 my ($str, $offs) = @_;
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
64 if ($str =~ /^(\d+):(\d+)$/)
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
65 {
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
66 return $offs + (60 * 60 * $1) + ($2 * 60);
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
67 }
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
68 else
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
69 {
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
70 return $offs;
a38c123921a9 Change the timestamp parsing.
Matti Hamalainen <ccr@tnsp.org>
parents: 195
diff changeset
71 }
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
72 }
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
73
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
74
194
55ed3db51ee7 Rename a function.
Matti Hamalainen <ccr@tnsp.org>
parents: 193
diff changeset
75 sub format_time_gmt($)
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
76 {
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
77 # 2012-02-27T00:00:00Z
199
95012f8f687a Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents: 198
diff changeset
78 return time2str("%Y-%m-%dT%TZ", $_[0], "UTC");
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
79 }
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
80
193
93ec73deebc5 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 192
diff changeset
81
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
82 sub str_trim($)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
83 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
84 my $str = $_[0];
193
93ec73deebc5 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 192
diff changeset
85 if (defined($str))
93ec73deebc5 Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 192
diff changeset
86 {
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
87 $str =~ s/^\s*//;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
88 $str =~ s/\s*$//;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
89 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
90 return $str;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
91 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
92
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
93
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
94 ###
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
95 ### Loose HTML parser
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
96 ###
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
97 sub pop_token_a($)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
98 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
99 my $tokens = $_[0];
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
100 return shift(@$tokens);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
101 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
102
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
103
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
104 sub pop_token($)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
105 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
106 return str_trim(pop_token_a($_[0]));
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
107 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
108
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
109
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
110 sub parse_html_str($)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
111 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
112 my $tokens = $_[0];
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
113 my $token = pop_token($tokens);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
114 my $str = "";
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
115 $token =~ s/^\s*//;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
116
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
117 return undef unless (substr($token, 0, 1) eq '"');
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
118 $token = substr($token, 1);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
119
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
120 while (defined($token)) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
121 my $tmp = $token;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
122 $tmp =~ s/\s*$//;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
123 if (substr($tmp, -1) eq '"') {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
124 $str .= substr($tmp, 0, -1);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
125 return $str;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
126 } else {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
127 $str .= $token;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
128 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
129 $token = shift(@$tokens);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
130 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
131 return undef;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
132 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
133
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
134
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
135 sub parse_html_tree($$);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
136
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
137 sub parse_html_tree($$)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
138 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
139 my ($tokens, $tree) = @_;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
140
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
141 while (my $token = pop_token($tokens)) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
142 if ($token =~ /^<[!\/]?[a-zA-Z]+/) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
143 $token = lc($token);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
144 if ($token =~ /^<\!.*>$/) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
145 # Ignore comments etc.
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
146 } elsif ($token =~ /^<([a-z]+[1-9]*)(.*)>$/) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
147 my ($name, $args) = ($1, $2);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
148 if ($name eq "meta" || $name eq "img") {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
149 my $tmp = {};
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
150 parse_html_tree($tokens, $tree);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
151 return $tree;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
152 } else {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
153 my $tmp = { "name" => $name, "args" => str_trim($args) };
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
154 parse_html_tree($tokens, $tmp);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
155 push(@{$$tree{"nodes"}}, $tmp);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
156 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
157 } elsif ($token =~ /^<\/([a-z]+[1-9]*)>$/) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
158 return $tree;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
159 } else {
232
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
160 print STDERR "ERROR: Failed to parse '$token'\n";
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
161 return undef;
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
162 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
163 } else {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
164 $token = str_trim(decode_entities($token));
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
165 push(@{$$tree{"nodes"}}, { "name" => "text", "args" => "", "text" => $token }) if length($token) > 0;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
166 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
167 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
168
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
169 return $tree;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
170 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
171
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
172
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
173 sub parse_html($)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
174 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
175 return undef unless defined($_[0]);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
176 my $str = $_[0];
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
177 my $res = { "name" => "", "args" => "" };
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
178 $str =~ tr/\r/ /;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
179 $str =~ tr/\n/ /;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
180 my @tokens = grep { !($_ =~ /^\s*$/) } split(/(<\/?[a-zA-Z]+.*?>)/, $str);
232
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
181 if (defined(parse_html_tree(\@tokens, $res)))
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
182 {
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
183 return $res;
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
184 }
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
185 else
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
186 {
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
187 return undef;
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
188 }
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
189 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
190
197
0fd2435371eb Cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents: 196
diff changeset
191
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
192 sub html_find_node($$$);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
193
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
194 sub html_find_node($$$)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
195 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
196 my ($node, $name, $args) = @_;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
197
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
198 if (defined($node)) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
199 if (ref($node) eq "ARRAY") {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
200 foreach my $n (@$node) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
201 my $tmp = html_find_node($n, $name, $args);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
202 # Must do it like this, in order not to break the loop
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
203 return $tmp if defined($tmp);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
204 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
205 } elsif (ref($node) eq "HASH") {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
206 if (defined($$node{"name"})) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
207 if ($$node{"name"} eq $name) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
208 if ($args ne "") {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
209 if (defined($$node{"args"}) && $$node{"args"} =~ /$args/) {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
210 } else {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
211 return html_find_node($$node{"nodes"}, $name, $args);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
212 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
213 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
214 return $node;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
215 } else {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
216 return html_find_node($$node{"nodes"}, $name, $args);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
217 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
218 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
219 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
220 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
221 return undef;
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
222 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
223
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
224
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
225 ###
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
226 ### Helper functions for locating/traversing nodes
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
227 ### in the parsed data tree structure.
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
228 ###
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
229
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
230 sub fnodep(@)
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
231 {
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
232 my $res = html_find_node(shift, shift, "");
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
233 if (defined(my $tmp = shift))
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
234 {
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
235 return defined($res) ? $res->{$tmp} : undef;
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
236 }
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
237 else
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
238 {
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
239 return $res;
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
240 }
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
241 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
242
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
243
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
244 sub fnodea($$$)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
245 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
246 return html_find_node($_[0], $_[1], $_[2]);
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
247 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
248
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
249
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
250 sub get_node($$$)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
251 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
252 return defined($_[0]->[$_[2]]{"nodes"}[0]{$_[1]}) ? $_[0]->[$_[2]]{"nodes"}[0]{$_[1]} : "";
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
253 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
254
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
255
212
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
256 my %th_rain_states =
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
257 (
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
258 "Pouta" => "Poutaa",
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
259 "Heikko" => "Heikkoa sadetta",
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
260 "Kohtalainen" => "Kohtalaista sadetta",
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
261 "Voimakas" => "Voimakasta sadetta",
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
262 );
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
263
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
264 my $th_rain_states_k = join("|", map {quotemeta} sort { length($b)<=>length($a) } keys %th_rain_states);
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
265
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
266 sub translate_th_rain($)
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
267 {
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
268 my $tmp = $_[0];
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
269 $tmp =~ s/($th_rain_states_k)/$th_rain_states{$1}/go;
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
270 return $tmp;
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
271 }
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
272
5f04ec60e8d9 weather: "Translate" rain state from Tiehallitus data.
Matti Hamalainen <ccr@tnsp.org>
parents: 208
diff changeset
273
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
274 sub plonk_data($)
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
275 {
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
276 return defined($_[0]) ? $_[0] : "";
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
277 }
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
278
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
279
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
280 ###
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
281 ### Configuration handling
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
282 ###
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
283 sub opt_chk_bool($)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
284 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
285 if (defined($settings{$_[0]}))
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
286 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
287 my $val = $settings{$_[0]};
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
288 return ($val == 1 || $val eq "true" || $val eq "on" || $val eq "1");
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
289 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
290 else
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
291 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
292 return 0;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
293 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
294 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
295
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
296
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
297 sub opt_chk_valid($$)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
298 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
299 if (defined($settings{$_[0]}))
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
300 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
301 my $val = $settings{$_[0]};
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
302 return length($val) >= $_[1];
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
303 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
304 else
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
305 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
306 return 0;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
307 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
308 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
309
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
310
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
311 sub opt_get($)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
312 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
313 if (defined($settings{$_[0]}))
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
314 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
315 return $settings{$_[0]};
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
316 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
317 else
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
318 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
319 return undef;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
320 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
321 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
322
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
323
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
324 sub opt_read_config($)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
325 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
326 my $filename = $_[0];
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
327 my $errors = 0;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
328 my $line = 0;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
329
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
330 open(CONFFILE, "<", $filename) or die("Could not open configuration '".$filename."'!\n");
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
331 while (<CONFFILE>)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
332 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
333 $line++;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
334 chomp;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
335 if (/(^\s*#|^\s*$)/) {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
336 # Ignore comments and empty lines
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
337 } elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*(\d+),?\s*$/) {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
338 my $key = lc($1);
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
339 my $value = $2;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
340 if (defined($settings{$key})) {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
341 $settings{$key} = $value;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
342 } else {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
343 mlog("[$filename:$line] Unknown setting '$key' = $value\n");
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
344 $errors = 1;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
345 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
346 } elsif (/^\s*\"?([a-zA-Z0-9_]+)\"?\s*=>?\s*\"(.*?)\",?\s*$/) {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
347 my $key = lc($1);
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
348 my $value = $2;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
349 if (defined($settings{$key})) {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
350 $settings{$key} = $value;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
351 } else {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
352 mlog("[$filename:$line] Unknown setting '$key' = '$value'\n");
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
353 $errors = 1;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
354 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
355 } else {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
356 mlog("[$filename:$line] Syntax error: $_\n");
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
357 $errors = 1;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
358 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
359 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
360 close(CONFFILE);
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
361 return $errors;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
362 }
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
363
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
364
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
365 ###
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
366 ### Main program begins
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
367 ###
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
368 my $weatherdata = {};
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
369
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
370 die(
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
371 "Weather Fetch v0.1 by ccr/TNSP <ccr\@tnsp.org>\n".
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
372 "Usage: $0 <config file>\n"
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
373 ) unless scalar(@ARGV) >= 1;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
374
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
375 my $cfgfile = shift;
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
376 opt_read_config($cfgfile) == 0 or die("Errors while parsing configuration file '".$cfgfile."'.\n");
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
377
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
378
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
379 ###
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
380 ### Fetch tiehallinto road weather measurement data
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
381 ###
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
382 if (opt_chk_bool("opt_tiehallinto"))
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
383 {
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
384 for (my $i = 1; $i <= 22; $i++)
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
385 {
232
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
386 my $qurl = "http://alk.tiehallinto.fi/alk/tiesaa/tiesaa_maak_".$i.".html";
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
387 my $res = fetch_http($qurl);
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
388 if ($res->code >= 200 && $res->code <= 201)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
389 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
390 my $data = $res->decoded_content;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
391
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
392 # Filter out crap tags we don't want or need
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
393 $data =~ s/\n/§/g;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
394 $data =~ s/<!--.*?-->//ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
395 $data =~ s/<map[^>]*>.*?<\/map>//ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
396 $data =~ s/<form[^>]*>.*?<\/form>//ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
397 $data =~ s/<script[^>]*>.*?<\/script>//ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
398 $data =~ s/<meta[^>]*>//ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
399 $data =~ s/<font[^>]*>//ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
400 $data =~ s/<\/font>//ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
401 $data =~ s/<span[^>]*>//ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
402 $data =~ s/<\/span>//ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
403 $data =~ s/<\/?b>//ig;
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
404
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
405 $data =~ s/<br>//ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
406 $data =~ s/&nbsp;/ /ig;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
407 $data =~ s/§/\n/g;
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
408
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
409 # Parse the HTML mess
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
410 my $otree = parse_html($data);
232
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
411 if (!defined($otree))
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
412 {
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
413 print STDERR "ERROR: Failed to parse file '".$qurl."'.\n";
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
414 next;
0dc95d66ba8d Improve debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 228
diff changeset
415 }
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
416
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
417 # Find our desired element nodes
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
418 my $odata = fnodea(fnodep($otree, "body"), "div", "class=elementc");
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
419 my $oupdate = fnodep($odata, "p");
199
95012f8f687a Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents: 198
diff changeset
420 my $time_base = str2time("00:00");
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
421 if ($oupdate) {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
422 my $tmp = $oupdate->{"nodes"}[0]{"text"};
199
95012f8f687a Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents: 198
diff changeset
423 if ($tmp =~ /:\s+(\d\d)\.(\d\d)\.(\d\d\d\d)\s+(\d\d:\d\d)/) {
95012f8f687a Another round of timestamp parsing changes, urgh.
Matti Hamalainen <ccr@tnsp.org>
parents: 198
diff changeset
424 $time_base = str2time($3."-".$2."-".$1);
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
425 }
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
426 }
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
427
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
428 my $oelems = fnodep($odata, "table", "nodes");
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
429 if (defined($oelems))
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
430 {
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
431 for (my $n = 1; $n < scalar(@$oelems); $n++)
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
432 {
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
433 my $fdata = @$oelems[$n]->{"nodes"};
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
434 $weatherdata->{get_node($fdata, "text", 0)} =
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
435 [
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
436 0,
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
437 parse_timestamp(get_node($fdata, "text", 1), $time_base),
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
438 get_node($fdata, "text", 2),
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
439 get_node($fdata, "text", 3),
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
440 translate_th_rain(get_node($fdata, "text", 4)),
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
441 "",
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
442 get_node($fdata, "text", 5),
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
443 ];
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
444 }
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
445 }
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
446 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
447 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
448 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
449
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
450
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
451 ###
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
452 ### Fetch FMI data
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
453 ###
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
454 if (opt_chk_bool("opt_fmi"))
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
455 {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
456 die("FMI data scrape enabled, but no API key set.\n") unless opt_chk_valid("fmi_api_key", 10);
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
457
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
458 my $uri = "http://data.fmi.fi/fmi-apikey/".opt_get("fmi_api_key").
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
459 "/wfs?request=getFeature&storedquery_id=fmi::observations::weather::".
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
460 "multipointcoverage".
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
461 # "timevaluepair".
194
55ed3db51ee7 Rename a function.
Matti Hamalainen <ccr@tnsp.org>
parents: 193
diff changeset
462 "&starttime=".format_time_gmt(time() - 10*60)."&endtime=".format_time_gmt(time()).
236
f36c26b3c38b Adjust FMI query.
Matti Hamalainen <ccr@tnsp.org>
parents: 233
diff changeset
463 "&parameters=temperature,humidity&maxlocations=100&bbox=19,59,32,75";
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
464
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
465 print STDERR "FMI URI: ".$uri."\n" if opt_chk_bool("debug");
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
466
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
467 my $res = fetch_http($uri);
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
468 if ($res->code >= 200 && $res->code <= 201)
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
469 {
170
e4faa3746869 Remove useless parameters from XMLin().
Matti Hamalainen <ccr@tnsp.org>
parents: 169
diff changeset
470 my $xml = XMLin($res->decoded_content);
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
471 my $time_base = time();
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
472
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
473 if (defined($xml->{"wfs:member"}{"omso:GridSeriesObservation"}))
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
474 {
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
475 my $fdata = $xml->{"wfs:member"}{"omso:GridSeriesObservation"};
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
476 my $fshit = $fdata->{"om:result"}{"gmlcov:MultiPointCoverage"};
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
477 my @farray = ();
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
478
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
479 foreach my $fline (split(/\n/, $fshit->{"gml:domainSet"}{"gmlcov:SimpleMultiPoint"}{"gmlcov:positions"}))
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
480 {
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
481 if ($fline =~ /^\s*([\+\-]?\d+\.\d*)\s+([\+\-]?\d+\.\d*)\s+(\d+)\s*$/)
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
482 {
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
483 push(@farray, {"lat" => $1, "long" => $2, "time" => $3});
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
484 }
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
485 }
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
486
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
487 my $findex = 0;
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
488 foreach my $fline (split(/\n/, $fshit->{"gml:rangeSet"}{"gml:DataBlock"}{"gml:doubleOrNilReasonTupleList"}))
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
489 {
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
490 if ($fline =~ /^\s*([\+\-]?\d+\.\d*|NaN)\s+([\+\-]?\d+\.\d*|NaN)\s*$/)
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
491 {
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
492 $farray[$findex]{"temp"} = $1 if ($1 ne "NaN");
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
493 $farray[$findex]{"humidity"} = $2 if ($2 ne "NaN");
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
494 $findex++;
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
495 }
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
496 }
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
497 # XXX Hashify the array into lat/long keys
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
498
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
499 # This is horrible :S
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
500 my $fcrap = $fdata->{"om:featureOfInterest"}{"sams:SF_SpatialSamplingFeature"}{"sams:shape"}{"gml:MultiPoint"}{"gml:pointMember"};
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
501 foreach my $xnode (@{$fcrap})
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
502 {
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
503 my $floc = $xnode->{"gml:Point"};
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
504 if ($floc->{"gml:pos"} =~ /^\s*([\+\-]?\d+\.\d*)\s+([\+\-]?\d+\.\d*)\s*$/)
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
505 {
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
506 my ($flat, $flong) = ($1, $2);
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
507 # Should use a hash -
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
508 foreach my $flol (@farray)
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
509 {
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
510 if ($flol->{"lat"} == $flat && $flol->{"long"} == $flong)
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
511 {
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
512 $weatherdata->{$floc->{"gml:name"}} =
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
513 [
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
514 1,
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
515 plonk_data($flol->{"time"}),
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
516 plonk_data($flol->{"temp"}),
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
517 "",
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
518 "",
228
fe420576f284 The kludges intensify.
Matti Hamalainen <ccr@tnsp.org>
parents: 213
diff changeset
519 plonk_data($flol->{"humidity"}),
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
520 ];
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
521 }
189
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
522 }
3af8ea19a3e2 Improved weather fetching.
Matti Hamalainen <ccr@tnsp.org>
parents: 186
diff changeset
523 }
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
524 }
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
525 } else {
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
526 # defined
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
527 print STDERR "Invalid XML received:\n";
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
528 print STDERR $res->decoded_content."\n\n";
169
c12e422a543a FMI data parsing kinda almost works. It is horrible, though. XML ... :Q
Matti Hamalainen <ccr@tnsp.org>
parents: 168
diff changeset
529 }
208
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
530 } else {
c6bbf69afc57 Add some error handling and debugging.
Matti Hamalainen <ccr@tnsp.org>
parents: 206
diff changeset
531 print STDERR "Error fetching FMI XML: ".$res->status_line."\n";
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
532 }
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
533 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
534
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
535
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
536 ###
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
537 ### Output
178
9b8ec700ede4 Clean up the weather data parser backend a bit.
Matti Hamalainen <ccr@tnsp.org>
parents: 170
diff changeset
538 ###
164
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
539 if (opt_chk_valid("outfile", 1)) {
d5a0e4248f3e weather: Implement configuration file support for weather fetcher backend.
Matti Hamalainen <ccr@tnsp.org>
parents: 162
diff changeset
540 open(STDOUT, '>', opt_get("outfile")) or die("Could not open output file '".opt_get("outfile")."'.\n");
158
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
541 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
542
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
543 binmode STDOUT, ':encoding(utf-8)';
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
544
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
545 foreach my $key (sort { $a cmp $b } keys %$weatherdata)
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
546 {
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
547 print STDOUT $key."|".join("|", @{$weatherdata->{$key}})."\n";
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
548 }
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
549
db2b1f74d994 Initial import of weather data fetching backend, written in Perl.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff changeset
550 close(STDOUT);