Mercurial > hg > lukkari
annotate parsedata.pl @ 4:dd2bce7ec0c8
Removed http fetching feature, it's not really useful here.
author | Matti Hamalainen <ccr@tnsp.org> |
---|---|
date | Tue, 11 Jan 2011 20:55:09 +0200 |
parents | 3d0904d29591 |
children | a52a0bdb5ea1 |
rev | line source |
---|---|
1 | 1 #!/usr/bin/perl -w |
2 # | |
3 # Fetch and parse HTML format class timetable into more sane formats | |
4 # (C) Copyright 2010-2010 Matti Hämäläinen <ccr@tnsp.org> | |
5 # | |
6 use strict; | |
7 use Data::Dumper; | |
8 use HTML::Entities; | |
9 | |
10 | |
11 sub urlencode($) | |
12 { | |
13 my $value = $_[0]; | |
14 $value =~ s/([^a-zA-Z_0-9 ])/"%" . uc(sprintf "%lx" , unpack("C", $1))/eg; | |
15 $value =~ tr/ /+/; | |
16 return $value; | |
17 } | |
18 | |
19 | |
20 sub str_trim($) | |
21 { | |
22 my $str = $_[0]; | |
23 if (defined($str)) { | |
24 $str =~ s/^\s*//; | |
25 $str =~ s/\s*$//; | |
26 } | |
27 return $str; | |
28 } | |
29 | |
30 | |
31 sub pop_token_a($) | |
32 { | |
33 my $tokens = $_[0]; | |
34 return shift(@$tokens); | |
35 } | |
36 | |
37 | |
38 sub pop_token($) | |
39 { | |
40 return str_trim(pop_token_a($_[0])); | |
41 } | |
42 | |
43 | |
44 sub parse_html_str($) | |
45 { | |
46 my $tokens = $_[0]; | |
47 my $token = pop_token($tokens); | |
48 my $str = ""; | |
49 $token =~ s/^\s*//; | |
50 | |
51 return undef unless (substr($token, 0, 1) eq '"'); | |
52 $token = substr($token, 1); | |
53 | |
54 while (defined($token)) { | |
55 if (substr($token, -1) eq '"') { | |
56 $str .= substr($token, 0, -1); | |
57 return $str; | |
58 } else { | |
59 $str .= $token; | |
60 } | |
61 $token = shift(@$tokens); | |
62 } | |
63 return undef; | |
64 } | |
65 | |
66 | |
67 sub parse_html_tree($$); | |
68 | |
69 sub parse_html_tree($$) | |
70 { | |
71 my ($tokens, $tree) = @_; | |
72 | |
73 while (my $token = pop_token($tokens)) { | |
74 if ($token =~ /^<[!\/]?[a-zA-Z]+/) { | |
75 $token = lc($token); | |
76 if ($token =~ /^<\!.*>$/) { | |
77 # Ignore comments etc. | |
78 } elsif ($token =~ /^<([a-z]+)(.*)>$/) { | |
79 my ($name, $args) = ($1, $2); | |
80 if ($name eq "meta" || $name eq "img") { | |
81 my $tmp = {}; | |
82 parse_html_tree($tokens, $tree); | |
83 return $tree; | |
84 } else { | |
85 my $tmp = { "name" => $name, "args" => str_trim($args) }; | |
86 parse_html_tree($tokens, $tmp); | |
87 push(@{$$tree{"nodes"}}, $tmp); | |
88 } | |
89 } elsif ($token =~ /^<\/([a-z]+)>$/) { | |
90 return $tree; | |
91 } else { | |
92 die("HORROR TERROR ELITE: $token\n"); | |
93 } | |
94 } else { | |
95 $token = str_trim(decode_entities($token)); | |
96 push(@{$$tree{"nodes"}}, { "name" => "text", "args" => "", "text" => $token }) if length($token) > 0; | |
97 } | |
98 } | |
99 | |
100 return $tree; | |
101 } | |
102 | |
103 | |
104 sub parse_html($) | |
105 { | |
106 return undef unless defined($_[0]); | |
107 my $str = $_[0]; | |
108 my $res = { "name" => "", "args" => "" }; | |
109 $str =~ tr/\r/ /; | |
110 $str =~ tr/\n/ /; | |
111 my @tokens = grep { !($_ =~ /^\s*$/) } split(/(<\/?[a-zA-Z]+.*?>)/, $str); | |
112 parse_html_tree(\@tokens, $res); | |
113 return $res; | |
114 } | |
115 | |
116 sub html_find_node($$$); | |
117 | |
118 sub html_find_node($$$) | |
119 { | |
120 my ($node, $name, $args) = @_; | |
121 | |
122 if (defined($node)) { | |
123 if (ref($node) eq "ARRAY") { | |
124 foreach my $n (@$node) { | |
125 my $tmp = html_find_node($n, $name, $args); | |
126 # Must do it like this, in order not to break the loop | |
127 return $tmp if defined($tmp); | |
128 } | |
129 } elsif (ref($node) eq "HASH") { | |
130 if (defined($$node{"name"})) { | |
131 if ($$node{"name"} eq $name) { | |
132 if ($args ne "") { | |
133 if (defined($$node{"args"}) && $$node{"args"} =~ /$args/) { | |
134 } else { | |
135 return undef; | |
136 } | |
137 } | |
138 return $node; | |
139 } else { | |
140 return html_find_node($$node{"nodes"}, $name, $args); | |
141 } | |
142 } | |
143 } | |
144 } | |
145 return undef; | |
146 } | |
147 | |
148 | |
149 sub fnode($$) | |
150 { | |
151 return html_find_node($_[0], $_[1], ""); | |
152 } | |
153 | |
154 | |
155 sub fnodea($$$) | |
156 { | |
157 return html_find_node($_[0], $_[1], $_[2]); | |
158 } | |
159 | |
160 | |
161 sub escape($) | |
162 { | |
163 my $s = $_[0]; | |
164 $s =~ s/(['"])/\\$1/g; | |
165 return $s; | |
166 } | |
167 | |
168 | |
169 sub html_collapse($$); | |
170 | |
171 sub html_collapse($$) | |
172 { | |
173 my ($node, $strip) = @_; | |
174 my $str = ""; | |
175 | |
176 if ($$node{"name"} eq "text") { | |
177 $str .= $$node{"text"}; | |
178 } else { | |
179 $str .= "<".$$node{"name"}.">" unless ($strip); | |
180 foreach my $n (@{$$node{"nodes"}}) { | |
181 $str .= html_collapse($n, $strip); | |
182 } | |
183 $str .= "</".$$node{"name"}.">" unless ($strip); | |
184 } | |
185 | |
186 return $str; | |
187 } | |
188 | |
189 | |
190 ### | |
191 ### Main program | |
192 ### | |
193 my $modes = "simple|php|xml"; | |
194 my $opt_mode = "php"; | |
195 my $opt_dump = 0; | |
196 my $opt_filename; | |
197 my $opt_outfile; | |
198 | |
199 while (defined(my $arg = shift)) { | |
200 if (substr($arg, 0, 1) eq "-") { | |
201 if ($arg =~ /^-($modes)$/o) { | |
202 $opt_mode = $1; | |
203 } | |
204 elsif ($arg eq "-dump") { | |
205 $opt_dump = 1; | |
206 } | |
207 elsif ($arg eq "-o") { | |
208 $opt_outfile = shift or die("Output filename option -o requires an argument.\n"); | |
209 } else { | |
210 die("Invalid option '$arg'.\n"); | |
211 } | |
212 } else { | |
213 $opt_filename = $arg; | |
214 } | |
215 } | |
216 | |
4
dd2bce7ec0c8
Removed http fetching feature, it's not really useful here.
Matti Hamalainen <ccr@tnsp.org>
parents:
3
diff
changeset
|
217 die("Usage: $0 [options] <filename> |
1 | 218 |
219 -php Output a PHP include file with data in arrays (default) | |
220 -simple Output simple tabled output for easy parsing. | |
221 -xml Output XML. | |
222 | |
223 -o <filename> Set output filename. Default is to use stdout. | |
224 | |
225 -dump Dump HTML tree to stdout and quit. | |
226 | |
227 ") unless defined($opt_filename); | |
228 | |
229 | |
230 my $data; | |
4
dd2bce7ec0c8
Removed http fetching feature, it's not really useful here.
Matti Hamalainen <ccr@tnsp.org>
parents:
3
diff
changeset
|
231 open(my $fh, '<', $opt_filename) or die("Error opening '$opt_filename': $!\n"); |
dd2bce7ec0c8
Removed http fetching feature, it's not really useful here.
Matti Hamalainen <ccr@tnsp.org>
parents:
3
diff
changeset
|
232 $data = do { local $/; <$fh> }; |
dd2bce7ec0c8
Removed http fetching feature, it's not really useful here.
Matti Hamalainen <ccr@tnsp.org>
parents:
3
diff
changeset
|
233 close($fh); |
1 | 234 |
235 die("No data in input.\n") unless (defined($data) && $data ne ""); | |
236 | |
237 | |
238 # Filter out certain unneeded elements | |
239 $data =~ s/<font[^>]*>//ig; | |
240 $data =~ s/<\/font>//ig; | |
241 $data =~ s/<\/?center>//ig; | |
242 $data =~ s/<br>//ig; | |
243 $data =~ s/ / /ig; | |
244 | |
245 ### Get some general information | |
246 my $otree = parse_html($data); | |
247 if ($opt_dump) { | |
248 print Dumper(fnode($otree, "html")); | |
249 exit; | |
250 } | |
251 | |
252 my %class = (); | |
253 my $body = fnode($otree, "body"); | |
254 if (defined($body) && defined($$body{"nodes"})) { | |
255 foreach my $n (@{$$body{"nodes"}}) { | |
256 if ($$n{"name"} eq "text") { | |
257 push(@{$class{"info"}}, $$n{"text"}); | |
258 } | |
259 elsif ($$n{"name"} eq "b") { | |
260 push(@{$class{"data"}}, $n); | |
261 } | |
262 } | |
263 } | |
264 | |
265 # Filter out some more, for easier tree access during table parsing | |
266 $data =~ s/<\/?b>//ig; | |
267 my $tree = parse_html($data); | |
268 my $node = fnode(fnode($tree, "body"), "table"); | |
269 die("No table element found in document. Perhaps the format has changed? :(\n") unless defined($node); | |
270 | |
271 | |
272 ### Parse through the HTML document node tree to find the data we need | |
273 my $id = 0; | |
274 my $q = $$node{"nodes"}; | |
275 my $tunnit = {}; | |
276 my $taulu = {}; | |
277 my $maxdays = 6; | |
278 my $maxhours = 0; | |
279 | |
280 # Skip zero this way | |
281 for (my $i = 1; $i < scalar(@{$q}); $i++) { | |
282 my $d = $$q[$i]{"nodes"}; | |
283 if (defined($d)) { | |
284 foreach my $n (@{$d}) { | |
285 my $l = $$n{"nodes"}[0]{"nodes"}; | |
286 if (defined($l) && $$n{"args"} =~ /colspan=6\s+rowspan=(\d+)/) { | |
287 my $tuntia = $1 / 2; | |
288 my $data = []; | |
289 my $grouped = 0; | |
290 foreach my $h (@{$l}) { | |
291 if (defined($$h{"nodes"})) { | |
292 foreach my $b (@{$$h{"nodes"}}) { | |
293 if (defined($$b{"nodes"})) { | |
294 my $text = $$b{"nodes"}[0]{"text"}; | |
295 $text =~ s/\.$//; | |
296 | |
297 $grouped = 1 if ($text =~ /vuorov/); | |
298 | |
299 push(@$data, $text); | |
300 } | |
301 } | |
302 } | |
303 } | |
304 | |
305 my $tid; | |
306 if (scalar(@$data) > 0) { | |
307 $id++; | |
308 $tid = $id; | |
309 } else { | |
310 $tid = 0; | |
311 } | |
312 | |
313 my $tpd = 0; | |
314 for (my $x = 0; $x < $maxdays; $x++) { | |
315 if (!defined($$taulu{$maxhours}{$x})) { | |
316 $tpd = $x; | |
317 last; | |
318 } | |
319 } | |
320 for (my $t = 0; $t < $tuntia; $t++) { | |
321 $$taulu{$maxhours + $t}{$tpd} = $tid; | |
322 } | |
323 | |
324 if (scalar(@$data) > 0) { | |
325 # Grouped, if there is another class ID in second slot | |
326 $grouped = 1 if ($$data[1] =~ /^[A-Z]\d{6}$/); | |
327 $$tunnit{$id} = { "grouped" => $grouped, "day" => $tpd, "start" => $maxhours, "hours" => $tuntia, "data" => $data }; | |
328 } | |
329 } | |
330 } | |
331 $maxhours++; | |
332 } | |
333 } | |
334 | |
335 | |
336 ### Open output file, if specified | |
337 if (defined($opt_outfile)) { | |
338 open(STDOUT, '>', $opt_outfile) or die("Could not open output file '$opt_outfile'.\n"); | |
339 } | |
340 | |
341 | |
342 ### Output data in desired format | |
343 if ($opt_mode eq "php") { | |
344 print "<?\n". | |
345 "\$classInfo = array(\n". | |
346 " \"general\" => array(".join(", ", map { "\"".escape($_)."\""; } @{$class{"info"}})."),\n". | |
347 " \"info\" => array(".join(", ", map { "\"".escape(html_collapse($_, 1))."\""; } @{$class{"data"}})."),\n". | |
348 " \"info_tags\" => array(".join(", ", map { "\"".escape(html_collapse($_, 0))."\""; } @{$class{"data"}})."),\n". | |
349 " \"maxdays\" => $maxdays,\n". | |
350 " \"maxhours\" => $maxhours,\n". | |
351 ");\n\n"; | |
352 | |
353 print "\$classDefs = array(\n"; | |
354 foreach my $id (sort { $a <=> $b } keys %{$tunnit}) { | |
355 print " $id => array("; | |
356 foreach my $key (keys %{$$tunnit{$id}}) { | |
357 my $a = $$tunnit{$id}{$key}; | |
358 print "\"$key\" => "; | |
359 if (ref($a) eq "ARRAY") { | |
360 print "array(".join(", ", map { "\"".escape($_)."\""; } @$a).")"; | |
361 } | |
362 elsif ($a =~ /^\d+$/) { | |
363 print $a; | |
364 } else { | |
365 print "\"".escape($a)."\""; | |
366 } | |
367 print ", "; | |
368 } | |
369 print "),\n"; | |
370 } | |
371 | |
372 print ");\n". | |
373 "\n". | |
374 "\$classHourTable = array(\n"; | |
375 for (my $y = 0; $y < $maxhours; $y++) { | |
376 my $str = ""; | |
377 for (my $x = 0; $x < $maxdays; $x++) { | |
378 $str .= ", " unless ($str eq ""); | |
379 $str .= sprintf "%3d", $$taulu{$y}{$x}; | |
380 } | |
381 print " array(".$str."),\n"; | |
382 } | |
383 print ");\n?>\n"; | |
384 } | |
385 | |
386 elsif ($opt_mode eq "xml") { | |
387 print "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n". | |
388 "<timetable>\n". | |
389 " <class>\n". | |
390 " <general>".join("", map { "<node>".encode_entities($_)."</node>"; } @{$class{"info"}})."</general>\n". | |
391 " <info>".join("", map { "<node>".encode_entities(html_collapse($_, 1))."</node>"; } @{$class{"data"}})."</info>\n". | |
392 " <maxdays>$maxdays</maxdays>\n". | |
393 " <maxhours>$maxhours</maxhours>\n". | |
394 " </class>\n"; | |
395 | |
396 | |
397 print "</timetable>\n"; | |
398 } | |
399 | |
400 | |
401 | |
402 elsif ($opt_mode eq "simple") { | |
403 for (my $y = 0; $y < $maxhours; $y++) { | |
404 for (my $x = 0; $x < $maxdays; $x++) { | |
405 printf "%3d ", $$taulu{$y}{$x}; | |
406 } | |
407 print "\n"; | |
408 } | |
409 } | |
410 | |
411 close (STDOUT); |