Mercurial > hg > lukkari
annotate update.sh @ 82:c553ad61e9c2
Make original data links work for both current and next period mode.
author | Matti Hamalainen <ccr@tnsp.org> |
---|---|
date | Wed, 24 Oct 2012 12:24:48 +0300 |
parents | d1b65d9903ab |
children | acb600abc0c7 |
rev | line source |
---|---|
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
1 #!/bin/sh |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
2 URLSUFFIX=".htm" |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
3 |
76
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
4 function createdir() |
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
5 { |
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
6 if test ! -d "$1"; then |
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
7 mkdir -p "$1" && chmod 751 "$1" |
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
8 fi |
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
9 } |
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
10 |
24
1b8260151e99
Get updates / data from upcoming work-in-progress timetables for next period also.
Matti Hamalainen <ccr@tnsp.org>
parents:
22
diff
changeset
|
11 function parse() |
1b8260151e99
Get updates / data from upcoming work-in-progress timetables for next period also.
Matti Hamalainen <ccr@tnsp.org>
parents:
22
diff
changeset
|
12 { |
46
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
13 # Create cache directories, if they do not exist |
47
ceca06576918
Change backup (old) cache directories.
Matti Hamalainen <ccr@tnsp.org>
parents:
46
diff
changeset
|
14 OLDCACHEDIR="${CACHEDIR}old/" |
76
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
15 createdir "${OLDCACHEDIR}" |
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
16 createdir "${CACHEDIR}" |
46
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
17 |
24
1b8260151e99
Get updates / data from upcoming work-in-progress timetables for next period also.
Matti Hamalainen <ccr@tnsp.org>
parents:
22
diff
changeset
|
18 URLPREFIX="$1" |
1b8260151e99
Get updates / data from upcoming work-in-progress timetables for next period also.
Matti Hamalainen <ccr@tnsp.org>
parents:
22
diff
changeset
|
19 CLASSFILE="$2" |
1b8260151e99
Get updates / data from upcoming work-in-progress timetables for next period also.
Matti Hamalainen <ccr@tnsp.org>
parents:
22
diff
changeset
|
20 LISTFILE="$2.tmp" |
29
ac51fc10414f
Add support for URL prefix pattern in updates.
Matti Hamalainen <ccr@tnsp.org>
parents:
24
diff
changeset
|
21 PATPREFIX="$3" |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
22 |
82
c553ad61e9c2
Make original data links work for both current and next period mode.
Matti Hamalainen <ccr@tnsp.org>
parents:
76
diff
changeset
|
23 echo "<? \$origBaseURI = \"${URLPREFIX}${PATPREFIX}\"; \$origBaseExt = \".htm\"; ?>" > "${CACHEDIR}baseuri.data"; |
c553ad61e9c2
Make original data links work for both current and next period mode.
Matti Hamalainen <ccr@tnsp.org>
parents:
76
diff
changeset
|
24 |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
25 if wget -q -O "$LISTFILE" "$URLPREFIX"; then |
29
ac51fc10414f
Add support for URL prefix pattern in updates.
Matti Hamalainen <ccr@tnsp.org>
parents:
24
diff
changeset
|
26 perl -ne "if (/<a href=\"${PATPREFIX}([A-Z]{3}\d\S+)${URLSUFFIX}\">/) { print \"\$1\n\"; }" < "$LISTFILE" > "$CLASSFILE" |
34 | 27 echo -n "* Fetched classfile $CLASSFILE: " |
28 cat "$CLASSFILE" | wc -l | |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
29 fi |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
30 |
24
1b8260151e99
Get updates / data from upcoming work-in-progress timetables for next period also.
Matti Hamalainen <ccr@tnsp.org>
parents:
22
diff
changeset
|
31 if test -e "$CLASSFILE"; then |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
32 cat "$CLASSFILE" | while read i; do |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
33 parse=no |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
34 INFILE="${CACHEDIR}${i}.html" |
46
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
35 ONFILE="${OLDCACHEDIR}${i}.html" |
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
36 DATAFILE="${i}.data" |
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
37 wget -q -O "${INFILE}.new" "${URLPREFIX}${PATPREFIX}${i}${URLSUFFIX}" |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
38 |
46
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
39 if test -e "${INFILE}.new"; then |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
40 # New data fetched, does old file exist? |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
41 if test -e "$INFILE"; then |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
42 # Yes, do a diff |
22
f025d36926a1
Make backup copies of old versions of data files when updating.
Matti Hamalainen <ccr@tnsp.org>
parents:
21
diff
changeset
|
43 if ! diff -u "$INFILE" "$INFILE.new" > "$INFILE.diff"; then |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
44 # There were differences, do a parse |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
45 parse=yes |
46
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
46 mv "$INFILE" "$ONFILE" && \ |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
47 mv "$INFILE.new" "$INFILE" |
46
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
48 else |
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
49 # No changes, apparently .. remove the new one |
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
50 rm -f "$INFILE.new" "$INFILE.diff" |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
51 fi |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
52 else |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
53 # No old file, parse new data |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
54 mv "$INFILE.new" "$INFILE" |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
55 parse=yes |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
56 fi |
46
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
57 fi |
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
58 # No new file fetched, does datafile exist? |
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
59 if test ! -e "${CACHEDIR}${DATAFILE}"; then |
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
60 # No, try to parse it if old file input exists |
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
61 parse=yes |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
62 fi |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
63 |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
64 # Parsing of old data requested? |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
65 if test "x$parse" = "xyes" -a -e "$INFILE"; then |
46
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
66 if test -e "${CACHEDIR}${DATAFILE}"; then |
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
67 mv "${CACHEDIR}${DATAFILE}" "${OLDCACHEDIR}${DATAFILE}" |
22
f025d36926a1
Make backup copies of old versions of data files when updating.
Matti Hamalainen <ccr@tnsp.org>
parents:
21
diff
changeset
|
68 fi |
34 | 69 echo "Parsing $i" |
46
9b5fa0f3812b
Improvements in the update logic.
Matti Hamalainen <ccr@tnsp.org>
parents:
42
diff
changeset
|
70 perl parsedata.pl -php "$INFILE" -o "${CACHEDIR}${DATAFILE}" |
75
3d9e42477367
More improvements in the parsing and XML output.
Matti Hamalainen <ccr@tnsp.org>
parents:
65
diff
changeset
|
71 perl parsedata.pl -xml "$INFILE" -o "${CACHEDIR}${i}.xml" |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
72 fi |
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
73 done |
24
1b8260151e99
Get updates / data from upcoming work-in-progress timetables for next period also.
Matti Hamalainen <ccr@tnsp.org>
parents:
22
diff
changeset
|
74 fi |
1b8260151e99
Get updates / data from upcoming work-in-progress timetables for next period also.
Matti Hamalainen <ccr@tnsp.org>
parents:
22
diff
changeset
|
75 } |
6
7fca87c41e17
Added data fetching and updating shellscript.
Matti Hamalainen <ccr@tnsp.org>
parents:
diff
changeset
|
76 |
76
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
77 # Fix permissions |
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
78 chmod 751 . && chmod 644 *.php *.css && chmod 600 coursecache.txt classes.txt classes_next.txt |
d1b65d9903ab
Clean up the update script slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
75
diff
changeset
|
79 |
35
4d9354abda73
Update fetching URLs and parameters to match the latest changes on OAMK's web.
Matti Hamalainen <ccr@tnsp.org>
parents:
34
diff
changeset
|
80 CACHEDIR="cache/" |
65 | 81 parse "http://www.oamk.fi/tyojarjestykset/otek/luokat/" "classes.txt" "OR_" |
82 #parse "http://www.oamk.fi/~heikkim/riihi2/Oppilaat/" "classes.txt" "Ryh._" | |
29
ac51fc10414f
Add support for URL prefix pattern in updates.
Matti Hamalainen <ccr@tnsp.org>
parents:
24
diff
changeset
|
83 |
31
dbe7ff545293
Add support for fetching and showing data for next/upcoming period.
Matti Hamalainen <ccr@tnsp.org>
parents:
29
diff
changeset
|
84 CACHEDIR="cache-next/" |
65 | 85 parse "http://www.oamk.fi/~heikkim/riihi2/Oppilaat/" "classes_next.txt" "Ryh._" |