Mercurial > hg > lukkari
changeset 6:7fca87c41e17
Added data fetching and updating shellscript.
author | Matti Hamalainen <ccr@tnsp.org> |
---|---|
date | Tue, 11 Jan 2011 21:58:52 +0200 |
parents | 3d6a83eaa2ba |
children | 2bb40c5945bc |
files | update.sh |
diffstat | 1 files changed, 46 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/update.sh Tue Jan 11 21:58:52 2011 +0200 @@ -0,0 +1,46 @@ +#!/bin/sh +URLPREFIX="http://www.oamk.fi/tyojarjestykset/otek/luokat/" +URLSUFFIX=".htm" +CACHEDIR="cache/" +LISTFILE="luokat.tmp" +CLASSFILE="luokat.txt" + + +if wget -q -O "$LISTFILE" "$URLPREFIX"; then + perl -ne 'if (/<a href="OR_([A-Z]{3}\d\S+)\.htm">/) { print "$1\n"; }' < "$LISTFILE" > "$CLASSFILE" +fi + + +cat "$CLASSFILE" | while read i; do + parse=no + INFILE="${CACHEDIR}${i}.html" + wget -q -O "$INFILE.new" "${URLPREFIX}OR_${i}${URLSUFFIX}" + + if test -e "$INFILE.new"; then + # New data fetched, does old file exist? + if test -e "$INFILE"; then + # Yes, do a diff + if diff "$INFILE" "$INFILE.new"; then + # There were differences, do a parse + parse=yes + mv "$INFILE.new" "$INFILE" + fi + else + # No old file, parse new data + mv "$INFILE.new" "$INFILE" + parse=yes + fi + else + # No new file fetched, does datafile exist? + if test ! -e "$i.data"; then + # No, try to parse it if old file input exists + parse=yes + fi + fi + + # Parsing of old data requested? + if test "x$parse" = "xyes" -a -e "$INFILE"; then + perl parsedata.pl -php "$INFILE" -o "${CACHEDIR}/$i.data" + fi +done +