view update.sh @ 193:16ce445c499a v3 tip

Import v3 branch.
author Matti Hamalainen <ccr@tnsp.org>
date Sun, 22 Jan 2017 02:31:10 +0200
parents b4a07ea2d739
children
line wrap: on
line source

#!/bin/sh
URLSUFFIX=".htm"

createdir()
{
  test ! -d "$1" && mkdir -p "$1" && chmod 751 "$1"
}


fetch()
{
  URLPREFIX="$1"
  CLASSFILE="$2"
  LISTFILE="$2.tmp"
  PATPREFIX="$3"

  if wget -q -O "$LISTFILE" "$URLPREFIX"; then
    perl -w ./parselist.pl "$PATPREFIX" "$URLSUFFIX" < "$LISTFILE" > "$CLASSFILE"
    rm -f "$LISTFILE"
  fi
}


parse()
{
  # Create cache directories, if they do not exist
  OLDCACHEDIR="${CACHEDIR}old/"
  createdir "${OLDCACHEDIR}"
  createdir "${CACHEDIR}"

  URLPREFIX="$1"
  CLASSFILE="$2"
  PATPREFIX="$3"

  echo "<? \$origBaseURI = \"${URLPREFIX}${PATPREFIX}\"; \$origBaseExt = \".htm\"; ?>" > "${CACHEDIR}baseuri.data";

  test -e "$CLASSFILE" || return 0
  
  echo "* Processing $CLASSFILE ..."
  cat "$CLASSFILE" | while read i; do
    parse=no
    INFILE="${CACHEDIR}${i}.html"
    ONFILE="${OLDCACHEDIR}${i}.html"
    DATAFILE="${i}.data"
    wget -q -O "${INFILE}.new" "${URLPREFIX}${PATPREFIX}${i}${URLSUFFIX}"

    if test -e "${INFILE}.new"; then
      # New data fetched, does old file exist?
      if test -e "$INFILE"; then
        # Yes, do a diff
        if ! diff -u "$INFILE" "$INFILE.new" > "$INFILE.diff"; then
          # There were differences, do a parse
          parse=yes
          mv "$INFILE" "$ONFILE" && \
          mv "$INFILE.new" "$INFILE"
        else
          # No changes, apparently .. remove the new one
          rm -f "$INFILE.new" "$INFILE.diff"
        fi
      else
        # No old file, parse new data
        mv "$INFILE.new" "$INFILE"
        parse=yes
      fi
    fi

    # No new file fetched, does datafile exist?
    if test ! -e "${CACHEDIR}${DATAFILE}"; then
      # No, try to parse it if old file input exists
      parse=yes
    fi

    # Parsing of old data requested?
    if test "x$parse" = "xyes" -a -e "$INFILE"; then
      if test -e "${CACHEDIR}${DATAFILE}"; then
        mv "${CACHEDIR}${DATAFILE}" "${OLDCACHEDIR}${DATAFILE}"
      fi
      echo "Parsing $i"
      perl parsedata.pl -php "$INFILE" -o "${CACHEDIR}${DATAFILE}"
      #perl parsedata.pl -xml "$INFILE" -o "${CACHEDIR}${i}.xml"
    fi
  done
}

fix_permissions()
{
  chmod 751 . && chmod 644 *.php *.css && chmod 600 coursecache.txt classes.txt classes_next.txt
}

fix_permissions
CACHEDIR="cache/"
fetch "http://www.oamk.fi/~heikkim/riihi1/luokat/" "classes.txt" "Ryh._"
parse "http://www.oamk.fi/~heikkim/riihi1/luokat/" "classes.txt" "Ryh._"
#parse "http://www.oamk.fi/tyojarjestykset/otek/luokat/" "classes.txt" "OR_"


CACHEDIR="cache-next/"
#fetch "http://www.oamk.fi/~heikkim/riihi1/luokat/" "classes.txt" "Ryh._"
#parse "http://www.oamk.fi/~heikkim/riihi1/luokat/" "classes.txt" "Ryh._"

fix_permissions
diff -u "classes.txt" "classes_next.txt"

# http://www.oamk.fi/~heikkim/riihi[1-5]/
# http://www.oamk.fi/~heikkim/Luhti[1-5]/