changeset 139:3305e142eecc

Change feed fetcher to use SQLite3 backend.
author Matti Hamalainen <ccr@tnsp.org>
date Tue, 04 Jun 2013 13:35:46 +0300
parents 743c1bea2498
children b0648e05c855
files hae_feedit.tcl
diffstat 1 files changed, 26 insertions(+), 45 deletions(-) [+]
line wrap: on
line diff
--- a/hae_feedit.tcl	Tue Jun 04 13:24:30 2013 +0300
+++ b/hae_feedit.tcl	Tue Jun 04 13:35:46 2013 +0300
@@ -4,15 +4,17 @@
 #
 ##########################################################################
 #
-# FeedCheck fetcher v0.7 by Matti 'ccr' Hamalainen <ccr@tnsp.org>
-# (C) Copyright 2008-2011 Tecnic Software productions (TNSP) 
+# FeedCheck fetcher v0.8 by Matti 'ccr' Hamalainen <ccr@tnsp.org>
+# (C) Copyright 2008-2013 Tecnic Software productions (TNSP) 
 #
 # This script is freely distributable under GNU GPL (version 2) license.
 #
 ##########################################################################
+package require sqlite3
+source [file dirname [info script]]/util_convert.tcl
 
-# Datafile, MUST be set to same as in feeds.tcl
-set datafile "/home/niinuska/bot/data.feeds"
+# SQLite3 database, MUST be set to same as in feeds.tcl
+set feeds_db "/home/niinuska/bot/feeds.sqlite"
 
 # Use a HTTP proxy? 1 = yes, 0 = no
 set http_proxy 0
@@ -23,6 +25,7 @@
 
 
 ##########################################################################
+
 set html_ent [split "&#160;| |&raquo;|>>|&quot;|\"|&auml;|ä|&ouml;|ö|&Auml;|Ä|&Ouml;|Ö|&amp;|&|&lt;|<|&gt;|>|ä|ä|ö|ö|Ä|Ä" "|"]
 
 package require http
@@ -38,13 +41,17 @@
 }
 
 
-proc add_entry {uname uprefix uurl udesc} {
-  global entries isentries newurls currclock
+proc add_entry {uname uprefix uurl utitle} {
+  global currclock dbh
   set utest "$uprefix[convert_ent $uurl]"
-  set isentries($utest) 1
-  if {[catch {set utmp $entries($utest)}]} {
-    set entries($utest) [list $currclock $uname $utest [convert_ent $udesc]]
-    incr newurls
+
+  set usql "SELECT title FROM feed_data WHERE url='[escape $utest]' AND feed='[escape $uname]'"
+  if {![dbh exists $usql]} {
+    set usql "INSERT INTO feed_data (feed,utime,url,title) VALUES ('[escape $uname]', $currclock, '[escape $utest]', '[escape $utitle]')"
+    if {[catch {dbh eval $usql} uerrmsg]} {
+      puts "\nError: $uerrmsg on:\n$usql"
+      exit 15
+    }
   }
 }
 
@@ -82,23 +89,13 @@
 }
 
 
-##########################################################################
-### Read in old data
-set oldurls 0
-set newurls 0
-if {![catch {set ufile [open $datafile r 0600]} uerrmsg]} {
-  while {![eof $ufile]} {
-    gets $ufile uline
-    set urec [split $uline "½"]
-    if {[llength $urec] == 4} {
-      set entries([lindex $urec 2]) $urec
-      set isentries([lindex $urec 2]) 0
-      incr oldurls
-    }
-  }
-  close $ufile
+### Open database, etc
+set currclock [clock seconds]
+global dbh
+if {[catch {sqlite3 dbh $feeds_db} uerrmsg]} {
+  puts "Could not open SQLite3 database '$feeds_db': $uerrmsg."
+  exit 2
 }
-set currclock [clock seconds]
 
 
 ##############################################################################
@@ -172,23 +169,7 @@
 
 #add_rss_feed "http://lehti.samizdat.info/feed/" "Lehti" ""
 
-##############################################################################
-### Open data file for writing
-set tmpfname "$datafile.tmp"
-if {[catch {set outfile [open $tmpfname w 0600]} uerrmsg]} {
-  puts "Error opening $tmpfname for writing: $uerrmsg"
-  return 1
-}
+
 
-set uexpire [expr [clock seconds] - (7*24*60*60)]
-foreach {ukey udata} [array get entries] {
-#  if {$isentries($ukey) != 0 || [lindex $udata 0] >= $uexpire} {
-    puts $outfile [join $udata "½"]
-#  }
-}
-
-close $outfile
-if {[catch {file rename -force -- $tmpfname $datafile} uerrmsg]} {
-  puts "Error renaming $tmpfname to $datafile: $uerrmsg"
-}
-#puts "$newurls new entries."
+### Close database
+dbh close