Mercurial > hg > egg-tcls

changeset 141:c99df41a691a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Work on making the feeds script use the SQLite3 backend.
author Matti Hamalainen <ccr@tnsp.org>
date Tue, 04 Jun 2013 15:52:32 +0300
parents b0648e05c855
children 4c51eeba993f
files feeds.tcl
diffstat 1 files changed, 74 insertions(+), 84 deletions(-) [+]
line wrap: on
line diff
--- a/feeds.tcl	Tue Jun 04 13:45:16 2013 +0300
+++ b/feeds.tcl	Tue Jun 04 15:52:32 2013 +0300
@@ -1,7 +1,7 @@
 ##########################################################################
 #
-# FeedCheck v0.7 by Matti 'ccr' Hamalainen <ccr@tnsp.org>
-# (C) Copyright 2008-2010 Tecnic Software productions (TNSP) 
+# FeedCheck v0.8 by Matti 'ccr' Hamalainen <ccr@tnsp.org>
+# (C) Copyright 2008-2013 Tecnic Software productions (TNSP)
 #
 # Requires get_feeds.tcl to be run as a cronjob, for example
 # 15 * * * *     /absolute/path/to/get_feeds.tcl
@@ -10,15 +10,16 @@
 #
 ##########################################################################
 
-# Datafile, MUST be set to same as in get_feeds.tcl
-set feeds_datafile "/home/niinuska/bot/data.feeds"
+# SQLite3 database, MUST be set to same as in feeds.tcl
+set feeds_dbfile "/home/niinuska/bot/feeds.sqlite"
+
 
 # Set channels and feed filters here:
 # "channel" "feedname|feedname2"
 # feednames can use matching, e.g. "mtv3*" would match all beginning with "mtv3"
 array set feeds_channels {
   "mazmlame" "the adventurers|oots|mestari|blastwave"
-  "fireball" "mestari"
+  "fireball" "mestari|mtv3/rikos"
   "tnsp" "the adventurers|oots|mestari"
 }
 
@@ -30,7 +31,7 @@
 # No need to look below this line
 ##########################################################################
 set feeds_name "FeedCheck"
-set feeds_message "$feeds_name v0.7 by ccr/TNSP"
+set feeds_message "$feeds_name v0.8 by ccr/TNSP"
 
 putlog "$feeds_message"
 bind pub - !feeds feeds_pubfetch
@@ -38,6 +39,10 @@
 
 
 # ------------------------------------------------------------------------
+proc feeds_escape { str } {
+  return [string map {' ''} $str]
+}
+
 proc feeds_smsg {uchan umsg} {
   global feeds_preferredmsg
   putserv "$feeds_preferredmsg $uchan :$umsg"
@@ -48,13 +53,13 @@
   putlog "$feeds_name: $umsg"
 }
 
-proc feeds_msg {urec} {
+proc feeds_msg {utime ufeed uurl utitle} {
   global feeds_channels
-  set uname [string tolower [lindex $urec 1]]
+  feeds_log "$utime $ufeed $uurl $utitle"
   foreach {uchan ufilter} [array get feeds_channels] {
     foreach umatch [split [string tolower $ufilter] "|"] {
-      if {[string match $umatch $uname]} {
-        feeds_smsg "#$uchan" "[lindex $urec 1]: \002[lindex $urec 3]\002 -- [lindex $urec 2]"
+      if {[string match $umatch $ufeed]} {
+        feeds_smsg "#$uchan" "$ufeed: \002$utitle\002 -- $uurl"
       }
     }
   }
@@ -68,59 +73,57 @@
 }
 
 
+### Open database, etc
+global feeds_dbh
+if {[catch {sqlite3 feeds_dbh $feeds_dbfile} uerrmsg]} {
+  puts "Could not open SQLite3 database '$feeds_dbfile': $uerrmsg."
+  exit 2
+}
+
+
 # ------------------------------------------------------------------------
-proc feeds_check {} {
-  global feeds_datafile
-  
-  set results {}
-  
-  # Otetaan aika
-  set newtime [clock seconds]
-  
-  # Haetaan edellinen
+proc feeds_check_start {} {
+  global feeds_dbfile
+
   set oldtime 0
-  if {![catch {set ufile [open "$feeds_datafile.time" r 0600]} uerrmsg]} {
+  if {![catch {set ufile [open "$feeds_dbfile.time" r 0600]} uerrmsg]} {
     gets $ufile oldtime
     close $ufile
   } else {
     feeds_log "Could not open timefile: $uerrmsg"
+    set oldtime [clock seconds]
   }
+
+  return $oldtime
+}
+
+
+proc feeds_check_end {} {
+  global feeds_dbfile
   
-  # Tarkistetaan
-  if {![catch {set ufile [open "$feeds_datafile" r 0600]} uerrmsg]} {
-    while {![eof $ufile]} {
-      gets $ufile uline
-      set urec [split $uline "½"]
-      if {[llength $urec] == 4 && [lindex $urec 0] > $oldtime} {
-        lappend results $urec
-      }
-    }
+  if {![catch {set ufile [open "$feeds_dbfile.time" w 0600]} uerrmsg]} {
+    puts $ufile [clock seconds]
     close $ufile
-  } else {
-    feeds_log "Could not open datafile: $uerrmsg"
   }
-  
-  # Talletetaan
-  if {[llength $results] > 0} {
-    if {![catch {set ufile [open "$feeds_datafile.time" w 0600]} uerrmsg]} {
-      puts $ufile $newtime
-      close $ufile
-    }
-  }
-  
-  return $results
 }
 
 # ------------------------------------------------------------------------
 proc feeds_exec {} {
-#  feeds_log "Checking for new entries."
-  set uresult [feeds_check]
-  set nresult [llength $uresult]
-  if {$nresult >= 20} {
-    feeds_log "$nresult entries, probably broken datafile. Ignoring."
-  } else {
-    foreach urec $uresult { feeds_msg $urec }
+  global feeds_dbh
+
+  set oldtime [feeds_check_start]
+  set found 0
+  
+  set usql "SELECT feed AS ufeed, title AS utitle, url AS uurl, utime AS utime FROM feed_data WHERE utime > $oldtime ORDER BY utime ASC"
+  feeds_dbh eval $usql {
+    feeds_msg $utime $ufeed $uurl $utitle
+    set found 1
   }
+
+  if {$found} {
+    feeds_check_end
+  }
+
   timer 5 feeds_exec
 }
 
@@ -130,59 +133,46 @@
   set feeds_running 1
 }
 
+
 # ------------------------------------------------------------------------
 proc feeds_pubfetch {unick uhost uhand uchan utext} {
-  feeds_log "Manual check invoked on $uchan."
-  set uresult [feeds_check]
-  set nresult [llength $uresult]
+
+  global feeds_dbh
+#  feeds_log "Manual check invoked on $uchan."
+
+  set oldtime [feeds_check_start]
+  set nresult [feeds_dbh onecolumn "SELECT COUNT(*) FROM feed_data WHERE utime > $oldtime"]
+
   if {$nresult > 0} {
+    feeds_check_end
     if {$nresult >= 20} {
       feeds_smsg $uchan "$nresult uutta, tod. näk. epäsynkissä. Ignoorataan."
     } else {
-      foreach urec $uresult { feeds_msg $urec }
+      set usql "SELECT feed AS ufeed, title AS utitle, url AS uurl, utime AS utime FROM feed_data WHERE utime > $oldtime ORDER BY utime ASC"
+      feeds_dbh eval $usql {
+        feeds_msg $utime $ufeed $uurl $utitle
+        set found 1
+      }
       feeds_smsg $uchan "$nresult uutta."
     }
   } else {
     feeds_smsg $uchan "Ei uusia."
   }
-  
 }
 
 
 # ------------------------------------------------------------------------
-proc feeds_checklast {ufeed} {
-  global feeds_datafile
-  
-  set result ""
-  set oldtime 0
-  set matsi [string tolower "*$ufeed*"]
-  
-  # Tarkistetaan
-  if {![catch {set ufile [open "$feeds_datafile" r 0600]} uerrmsg]} {
-    while {![eof $ufile]} {
-      gets $ufile uline
-      set urec [split $uline "½"]
-      if {[llength $urec] == 4 && [lindex $urec 0] >= $oldtime} {
-        if {[string match $matsi [string tolower [lindex $urec 1]]]} {
-          set result $urec
-          set oldtime [lindex $urec 0]
-        }
-      }
-    }
-    close $ufile
-  } else {
-    feeds_log "Could not open datafile: $uerrmsg"
+proc feeds_publast {unick uhost uhand uchan utext} {
+  global feeds_dbh
+  set ufound 0
+
+  set usql "SELECT url AS uurl, feed AS ufeed, title AS utitle, utime AS utime FROM feed_data WHERE feed LIKE '%[string tolower [feeds_escape $utext]]%' ORDER BY utime DESC LIMIT 1"
+  feeds_dbh eval $usql {
+    feeds_smsg $uchan "Uusin '$ufeed' / [feeds_ctime $utime]: $utitle -- $uurl"
+    set ufound 1
   }
-  
-  return $result
-}
 
-
-proc feeds_publast {unick uhost uhand uchan utext} {
-  set uresult [feeds_checklast $utext]
-  if {$uresult != ""} {
-    feeds_smsg $uchan "Uusin '[lindex $uresult 1]' / [feeds_ctime [lindex $uresult 0]]: [lindex $uresult 3] -- [lindex $uresult 2]"
-  } else {
+  if {$ufound == 0} {
     feeds_smsg $uchan "Ei osumia haulla '$utext'."
   }
 }