Mercurial > hg > egg-tcls
changeset 139:3305e142eecc
Change feed fetcher to use SQLite3 backend.
author | Matti Hamalainen <ccr@tnsp.org> |
---|---|
date | Tue, 04 Jun 2013 13:35:46 +0300 |
parents | 743c1bea2498 |
children | b0648e05c855 |
files | hae_feedit.tcl |
diffstat | 1 files changed, 26 insertions(+), 45 deletions(-) [+] |
line wrap: on
line diff
--- a/hae_feedit.tcl Tue Jun 04 13:24:30 2013 +0300 +++ b/hae_feedit.tcl Tue Jun 04 13:35:46 2013 +0300 @@ -4,15 +4,17 @@ # ########################################################################## # -# FeedCheck fetcher v0.7 by Matti 'ccr' Hamalainen <ccr@tnsp.org> -# (C) Copyright 2008-2011 Tecnic Software productions (TNSP) +# FeedCheck fetcher v0.8 by Matti 'ccr' Hamalainen <ccr@tnsp.org> +# (C) Copyright 2008-2013 Tecnic Software productions (TNSP) # # This script is freely distributable under GNU GPL (version 2) license. # ########################################################################## +package require sqlite3 +source [file dirname [info script]]/util_convert.tcl -# Datafile, MUST be set to same as in feeds.tcl -set datafile "/home/niinuska/bot/data.feeds" +# SQLite3 database, MUST be set to same as in feeds.tcl +set feeds_db "/home/niinuska/bot/feeds.sqlite" # Use a HTTP proxy? 1 = yes, 0 = no set http_proxy 0 @@ -23,6 +25,7 @@ ########################################################################## + set html_ent [split " | |»|>>|"|\"|ä|ä|ö|ö|Ä|Ä|Ö|Ö|&|&|<|<|>|>|ä|ä|ö|ö|Ä|Ä" "|"] package require http @@ -38,13 +41,17 @@ } -proc add_entry {uname uprefix uurl udesc} { - global entries isentries newurls currclock +proc add_entry {uname uprefix uurl utitle} { + global currclock dbh set utest "$uprefix[convert_ent $uurl]" - set isentries($utest) 1 - if {[catch {set utmp $entries($utest)}]} { - set entries($utest) [list $currclock $uname $utest [convert_ent $udesc]] - incr newurls + + set usql "SELECT title FROM feed_data WHERE url='[escape $utest]' AND feed='[escape $uname]'" + if {![dbh exists $usql]} { + set usql "INSERT INTO feed_data (feed,utime,url,title) VALUES ('[escape $uname]', $currclock, '[escape $utest]', '[escape $utitle]')" + if {[catch {dbh eval $usql} uerrmsg]} { + puts "\nError: $uerrmsg on:\n$usql" + exit 15 + } } } @@ -82,23 +89,13 @@ } -########################################################################## -### Read in old data -set oldurls 0 -set newurls 0 -if {![catch {set ufile [open $datafile r 0600]} uerrmsg]} { - while {![eof $ufile]} { - gets $ufile uline - set urec [split $uline "½"] - if {[llength $urec] == 4} { - set entries([lindex $urec 2]) $urec - set isentries([lindex $urec 2]) 0 - incr oldurls - } - } - close $ufile +### Open database, etc +set currclock [clock seconds] +global dbh +if {[catch {sqlite3 dbh $feeds_db} uerrmsg]} { + puts "Could not open SQLite3 database '$feeds_db': $uerrmsg." + exit 2 } -set currclock [clock seconds] ############################################################################## @@ -172,23 +169,7 @@ #add_rss_feed "http://lehti.samizdat.info/feed/" "Lehti" "" -############################################################################## -### Open data file for writing -set tmpfname "$datafile.tmp" -if {[catch {set outfile [open $tmpfname w 0600]} uerrmsg]} { - puts "Error opening $tmpfname for writing: $uerrmsg" - return 1 -} + -set uexpire [expr [clock seconds] - (7*24*60*60)] -foreach {ukey udata} [array get entries] { -# if {$isentries($ukey) != 0 || [lindex $udata 0] >= $uexpire} { - puts $outfile [join $udata "½"] -# } -} - -close $outfile -if {[catch {file rename -force -- $tmpfname $datafile} uerrmsg]} { - puts "Error renaming $tmpfname to $datafile: $uerrmsg" -} -#puts "$newurls new entries." +### Close database +dbh close