Mercurial > hg > egg-tcls
annotate hae_feedit.tcl @ 140:b0648e05c855
Change some variable names, etc.
| author | Matti Hamalainen <ccr@tnsp.org> |
|---|---|
| date | Tue, 04 Jun 2013 13:45:16 +0300 |
| parents | 3305e142eecc |
| children | 4c51eeba993f |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/tclsh |
| 1 | 2 # |
| 3 # NOTICE! Change above path to correct tclsh binary path! | |
| 4 # | |
|
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
5 ########################################################################## |
| 0 | 6 # |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
7 # FeedCheck fetcher v0.8 by Matti 'ccr' Hamalainen <ccr@tnsp.org> |
|
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
8 # (C) Copyright 2008-2013 Tecnic Software productions (TNSP) |
| 0 | 9 # |
| 10 # This script is freely distributable under GNU GPL (version 2) license. | |
| 11 # | |
|
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
12 ########################################################################## |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
13 package require sqlite3 |
|
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
14 source [file dirname [info script]]/util_convert.tcl |
| 0 | 15 |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
16 # SQLite3 database, MUST be set to same as in feeds.tcl |
|
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
17 set feeds_dbfile "/home/niinuska/bot/feeds.sqlite" |
| 0 | 18 |
| 19 # Use a HTTP proxy? 1 = yes, 0 = no | |
| 20 set http_proxy 0 | |
| 21 | |
| 22 # HTTP proxy address and port | |
| 23 set http_proxy_host "cache.inet.fi" | |
| 24 set http_proxy_port 800 | |
| 25 | |
| 26 | |
|
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
27 ########################################################################## |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
28 |
| 0 | 29 set html_ent [split " | |»|>>|"|\"|ä|ä|ö|ö|Ä|Ä|Ö|Ö|&|&|<|<|>|>|ä|ä|ö|ö|Ä|Ä" "|"] |
| 30 | |
| 31 package require http | |
| 32 ::http::config -urlencoding iso8859-1 -useragent "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) Opera 9.5" | |
| 33 if {$http_proxy != 0} { | |
| 63 | 34 ::http::config -proxyhost $http_proxy_host -proxyport $http_proxy_port |
| 0 | 35 } |
| 36 | |
| 37 | |
| 38 proc convert_ent {udata} { | |
| 63 | 39 global html_ent |
| 40 return [string map $html_ent $udata] | |
| 0 | 41 } |
| 42 | |
| 43 | |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
44 proc add_entry {uname uprefix uurl utitle} { |
|
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
45 global currclock feeds_db |
| 63 | 46 set utest "$uprefix[convert_ent $uurl]" |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
47 |
|
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
48 set usql "SELECT title FROM feed_data WHERE url='[escape $utest]' AND feed='[escape $uname]'" |
|
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
49 if {![feeds_db exists $usql]} { |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
50 set usql "INSERT INTO feed_data (feed,utime,url,title) VALUES ('[escape $uname]', $currclock, '[escape $utest]', '[escape $utitle]')" |
|
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
51 if {[catch {feeds_db eval $usql} uerrmsg]} { |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
52 puts "\nError: $uerrmsg on:\n$usql" |
|
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
53 exit 15 |
|
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
54 } |
| 63 | 55 } |
| 0 | 56 } |
| 57 | |
| 58 | |
| 59 proc add_rss_feed {datauri dataname dataprefix} { | |
| 63 | 60 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { |
| 61 puts "Error getting $datauri: $uerrmsg" | |
| 62 return 1 | |
| 63 } | |
| 64 set upage [::http::data $utoken] | |
| 65 ::http::cleanup $utoken | |
| 66 | |
| 67 set umatches [regexp -all -nocase -inline -- "<item>.\*\?<title><..CDATA.(.\*\?)\\\]\\\]></title>.\*\?<link>(http.\*\?)</link>.\*\?</item>" $upage] | |
| 68 set nmatches [llength $umatches] | |
| 69 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 70 add_entry $dataname $dataprefix [lindex $umatches [expr $n+2]] [lindex $umatches [expr $n+1]] | |
| 71 } | |
| 72 | |
| 73 if {$nmatches == 0} { | |
| 74 set umatches [regexp -all -nocase -inline -- "<item>.\*\?<title>(.\*\?)</title>.\*\?<link>(http.\*\?)</link>.\*\?</item>" $upage] | |
| 75 set nmatches [llength $umatches] | |
| 76 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 77 add_entry $dataname $dataprefix [lindex $umatches [expr $n+2]] [lindex $umatches [expr $n+1]] | |
| 78 } | |
| 79 } | |
| 0 | 80 |
| 63 | 81 if {$nmatches == 0} { |
| 82 set umatches [regexp -all -nocase -inline -- "<item \[^>\]*>.\*\?<title>(.\*\?)</title>.\*\?<link>(http.\*\?)</link>.\*\?</item>" $upage] | |
| 83 set nmatches [llength $umatches] | |
| 84 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 85 add_entry $dataname $dataprefix [lindex $umatches [expr $n+2]] [lindex $umatches [expr $n+1]] | |
| 86 } | |
| 87 } | |
| 88 return 0 | |
| 0 | 89 } |
| 90 | |
| 91 | |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
92 ### Open database, etc |
|
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
93 set currclock [clock seconds] |
|
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
94 global feeds_db |
|
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
95 if {[catch {sqlite3 feeds_db $feeds_dbfile} uerrmsg]} { |
|
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
96 puts "Could not open SQLite3 database '$feeds_dbfile': $uerrmsg." |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
97 exit 2 |
| 0 | 98 } |
| 99 | |
| 100 | |
| 101 ############################################################################## | |
|
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
102 ### Fetch and parse Halla-aho's blog page data |
| 0 | 103 set datauri "http://www.halla-aho.com/scripta/"; |
| 104 set dataname "Mestari" | |
| 105 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { | |
| 63 | 106 puts "Error getting $datauri: $uerrmsg" |
| 0 | 107 } else { |
| 63 | 108 set upage [::http::data $utoken] |
| 109 ::http::cleanup $utoken | |
| 110 | |
| 111 set umatches [regexp -all -nocase -inline -- "<a href=\"(\[^\"\]+\.html)\"><b>(\[^<\]+)</b>" $upage] | |
| 112 set nmatches [llength $umatches] | |
| 113 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 114 add_entry $dataname $datauri [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
| 115 } | |
| 0 | 116 |
| 63 | 117 set umatches [regexp -all -nocase -inline -- "<a href=\"(\[^\"\]+\.html)\">(\[^<\]\[^b\]\[^<\]+)</a>" $upage] |
| 118 set nmatches [llength $umatches] | |
| 119 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 120 add_entry $dataname $datauri [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
| 121 } | |
| 0 | 122 } |
| 123 | |
| 124 | |
| 125 ### The Adventurers | |
| 126 set datauri "http://www.peldor.com/chapters/index_sidebar.html"; | |
| 127 set dataname "The Adventurers" | |
| 128 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { | |
| 63 | 129 puts "Error getting $datauri: $uerrmsg" |
| 0 | 130 } else { |
| 63 | 131 set upage [::http::data $utoken] |
| 132 ::http::cleanup $utoken | |
| 133 | |
| 134 set umatches [regexp -all -nocase -inline -- "<a href=\"(\[^\"\]+)\">(\[^<\]+)</a>" $upage] | |
| 135 set nmatches [llength $umatches] | |
| 136 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 137 add_entry $dataname "http://www.peldor.com/" [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
| 138 } | |
| 0 | 139 } |
| 140 | |
| 141 | |
| 142 ### Order of the Stick | |
| 143 set datauri "http://www.giantitp.com/comics/oots.html"; | |
| 144 set dataname "OOTS" | |
| 145 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { | |
| 63 | 146 puts "Error getting $datauri: $uerrmsg" |
| 0 | 147 } else { |
| 63 | 148 set upage [::http::data $utoken] |
| 149 ::http::cleanup $utoken | |
| 150 | |
| 151 set umatches [regexp -all -nocase -inline -- "<a href=\"(/comics/oots\[0-9\]+\.html)\">(\[^<\]+)</a>" $upage] | |
| 152 set nmatches [llength $umatches] | |
| 153 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 154 add_entry $dataname "http://www.giantitp.com" [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
| 155 } | |
| 0 | 156 } |
| 157 | |
| 158 | |
|
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
159 ### Generic RSS-feed fetching |
| 0 | 160 add_rss_feed "http://www.kaleva.fi/rss/145.xml" "Kaleva/Tiede" "" |
| 161 | |
| 162 #add_rss_feed "http://sektori.com/?tpl=rssNewsFeed" "Sektori" "" | |
| 163 | |
| 164 add_rss_feed "http://www.effi.org/xml/uutiset.rss" "EFFI" "" | |
| 165 | |
| 166 add_rss_feed "http://www.mtv3.fi/rss/uutiset_rikos.rss" "MTV3/Rikos" "" | |
| 167 | |
| 168 add_rss_feed "http://www.blastwave-comic.com/rss/blastwave.xml" "Blastwave" "" | |
| 169 | |
| 170 #add_rss_feed "http://lehti.samizdat.info/feed/" "Lehti" "" | |
| 171 | |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
172 |
| 0 | 173 |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
174 ### Close database |
|
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
175 feeds_db close |