Mercurial > hg > egg-tcls
annotate fetch_feeds.tcl @ 296:a7455b0dc144
Fix feed fetching.
| author | Matti Hamalainen <ccr@tnsp.org> |
|---|---|
| date | Mon, 26 Jan 2015 14:14:17 +0200 |
| parents | 9f90d6918626 |
| children | d8b957796121 |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/tclsh |
| 1 | 2 # |
| 3 # NOTICE! Change above path to correct tclsh binary path! | |
| 4 # | |
|
268
96310b1c88fa
feeds: Improve config resiliency.
Matti Hamalainen <ccr@tnsp.org>
parents:
265
diff
changeset
|
5 ############################################################################## |
| 0 | 6 # |
|
265
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
7 # FeedCheck fetcher v0.9 by Matti 'ccr' Hamalainen <ccr@tnsp.org> |
|
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
8 # (C) Copyright 2008-2015 Tecnic Software productions (TNSP) |
| 0 | 9 # |
| 10 # This script is freely distributable under GNU GPL (version 2) license. | |
| 11 # | |
|
268
96310b1c88fa
feeds: Improve config resiliency.
Matti Hamalainen <ccr@tnsp.org>
parents:
265
diff
changeset
|
12 ############################################################################## |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
13 package require sqlite3 |
|
265
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
14 source [file dirname [info script]]/utillib.tcl |
| 0 | 15 |
|
265
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
16 ### The configuration should be in config.feeds in same directory |
|
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
17 ### as this script. Or change the line below to point where ever |
|
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
18 ### you wish. See "config.feeds.example" for an example config file. |
|
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
19 source [file dirname [info script]]/config.feeds |
| 0 | 20 |
| 21 | |
|
268
96310b1c88fa
feeds: Improve config resiliency.
Matti Hamalainen <ccr@tnsp.org>
parents:
265
diff
changeset
|
22 ############################################################################## |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
23 |
| 0 | 24 package require http |
| 271 | 25 |
|
265
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
26 if {[info exists http_user_agent] && $http_user_agent != ""} { |
| 296 | 27 ::http::config -urlencoding utf8 -useragent $http_user_agent |
|
265
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
28 } else { |
| 296 | 29 ::http::config -urlencoding utf8 -useragent "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) Opera 9.5" |
|
265
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
30 } |
| 271 | 31 |
|
268
96310b1c88fa
feeds: Improve config resiliency.
Matti Hamalainen <ccr@tnsp.org>
parents:
265
diff
changeset
|
32 if {[info exists http_use_proxy] && $http_use_proxy != 0} { |
| 63 | 33 ::http::config -proxyhost $http_proxy_host -proxyport $http_proxy_port |
| 0 | 34 } |
| 35 | |
|
268
96310b1c88fa
feeds: Improve config resiliency.
Matti Hamalainen <ccr@tnsp.org>
parents:
265
diff
changeset
|
36 if {[info exists http_tls_support] && $http_tls_support != 0} { |
|
265
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
37 package require tls |
|
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
38 ::http::register https 443 [list ::tls::socket -request 1 -require 1 -tls1 1 -cadir $http_tls_cadir] |
|
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
39 } |
|
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
159
diff
changeset
|
40 |
| 0 | 41 |
|
268
96310b1c88fa
feeds: Improve config resiliency.
Matti Hamalainen <ccr@tnsp.org>
parents:
265
diff
changeset
|
42 ############################################################################## |
|
96310b1c88fa
feeds: Improve config resiliency.
Matti Hamalainen <ccr@tnsp.org>
parents:
265
diff
changeset
|
43 |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
44 proc add_entry {uname uprefix uurl utitle} { |
| 142 | 45 global currclock feeds_db nitems |
|
292
9f90d6918626
feeds: Also use the html entity conversion from utillib here.
Matti Hamalainen <ccr@tnsp.org>
parents:
271
diff
changeset
|
46 set utmp [utl_convert_html_ent $uurl] |
| 147 | 47 if {[string match "http://*" $utmp] || [string match "https://*" $utmp]} { |
| 48 set utest "$utmp" | |
| 49 } else { | |
| 50 set utest "$uprefix$utmp" | |
| 51 } | |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
52 |
| 296 | 53 set usql "SELECT title FROM feeds WHERE url='[utl_escape $utest]' AND feed='[utl_escape $uname]'" |
|
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
54 if {![feeds_db exists $usql]} { |
| 296 | 55 set usql "INSERT INTO feeds (feed,utime,url,title) VALUES ('[utl_escape $uname]', $currclock, '[utl_escape $utest]', '[utl_escape $utitle]')" |
| 142 | 56 incr nitems |
|
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
57 if {[catch {feeds_db eval $usql} uerrmsg]} { |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
58 puts "\nError: $uerrmsg on:\n$usql" |
|
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
59 exit 15 |
|
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
60 } |
| 63 | 61 } |
| 0 | 62 } |
| 63 | |
| 64 | |
| 65 proc add_rss_feed {datauri dataname dataprefix} { | |
| 63 | 66 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { |
| 67 puts "Error getting $datauri: $uerrmsg" | |
| 68 return 1 | |
| 69 } | |
| 70 set upage [::http::data $utoken] | |
| 71 ::http::cleanup $utoken | |
| 72 | |
| 73 set umatches [regexp -all -nocase -inline -- "<item>.\*\?<title><..CDATA.(.\*\?)\\\]\\\]></title>.\*\?<link>(http.\*\?)</link>.\*\?</item>" $upage] | |
| 74 set nmatches [llength $umatches] | |
| 75 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 76 add_entry $dataname $dataprefix [lindex $umatches [expr $n+2]] [lindex $umatches [expr $n+1]] | |
| 77 } | |
| 78 | |
| 79 if {$nmatches == 0} { | |
| 80 set umatches [regexp -all -nocase -inline -- "<item>.\*\?<title>(.\*\?)</title>.\*\?<link>(http.\*\?)</link>.\*\?</item>" $upage] | |
| 81 set nmatches [llength $umatches] | |
| 82 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 83 add_entry $dataname $dataprefix [lindex $umatches [expr $n+2]] [lindex $umatches [expr $n+1]] | |
| 84 } | |
| 85 } | |
| 0 | 86 |
| 63 | 87 if {$nmatches == 0} { |
| 88 set umatches [regexp -all -nocase -inline -- "<item \[^>\]*>.\*\?<title>(.\*\?)</title>.\*\?<link>(http.\*\?)</link>.\*\?</item>" $upage] | |
| 89 set nmatches [llength $umatches] | |
| 90 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 91 add_entry $dataname $dataprefix [lindex $umatches [expr $n+2]] [lindex $umatches [expr $n+1]] | |
| 92 } | |
| 93 } | |
| 143 | 94 |
| 63 | 95 return 0 |
| 0 | 96 } |
| 97 | |
| 98 | |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
99 ### Open database, etc |
| 142 | 100 set nitems 0 |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
101 set currclock [clock seconds] |
|
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
102 global feeds_db |
|
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
103 if {[catch {sqlite3 feeds_db $feeds_dbfile} uerrmsg]} { |
|
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
104 puts "Could not open SQLite3 database '$feeds_dbfile': $uerrmsg." |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
105 exit 2 |
| 0 | 106 } |
| 107 | |
| 108 | |
| 109 ############################################################################## | |
|
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
110 ### Fetch and parse Halla-aho's blog page data |
| 0 | 111 set datauri "http://www.halla-aho.com/scripta/"; |
| 112 set dataname "Mestari" | |
| 113 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { | |
| 63 | 114 puts "Error getting $datauri: $uerrmsg" |
| 0 | 115 } else { |
| 63 | 116 set upage [::http::data $utoken] |
| 117 ::http::cleanup $utoken | |
| 118 | |
| 119 set umatches [regexp -all -nocase -inline -- "<a href=\"(\[^\"\]+\.html)\"><b>(\[^<\]+)</b>" $upage] | |
| 120 set nmatches [llength $umatches] | |
| 121 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 122 add_entry $dataname $datauri [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
| 123 } | |
| 0 | 124 |
| 63 | 125 set umatches [regexp -all -nocase -inline -- "<a href=\"(\[^\"\]+\.html)\">(\[^<\]\[^b\]\[^<\]+)</a>" $upage] |
| 126 set nmatches [llength $umatches] | |
| 127 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 128 add_entry $dataname $datauri [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
| 129 } | |
| 0 | 130 } |
| 131 | |
| 132 | |
| 133 ### The Adventurers | |
| 134 set datauri "http://www.peldor.com/chapters/index_sidebar.html"; | |
| 135 set dataname "The Adventurers" | |
| 136 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { | |
| 63 | 137 puts "Error getting $datauri: $uerrmsg" |
| 0 | 138 } else { |
| 63 | 139 set upage [::http::data $utoken] |
| 140 ::http::cleanup $utoken | |
| 141 | |
| 142 set umatches [regexp -all -nocase -inline -- "<a href=\"(\[^\"\]+)\">(\[^<\]+)</a>" $upage] | |
| 143 set nmatches [llength $umatches] | |
| 144 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 145 add_entry $dataname "http://www.peldor.com/" [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
| 146 } | |
| 0 | 147 } |
| 148 | |
| 149 | |
| 150 ### Order of the Stick | |
| 151 set datauri "http://www.giantitp.com/comics/oots.html"; | |
| 152 set dataname "OOTS" | |
| 153 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { | |
| 63 | 154 puts "Error getting $datauri: $uerrmsg" |
| 0 | 155 } else { |
| 63 | 156 set upage [::http::data $utoken] |
| 157 ::http::cleanup $utoken | |
| 158 | |
| 159 set umatches [regexp -all -nocase -inline -- "<a href=\"(/comics/oots\[0-9\]+\.html)\">(\[^<\]+)</a>" $upage] | |
| 160 set nmatches [llength $umatches] | |
| 161 for {set n 0} {$n < $nmatches} {incr n 3} { | |
| 162 add_entry $dataname "http://www.giantitp.com" [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
| 163 } | |
| 0 | 164 } |
| 165 | |
| 166 | |
|
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
167 ### Generic RSS-feed fetching |
| 143 | 168 #add_rss_feed "http://www.kaleva.fi/rss/145.xml" "Kaleva/Tiede" "" |
| 0 | 169 |
| 170 add_rss_feed "http://www.effi.org/xml/uutiset.rss" "EFFI" "" | |
| 171 | |
| 143 | 172 add_rss_feed "http://static.mtv3.fi/rss/uutiset_rikos.rss" "MTV3/Rikos" "" |
| 0 | 173 |
| 174 add_rss_feed "http://www.blastwave-comic.com/rss/blastwave.xml" "Blastwave" "" | |
| 175 | |
| 176 #add_rss_feed "http://lehti.samizdat.info/feed/" "Lehti" "" | |
| 177 | |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
178 |
| 0 | 179 |
|
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
180 ### Close database |
|
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
181 feeds_db close |
| 142 | 182 |
| 183 puts "$nitems new items." |