Mercurial > hg > egg-tcls
annotate get_feeds.tcl @ 146:7106dd8db4de
Improve entity parsing, etc.
author | Matti Hamalainen <ccr@tnsp.org> |
---|---|
date | Wed, 05 Jun 2013 16:00:16 +0300 |
parents | c94b4e1a2ed4 |
children | 48460e925a8c |
rev | line source |
---|---|
0 | 1 #!/usr/bin/tclsh |
1 | 2 # |
3 # NOTICE! Change above path to correct tclsh binary path! | |
4 # | |
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
5 ########################################################################## |
0 | 6 # |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
7 # FeedCheck fetcher v0.8 by Matti 'ccr' Hamalainen <ccr@tnsp.org> |
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
8 # (C) Copyright 2008-2013 Tecnic Software productions (TNSP) |
0 | 9 # |
10 # This script is freely distributable under GNU GPL (version 2) license. | |
11 # | |
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
12 ########################################################################## |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
13 package require sqlite3 |
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
14 source [file dirname [info script]]/util_convert.tcl |
0 | 15 |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
16 # SQLite3 database, MUST be set to same as in feeds.tcl |
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
17 set feeds_dbfile "/home/niinuska/bot/feeds.sqlite" |
0 | 18 |
19 # Use a HTTP proxy? 1 = yes, 0 = no | |
20 set http_proxy 0 | |
21 | |
22 # HTTP proxy address and port | |
23 set http_proxy_host "cache.inet.fi" | |
24 set http_proxy_port 800 | |
25 | |
26 | |
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
27 ########################################################################## |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
28 |
146
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
29 set feeds_ent_str "-|-|'|'|—|-|‏||—|-|–|--|‪||‬|" |
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
30 append feeds_ent_str "|‎||å|å|Å|Å|é|é|:|:| | " |
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
31 append feeds_ent_str "|”|\"|“|\"|«|<<|»|>>|"|\"" |
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
32 append feeds_ent_str "|ä|ä|ö|ö|Ä|Ä|Ö|Ö|&|&|<|<|>|>" |
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
33 append feeds_ent_str "|ä|ä|å|ö|—|-|'|'|–|-|"|\"" |
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
34 append feeds_ent_str "|||-|’|'|ü|ü|Ü|Ü|•|*|€|€" |
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
35 append feeds_ent_str "|”|\"" |
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
36 set html_ent [split [encoding convertfrom "utf-8" $feeds_ent_str] "|"] |
0 | 37 |
38 package require http | |
39 ::http::config -urlencoding iso8859-1 -useragent "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) Opera 9.5" | |
40 if {$http_proxy != 0} { | |
63 | 41 ::http::config -proxyhost $http_proxy_host -proxyport $http_proxy_port |
0 | 42 } |
43 | |
44 | |
45 proc convert_ent {udata} { | |
63 | 46 global html_ent |
146
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
47 return [string map -nocase $html_ent [string map $html_ent $udata]] |
0 | 48 } |
49 | |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
50 proc add_entry {uname uprefix uurl utitle} { |
142 | 51 global currclock feeds_db nitems |
63 | 52 set utest "$uprefix[convert_ent $uurl]" |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
53 |
142 | 54 set usql "SELECT title FROM feeds WHERE url='[escape $utest]' AND feed='[escape $uname]'" |
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
55 if {![feeds_db exists $usql]} { |
142 | 56 set usql "INSERT INTO feeds (feed,utime,url,title) VALUES ('[escape $uname]', $currclock, '[escape $utest]', '[escape $utitle]')" |
57 incr nitems | |
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
58 if {[catch {feeds_db eval $usql} uerrmsg]} { |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
59 puts "\nError: $uerrmsg on:\n$usql" |
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
60 exit 15 |
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
61 } |
63 | 62 } |
0 | 63 } |
64 | |
65 | |
66 proc add_rss_feed {datauri dataname dataprefix} { | |
63 | 67 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { |
68 puts "Error getting $datauri: $uerrmsg" | |
69 return 1 | |
70 } | |
71 set upage [::http::data $utoken] | |
72 ::http::cleanup $utoken | |
73 | |
74 set umatches [regexp -all -nocase -inline -- "<item>.\*\?<title><..CDATA.(.\*\?)\\\]\\\]></title>.\*\?<link>(http.\*\?)</link>.\*\?</item>" $upage] | |
75 set nmatches [llength $umatches] | |
76 for {set n 0} {$n < $nmatches} {incr n 3} { | |
77 add_entry $dataname $dataprefix [lindex $umatches [expr $n+2]] [lindex $umatches [expr $n+1]] | |
78 } | |
79 | |
80 if {$nmatches == 0} { | |
81 set umatches [regexp -all -nocase -inline -- "<item>.\*\?<title>(.\*\?)</title>.\*\?<link>(http.\*\?)</link>.\*\?</item>" $upage] | |
82 set nmatches [llength $umatches] | |
83 for {set n 0} {$n < $nmatches} {incr n 3} { | |
84 add_entry $dataname $dataprefix [lindex $umatches [expr $n+2]] [lindex $umatches [expr $n+1]] | |
85 } | |
86 } | |
0 | 87 |
63 | 88 if {$nmatches == 0} { |
89 set umatches [regexp -all -nocase -inline -- "<item \[^>\]*>.\*\?<title>(.\*\?)</title>.\*\?<link>(http.\*\?)</link>.\*\?</item>" $upage] | |
90 set nmatches [llength $umatches] | |
91 for {set n 0} {$n < $nmatches} {incr n 3} { | |
92 add_entry $dataname $dataprefix [lindex $umatches [expr $n+2]] [lindex $umatches [expr $n+1]] | |
93 } | |
94 } | |
143 | 95 |
63 | 96 return 0 |
0 | 97 } |
98 | |
99 | |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
100 ### Open database, etc |
142 | 101 set nitems 0 |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
102 set currclock [clock seconds] |
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
103 global feeds_db |
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
104 if {[catch {sqlite3 feeds_db $feeds_dbfile} uerrmsg]} { |
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
105 puts "Could not open SQLite3 database '$feeds_dbfile': $uerrmsg." |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
106 exit 2 |
0 | 107 } |
108 | |
109 | |
110 ############################################################################## | |
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
111 ### Fetch and parse Halla-aho's blog page data |
0 | 112 set datauri "http://www.halla-aho.com/scripta/"; |
113 set dataname "Mestari" | |
114 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { | |
63 | 115 puts "Error getting $datauri: $uerrmsg" |
0 | 116 } else { |
63 | 117 set upage [::http::data $utoken] |
118 ::http::cleanup $utoken | |
119 | |
120 set umatches [regexp -all -nocase -inline -- "<a href=\"(\[^\"\]+\.html)\"><b>(\[^<\]+)</b>" $upage] | |
121 set nmatches [llength $umatches] | |
122 for {set n 0} {$n < $nmatches} {incr n 3} { | |
123 add_entry $dataname $datauri [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
124 } | |
0 | 125 |
63 | 126 set umatches [regexp -all -nocase -inline -- "<a href=\"(\[^\"\]+\.html)\">(\[^<\]\[^b\]\[^<\]+)</a>" $upage] |
127 set nmatches [llength $umatches] | |
128 for {set n 0} {$n < $nmatches} {incr n 3} { | |
129 add_entry $dataname $datauri [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
130 } | |
0 | 131 } |
132 | |
133 | |
134 ### The Adventurers | |
135 set datauri "http://www.peldor.com/chapters/index_sidebar.html"; | |
136 set dataname "The Adventurers" | |
137 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { | |
63 | 138 puts "Error getting $datauri: $uerrmsg" |
0 | 139 } else { |
63 | 140 set upage [::http::data $utoken] |
141 ::http::cleanup $utoken | |
142 | |
143 set umatches [regexp -all -nocase -inline -- "<a href=\"(\[^\"\]+)\">(\[^<\]+)</a>" $upage] | |
144 set nmatches [llength $umatches] | |
145 for {set n 0} {$n < $nmatches} {incr n 3} { | |
146 add_entry $dataname "http://www.peldor.com/" [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
147 } | |
0 | 148 } |
149 | |
150 | |
151 ### Order of the Stick | |
152 set datauri "http://www.giantitp.com/comics/oots.html"; | |
153 set dataname "OOTS" | |
154 if {[catch {set utoken [::http::geturl $datauri -binary true -timeout 5000]} uerrmsg]} { | |
63 | 155 puts "Error getting $datauri: $uerrmsg" |
0 | 156 } else { |
63 | 157 set upage [::http::data $utoken] |
158 ::http::cleanup $utoken | |
159 | |
160 set umatches [regexp -all -nocase -inline -- "<a href=\"(/comics/oots\[0-9\]+\.html)\">(\[^<\]+)</a>" $upage] | |
161 set nmatches [llength $umatches] | |
162 for {set n 0} {$n < $nmatches} {incr n 3} { | |
163 add_entry $dataname "http://www.giantitp.com" [lindex $umatches [expr $n+1]] [lindex $umatches [expr $n+2]] | |
164 } | |
0 | 165 } |
166 | |
167 | |
69
df3230f8aa46
Translate some comments to english and cosmetic fixes.
Matti Hamalainen <ccr@tnsp.org>
parents:
63
diff
changeset
|
168 ### Generic RSS-feed fetching |
143 | 169 #add_rss_feed "http://www.kaleva.fi/rss/145.xml" "Kaleva/Tiede" "" |
0 | 170 |
171 add_rss_feed "http://www.effi.org/xml/uutiset.rss" "EFFI" "" | |
172 | |
143 | 173 add_rss_feed "http://static.mtv3.fi/rss/uutiset_rikos.rss" "MTV3/Rikos" "" |
0 | 174 |
175 add_rss_feed "http://www.blastwave-comic.com/rss/blastwave.xml" "Blastwave" "" | |
176 | |
177 #add_rss_feed "http://lehti.samizdat.info/feed/" "Lehti" "" | |
178 | |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
179 |
0 | 180 |
139
3305e142eecc
Change feed fetcher to use SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
181 ### Close database |
140
b0648e05c855
Change some variable names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
139
diff
changeset
|
182 feeds_db close |
142 | 183 |
184 puts "$nitems new items." |