Mercurial > hg > egg-tcls
annotate feeds.tcl @ 424:825cac46b1cb
Cosmetic / stray trailing whitespace cleanup.
author | Matti Hamalainen <ccr@tnsp.org> |
---|---|
date | Sun, 08 Jan 2017 03:55:55 +0200 |
parents | 44c9128097cd |
children | e5810c52d376 |
rev | line source |
---|---|
70 | 1 ########################################################################## |
0 | 2 # |
323 | 3 # FeedCheck v1.0 by Matti 'ccr' Hamalainen <ccr@tnsp.org> |
265
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
226
diff
changeset
|
4 # (C) Copyright 2008-2015 Tecnic Software productions (TNSP) |
0 | 5 # |
265
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
226
diff
changeset
|
6 # Requires fetch_feeds.tcl to be run as a cronjob, for example |
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
226
diff
changeset
|
7 # 15 * * * * /absolute/path/to/fetch_feeds.tcl |
0 | 8 # |
145 | 9 # See also create_feeds_db.tcl OR convert_feeds_db.tcl, as you will |
10 # need to either create a SQLite3 database or convert old text flat | |
11 # file to SQLite3. | |
12 # | |
0 | 13 # This script is freely distributable under GNU GPL (version 2) license. |
14 # | |
70 | 15 ########################################################################## |
0 | 16 |
265
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
226
diff
changeset
|
17 ### The configuration should be in config.feeds in same directory |
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
226
diff
changeset
|
18 ### as this script. Or change the line below to point where ever |
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
226
diff
changeset
|
19 ### you wish. See "config.feeds.example" for an example config file. |
908edc54005a
feeds: Move configuration to separate file.
Matti Hamalainen <ccr@tnsp.org>
parents:
226
diff
changeset
|
20 source [file dirname [info script]]/config.feeds |
0 | 21 |
422
880a07485275
Add utl_ctime() to utillib and use it elsewhere.
Matti Hamalainen <ccr@tnsp.org>
parents:
393
diff
changeset
|
22 ### Required utillib.tcl |
880a07485275
Add utl_ctime() to utillib and use it elsewhere.
Matti Hamalainen <ccr@tnsp.org>
parents:
393
diff
changeset
|
23 source [file dirname [info script]]/utillib.tcl |
880a07485275
Add utl_ctime() to utillib and use it elsewhere.
Matti Hamalainen <ccr@tnsp.org>
parents:
393
diff
changeset
|
24 |
0 | 25 |
70 | 26 ########################################################################## |
0 | 27 # No need to look below this line |
70 | 28 ########################################################################## |
423
44c9128097cd
feeds: Remember to require sqlite3 package.
Matti Hamalainen <ccr@tnsp.org>
parents:
422
diff
changeset
|
29 package require sqlite3 |
44c9128097cd
feeds: Remember to require sqlite3 package.
Matti Hamalainen <ccr@tnsp.org>
parents:
422
diff
changeset
|
30 |
0 | 31 set feeds_name "FeedCheck" |
323 | 32 set feeds_message "$feeds_name v1.0 by ccr/TNSP" |
0 | 33 |
34 putlog "$feeds_message" | |
35 bind pub - !feeds feeds_pubfetch | |
36 bind pub - !last feeds_publast | |
37 | |
38 | |
70 | 39 # ------------------------------------------------------------------------ |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
40 proc feeds_escape { str } { |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
41 return [string map {' ''} $str] |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
42 } |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
43 |
0 | 44 proc feeds_smsg {uchan umsg} { |
45 global feeds_preferredmsg | |
46 putserv "$feeds_preferredmsg $uchan :$umsg" | |
47 } | |
48 | |
49 proc feeds_log {umsg} { | |
50 global feeds_name | |
51 putlog "$feeds_name: $umsg" | |
52 } | |
53 | |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
54 proc feeds_msg {utime ufeed uurl utitle} { |
0 | 55 global feeds_channels |
56 foreach {uchan ufilter} [array get feeds_channels] { | |
143 | 57 foreach umatch [split $ufilter "|"] { |
58 if {[string match -nocase $umatch $ufeed]} { | |
226
0ff78e418e77
feeds: Don't add # to channel names automatically.
Matti Hamalainen <ccr@tnsp.org>
parents:
154
diff
changeset
|
59 feeds_smsg "$uchan" "$ufeed: \002$utitle\002 -- $uurl" |
0 | 60 } |
61 } | |
62 } | |
63 } | |
64 | |
65 | |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
66 ### Open database, etc |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
67 if {[catch {sqlite3 feeds_dbh $feeds_dbfile} uerrmsg]} { |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
68 puts "Could not open SQLite3 database '$feeds_dbfile': $uerrmsg." |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
69 exit 2 |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
70 } |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
71 |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
72 |
70 | 73 # ------------------------------------------------------------------------ |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
74 proc feeds_check_start {} { |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
75 global feeds_dbfile |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
76 |
0 | 77 set oldtime 0 |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
78 if {![catch {set ufile [open "$feeds_dbfile.time" r 0600]} uerrmsg]} { |
0 | 79 gets $ufile oldtime |
80 close $ufile | |
81 } else { | |
82 feeds_log "Could not open timefile: $uerrmsg" | |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
83 set oldtime [clock seconds] |
0 | 84 } |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
85 |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
86 return $oldtime |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
87 } |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
88 |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
89 |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
90 proc feeds_check_end {} { |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
91 global feeds_dbfile |
424
825cac46b1cb
Cosmetic / stray trailing whitespace cleanup.
Matti Hamalainen <ccr@tnsp.org>
parents:
423
diff
changeset
|
92 |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
93 if {![catch {set ufile [open "$feeds_dbfile.time" w 0600]} uerrmsg]} { |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
94 puts $ufile [clock seconds] |
0 | 95 close $ufile |
96 } | |
97 } | |
98 | |
70 | 99 # ------------------------------------------------------------------------ |
0 | 100 proc feeds_exec {} { |
351
0f55bbb7fea3
feeds: Improve safety checks and add new configuration setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
323
diff
changeset
|
101 global feeds_dbh feeds_check_period feeds_running feeds_sync_limit |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
102 |
146
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
103 # feeds_log "Timed feed check." |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
104 set oldtime [feeds_check_start] |
142 | 105 set usql "SELECT feed AS ufeed, title AS utitle, url AS uurl, utime AS utime FROM feeds WHERE utime > $oldtime ORDER BY utime ASC" |
351
0f55bbb7fea3
feeds: Improve safety checks and add new configuration setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
323
diff
changeset
|
106 set nresult [feeds_dbh onecolumn "SELECT COUNT(*) FROM feeds WHERE utime > $oldtime"] |
424
825cac46b1cb
Cosmetic / stray trailing whitespace cleanup.
Matti Hamalainen <ccr@tnsp.org>
parents:
423
diff
changeset
|
107 |
361
299cdd79b7ee
feeds: Fix some limit checks.
Matti Hamalainen <ccr@tnsp.org>
parents:
351
diff
changeset
|
108 if {$nresult > $feeds_sync_limit} { |
393 | 109 feeds_log "$nresult new entries, probably unsynchronized. Ignoring." |
351
0f55bbb7fea3
feeds: Improve safety checks and add new configuration setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
323
diff
changeset
|
110 } else { |
393 | 111 feeds_log "$nresult new entries since $oldtime .." |
351
0f55bbb7fea3
feeds: Improve safety checks and add new configuration setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
323
diff
changeset
|
112 feeds_dbh eval $usql { |
0f55bbb7fea3
feeds: Improve safety checks and add new configuration setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
323
diff
changeset
|
113 feeds_msg $utime $ufeed $uurl $utitle |
0f55bbb7fea3
feeds: Improve safety checks and add new configuration setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
323
diff
changeset
|
114 } |
0 | 115 } |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
116 |
351
0f55bbb7fea3
feeds: Improve safety checks and add new configuration setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
323
diff
changeset
|
117 if {$nresult > 0} { |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
118 feeds_check_end |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
119 } |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
120 |
144 | 121 set feeds_running [clock seconds] |
122 timer $feeds_check_period feeds_exec | |
0 | 123 } |
124 | |
146
7106dd8db4de
Improve entity parsing, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
145
diff
changeset
|
125 |
144 | 126 if {[info exists feeds_running]} { |
127 set feeds_last [expr [clock seconds] - $feeds_running] | |
128 } else { | |
129 set feeds_last -1 | |
130 } | |
131 | |
132 if {$feeds_last < 0 || $feeds_last > [expr $feeds_check_period * 60]} { | |
0 | 133 feeds_log "Starting timed feed check." |
154
7a549487fba3
feeds: Cleanups, minor bugfix in feed update checking.
Matti Hamalainen <ccr@tnsp.org>
parents:
146
diff
changeset
|
134 feeds_exec |
0 | 135 } |
136 | |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
137 |
70 | 138 # ------------------------------------------------------------------------ |
0 | 139 proc feeds_pubfetch {unick uhost uhand uchan utext} { |
351
0f55bbb7fea3
feeds: Improve safety checks and add new configuration setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
323
diff
changeset
|
140 global feeds_dbh feeds_sync_limit |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
141 # feeds_log "Manual check invoked on $uchan." |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
142 |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
143 set oldtime [feeds_check_start] |
142 | 144 set nresult [feeds_dbh onecolumn "SELECT COUNT(*) FROM feeds WHERE utime > $oldtime"] |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
145 |
0 | 146 if {$nresult > 0} { |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
147 feeds_check_end |
361
299cdd79b7ee
feeds: Fix some limit checks.
Matti Hamalainen <ccr@tnsp.org>
parents:
351
diff
changeset
|
148 if {$nresult > $feeds_sync_limit} { |
351
0f55bbb7fea3
feeds: Improve safety checks and add new configuration setting.
Matti Hamalainen <ccr@tnsp.org>
parents:
323
diff
changeset
|
149 feeds_smsg $uchan "$nresult uutta, tod. näk. epäsynkissä. Ignoorataan." |
0 | 150 } else { |
142 | 151 set usql "SELECT feed AS ufeed, title AS utitle, url AS uurl, utime AS utime FROM feeds WHERE utime > $oldtime ORDER BY utime ASC" |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
152 feeds_dbh eval $usql { |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
153 feeds_msg $utime $ufeed $uurl $utitle |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
154 set found 1 |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
155 } |
0 | 156 feeds_smsg $uchan "$nresult uutta." |
157 } | |
158 } else { | |
159 feeds_smsg $uchan "Ei uusia." | |
160 } | |
161 } | |
162 | |
163 | |
70 | 164 # ------------------------------------------------------------------------ |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
165 proc feeds_publast {unick uhost uhand uchan utext} { |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
166 global feeds_dbh |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
167 set ufound 0 |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
168 |
142 | 169 set usql "SELECT url AS uurl, feed AS ufeed, title AS utitle, utime AS utime FROM feeds WHERE feed LIKE '%[string tolower [feeds_escape $utext]]%' ORDER BY utime DESC LIMIT 1" |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
170 feeds_dbh eval $usql { |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
171 feeds_smsg $uchan "Uusin '$ufeed' / [feeds_ctime $utime]: $utitle -- $uurl" |
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
172 set ufound 1 |
0 | 173 } |
174 | |
141
c99df41a691a
Work on making the feeds script use the SQLite3 backend.
Matti Hamalainen <ccr@tnsp.org>
parents:
114
diff
changeset
|
175 if {$ufound == 0} { |
0 | 176 feeds_smsg $uchan "Ei osumia haulla '$utext'." |
177 } | |
178 } |