Mercurial > hg > egg-tcls
annotate urllog.tcl @ 651:ecd8fb2b9ad3
urllog: Use regexp splitx instead of string split.
| author | Matti Hamalainen <ccr@tnsp.org> |
|---|---|
| date | Wed, 17 Feb 2021 11:00:49 +0200 |
| parents | 577763cb8864 |
| children | 10ea2c1101b3 |
| rev | line source |
|---|---|
| 0 | 1 ########################################################################## |
| 2 # | |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
3 # URLLog v2.7.0 by Matti 'ccr' Hamalainen <ccr@tnsp.org> |
| 578 | 4 # (C) Copyright 2000-2021 Tecnic Software productions (TNSP) |
| 0 | 5 # |
|
113
077c7383f36f
urllog: Add line about the script's license.
Matti Hamalainen <ccr@tnsp.org>
parents:
112
diff
changeset
|
6 # This script is freely distributable under GNU GPL (version 2) license. |
|
077c7383f36f
urllog: Add line about the script's license.
Matti Hamalainen <ccr@tnsp.org>
parents:
112
diff
changeset
|
7 # |
| 0 | 8 ########################################################################## |
| 9 # | |
|
591
b11484cb3623
urllog: Update instructions a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
590
diff
changeset
|
10 # URL-logger script for EggDrop IRC robot, utilizing TCL SQLite3 database |
|
b11484cb3623
urllog: Update instructions a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
590
diff
changeset
|
11 # interface. Requirements for this script are as follows: |
|
458
cfbe6acc1d73
urllog: tcl-tls 1.7.x (tested with 1.7.13) is now a requirement. It is
Matti Hamalainen <ccr@tnsp.org>
parents:
457
diff
changeset
|
12 # |
|
591
b11484cb3623
urllog: Update instructions a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
590
diff
changeset
|
13 # - utillib.tcl (available from same repository as this script) |
|
b11484cb3623
urllog: Update instructions a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
590
diff
changeset
|
14 # - tcl-tls 1.7.13+ for SSL/TLS support |
|
b11484cb3623
urllog: Update instructions a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
590
diff
changeset
|
15 # - TCL 8.6 or later |
|
b11484cb3623
urllog: Update instructions a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
590
diff
changeset
|
16 # - Eggdrop 1.6.20 or later |
|
b11484cb3623
urllog: Update instructions a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
590
diff
changeset
|
17 # - SQLite3 and TCL bindings for it. |
|
b11484cb3623
urllog: Update instructions a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
590
diff
changeset
|
18 # |
|
b11484cb3623
urllog: Update instructions a bit.
Matti Hamalainen <ccr@tnsp.org>
parents:
590
diff
changeset
|
19 # On Debian: tcl8.6 tcl-tls libsqlite3-tcl eggdrop eggdrop-data |
|
50
f69363fc1f61
Update some comments and add a bit of documentation.
Matti Hamalainen <ccr@tnsp.org>
parents:
49
diff
changeset
|
20 # |
|
479
fc3b6bc37927
urllog: Improve documentation slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
473
diff
changeset
|
21 # If you are doing a fresh install, you will need to create the initial |
|
fc3b6bc37927
urllog: Improve documentation slightly.
Matti Hamalainen <ccr@tnsp.org>
parents:
473
diff
changeset
|
22 # database with the required table schemas. You can do that by running |
|
503
fdd1f0b83685
urllog: Migration from old versions hasn't been supported in reality for a
Matti Hamalainen <ccr@tnsp.org>
parents:
493
diff
changeset
|
23 # "create_urllog_db.tcl". You also need to set up the configuration in |
|
fdd1f0b83685
urllog: Migration from old versions hasn't been supported in reality for a
Matti Hamalainen <ccr@tnsp.org>
parents:
493
diff
changeset
|
24 # "config.urllog" file. |
| 0 | 25 # |
| 26 ########################################################################## | |
|
13
e06d41fb69d5
Begin work on converting urllog.tcl to use an SQLite3 database instead of flat file.
Matti Hamalainen <ccr@tnsp.org>
parents:
8
diff
changeset
|
27 |
|
263
f01d60175c44
urllog: Move configuration to external file.
Matti Hamalainen <ccr@tnsp.org>
parents:
260
diff
changeset
|
28 ### The configuration should be in config.urllog in same directory |
|
f01d60175c44
urllog: Move configuration to external file.
Matti Hamalainen <ccr@tnsp.org>
parents:
260
diff
changeset
|
29 ### as this script. Or change the line below to point where ever |
|
f01d60175c44
urllog: Move configuration to external file.
Matti Hamalainen <ccr@tnsp.org>
parents:
260
diff
changeset
|
30 ### you wish. See "config.urllog.example" for an example config file. |
|
f01d60175c44
urllog: Move configuration to external file.
Matti Hamalainen <ccr@tnsp.org>
parents:
260
diff
changeset
|
31 source [file dirname [info script]]/config.urllog |
| 0 | 32 |
|
291
54d34d086b47
urllog: Use the utility lib for entity conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
289
diff
changeset
|
33 ### Required utillib.tcl |
|
54d34d086b47
urllog: Use the utility lib for entity conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
289
diff
changeset
|
34 source [file dirname [info script]]/utillib.tcl |
|
54d34d086b47
urllog: Use the utility lib for entity conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
289
diff
changeset
|
35 |
| 0 | 36 |
| 37 ########################################################################## | |
| 38 # No need to look below this line | |
| 39 ########################################################################## | |
| 560 | 40 package require sqlite3 |
| 41 package require http | |
|
651
ecd8fb2b9ad3
urllog: Use regexp splitx instead of string split.
Matti Hamalainen <ccr@tnsp.org>
parents:
603
diff
changeset
|
42 package require textutil::split |
| 560 | 43 |
| 0 | 44 set urllog_name "URLLog" |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
45 set urllog_version "2.7.0" |
| 578 | 46 set urllog_message "$urllog_name v$urllog_version (C) 2000-2021 ccr/TNSP" |
| 560 | 47 |
| 0 | 48 |
|
424
825cac46b1cb
Cosmetic / stray trailing whitespace cleanup.
Matti Hamalainen <ccr@tnsp.org>
parents:
422
diff
changeset
|
49 set urllog_httprep [split "\@|%40|{|%7B|}|%7D|\[|%5B|\]|%5D" "|"] |
|
592
ac72a228089b
urllog: Rename urllog_shorturl_str to urllog_shorturl_codes.
Matti Hamalainen <ccr@tnsp.org>
parents:
591
diff
changeset
|
50 set urllog_shorturl_codes "ABCDEFGHIJKLNMOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" |
| 0 | 51 |
|
102
5425dc418505
urllog: Entity data is now in UTF-8, but TCL source files are interpreted with current system locale, which may not be UTF-8. We must therefore "convert" the entity mapping string to UTF-8 to be certain of TCL's interpretation of its encoding.
Matti Hamalainen <ccr@tnsp.org>
parents:
101
diff
changeset
|
52 |
| 0 | 53 #------------------------------------------------------------------------- |
| 54 ### Utility functions | |
| 560 | 55 proc urllog_log {umsg} { |
|
267
da239a953e24
urllog: Change some setting names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
264
diff
changeset
|
56 global urllog_log_enable urllog_name |
| 0 | 57 |
|
267
da239a953e24
urllog: Change some setting names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
264
diff
changeset
|
58 if {$urllog_log_enable != 0} { |
| 595 | 59 putlog "${urllog_name}: $umsg" |
| 28 | 60 } |
| 0 | 61 } |
| 62 | |
| 63 | |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
64 proc urllog_qm {uid} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
65 global urllog_messages |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
66 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
67 if {[info exists urllog_messages($uid)]} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
68 return $urllog_messages($uid) |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
69 } else { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
70 return $uid |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
71 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
72 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
73 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
74 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
75 proc urllog_smsg {apublic anick achan amsg {aargs {}}} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
76 global urllog_preferredmsg urllog_cmd_name |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
77 set amsg [string map [list "@cmd@" $urllog_cmd_name] $amsg] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
78 utl_msg_args $urllog_preferredmsg $apublic $anick $achan $amsg $aargs |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
79 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
80 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
81 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
82 proc urllog_msg {apublic anick achan aid {aargs {}}} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
83 urllog_smsg $apublic $anick $achan [urllog_qm $aid] $aargs |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
84 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
85 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
86 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
87 proc urllog_verb_msg {apublic anick achan aid {aargs {}}} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
88 global urllog_verbose |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
89 if {$urllog_verbose != 0} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
90 urllog_msg $apublic $anick $achan $aid $aargs |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
91 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
92 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
93 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
94 |
| 0 | 95 proc urllog_isnumber {uarg} { |
| 28 | 96 foreach i [split $uarg {}] { |
|
65
31c8c4f50aa6
urllog: Improve urllog_isnumber function.
Matti Hamalainen <ccr@tnsp.org>
parents:
62
diff
changeset
|
97 if {![string match \[0-9\] $i]} { return 0 } |
| 28 | 98 } |
|
65
31c8c4f50aa6
urllog: Improve urllog_isnumber function.
Matti Hamalainen <ccr@tnsp.org>
parents:
62
diff
changeset
|
99 return 1 |
| 0 | 100 } |
| 101 | |
| 102 | |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
103 proc urllog_is_enabled {uval} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
104 if {$uval} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
105 return "ON." |
| 28 | 106 } else { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
107 return "OFF." |
| 28 | 108 } |
| 0 | 109 } |
| 110 | |
| 111 | |
|
116
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
112 proc urllog_sanitize_encoding {uencoding} { |
|
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
113 regsub -- "^\[a-z\]\[a-z\]_\[A-Z\]\[A-Z\]\." $uencoding "" uencoding |
|
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
114 set uencoding [string tolower $uencoding] |
|
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
115 regsub -- "^iso-" $uencoding "iso" uencoding |
|
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
116 return $uencoding |
|
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
117 } |
|
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
118 |
| 294 | 119 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
120 proc urllog_get_ss {uindex} { |
|
592
ac72a228089b
urllog: Rename urllog_shorturl_str to urllog_shorturl_codes.
Matti Hamalainen <ccr@tnsp.org>
parents:
591
diff
changeset
|
121 global urllog_shorturl_codes |
|
ac72a228089b
urllog: Rename urllog_shorturl_str to urllog_shorturl_codes.
Matti Hamalainen <ccr@tnsp.org>
parents:
591
diff
changeset
|
122 return [string index $urllog_shorturl_codes $uindex] |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
123 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
124 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
125 |
|
150
52350ed97775
urllog: Cleanups, rename/move some global variables.
Matti Hamalainen <ccr@tnsp.org>
parents:
136
diff
changeset
|
126 proc urllog_get_short {utime} { |
|
592
ac72a228089b
urllog: Rename urllog_shorturl_str to urllog_shorturl_codes.
Matti Hamalainen <ccr@tnsp.org>
parents:
591
diff
changeset
|
127 global urllog_shorturl_prefix urllog_shorturl_codes |
|
150
52350ed97775
urllog: Cleanups, rename/move some global variables.
Matti Hamalainen <ccr@tnsp.org>
parents:
136
diff
changeset
|
128 |
|
592
ac72a228089b
urllog: Rename urllog_shorturl_str to urllog_shorturl_codes.
Matti Hamalainen <ccr@tnsp.org>
parents:
591
diff
changeset
|
129 set ulen [string length $urllog_shorturl_codes] |
| 0 | 130 |
| 28 | 131 set u1 [expr $utime / ($ulen * $ulen)] |
| 132 set utmp [expr $utime % ($ulen * $ulen)] | |
| 133 set u2 [expr $utmp / $ulen] | |
| 134 set u3 [expr $utmp % $ulen] | |
| 0 | 135 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
136 return "\[ $urllog_shorturl_prefix[urllog_get_ss $u1][urllog_get_ss $u2][urllog_get_ss $u3] \]" |
|
150
52350ed97775
urllog: Cleanups, rename/move some global variables.
Matti Hamalainen <ccr@tnsp.org>
parents:
136
diff
changeset
|
137 } |
| 0 | 138 |
| 139 | |
| 594 | 140 proc urllog_chop_str {ustr umax} { |
| 141 if {[string length $ustr] > $umax} { | |
| 142 return "[string range $ustr 0 $umax]..." | |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
143 } else { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
144 return $ustr |
| 28 | 145 } |
| 0 | 146 } |
| 147 | |
| 241 | 148 |
| 0 | 149 #------------------------------------------------------------------------- |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
150 proc urllog_add_url {urlStr urlNick urlHost urlChan urlTitle} { |
| 594 | 151 global urllog_db urllog_shorturl_enable urllog_pub_channels |
| 152 global urllog_add_title_max urllog_add_url_max | |
| 0 | 153 |
|
18
1e2232135354
More changes for SQLite support.
Matti Hamalainen <ccr@tnsp.org>
parents:
13
diff
changeset
|
154 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
155 ### Does the URL already exist? |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
156 set usql "SELECT id AS uid, utime AS utime, url AS uurl, user AS uuser, host AS uhost, chan AS uchan, title AS utitle FROM urls WHERE url='[utl_escape $urlStr]'" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
157 urllog_db eval $usql { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
158 urllog_log "URL said by $urlNick ($urlStr) already known" |
|
83
f171a9fb7b7b
urllog: Split urllog_add function to urllog_exists for checking whether given URL already exists in the database. Use urllog_exists where appropriate.
Matti Hamalainen <ccr@tnsp.org>
parents:
82
diff
changeset
|
159 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
160 if {[utl_match_delim_list $urllog_pub_channels $uchan]} { |
|
603
577763cb8864
urllog: Improve title handling in "URL is known" messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
598
diff
changeset
|
161 if {[string length $utitle] > 0} { |
|
577763cb8864
urllog: Improve title handling in "URL is known" messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
598
diff
changeset
|
162 set stitle [utl_str_map_values [urllog_qm "url_known_has_title"] [list $utitle [urllog_chop_str $utitle $urllog_add_title_max]]] |
|
577763cb8864
urllog: Improve title handling in "URL is known" messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
598
diff
changeset
|
163 } else { |
|
577763cb8864
urllog: Improve title handling in "URL is known" messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
598
diff
changeset
|
164 set stitle [urllog_qm "url_known_no_title"] |
|
577763cb8864
urllog: Improve title handling in "URL is known" messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
598
diff
changeset
|
165 } |
|
577763cb8864
urllog: Improve title handling in "URL is known" messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
598
diff
changeset
|
166 |
| 594 | 167 if {$urllog_shorturl_enable != 0} { |
|
603
577763cb8864
urllog: Improve title handling in "URL is known" messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
598
diff
changeset
|
168 urllog_verb_msg 1 $urlNick $urlChan "url_known_short" [list $uuser $uchan $uhost [utl_ctime $utime] $stitle $uurl [urllog_get_short $uid]] |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
169 } else { |
|
603
577763cb8864
urllog: Improve title handling in "URL is known" messages.
Matti Hamalainen <ccr@tnsp.org>
parents:
598
diff
changeset
|
170 urllog_verb_msg 1 $urlNick $urlChan "url_known_long" [list $uuser $uchan $uhost [utl_ctime $utime] $stitle $uurl] |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
171 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
172 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
173 return 1 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
174 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
175 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
176 ### Validate title |
|
93
4e02c0219afe
urllog: Insert NULL into title column when we didn't get a title.
Matti Hamalainen <ccr@tnsp.org>
parents:
92
diff
changeset
|
177 if {$urlTitle == ""} { |
|
4e02c0219afe
urllog: Insert NULL into title column when we didn't get a title.
Matti Hamalainen <ccr@tnsp.org>
parents:
92
diff
changeset
|
178 set uins "NULL" |
|
4e02c0219afe
urllog: Insert NULL into title column when we didn't get a title.
Matti Hamalainen <ccr@tnsp.org>
parents:
92
diff
changeset
|
179 } else { |
|
295
141bb4a2b76f
utillib: utl_escape (which will be deprecated soon).
Matti Hamalainen <ccr@tnsp.org>
parents:
294
diff
changeset
|
180 set uins "'[utl_escape $urlTitle]'" |
|
93
4e02c0219afe
urllog: Insert NULL into title column when we didn't get a title.
Matti Hamalainen <ccr@tnsp.org>
parents:
92
diff
changeset
|
181 } |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
182 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
183 ### Attempt to insert into database |
|
295
141bb4a2b76f
utillib: utl_escape (which will be deprecated soon).
Matti Hamalainen <ccr@tnsp.org>
parents:
294
diff
changeset
|
184 set usql "INSERT INTO urls (utime,url,user,host,chan,title) VALUES ([unixtime], '[utl_escape $urlStr]', '[utl_escape $urlNick]', '[utl_escape $urlHost]', '[utl_escape $urlChan]', $uins)" |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
185 if {[catch {urllog_db eval $usql} uerrmsg]} { |
|
83
f171a9fb7b7b
urllog: Split urllog_add function to urllog_exists for checking whether given URL already exists in the database. Use urllog_exists where appropriate.
Matti Hamalainen <ccr@tnsp.org>
parents:
82
diff
changeset
|
186 urllog_log "$uerrmsg on SQL:\n$usql" |
| 28 | 187 return 0 |
| 188 } | |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
189 set uid [urllog_db last_insert_rowid] |
| 28 | 190 urllog_log "Added URL ($urlNick@$urlChan): $urlStr" |
| 0 | 191 |
| 192 | |
| 28 | 193 ### Let's say something, to confirm that everything went well. |
| 594 | 194 if {$urllog_shorturl_enable != 0} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
195 set urlShort [urllog_get_short $uid] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
196 set ushort "short" |
| 28 | 197 } else { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
198 set urlShort "" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
199 set ushort "long" |
| 28 | 200 } |
| 0 | 201 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
202 if {[string length $urlTitle] > 0} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
203 set umode "url_added_${ushort}_has_title" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
204 } else { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
205 set umode "url_added_${ushort}_no_title" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
206 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
207 |
| 594 | 208 urllog_verb_msg 1 $urlNick $urlChan $umode [list $urlTitle [urllog_chop_str $urlTitle $urllog_add_title_max] $urlStr [urllog_chop_str $urlStr $urllog_add_url_max] $urlShort] |
| 28 | 209 return 1 |
| 0 | 210 } |
| 211 | |
| 212 | |
| 213 #------------------------------------------------------------------------- | |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
214 proc urllog_clear_request { urlStatus urlSCode urlCode urlData urlMeta } { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
215 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
216 ### Clear the request data |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
217 upvar $urlStatus ustatus |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
218 upvar $urlSCode uscode |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
219 upvar $urlCode ucode |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
220 upvar $urlData udata |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
221 upvar $urlMeta umeta |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
222 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
223 unset ustatus |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
224 unset uscode |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
225 unset ucode |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
226 unset udata |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
227 array unset umeta |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
228 } |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
229 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
230 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
231 #------------------------------------------------------------------------- |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
232 proc urllog_do_request { urlNick urlChan urlStr urlStatus urlSCode urlCode urlData urlMeta } { |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
233 |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
234 upvar $urlStatus ustatus |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
235 upvar $urlSCode uscode |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
236 upvar $urlCode ucode |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
237 upvar $urlData udata |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
238 upvar $urlMeta umeta |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
239 |
|
456
102dc89488af
urllog: Improve how http headers are formed.
Matti Hamalainen <ccr@tnsp.org>
parents:
425
diff
changeset
|
240 set urlHeaders {} |
|
102dc89488af
urllog: Improve how http headers are formed.
Matti Hamalainen <ccr@tnsp.org>
parents:
425
diff
changeset
|
241 lappend urlHeaders "Accept-Encoding" "identity" |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
242 #lappend urlHeaders "Connection" "keep-alive" |
|
456
102dc89488af
urllog: Improve how http headers are formed.
Matti Hamalainen <ccr@tnsp.org>
parents:
425
diff
changeset
|
243 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
244 ### Perform request |
|
456
102dc89488af
urllog: Improve how http headers are formed.
Matti Hamalainen <ccr@tnsp.org>
parents:
425
diff
changeset
|
245 if {[catch {set utoken [::http::geturl $urlStr -timeout 6000 -binary 1 -headers $urlHeaders]} uerrmsg]} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
246 urllog_verb_msg 1 $urlNick $urlChan "err_http_get" [list $urlStr $uerrmsg] |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
247 urllog_log "HTTP request failed: $uerrmsg" |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
248 return 0 |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
249 } |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
250 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
251 ### Check status |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
252 set ustatus [::http::status $utoken] |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
253 set uscode [::http::code $utoken] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
254 set ucode [::http::ncode $utoken] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
255 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
256 if {$ustatus != "ok"} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
257 urllog_verb_msg 1 $urlNick $urlChan "err_http_status" [list $urlStr $ustatus $uscode $ucode] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
258 urllog_log "Error in HTTP request: $ustatus / $uscode ($urlStr)" |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
259 return 0 |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
260 } |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
261 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
262 ### Get data |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
263 set udata [::http::data $utoken] |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
264 array set umeta [::http::meta $utoken] |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
265 ::http::cleanup $utoken |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
266 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
267 ### Sanitize the metadata KEYS |
|
584
9b64f201b3a7
urllog: Use lowercase HTTP metadata keys and sanitize them.
Matti Hamalainen <ccr@tnsp.org>
parents:
583
diff
changeset
|
268 foreach {ukey uvalue} [array get umeta] { |
|
9b64f201b3a7
urllog: Use lowercase HTTP metadata keys and sanitize them.
Matti Hamalainen <ccr@tnsp.org>
parents:
583
diff
changeset
|
269 set ukey [string tolower $ukey] |
|
9b64f201b3a7
urllog: Use lowercase HTTP metadata keys and sanitize them.
Matti Hamalainen <ccr@tnsp.org>
parents:
583
diff
changeset
|
270 set umeta($ukey) $uvalue |
|
9b64f201b3a7
urllog: Use lowercase HTTP metadata keys and sanitize them.
Matti Hamalainen <ccr@tnsp.org>
parents:
583
diff
changeset
|
271 } |
|
9b64f201b3a7
urllog: Use lowercase HTTP metadata keys and sanitize them.
Matti Hamalainen <ccr@tnsp.org>
parents:
583
diff
changeset
|
272 |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
273 return 1 |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
274 } |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
275 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
276 |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
277 #------------------------------------------------------------------------- |
|
327
a5282cdc56e6
urllog: Fix redirection handling for HTTP 1.1.
Matti Hamalainen <ccr@tnsp.org>
parents:
319
diff
changeset
|
278 proc urllog_validate_url { urlNick urlChan urlMStr urlMProto urlMHostName } { |
| 594 | 279 global urllog_httprep urllog_shorturl_prefix urllog_shorturl_enable |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
280 |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
281 upvar $urlMStr urlStr |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
282 upvar $urlMProto urlProto |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
283 upvar $urlMHostName urlHostName |
|
3
8003090caa35
Lots of code cleanups, add "fixer" for RasiaTube links (which suck) to point directly to Youtube.
Matti Hamalainen <ccr@tnsp.org>
parents:
0
diff
changeset
|
284 |
|
571
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
285 ### Hack for removing parenthesis around an URL |
|
598
40db54427950
urllog: Clean up regexp statements.
Matti Hamalainen <ccr@tnsp.org>
parents:
595
diff
changeset
|
286 if {[regexp {^\((.+)\)$} $urlStr -> urlClean]} { |
|
571
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
287 set urlStr $urlClean |
|
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
288 } |
|
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
289 |
|
598
40db54427950
urllog: Clean up regexp statements.
Matti Hamalainen <ccr@tnsp.org>
parents:
595
diff
changeset
|
290 if {[regexp {^\[(.+)\]$} $urlStr -> urlClean]} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
291 set urlStr $urlClean |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
292 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
293 |
|
571
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
294 ### Clean excess stuff, if any, and attempt to |
|
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
295 ### guess the URL protocol component if it is missing |
|
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
296 if {[regexp "(\[a-z\]+)://\[^ \]+" $urlStr urlMatch urlProto]} { |
|
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
297 set urlStr $urlMatch |
|
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
298 } elseif {[regexp "www\.\[^ \]+" $urlStr urlMatch]} { |
|
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
299 set urlStr "http://$urlMatch" |
|
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
300 } elseif {[regexp "ftp\.\[^ \]+" $urlStr urlMatch]} { |
|
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
301 set urlStr "ftp://$urlMatch" |
| 28 | 302 } |
| 0 | 303 |
|
95
687bdd74dfac
urllog: Check if TLS support is enabled when checking if we can fetch title information via HTTP or SSL/HTTP.
Matti Hamalainen <ccr@tnsp.org>
parents:
93
diff
changeset
|
304 ### Handle URLs that have an IPv4-address |
|
598
40db54427950
urllog: Clean up regexp statements.
Matti Hamalainen <ccr@tnsp.org>
parents:
595
diff
changeset
|
305 if {[regexp "(\[a-z\]+)://(\[0-9\]{1,3})\\.(\[0-9\]{1,3})\\.(\[0-9\]{1,3})\\.(\[0-9\]{1,3})" $urlStr -> urlProto ni1 ni2 ni3 ni4]} { |
| 28 | 306 # Check if the IP is on local network |
|
572
295c225e3152
urllog: Improve invalid/local IPv4 network check
Matti Hamalainen <ccr@tnsp.org>
parents:
571
diff
changeset
|
307 if {$ni1 == 127 || $ni1 == 10 || ($ni1 == 192 && $ni2 == 168)} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
308 urllog_verb_msg 1 $urlNick $urlChan "err_url_local_net" [list $urlStr] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
309 urllog_log "URL pointing to local network, ignored (${urlStr})." |
| 28 | 310 return 0 |
| 311 } | |
|
572
295c225e3152
urllog: Improve invalid/local IPv4 network check
Matti Hamalainen <ccr@tnsp.org>
parents:
571
diff
changeset
|
312 if {$ni1 == 0 || $ni1 >= 255 || $ni2 >= 255 || $ni3 >= 255 || $ni4 >= 255} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
313 urllog_verb_msg 1 $urlNick $urlChan "err_url_invalid_net" [list $urlStr] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
314 urllog_log "URL pointing to invalid network, ignored (${urlStr})." |
|
571
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
315 return 0 |
|
d4d2fda12308
urllog: Improve URL parsing/validation and protocol guessing.
Matti Hamalainen <ccr@tnsp.org>
parents:
570
diff
changeset
|
316 } |
| 28 | 317 } |
| 0 | 318 |
|
96
e5a6c27be365
urllog: Comments and cosmetics.
Matti Hamalainen <ccr@tnsp.org>
parents:
95
diff
changeset
|
319 ### Check now if we have an ShortURL here ... |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
320 if {[string match "${urllog_shorturl_prefix}*" $urlStr]} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
321 urllog_log "Ignoring ShortURL: ${urlStr}" |
|
252
eb2fce89b8ab
urllog: Comment out some currently unused code.
Matti Hamalainen <ccr@tnsp.org>
parents:
251
diff
changeset
|
322 # set uud "" |
|
eb2fce89b8ab
urllog: Comment out some currently unused code.
Matti Hamalainen <ccr@tnsp.org>
parents:
251
diff
changeset
|
323 # set usql "SELECT id AS uid, url AS uurl, user AS uuser, host AS uhost, chan AS uchan, title AS utitle FROM urls WHERE utime=$uud" |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
324 # urllog_db eval $usql { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
325 # urllog_msg 1 $urlNick $urlChan "'$utitle' - $uurl" |
|
252
eb2fce89b8ab
urllog: Comment out some currently unused code.
Matti Hamalainen <ccr@tnsp.org>
parents:
251
diff
changeset
|
326 # return 1 |
|
eb2fce89b8ab
urllog: Comment out some currently unused code.
Matti Hamalainen <ccr@tnsp.org>
parents:
251
diff
changeset
|
327 # } |
| 28 | 328 return 0 |
| 329 } | |
| 0 | 330 |
|
95
687bdd74dfac
urllog: Check if TLS support is enabled when checking if we can fetch title information via HTTP or SSL/HTTP.
Matti Hamalainen <ccr@tnsp.org>
parents:
93
diff
changeset
|
331 ### Get URL protocol component |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
332 set urlProto "" |
|
598
40db54427950
urllog: Clean up regexp statements.
Matti Hamalainen <ccr@tnsp.org>
parents:
595
diff
changeset
|
333 if {[regexp "(\[a-z\]+)://" $urlStr -> urlProto]} { |
|
464
506977ea9d0c
urllog: Improve URL validation.
Matti Hamalainen <ccr@tnsp.org>
parents:
458
diff
changeset
|
334 ### Is it a http or ftp url? |
|
506977ea9d0c
urllog: Improve URL validation.
Matti Hamalainen <ccr@tnsp.org>
parents:
458
diff
changeset
|
335 if {$urlProto != "http" && $urlProto != "https" && $urlProto != "ftp"} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
336 urllog_verb_msg 1 $urlNick $urlChan "err_url_proto_class" [list $urlStr $urlProto] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
337 urllog_log "Broken URL: ${urlStr} - unsupported protocol class (${urlProto})." |
|
464
506977ea9d0c
urllog: Improve URL validation.
Matti Hamalainen <ccr@tnsp.org>
parents:
458
diff
changeset
|
338 return 0 |
|
506977ea9d0c
urllog: Improve URL validation.
Matti Hamalainen <ccr@tnsp.org>
parents:
458
diff
changeset
|
339 } |
|
506977ea9d0c
urllog: Improve URL validation.
Matti Hamalainen <ccr@tnsp.org>
parents:
458
diff
changeset
|
340 } else { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
341 urllog_verb_msg 1 $urlNick $urlChan "err_url_proto_no_class" [list $urlStr] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
342 urllog_log "Broken URL: ${urlStr} - no protocol specifier." |
|
464
506977ea9d0c
urllog: Improve URL validation.
Matti Hamalainen <ccr@tnsp.org>
parents:
458
diff
changeset
|
343 return 0 |
|
506977ea9d0c
urllog: Improve URL validation.
Matti Hamalainen <ccr@tnsp.org>
parents:
458
diff
changeset
|
344 } |
|
95
687bdd74dfac
urllog: Check if TLS support is enabled when checking if we can fetch title information via HTTP or SSL/HTTP.
Matti Hamalainen <ccr@tnsp.org>
parents:
93
diff
changeset
|
345 |
| 28 | 346 ### Check the PORT (if the ":" is there) |
|
327
a5282cdc56e6
urllog: Fix redirection handling for HTTP 1.1.
Matti Hamalainen <ccr@tnsp.org>
parents:
319
diff
changeset
|
347 set urlRecord [split $urlStr "/"] |
|
a5282cdc56e6
urllog: Fix redirection handling for HTTP 1.1.
Matti Hamalainen <ccr@tnsp.org>
parents:
319
diff
changeset
|
348 set urlHostName [lindex $urlRecord 2] |
|
a5282cdc56e6
urllog: Fix redirection handling for HTTP 1.1.
Matti Hamalainen <ccr@tnsp.org>
parents:
319
diff
changeset
|
349 set urlPort [lindex [split $urlHostName ":"] end] |
| 0 | 350 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
351 if {$urlPort != "" && ![urllog_isnumber $urlPort] && $urlPort != $urlHostName} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
352 urllog_verb_msg 1 $urlNick $urlChan "err_url_invalid_port" [list $urlStr $urlPort] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
353 urllog_log "Broken URL: ${urlStr} - illegal or invalid port '${urlPort}'" |
| 28 | 354 return 0 |
| 355 } | |
| 0 | 356 |
| 357 | |
| 28 | 358 set urlStr [string map $urllog_httprep $urlStr] |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
359 return 1 |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
360 } |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
361 |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
362 |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
363 #------------------------------------------------------------------------- |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
364 proc urllog_handle_redirect {urlNick urlHost urlChan urlRedirLevel urlProto urlHostName urlStr urlStatus urlSCode urlCode urlData urlMeta} { |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
365 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
366 upvar $urlProto uproto |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
367 upvar $urlHostName uhostname |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
368 upvar $urlStr ustr |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
369 upvar $urlStatus ustatus |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
370 upvar $urlSCode uscode |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
371 upvar $urlCode ucode |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
372 upvar $urlData udata |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
373 upvar $urlMeta umeta |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
374 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
375 ### Was result a redirect? |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
376 if {$ucode >= 301 && $ucode <= 303} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
377 ### Check that we have a location header |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
378 if {![info exists umeta(location)]} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
379 urllog_verb_msg 1 $urlNick $urlChan "err_redirect_invalid" [list $ustr $ustatus $uscode $ucode $urlRedirLevel] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
380 urllog_log "Invalid redirect without location header: status=$ustatus, code=$ucode, scode=$uscode, url=$ustr, redirLevel=$urlRedirLevel" |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
381 return 0 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
382 } |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
383 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
384 ### Fix up location URI |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
385 set nustr $umeta(location) |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
386 if {![regexp "\[a-z\]+://" $nustr]} { |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
387 if {[string range $nustr 0 0] != "/"} { |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
388 append nustr "/" |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
389 } |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
390 set nustr "${uproto}://${uhostname}${nustr}" |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
391 } |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
392 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
393 ### Validate the target URI |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
394 urllog_log "Redirection #${urlRedirLevel}: $ustr -> $nustr" |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
395 set ustr $nustr |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
396 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
397 if {![urllog_validate_url $urlNick $urlChan ustr uproto uhostname]} { |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
398 return 0 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
399 } |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
400 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
401 ### Attempt to fetch redirection target |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
402 urllog_clear_request ustatus uscode ucode udata umeta |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
403 if {![urllog_do_request $urlNick $urlChan $ustr ustatus uscode ucode udata umeta]} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
404 urllog_verb_msg 1 $urlNick $urlChan "err_redirect_fail" [list $ustr $ustatus $uscode $ucode $urlRedirLevel] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
405 urllog_log "Error fetching redirect: status=$ustatus, code=$ucode, scode=$uscode, url=$ustr, redirLevel=$urlRedirLevel" |
|
585
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
406 return 0 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
407 } |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
408 } |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
409 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
410 return 1 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
411 } |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
412 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
413 |
|
a5dc31f5b44e
urllog: Clean up redirection handling and improve error checking and handling.
Matti Hamalainen <ccr@tnsp.org>
parents:
584
diff
changeset
|
414 #------------------------------------------------------------------------- |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
415 proc urllog_check_url {urlStr urlNick urlHost urlChan} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
416 global urllog_encoding http_tls_support |
|
581
148c7553c50f
urllog: Remove rasiatube hack setting, it's not used.
Matti Hamalainen <ccr@tnsp.org>
parents:
579
diff
changeset
|
417 global urllog_extra_checks urllog_extra_strict |
|
3
8003090caa35
Lots of code cleanups, add "fixer" for RasiaTube links (which suck) to point directly to Youtube.
Matti Hamalainen <ccr@tnsp.org>
parents:
0
diff
changeset
|
418 |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
419 ### Validate URL compoments, etc. |
|
327
a5282cdc56e6
urllog: Fix redirection handling for HTTP 1.1.
Matti Hamalainen <ccr@tnsp.org>
parents:
319
diff
changeset
|
420 set urlProto "" |
|
a5282cdc56e6
urllog: Fix redirection handling for HTTP 1.1.
Matti Hamalainen <ccr@tnsp.org>
parents:
319
diff
changeset
|
421 set urlHostName "" |
|
a5282cdc56e6
urllog: Fix redirection handling for HTTP 1.1.
Matti Hamalainen <ccr@tnsp.org>
parents:
319
diff
changeset
|
422 if {![urllog_validate_url $urlNick $urlChan urlStr urlProto urlHostName]} { |
|
251
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
423 return 1 |
|
e59f0c3ea0f4
urllog: Handle first and second level redirects.
Matti Hamalainen <ccr@tnsp.org>
parents:
250
diff
changeset
|
424 } |
| 0 | 425 |
|
267
da239a953e24
urllog: Change some setting names, etc.
Matti Hamalainen <ccr@tnsp.org>
parents:
264
diff
changeset
|
426 ### Do we perform additional checks? |
|
327
a5282cdc56e6
urllog: Fix redirection handling for HTTP 1.1.
Matti Hamalainen <ccr@tnsp.org>
parents:
319
diff
changeset
|
427 if {$urllog_extra_checks == 0 || !(($http_tls_support != 0 && $urlProto == "https") || $urlProto == "http")} { |
| 230 | 428 # No optional checks, or it's not http/https. |
| 306 | 429 if {$urllog_extra_strict == 0} { |
|
304
f1589fe20732
urllog: Added urllog_extra_strict option.
Matti Hamalainen <ccr@tnsp.org>
parents:
302
diff
changeset
|
430 # Strict checking disabled, so add the URL, if it does not exist already. |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
431 urllog_add_url $urlStr $urlNick $urlHost $urlChan "" |
|
304
f1589fe20732
urllog: Added urllog_extra_strict option.
Matti Hamalainen <ccr@tnsp.org>
parents:
302
diff
changeset
|
432 return 1 |
|
327
a5282cdc56e6
urllog: Fix redirection handling for HTTP 1.1.
Matti Hamalainen <ccr@tnsp.org>
parents:
319
diff
changeset
|
433 } elseif {$http_tls_support == 0 && $urlProto == "https"} { |
|
304
f1589fe20732
urllog: Added urllog_extra_strict option.
Matti Hamalainen <ccr@tnsp.org>
parents:
302
diff
changeset
|
434 # Strict ENABLED: If TLS support is disabled and we have https, do nothing |
|
f1589fe20732
urllog: Added urllog_extra_strict option.
Matti Hamalainen <ccr@tnsp.org>
parents:
302
diff
changeset
|
435 return 1 |
|
327
a5282cdc56e6
urllog: Fix redirection handling for HTTP 1.1.
Matti Hamalainen <ccr@tnsp.org>
parents:
319
diff
changeset
|
436 } elseif {$urlProto != "http" && $urlProto != "https"} { |
|
304
f1589fe20732
urllog: Added urllog_extra_strict option.
Matti Hamalainen <ccr@tnsp.org>
parents:
302
diff
changeset
|
437 # Strict ENABLED: It's not http, or https |
|
f1589fe20732
urllog: Added urllog_extra_strict option.
Matti Hamalainen <ccr@tnsp.org>
parents:
302
diff
changeset
|
438 return 1 |
|
f1589fe20732
urllog: Added urllog_extra_strict option.
Matti Hamalainen <ccr@tnsp.org>
parents:
302
diff
changeset
|
439 } |
|
7
50b52294e93e
urllog: Strip ‏ entities from titles; Some work on SSL/https support.
Matti Hamalainen <ccr@tnsp.org>
parents:
4
diff
changeset
|
440 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
441 } else { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
442 ### Does the document pointed by the URL exist? |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
443 if {![urllog_do_request $urlNick $urlChan $urlStr ustatus uscode ucode udata umeta]} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
444 return 1 |
|
116
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
445 } |
|
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
446 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
447 ### Handle redirects of 3 levels |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
448 if {![urllog_handle_redirect $urlNick $urlHost $urlChan 1 urlProto urlHostName urlStr ustatus uscode ucode udata umeta]} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
449 return 1 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
450 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
451 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
452 if {![urllog_handle_redirect $urlNick $urlHost $urlChan 2 urlProto urlHostName urlStr ustatus uscode ucode udata umeta]} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
453 return 1 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
454 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
455 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
456 if {![urllog_handle_redirect $urlNick $urlHost $urlChan 3 urlProto urlHostName urlStr ustatus uscode ucode udata umeta]} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
457 return 1 |
|
116
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
458 } |
|
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
459 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
460 # Final document |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
461 if {$ucode >= 200 && $ucode <= 205} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
462 set uenc_doc "" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
463 set uenc_http "" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
464 set uencoding "" |
|
116
4f3edcf72987
urllog: Improvements in document / HTTP encoding handling and conversion.
Matti Hamalainen <ccr@tnsp.org>
parents:
115
diff
changeset
|
465 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
466 # Get information about specified character encodings |
|
598
40db54427950
urllog: Clean up regexp statements.
Matti Hamalainen <ccr@tnsp.org>
parents:
595
diff
changeset
|
467 if {[info exists umeta(content-type)] && [regexp -nocase {charset\s*=\s*([a-z0-9._-]+)} $umeta(content-type) -> uenc_http]} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
468 # Found character set encoding information in HTTP headers |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
469 } |
|
470
2faf2eb18f26
urllog: Add support for Twitter meta titles.
Matti Hamalainen <ccr@tnsp.org>
parents:
469
diff
changeset
|
470 |
|
598
40db54427950
urllog: Clean up regexp statements.
Matti Hamalainen <ccr@tnsp.org>
parents:
595
diff
changeset
|
471 if {[regexp -nocase -- "<meta.\*\?content=\"text/html.\*\?charset=(\[^\"\]*)\".\*\?/\?>" $udata -> uenc_doc]} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
472 # Found old style HTML meta tag with character set information |
|
598
40db54427950
urllog: Clean up regexp statements.
Matti Hamalainen <ccr@tnsp.org>
parents:
595
diff
changeset
|
473 } elseif {[regexp -nocase -- "<meta.\*\?charset=\"(\[^\"\]*)\".\*\?/\?>" $udata -> uenc_doc]} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
474 # Found HTML5 style meta tag with character set information |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
475 } |
|
313
8175ef52889b
urllog: Improve URL title functionality.
Matti Hamalainen <ccr@tnsp.org>
parents:
312
diff
changeset
|
476 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
477 # Make sanitized versions of the encoding strings |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
478 set uenc_http2 [urllog_sanitize_encoding $uenc_http] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
479 set uenc_doc2 [urllog_sanitize_encoding $uenc_doc] |
|
313
8175ef52889b
urllog: Improve URL title functionality.
Matti Hamalainen <ccr@tnsp.org>
parents:
312
diff
changeset
|
480 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
481 # Check if the document has specified encoding |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
482 # KLUDGE! |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
483 set uencoding $uenc_http2 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
484 if {$uencoding == "" && $uenc_doc2 != ""} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
485 set uencoding $uenc_doc2 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
486 } elseif {$uencoding == ""} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
487 # If _NO_ known encoding of any kind, assume the default of iso8859-1 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
488 set uencoding "iso8859-1" |
| 28 | 489 } |
|
150
52350ed97775
urllog: Cleanups, rename/move some global variables.
Matti Hamalainen <ccr@tnsp.org>
parents:
136
diff
changeset
|
490 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
491 urllog_log "Charsets: http='$uenc_http', doc='$uenc_doc' / sanitized http='$uenc_http2', doc='$uenc_doc2' -> '$uencoding'" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
492 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
493 # Get the document title, if any |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
494 set urlTitle "" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
495 |
|
598
40db54427950
urllog: Clean up regexp statements.
Matti Hamalainen <ccr@tnsp.org>
parents:
595
diff
changeset
|
496 if {[regexp -nocase -- "<meta name=\"twitter:title\" content=\"(.\*\?)\"\\s\*\/\?>" $udata -> urlTitle]} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
497 # ... |
|
598
40db54427950
urllog: Clean up regexp statements.
Matti Hamalainen <ccr@tnsp.org>
parents:
595
diff
changeset
|
498 } elseif {[regexp -nocase -- "<title.\*\?>(.\*\?)</title>" $udata -> urlTitle]} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
499 # ... |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
500 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
501 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
502 # If facebook, get meta info |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
503 if {[regexp -nocase -- "(http|https):\/\/www.facebook.com" $urlStr]} { |
|
598
40db54427950
urllog: Clean up regexp statements.
Matti Hamalainen <ccr@tnsp.org>
parents:
595
diff
changeset
|
504 if {[regexp -nocase -- "<meta name=\"description\" content=\"(.\*\?)\"" $udata -> urlTmp]} { |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
505 if {$urlTitle != ""} { append urlTitle " :: " } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
506 append urlTitle $urlTmp |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
507 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
508 } |
|
3
8003090caa35
Lots of code cleanups, add "fixer" for RasiaTube links (which suck) to point directly to Youtube.
Matti Hamalainen <ccr@tnsp.org>
parents:
0
diff
changeset
|
509 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
510 # If character set conversion is required, do it now |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
511 if {$urlTitle != "" && $uencoding != ""} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
512 if {[catch {set urlTitle [encoding convertfrom $uencoding $urlTitle]} cerrmsg]} { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
513 urllog_verb_msg 1 $urlNick $urlChan "err_charset" [list $urlStr $urlTitle $cerrmsg] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
514 urllog_log "Error in charset conversion: $urlStr / '$urlTitle': $cerrmsg" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
515 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
516 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
517 # Convert some HTML entities to plaintext and do some cleanup |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
518 set utmp [utl_convert_html_ent $urlTitle] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
519 regsub -all "\r|\n|\t" $utmp " " utmp |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
520 regsub -all " *" $utmp " " utmp |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
521 set urlTitle [string trim $utmp] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
522 } |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
523 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
524 # Check if the URL already exists, just in case we had some redirects |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
525 urllog_add_url $urlStr $urlNick $urlHost $urlChan $urlTitle |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
526 } else { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
527 urllog_verb_msg 1 $urlNick $urlChan "err_http_fail" [list $urlStr $ustatus $uscode $ucode] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
528 urllog_log "Error fetching document: status=$ustatus, code=$ucode, scode=$uscode, url=$urlStr" |
|
83
f171a9fb7b7b
urllog: Split urllog_add function to urllog_exists for checking whether given URL already exists in the database. Use urllog_exists where appropriate.
Matti Hamalainen <ccr@tnsp.org>
parents:
82
diff
changeset
|
529 } |
| 28 | 530 } |
| 0 | 531 } |
| 532 | |
| 533 | |
| 534 #------------------------------------------------------------------------- | |
| 249 | 535 proc urllog_check_line {unick uhost uhand uchan utext} { |
|
219
4e09bcc48851
urllog: Add settings for specifying channels where URL logging is active, and where !urlfind functionality works (separately, if so desired.)
Matti Hamalainen <ccr@tnsp.org>
parents:
218
diff
changeset
|
536 global urllog_log_channels |
|
4e09bcc48851
urllog: Add settings for specifying channels where URL logging is active, and where !urlfind functionality works (separately, if so desired.)
Matti Hamalainen <ccr@tnsp.org>
parents:
218
diff
changeset
|
537 |
| 28 | 538 ### Check the nick |
| 87 | 539 if {$unick == "*"} { |
| 28 | 540 return 0 |
| 541 } | |
| 0 | 542 |
|
219
4e09bcc48851
urllog: Add settings for specifying channels where URL logging is active, and where !urlfind functionality works (separately, if so desired.)
Matti Hamalainen <ccr@tnsp.org>
parents:
218
diff
changeset
|
543 ### Check the channel |
|
315
7a987b22a817
urllog: Add new configuration option urllog_msg_channels.
Matti Hamalainen <ccr@tnsp.org>
parents:
313
diff
changeset
|
544 if {[utl_match_delim_list $urllog_log_channels $uchan]} { |
|
7a987b22a817
urllog: Add new configuration option urllog_msg_channels.
Matti Hamalainen <ccr@tnsp.org>
parents:
313
diff
changeset
|
545 ### Do the URL checking |
|
651
ecd8fb2b9ad3
urllog: Use regexp splitx instead of string split.
Matti Hamalainen <ccr@tnsp.org>
parents:
603
diff
changeset
|
546 foreach str [::textutil::split::splitx $utext {\s+}] { |
|
464
506977ea9d0c
urllog: Improve URL validation.
Matti Hamalainen <ccr@tnsp.org>
parents:
458
diff
changeset
|
547 if {[regexp "(\[a-z]+://\[^\[:space:\]\]+|^(www|ftp)\.\[^\[:space:\]\]+)" $str ulink]} { |
|
315
7a987b22a817
urllog: Add new configuration option urllog_msg_channels.
Matti Hamalainen <ccr@tnsp.org>
parents:
313
diff
changeset
|
548 urllog_check_url $str $unick $uhost $uchan |
|
219
4e09bcc48851
urllog: Add settings for specifying channels where URL logging is active, and where !urlfind functionality works (separately, if so desired.)
Matti Hamalainen <ccr@tnsp.org>
parents:
218
diff
changeset
|
549 } |
| 28 | 550 } |
| 551 } | |
| 0 | 552 |
| 28 | 553 return 0 |
| 0 | 554 } |
| 555 | |
| 556 | |
| 557 #------------------------------------------------------------------------- | |
| 558 ### Parse arguments, find and show the results | |
| 594 | 559 proc urllog_cmd_search {unick uhand uchan utext upublic} { |
| 560 global urllog_db urllog_shorturl_enable urllog_showmax_pub urllog_showmax_priv | |
| 561 global urllog_search_title_max urllog_search_url_max | |
| 0 | 562 |
| 28 | 563 if {$upublic == 0} { |
| 564 set ulimit 5 | |
| 565 } else { | |
| 566 set ulimit 3 | |
| 567 } | |
|
19
9cf22053e5da
Repair !urlfind functionality.
Matti Hamalainen <ccr@tnsp.org>
parents:
18
diff
changeset
|
568 |
| 28 | 569 ### Parse the given command |
|
651
ecd8fb2b9ad3
urllog: Use regexp splitx instead of string split.
Matti Hamalainen <ccr@tnsp.org>
parents:
603
diff
changeset
|
570 urllog_log "${unick}/${uhand} searched URL: ${utext}" |
| 0 | 571 |
|
651
ecd8fb2b9ad3
urllog: Use regexp splitx instead of string split.
Matti Hamalainen <ccr@tnsp.org>
parents:
603
diff
changeset
|
572 set ftokens [::textutil::split::splitx $utext {\s+}] |
| 28 | 573 set fpatlist "" |
| 574 foreach ftoken $ftokens { | |
| 575 set fprefix [string range $ftoken 0 0] | |
| 576 set fpattern [string range $ftoken 1 end] | |
|
295
141bb4a2b76f
utillib: utl_escape (which will be deprecated soon).
Matti Hamalainen <ccr@tnsp.org>
parents:
294
diff
changeset
|
577 set qpattern "'%[utl_escape $fpattern]%'" |
| 0 | 578 |
| 28 | 579 if {$fprefix == "-"} { |
|
128
0d21b9d1d2b9
urllog: Improve search functionality.
Matti Hamalainen <ccr@tnsp.org>
parents:
127
diff
changeset
|
580 lappend fpatlist "(url NOT LIKE $qpattern OR title NOT LIKE $qpattern)" |
| 28 | 581 } elseif {$fprefix == "%"} { |
|
128
0d21b9d1d2b9
urllog: Improve search functionality.
Matti Hamalainen <ccr@tnsp.org>
parents:
127
diff
changeset
|
582 lappend fpatlist "user LIKE $qpattern" |
| 28 | 583 } elseif {$fprefix == "@"} { |
| 584 # foo | |
|
112
fae3dd7a8b20
urllog: Oops, a typo in variable name. Fixed.
Matti Hamalainen <ccr@tnsp.org>
parents:
111
diff
changeset
|
585 } elseif {$fprefix == "+"} { |
|
128
0d21b9d1d2b9
urllog: Improve search functionality.
Matti Hamalainen <ccr@tnsp.org>
parents:
127
diff
changeset
|
586 lappend fpatlist "(url LIKE $qpattern OR title LIKE $qpattern)" |
| 28 | 587 } else { |
|
295
141bb4a2b76f
utillib: utl_escape (which will be deprecated soon).
Matti Hamalainen <ccr@tnsp.org>
parents:
294
diff
changeset
|
588 set qpattern "'%[utl_escape $ftoken]%'" |
|
128
0d21b9d1d2b9
urllog: Improve search functionality.
Matti Hamalainen <ccr@tnsp.org>
parents:
127
diff
changeset
|
589 lappend fpatlist "(url LIKE $qpattern OR title LIKE $qpattern)" |
| 28 | 590 } |
| 591 } | |
|
19
9cf22053e5da
Repair !urlfind functionality.
Matti Hamalainen <ccr@tnsp.org>
parents:
18
diff
changeset
|
592 |
|
27
6e381916b016
Some fixes in the query mechanisms of QuoteDB and URLLog.
Matti Hamalainen <ccr@tnsp.org>
parents:
20
diff
changeset
|
593 if {[llength $fpatlist] > 0} { |
|
6e381916b016
Some fixes in the query mechanisms of QuoteDB and URLLog.
Matti Hamalainen <ccr@tnsp.org>
parents:
20
diff
changeset
|
594 set fquery "WHERE [join $fpatlist " AND "]" |
|
6e381916b016
Some fixes in the query mechanisms of QuoteDB and URLLog.
Matti Hamalainen <ccr@tnsp.org>
parents:
20
diff
changeset
|
595 } else { |
|
6e381916b016
Some fixes in the query mechanisms of QuoteDB and URLLog.
Matti Hamalainen <ccr@tnsp.org>
parents:
20
diff
changeset
|
596 set fquery "" |
|
6e381916b016
Some fixes in the query mechanisms of QuoteDB and URLLog.
Matti Hamalainen <ccr@tnsp.org>
parents:
20
diff
changeset
|
597 } |
| 68 | 598 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
599 ### Perform SQL query and show results if any |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
600 set nresult 0 |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
601 set usql "SELECT id AS uid, utime AS utime, url AS uurl, user AS uuser, host AS uhost, title AS utitle FROM urls $fquery ORDER BY utime DESC LIMIT $ulimit" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
602 urllog_db eval $usql { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
603 incr nresult |
| 594 | 604 if {[string length $utitle] > 0} { |
| 605 set stitle [utl_str_map_values [urllog_qm "search_result_has_title"] [list $utitle [urllog_chop_str $utitle $urllog_search_title_max]]] | |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
606 } else { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
607 set stitle [urllog_qm "search_result_no_title"] |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
608 } |
| 594 | 609 |
| 610 if {$urllog_shorturl_enable != 0 && $uid != ""} { | |
| 611 urllog_msg $upublic $unick $uchan "search_result_short" [list $nresult $uuser [utl_ctime $utime] $stitle [urllog_chop_str $uurl $urllog_search_url_max] [urllog_get_short $uid]] | |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
612 } else { |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
613 urllog_msg $upublic $unick $uchan "search_result_long" [list $nresult $uuser [utl_ctime $utime] $stitle $uurl] |
| 28 | 614 } |
| 615 } | |
|
424
825cac46b1cb
Cosmetic / stray trailing whitespace cleanup.
Matti Hamalainen <ccr@tnsp.org>
parents:
422
diff
changeset
|
616 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
617 if {$nresult == 0} { |
| 28 | 618 # If no URLs were found |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
619 urllog_msg $upublic $unick $uchan "search_no_match" [list $utext] |
| 28 | 620 } |
| 0 | 621 |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
622 return 1 |
| 0 | 623 } |
| 624 | |
| 625 | |
| 626 #------------------------------------------------------------------------- | |
| 627 ### Finding binded functions | |
| 594 | 628 proc urllog_cmd_pub_search {unick uhost uhand uchan utext} { |
|
219
4e09bcc48851
urllog: Add settings for specifying channels where URL logging is active, and where !urlfind functionality works (separately, if so desired.)
Matti Hamalainen <ccr@tnsp.org>
parents:
218
diff
changeset
|
629 global urllog_search_channels |
|
4e09bcc48851
urllog: Add settings for specifying channels where URL logging is active, and where !urlfind functionality works (separately, if so desired.)
Matti Hamalainen <ccr@tnsp.org>
parents:
218
diff
changeset
|
630 |
|
315
7a987b22a817
urllog: Add new configuration option urllog_msg_channels.
Matti Hamalainen <ccr@tnsp.org>
parents:
313
diff
changeset
|
631 if {[utl_match_delim_list $urllog_search_channels $uchan]} { |
| 594 | 632 return [urllog_cmd_search $unick $uhand $uchan $utext 1] |
|
219
4e09bcc48851
urllog: Add settings for specifying channels where URL logging is active, and where !urlfind functionality works (separately, if so desired.)
Matti Hamalainen <ccr@tnsp.org>
parents:
218
diff
changeset
|
633 } |
| 28 | 634 return 0 |
| 0 | 635 } |
| 636 | |
| 637 | |
| 594 | 638 proc urllog_cmd_msg_search {unick uhost uhand utext} { |
| 639 return [urllog_cmd_search $unick $uhand "" $utext 0] | |
|
3
8003090caa35
Lots of code cleanups, add "fixer" for RasiaTube links (which suck) to point directly to Youtube.
Matti Hamalainen <ccr@tnsp.org>
parents:
0
diff
changeset
|
640 } |
| 0 | 641 |
| 560 | 642 |
| 643 #------------------------------------------------------------------------- | |
| 644 # Script initialization | |
| 645 #------------------------------------------------------------------------- | |
| 570 | 646 ### Initialization messages |
| 560 | 647 putlog "$urllog_message" |
| 648 | |
| 570 | 649 |
| 650 ### Miscellaneous init messages | |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
651 putlog " - Log messages [urllog_is_enabled $urllog_log_enable]" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
652 putlog " - Verbose mode [urllog_is_enabled $urllog_verbose]" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
653 putlog " - Additional URL validity checks [urllog_is_enabled $urllog_extra_checks]" |
|
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
654 putlog " - Strict checks [urllog_is_enabled $urllog_extra_strict]" |
| 560 | 655 |
| 570 | 656 |
| 560 | 657 ### HTTP module initialization |
| 658 if {[info exists http_user_agent] && $http_user_agent != ""} { | |
| 659 ::http::config -useragent $http_user_agent | |
| 660 } else { | |
| 661 ::http::config -useragent "$urllog_name/$urllog_version" | |
| 662 } | |
| 663 | |
| 664 if {[info exists http_use_proxy] && $http_use_proxy != 0} { | |
| 665 ::http::config -proxyhost $http_proxy_host -proxyport $http_proxy_port | |
| 666 putlog " - Using proxy $http_proxy_host:$http_proxy_port" | |
| 667 } | |
| 668 | |
| 669 if {[info exists http_tls_support] && $http_tls_support != 0} { | |
| 670 package require tls | |
| 671 ::http::register https 443 [list ::tls::socket -request true -require true -ssl2 false -ssl3 false -tls1 true -tls1.1 true -tls1.2 true -cadir $http_tls_cadir -autoservername true] | |
| 672 putlog " - TLS/SSL support enabled." | |
| 673 } | |
| 674 | |
| 675 | |
| 676 ### SQLite database initialization | |
|
590
2294b73df2cf
urllog: Massive refactoring to use similar message system as other scripts,
Matti Hamalainen <ccr@tnsp.org>
parents:
586
diff
changeset
|
677 if {[catch {sqlite3 urllog_db $urllog_db_file} uerrmsg]} { |
| 561 | 678 putlog "Could not open SQLite3 database '${urllog_db_file}': ${uerrmsg}" |
| 560 | 679 exit 2 |
| 680 } | |
| 681 | |
| 682 | |
| 0 | 683 # end of script |