changeset 10:fbf718c24de4

Add script for converting URLLog text database to SQLite.
author Matti Hamalainen <ccr@tnsp.org>
date Mon, 05 Sep 2011 14:10:07 +0300
parents 47c73165754b
children 06f5e5b25930
files convert_urllog_db.tcl
diffstat 1 files changed, 54 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/convert_urllog_db.tcl	Mon Sep 05 14:10:07 2011 +0300
@@ -0,0 +1,54 @@
+#!/usr/bin/tclsh
+package require sqlite3
+
+set urllog_orig_file "data.urllog"
+set urllog_db_file "urllog.sqlite"
+
+
+proc escape { str } {
+	return [string map {' ''} $str]
+}
+
+proc lescape { lst pos } {
+	return [escape [lindex $lst $pos]]
+}
+
+
+if {[catch {set fd [open $urllog_orig_file r]} uerrmsg]} {
+	puts "Could not open '$urllog_orig_file' for reading: $uerrmsg"
+	exit 1
+}
+
+if {[catch {sqlite3 urldb $urllog_db_file} uerrmsg]} {
+	puts "Could not open SQLite3 database '$urllog_db_file': $uerrmsg"
+	exit 2
+}
+
+urldb eval "CREATE TABLE urls(id INT, utime INT, url VARCHAR(2048), user VARCHAR(32), host VARCHAR(256))"
+
+puts -nonewline "Converting database, please wait ..."
+set nline 0
+while {![eof $fd]} {
+	incr nline
+	set line [gets $fd]
+	set items [split $line " "]
+	if {$line != ""} {
+		set host [lindex $items 3]
+		if {[regexp {^\((.+)\)$} $host ures uhost]} {
+			set host $uhost
+		}
+		set sql "INSERT INTO urls VALUES([lindex $items 4], [lindex $items 1], '[lescape $items 0]', '[lescape $items 2]', '[escape $host]')"
+		if {[catch {urldb eval $sql} uerrmsg]} {
+			puts "\nError: $uerrmsg on:\n$sql"
+		}
+	}
+	if {[expr $nline % 10] == 1} {
+		puts -nonewline "."
+		flush stdout
+	}
+}
+urldb close
+close $fd
+
+puts "OK"
+puts "New SQLite3 database is in file '$urllog_db_file'"
\ No newline at end of file