asdf

lib/db.py

@@ -3,7 +3,7 @@ import mmh3
 import sys
 #import requests
 import httplib2
-DEBUG = False
+DEBUG = True
 
 def log(*s):
     if DEBUG:
@@ -67,13 +67,13 @@ def createnwview(file):
         createFilterTable(file)
 
 def createFilterTable(file):
-    with sqlite3.connect(file) as connection:
+    with sqlite3.connect(file,timeout=10) as connection:
         cmd_create_filter_table = f"""CREATE TABLE IF NOT EXISTS filters(cmd TEXT);"""
         cursor = connection.cursor()
         cursor.execute(cmd_create_filter_table)
         print("db connection:",connection.total_changes)
 def addFineFilter(file,table,filterstr):
-    with sqlite3.connect(file) as connection:
+    with sqlite3.connect(file,timeout=10) as connection:
         cmd_createFineFilter = f"""INSERT INTO {table}(cmd) VALUES(?);"""
         cmd_checkIfExists = f"""SELECT * FROM {table} WHERE cmd = ?"""
         cursor = connection.cursor()
@@ -81,7 +81,8 @@ def addFineFilter(file,table,filterstr):
             cursor.execute(cmd_createFineFilter,(filterstr,))
 
 def writedb(jobs):
-        with sqlite3.connect("../db/sqlite3.db") as connection:
+        with sqlite3.connect("../db/sqlite3.db",timeout=10) as connection:
+            connection.execute("pragma journal_mode=wal")
             print("db connection", connection.total_changes)
             cursor = connection.cursor()
         #    cursor.execute("CREATE TABLE jobs (title TEXT, location TEXT, company TEXT,link TEXT,hash INT)")
@@ -94,16 +95,19 @@ def writedb(jobs):
                     print("NEW_ENTRY")
                     cursor.execute("INSERT INTO jobs (star,tag,title,company,location,link,pubdate,hash) VALUES (?,?,?,?,?,?,?,?)",(job.starred,job.tag,job.title,job.company,job.location,job.link,job.date,hash1))
 
-def isStillValid(file):
+def isStillValid(file,skiprows):
     rows = [0,0,0]
-    with sqlite3.connect(file) as connection:
+    with sqlite3.connect(file,timeout=10) as connection:
         cmd_read_chunk = f"""SELECT link from jobs;"""
+        connection.execute("pragma journal_mode=wal")
         cursor = connection.cursor()
         cursor.execute(cmd_read_chunk)
         
+        #cursor.fetchmany(skiprows)#drop rows
         while(len(rows)!=0):
             isLink = True
             rows = []
+
             rows = cursor.fetchmany(256)
             h = httplib2.Http()
             for row in rows:
@@ -135,3 +139,5 @@ def isStillValid(file):
                                 rm_itm = rm_cursor.execute(f"""DELETE from jobs WHERE link = ?;""",(row[0],))
                                 print ("Deletion resultet in: ", rm_itm)
             print("result of commit: ", connection.commit())
+            return 0
+