Browse Source

More additions for sqlite support

master
Ben Kurtovic 5 years ago
parent
commit
8321a07ba9
3 changed files with 41 additions and 6 deletions
  1. +11
    -5
      copyvios/checker.py
  2. +10
    -1
      copyvios/misc.py
  3. +20
    -0
      schema.sql

+ 11
- 5
copyvios/checker.py View File

@@ -9,7 +9,7 @@ from earwigbot.wiki.copyvios.markov import EMPTY, MarkovChain
from earwigbot.wiki.copyvios.parsers import ArticleTextParser
from earwigbot.wiki.copyvios.result import CopyvioSource, CopyvioCheckResult

from .misc import Query, get_db, get_cursor
from .misc import Query, get_db, get_cursor, sql_dialect
from .sites import get_site
from .turnitin import search_turnitin

@@ -129,8 +129,9 @@ def _get_page_by_revid(site, revid):
return page

def _get_cached_results(page, conn, mode, noskip):
query1 = """DELETE FROM cache
WHERE cache_time < DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 3 DAY)"""
expiry = sql_dialect(mysql="DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 3 DAY)",
sqlite="STRFTIME('%s', 'now', '-3 days')")
query1 = "DELETE FROM cache WHERE cache_time < %s" % expiry
query2 = """SELECT cache_time, cache_queries, cache_process_time,
cache_possible_miss
FROM cache
@@ -149,6 +150,8 @@ def _get_cached_results(page, conn, mode, noskip):
cache_time, queries, check_time, possible_miss = results[0]
if possible_miss and noskip:
return None
if not isinstance(cache_time, datetime):
cache_time = datetime.utcfromtimestamp(cache_time)
cursor.execute(query3, (cache_id,))
data = cursor.fetchall()

@@ -196,8 +199,11 @@ def _format_date(cache_time):

def _cache_result(page, result, conn, mode):
query1 = "DELETE FROM cache WHERE cache_id = ?"
query2 = "INSERT INTO cache VALUES (?, DEFAULT, ?, ?, ?)"
query3 = "INSERT INTO cache_data VALUES (DEFAULT, ?, ?, ?, ?, ?)"
query2 = """INSERT INTO cache (cache_id, cache_queries, cache_process_time,
cache_possible_miss) VALUES (?, ?, ?, ?)"""
query3 = """INSERT INTO cache_data (cdata_cache_id, cdata_url,
cdata_confidence, cdata_skipped,
cdata_excluded) VALUES (?, ?, ?, ?, ?)"""
cache_id = buffer(sha256(mode + page.get().encode("utf8")).digest())
data = [(cache_id, source.url[:1024], source.confidence, source.skipped,
source.excluded)


+ 10
- 1
copyvios/misc.py View File

@@ -51,7 +51,9 @@ def _connect_to_db(engine, args):
if engine == "sqlite":
import apsw
dbpath = join(cache.bot.config.root_dir, "copyvios.db")
return apsw.Connection(dbpath)
conn = apsw.Connection(dbpath)
conn.cursor().execute("PRAGMA foreign_keys = ON")
return conn
raise ValueError("Unknown engine: %s" % engine)

def get_db():
@@ -72,6 +74,13 @@ def get_cursor(conn):
else:
raise ValueError("Unknown engine: %s" % g._engine)

def sql_dialect(mysql, sqlite):
if g._engine == "mysql":
return mysql
if g._engine == "sqlite":
return sqlite
raise ValueError("Unknown engine: %s" % g._engine)

def get_notice():
try:
with open(expanduser("~/copyvios_notice.html")) as fp:


+ 20
- 0
schema.sql View File

@@ -0,0 +1,20 @@
CREATE TABLE cache (
cache_id BLOB NOT NULL,
cache_time INTEGER NOT NULL DEFAULT (STRFTIME('%s', 'now')),
cache_queries INTEGER NOT NULL DEFAULT 0,
cache_process_time REAL NOT NULL DEFAULT 0,
cache_possible_miss INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (cache_id)
);
CREATE INDEX cache_time_idx ON cache (cache_time);
CREATE TABLE cache_data (
cdata_id ROWID,
cdata_cache_id BLOB NOT NULL,
cdata_url TEXT NOT NULL,
cdata_confidence REAL NOT NULL DEFAULT 0,
cdata_skipped INTEGER NOT NULL DEFAULT 0,
cdata_excluded INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (cdata_id),
FOREIGN KEY (cdata_cache_id) REFERENCES cache (cache_id)
ON DELETE CASCADE ON UPDATE CASCADE
);

Loading…
Cancel
Save