浏览代码

Update schema to v2; database updates.

tags/v1.0^2
Ben Kurtovic 10 年前
父节点
当前提交
22d6b62547
共有 3 个文件被更改,包括 73 次插入25 次删除
  1. +40
    -22
      bitshift/database/__init__.py
  2. +23
    -0
      bitshift/database/migration.py
  3. +10
    -3
      bitshift/database/schema.sql

+ 40
- 22
bitshift/database/__init__.py 查看文件

@@ -8,15 +8,16 @@ import os
import mmh3
import oursql

# from ..languages import ...
from .migration import VERSION, MIGRATIONS

__all__ = ["Database"]

class Database(object):
"""Represents the MySQL database."""

def __init__(self):
def __init__(self, migrate=False):
self._connect()
self._check_version(migrate)

def _connect(self):
"""Establish a connection to the database."""
@@ -25,6 +26,33 @@ class Database(object):
self._conn = oursql.connect(read_default_file=default_file,
autoping=True, autoreconnect=True)

def _migrate(self, cursor, current):
"""Migrate the database to the latest schema version."""
for version in xrange(current, VERSION):
for query in MIGRATIONS[version - 1]:
cursor.execute(query)

def _check_version(self, migrate):
"""Check the database schema version and respond accordingly.

If the schema is out of date, migrate if *migrate* is True, else raise
an exception.
"""
with self._conn.cursor() as cursor:
cursor.execute("SELECT version FROM version")
version = cursor.fetchone()[0]
if version < VERSION:
if migrate:
self._migrate(cursor, version)
else:
err = "Database schema out of date. " \
"Run `python -m bitshift.database.migration`."
raise RuntimeError(err)

def close(self):
"""Disconnect from the database."""
self._conn.close()

def search(self, query, page=1):
"""
Search the database for a query and return the *n*\ th page of results.
@@ -55,19 +83,14 @@ class Database(object):
:param codelet: The codelet to insert.
:type codelet: :py:class:`.Codelet`
"""
frag_size = 16384 # 16 kB
query_slt1 = """SELECT code_id, LEFT(code_code, {0})
FROM code WHERE code_hash = ?""".format(frag_size)
query_ins1 = "INSERT INTO code VALUES (?, ?)"
query_ins2 = "INSERT INTO codelets VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
query_ins3 = "INSERT INTO authors VALUES", " (?, ?, ?)"
query_ins4 = "INSERT INTO symbols VALUES", " (?, ?, ?, ?, ?)"
query1 = """INSERT INTO code VALUES (?, ?)
ON DUPLICATE KEY UPDATE code_id=code_id"""
query2 = "INSERT INTO codelets VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
query3 = "INSERT INTO authors VALUES", " (?, ?, ?)"
query4 = "INSERT INTO symbols VALUES", " (?, ?, ?, ?, ?)"

# LAST_INSERT_ID()

code_id = None
code_hash = mmh3.hash64(codelet.code.encode("utf8"))[0]

# codelet_id -- auto_increment used here
codelet_name
codelet_code_id
@@ -88,14 +111,9 @@ class Database(object):
codelet.date_created
codelet.date_modified

#######################################################################

code_id = mmh3.hash64(codelet.code.encode("utf8"))[0]

with self._conn.cursor() as cursor:
# Retrieve the ID of the source code if it's already in the DB:
cursor.execute(query_slt1, (code_hash,))
for c_id, c_code_frag in cursor.fetchall():
if c_code_frag == codelet.code[:frag_size]:
code_id = c_id
break

# If the source code isn't already in the DB, add it:
if not code_id:
cursor.execute()
cursor.execute(query1, (code_id, codelet.code))

+ 23
- 0
bitshift/database/migration.py 查看文件

@@ -0,0 +1,23 @@
"""
Contains information about database schema versions, and SQL queries to update
between them.
"""

VERSION = 2

MIGRATIONS = [
# 1 -> 2
[
# drop index on code_hash
"ALTER TABLE code DROP COLUMN code_hash",
# change code_id to BIGINT NOT NULL,
# add key on codelets to codelet_lang
# add symbol_end_row INT UNSIGNED NOT NULL
# add symbol_end_col INT UNSIGNED NOT NULL
]
]

if __name__ == "__main__":
from . import Database

Database(migrate=True).close()

+ 10
- 3
bitshift/database/schema.sql 查看文件

@@ -1,6 +1,12 @@
-- Schema version 2

CREATE DATABASE `bitshift` DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
USE `bitshift`;

CREATE TABLE `version` (
`version` INT UNSIGNED NOT NULL
) ENGINE=InnoDB;

CREATE TABLE `origins` (
`origin_id` TINYINT UNSIGNED NOT NULL AUTO_INCREMENT,
`origin_name` VARCHAR(64) NOT NULL,
@@ -11,11 +17,9 @@ CREATE TABLE `origins` (
) ENGINE=InnoDB;

CREATE TABLE `code` (
`code_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`code_hash` BIGINT NOT NULL,
`code_id` BIGINT NOT NULL,
`code_code` MEDIUMTEXT NOT NULL,
PRIMARY KEY (`code_id`),
KEY (`code_hash`),
FULLTEXT KEY (`code_code`)
) ENGINE=InnoDB;

@@ -31,6 +35,7 @@ CREATE TABLE `codelets` (
`codelet_date_modified` DATETIME DEFAULT NULL,
PRIMARY KEY (`codelet_id`),
FULLTEXT KEY (`codelet_name`),
KEY (`codelet_lang`),
KEY (`codelet_rank`),
KEY (`codelet_date_created`),
KEY (`codelet_date_modified`),
@@ -61,6 +66,8 @@ CREATE TABLE `symbols` (
`symbol_name` VARCHAR(512) NOT NULL,
`symbol_row` INT UNSIGNED NOT NULL,
`symbol_col` INT UNSIGNED NOT NULL,
`symbol_end_row` INT UNSIGNED NOT NULL,
`symbol_end_col` INT UNSIGNED NOT NULL,
PRIMARY KEY (`symbol_id`),
KEY (`symbol_type`, `symbol_name`(32)),
FOREIGN KEY (`symbol_codelet`)


正在加载...
取消
保存