From 1cbe669c0247446fba178c07d3f8daf86e73e5ca Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Thu, 17 Apr 2014 19:25:42 -0400 Subject: [PATCH] More work on db schema; all except FTS indices. --- bitshift/database.py | 12 +++--- schema.sql | 108 ++++++++++++++++++++++++++++++++------------------- 2 files changed, 74 insertions(+), 46 deletions(-) diff --git a/bitshift/database.py b/bitshift/database.py index 07c71c2..b86b05a 100644 --- a/bitshift/database.py +++ b/bitshift/database.py @@ -28,13 +28,15 @@ class Database(object): :return: A list of search results. :rtype: list of :py:class:`.Codelet`\ s """ - # query tree hash + page -> cached? + # search for cache_hash = mmh3.hash(query.serialize() + str(page)) # cache HIT: - # if qcache_created is too old: invalidate cache, goto cache MISS - # update qcache_last_used - # parse qcache_results, fetch codelets + # update cache_last_used + # return codelets # cache MISS: # build complex search query + # fetch codelets + # cache results + # return codelets pass def insert(self, codelet): @@ -44,5 +46,5 @@ class Database(object): :param codelet: The codelet to insert. :type codelet: :py:class:`.Codelet` """ - # code_hash = mmh3.hash64(codelet.code)[0] + # code_hash = mmh3.hash64(codelet.code.encode("utf8"))[0] pass diff --git a/schema.sql b/schema.sql index d49fc6e..21c9c07 100644 --- a/schema.sql +++ b/schema.sql @@ -2,67 +2,93 @@ CREATE DATABASE `bitshift` DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci; USE `bitshift`; CREATE TABLE `languages` ( - `language_id` SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE, + `language_id` SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, `language_name` VARCHAR(64) NOT NULL, PRIMARY KEY (`language_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +) ENGINE=InnoDB; CREATE TABLE `origins` ( - `origin_id` TINYINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE, + `origin_id` TINYINT UNSIGNED NOT NULL AUTO_INCREMENT, `origin_name` VARCHAR(64) NOT NULL, `origin_url` VARCHAR(512) NOT NULL, `origin_url_base` VARCHAR(512) NOT NULL, - `origin_image` TINYBLOB DEFAULT NULL, -- TODO: verify size (<64kB) + `origin_image` BLOB DEFAULT NULL, PRIMARY KEY (`origin_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +) ENGINE=InnoDB; + +CREATE TABLE `code` ( + `code_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `code_hash` BIGINT NOT NULL, + `code_code` MEDIUMTEXT NOT NULL, -- TODO: full-text search index + PRIMARY KEY (`code_id`), + KEY (`code_hash`) +) ENGINE=InnoDB; CREATE TABLE `codelets` ( - `codelet_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE, - `codelet_name` VARCHAR(512) NOT NULL, + `codelet_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `codelet_name` VARCHAR(300) NOT NULL, -- TODO: full-text search index `codelet_code_id` BIGINT UNSIGNED NOT NULL, - `codelet_lang` SMALLINT UNSIGNED DEFAULT NULL, -- TODO: needs index + `codelet_lang` SMALLINT UNSIGNED DEFAULT NULL, `codelet_origin` TINYINT UNSIGNED NOT NULL, `codelet_url` VARCHAR(512) NOT NULL, - `codelet_date_created` DATETIME DEFAULT NULL, -- TODO: needs index - `codelet_date_modified` DATETIME DEFAULT NULL, -- TODO: needs index - PRIMARY KEY (`codelet_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; - -CREATE TABLE `code` ( - `code_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE, - `code_hash` BIGINT NOT NULL, -- TODO: needs index - `code_code` MEDIUMTEXT NOT NULL, -- TODO: verify size (16mB?) - PRIMARY KEY (`code_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + `codelet_date_created` DATETIME DEFAULT NULL, + `codelet_date_modified` DATETIME DEFAULT NULL, + PRIMARY KEY (`codelet_id`), + KEY (`codelet_date_created`), + KEY (`codelet_date_modified`), + FOREIGN KEY (`codelet_code_id`) + REFERENCES `code` (`code_id`) + ON DELETE RESTRICT ON UPDATE CASCADE, + FOREIGN KEY (`codelet_lang`) + REFERENCES `languages` (`language_id`) + ON DELETE RESTRICT ON UPDATE CASCADE, + FOREIGN KEY (`codelet_origin`) + REFERENCES `origins` (`origin_id`) + ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB; CREATE TABLE `authors` ( - `author_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE, - `author_codelet` BIGINT UNSIGNED NOT NULL, -- TODO: foreign index? - `author_name` VARCHAR(128) NOT NULL, -- TODO: needs index + `author_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `author_codelet` BIGINT UNSIGNED NOT NULL, + `author_name` VARCHAR(128) NOT NULL, -- TODO: full-text search index `author_url` VARCHAR(512) DEFAULT NULL, - PRIMARY KEY (`author_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + PRIMARY KEY (`author_id`), + FOREIGN KEY (`author_codelet`) + REFERENCES `codelet` (`codelet_id`) + ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB; CREATE TABLE `symbols` ( - `symbol_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE, - `symbol_codelet` BIGINT UNSIGNED NOT NULL, -- TODO: foreign index? - `symbol_type` TINYINT UNSIGNED NOT NULL, -- TODO: multi-column index? - `symbol_name` VARCHAR(512) NOT NULL, -- TODO: needs index + `symbol_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `symbol_codelet` BIGINT UNSIGNED NOT NULL, + `symbol_type` TINYINT UNSIGNED NOT NULL, + `symbol_name` VARCHAR(512) NOT NULL, `symbol_row` INT UNSIGNED NOT NULL, `symbol_col` INT UNSIGNED NOT NULL, - PRIMARY KEY (`symbol_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + PRIMARY KEY (`symbol_id`), + KEY (`symbol_type`, `symbol_name`(32)), + FOREIGN KEY (`symbol_codelet`) + REFERENCES `codelet` (`codelet_id`) + ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB; -CREATE TABLE `query_cache` ( - `qcache_id` INT NOT NULL UNIQUE, - `qcache_query` VARCHAR(512) NOT NULL, - `qcache_results` BLOB NOT NULL, -- TODO: verify; perhaps use some kind of array - `qcache_page` TINYINT UNSIGNED NOT NULL, - `qcache_count_mnt` TINYINT UNSIGNED NOT NULL, - `qcache_count_exp` TINYINT UNSIGNED NOT NULL, - `qcache_created` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- TODO: verify - `qcache_last_used` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- TODO: verify +CREATE TABLE `cache` ( + `cache_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `cache_hash` BIGINT NOT NULL, + `cache_count_mnt` TINYINT UNSIGNED NOT NULL, + `cache_count_exp` TINYINT UNSIGNED NOT NULL, + `cache_created` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + `cache_last_used` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`cache_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +) ENGINE=InnoDB; --- TODO: full-text search index table +CREATE TABLE `cache_data` ( + `cdata_cache` INT UNSIGNED NOT NULL, + `cdata_codelet` BIGINT UNSIGNED NOT NULL, + FOREIGN KEY (`cdata_cache`) + REFERENCES `cache` (`cache_id`) + ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY (`cdata_codelet`) + REFERENCES `codelet` (`codelet_id`) + ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB;