More work on db schema; all except FTS indices.

bitshift/database.py

@@ -28,13 +28,15 @@ class Database(object):
         :return: A list of search results.
         :rtype: list of :py:class:`.Codelet`\ s
         """
-        # query tree hash + page -> cached?
+        # search for cache_hash = mmh3.hash(query.serialize() + str(page))
         #   cache HIT:
-        #       if qcache_created is too old: invalidate cache, goto cache MISS
-        #       update qcache_last_used
-        #       parse qcache_results, fetch codelets
+        #       update cache_last_used
+        #       return codelets
         #   cache MISS:
         #       build complex search query
+        #       fetch codelets
+        #       cache results
+        #       return codelets
         pass
 
     def insert(self, codelet):
@@ -44,5 +46,5 @@ class Database(object):
         :param codelet: The codelet to insert.
         :type codelet: :py:class:`.Codelet`
         """
-        # code_hash = mmh3.hash64(codelet.code)[0]
+        # code_hash = mmh3.hash64(codelet.code.encode("utf8"))[0]
         pass

schema.sql

@@ -2,67 +2,93 @@ CREATE DATABASE `bitshift` DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 USE `bitshift`;
 
 CREATE TABLE `languages` (
-    `language_id` SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
+    `language_id` SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT,
     `language_name` VARCHAR(64) NOT NULL,
     PRIMARY KEY (`language_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+) ENGINE=InnoDB;
 
 CREATE TABLE `origins` (
-    `origin_id` TINYINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
+    `origin_id` TINYINT UNSIGNED NOT NULL AUTO_INCREMENT,
     `origin_name` VARCHAR(64) NOT NULL,
     `origin_url` VARCHAR(512) NOT NULL,
     `origin_url_base` VARCHAR(512) NOT NULL,
-    `origin_image` TINYBLOB DEFAULT NULL, -- TODO: verify size (<64kB)
+    `origin_image` BLOB DEFAULT NULL,
     PRIMARY KEY (`origin_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+) ENGINE=InnoDB;
+
+CREATE TABLE `code` (
+    `code_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+    `code_hash` BIGINT NOT NULL,
+    `code_code` MEDIUMTEXT NOT NULL, -- TODO: full-text search index
+    PRIMARY KEY (`code_id`),
+    KEY (`code_hash`)
+) ENGINE=InnoDB;
 
 CREATE TABLE `codelets` (
-    `codelet_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
-    `codelet_name` VARCHAR(512) NOT NULL,
+    `codelet_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+    `codelet_name` VARCHAR(300) NOT NULL, -- TODO: full-text search index
     `codelet_code_id` BIGINT UNSIGNED NOT NULL,
-    `codelet_lang` SMALLINT UNSIGNED DEFAULT NULL, -- TODO: needs index
+    `codelet_lang` SMALLINT UNSIGNED DEFAULT NULL,
     `codelet_origin` TINYINT UNSIGNED NOT NULL,
     `codelet_url` VARCHAR(512) NOT NULL,
-    `codelet_date_created` DATETIME DEFAULT NULL, -- TODO: needs index
-    `codelet_date_modified` DATETIME DEFAULT NULL, -- TODO: needs index
-    PRIMARY KEY (`codelet_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
-
-CREATE TABLE `code` (
-    `code_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
-    `code_hash` BIGINT NOT NULL, -- TODO: needs index
-    `code_code` MEDIUMTEXT NOT NULL, -- TODO: verify size (16mB?)
-    PRIMARY KEY (`code_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+    `codelet_date_created` DATETIME DEFAULT NULL,
+    `codelet_date_modified` DATETIME DEFAULT NULL,
+    PRIMARY KEY (`codelet_id`),
+    KEY (`codelet_date_created`),
+    KEY (`codelet_date_modified`),
+    FOREIGN KEY (`codelet_code_id`)
+        REFERENCES `code` (`code_id`)
+        ON DELETE RESTRICT ON UPDATE CASCADE,
+    FOREIGN KEY (`codelet_lang`)
+        REFERENCES `languages` (`language_id`)
+        ON DELETE RESTRICT ON UPDATE CASCADE,
+    FOREIGN KEY (`codelet_origin`)
+        REFERENCES `origins` (`origin_id`)
+        ON DELETE RESTRICT ON UPDATE CASCADE
+) ENGINE=InnoDB;
 
 CREATE TABLE `authors` (
-    `author_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
-    `author_codelet` BIGINT UNSIGNED NOT NULL, -- TODO: foreign index?
-    `author_name` VARCHAR(128) NOT NULL, -- TODO: needs index
+    `author_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+    `author_codelet` BIGINT UNSIGNED NOT NULL,
+    `author_name` VARCHAR(128) NOT NULL, -- TODO: full-text search index
     `author_url` VARCHAR(512) DEFAULT NULL,
-    PRIMARY KEY (`author_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+    PRIMARY KEY (`author_id`),
+    FOREIGN KEY (`author_codelet`)
+        REFERENCES `codelet` (`codelet_id`)
+        ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB;
 
 CREATE TABLE `symbols` (
-    `symbol_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
-    `symbol_codelet` BIGINT UNSIGNED NOT NULL, -- TODO: foreign index?
-    `symbol_type` TINYINT UNSIGNED NOT NULL, -- TODO: multi-column index?
-    `symbol_name` VARCHAR(512) NOT NULL, -- TODO: needs index
+    `symbol_id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+    `symbol_codelet` BIGINT UNSIGNED NOT NULL,
+    `symbol_type` TINYINT UNSIGNED NOT NULL,
+    `symbol_name` VARCHAR(512) NOT NULL,
     `symbol_row` INT UNSIGNED NOT NULL,
     `symbol_col` INT UNSIGNED NOT NULL,
-    PRIMARY KEY (`symbol_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+    PRIMARY KEY (`symbol_id`),
+    KEY (`symbol_type`, `symbol_name`(32)),
+    FOREIGN KEY (`symbol_codelet`)
+        REFERENCES `codelet` (`codelet_id`)
+        ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB;
 
-CREATE TABLE `query_cache` (
-    `qcache_id` INT NOT NULL UNIQUE,
-    `qcache_query` VARCHAR(512) NOT NULL,
-    `qcache_results` BLOB NOT NULL, -- TODO: verify; perhaps use some kind of array
-    `qcache_page` TINYINT UNSIGNED NOT NULL,
-    `qcache_count_mnt` TINYINT UNSIGNED NOT NULL,
-    `qcache_count_exp` TINYINT UNSIGNED NOT NULL,
-    `qcache_created` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- TODO: verify
-    `qcache_last_used` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- TODO: verify
+CREATE TABLE `cache` (
+    `cache_id` INT UNSIGNED NOT NULL AUTO_INCREMENT,
+    `cache_hash` BIGINT NOT NULL,
+    `cache_count_mnt` TINYINT UNSIGNED NOT NULL,
+    `cache_count_exp` TINYINT UNSIGNED NOT NULL,
+    `cache_created` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `cache_last_used` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     PRIMARY KEY (`cache_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+) ENGINE=InnoDB;
 
+CREATE TABLE `cache_data` (
+    `cdata_cache` INT UNSIGNED NOT NULL,
+    `cdata_codelet` BIGINT UNSIGNED NOT NULL,
+    FOREIGN KEY (`cdata_cache`)
+        REFERENCES `cache` (`cache_id`)
+        ON DELETE CASCADE ON UPDATE CASCADE,
+    FOREIGN KEY (`cdata_codelet`)
+        REFERENCES `codelet` (`codelet_id`)
+        ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB;