Start working on new language system.

.gitignore

@@ -52,3 +52,4 @@ target
 # Ctags
 */tags
 logs
+Gemfile.lock

bitshift/crawler/crawl.py

@@ -8,6 +8,7 @@ import logging
 import logging.handlers
 import os
 import Queue
+import sys
 import time
 from threading import Event
 
@@ -37,9 +38,9 @@ def crawl():
                crawler.BitbucketCrawler(repo_clone_queue, run_event),
                indexer.GitIndexer(repo_clone_queue, run_event)]
 
+    parse_servers = start_parse_servers()
     for thread in threads:
         thread.start()
-    parse_servers = start_parse_servers()
 
     try:
         while 1:
@@ -77,5 +78,4 @@ def _configure_logging():
     root_logger.setLevel(logging.NOTSET)
 
 if __name__ == "__main__":
-    _configure_logging()
     crawl()

bitshift/crawler/crawler.py

@@ -86,8 +86,8 @@ class GitHubCrawler(threading.Thread):
                     time.sleep(1)
 
                 self.clone_queue.put(indexer.GitRepository(
-                        repo["html_url"], repo["full_name"].replace("/", ""),
-                        "GitHub", repo_ranks[repo["full_name"]]))
+                        repo["html_url"], repo["full_name"], "GitHub",
+                        repo_ranks[repo["full_name"]]))
 
             if int(resp.headers["x-ratelimit-remaining"]) == 0:
                 time.sleep(int(resp.headers["x-ratelimit-reset"]) -

bitshift/languages.json

@@ -1,283 +0,0 @@
-{
-    "_comment" : "A list of programming languages supported by `bitshift`.",
-    "languages" : [
-        "Debian Sourcelist",
-        "Delphi",
-        "JavaScript+Mako",
-        "Brainfuck",
-        "Ceylon",
-        "JavaScript+Django/Jinja",
-        "HTML+Evoque",
-        "NumPy",
-        "Modula-2",
-        "LiveScript",
-        "Nimrod",
-        "Bash",
-        "HTML+Django/Jinja",
-        "CSS+PHP",
-        "XML+Lasso",
-        "VimL",
-        "CSS+Genshi Text",
-        "Fancy",
-        "Coldfusion HTML",
-        "cfstatement",
-        "Scalate Server Page",
-        "Smarty",
-        "XML+Evoque",
-        "haXe",
-        "PowerShell",
-        "Tea",
-        "HTML+Cheetah",
-        "Mason",
-        "Django/Jinja",
-        "JAGS",
-        "ApacheConf",
-        "DTD",
-        "Lighttpd configuration file",
-        "Java",
-        "JavaScript+Genshi Text",
-        "Scheme",
-        "Nemerle",
-        "RHTML",
-        "Ragel in Java Host",
-        "Darcs Patch",
-        "Puppet",
-        "Octave",
-        "CoffeeScript",
-        "Ragel in D Host",
-        "Scilab",
-        "Monkey",
-        "HTML+Myghty",
-        "CSS",
-        "JavaScript+Smarty",
-        "Io",
-        "COBOLFree",
-        "Asymptote",
-        "vhdl",
-        "CSS+Ruby",
-        "Fortran",
-        "d-objdump",
-        "MySQL",
-        "REBOL",
-        "C++",
-        "ERB",
-        "CBM BASIC V2",
-        "Befunge",
-        "Julia",
-        "MoonScript",
-        "Ruby",
-        "XML+Smarty",
-        "Dylan",
-        "Groovy",
-        "MoinMoin/Trac Wiki markup",
-        "autohotkey",
-        "C",
-        "HTML",
-        "Felix",
-        "CMake",
-        "NSIS",
-        "SourcePawn",
-        "Mako",
-        "VGL",
-        "Velocity",
-        "Koka",
-        "CUDA",
-        "Gnuplot",
-        "IRC logs",
-        "Prolog",
-        "Python",
-        "CSS+Django/Jinja",
-        "verilog",
-        "Smalltalk",
-        "JavaScript+Myghty",
-        "YAML",
-        "Julia console",
-        "ANTLR With ActionScript Target",
-        "XML+Mako",
-        "XSLT",
-        "UrbiScript",
-        "Scaml",
-        "S",
-        "DylanLID",
-        "MAQL",
-        "sqlite3con",
-        "Boo",
-        "OCaml",
-        "eC",
-        "ActionScript",
-        "VB.net",
-        "SquidConf",
-        "XQuery",
-        "D",
-        "Fantom",
-        "Gettext Catalog",
-        "Logos",
-        "Lasso",
-        "SCSS",
-        "BBCode",
-        "Haml",
-        "FoxPro",
-        "MuPAD",
-        "XML+Ruby",
-        "Dart",
-        "IDL",
-        "dg",
-        "Evoque",
-        "Jade",
-        "c-objdump",
-        "Kconfig",
-        "Java Server Page",
-        "reg",
-        "ABAP",
-        "XML+Velocity",
-        "JavaScript+Cheetah",
-        "HTML+Mako",
-        "Ragel in Ruby Host",
-        "RobotFramework",
-        "Protocol Buffer",
-        "CFEngine3",
-        "Ragel",
-        "GLSL",
-        "COBOL",
-        "TypeScript",
-        "Ada",
-        "PostgreSQL SQL dialect",
-        "Xtend",
-        "Logtalk",
-        "objdump",
-        "CSS+Mako",
-        "ca65",
-        "Objective-C++",
-        "Gherkin",
-        "HTML+PHP",
-        "Makefile",
-        "PostScript",
-        "Hxml",
-        "Kotlin",
-        "PL/pgSQL",
-        "Vala",
-        "Haskell",
-        "Bro",
-        "Lua",
-        "POVRay",
-        "Sass",
-        "ANTLR With Java Target",
-        "Tcl",
-        "ANTLR With ObjectiveC Target",
-        "JavaScript+Ruby",
-        "Racket",
-        "AspectJ",
-        "Base Makefile",
-        "ANTLR With Python Target",
-        "cpp-objdump",
-        "Genshi Text",
-        "Ioke",
-        "PyPy Log",
-        "Croc",
-        "Objective-J",
-        "GAS",
-        "Batchfile",
-        "Snobol",
-        "XML",
-        "ANTLR",
-        "Opa",
-        "XML+Cheetah",
-        "Go",
-        "Diff",
-        "MiniD",
-        "Cython",
-        "Ragel in C Host",
-        "Erlang",
-        "Debian Control file",
-        "aspx-vb",
-        "BUGS",
-        "Ragel in CPP Host",
-        "aspx-cs",
-        "Properties",
-        "Groff",
-        "Clojure",
-        "Modelica",
-        "QML",
-        "JavaScript+Lasso",
-        "ANTLR With Perl Target",
-        "Genshi",
-        "BlitzMax",
-        "Treetop",
-        "Matlab",
-        "Myghty",
-        "HTML+Genshi",
-        "Duel",
-        "Perl",
-        "FSharp",
-        "reStructuredText",
-        "NewLisp",
-        "Scala",
-        "CSS+Lasso",
-        "XML+PHP",
-        "Stan",
-        "INI",
-        "MOOCode",
-        "Shell Session",
-        "RPMSpec",
-        "Newspeak",
-        "Bash Session",
-        "Coq",
-        "Raw token data",
-        "Tcsh",
-        "HTML+Lasso",
-        "C#",
-        "Gosu Template",
-        "RConsole",
-        "MXML",
-        "TeX",
-        "CSS+Smarty",
-        "Text only",
-        "ANTLR With C# Target",
-        "OpenEdge ABL",
-        "Cheetah",
-        "Smali",
-        "CSS+Myghty",
-        "Rd",
-        "LLVM",
-        "Standard ML",
-        "Elixir",
-        "Nginx configuration file",
-        "GoodData-CL",
-        "AppleScript",
-        "HTML+Smarty",
-        "Objective-C",
-        "JavaScript",
-        "Rust",
-        "Common Lisp",
-        "Embedded Ragel",
-        "ActionScript 3",
-        "systemverilog",
-        "Literate Haskell",
-        "PHP",
-        "ANTLR With CPP Target",
-        "Gosu",
-        "Hybris",
-        "JavaScript+PHP",
-        "Factor",
-        "HTML+Velocity",
-        "Mscgen",
-        "Ooc",
-        "SQL",
-        "HTTP",
-        "ECL",
-        "Redcode",
-        "Ragel in Objective C Host",
-        "XML+Django/Jinja",
-        "Awk",
-        "JSON",
-        "NASM",
-        "ANTLR With Ruby Target",
-        "XML+Myghty",
-        "AutoIt",
-        "Mako",
-        "CSS+Mako",
-        "HTML+Mako",
-        "XML+Mako",
-        "JavaScript+Mako"
-    ]
-}

bitshift/languages.py

@@ -1,5 +1,6 @@
-import json
 from os import path
 
-with open(path.join(path.dirname(__file__), "languages.json")) as lang_json:
-    LANGS = [lang for lang in json.load(lang_json)["languages"]]
+import yaml
+
+with open(path.join(path.dirname(__file__), "languages.yml")) as lang_yaml:
+    LANGS = [lang for lang in yaml.load(lang_yaml)["languages"]]

bitshift/languages.yml

@@ -0,0 +1,129 @@
+# A list of programming languages supported by bitshift:
+
+languages:
+    # With parsers:
+    - Python:
+        - Python
+        - Python 3
+        - Python 3.0 Traceback
+        - Python console session
+        - Python Traceback
+    - C
+    - Java
+    - Ruby
+
+    # Without parsers:
+    - ABAP
+    - APL
+    - ActionScript:
+        - ActionScript
+        - ActionScript 3
+    - ANTLR:
+        - ANTLR
+        - ANTLR With ActionScript Target
+        - ANTLR With CPP Target
+        - "ANTLR With C# Target"
+        - ANTLR With Java Target
+        - ANTLR With ObjectiveC Target
+        - ANTLR With Perl Target
+        - ANTLR With Python Target
+        - ANTLR With Ruby Target
+    - Ada
+    - Agda
+    - Alloy
+    - AmbientTalk
+    - ApacheConf
+    - AppleScript
+    - AspectJ
+    - Asymptote
+    - AutoIt
+    - Awk
+    - BBCode
+    - BUGS
+    - Bash:
+        - Bash
+        - Bash Session
+    - Batchfile
+    - Befunge
+    - BlitzBasic:
+        - BlitzBasic
+        - BlitzMax
+    - Boo
+    - Brainfuck
+    - Bro
+    - "C#"
+    - C++
+
+    - CSS:
+        - CSS
+        - CSS+Django/Jinja
+        - CSS+Genshi Text
+        - CSS+Lasso
+        - CSS+Mako
+        - CSS+Mako
+        - CSS+Myghty
+        - CSS+PHP
+        - CSS+Ruby
+        - CSS+Smarty
+    - Haskell:
+        - Haskell
+        - Literate Haskell
+    - HTML:
+        - HTML
+        - HTML+Cheetah
+        - HTML+Django/Jinja
+        - HTML+Evoque
+        - HTML+Genshi
+        - HTML+Lasso
+        - HTML+Mako
+        - HTML+Mako
+        - HTML+Myghty
+        - HTML+PHP
+        - HTML+Smarty
+        - HTML+Velocity
+    - JavaScript:
+        - JavaScript
+        - JavaScript+Cheetah
+        - JavaScript+Django/Jinja
+        - JavaScript+Genshi Text
+        - JavaScript+Lasso
+        - JavaScript+Mak
+        - JavaScript+Mako
+        - JavaScript+Myghty
+        - JavaScript+PHP
+        - JavaScript+Ruby
+        - JavaScript+Smarty
+    - Julia:
+        - Julia
+        - Julia console
+    - Makefile
+        - Makefile
+        - Base Makefile
+    - Objective-C
+    - Objective-C++
+    - Objective-J
+    - Ragel:
+        - Ragel
+        - Ragel in C Host
+        - Ragel in CPP Host
+        - Ragel in D Host
+        - Ragel in Java Host
+        - Ragel in Objective C Host
+        - Ragel in Ruby Host
+    - Sass:
+        - Sass
+        - SCSS
+    - XML:
+        - XML
+        - XML+Cheetah
+        - XML+Django/Jinja
+        - XML+Evoque
+        - XML+Lasso
+        - XML+Mako
+        - XML+Mako
+        - XML+Myghty
+        - XML+PHP
+        - XML+Ruby
+        - XML+Smarty
+        - XML+Velocity
+    - YAML

setup.py

@@ -7,7 +7,7 @@ setup(
     install_requires = [
         "Flask>=0.10.1", "gunicorn>=18.0", "pygments>=1.6", "requests>=2.2.0",
         "beautifulsoup4>=3.2.1", "oursql>=0.9.3.1", "mmh3>=2.3",
-        "python-dateutil>=2.2"],
+        "PyYAML>=3.11", "python-dateutil>=2.2"],
     author = "Benjamin Attal, Ben Kurtovic, Severyn Kozak",
     license = "MIT",
     url = "https://github.com/earwig/bitshift"