Pārlūkot izejas kodu

Add parse() and insert() calls to crawler.

Add:
    bitshift/crawler/indexer.py
        -Add `parse()` and `insert()` calls to `_insert_repository_codelets()`.
tags/v1.0^2
Severyn Kozak pirms 12 gadiem
vecāks
revīzija
b698a16c98
1 mainītis faili ar 4 papildinājumiem un 0 dzēšanām
  1. +4
    -0
      bitshift/crawler/indexer.py

+ 4
- 0
bitshift/crawler/indexer.py Parādīt failu

@@ -7,6 +7,7 @@ import bs4, datetime, logging, os, Queue, re, shutil, string, subprocess, time,\
threading threading


from ..database import Database from ..database import Database
from ..parser import parse
from ..codelet import Codelet from ..codelet import Codelet


GIT_CLONE_DIR = "/tmp/bitshift" GIT_CLONE_DIR = "/tmp/bitshift"
@@ -73,6 +74,7 @@ class GitIndexer(threading.Thread):
self.index_queue = Queue.Queue(maxsize=MAX_INDEX_QUEUE_SIZE) self.index_queue = Queue.Queue(maxsize=MAX_INDEX_QUEUE_SIZE)
self.git_cloner = _GitCloner(clone_queue, self.index_queue) self.git_cloner = _GitCloner(clone_queue, self.index_queue)
self.git_cloner.start() self.git_cloner.start()
self.database = Database()
self._logger = logging.getLogger("%s.%s" % self._logger = logging.getLogger("%s.%s" %
(__name__, self.__class__.__name__)) (__name__, self.__class__.__name__))
self._logger.info("Starting.") self._logger.info("Starting.")
@@ -158,6 +160,8 @@ class GitIndexer(threading.Thread):
commits_meta[filename]["time_created"], commits_meta[filename]["time_created"],
commits_meta[filename]["time_last_modified"], commits_meta[filename]["time_last_modified"],
repo.rank) repo.rank)
parse(codelet)
self.database.insert(codelet)


def _generate_file_url(self, filename, repo_url, framework_name): def _generate_file_url(self, filename, repo_url, framework_name):
""" """


Notiek ielāde…
Atcelt
Saglabāt