| @@ -45,13 +45,15 @@ _WorkingResult = namedtuple("_WorkingResult", ["url", "confidence", "chains"]) | |||||
| class _CopyvioWorkspace(object): | class _CopyvioWorkspace(object): | ||||
| """Manages a single copyvio check distributed across threads.""" | """Manages a single copyvio check distributed across threads.""" | ||||
| def __init__(self, article, min_confidence, until, headers, url_timeout=5): | |||||
| def __init__(self, article, min_confidence, until, logger, headers, | |||||
| url_timeout=5): | |||||
| self.best = _WorkingResult(None, 0.0, (EMPTY, EMPTY_INTERSECTION)) | self.best = _WorkingResult(None, 0.0, (EMPTY, EMPTY_INTERSECTION)) | ||||
| self.until = until | self.until = until | ||||
| self._article = article | self._article = article | ||||
| self._handled_urls = [] | self._handled_urls = [] | ||||
| self._headers = headers | self._headers = headers | ||||
| self._logger = logger.getChild("copyvios") | |||||
| self._min_confidence = min_confidence | self._min_confidence = min_confidence | ||||
| self._queue = Queue() | self._queue = Queue() | ||||
| self._result_lock = Lock() | self._result_lock = Lock() | ||||
| @@ -115,6 +117,7 @@ class _CopyvioWorkspace(object): | |||||
| """Compare a source to the article, and update the working result.""" | """Compare a source to the article, and update the working result.""" | ||||
| delta = MarkovChainIntersection(self._article, source) | delta = MarkovChainIntersection(self._article, source) | ||||
| confidence = self._calculate_confidence(delta) | confidence = self._calculate_confidence(delta) | ||||
| self._logger.debug("{0} -> {1}".format(url, confidence)) | |||||
| with self._result_lock: | with self._result_lock: | ||||
| if confidence > self.best.confidence: | if confidence > self.best.confidence: | ||||
| self.best = _WorkingResult(url, confidence, (source, delta)) | self.best = _WorkingResult(url, confidence, (source, delta)) | ||||
| @@ -266,7 +269,7 @@ class CopyvioMixIn(object): | |||||
| parser = ArticleTextParser(self.get()) | parser = ArticleTextParser(self.get()) | ||||
| article = MarkovChain(parser.strip()) | article = MarkovChain(parser.strip()) | ||||
| workspace = _CopyvioWorkspace(article, min_confidence, | workspace = _CopyvioWorkspace(article, min_confidence, | ||||
| until, self._addheaders) | |||||
| until, self._logger, self._addheaders) | |||||
| if self._exclusions_db: | if self._exclusions_db: | ||||
| self._exclusions_db.sync(self.site.name) | self._exclusions_db.sync(self.site.name) | ||||
| exclude = lambda u: self._exclusions_db.check(self.site.name, u) | exclude = lambda u: self._exclusions_db.check(self.site.name, u) | ||||
| @@ -276,7 +279,7 @@ class CopyvioMixIn(object): | |||||
| if article.size() < 20: # Auto-fail very small articles | if article.size() < 20: # Auto-fail very small articles | ||||
| result = CopyvioCheckResult(False, 0.0, None, 0, 0, article, | result = CopyvioCheckResult(False, 0.0, None, 0, 0, article, | ||||
| workspace.best.chains) | workspace.best.chains) | ||||
| self._logger.debug(result.get_log_message(self.title)) | |||||
| self._logger.info(result.get_log_message(self.title)) | |||||
| return result | return result | ||||
| workspace.spawn(worker_threads) | workspace.spawn(worker_threads) | ||||
| @@ -296,7 +299,7 @@ class CopyvioMixIn(object): | |||||
| workspace.best.confidence >= min_confidence, | workspace.best.confidence >= min_confidence, | ||||
| workspace.best.confidence, workspace.best.url, len(chunks), | workspace.best.confidence, workspace.best.url, len(chunks), | ||||
| time() - start_time, article, workspace.best.chains) | time() - start_time, article, workspace.best.chains) | ||||
| self._logger.debug(result.get_log_message(self.title)) | |||||
| self._logger.info(result.get_log_message(self.title)) | |||||
| return result | return result | ||||
| def copyvio_compare(self, url, min_confidence=0.5, max_time=30): | def copyvio_compare(self, url, min_confidence=0.5, max_time=30): | ||||
| @@ -324,12 +327,12 @@ class CopyvioMixIn(object): | |||||
| until = (start_time + max_time) if max_time > 0 else None | until = (start_time + max_time) if max_time > 0 else None | ||||
| article = MarkovChain(ArticleTextParser(self.get()).strip()) | article = MarkovChain(ArticleTextParser(self.get()).strip()) | ||||
| workspace = _CopyvioWorkspace(article, min_confidence, | workspace = _CopyvioWorkspace(article, min_confidence, | ||||
| until, self._addheaders) | |||||
| until, self._logger, self._addheaders) | |||||
| workspace.enqueue([url]) | workspace.enqueue([url]) | ||||
| workspace.spawn(1) | workspace.spawn(1) | ||||
| workspace.wait() | workspace.wait() | ||||
| url, conf, chains = workspace.best | url, conf, chains = workspace.best | ||||
| result = CopyvioCheckResult(conf >= min_confidence, conf, url, 0, | result = CopyvioCheckResult(conf >= min_confidence, conf, url, 0, | ||||
| time() - start_time, article, chains) | time() - start_time, article, chains) | ||||
| self._logger.debug(result.get_log_message(self.title)) | |||||
| self._logger.info(result.get_log_message(self.title)) | |||||
| return result | return result | ||||