Ver código fonte

Handle empty queues correctly. Remove some log messages.

tags/v0.2
Ben Kurtovic 9 anos atrás
pai
commit
dbb1ae5483
1 arquivos alterados com 2 adições e 3 exclusões
  1. +2
    -3
      earwigbot/wiki/copyvios/workers.py

+ 2
- 3
earwigbot/wiki/copyvios/workers.py Ver arquivo

@@ -216,7 +216,7 @@ class _CopyvioWorker(object):


def _dequeue(self): def _dequeue(self):
"""Remove a source from one of the queues.""" """Remove a source from one of the queues."""
if not self._queue:
if not self._site:
self._acquire_new_site() self._acquire_new_site()


logmsg = u"Fetching source URL from queue {0}" logmsg = u"Fetching source URL from queue {0}"
@@ -227,6 +227,7 @@ class _CopyvioWorker(object):
except IndexError: except IndexError:
self._logger.debug("Queue is empty") self._logger.debug("Queue is empty")
del self._queues.sites[self._site] del self._queues.sites[self._site]
self._site = None
self._queue = None self._queue = None
self._queues.lock.release() self._queues.lock.release()
return self._dequeue() return self._dequeue()
@@ -369,12 +370,10 @@ class CopyvioWorkspace(object):
"""Wait for the workers to finish handling the sources.""" """Wait for the workers to finish handling the sources."""
self._logger.debug("Waiting on {0} sources".format(len(self.sources))) self._logger.debug("Waiting on {0} sources".format(len(self.sources)))
for source in self.sources: for source in self.sources:
self._logger.debug("Waiting on source: {0}".format(source.url))
source.join(self._until) source.join(self._until)
if not _is_globalized: if not _is_globalized:
for i in xrange(self._num_workers): for i in xrange(self._num_workers):
self._queues.unassigned.put((StopIteration, None)) self._queues.unassigned.put((StopIteration, None))
self._logger.debug("Done waiting")


def compare(self, source, source_chain): def compare(self, source, source_chain):
"""Compare a source to the article, and update the best known one.""" """Compare a source to the article, and update the best known one."""


Carregando…
Cancelar
Salvar