Sfoglia il codice sorgente

Try something.

tags/v1.0^2
Ben Kurtovic 10 anni fa
parent
commit
91ab08f99c
1 ha cambiato i file con 3 aggiunte e 21 eliminazioni
  1. +3
    -21
      bitshift/crawler/indexer.py

+ 3
- 21
bitshift/crawler/indexer.py Vedi File

@@ -434,38 +434,20 @@ class _GitCloner(threading.Thread):
""" """


GIT_CLONE_TIMEOUT = 500 GIT_CLONE_TIMEOUT = 500

queue_percent_full = (float(self.index_queue.qsize()) / queue_percent_full = (float(self.index_queue.qsize()) /
self.index_queue.maxsize) * 100 self.index_queue.maxsize) * 100


exit_code = None
command = ["perl", "-e", "alarm shift @ARGV; exec @ARGV", command = ["perl", "-e", "alarm shift @ARGV; exec @ARGV",
str(GIT_CLONE_TIMEOUT), "git", "clone", "--single-branch", str(GIT_CLONE_TIMEOUT), "git", "clone", "--single-branch",
repo.url, GIT_CLONE_DIR + "/" + repo.dirname, "||", "pkill",
"-f", "git"]

command_attempt = 0
while exit_code is None:
try:
exit_code = subprocess.call(command)
except Exception: # TODO: subprocess.CalledProcessError instead?
time.sleep(1)
command_attempt += 1
if command_attempt == 20:
break
else:
continue
else:
break

if exit_code != 0:
repo.url, GIT_CLONE_DIR + "/" + repo.dirname]
if subprocess.call(command) != 0:
subprocess.call(["pkill", "-f", "git"]) # This makes Ben K upset
if os.path.isdir("%s/%s" % (GIT_CLONE_DIR, repo.dirname)): if os.path.isdir("%s/%s" % (GIT_CLONE_DIR, repo.dirname)):
shutil.rmtree("%s/%s" % (GIT_CLONE_DIR, repo.dirname)) shutil.rmtree("%s/%s" % (GIT_CLONE_DIR, repo.dirname))
return return


while self.index_queue.full(): while self.index_queue.full():
time.sleep(THREAD_QUEUE_SLEEP) time.sleep(THREAD_QUEUE_SLEEP)

self.index_queue.put(repo) self.index_queue.put(repo)


class _ChangeDir(object): class _ChangeDir(object):


Caricamento…
Annulla
Salva