|
@@ -25,8 +25,9 @@ from gzip import GzipFile |
|
|
from logging import getLogger |
|
|
from logging import getLogger |
|
|
from math import log |
|
|
from math import log |
|
|
from Queue import Empty, Queue |
|
|
from Queue import Empty, Queue |
|
|
from socket import error |
|
|
|
|
|
|
|
|
from socket import error as socket_error |
|
|
from StringIO import StringIO |
|
|
from StringIO import StringIO |
|
|
|
|
|
from struct import error as struct_error |
|
|
from threading import Lock, Thread |
|
|
from threading import Lock, Thread |
|
|
from time import time |
|
|
from time import time |
|
|
from urllib2 import build_opener, URLError |
|
|
from urllib2 import build_opener, URLError |
|
@@ -125,7 +126,7 @@ class _CopyvioWorker(object): |
|
|
url = source.url.encode("utf8") |
|
|
url = source.url.encode("utf8") |
|
|
try: |
|
|
try: |
|
|
response = self._opener.open(url, timeout=source.timeout) |
|
|
response = self._opener.open(url, timeout=source.timeout) |
|
|
except (URLError, error): |
|
|
|
|
|
|
|
|
except (URLError, socket_error): |
|
|
return None |
|
|
return None |
|
|
|
|
|
|
|
|
try: |
|
|
try: |
|
@@ -146,7 +147,7 @@ class _CopyvioWorker(object): |
|
|
|
|
|
|
|
|
try: |
|
|
try: |
|
|
content = response.read() |
|
|
content = response.read() |
|
|
except (URLError, error): |
|
|
|
|
|
|
|
|
except (URLError, socket_error): |
|
|
return None |
|
|
return None |
|
|
|
|
|
|
|
|
if response.headers.get("Content-Encoding") == "gzip": |
|
|
if response.headers.get("Content-Encoding") == "gzip": |
|
@@ -154,7 +155,7 @@ class _CopyvioWorker(object): |
|
|
gzipper = GzipFile(fileobj=stream) |
|
|
gzipper = GzipFile(fileobj=stream) |
|
|
try: |
|
|
try: |
|
|
content = gzipper.read(2 * 1024 ** 2) |
|
|
content = gzipper.read(2 * 1024 ** 2) |
|
|
except IOError: |
|
|
|
|
|
|
|
|
except (IOError, struct_error): |
|
|
return None |
|
|
return None |
|
|
|
|
|
|
|
|
return handler(content) |
|
|
return handler(content) |
|
|