Browse Source

Handle struct.error from GzipFile.read() (Python bug?)

tags/v0.2
Ben Kurtovic 9 years ago
parent
commit
f10908e34e
1 changed files with 5 additions and 4 deletions
  1. +5
    -4
      earwigbot/wiki/copyvios/workers.py

+ 5
- 4
earwigbot/wiki/copyvios/workers.py View File

@@ -25,8 +25,9 @@ from gzip import GzipFile
from logging import getLogger from logging import getLogger
from math import log from math import log
from Queue import Empty, Queue from Queue import Empty, Queue
from socket import error
from socket import error as socket_error
from StringIO import StringIO from StringIO import StringIO
from struct import error as struct_error
from threading import Lock, Thread from threading import Lock, Thread
from time import time from time import time
from urllib2 import build_opener, URLError from urllib2 import build_opener, URLError
@@ -125,7 +126,7 @@ class _CopyvioWorker(object):
url = source.url.encode("utf8") url = source.url.encode("utf8")
try: try:
response = self._opener.open(url, timeout=source.timeout) response = self._opener.open(url, timeout=source.timeout)
except (URLError, error):
except (URLError, socket_error):
return None return None


try: try:
@@ -146,7 +147,7 @@ class _CopyvioWorker(object):


try: try:
content = response.read() content = response.read()
except (URLError, error):
except (URLError, socket_error):
return None return None


if response.headers.get("Content-Encoding") == "gzip": if response.headers.get("Content-Encoding") == "gzip":
@@ -154,7 +155,7 @@ class _CopyvioWorker(object):
gzipper = GzipFile(fileobj=stream) gzipper = GzipFile(fileobj=stream)
try: try:
content = gzipper.read(2 * 1024 ** 2) content = gzipper.read(2 * 1024 ** 2)
except IOError:
except (IOError, struct_error):
return None return None


return handler(content) return handler(content)


Loading…
Cancel
Save