|
@@ -109,7 +109,7 @@ class TokenizerTestCase(object): |
|
|
def build(cls): |
|
|
def build(cls): |
|
|
"""Load and install all tests from the 'tokenizer' directory.""" |
|
|
"""Load and install all tests from the 'tokenizer' directory.""" |
|
|
def load_file(filename): |
|
|
def load_file(filename): |
|
|
with open(filename, "r") as fp: |
|
|
|
|
|
|
|
|
with open(filename, "rU") as fp: |
|
|
text = fp.read() |
|
|
text = fp.read() |
|
|
if not py3k: |
|
|
if not py3k: |
|
|
text = text.decode("utf8") |
|
|
text = text.decode("utf8") |
|
|