diff --git a/tests/_test_tokenizer.py b/tests/_test_tokenizer.py index 7487241..313b959 100644 --- a/tests/_test_tokenizer.py +++ b/tests/_test_tokenizer.py @@ -25,6 +25,8 @@ from os import listdir, path import sys from mwparserfromhell.compat import py3k +if not py3k: + from codecs import open from mwparserfromhell.parser import tokens class _TestParseError(Exception): @@ -109,10 +111,8 @@ class TokenizerTestCase(object): def build(cls): """Load and install all tests from the 'tokenizer' directory.""" def load_file(filename): - with open(filename, "rU") as fp: + with open(filename, "rU", encoding='utf8') as fp: text = fp.read() - if not py3k: - text = text.decode("utf8") name = path.split(filename)[1][:0-len(extension)] cls._load_tests(filename, name, text)