_test_tokenizer: force utf-8 file encoding

tests/_test_tokenizer.py

@@ -25,6 +25,8 @@ from os import listdir, path
 import sys
 
 from mwparserfromhell.compat import py3k
+if not py3k:
+	from codecs import open
 from mwparserfromhell.parser import tokens
 
 class _TestParseError(Exception):
@@ -109,10 +111,8 @@ class TokenizerTestCase(object):
     def build(cls):
         """Load and install all tests from the 'tokenizer' directory."""
         def load_file(filename):
-            with open(filename, "rU") as fp:
+            with open(filename, "rU", encoding='utf8') as fp:
                 text = fp.read()
-                if not py3k:
-                    text = text.decode("utf8")
                 name = path.split(filename)[1][:0-len(extension)]
                 cls._load_tests(filename, name, text)