From ec7f98ddb60b14fdcb7a74da888aec54f53849d6 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben@benkurtovic.com>
Date: Sat, 28 Jun 2025 19:35:44 -0400
Subject: [PATCH] Encoding required on Windows

---
 tests/test_tokenizer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
index 8cab57c..5864b48 100644
--- a/tests/test_tokenizer.py
+++ b/tests/test_tokenizer.py
@@ -110,7 +110,7 @@ def build():
         if not filename.endswith(extension):
             continue
         fullname = os.path.join(directory, filename)
-        with open(fullname) as fp:
+        with open(fullname, encoding="utf8") as fp:
             text = fp.read()
             name = os.path.split(fullname)[1][: -len(extension)]
             yield from _load_tests(fullname, name, text)