|
@@ -183,7 +183,7 @@ class Tokenizer(object): |
|
|
while True: |
|
|
while True: |
|
|
this = self._read() |
|
|
this = self._read() |
|
|
if this not in self.SENTINELS: |
|
|
if this not in self.SENTINELS: |
|
|
self._write(self._read(), text=True) |
|
|
|
|
|
|
|
|
self._write(this, text=True) |
|
|
self._head += 1 |
|
|
self._head += 1 |
|
|
continue |
|
|
continue |
|
|
if this is self.END: |
|
|
if this is self.END: |
|
|