A Python parser for MediaWiki wikicode https://mwparserfromhell.readthedocs.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

346 lines
8.8 KiB

  1. /*
  2. Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
  3. Permission is hereby granted, free of charge, to any person obtaining a copy of
  4. this software and associated documentation files (the "Software"), to deal in
  5. the Software without restriction, including without limitation the rights to
  6. use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
  7. of the Software, and to permit persons to whom the Software is furnished to do
  8. so, subject to the following conditions:
  9. The above copyright notice and this permission notice shall be included in all
  10. copies or substantial portions of the Software.
  11. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  12. IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  13. FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  14. AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  15. LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  16. OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  17. SOFTWARE.
  18. */
  19. #include "tok_support.h"
  20. #include "textbuffer.h"
  21. #include "tokens.h"
  22. /*
  23. Add a new token stack, context, and textbuffer to the list.
  24. */
  25. int Tokenizer_push(Tokenizer* self, uint64_t context)
  26. {
  27. Stack* top = malloc(sizeof(Stack));
  28. if (!top) {
  29. PyErr_NoMemory();
  30. return -1;
  31. }
  32. top->stack = PyList_New(0);
  33. top->context = context;
  34. top->textbuffer = Textbuffer_new(&self->text);
  35. if (!top->textbuffer)
  36. return -1;
  37. top->next = self->topstack;
  38. self->topstack = top;
  39. self->depth++;
  40. self->cycles++;
  41. return 0;
  42. }
  43. /*
  44. Push the textbuffer onto the stack as a Text node and clear it.
  45. */
  46. int Tokenizer_push_textbuffer(Tokenizer* self)
  47. {
  48. PyObject *text, *kwargs, *token;
  49. Textbuffer* buffer = self->topstack->textbuffer;
  50. if (buffer->length == 0)
  51. return 0;
  52. text = Textbuffer_render(buffer);
  53. if (!text)
  54. return -1;
  55. kwargs = PyDict_New();
  56. if (!kwargs) {
  57. Py_DECREF(text);
  58. return -1;
  59. }
  60. PyDict_SetItemString(kwargs, "text", text);
  61. Py_DECREF(text);
  62. token = PyObject_Call(Text, NOARGS, kwargs);
  63. Py_DECREF(kwargs);
  64. if (!token)
  65. return -1;
  66. if (PyList_Append(self->topstack->stack, token)) {
  67. Py_DECREF(token);
  68. return -1;
  69. }
  70. Py_DECREF(token);
  71. if (Textbuffer_reset(buffer))
  72. return -1;
  73. return 0;
  74. }
  75. /*
  76. Pop and deallocate the top token stack/context/textbuffer.
  77. */
  78. void Tokenizer_delete_top_of_stack(Tokenizer* self)
  79. {
  80. Stack* top = self->topstack;
  81. Py_DECREF(top->stack);
  82. Textbuffer_dealloc(top->textbuffer);
  83. self->topstack = top->next;
  84. free(top);
  85. self->depth--;
  86. }
  87. /*
  88. Pop the current stack/context/textbuffer, returing the stack.
  89. */
  90. PyObject* Tokenizer_pop(Tokenizer* self)
  91. {
  92. PyObject* stack;
  93. if (Tokenizer_push_textbuffer(self))
  94. return NULL;
  95. stack = self->topstack->stack;
  96. Py_INCREF(stack);
  97. Tokenizer_delete_top_of_stack(self);
  98. return stack;
  99. }
  100. /*
  101. Pop the current stack/context/textbuffer, returing the stack. We will also
  102. replace the underlying stack's context with the current stack's.
  103. */
  104. PyObject* Tokenizer_pop_keeping_context(Tokenizer* self)
  105. {
  106. PyObject* stack;
  107. uint64_t context;
  108. if (Tokenizer_push_textbuffer(self))
  109. return NULL;
  110. stack = self->topstack->stack;
  111. Py_INCREF(stack);
  112. context = self->topstack->context;
  113. Tokenizer_delete_top_of_stack(self);
  114. self->topstack->context = context;
  115. return stack;
  116. }
  117. /*
  118. Fail the current tokenization route. Discards the current
  119. stack/context/textbuffer and sets the BAD_ROUTE flag.
  120. */
  121. void* Tokenizer_fail_route(Tokenizer* self)
  122. {
  123. uint64_t context = self->topstack->context;
  124. PyObject* stack = Tokenizer_pop(self);
  125. Py_XDECREF(stack);
  126. FAIL_ROUTE(context);
  127. return NULL;
  128. }
  129. /*
  130. Write a token to the current token stack.
  131. */
  132. int Tokenizer_emit_token(Tokenizer* self, PyObject* token, int first)
  133. {
  134. PyObject* instance;
  135. if (Tokenizer_push_textbuffer(self))
  136. return -1;
  137. instance = PyObject_CallObject(token, NULL);
  138. if (!instance)
  139. return -1;
  140. if (first ? PyList_Insert(self->topstack->stack, 0, instance) :
  141. PyList_Append(self->topstack->stack, instance)) {
  142. Py_DECREF(instance);
  143. return -1;
  144. }
  145. Py_DECREF(instance);
  146. return 0;
  147. }
  148. /*
  149. Write a token to the current token stack, with kwargs. Steals a reference
  150. to kwargs.
  151. */
  152. int Tokenizer_emit_token_kwargs(Tokenizer* self, PyObject* token,
  153. PyObject* kwargs, int first)
  154. {
  155. PyObject* instance;
  156. if (Tokenizer_push_textbuffer(self)) {
  157. Py_DECREF(kwargs);
  158. return -1;
  159. }
  160. instance = PyObject_Call(token, NOARGS, kwargs);
  161. if (!instance) {
  162. Py_DECREF(kwargs);
  163. return -1;
  164. }
  165. if (first ? PyList_Insert(self->topstack->stack, 0, instance):
  166. PyList_Append(self->topstack->stack, instance)) {
  167. Py_DECREF(instance);
  168. Py_DECREF(kwargs);
  169. return -1;
  170. }
  171. Py_DECREF(instance);
  172. Py_DECREF(kwargs);
  173. return 0;
  174. }
  175. /*
  176. Write a Unicode codepoint to the current textbuffer.
  177. */
  178. int Tokenizer_emit_char(Tokenizer* self, Unicode code)
  179. {
  180. return Textbuffer_write(self->topstack->textbuffer, code);
  181. }
  182. /*
  183. Write a string of text to the current textbuffer.
  184. */
  185. int Tokenizer_emit_text(Tokenizer* self, const char* text)
  186. {
  187. int i = 0;
  188. while (text[i]) {
  189. if (Tokenizer_emit_char(self, text[i]))
  190. return -1;
  191. i++;
  192. }
  193. return 0;
  194. }
  195. /*
  196. Write the contents of another textbuffer to the current textbuffer,
  197. deallocating it in the process.
  198. */
  199. int Tokenizer_emit_textbuffer(Tokenizer* self, Textbuffer* buffer)
  200. {
  201. int retval = Textbuffer_concat(self->topstack->textbuffer, buffer);
  202. Textbuffer_dealloc(buffer);
  203. return retval;
  204. }
  205. /*
  206. Write a series of tokens to the current stack at once.
  207. */
  208. int Tokenizer_emit_all(Tokenizer* self, PyObject* tokenlist)
  209. {
  210. int pushed = 0;
  211. PyObject *stack, *token, *left, *right, *text;
  212. Textbuffer* buffer;
  213. Py_ssize_t size;
  214. if (PyList_GET_SIZE(tokenlist) > 0) {
  215. token = PyList_GET_ITEM(tokenlist, 0);
  216. switch (PyObject_IsInstance(token, Text)) {
  217. case 0:
  218. break;
  219. case 1: {
  220. pushed = 1;
  221. buffer = self->topstack->textbuffer;
  222. if (buffer->length == 0)
  223. break;
  224. left = Textbuffer_render(buffer);
  225. if (!left)
  226. return -1;
  227. right = PyObject_GetAttrString(token, "text");
  228. if (!right)
  229. return -1;
  230. text = PyUnicode_Concat(left, right);
  231. Py_DECREF(left);
  232. Py_DECREF(right);
  233. if (!text)
  234. return -1;
  235. if (PyObject_SetAttrString(token, "text", text)) {
  236. Py_DECREF(text);
  237. return -1;
  238. }
  239. Py_DECREF(text);
  240. if (Textbuffer_reset(buffer))
  241. return -1;
  242. break;
  243. }
  244. case -1:
  245. return -1;
  246. }
  247. }
  248. if (!pushed) {
  249. if (Tokenizer_push_textbuffer(self))
  250. return -1;
  251. }
  252. stack = self->topstack->stack;
  253. size = PyList_GET_SIZE(stack);
  254. if (PyList_SetSlice(stack, size, size, tokenlist))
  255. return -1;
  256. return 0;
  257. }
  258. /*
  259. Pop the current stack, write text, and then write the stack. 'text' is a
  260. NULL-terminated array of chars.
  261. */
  262. int Tokenizer_emit_text_then_stack(Tokenizer* self, const char* text)
  263. {
  264. PyObject* stack = Tokenizer_pop(self);
  265. if (Tokenizer_emit_text(self, text)) {
  266. Py_DECREF(stack);
  267. return -1;
  268. }
  269. if (stack) {
  270. if (PyList_GET_SIZE(stack) > 0) {
  271. if (Tokenizer_emit_all(self, stack)) {
  272. Py_DECREF(stack);
  273. return -1;
  274. }
  275. }
  276. Py_DECREF(stack);
  277. }
  278. self->head--;
  279. return 0;
  280. }
  281. /*
  282. Internal function to read the codepoint at the given index from the input.
  283. */
  284. static Unicode read_codepoint(TokenizerInput* text, Py_ssize_t index)
  285. {
  286. #ifdef PEP_393
  287. return PyUnicode_READ(text->kind, text->data, index);
  288. #else
  289. return text->buf[index];
  290. #endif
  291. }
  292. /*
  293. Read the value at a relative point in the wikicode, forwards.
  294. */
  295. Unicode Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
  296. {
  297. Py_ssize_t index = self->head + delta;
  298. if (index >= self->text.length)
  299. return '\0';
  300. return read_codepoint(&self->text, index);
  301. }
  302. /*
  303. Read the value at a relative point in the wikicode, backwards.
  304. */
  305. Unicode Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
  306. {
  307. Py_ssize_t index;
  308. if (delta > self->head)
  309. return '\0';
  310. index = self->head - delta;
  311. return read_codepoint(&self->text, index);
  312. }