A Python parser for MediaWiki wikicode https://mwparserfromhell.readthedocs.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

2819 rivejä
83 KiB

  1. /*
  2. Copyright (C) 2012-2019 Ben Kurtovic <ben.kurtovic@gmail.com>
  3. Permission is hereby granted, free of charge, to any person obtaining a copy of
  4. this software and associated documentation files (the "Software"), to deal in
  5. the Software without restriction, including without limitation the rights to
  6. use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
  7. of the Software, and to permit persons to whom the Software is furnished to do
  8. so, subject to the following conditions:
  9. The above copyright notice and this permission notice shall be included in all
  10. copies or substantial portions of the Software.
  11. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  12. IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  13. FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  14. AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  15. LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  16. OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  17. SOFTWARE.
  18. */
  19. #include "tok_parse.h"
  20. #include "contexts.h"
  21. #include "definitions.h"
  22. #include "tag_data.h"
  23. #include "tok_support.h"
  24. #include "tokens.h"
  25. #define DIGITS "0123456789"
  26. #define HEXDIGITS "0123456789abcdefABCDEF"
  27. #define ALPHANUM "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
  28. #define URISCHEME "abcdefghijklmnopqrstuvwxyz0123456789+.-"
  29. #define MAX_BRACES 255
  30. #define MAX_ENTITY_SIZE 8
  31. typedef struct {
  32. PyObject* title;
  33. int level;
  34. } HeadingData;
  35. /* Forward declarations */
  36. static PyObject* Tokenizer_really_parse_external_link(
  37. Tokenizer*, int, Textbuffer*);
  38. static int Tokenizer_parse_entity(Tokenizer*);
  39. static int Tokenizer_parse_comment(Tokenizer*);
  40. static int Tokenizer_handle_dl_term(Tokenizer*);
  41. static int Tokenizer_parse_tag(Tokenizer*);
  42. /*
  43. Determine whether the given code point is a marker.
  44. */
  45. static int is_marker(Unicode this)
  46. {
  47. int i;
  48. for (i = 0; i < NUM_MARKERS; i++) {
  49. if (MARKERS[i] == this)
  50. return 1;
  51. }
  52. return 0;
  53. }
  54. /*
  55. Given a context, return the heading level encoded within it.
  56. */
  57. static int heading_level_from_context(uint64_t n)
  58. {
  59. int level;
  60. n /= LC_HEADING_LEVEL_1;
  61. for (level = 1; n > 1; n >>= 1)
  62. level++;
  63. return level;
  64. }
  65. /*
  66. Sanitize the name of a tag so it can be compared with others for equality.
  67. */
  68. static PyObject* strip_tag_name(PyObject* token, int take_attr)
  69. {
  70. PyObject *text, *rstripped, *lowered;
  71. if (take_attr) {
  72. text = PyObject_GetAttrString(token, "text");
  73. if (!text)
  74. return NULL;
  75. rstripped = PyObject_CallMethod(text, "rstrip", NULL);
  76. Py_DECREF(text);
  77. }
  78. else
  79. rstripped = PyObject_CallMethod(token, "rstrip", NULL);
  80. if (!rstripped)
  81. return NULL;
  82. lowered = PyObject_CallMethod(rstripped, "lower", NULL);
  83. Py_DECREF(rstripped);
  84. return lowered;
  85. }
  86. /*
  87. Parse a template at the head of the wikicode string.
  88. */
  89. static int Tokenizer_parse_template(Tokenizer* self, int has_content)
  90. {
  91. PyObject *template;
  92. Py_ssize_t reset = self->head;
  93. uint64_t context = LC_TEMPLATE_NAME;
  94. if (has_content)
  95. context |= LC_HAS_TEMPLATE;
  96. template = Tokenizer_parse(self, context, 1);
  97. if (BAD_ROUTE) {
  98. self->head = reset;
  99. return 0;
  100. }
  101. if (!template)
  102. return -1;
  103. if (Tokenizer_emit_first(self, TemplateOpen)) {
  104. Py_DECREF(template);
  105. return -1;
  106. }
  107. if (Tokenizer_emit_all(self, template)) {
  108. Py_DECREF(template);
  109. return -1;
  110. }
  111. Py_DECREF(template);
  112. if (Tokenizer_emit(self, TemplateClose))
  113. return -1;
  114. return 0;
  115. }
  116. /*
  117. Parse an argument at the head of the wikicode string.
  118. */
  119. static int Tokenizer_parse_argument(Tokenizer* self)
  120. {
  121. PyObject *argument;
  122. Py_ssize_t reset = self->head;
  123. argument = Tokenizer_parse(self, LC_ARGUMENT_NAME, 1);
  124. if (BAD_ROUTE) {
  125. self->head = reset;
  126. return 0;
  127. }
  128. if (!argument)
  129. return -1;
  130. if (Tokenizer_emit_first(self, ArgumentOpen)) {
  131. Py_DECREF(argument);
  132. return -1;
  133. }
  134. if (Tokenizer_emit_all(self, argument)) {
  135. Py_DECREF(argument);
  136. return -1;
  137. }
  138. Py_DECREF(argument);
  139. if (Tokenizer_emit(self, ArgumentClose))
  140. return -1;
  141. return 0;
  142. }
  143. /*
  144. Parse a template or argument at the head of the wikicode string.
  145. */
  146. static int Tokenizer_parse_template_or_argument(Tokenizer* self)
  147. {
  148. unsigned int braces = 2, i;
  149. int has_content = 0;
  150. PyObject *tokenlist;
  151. self->head += 2;
  152. while (Tokenizer_read(self, 0) == '{' && braces < MAX_BRACES) {
  153. self->head++;
  154. braces++;
  155. }
  156. if (Tokenizer_push(self, 0))
  157. return -1;
  158. while (braces) {
  159. if (braces == 1) {
  160. if (Tokenizer_emit_text_then_stack(self, "{"))
  161. return -1;
  162. return 0;
  163. }
  164. if (braces == 2) {
  165. if (Tokenizer_parse_template(self, has_content))
  166. return -1;
  167. if (BAD_ROUTE) {
  168. RESET_ROUTE();
  169. if (Tokenizer_emit_text_then_stack(self, "{{"))
  170. return -1;
  171. return 0;
  172. }
  173. break;
  174. }
  175. if (Tokenizer_parse_argument(self))
  176. return -1;
  177. if (BAD_ROUTE) {
  178. RESET_ROUTE();
  179. if (Tokenizer_parse_template(self, has_content))
  180. return -1;
  181. if (BAD_ROUTE) {
  182. char text[MAX_BRACES + 1];
  183. RESET_ROUTE();
  184. for (i = 0; i < braces; i++) text[i] = '{';
  185. text[braces] = '\0';
  186. if (Tokenizer_emit_text_then_stack(self, text))
  187. return -1;
  188. return 0;
  189. }
  190. else
  191. braces -= 2;
  192. }
  193. else
  194. braces -= 3;
  195. if (braces) {
  196. has_content = 1;
  197. self->head++;
  198. }
  199. }
  200. tokenlist = Tokenizer_pop(self);
  201. if (!tokenlist)
  202. return -1;
  203. if (Tokenizer_emit_all(self, tokenlist)) {
  204. Py_DECREF(tokenlist);
  205. return -1;
  206. }
  207. Py_DECREF(tokenlist);
  208. if (self->topstack->context & LC_FAIL_NEXT)
  209. self->topstack->context ^= LC_FAIL_NEXT;
  210. return 0;
  211. }
  212. /*
  213. Handle a template parameter at the head of the string.
  214. */
  215. static int Tokenizer_handle_template_param(Tokenizer* self)
  216. {
  217. PyObject *stack;
  218. if (self->topstack->context & LC_TEMPLATE_NAME) {
  219. if (!(self->topstack->context & (LC_HAS_TEXT | LC_HAS_TEMPLATE))) {
  220. Tokenizer_fail_route(self);
  221. return -1;
  222. }
  223. self->topstack->context ^= LC_TEMPLATE_NAME;
  224. }
  225. else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE)
  226. self->topstack->context ^= LC_TEMPLATE_PARAM_VALUE;
  227. if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
  228. stack = Tokenizer_pop(self);
  229. if (!stack)
  230. return -1;
  231. if (Tokenizer_emit_all(self, stack)) {
  232. Py_DECREF(stack);
  233. return -1;
  234. }
  235. Py_DECREF(stack);
  236. }
  237. else
  238. self->topstack->context |= LC_TEMPLATE_PARAM_KEY;
  239. if (Tokenizer_emit(self, TemplateParamSeparator))
  240. return -1;
  241. if (Tokenizer_push(self, self->topstack->context))
  242. return -1;
  243. return 0;
  244. }
  245. /*
  246. Handle a template parameter's value at the head of the string.
  247. */
  248. static int Tokenizer_handle_template_param_value(Tokenizer* self)
  249. {
  250. PyObject *stack;
  251. stack = Tokenizer_pop(self);
  252. if (!stack)
  253. return -1;
  254. if (Tokenizer_emit_all(self, stack)) {
  255. Py_DECREF(stack);
  256. return -1;
  257. }
  258. Py_DECREF(stack);
  259. self->topstack->context ^= LC_TEMPLATE_PARAM_KEY;
  260. self->topstack->context |= LC_TEMPLATE_PARAM_VALUE;
  261. if (Tokenizer_emit(self, TemplateParamEquals))
  262. return -1;
  263. return 0;
  264. }
  265. /*
  266. Handle the end of a template at the head of the string.
  267. */
  268. static PyObject* Tokenizer_handle_template_end(Tokenizer* self)
  269. {
  270. PyObject* stack;
  271. if (self->topstack->context & LC_TEMPLATE_NAME) {
  272. if (!(self->topstack->context & (LC_HAS_TEXT | LC_HAS_TEMPLATE)))
  273. return Tokenizer_fail_route(self);
  274. }
  275. else if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
  276. stack = Tokenizer_pop(self);
  277. if (!stack)
  278. return NULL;
  279. if (Tokenizer_emit_all(self, stack)) {
  280. Py_DECREF(stack);
  281. return NULL;
  282. }
  283. Py_DECREF(stack);
  284. }
  285. self->head++;
  286. stack = Tokenizer_pop(self);
  287. return stack;
  288. }
  289. /*
  290. Handle the separator between an argument's name and default.
  291. */
  292. static int Tokenizer_handle_argument_separator(Tokenizer* self)
  293. {
  294. self->topstack->context ^= LC_ARGUMENT_NAME;
  295. self->topstack->context |= LC_ARGUMENT_DEFAULT;
  296. if (Tokenizer_emit(self, ArgumentSeparator))
  297. return -1;
  298. return 0;
  299. }
  300. /*
  301. Handle the end of an argument at the head of the string.
  302. */
  303. static PyObject* Tokenizer_handle_argument_end(Tokenizer* self)
  304. {
  305. PyObject* stack = Tokenizer_pop(self);
  306. self->head += 2;
  307. return stack;
  308. }
  309. /*
  310. Parse an internal wikilink at the head of the wikicode string.
  311. */
  312. static int Tokenizer_parse_wikilink(Tokenizer* self)
  313. {
  314. Py_ssize_t reset;
  315. PyObject *extlink, *wikilink, *kwargs;
  316. reset = self->head + 1;
  317. self->head += 2;
  318. // If the wikilink looks like an external link, parse it as such:
  319. extlink = Tokenizer_really_parse_external_link(self, 1, NULL);
  320. if (BAD_ROUTE) {
  321. RESET_ROUTE();
  322. self->head = reset + 1;
  323. // Otherwise, actually parse it as a wikilink:
  324. wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE, 1);
  325. if (BAD_ROUTE) {
  326. RESET_ROUTE();
  327. self->head = reset;
  328. if (Tokenizer_emit_text(self, "[["))
  329. return -1;
  330. return 0;
  331. }
  332. if (!wikilink)
  333. return -1;
  334. if (Tokenizer_emit(self, WikilinkOpen)) {
  335. Py_DECREF(wikilink);
  336. return -1;
  337. }
  338. if (Tokenizer_emit_all(self, wikilink)) {
  339. Py_DECREF(wikilink);
  340. return -1;
  341. }
  342. Py_DECREF(wikilink);
  343. if (Tokenizer_emit(self, WikilinkClose))
  344. return -1;
  345. return 0;
  346. }
  347. if (!extlink)
  348. return -1;
  349. if (self->topstack->context & LC_EXT_LINK_TITLE) {
  350. // In this exceptional case, an external link that looks like a
  351. // wikilink inside of an external link is parsed as text:
  352. Py_DECREF(extlink);
  353. self->head = reset;
  354. if (Tokenizer_emit_text(self, "[["))
  355. return -1;
  356. return 0;
  357. }
  358. if (Tokenizer_emit_text(self, "[")) {
  359. Py_DECREF(extlink);
  360. return -1;
  361. }
  362. kwargs = PyDict_New();
  363. if (!kwargs) {
  364. Py_DECREF(extlink);
  365. return -1;
  366. }
  367. PyDict_SetItemString(kwargs, "brackets", Py_True);
  368. if (Tokenizer_emit_kwargs(self, ExternalLinkOpen, kwargs)) {
  369. Py_DECREF(extlink);
  370. return -1;
  371. }
  372. if (Tokenizer_emit_all(self, extlink)) {
  373. Py_DECREF(extlink);
  374. return -1;
  375. }
  376. Py_DECREF(extlink);
  377. if (Tokenizer_emit(self, ExternalLinkClose))
  378. return -1;
  379. return 0;
  380. }
  381. /*
  382. Handle the separator between a wikilink's title and its text.
  383. */
  384. static int Tokenizer_handle_wikilink_separator(Tokenizer* self)
  385. {
  386. self->topstack->context ^= LC_WIKILINK_TITLE;
  387. self->topstack->context |= LC_WIKILINK_TEXT;
  388. if (Tokenizer_emit(self, WikilinkSeparator))
  389. return -1;
  390. return 0;
  391. }
  392. /*
  393. Handle the end of a wikilink at the head of the string.
  394. */
  395. static PyObject* Tokenizer_handle_wikilink_end(Tokenizer* self)
  396. {
  397. PyObject* stack = Tokenizer_pop(self);
  398. self->head += 1;
  399. return stack;
  400. }
  401. /*
  402. Parse the URI scheme of a bracket-enclosed external link.
  403. */
  404. static int Tokenizer_parse_bracketed_uri_scheme(Tokenizer* self)
  405. {
  406. static const char* valid = URISCHEME;
  407. Textbuffer* buffer;
  408. PyObject* scheme;
  409. Unicode this;
  410. int slashes, i;
  411. if (Tokenizer_check_route(self, LC_EXT_LINK_URI) < 0)
  412. return 0;
  413. if (Tokenizer_push(self, LC_EXT_LINK_URI))
  414. return -1;
  415. if (Tokenizer_read(self, 0) == '/' && Tokenizer_read(self, 1) == '/') {
  416. if (Tokenizer_emit_text(self, "//"))
  417. return -1;
  418. self->head += 2;
  419. }
  420. else {
  421. buffer = Textbuffer_new(&self->text);
  422. if (!buffer)
  423. return -1;
  424. while ((this = Tokenizer_read(self, 0))) {
  425. i = 0;
  426. while (1) {
  427. if (!valid[i])
  428. goto end_of_loop;
  429. if (this == (Unicode) valid[i])
  430. break;
  431. i++;
  432. }
  433. Textbuffer_write(buffer, this);
  434. if (Tokenizer_emit_char(self, this)) {
  435. Textbuffer_dealloc(buffer);
  436. return -1;
  437. }
  438. self->head++;
  439. }
  440. end_of_loop:
  441. if (this != ':') {
  442. Textbuffer_dealloc(buffer);
  443. Tokenizer_fail_route(self);
  444. return 0;
  445. }
  446. if (Tokenizer_emit_char(self, ':')) {
  447. Textbuffer_dealloc(buffer);
  448. return -1;
  449. }
  450. self->head++;
  451. slashes = (Tokenizer_read(self, 0) == '/' &&
  452. Tokenizer_read(self, 1) == '/');
  453. if (slashes) {
  454. if (Tokenizer_emit_text(self, "//")) {
  455. Textbuffer_dealloc(buffer);
  456. return -1;
  457. }
  458. self->head += 2;
  459. }
  460. scheme = Textbuffer_render(buffer);
  461. Textbuffer_dealloc(buffer);
  462. if (!scheme)
  463. return -1;
  464. if (!is_scheme(scheme, slashes)) {
  465. Py_DECREF(scheme);
  466. Tokenizer_fail_route(self);
  467. return 0;
  468. }
  469. Py_DECREF(scheme);
  470. }
  471. return 0;
  472. }
  473. /*
  474. Parse the URI scheme of a free (no brackets) external link.
  475. */
  476. static int Tokenizer_parse_free_uri_scheme(Tokenizer* self)
  477. {
  478. static const char* valid = URISCHEME;
  479. Textbuffer *scheme_buffer = Textbuffer_new(&self->text);
  480. PyObject *scheme;
  481. Unicode chunk;
  482. Py_ssize_t i;
  483. int slashes, j;
  484. uint64_t new_context;
  485. if (!scheme_buffer)
  486. return -1;
  487. // We have to backtrack through the textbuffer looking for our scheme since
  488. // it was just parsed as text:
  489. for (i = self->topstack->textbuffer->length - 1; i >= 0; i--) {
  490. chunk = Textbuffer_read(self->topstack->textbuffer, i);
  491. if (Py_UNICODE_ISSPACE(chunk) || is_marker(chunk))
  492. goto end_of_loop;
  493. j = 0;
  494. do {
  495. if (!valid[j]) {
  496. Textbuffer_dealloc(scheme_buffer);
  497. FAIL_ROUTE(0);
  498. return 0;
  499. }
  500. } while (chunk != (Unicode) valid[j++]);
  501. Textbuffer_write(scheme_buffer, chunk);
  502. }
  503. end_of_loop:
  504. Textbuffer_reverse(scheme_buffer);
  505. scheme = Textbuffer_render(scheme_buffer);
  506. if (!scheme) {
  507. Textbuffer_dealloc(scheme_buffer);
  508. return -1;
  509. }
  510. slashes = (Tokenizer_read(self, 0) == '/' &&
  511. Tokenizer_read(self, 1) == '/');
  512. if (!is_scheme(scheme, slashes)) {
  513. Py_DECREF(scheme);
  514. Textbuffer_dealloc(scheme_buffer);
  515. FAIL_ROUTE(0);
  516. return 0;
  517. }
  518. Py_DECREF(scheme);
  519. new_context = self->topstack->context | LC_EXT_LINK_URI;
  520. if (Tokenizer_check_route(self, new_context) < 0) {
  521. Textbuffer_dealloc(scheme_buffer);
  522. return 0;
  523. }
  524. if (Tokenizer_push(self, new_context)) {
  525. Textbuffer_dealloc(scheme_buffer);
  526. return -1;
  527. }
  528. if (Tokenizer_emit_textbuffer(self, scheme_buffer))
  529. return -1;
  530. if (Tokenizer_emit_char(self, ':'))
  531. return -1;
  532. if (slashes) {
  533. if (Tokenizer_emit_text(self, "//"))
  534. return -1;
  535. self->head += 2;
  536. }
  537. return 0;
  538. }
  539. /*
  540. Handle text in a free external link, including trailing punctuation.
  541. */
  542. static int Tokenizer_handle_free_link_text(
  543. Tokenizer* self, int* parens, Textbuffer* tail, Unicode this)
  544. {
  545. #define PUSH_TAIL_BUFFER(tail, error) \
  546. if (tail && tail->length > 0) { \
  547. if (Textbuffer_concat(self->topstack->textbuffer, tail)) \
  548. return error; \
  549. if (Textbuffer_reset(tail)) \
  550. return error; \
  551. }
  552. if (this == '(' && !(*parens)) {
  553. *parens = 1;
  554. PUSH_TAIL_BUFFER(tail, -1)
  555. }
  556. else if (this == ',' || this == ';' || this == '\\' || this == '.' ||
  557. this == ':' || this == '!' || this == '?' ||
  558. (!(*parens) && this == ')'))
  559. return Textbuffer_write(tail, this);
  560. else
  561. PUSH_TAIL_BUFFER(tail, -1)
  562. return Tokenizer_emit_char(self, this);
  563. }
  564. /*
  565. Return whether the current head is the end of a free link.
  566. */
  567. static int
  568. Tokenizer_is_free_link(Tokenizer* self, Unicode this, Unicode next)
  569. {
  570. // Built from Tokenizer_parse()'s end sentinels:
  571. Unicode after = Tokenizer_read(self, 2);
  572. uint64_t ctx = self->topstack->context;
  573. return (!this || this == '\n' || this == '[' || this == ']' ||
  574. this == '<' || this == '>' || (this == '\'' && next == '\'') ||
  575. (this == '|' && ctx & LC_TEMPLATE) ||
  576. (this == '=' && ctx & (LC_TEMPLATE_PARAM_KEY | LC_HEADING)) ||
  577. (this == '}' && next == '}' &&
  578. (ctx & LC_TEMPLATE || (after == '}' && ctx & LC_ARGUMENT))));
  579. }
  580. /*
  581. Really parse an external link.
  582. */
  583. static PyObject*
  584. Tokenizer_really_parse_external_link(Tokenizer* self, int brackets,
  585. Textbuffer* extra)
  586. {
  587. Unicode this, next;
  588. int parens = 0;
  589. if (brackets ? Tokenizer_parse_bracketed_uri_scheme(self) :
  590. Tokenizer_parse_free_uri_scheme(self))
  591. return NULL;
  592. if (BAD_ROUTE)
  593. return NULL;
  594. this = Tokenizer_read(self, 0);
  595. if (!this || this == '\n' || this == ' ' || this == ']')
  596. return Tokenizer_fail_route(self);
  597. if (!brackets && this == '[')
  598. return Tokenizer_fail_route(self);
  599. while (1) {
  600. this = Tokenizer_read(self, 0);
  601. next = Tokenizer_read(self, 1);
  602. if (this == '&') {
  603. PUSH_TAIL_BUFFER(extra, NULL)
  604. if (Tokenizer_parse_entity(self))
  605. return NULL;
  606. }
  607. else if (this == '<' && next == '!'
  608. && Tokenizer_read(self, 2) == '-'
  609. && Tokenizer_read(self, 3) == '-') {
  610. PUSH_TAIL_BUFFER(extra, NULL)
  611. if (Tokenizer_parse_comment(self))
  612. return NULL;
  613. }
  614. else if (!brackets && Tokenizer_is_free_link(self, this, next)) {
  615. self->head--;
  616. return Tokenizer_pop(self);
  617. }
  618. else if (!this || this == '\n')
  619. return Tokenizer_fail_route(self);
  620. else if (this == '{' && next == '{' && Tokenizer_CAN_RECURSE(self)) {
  621. PUSH_TAIL_BUFFER(extra, NULL)
  622. if (Tokenizer_parse_template_or_argument(self))
  623. return NULL;
  624. }
  625. else if (this == ']')
  626. return Tokenizer_pop(self);
  627. else if (this == ' ') {
  628. if (brackets) {
  629. if (Tokenizer_emit(self, ExternalLinkSeparator))
  630. return NULL;
  631. self->topstack->context ^= LC_EXT_LINK_URI;
  632. self->topstack->context |= LC_EXT_LINK_TITLE;
  633. self->head++;
  634. return Tokenizer_parse(self, 0, 0);
  635. }
  636. if (Textbuffer_write(extra, ' '))
  637. return NULL;
  638. return Tokenizer_pop(self);
  639. }
  640. else if (!brackets) {
  641. if (Tokenizer_handle_free_link_text(self, &parens, extra, this))
  642. return NULL;
  643. }
  644. else {
  645. if (Tokenizer_emit_char(self, this))
  646. return NULL;
  647. }
  648. self->head++;
  649. }
  650. }
  651. /*
  652. Remove the URI scheme of a new external link from the textbuffer.
  653. */
  654. static int
  655. Tokenizer_remove_uri_scheme_from_textbuffer(Tokenizer* self, PyObject* link)
  656. {
  657. PyObject *text = PyObject_GetAttrString(PyList_GET_ITEM(link, 0), "text"),
  658. *split, *scheme;
  659. Py_ssize_t length;
  660. if (!text)
  661. return -1;
  662. split = PyObject_CallMethod(text, "split", "si", ":", 1);
  663. Py_DECREF(text);
  664. if (!split)
  665. return -1;
  666. scheme = PyList_GET_ITEM(split, 0);
  667. length = PyUnicode_GET_LENGTH(scheme);
  668. Py_DECREF(split);
  669. self->topstack->textbuffer->length -= length;
  670. return 0;
  671. }
  672. /*
  673. Parse an external link at the head of the wikicode string.
  674. */
  675. static int Tokenizer_parse_external_link(Tokenizer* self, int brackets)
  676. {
  677. #define NOT_A_LINK \
  678. if (!brackets && self->topstack->context & LC_DLTERM) \
  679. return Tokenizer_handle_dl_term(self); \
  680. return Tokenizer_emit_char(self, Tokenizer_read(self, 0))
  681. Py_ssize_t reset = self->head;
  682. PyObject *link, *kwargs;
  683. Textbuffer *extra;
  684. if (self->topstack->context & AGG_NO_EXT_LINKS ||
  685. !(Tokenizer_CAN_RECURSE(self))) {
  686. NOT_A_LINK;
  687. }
  688. extra = Textbuffer_new(&self->text);
  689. if (!extra)
  690. return -1;
  691. self->head++;
  692. link = Tokenizer_really_parse_external_link(self, brackets, extra);
  693. if (BAD_ROUTE) {
  694. RESET_ROUTE();
  695. self->head = reset;
  696. Textbuffer_dealloc(extra);
  697. NOT_A_LINK;
  698. }
  699. if (!link) {
  700. Textbuffer_dealloc(extra);
  701. return -1;
  702. }
  703. if (!brackets) {
  704. if (Tokenizer_remove_uri_scheme_from_textbuffer(self, link)) {
  705. Textbuffer_dealloc(extra);
  706. Py_DECREF(link);
  707. return -1;
  708. }
  709. }
  710. kwargs = PyDict_New();
  711. if (!kwargs) {
  712. Textbuffer_dealloc(extra);
  713. Py_DECREF(link);
  714. return -1;
  715. }
  716. PyDict_SetItemString(kwargs, "brackets", brackets ? Py_True : Py_False);
  717. if (Tokenizer_emit_kwargs(self, ExternalLinkOpen, kwargs)) {
  718. Textbuffer_dealloc(extra);
  719. Py_DECREF(link);
  720. return -1;
  721. }
  722. if (Tokenizer_emit_all(self, link)) {
  723. Textbuffer_dealloc(extra);
  724. Py_DECREF(link);
  725. return -1;
  726. }
  727. Py_DECREF(link);
  728. if (Tokenizer_emit(self, ExternalLinkClose)) {
  729. Textbuffer_dealloc(extra);
  730. return -1;
  731. }
  732. if (extra->length > 0)
  733. return Tokenizer_emit_textbuffer(self, extra);
  734. Textbuffer_dealloc(extra);
  735. return 0;
  736. }
  737. /*
  738. Parse a section heading at the head of the wikicode string.
  739. */
  740. static int Tokenizer_parse_heading(Tokenizer* self)
  741. {
  742. Py_ssize_t reset = self->head;
  743. int best = 1, i, context, diff;
  744. HeadingData *heading;
  745. PyObject *level, *kwargs;
  746. self->global |= GL_HEADING;
  747. self->head += 1;
  748. while (Tokenizer_read(self, 0) == '=') {
  749. best++;
  750. self->head++;
  751. }
  752. context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
  753. heading = (HeadingData*) Tokenizer_parse(self, context, 1);
  754. if (BAD_ROUTE) {
  755. RESET_ROUTE();
  756. self->head = reset + best - 1;
  757. for (i = 0; i < best; i++) {
  758. if (Tokenizer_emit_char(self, '='))
  759. return -1;
  760. }
  761. self->global ^= GL_HEADING;
  762. return 0;
  763. }
  764. if (!heading) {
  765. return -1;
  766. }
  767. #ifdef IS_PY3K
  768. level = PyLong_FromSsize_t(heading->level);
  769. #else
  770. level = PyInt_FromSsize_t(heading->level);
  771. #endif
  772. if (!level) {
  773. Py_DECREF(heading->title);
  774. free(heading);
  775. return -1;
  776. }
  777. kwargs = PyDict_New();
  778. if (!kwargs) {
  779. Py_DECREF(level);
  780. Py_DECREF(heading->title);
  781. free(heading);
  782. return -1;
  783. }
  784. PyDict_SetItemString(kwargs, "level", level);
  785. Py_DECREF(level);
  786. if (Tokenizer_emit_kwargs(self, HeadingStart, kwargs)) {
  787. Py_DECREF(heading->title);
  788. free(heading);
  789. return -1;
  790. }
  791. if (heading->level < best) {
  792. diff = best - heading->level;
  793. for (i = 0; i < diff; i++) {
  794. if (Tokenizer_emit_char(self, '=')) {
  795. Py_DECREF(heading->title);
  796. free(heading);
  797. return -1;
  798. }
  799. }
  800. }
  801. if (Tokenizer_emit_all(self, heading->title)) {
  802. Py_DECREF(heading->title);
  803. free(heading);
  804. return -1;
  805. }
  806. Py_DECREF(heading->title);
  807. free(heading);
  808. if (Tokenizer_emit(self, HeadingEnd))
  809. return -1;
  810. self->global ^= GL_HEADING;
  811. return 0;
  812. }
  813. /*
  814. Handle the end of a section heading at the head of the string.
  815. */
  816. static HeadingData* Tokenizer_handle_heading_end(Tokenizer* self)
  817. {
  818. Py_ssize_t reset = self->head;
  819. int best, i, current, level, diff;
  820. HeadingData *after, *heading;
  821. PyObject *stack;
  822. self->head += 1;
  823. best = 1;
  824. while (Tokenizer_read(self, 0) == '=') {
  825. best++;
  826. self->head++;
  827. }
  828. current = heading_level_from_context(self->topstack->context);
  829. level = current > best ? (best > 6 ? 6 : best) :
  830. (current > 6 ? 6 : current);
  831. after = (HeadingData*) Tokenizer_parse(self, self->topstack->context, 1);
  832. if (BAD_ROUTE) {
  833. RESET_ROUTE();
  834. if (level < best) {
  835. diff = best - level;
  836. for (i = 0; i < diff; i++) {
  837. if (Tokenizer_emit_char(self, '='))
  838. return NULL;
  839. }
  840. }
  841. self->head = reset + best - 1;
  842. }
  843. else {
  844. if (!after) {
  845. return NULL;
  846. }
  847. for (i = 0; i < best; i++) {
  848. if (Tokenizer_emit_char(self, '=')) {
  849. Py_DECREF(after->title);
  850. free(after);
  851. return NULL;
  852. }
  853. }
  854. if (Tokenizer_emit_all(self, after->title)) {
  855. Py_DECREF(after->title);
  856. free(after);
  857. return NULL;
  858. }
  859. Py_DECREF(after->title);
  860. level = after->level;
  861. free(after);
  862. }
  863. stack = Tokenizer_pop(self);
  864. if (!stack)
  865. return NULL;
  866. heading = malloc(sizeof(HeadingData));
  867. if (!heading) {
  868. PyErr_NoMemory();
  869. return NULL;
  870. }
  871. heading->title = stack;
  872. heading->level = level;
  873. return heading;
  874. }
  875. /*
  876. Actually parse an HTML entity and ensure that it is valid.
  877. */
  878. static int Tokenizer_really_parse_entity(Tokenizer* self)
  879. {
  880. PyObject *kwargs, *charobj, *textobj;
  881. Unicode this;
  882. int numeric, hexadecimal, i, j, zeroes, test;
  883. char *valid, *text, *buffer, *def;
  884. #define FAIL_ROUTE_AND_EXIT() { \
  885. Tokenizer_fail_route(self); \
  886. free(text); \
  887. return 0; \
  888. }
  889. if (Tokenizer_emit(self, HTMLEntityStart))
  890. return -1;
  891. self->head++;
  892. this = Tokenizer_read(self, 0);
  893. if (!this) {
  894. Tokenizer_fail_route(self);
  895. return 0;
  896. }
  897. if (this == '#') {
  898. numeric = 1;
  899. if (Tokenizer_emit(self, HTMLEntityNumeric))
  900. return -1;
  901. self->head++;
  902. this = Tokenizer_read(self, 0);
  903. if (!this) {
  904. Tokenizer_fail_route(self);
  905. return 0;
  906. }
  907. if (this == 'x' || this == 'X') {
  908. hexadecimal = 1;
  909. kwargs = PyDict_New();
  910. if (!kwargs)
  911. return -1;
  912. if (!(charobj = PyUnicode_FROM_SINGLE(this))) {
  913. Py_DECREF(kwargs);
  914. return -1;
  915. }
  916. PyDict_SetItemString(kwargs, "char", charobj);
  917. Py_DECREF(charobj);
  918. if (Tokenizer_emit_kwargs(self, HTMLEntityHex, kwargs))
  919. return -1;
  920. self->head++;
  921. }
  922. else
  923. hexadecimal = 0;
  924. }
  925. else
  926. numeric = hexadecimal = 0;
  927. if (hexadecimal)
  928. valid = HEXDIGITS;
  929. else if (numeric)
  930. valid = DIGITS;
  931. else
  932. valid = ALPHANUM;
  933. text = calloc(MAX_ENTITY_SIZE, sizeof(char));
  934. if (!text) {
  935. PyErr_NoMemory();
  936. return -1;
  937. }
  938. i = 0;
  939. zeroes = 0;
  940. while (1) {
  941. this = Tokenizer_read(self, 0);
  942. if (this == ';') {
  943. if (i == 0)
  944. FAIL_ROUTE_AND_EXIT()
  945. break;
  946. }
  947. if (i == 0 && this == '0') {
  948. zeroes++;
  949. self->head++;
  950. continue;
  951. }
  952. if (i >= MAX_ENTITY_SIZE)
  953. FAIL_ROUTE_AND_EXIT()
  954. if (is_marker(this))
  955. FAIL_ROUTE_AND_EXIT()
  956. j = 0;
  957. while (1) {
  958. if (!valid[j])
  959. FAIL_ROUTE_AND_EXIT()
  960. if (this == (Unicode) valid[j])
  961. break;
  962. j++;
  963. }
  964. text[i] = (char) this;
  965. self->head++;
  966. i++;
  967. }
  968. if (numeric) {
  969. sscanf(text, (hexadecimal ? "%x" : "%d"), &test);
  970. if (test < 1 || test > 0x10FFFF)
  971. FAIL_ROUTE_AND_EXIT()
  972. }
  973. else {
  974. i = 0;
  975. while (1) {
  976. def = entitydefs[i];
  977. if (!def) // We've reached the end of the defs without finding it
  978. FAIL_ROUTE_AND_EXIT()
  979. if (strcmp(text, def) == 0)
  980. break;
  981. i++;
  982. }
  983. }
  984. if (zeroes) {
  985. buffer = calloc(strlen(text) + zeroes + 1, sizeof(char));
  986. if (!buffer) {
  987. free(text);
  988. PyErr_NoMemory();
  989. return -1;
  990. }
  991. for (i = 0; i < zeroes; i++)
  992. strcat(buffer, "0");
  993. strcat(buffer, text);
  994. free(text);
  995. text = buffer;
  996. }
  997. textobj = PyUnicode_FromString(text);
  998. if (!textobj) {
  999. free(text);
  1000. return -1;
  1001. }
  1002. free(text);
  1003. kwargs = PyDict_New();
  1004. if (!kwargs) {
  1005. Py_DECREF(textobj);
  1006. return -1;
  1007. }
  1008. PyDict_SetItemString(kwargs, "text", textobj);
  1009. Py_DECREF(textobj);
  1010. if (Tokenizer_emit_kwargs(self, Text, kwargs))
  1011. return -1;
  1012. if (Tokenizer_emit(self, HTMLEntityEnd))
  1013. return -1;
  1014. return 0;
  1015. }
  1016. /*
  1017. Parse an HTML entity at the head of the wikicode string.
  1018. */
  1019. static int Tokenizer_parse_entity(Tokenizer* self)
  1020. {
  1021. Py_ssize_t reset = self->head;
  1022. PyObject *tokenlist;
  1023. if (Tokenizer_check_route(self, LC_HTML_ENTITY) < 0)
  1024. goto on_bad_route;
  1025. if (Tokenizer_push(self, LC_HTML_ENTITY))
  1026. return -1;
  1027. if (Tokenizer_really_parse_entity(self))
  1028. return -1;
  1029. if (BAD_ROUTE) {
  1030. on_bad_route:
  1031. RESET_ROUTE();
  1032. self->head = reset;
  1033. if (Tokenizer_emit_char(self, '&'))
  1034. return -1;
  1035. return 0;
  1036. }
  1037. tokenlist = Tokenizer_pop(self);
  1038. if (!tokenlist)
  1039. return -1;
  1040. if (Tokenizer_emit_all(self, tokenlist)) {
  1041. Py_DECREF(tokenlist);
  1042. return -1;
  1043. }
  1044. Py_DECREF(tokenlist);
  1045. return 0;
  1046. }
  1047. /*
  1048. Parse an HTML comment at the head of the wikicode string.
  1049. */
  1050. static int Tokenizer_parse_comment(Tokenizer* self)
  1051. {
  1052. Py_ssize_t reset = self->head + 3;
  1053. PyObject *comment;
  1054. Unicode this;
  1055. self->head += 4;
  1056. if (Tokenizer_push(self, 0))
  1057. return -1;
  1058. while (1) {
  1059. this = Tokenizer_read(self, 0);
  1060. if (!this) {
  1061. comment = Tokenizer_pop(self);
  1062. Py_XDECREF(comment);
  1063. self->head = reset;
  1064. return Tokenizer_emit_text(self, "<!--");
  1065. }
  1066. if (this == '-' && Tokenizer_read(self, 1) == this &&
  1067. Tokenizer_read(self, 2) == '>') {
  1068. if (Tokenizer_emit_first(self, CommentStart))
  1069. return -1;
  1070. if (Tokenizer_emit(self, CommentEnd))
  1071. return -1;
  1072. comment = Tokenizer_pop(self);
  1073. if (!comment)
  1074. return -1;
  1075. if (Tokenizer_emit_all(self, comment))
  1076. return -1;
  1077. Py_DECREF(comment);
  1078. self->head += 2;
  1079. if (self->topstack->context & LC_FAIL_NEXT) {
  1080. /* _verify_safe() sets this flag while parsing a template or
  1081. link when it encounters what might be a comment -- we must
  1082. unset it to let _verify_safe() know it was correct: */
  1083. self->topstack->context ^= LC_FAIL_NEXT;
  1084. }
  1085. return 0;
  1086. }
  1087. if (Tokenizer_emit_char(self, this))
  1088. return -1;
  1089. self->head++;
  1090. }
  1091. }
  1092. /*
  1093. Write a pending tag attribute from data to the stack.
  1094. */
  1095. static int Tokenizer_push_tag_buffer(Tokenizer* self, TagData* data)
  1096. {
  1097. PyObject *tokens, *kwargs, *tmp, *pad_first, *pad_before_eq, *pad_after_eq;
  1098. if (data->context & TAG_QUOTED) {
  1099. kwargs = PyDict_New();
  1100. if (!kwargs)
  1101. return -1;
  1102. tmp = PyUnicode_FROM_SINGLE(data->quoter);
  1103. if (!tmp)
  1104. return -1;
  1105. PyDict_SetItemString(kwargs, "char", tmp);
  1106. Py_DECREF(tmp);
  1107. if (Tokenizer_emit_first_kwargs(self, TagAttrQuote, kwargs))
  1108. return -1;
  1109. tokens = Tokenizer_pop(self);
  1110. if (!tokens)
  1111. return -1;
  1112. if (Tokenizer_emit_all(self, tokens)) {
  1113. Py_DECREF(tokens);
  1114. return -1;
  1115. }
  1116. Py_DECREF(tokens);
  1117. }
  1118. pad_first = Textbuffer_render(data->pad_first);
  1119. pad_before_eq = Textbuffer_render(data->pad_before_eq);
  1120. pad_after_eq = Textbuffer_render(data->pad_after_eq);
  1121. if (!pad_first || !pad_before_eq || !pad_after_eq)
  1122. return -1;
  1123. kwargs = PyDict_New();
  1124. if (!kwargs)
  1125. return -1;
  1126. PyDict_SetItemString(kwargs, "pad_first", pad_first);
  1127. PyDict_SetItemString(kwargs, "pad_before_eq", pad_before_eq);
  1128. PyDict_SetItemString(kwargs, "pad_after_eq", pad_after_eq);
  1129. Py_DECREF(pad_first);
  1130. Py_DECREF(pad_before_eq);
  1131. Py_DECREF(pad_after_eq);
  1132. if (Tokenizer_emit_first_kwargs(self, TagAttrStart, kwargs))
  1133. return -1;
  1134. tokens = Tokenizer_pop(self);
  1135. if (!tokens)
  1136. return -1;
  1137. if (Tokenizer_emit_all(self, tokens)) {
  1138. Py_DECREF(tokens);
  1139. return -1;
  1140. }
  1141. Py_DECREF(tokens);
  1142. if (TagData_reset_buffers(data))
  1143. return -1;
  1144. return 0;
  1145. }
  1146. /*
  1147. Handle whitespace inside of an HTML open tag.
  1148. */
  1149. static int Tokenizer_handle_tag_space(
  1150. Tokenizer* self, TagData* data, Unicode text)
  1151. {
  1152. uint64_t ctx = data->context;
  1153. uint64_t end_of_value = (ctx & TAG_ATTR_VALUE &&
  1154. !(ctx & (TAG_QUOTED | TAG_NOTE_QUOTE)));
  1155. if (end_of_value || (ctx & TAG_QUOTED && ctx & TAG_NOTE_SPACE)) {
  1156. if (Tokenizer_push_tag_buffer(self, data))
  1157. return -1;
  1158. data->context = TAG_ATTR_READY;
  1159. }
  1160. else if (ctx & TAG_NOTE_SPACE)
  1161. data->context = TAG_ATTR_READY;
  1162. else if (ctx & TAG_ATTR_NAME) {
  1163. data->context |= TAG_NOTE_EQUALS;
  1164. if (Textbuffer_write(data->pad_before_eq, text))
  1165. return -1;
  1166. }
  1167. if (ctx & TAG_QUOTED && !(ctx & TAG_NOTE_SPACE)) {
  1168. if (Tokenizer_emit_char(self, text))
  1169. return -1;
  1170. }
  1171. else if (data->context & TAG_ATTR_READY)
  1172. return Textbuffer_write(data->pad_first, text);
  1173. else if (data->context & TAG_ATTR_VALUE)
  1174. return Textbuffer_write(data->pad_after_eq, text);
  1175. return 0;
  1176. }
  1177. /*
  1178. Handle regular text inside of an HTML open tag.
  1179. */
  1180. static int Tokenizer_handle_tag_text(Tokenizer* self, Unicode text)
  1181. {
  1182. Unicode next = Tokenizer_read(self, 1);
  1183. if (!is_marker(text) || !Tokenizer_CAN_RECURSE(self))
  1184. return Tokenizer_emit_char(self, text);
  1185. else if (text == next && next == '{')
  1186. return Tokenizer_parse_template_or_argument(self);
  1187. else if (text == next && next == '[')
  1188. return Tokenizer_parse_wikilink(self);
  1189. else if (text == '<')
  1190. return Tokenizer_parse_tag(self);
  1191. return Tokenizer_emit_char(self, text);
  1192. }
  1193. /*
  1194. Handle all sorts of text data inside of an HTML open tag.
  1195. */
  1196. static int Tokenizer_handle_tag_data(
  1197. Tokenizer* self, TagData* data, Unicode chunk)
  1198. {
  1199. PyObject *trash;
  1200. int first_time, escaped;
  1201. if (data->context & TAG_NAME) {
  1202. first_time = !(data->context & TAG_NOTE_SPACE);
  1203. if (is_marker(chunk) || (Py_UNICODE_ISSPACE(chunk) && first_time)) {
  1204. // Tags must start with text, not spaces
  1205. Tokenizer_fail_route(self);
  1206. return 0;
  1207. }
  1208. else if (first_time)
  1209. data->context |= TAG_NOTE_SPACE;
  1210. else if (Py_UNICODE_ISSPACE(chunk)) {
  1211. data->context = TAG_ATTR_READY;
  1212. return Tokenizer_handle_tag_space(self, data, chunk);
  1213. }
  1214. }
  1215. else if (Py_UNICODE_ISSPACE(chunk))
  1216. return Tokenizer_handle_tag_space(self, data, chunk);
  1217. else if (data->context & TAG_NOTE_SPACE) {
  1218. if (data->context & TAG_QUOTED) {
  1219. data->context = TAG_ATTR_VALUE;
  1220. Tokenizer_memoize_bad_route(self);
  1221. trash = Tokenizer_pop(self);
  1222. Py_XDECREF(trash);
  1223. self->head = data->reset - 1; // Will be auto-incremented
  1224. }
  1225. else
  1226. Tokenizer_fail_route(self);
  1227. return 0;
  1228. }
  1229. else if (data->context & TAG_ATTR_READY) {
  1230. data->context = TAG_ATTR_NAME;
  1231. if (Tokenizer_push(self, LC_TAG_ATTR))
  1232. return -1;
  1233. }
  1234. else if (data->context & TAG_ATTR_NAME) {
  1235. if (chunk == '=') {
  1236. data->context = TAG_ATTR_VALUE | TAG_NOTE_QUOTE;
  1237. if (Tokenizer_emit(self, TagAttrEquals))
  1238. return -1;
  1239. return 0;
  1240. }
  1241. if (data->context & TAG_NOTE_EQUALS) {
  1242. if (Tokenizer_push_tag_buffer(self, data))
  1243. return -1;
  1244. data->context = TAG_ATTR_NAME;
  1245. if (Tokenizer_push(self, LC_TAG_ATTR))
  1246. return -1;
  1247. }
  1248. }
  1249. else { // data->context & TAG_ATTR_VALUE assured
  1250. escaped = (Tokenizer_read_backwards(self, 1) == '\\' &&
  1251. Tokenizer_read_backwards(self, 2) != '\\');
  1252. if (data->context & TAG_NOTE_QUOTE) {
  1253. data->context ^= TAG_NOTE_QUOTE;
  1254. if ((chunk == '"' || chunk == '\'') && !escaped) {
  1255. data->context |= TAG_QUOTED;
  1256. data->quoter = chunk;
  1257. data->reset = self->head;
  1258. if (Tokenizer_check_route(self, self->topstack->context) < 0) {
  1259. RESET_ROUTE();
  1260. data->context = TAG_ATTR_VALUE;
  1261. self->head--;
  1262. }
  1263. else if (Tokenizer_push(self, self->topstack->context))
  1264. return -1;
  1265. return 0;
  1266. }
  1267. }
  1268. else if (data->context & TAG_QUOTED) {
  1269. if (chunk == data->quoter && !escaped) {
  1270. data->context |= TAG_NOTE_SPACE;
  1271. return 0;
  1272. }
  1273. }
  1274. }
  1275. return Tokenizer_handle_tag_text(self, chunk);
  1276. }
  1277. /*
  1278. Handle the closing of a open tag (<foo>).
  1279. */
  1280. static int
  1281. Tokenizer_handle_tag_close_open(Tokenizer* self, TagData* data, PyObject* cls)
  1282. {
  1283. PyObject *padding, *kwargs;
  1284. if (data->context & (TAG_ATTR_NAME | TAG_ATTR_VALUE)) {
  1285. if (Tokenizer_push_tag_buffer(self, data))
  1286. return -1;
  1287. }
  1288. padding = Textbuffer_render(data->pad_first);
  1289. if (!padding)
  1290. return -1;
  1291. kwargs = PyDict_New();
  1292. if (!kwargs) {
  1293. Py_DECREF(padding);
  1294. return -1;
  1295. }
  1296. PyDict_SetItemString(kwargs, "padding", padding);
  1297. Py_DECREF(padding);
  1298. if (Tokenizer_emit_kwargs(self, cls, kwargs))
  1299. return -1;
  1300. self->head++;
  1301. return 0;
  1302. }
  1303. /*
  1304. Handle the opening of a closing tag (</foo>).
  1305. */
  1306. static int Tokenizer_handle_tag_open_close(Tokenizer* self)
  1307. {
  1308. if (Tokenizer_emit(self, TagOpenClose))
  1309. return -1;
  1310. if (Tokenizer_push(self, LC_TAG_CLOSE))
  1311. return -1;
  1312. self->head++;
  1313. return 0;
  1314. }
  1315. /*
  1316. Handle the ending of a closing tag (</foo>).
  1317. */
  1318. static PyObject* Tokenizer_handle_tag_close_close(Tokenizer* self)
  1319. {
  1320. PyObject *closing, *first, *so, *sc;
  1321. int valid = 1;
  1322. closing = Tokenizer_pop(self);
  1323. if (!closing)
  1324. return NULL;
  1325. if (PyList_GET_SIZE(closing) != 1)
  1326. valid = 0;
  1327. else {
  1328. first = PyList_GET_ITEM(closing, 0);
  1329. switch (PyObject_IsInstance(first, Text)) {
  1330. case 0:
  1331. valid = 0;
  1332. break;
  1333. case 1: {
  1334. so = strip_tag_name(first, 1);
  1335. sc = strip_tag_name(
  1336. PyList_GET_ITEM(self->topstack->stack, 1), 1);
  1337. if (so && sc) {
  1338. if (PyUnicode_Compare(so, sc))
  1339. valid = 0;
  1340. Py_DECREF(so);
  1341. Py_DECREF(sc);
  1342. break;
  1343. }
  1344. Py_XDECREF(so);
  1345. Py_XDECREF(sc);
  1346. }
  1347. case -1:
  1348. Py_DECREF(closing);
  1349. return NULL;
  1350. }
  1351. }
  1352. if (!valid) {
  1353. Py_DECREF(closing);
  1354. return Tokenizer_fail_route(self);
  1355. }
  1356. if (Tokenizer_emit_all(self, closing)) {
  1357. Py_DECREF(closing);
  1358. return NULL;
  1359. }
  1360. Py_DECREF(closing);
  1361. if (Tokenizer_emit(self, TagCloseClose))
  1362. return NULL;
  1363. return Tokenizer_pop(self);
  1364. }
  1365. /*
  1366. Handle the body of an HTML tag that is parser-blacklisted.
  1367. */
  1368. static PyObject* Tokenizer_handle_blacklisted_tag(Tokenizer* self)
  1369. {
  1370. Textbuffer* buffer;
  1371. PyObject *buf_tmp, *end_tag, *start_tag;
  1372. Unicode this, next;
  1373. Py_ssize_t reset;
  1374. int cmp;
  1375. while (1) {
  1376. this = Tokenizer_read(self, 0);
  1377. next = Tokenizer_read(self, 1);
  1378. if (!this)
  1379. return Tokenizer_fail_route(self);
  1380. else if (this == '<' && next == '/') {
  1381. self->head += 2;
  1382. reset = self->head - 1;
  1383. buffer = Textbuffer_new(&self->text);
  1384. if (!buffer)
  1385. return NULL;
  1386. while ((this = Tokenizer_read(self, 0)), 1) {
  1387. if (this == '>') {
  1388. buf_tmp = Textbuffer_render(buffer);
  1389. if (!buf_tmp)
  1390. return NULL;
  1391. end_tag = strip_tag_name(buf_tmp, 0);
  1392. Py_DECREF(buf_tmp);
  1393. if (!end_tag)
  1394. return NULL;
  1395. start_tag = strip_tag_name(
  1396. PyList_GET_ITEM(self->topstack->stack, 1), 1);
  1397. if (!start_tag)
  1398. return NULL;
  1399. cmp = PyUnicode_Compare(start_tag, end_tag);
  1400. Py_DECREF(end_tag);
  1401. Py_DECREF(start_tag);
  1402. if (cmp)
  1403. goto no_matching_end;
  1404. if (Tokenizer_emit(self, TagOpenClose))
  1405. return NULL;
  1406. if (Tokenizer_emit_textbuffer(self, buffer))
  1407. return NULL;
  1408. if (Tokenizer_emit(self, TagCloseClose))
  1409. return NULL;
  1410. return Tokenizer_pop(self);
  1411. }
  1412. if (!this || this == '\n') {
  1413. no_matching_end:
  1414. Textbuffer_dealloc(buffer);
  1415. self->head = reset;
  1416. if (Tokenizer_emit_text(self, "</"))
  1417. return NULL;
  1418. break;
  1419. }
  1420. Textbuffer_write(buffer, this);
  1421. self->head++;
  1422. }
  1423. }
  1424. else if (this == '&') {
  1425. if (Tokenizer_parse_entity(self))
  1426. return NULL;
  1427. }
  1428. else if (Tokenizer_emit_char(self, this))
  1429. return NULL;
  1430. self->head++;
  1431. }
  1432. }
  1433. /*
  1434. Handle the end of an implicitly closing single-only HTML tag.
  1435. */
  1436. static PyObject* Tokenizer_handle_single_only_tag_end(Tokenizer* self)
  1437. {
  1438. PyObject *top, *padding, *kwargs;
  1439. top = PyObject_CallMethod(self->topstack->stack, "pop", NULL);
  1440. if (!top)
  1441. return NULL;
  1442. padding = PyObject_GetAttrString(top, "padding");
  1443. Py_DECREF(top);
  1444. if (!padding)
  1445. return NULL;
  1446. kwargs = PyDict_New();
  1447. if (!kwargs) {
  1448. Py_DECREF(padding);
  1449. return NULL;
  1450. }
  1451. PyDict_SetItemString(kwargs, "padding", padding);
  1452. PyDict_SetItemString(kwargs, "implicit", Py_True);
  1453. Py_DECREF(padding);
  1454. if (Tokenizer_emit_kwargs(self, TagCloseSelfclose, kwargs))
  1455. return NULL;
  1456. self->head--; // Offset displacement done by handle_tag_close_open
  1457. return Tokenizer_pop(self);
  1458. }
  1459. /*
  1460. Handle the stream end when inside a single-supporting HTML tag.
  1461. */
  1462. static PyObject* Tokenizer_handle_single_tag_end(Tokenizer* self)
  1463. {
  1464. PyObject *token = 0, *padding, *kwargs;
  1465. Py_ssize_t len, index;
  1466. int depth = 1, is_instance;
  1467. len = PyList_GET_SIZE(self->topstack->stack);
  1468. for (index = 2; index < len; index++) {
  1469. token = PyList_GET_ITEM(self->topstack->stack, index);
  1470. is_instance = PyObject_IsInstance(token, TagOpenOpen);
  1471. if (is_instance == -1)
  1472. return NULL;
  1473. else if (is_instance == 1)
  1474. depth++;
  1475. is_instance = PyObject_IsInstance(token, TagCloseOpen);
  1476. if (is_instance == -1)
  1477. return NULL;
  1478. else if (is_instance == 1) {
  1479. depth--;
  1480. if (depth == 0)
  1481. break;
  1482. }
  1483. is_instance = PyObject_IsInstance(token, TagCloseSelfclose);
  1484. if (is_instance == -1)
  1485. return NULL;
  1486. else if (is_instance == 1) {
  1487. depth--;
  1488. if (depth == 0) // Should never happen
  1489. return NULL;
  1490. }
  1491. }
  1492. if (!token || depth > 0)
  1493. return NULL;
  1494. padding = PyObject_GetAttrString(token, "padding");
  1495. if (!padding)
  1496. return NULL;
  1497. kwargs = PyDict_New();
  1498. if (!kwargs) {
  1499. Py_DECREF(padding);
  1500. return NULL;
  1501. }
  1502. PyDict_SetItemString(kwargs, "padding", padding);
  1503. PyDict_SetItemString(kwargs, "implicit", Py_True);
  1504. Py_DECREF(padding);
  1505. token = PyObject_Call(TagCloseSelfclose, NOARGS, kwargs);
  1506. Py_DECREF(kwargs);
  1507. if (!token)
  1508. return NULL;
  1509. if (PyList_SetItem(self->topstack->stack, index, token)) {
  1510. Py_DECREF(token);
  1511. return NULL;
  1512. }
  1513. return Tokenizer_pop(self);
  1514. }
  1515. /*
  1516. Actually parse an HTML tag, starting with the open (<foo>).
  1517. */
  1518. static PyObject* Tokenizer_really_parse_tag(Tokenizer* self)
  1519. {
  1520. TagData *data = TagData_new(&self->text);
  1521. PyObject *token, *text, *trash;
  1522. Unicode this, next;
  1523. int can_exit;
  1524. if (!data)
  1525. return NULL;
  1526. if (Tokenizer_check_route(self, LC_TAG_OPEN) < 0)
  1527. return NULL;
  1528. if (Tokenizer_push(self, LC_TAG_OPEN)) {
  1529. TagData_dealloc(data);
  1530. return NULL;
  1531. }
  1532. if (Tokenizer_emit(self, TagOpenOpen)) {
  1533. TagData_dealloc(data);
  1534. return NULL;
  1535. }
  1536. while (1) {
  1537. this = Tokenizer_read(self, 0);
  1538. next = Tokenizer_read(self, 1);
  1539. can_exit = (!(data->context & (TAG_QUOTED | TAG_NAME)) ||
  1540. data->context & TAG_NOTE_SPACE);
  1541. if (!this) {
  1542. if (self->topstack->context & LC_TAG_ATTR) {
  1543. if (data->context & TAG_QUOTED) {
  1544. // Unclosed attribute quote: reset, don't die
  1545. data->context = TAG_ATTR_VALUE;
  1546. Tokenizer_memoize_bad_route(self);
  1547. trash = Tokenizer_pop(self);
  1548. Py_XDECREF(trash);
  1549. self->head = data->reset;
  1550. continue;
  1551. }
  1552. trash = Tokenizer_pop(self);
  1553. Py_XDECREF(trash);
  1554. }
  1555. TagData_dealloc(data);
  1556. return Tokenizer_fail_route(self);
  1557. }
  1558. else if (this == '>' && can_exit) {
  1559. if (Tokenizer_handle_tag_close_open(self, data, TagCloseOpen)) {
  1560. TagData_dealloc(data);
  1561. return NULL;
  1562. }
  1563. TagData_dealloc(data);
  1564. self->topstack->context = LC_TAG_BODY;
  1565. token = PyList_GET_ITEM(self->topstack->stack, 1);
  1566. text = PyObject_GetAttrString(token, "text");
  1567. if (!text)
  1568. return NULL;
  1569. if (is_single_only(text)) {
  1570. Py_DECREF(text);
  1571. return Tokenizer_handle_single_only_tag_end(self);
  1572. }
  1573. if (is_parsable(text)) {
  1574. Py_DECREF(text);
  1575. return Tokenizer_parse(self, 0, 0);
  1576. }
  1577. Py_DECREF(text);
  1578. return Tokenizer_handle_blacklisted_tag(self);
  1579. }
  1580. else if (this == '/' && next == '>' && can_exit) {
  1581. if (Tokenizer_handle_tag_close_open(self, data,
  1582. TagCloseSelfclose)) {
  1583. TagData_dealloc(data);
  1584. return NULL;
  1585. }
  1586. TagData_dealloc(data);
  1587. return Tokenizer_pop(self);
  1588. }
  1589. else {
  1590. if (Tokenizer_handle_tag_data(self, data, this) || BAD_ROUTE) {
  1591. TagData_dealloc(data);
  1592. return NULL;
  1593. }
  1594. }
  1595. self->head++;
  1596. }
  1597. }
  1598. /*
  1599. Handle the (possible) start of an implicitly closing single tag.
  1600. */
  1601. static int Tokenizer_handle_invalid_tag_start(Tokenizer* self)
  1602. {
  1603. Py_ssize_t reset = self->head + 1, pos = 0;
  1604. Textbuffer* buf;
  1605. PyObject *name, *tag;
  1606. Unicode this;
  1607. self->head += 2;
  1608. buf = Textbuffer_new(&self->text);
  1609. if (!buf)
  1610. return -1;
  1611. while (1) {
  1612. this = Tokenizer_read(self, pos);
  1613. if (Py_UNICODE_ISSPACE(this) || is_marker(this)) {
  1614. name = Textbuffer_render(buf);
  1615. if (!name) {
  1616. Textbuffer_dealloc(buf);
  1617. return -1;
  1618. }
  1619. if (!is_single_only(name))
  1620. FAIL_ROUTE(0);
  1621. Py_DECREF(name);
  1622. break;
  1623. }
  1624. Textbuffer_write(buf, this);
  1625. pos++;
  1626. }
  1627. Textbuffer_dealloc(buf);
  1628. if (!BAD_ROUTE)
  1629. tag = Tokenizer_really_parse_tag(self);
  1630. if (BAD_ROUTE) {
  1631. RESET_ROUTE();
  1632. self->head = reset;
  1633. return Tokenizer_emit_text(self, "</");
  1634. }
  1635. if (!tag)
  1636. return -1;
  1637. // Set invalid=True flag of TagOpenOpen
  1638. if (PyObject_SetAttrString(PyList_GET_ITEM(tag, 0), "invalid", Py_True))
  1639. return -1;
  1640. if (Tokenizer_emit_all(self, tag)) {
  1641. Py_DECREF(tag);
  1642. return -1;
  1643. }
  1644. Py_DECREF(tag);
  1645. return 0;
  1646. }
  1647. /*
  1648. Parse an HTML tag at the head of the wikicode string.
  1649. */
  1650. static int Tokenizer_parse_tag(Tokenizer* self)
  1651. {
  1652. Py_ssize_t reset = self->head;
  1653. PyObject* tag;
  1654. self->head++;
  1655. tag = Tokenizer_really_parse_tag(self);
  1656. if (BAD_ROUTE) {
  1657. RESET_ROUTE();
  1658. self->head = reset;
  1659. return Tokenizer_emit_char(self, '<');
  1660. }
  1661. if (!tag) {
  1662. return -1;
  1663. }
  1664. if (Tokenizer_emit_all(self, tag)) {
  1665. Py_DECREF(tag);
  1666. return -1;
  1667. }
  1668. Py_DECREF(tag);
  1669. return 0;
  1670. }
  1671. /*
  1672. Write the body of a tag and the tokens that should surround it.
  1673. */
  1674. static int Tokenizer_emit_style_tag(Tokenizer* self, const char* tag,
  1675. const char* ticks, PyObject* body)
  1676. {
  1677. PyObject *markup, *kwargs;
  1678. markup = PyUnicode_FromString(ticks);
  1679. if (!markup)
  1680. return -1;
  1681. kwargs = PyDict_New();
  1682. if (!kwargs) {
  1683. Py_DECREF(markup);
  1684. return -1;
  1685. }
  1686. PyDict_SetItemString(kwargs, "wiki_markup", markup);
  1687. Py_DECREF(markup);
  1688. if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
  1689. return -1;
  1690. if (Tokenizer_emit_text(self, tag))
  1691. return -1;
  1692. if (Tokenizer_emit(self, TagCloseOpen))
  1693. return -1;
  1694. if (Tokenizer_emit_all(self, body))
  1695. return -1;
  1696. Py_DECREF(body);
  1697. if (Tokenizer_emit(self, TagOpenClose))
  1698. return -1;
  1699. if (Tokenizer_emit_text(self, tag))
  1700. return -1;
  1701. if (Tokenizer_emit(self, TagCloseClose))
  1702. return -1;
  1703. return 0;
  1704. }
  1705. /*
  1706. Parse wiki-style italics.
  1707. */
  1708. static int Tokenizer_parse_italics(Tokenizer* self)
  1709. {
  1710. Py_ssize_t reset = self->head;
  1711. uint64_t context;
  1712. PyObject *stack;
  1713. stack = Tokenizer_parse(self, LC_STYLE_ITALICS, 1);
  1714. if (BAD_ROUTE) {
  1715. RESET_ROUTE();
  1716. self->head = reset;
  1717. if (BAD_ROUTE_CONTEXT & LC_STYLE_PASS_AGAIN) {
  1718. context = LC_STYLE_ITALICS | LC_STYLE_SECOND_PASS;
  1719. stack = Tokenizer_parse(self, context, 1);
  1720. if (BAD_ROUTE) {
  1721. RESET_ROUTE();
  1722. self->head = reset;
  1723. return Tokenizer_emit_text(self, "''");
  1724. }
  1725. }
  1726. else
  1727. return Tokenizer_emit_text(self, "''");
  1728. }
  1729. if (!stack)
  1730. return -1;
  1731. return Tokenizer_emit_style_tag(self, "i", "''", stack);
  1732. }
  1733. /*
  1734. Parse wiki-style bold.
  1735. */
  1736. static int Tokenizer_parse_bold(Tokenizer* self)
  1737. {
  1738. Py_ssize_t reset = self->head;
  1739. PyObject *stack;
  1740. stack = Tokenizer_parse(self, LC_STYLE_BOLD, 1);
  1741. if (BAD_ROUTE) {
  1742. RESET_ROUTE();
  1743. self->head = reset;
  1744. if (self->topstack->context & LC_STYLE_SECOND_PASS)
  1745. return Tokenizer_emit_char(self, '\'') ? -1 : 1;
  1746. if (self->topstack->context & LC_STYLE_ITALICS) {
  1747. self->topstack->context |= LC_STYLE_PASS_AGAIN;
  1748. return Tokenizer_emit_text(self, "'''");
  1749. }
  1750. if (Tokenizer_emit_char(self, '\''))
  1751. return -1;
  1752. return Tokenizer_parse_italics(self);
  1753. }
  1754. if (!stack)
  1755. return -1;
  1756. return Tokenizer_emit_style_tag(self, "b", "'''", stack);
  1757. }
  1758. /*
  1759. Parse wiki-style italics and bold together (i.e., five ticks).
  1760. */
  1761. static int Tokenizer_parse_italics_and_bold(Tokenizer* self)
  1762. {
  1763. Py_ssize_t reset = self->head;
  1764. PyObject *stack, *stack2;
  1765. stack = Tokenizer_parse(self, LC_STYLE_BOLD, 1);
  1766. if (BAD_ROUTE) {
  1767. RESET_ROUTE();
  1768. self->head = reset;
  1769. stack = Tokenizer_parse(self, LC_STYLE_ITALICS, 1);
  1770. if (BAD_ROUTE) {
  1771. RESET_ROUTE();
  1772. self->head = reset;
  1773. return Tokenizer_emit_text(self, "'''''");
  1774. }
  1775. if (!stack)
  1776. return -1;
  1777. reset = self->head;
  1778. stack2 = Tokenizer_parse(self, LC_STYLE_BOLD, 1);
  1779. if (BAD_ROUTE) {
  1780. RESET_ROUTE();
  1781. self->head = reset;
  1782. if (Tokenizer_emit_text(self, "'''"))
  1783. return -1;
  1784. return Tokenizer_emit_style_tag(self, "i", "''", stack);
  1785. }
  1786. if (!stack2)
  1787. return -1;
  1788. if (Tokenizer_push(self, 0))
  1789. return -1;
  1790. if (Tokenizer_emit_style_tag(self, "i", "''", stack))
  1791. return -1;
  1792. if (Tokenizer_emit_all(self, stack2))
  1793. return -1;
  1794. Py_DECREF(stack2);
  1795. stack2 = Tokenizer_pop(self);
  1796. if (!stack2)
  1797. return -1;
  1798. return Tokenizer_emit_style_tag(self, "b", "'''", stack2);
  1799. }
  1800. if (!stack)
  1801. return -1;
  1802. reset = self->head;
  1803. stack2 = Tokenizer_parse(self, LC_STYLE_ITALICS, 1);
  1804. if (BAD_ROUTE) {
  1805. RESET_ROUTE();
  1806. self->head = reset;
  1807. if (Tokenizer_emit_text(self, "''"))
  1808. return -1;
  1809. return Tokenizer_emit_style_tag(self, "b", "'''", stack);
  1810. }
  1811. if (!stack2)
  1812. return -1;
  1813. if (Tokenizer_push(self, 0))
  1814. return -1;
  1815. if (Tokenizer_emit_style_tag(self, "b", "'''", stack))
  1816. return -1;
  1817. if (Tokenizer_emit_all(self, stack2))
  1818. return -1;
  1819. Py_DECREF(stack2);
  1820. stack2 = Tokenizer_pop(self);
  1821. if (!stack2)
  1822. return -1;
  1823. return Tokenizer_emit_style_tag(self, "i", "''", stack2);
  1824. }
  1825. /*
  1826. Parse wiki-style formatting (''/''' for italics/bold).
  1827. */
  1828. static PyObject* Tokenizer_parse_style(Tokenizer* self)
  1829. {
  1830. uint64_t context = self->topstack->context, ticks = 2, i;
  1831. self->head += 2;
  1832. while (Tokenizer_read(self, 0) == '\'') {
  1833. self->head++;
  1834. ticks++;
  1835. }
  1836. if (ticks > 5) {
  1837. for (i = 0; i < ticks - 5; i++) {
  1838. if (Tokenizer_emit_char(self, '\''))
  1839. return NULL;
  1840. }
  1841. ticks = 5;
  1842. }
  1843. else if (ticks == 4) {
  1844. if (Tokenizer_emit_char(self, '\''))
  1845. return NULL;
  1846. ticks = 3;
  1847. }
  1848. if ((context & LC_STYLE_ITALICS && (ticks == 2 || ticks == 5)) ||
  1849. (context & LC_STYLE_BOLD && (ticks == 3 || ticks == 5))) {
  1850. if (ticks == 5)
  1851. self->head -= context & LC_STYLE_ITALICS ? 3 : 2;
  1852. return Tokenizer_pop(self);
  1853. }
  1854. if (!Tokenizer_CAN_RECURSE(self)) {
  1855. if (ticks == 3) {
  1856. if (context & LC_STYLE_SECOND_PASS) {
  1857. if (Tokenizer_emit_char(self, '\''))
  1858. return NULL;
  1859. return Tokenizer_pop(self);
  1860. }
  1861. if (context & LC_STYLE_ITALICS)
  1862. self->topstack->context |= LC_STYLE_PASS_AGAIN;
  1863. }
  1864. for (i = 0; i < ticks; i++) {
  1865. if (Tokenizer_emit_char(self, '\''))
  1866. return NULL;
  1867. }
  1868. }
  1869. else if (ticks == 2) {
  1870. if (Tokenizer_parse_italics(self))
  1871. return NULL;
  1872. }
  1873. else if (ticks == 3) {
  1874. switch (Tokenizer_parse_bold(self)) {
  1875. case 1:
  1876. return Tokenizer_pop(self);
  1877. case -1:
  1878. return NULL;
  1879. }
  1880. }
  1881. else {
  1882. if (Tokenizer_parse_italics_and_bold(self))
  1883. return NULL;
  1884. }
  1885. self->head--;
  1886. return Py_None;
  1887. }
  1888. /*
  1889. Handle a list marker at the head (#, *, ;, :).
  1890. */
  1891. static int Tokenizer_handle_list_marker(Tokenizer* self)
  1892. {
  1893. PyObject *kwargs, *markup;
  1894. Unicode code = Tokenizer_read(self, 0);
  1895. if (code == ';')
  1896. self->topstack->context |= LC_DLTERM;
  1897. kwargs = PyDict_New();
  1898. if (!kwargs)
  1899. return -1;
  1900. if (!(markup = PyUnicode_FROM_SINGLE(code))) {
  1901. Py_DECREF(kwargs);
  1902. return -1;
  1903. }
  1904. PyDict_SetItemString(kwargs, "wiki_markup", markup);
  1905. Py_DECREF(markup);
  1906. if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
  1907. return -1;
  1908. if (Tokenizer_emit_text(self, GET_HTML_TAG(code)))
  1909. return -1;
  1910. if (Tokenizer_emit(self, TagCloseSelfclose))
  1911. return -1;
  1912. return 0;
  1913. }
  1914. /*
  1915. Handle a wiki-style list (#, *, ;, :).
  1916. */
  1917. static int Tokenizer_handle_list(Tokenizer* self)
  1918. {
  1919. Unicode marker = Tokenizer_read(self, 1);
  1920. if (Tokenizer_handle_list_marker(self))
  1921. return -1;
  1922. while (marker == '#' || marker == '*' || marker == ';' ||
  1923. marker == ':') {
  1924. self->head++;
  1925. if (Tokenizer_handle_list_marker(self))
  1926. return -1;
  1927. marker = Tokenizer_read(self, 1);
  1928. }
  1929. return 0;
  1930. }
  1931. /*
  1932. Handle a wiki-style horizontal rule (----) in the string.
  1933. */
  1934. static int Tokenizer_handle_hr(Tokenizer* self)
  1935. {
  1936. PyObject *markup, *kwargs;
  1937. Textbuffer *buffer = Textbuffer_new(&self->text);
  1938. int i;
  1939. if (!buffer)
  1940. return -1;
  1941. self->head += 3;
  1942. for (i = 0; i < 4; i++) {
  1943. if (Textbuffer_write(buffer, '-'))
  1944. return -1;
  1945. }
  1946. while (Tokenizer_read(self, 1) == '-') {
  1947. if (Textbuffer_write(buffer, '-'))
  1948. return -1;
  1949. self->head++;
  1950. }
  1951. markup = Textbuffer_render(buffer);
  1952. Textbuffer_dealloc(buffer);
  1953. if (!markup)
  1954. return -1;
  1955. kwargs = PyDict_New();
  1956. if (!kwargs)
  1957. return -1;
  1958. PyDict_SetItemString(kwargs, "wiki_markup", markup);
  1959. Py_DECREF(markup);
  1960. if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
  1961. return -1;
  1962. if (Tokenizer_emit_text(self, "hr"))
  1963. return -1;
  1964. if (Tokenizer_emit(self, TagCloseSelfclose))
  1965. return -1;
  1966. return 0;
  1967. }
  1968. /*
  1969. Handle the term in a description list ('foo' in ';foo:bar').
  1970. */
  1971. static int Tokenizer_handle_dl_term(Tokenizer* self)
  1972. {
  1973. self->topstack->context ^= LC_DLTERM;
  1974. if (Tokenizer_read(self, 0) == ':')
  1975. return Tokenizer_handle_list_marker(self);
  1976. return Tokenizer_emit_char(self, '\n');
  1977. }
  1978. /*
  1979. Emit a table tag.
  1980. */
  1981. static int
  1982. Tokenizer_emit_table_tag(Tokenizer* self, const char* open_open_markup,
  1983. const char* tag, PyObject* style, PyObject* padding,
  1984. const char* close_open_markup, PyObject* contents,
  1985. const char* open_close_markup)
  1986. {
  1987. PyObject *open_open_kwargs, *open_open_markup_unicode, *close_open_kwargs,
  1988. *close_open_markup_unicode, *open_close_kwargs,
  1989. *open_close_markup_unicode;
  1990. open_open_kwargs = PyDict_New();
  1991. if (!open_open_kwargs)
  1992. goto fail_decref_all;
  1993. open_open_markup_unicode = PyUnicode_FromString(open_open_markup);
  1994. if (!open_open_markup_unicode) {
  1995. Py_DECREF(open_open_kwargs);
  1996. goto fail_decref_all;
  1997. }
  1998. PyDict_SetItemString(open_open_kwargs, "wiki_markup",
  1999. open_open_markup_unicode);
  2000. Py_DECREF(open_open_markup_unicode);
  2001. if (Tokenizer_emit_kwargs(self, TagOpenOpen, open_open_kwargs))
  2002. goto fail_decref_all;
  2003. if (Tokenizer_emit_text(self, tag))
  2004. goto fail_decref_all;
  2005. if (style) {
  2006. if (Tokenizer_emit_all(self, style))
  2007. goto fail_decref_all;
  2008. Py_DECREF(style);
  2009. }
  2010. close_open_kwargs = PyDict_New();
  2011. if (!close_open_kwargs)
  2012. goto fail_decref_padding_contents;
  2013. if (close_open_markup && strlen(close_open_markup) != 0) {
  2014. close_open_markup_unicode = PyUnicode_FromString(close_open_markup);
  2015. if (!close_open_markup_unicode) {
  2016. Py_DECREF(close_open_kwargs);
  2017. goto fail_decref_padding_contents;
  2018. }
  2019. PyDict_SetItemString(close_open_kwargs, "wiki_markup",
  2020. close_open_markup_unicode);
  2021. Py_DECREF(close_open_markup_unicode);
  2022. }
  2023. PyDict_SetItemString(close_open_kwargs, "padding", padding);
  2024. Py_DECREF(padding);
  2025. if (Tokenizer_emit_kwargs(self, TagCloseOpen, close_open_kwargs))
  2026. goto fail_decref_contents;
  2027. if (contents) {
  2028. if (Tokenizer_emit_all(self, contents))
  2029. goto fail_decref_contents;
  2030. Py_DECREF(contents);
  2031. }
  2032. open_close_kwargs = PyDict_New();
  2033. if (!open_close_kwargs)
  2034. return -1;
  2035. open_close_markup_unicode = PyUnicode_FromString(open_close_markup);
  2036. if (!open_close_markup_unicode) {
  2037. Py_DECREF(open_close_kwargs);
  2038. return -1;
  2039. }
  2040. PyDict_SetItemString(open_close_kwargs, "wiki_markup",
  2041. open_close_markup_unicode);
  2042. Py_DECREF(open_close_markup_unicode);
  2043. if (Tokenizer_emit_kwargs(self, TagOpenClose, open_close_kwargs))
  2044. return -1;
  2045. if (Tokenizer_emit_text(self, tag))
  2046. return -1;
  2047. if (Tokenizer_emit(self, TagCloseClose))
  2048. return -1;
  2049. return 0;
  2050. fail_decref_all:
  2051. Py_XDECREF(style);
  2052. fail_decref_padding_contents:
  2053. Py_DECREF(padding);
  2054. fail_decref_contents:
  2055. Py_DECREF(contents);
  2056. return -1;
  2057. }
  2058. /*
  2059. Handle style attributes for a table until an ending token.
  2060. */
  2061. static PyObject* Tokenizer_handle_table_style(Tokenizer* self, Unicode end_token)
  2062. {
  2063. TagData *data = TagData_new(&self->text);
  2064. PyObject *padding, *trash;
  2065. Unicode this;
  2066. int can_exit;
  2067. if (!data)
  2068. return NULL;
  2069. data->context = TAG_ATTR_READY;
  2070. while (1) {
  2071. this = Tokenizer_read(self, 0);
  2072. can_exit = (!(data->context & TAG_QUOTED) || data->context & TAG_NOTE_SPACE);
  2073. if (this == end_token && can_exit) {
  2074. if (data->context & (TAG_ATTR_NAME | TAG_ATTR_VALUE)) {
  2075. if (Tokenizer_push_tag_buffer(self, data)) {
  2076. TagData_dealloc(data);
  2077. return NULL;
  2078. }
  2079. }
  2080. if (Py_UNICODE_ISSPACE(this))
  2081. Textbuffer_write(data->pad_first, this);
  2082. padding = Textbuffer_render(data->pad_first);
  2083. TagData_dealloc(data);
  2084. if (!padding)
  2085. return NULL;
  2086. return padding;
  2087. }
  2088. else if (!this || this == end_token) {
  2089. if (self->topstack->context & LC_TAG_ATTR) {
  2090. if (data->context & TAG_QUOTED) {
  2091. // Unclosed attribute quote: reset, don't die
  2092. data->context = TAG_ATTR_VALUE;
  2093. Tokenizer_memoize_bad_route(self);
  2094. trash = Tokenizer_pop(self);
  2095. Py_XDECREF(trash);
  2096. self->head = data->reset;
  2097. continue;
  2098. }
  2099. trash = Tokenizer_pop(self);
  2100. Py_XDECREF(trash);
  2101. }
  2102. TagData_dealloc(data);
  2103. return Tokenizer_fail_route(self);
  2104. }
  2105. else {
  2106. if (Tokenizer_handle_tag_data(self, data, this) || BAD_ROUTE) {
  2107. TagData_dealloc(data);
  2108. return NULL;
  2109. }
  2110. }
  2111. self->head++;
  2112. }
  2113. }
  2114. /*
  2115. Parse a wikicode table by starting with the first line.
  2116. */
  2117. static int Tokenizer_parse_table(Tokenizer* self)
  2118. {
  2119. Py_ssize_t reset = self->head;
  2120. PyObject *style, *padding, *trash;
  2121. PyObject *table = NULL;
  2122. StackIdent restore_point;
  2123. self->head += 2;
  2124. if (Tokenizer_check_route(self, LC_TABLE_OPEN) < 0)
  2125. goto on_bad_route;
  2126. if (Tokenizer_push(self, LC_TABLE_OPEN))
  2127. return -1;
  2128. padding = Tokenizer_handle_table_style(self, '\n');
  2129. if (BAD_ROUTE) {
  2130. on_bad_route:
  2131. RESET_ROUTE();
  2132. self->head = reset;
  2133. if (Tokenizer_emit_char(self, '{'))
  2134. return -1;
  2135. return 0;
  2136. }
  2137. if (!padding)
  2138. return -1;
  2139. style = Tokenizer_pop(self);
  2140. if (!style) {
  2141. Py_DECREF(padding);
  2142. return -1;
  2143. }
  2144. self->head++;
  2145. restore_point = self->topstack->ident;
  2146. table = Tokenizer_parse(self, LC_TABLE_OPEN, 1);
  2147. if (BAD_ROUTE) {
  2148. RESET_ROUTE();
  2149. Py_DECREF(padding);
  2150. Py_DECREF(style);
  2151. while (!Tokenizer_IS_CURRENT_STACK(self, restore_point)) {
  2152. Tokenizer_memoize_bad_route(self);
  2153. trash = Tokenizer_pop(self);
  2154. Py_XDECREF(trash);
  2155. }
  2156. self->head = reset;
  2157. if (Tokenizer_emit_char(self, '{'))
  2158. return -1;
  2159. return 0;
  2160. }
  2161. if (!table) {
  2162. Py_DECREF(padding);
  2163. Py_DECREF(style);
  2164. return -1;
  2165. }
  2166. if (Tokenizer_emit_table_tag(self, "{|", "table", style, padding, NULL,
  2167. table, "|}"))
  2168. return -1;
  2169. // Offset displacement done by _parse()
  2170. self->head--;
  2171. return 0;
  2172. }
  2173. /*
  2174. Parse as style until end of the line, then continue.
  2175. */
  2176. static int Tokenizer_handle_table_row(Tokenizer* self)
  2177. {
  2178. PyObject *padding, *style, *row;
  2179. self->head += 2;
  2180. if (!Tokenizer_CAN_RECURSE(self)) {
  2181. if (Tokenizer_emit_text(self, "|-"))
  2182. return -1;
  2183. self->head -= 1;
  2184. return 0;
  2185. }
  2186. if (Tokenizer_check_route(self, LC_TABLE_OPEN | LC_TABLE_ROW_OPEN) < 0)
  2187. return 0;
  2188. if (Tokenizer_push(self, LC_TABLE_OPEN | LC_TABLE_ROW_OPEN))
  2189. return -1;
  2190. padding = Tokenizer_handle_table_style(self, '\n');
  2191. if (BAD_ROUTE)
  2192. return 0;
  2193. if (!padding)
  2194. return -1;
  2195. style = Tokenizer_pop(self);
  2196. if (!style) {
  2197. Py_DECREF(padding);
  2198. return -1;
  2199. }
  2200. // Don't parse the style separator
  2201. self->head++;
  2202. row = Tokenizer_parse(self, LC_TABLE_OPEN | LC_TABLE_ROW_OPEN, 1);
  2203. if (!row) {
  2204. Py_DECREF(padding);
  2205. Py_DECREF(style);
  2206. return -1;
  2207. }
  2208. if (Tokenizer_emit_table_tag(self, "|-", "tr", style, padding, NULL, row, ""))
  2209. return -1;
  2210. // Offset displacement done by _parse()
  2211. self->head--;
  2212. return 0;
  2213. }
  2214. /*
  2215. Parse as normal syntax unless we hit a style marker, then parse style
  2216. as HTML attributes and the remainder as normal syntax.
  2217. */
  2218. static int
  2219. Tokenizer_handle_table_cell(Tokenizer* self, const char *markup,
  2220. const char *tag, uint64_t line_context)
  2221. {
  2222. uint64_t old_context = self->topstack->context;
  2223. uint64_t cell_context;
  2224. Py_ssize_t reset;
  2225. PyObject *padding, *cell, *style = NULL;
  2226. const char *close_open_markup = NULL;
  2227. self->head += strlen(markup);
  2228. reset = self->head;
  2229. if (!Tokenizer_CAN_RECURSE(self)) {
  2230. if (Tokenizer_emit_text(self, markup))
  2231. return -1;
  2232. self->head--;
  2233. return 0;
  2234. }
  2235. cell = Tokenizer_parse(self, LC_TABLE_OPEN | LC_TABLE_CELL_OPEN |
  2236. LC_TABLE_CELL_STYLE | line_context, 1);
  2237. if (!cell)
  2238. return -1;
  2239. cell_context = self->topstack->context;
  2240. self->topstack->context = old_context;
  2241. if (cell_context & LC_TABLE_CELL_STYLE) {
  2242. Py_DECREF(cell);
  2243. self->head = reset;
  2244. if (Tokenizer_push(self, LC_TABLE_OPEN | LC_TABLE_CELL_OPEN |
  2245. line_context))
  2246. return -1;
  2247. padding = Tokenizer_handle_table_style(self, '|');
  2248. if (!padding)
  2249. return -1;
  2250. style = Tokenizer_pop(self);
  2251. if (!style) {
  2252. Py_DECREF(padding);
  2253. return -1;
  2254. }
  2255. // Don't parse the style separator
  2256. self->head++;
  2257. cell = Tokenizer_parse(self, LC_TABLE_OPEN | LC_TABLE_CELL_OPEN |
  2258. line_context, 1);
  2259. if (!cell) {
  2260. Py_DECREF(padding);
  2261. Py_DECREF(style);
  2262. return -1;
  2263. }
  2264. cell_context = self->topstack->context;
  2265. self->topstack->context = old_context;
  2266. }
  2267. else {
  2268. padding = PyUnicode_FromString("");
  2269. if (!padding) {
  2270. Py_DECREF(cell);
  2271. return -1;
  2272. }
  2273. }
  2274. if (style) {
  2275. close_open_markup = "|";
  2276. }
  2277. if (Tokenizer_emit_table_tag(self, markup, tag, style, padding,
  2278. close_open_markup, cell, ""))
  2279. return -1;
  2280. // Keep header/cell line contexts
  2281. self->topstack->context |= cell_context & (LC_TABLE_TH_LINE | LC_TABLE_TD_LINE);
  2282. // Offset displacement done by parse()
  2283. self->head--;
  2284. return 0;
  2285. }
  2286. /*
  2287. Returns the context, stack, and whether to reset the cell for style
  2288. in a tuple.
  2289. */
  2290. static PyObject*
  2291. Tokenizer_handle_table_cell_end(Tokenizer* self, int reset_for_style)
  2292. {
  2293. if (reset_for_style)
  2294. self->topstack->context |= LC_TABLE_CELL_STYLE;
  2295. else
  2296. self->topstack->context &= ~LC_TABLE_CELL_STYLE;
  2297. return Tokenizer_pop_keeping_context(self);
  2298. }
  2299. /*
  2300. Return the stack in order to handle the table row end.
  2301. */
  2302. static PyObject* Tokenizer_handle_table_row_end(Tokenizer* self)
  2303. {
  2304. return Tokenizer_pop(self);
  2305. }
  2306. /*
  2307. Return the stack in order to handle the table end.
  2308. */
  2309. static PyObject* Tokenizer_handle_table_end(Tokenizer* self)
  2310. {
  2311. self->head += 2;
  2312. return Tokenizer_pop(self);
  2313. }
  2314. /*
  2315. Handle the end of the stream of wikitext.
  2316. */
  2317. static PyObject* Tokenizer_handle_end(Tokenizer* self, uint64_t context)
  2318. {
  2319. PyObject *token, *text, *trash;
  2320. int single;
  2321. if (context & AGG_FAIL) {
  2322. if (context & LC_TAG_BODY) {
  2323. token = PyList_GET_ITEM(self->topstack->stack, 1);
  2324. text = PyObject_GetAttrString(token, "text");
  2325. if (!text)
  2326. return NULL;
  2327. single = is_single(text);
  2328. Py_DECREF(text);
  2329. if (single)
  2330. return Tokenizer_handle_single_tag_end(self);
  2331. }
  2332. else {
  2333. if (context & LC_TABLE_CELL_OPEN) {
  2334. trash = Tokenizer_pop(self);
  2335. Py_XDECREF(trash);
  2336. context = self->topstack->context;
  2337. }
  2338. if (context & AGG_DOUBLE) {
  2339. trash = Tokenizer_pop(self);
  2340. Py_XDECREF(trash);
  2341. }
  2342. }
  2343. return Tokenizer_fail_route(self);
  2344. }
  2345. return Tokenizer_pop(self);
  2346. }
  2347. /*
  2348. Make sure we are not trying to write an invalid character. Return 0 if
  2349. everything is safe, or -1 if the route must be failed.
  2350. */
  2351. static int
  2352. Tokenizer_verify_safe(Tokenizer* self, uint64_t context, Unicode data)
  2353. {
  2354. if (context & LC_FAIL_NEXT)
  2355. return -1;
  2356. if (context & LC_WIKILINK_TITLE) {
  2357. if (data == ']' || data == '{') {
  2358. self->topstack->context |= LC_FAIL_NEXT;
  2359. } else if (data == '\n' || data == '[' || data == '}' || data == '>') {
  2360. return -1;
  2361. } else if (data == '<') {
  2362. if (Tokenizer_read(self, 1) == '!')
  2363. self->topstack->context |= LC_FAIL_NEXT;
  2364. else
  2365. return -1;
  2366. }
  2367. return 0;
  2368. }
  2369. if (context & LC_EXT_LINK_TITLE)
  2370. return (data == '\n') ? -1 : 0;
  2371. if (context & LC_TAG_CLOSE)
  2372. return (data == '<') ? -1 : 0;
  2373. if (context & LC_TEMPLATE_NAME) {
  2374. if (data == '{') {
  2375. self->topstack->context |= LC_HAS_TEMPLATE | LC_FAIL_NEXT;
  2376. return 0;
  2377. }
  2378. if (data == '}' || (data == '<' && Tokenizer_read(self, 1) == '!')) {
  2379. self->topstack->context |= LC_FAIL_NEXT;
  2380. return 0;
  2381. }
  2382. if (data == '[' || data == ']' || data == '<' || data == '>') {
  2383. return -1;
  2384. }
  2385. if (data == '|')
  2386. return 0;
  2387. if (context & LC_HAS_TEXT) {
  2388. if (context & LC_FAIL_ON_TEXT) {
  2389. if (!Py_UNICODE_ISSPACE(data))
  2390. return -1;
  2391. }
  2392. else if (data == '\n')
  2393. self->topstack->context |= LC_FAIL_ON_TEXT;
  2394. }
  2395. else if (!Py_UNICODE_ISSPACE(data))
  2396. self->topstack->context |= LC_HAS_TEXT;
  2397. }
  2398. else {
  2399. if (context & LC_FAIL_ON_EQUALS) {
  2400. if (data == '=') {
  2401. return -1;
  2402. }
  2403. }
  2404. else if (context & LC_FAIL_ON_LBRACE) {
  2405. if (data == '{' || (Tokenizer_read_backwards(self, 1) == '{' &&
  2406. Tokenizer_read_backwards(self, 2) == '{')) {
  2407. if (context & LC_TEMPLATE)
  2408. self->topstack->context |= LC_FAIL_ON_EQUALS;
  2409. else
  2410. self->topstack->context |= LC_FAIL_NEXT;
  2411. return 0;
  2412. }
  2413. self->topstack->context ^= LC_FAIL_ON_LBRACE;
  2414. }
  2415. else if (context & LC_FAIL_ON_RBRACE) {
  2416. if (data == '}') {
  2417. self->topstack->context |= LC_FAIL_NEXT;
  2418. return 0;
  2419. }
  2420. self->topstack->context ^= LC_FAIL_ON_RBRACE;
  2421. }
  2422. else if (data == '{')
  2423. self->topstack->context |= LC_FAIL_ON_LBRACE;
  2424. else if (data == '}')
  2425. self->topstack->context |= LC_FAIL_ON_RBRACE;
  2426. }
  2427. return 0;
  2428. }
  2429. /*
  2430. Returns whether the current head has leading whitespace.
  2431. TODO: treat comments and templates as whitespace, allow fail on non-newline spaces.
  2432. */
  2433. static int Tokenizer_has_leading_whitespace(Tokenizer* self)
  2434. {
  2435. int offset = 1;
  2436. Unicode current_character;
  2437. while (1) {
  2438. current_character = Tokenizer_read_backwards(self, offset);
  2439. if (!current_character || current_character == '\n')
  2440. return 1;
  2441. else if (!Py_UNICODE_ISSPACE(current_character))
  2442. return 0;
  2443. offset++;
  2444. }
  2445. }
  2446. /*
  2447. Parse the wikicode string, using context for when to stop. If push is true,
  2448. we will push a new context, otherwise we won't and context will be ignored.
  2449. */
  2450. PyObject* Tokenizer_parse(Tokenizer* self, uint64_t context, int push)
  2451. {
  2452. uint64_t this_context;
  2453. Unicode this, next, next_next, last;
  2454. PyObject* temp;
  2455. if (push) {
  2456. if (Tokenizer_check_route(self, context) < 0)
  2457. return NULL;
  2458. if (Tokenizer_push(self, context))
  2459. return NULL;
  2460. }
  2461. while (1) {
  2462. this = Tokenizer_read(self, 0);
  2463. this_context = self->topstack->context;
  2464. if (this_context & AGG_UNSAFE) {
  2465. if (Tokenizer_verify_safe(self, this_context, this) < 0) {
  2466. if (this_context & AGG_DOUBLE) {
  2467. temp = Tokenizer_pop(self);
  2468. Py_XDECREF(temp);
  2469. }
  2470. return Tokenizer_fail_route(self);
  2471. }
  2472. }
  2473. if (!is_marker(this)) {
  2474. if (Tokenizer_emit_char(self, this))
  2475. return NULL;
  2476. self->head++;
  2477. continue;
  2478. }
  2479. if (!this)
  2480. return Tokenizer_handle_end(self, this_context);
  2481. if (PyErr_CheckSignals())
  2482. return NULL;
  2483. next = Tokenizer_read(self, 1);
  2484. last = Tokenizer_read_backwards(self, 1);
  2485. if (this == next && next == '{') {
  2486. if (Tokenizer_CAN_RECURSE(self)) {
  2487. if (Tokenizer_parse_template_or_argument(self))
  2488. return NULL;
  2489. }
  2490. else if (Tokenizer_emit_char(self, this))
  2491. return NULL;
  2492. }
  2493. else if (this == '|' && this_context & LC_TEMPLATE) {
  2494. if (Tokenizer_handle_template_param(self))
  2495. return NULL;
  2496. }
  2497. else if (this == '=' && this_context & LC_TEMPLATE_PARAM_KEY) {
  2498. if (Tokenizer_handle_template_param_value(self))
  2499. return NULL;
  2500. }
  2501. else if (this == next && next == '}' && this_context & LC_TEMPLATE)
  2502. return Tokenizer_handle_template_end(self);
  2503. else if (this == '|' && this_context & LC_ARGUMENT_NAME) {
  2504. if (Tokenizer_handle_argument_separator(self))
  2505. return NULL;
  2506. }
  2507. else if (this == next && next == '}' && this_context & LC_ARGUMENT) {
  2508. if (Tokenizer_read(self, 2) == '}') {
  2509. return Tokenizer_handle_argument_end(self);
  2510. }
  2511. if (Tokenizer_emit_char(self, this))
  2512. return NULL;
  2513. }
  2514. else if (this == next && next == '[' && Tokenizer_CAN_RECURSE(self)) {
  2515. if (!(this_context & AGG_NO_WIKILINKS)) {
  2516. if (Tokenizer_parse_wikilink(self))
  2517. return NULL;
  2518. }
  2519. else if (Tokenizer_emit_char(self, this))
  2520. return NULL;
  2521. }
  2522. else if (this == '|' && this_context & LC_WIKILINK_TITLE) {
  2523. if (Tokenizer_handle_wikilink_separator(self))
  2524. return NULL;
  2525. }
  2526. else if (this == next && next == ']' && this_context & LC_WIKILINK)
  2527. return Tokenizer_handle_wikilink_end(self);
  2528. else if (this == '[') {
  2529. if (Tokenizer_parse_external_link(self, 1))
  2530. return NULL;
  2531. }
  2532. else if (this == ':' && !is_marker(last)) {
  2533. if (Tokenizer_parse_external_link(self, 0))
  2534. return NULL;
  2535. }
  2536. else if (this == ']' && this_context & LC_EXT_LINK_TITLE)
  2537. return Tokenizer_pop(self);
  2538. else if (this == '=' && !(self->global & GL_HEADING)) {
  2539. if (!last || last == '\n') {
  2540. if (Tokenizer_parse_heading(self))
  2541. return NULL;
  2542. }
  2543. else if (Tokenizer_emit_char(self, this))
  2544. return NULL;
  2545. }
  2546. else if (this == '=' && this_context & LC_HEADING)
  2547. return (PyObject*) Tokenizer_handle_heading_end(self);
  2548. else if (this == '\n' && this_context & LC_HEADING)
  2549. return Tokenizer_fail_route(self);
  2550. else if (this == '&') {
  2551. if (Tokenizer_parse_entity(self))
  2552. return NULL;
  2553. }
  2554. else if (this == '<' && next == '!') {
  2555. next_next = Tokenizer_read(self, 2);
  2556. if (next_next == Tokenizer_read(self, 3) && next_next == '-') {
  2557. if (Tokenizer_parse_comment(self))
  2558. return NULL;
  2559. }
  2560. else if (Tokenizer_emit_char(self, this))
  2561. return NULL;
  2562. }
  2563. else if (this == '<' && next == '/' && Tokenizer_read(self, 2)) {
  2564. if (this_context & LC_TAG_BODY ?
  2565. Tokenizer_handle_tag_open_close(self) :
  2566. Tokenizer_handle_invalid_tag_start(self))
  2567. return NULL;
  2568. }
  2569. else if (this == '<' && !(this_context & LC_TAG_CLOSE)) {
  2570. if (Tokenizer_CAN_RECURSE(self)) {
  2571. if (Tokenizer_parse_tag(self))
  2572. return NULL;
  2573. }
  2574. else if (Tokenizer_emit_char(self, this))
  2575. return NULL;
  2576. }
  2577. else if (this == '>' && this_context & LC_TAG_CLOSE)
  2578. return Tokenizer_handle_tag_close_close(self);
  2579. else if (this == next && next == '\'' && !self->skip_style_tags) {
  2580. temp = Tokenizer_parse_style(self);
  2581. if (temp != Py_None)
  2582. return temp;
  2583. }
  2584. else if ((!last || last == '\n') && (this == '#' || this == '*' || this == ';' || this == ':')) {
  2585. if (Tokenizer_handle_list(self))
  2586. return NULL;
  2587. }
  2588. else if ((!last || last == '\n') && (this == '-' && this == next &&
  2589. this == Tokenizer_read(self, 2) &&
  2590. this == Tokenizer_read(self, 3))) {
  2591. if (Tokenizer_handle_hr(self))
  2592. return NULL;
  2593. }
  2594. else if ((this == '\n' || this == ':') && this_context & LC_DLTERM) {
  2595. if (Tokenizer_handle_dl_term(self))
  2596. return NULL;
  2597. // Kill potential table contexts
  2598. if (this == '\n')
  2599. self->topstack->context &= ~LC_TABLE_CELL_LINE_CONTEXTS;
  2600. }
  2601. // Start of table parsing
  2602. else if (this == '{' && next == '|' && Tokenizer_has_leading_whitespace(self)) {
  2603. if (Tokenizer_CAN_RECURSE(self)) {
  2604. if (Tokenizer_parse_table(self))
  2605. return NULL;
  2606. }
  2607. else if (Tokenizer_emit_char(self, this))
  2608. return NULL;
  2609. }
  2610. else if (this_context & LC_TABLE_OPEN) {
  2611. if (this == '|' && next == '|' && this_context & LC_TABLE_TD_LINE) {
  2612. if (this_context & LC_TABLE_CELL_OPEN)
  2613. return Tokenizer_handle_table_cell_end(self, 0);
  2614. else if (Tokenizer_handle_table_cell(self, "||", "td", LC_TABLE_TD_LINE))
  2615. return NULL;
  2616. }
  2617. else if (this == '|' && next == '|' && this_context & LC_TABLE_TH_LINE) {
  2618. if (this_context & LC_TABLE_CELL_OPEN)
  2619. return Tokenizer_handle_table_cell_end(self, 0);
  2620. else if (Tokenizer_handle_table_cell(self, "||", "th", LC_TABLE_TH_LINE))
  2621. return NULL;
  2622. }
  2623. else if (this == '!' && next == '!' && this_context & LC_TABLE_TH_LINE) {
  2624. if (this_context & LC_TABLE_CELL_OPEN)
  2625. return Tokenizer_handle_table_cell_end(self, 0);
  2626. else if (Tokenizer_handle_table_cell(self, "!!", "th", LC_TABLE_TH_LINE))
  2627. return NULL;
  2628. }
  2629. else if (this == '|' && this_context & LC_TABLE_CELL_STYLE) {
  2630. return Tokenizer_handle_table_cell_end(self, 1);
  2631. }
  2632. // On newline, clear out cell line contexts
  2633. else if (this == '\n' && this_context & LC_TABLE_CELL_LINE_CONTEXTS) {
  2634. self->topstack->context &= ~LC_TABLE_CELL_LINE_CONTEXTS;
  2635. if (Tokenizer_emit_char(self, this))
  2636. return NULL;
  2637. }
  2638. else if (Tokenizer_has_leading_whitespace(self)) {
  2639. if (this == '|' && next == '}') {
  2640. if (this_context & LC_TABLE_CELL_OPEN)
  2641. return Tokenizer_handle_table_cell_end(self, 0);
  2642. if (this_context & LC_TABLE_ROW_OPEN)
  2643. return Tokenizer_handle_table_row_end(self);
  2644. else
  2645. return Tokenizer_handle_table_end(self);
  2646. }
  2647. else if (this == '|' && next == '-') {
  2648. if (this_context & LC_TABLE_CELL_OPEN)
  2649. return Tokenizer_handle_table_cell_end(self, 0);
  2650. if (this_context & LC_TABLE_ROW_OPEN)
  2651. return Tokenizer_handle_table_row_end(self);
  2652. else if (Tokenizer_handle_table_row(self))
  2653. return NULL;
  2654. }
  2655. else if (this == '|') {
  2656. if (this_context & LC_TABLE_CELL_OPEN)
  2657. return Tokenizer_handle_table_cell_end(self, 0);
  2658. else if (Tokenizer_handle_table_cell(self, "|", "td", LC_TABLE_TD_LINE))
  2659. return NULL;
  2660. }
  2661. else if (this == '!') {
  2662. if (this_context & LC_TABLE_CELL_OPEN)
  2663. return Tokenizer_handle_table_cell_end(self, 0);
  2664. else if (Tokenizer_handle_table_cell(self, "!", "th", LC_TABLE_TH_LINE))
  2665. return NULL;
  2666. }
  2667. else if (Tokenizer_emit_char(self, this))
  2668. return NULL;
  2669. }
  2670. else if (Tokenizer_emit_char(self, this))
  2671. return NULL;
  2672. // Raise BadRoute to table start
  2673. if (BAD_ROUTE)
  2674. return NULL;
  2675. }
  2676. else if (Tokenizer_emit_char(self, this))
  2677. return NULL;
  2678. self->head++;
  2679. }
  2680. }