ben
/
mwparserfromhell
mirror of https://github.com/earwig/mwparserfromhell
1
1
Fork 0
Code Releases 24 Activity
A Python parser for MediaWiki wikicode https://mwparserfromhell.readthedocs.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
445 Commits
1 Branch
7.1 MiB
 
 
 
 
Tree: 2f3edde57a
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '2f3edde57a'
${ noResults }
mwparserfromhell/tests/test_wikicode.py

378 lines
18 KiB
Raw Blame History 

  1. # -*- coding: utf-8  -*-

  2. #

  3. # Copyright (C) 2012-2013 Ben Kurtovic <ben.kurtovic@verizon.net>

  4. #

  5. # Permission is hereby granted, free of charge, to any person obtaining a copy

  6. # of this software and associated documentation files (the "Software"), to deal

  7. # in the Software without restriction, including without limitation the rights

  8. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

  9. # copies of the Software, and to permit persons to whom the Software is

  10. # furnished to do so, subject to the following conditions:

  11. #

  12. # The above copyright notice and this permission notice shall be included in

  13. # all copies or substantial portions of the Software.

  14. #

  15. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

  16. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

  17. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

  18. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

  19. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

  20. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

  21. # SOFTWARE.

  22. 

  23. from __future__ import unicode_literals

  24. import re

  25. from types import GeneratorType

  26. import unittest

  27. 

  28. from mwparserfromhell.nodes import (Argument, Comment, Heading, HTMLEntity,

  29.                                     Node, Tag, Template, Text, Wikilink)

  30. from mwparserfromhell.smart_list import SmartList

  31. from mwparserfromhell.wikicode import Wikicode

  32. from mwparserfromhell import parse

  33. from mwparserfromhell.compat import py3k, str

  34. 

  35. from ._test_tree_equality import TreeEqualityTestCase, wrap, wraptext

  36. 

  37. class TestWikicode(TreeEqualityTestCase):

  38.     """Tests for the Wikicode class, which manages a list of nodes."""

  39. 

  40.     def test_unicode(self):

  41.         """test Wikicode.__unicode__()"""

  42.         code1 = parse("foobar")

  43.         code2 = parse("Have a {{template}} and a [[page|link]]")

  44.         self.assertEqual("foobar", str(code1))

  45.         self.assertEqual("Have a {{template}} and a [[page|link]]", str(code2))

  46. 

  47.     def test_nodes(self):

  48.         """test getter/setter for the nodes attribute"""

  49.         code = parse("Have a {{template}}")

  50.         self.assertEqual(["Have a ", "{{template}}"], code.nodes)

  51.         L1 = SmartList([Text("foobar"), Template(wraptext("abc"))])

  52.         L2 = [Text("barfoo"), Template(wraptext("cba"))]

  53.         L3 = "abc{{def}}"

  54.         code.nodes = L1

  55.         self.assertIs(L1, code.nodes)

  56.         code.nodes = L2

  57.         self.assertIs(L2, code.nodes)

  58.         code.nodes = L3

  59.         self.assertEqual(["abc", "{{def}}"], code.nodes)

  60.         self.assertRaises(ValueError, setattr, code, "nodes", object)

  61. 

  62.     def test_get(self):

  63.         """test Wikicode.get()"""

  64.         code = parse("Have a {{template}} and a [[page|link]]")

  65.         self.assertIs(code.nodes[0], code.get(0))

  66.         self.assertIs(code.nodes[2], code.get(2))

  67.         self.assertRaises(IndexError, code.get, 4)

  68. 

  69.     def test_set(self):

  70.         """test Wikicode.set()"""

  71.         code = parse("Have a {{template}} and a [[page|link]]")

  72.         code.set(1, "{{{argument}}}")

  73.         self.assertEqual("Have a {{{argument}}} and a [[page|link]]", code)

  74.         self.assertIsInstance(code.get(1), Argument)

  75.         code.set(2, None)

  76.         self.assertEqual("Have a {{{argument}}}[[page|link]]", code)

  77.         code.set(-3, "This is an ")

  78.         self.assertEqual("This is an {{{argument}}}[[page|link]]", code)

  79.         self.assertRaises(ValueError, code.set, 1, "foo {{bar}}")

  80.         self.assertRaises(IndexError, code.set, 3, "{{baz}}")

  81.         self.assertRaises(IndexError, code.set, -4, "{{baz}}")

  82. 

  83.     def test_index(self):

  84.         """test Wikicode.index()"""

  85.         code = parse("Have a {{template}} and a [[page|link]]")

  86.         self.assertEqual(0, code.index("Have a "))

  87.         self.assertEqual(3, code.index("[[page|link]]"))

  88.         self.assertEqual(1, code.index(code.get(1)))

  89.         self.assertRaises(ValueError, code.index, "foo")

  90. 

  91.         code = parse("{{foo}}{{bar|{{baz}}}}")

  92.         self.assertEqual(1, code.index("{{bar|{{baz}}}}"))

  93.         self.assertEqual(1, code.index("{{baz}}", recursive=True))

  94.         self.assertEqual(1, code.index(code.get(1).get(1).value,

  95.                                        recursive=True))

  96.         self.assertRaises(ValueError, code.index, "{{baz}}", recursive=False)

  97.         self.assertRaises(ValueError, code.index,

  98.                           code.get(1).get(1).value, recursive=False)

  99. 

  100.     def test_insert(self):

  101.         """test Wikicode.insert()"""

  102.         code = parse("Have a {{template}} and a [[page|link]]")

  103.         code.insert(1, "{{{argument}}}")

  104.         self.assertEqual(

  105.             "Have a {{{argument}}}{{template}} and a [[page|link]]", code)

  106.         self.assertIsInstance(code.get(1), Argument)

  107.         code.insert(2, None)

  108.         self.assertEqual(

  109.             "Have a {{{argument}}}{{template}} and a [[page|link]]", code)

  110.         code.insert(-3, Text("foo"))

  111.         self.assertEqual(

  112.             "Have a {{{argument}}}foo{{template}} and a [[page|link]]", code)

  113. 

  114.         code2 = parse("{{foo}}{{bar}}{{baz}}")

  115.         code2.insert(1, "abc{{def}}ghi[[jk]]")

  116.         self.assertEqual("{{foo}}abc{{def}}ghi[[jk]]{{bar}}{{baz}}", code2)

  117.         self.assertEqual(["{{foo}}", "abc", "{{def}}", "ghi", "[[jk]]",

  118.                           "{{bar}}", "{{baz}}"], code2.nodes)

  119. 

  120.         code3 = parse("{{foo}}bar")

  121.         code3.insert(1000, "[[baz]]")

  122.         code3.insert(-1000, "derp")

  123.         self.assertEqual("derp{{foo}}bar[[baz]]", code3)

  124. 

  125.     def test_insert_before(self):

  126.         """test Wikicode.insert_before()"""

  127.         code = parse("{{a}}{{b}}{{c}}{{d}}")

  128.         code.insert_before("{{b}}", "x", recursive=True)

  129.         code.insert_before("{{d}}", "[[y]]", recursive=False)

  130.         self.assertEqual("{{a}}x{{b}}{{c}}[[y]]{{d}}", code)

  131.         code.insert_before(code.get(2), "z")

  132.         self.assertEqual("{{a}}xz{{b}}{{c}}[[y]]{{d}}", code)

  133.         self.assertRaises(ValueError, code.insert_before, "{{r}}", "n",

  134.                           recursive=True)

  135.         self.assertRaises(ValueError, code.insert_before, "{{r}}", "n",

  136.                           recursive=False)

  137. 

  138.         code2 = parse("{{a|{{b}}|{{c|d={{f}}}}}}")

  139.         code2.insert_before(code2.get(0).params[0].value.get(0), "x",

  140.                             recursive=True)

  141.         code2.insert_before("{{f}}", "y", recursive=True)

  142.         self.assertEqual("{{a|x{{b}}|{{c|d=y{{f}}}}}}", code2)

  143.         self.assertRaises(ValueError, code2.insert_before, "{{f}}", "y",

  144.                           recursive=False)

  145. 

  146.     def test_insert_after(self):

  147.         """test Wikicode.insert_after()"""

  148.         code = parse("{{a}}{{b}}{{c}}{{d}}")

  149.         code.insert_after("{{b}}", "x", recursive=True)

  150.         code.insert_after("{{d}}", "[[y]]", recursive=False)

  151.         self.assertEqual("{{a}}{{b}}x{{c}}{{d}}[[y]]", code)

  152.         code.insert_after(code.get(2), "z")

  153.         self.assertEqual("{{a}}{{b}}xz{{c}}{{d}}[[y]]", code)

  154.         self.assertRaises(ValueError, code.insert_after, "{{r}}", "n",

  155.                           recursive=True)

  156.         self.assertRaises(ValueError, code.insert_after, "{{r}}", "n",

  157.                           recursive=False)

  158. 

  159.         code2 = parse("{{a|{{b}}|{{c|d={{f}}}}}}")

  160.         code2.insert_after(code2.get(0).params[0].value.get(0), "x",

  161.                            recursive=True)

  162.         code2.insert_after("{{f}}", "y", recursive=True)

  163.         self.assertEqual("{{a|{{b}}x|{{c|d={{f}}y}}}}", code2)

  164.         self.assertRaises(ValueError, code2.insert_after, "{{f}}", "y",

  165.                           recursive=False)

  166. 

  167.     def test_replace(self):

  168.         """test Wikicode.replace()"""

  169.         code = parse("{{a}}{{b}}{{c}}{{d}}")

  170.         code.replace("{{b}}", "x", recursive=True)

  171.         code.replace("{{d}}", "[[y]]", recursive=False)

  172.         self.assertEqual("{{a}}x{{c}}[[y]]", code)

  173.         code.replace(code.get(1), "z")

  174.         self.assertEqual("{{a}}z{{c}}[[y]]", code)

  175.         self.assertRaises(ValueError, code.replace, "{{r}}", "n",

  176.                           recursive=True)

  177.         self.assertRaises(ValueError, code.replace, "{{r}}", "n",

  178.                           recursive=False)

  179. 

  180.         code2 = parse("{{a|{{b}}|{{c|d={{f}}}}}}")

  181.         code2.replace(code2.get(0).params[0].value.get(0), "x", recursive=True)

  182.         code2.replace("{{f}}", "y", recursive=True)

  183.         self.assertEqual("{{a|x|{{c|d=y}}}}", code2)

  184.         self.assertRaises(ValueError, code2.replace, "y", "z", recursive=False)

  185. 

  186.     def test_append(self):

  187.         """test Wikicode.append()"""

  188.         code = parse("Have a {{template}}")

  189.         code.append("{{{argument}}}")

  190.         self.assertEqual("Have a {{template}}{{{argument}}}", code)

  191.         self.assertIsInstance(code.get(2), Argument)

  192.         code.append(None)

  193.         self.assertEqual("Have a {{template}}{{{argument}}}", code)

  194.         code.append(Text(" foo"))

  195.         self.assertEqual("Have a {{template}}{{{argument}}} foo", code)

  196.         self.assertRaises(ValueError, code.append, slice(0, 1))

  197. 

  198.     def test_remove(self):

  199.         """test Wikicode.remove()"""

  200.         code = parse("{{a}}{{b}}{{c}}{{d}}")

  201.         code.remove("{{b}}", recursive=True)

  202.         code.remove(code.get(1), recursive=True)

  203.         self.assertEqual("{{a}}{{d}}", code)

  204.         self.assertRaises(ValueError, code.remove, "{{r}}", recursive=True)

  205.         self.assertRaises(ValueError, code.remove, "{{r}}", recursive=False)

  206. 

  207.         code2 = parse("{{a|{{b}}|{{c|d={{f}}{{h}}}}}}")

  208.         code2.remove(code2.get(0).params[0].value.get(0), recursive=True)

  209.         code2.remove("{{f}}", recursive=True)

  210.         self.assertEqual("{{a||{{c|d={{h}}}}}}", code2)

  211.         self.assertRaises(ValueError, code2.remove, "{{h}}", recursive=False)

  212. 

  213.     def test_matches(self):

  214.         """test Wikicode.matches()"""

  215.         code1 = parse("Cleanup")

  216.         code2 = parse("\nstub<!-- TODO: make more specific -->")

  217.         self.assertTrue(code1.matches("Cleanup"))

  218.         self.assertTrue(code1.matches("cleanup"))

  219.         self.assertTrue(code1.matches("  cleanup\n"))

  220.         self.assertFalse(code1.matches("CLEANup"))

  221.         self.assertFalse(code1.matches("Blah"))

  222.         self.assertTrue(code2.matches("stub"))

  223.         self.assertTrue(code2.matches("Stub<!-- no, it's fine! -->"))

  224.         self.assertFalse(code2.matches("StuB"))

  225. 

  226.     def test_filter_family(self):

  227.         """test the Wikicode.i?filter() family of functions"""

  228.         def genlist(gen):

  229.             self.assertIsInstance(gen, GeneratorType)

  230.             return list(gen)

  231.         ifilter = lambda code: (lambda **kw: genlist(code.ifilter(**kw)))

  232. 

  233.         code = parse("a{{b}}c[[d]]{{{e}}}{{f}}[[g]]")

  234.         for func in (code.filter, ifilter(code)):

  235.             self.assertEqual(["a", "{{b}}", "b", "c", "[[d]]", "d", "{{{e}}}",

  236.                               "e", "{{f}}", "f", "[[g]]", "g"], func())

  237.             self.assertEqual(["{{{e}}}"], func(forcetype=Argument))

  238.             self.assertIs(code.get(4), func(forcetype=Argument)[0])

  239.             self.assertEqual(list("abcdefg"), func(forcetype=Text))

  240.             self.assertEqual([], func(forcetype=Heading))

  241.             self.assertRaises(TypeError, func, forcetype=True)

  242. 

  243.         funcs = [

  244.             lambda name, **kw: getattr(code, "filter_" + name)(**kw),

  245.             lambda name, **kw: genlist(getattr(code, "ifilter_" + name)(**kw))

  246.         ]

  247.         for get_filter in funcs:

  248.             self.assertEqual(["{{{e}}}"], get_filter("arguments"))

  249.             self.assertIs(code.get(4), get_filter("arguments")[0])

  250.             self.assertEqual([], get_filter("comments"))

  251.             self.assertEqual([], get_filter("headings"))

  252.             self.assertEqual([], get_filter("html_entities"))

  253.             self.assertEqual([], get_filter("tags"))

  254.             self.assertEqual(["{{b}}", "{{f}}"], get_filter("templates"))

  255.             self.assertEqual(list("abcdefg"), get_filter("text"))

  256.             self.assertEqual(["[[d]]", "[[g]]"], get_filter("wikilinks"))

  257. 

  258.         code2 = parse("{{a|{{b}}|{{c|d={{f}}{{h}}}}}}")

  259.         for func in (code2.filter, ifilter(code2)):

  260.             self.assertEqual(["{{a|{{b}}|{{c|d={{f}}{{h}}}}}}"],

  261.                              func(recursive=False, forcetype=Template))

  262.             self.assertEqual(["{{a|{{b}}|{{c|d={{f}}{{h}}}}}}", "{{b}}",

  263.                               "{{c|d={{f}}{{h}}}}", "{{f}}", "{{h}}"],

  264.                              func(recursive=True, forcetype=Template))

  265. 

  266.         code3 = parse("{{foobar}}{{FOO}}{{baz}}{{bz}}")

  267.         for func in (code3.filter, ifilter(code3)):

  268.             self.assertEqual(["{{foobar}}", "{{FOO}}"], func(recursive=False, matches=r"foo"))

  269.             self.assertEqual(["{{foobar}}", "{{FOO}}"],

  270.                              func(recursive=False, matches=r"^{{foo.*?}}"))

  271.             self.assertEqual(["{{foobar}}"],

  272.                              func(recursive=False, matches=r"^{{foo.*?}}", flags=re.UNICODE))

  273.             self.assertEqual(["{{baz}}", "{{bz}}"], func(recursive=False, matches=r"^{{b.*?z"))

  274.             self.assertEqual(["{{baz}}"], func(recursive=False, matches=r"^{{b.+?z}}"))

  275. 

  276.         self.assertEqual(["{{a|{{b}}|{{c|d={{f}}{{h}}}}}}"],

  277.                          code2.filter_templates(recursive=False))

  278.         self.assertEqual(["{{a|{{b}}|{{c|d={{f}}{{h}}}}}}", "{{b}}",

  279.                           "{{c|d={{f}}{{h}}}}", "{{f}}", "{{h}}"],

  280.                          code2.filter_templates(recursive=True))

  281.         self.assertEqual(["{{baz}}", "{{bz}}"],

  282.                          code3.filter_templates(matches=r"^{{b.*?z"))

  283.         self.assertEqual([], code3.filter_tags(matches=r"^{{b.*?z"))

  284.         self.assertEqual([], code3.filter_tags(matches=r"^{{b.*?z", flags=0))

  285. 

  286.         self.assertRaises(TypeError, code.filter_templates, 100)

  287.         self.assertRaises(TypeError, code.filter_templates, a=42)

  288.         self.assertRaises(TypeError, code.filter_templates, forcetype=Template)

  289. 

  290.     def test_get_sections(self):

  291.         """test Wikicode.get_sections()"""

  292.         page1 = parse("")

  293.         page2 = parse("==Heading==")

  294.         page3 = parse("===Heading===\nFoo bar baz\n====Gnidaeh====\n")

  295. 

  296.         p4_lead = "This is a lead.\n"

  297.         p4_IA = "=== Section I.A ===\nSection I.A [[body]].\n"

  298.         p4_IB1 = "==== Section I.B.1 ====\nSection I.B.1 body.\n\n&bull;Some content.\n\n"

  299.         p4_IB = "=== Section I.B ===\n" + p4_IB1

  300.         p4_I = "== Section I ==\nSection I body. {{and a|template}}\n" + p4_IA + p4_IB

  301.         p4_II = "== Section II ==\nSection II body.\n\n"

  302.         p4_IIIA1a = "===== Section III.A.1.a =====\nMore text.\n"

  303.         p4_IIIA2ai1 = "======= Section III.A.2.a.i.1 =======\nAn invalid section!"

  304.         p4_IIIA2 = "==== Section III.A.2 ====\nEven more text.\n" + p4_IIIA2ai1

  305.         p4_IIIA = "=== Section III.A ===\nText.\n" + p4_IIIA1a + p4_IIIA2

  306.         p4_III = "== Section III ==\n" + p4_IIIA

  307.         page4 = parse(p4_lead + p4_I + p4_II + p4_III)

  308. 

  309.         self.assertEqual([], page1.get_sections())

  310.         self.assertEqual(["", "==Heading=="], page2.get_sections())

  311.         self.assertEqual(["", "===Heading===\nFoo bar baz\n====Gnidaeh====\n",

  312.                           "====Gnidaeh====\n"], page3.get_sections())

  313.         self.assertEqual([p4_lead, p4_IA, p4_I, p4_IB, p4_IB1, p4_II,

  314.                           p4_IIIA1a, p4_III, p4_IIIA, p4_IIIA2, p4_IIIA2ai1],

  315.                          page4.get_sections())

  316. 

  317.         self.assertEqual(["====Gnidaeh====\n"], page3.get_sections(levels=[4]))

  318.         self.assertEqual(["===Heading===\nFoo bar baz\n====Gnidaeh====\n"],

  319.                          page3.get_sections(levels=(2, 3)))

  320.         self.assertEqual([], page3.get_sections(levels=[0]))

  321.         self.assertEqual(["", "====Gnidaeh====\n"],

  322.                          page3.get_sections(levels=[4], include_lead=True))

  323.         self.assertEqual(["===Heading===\nFoo bar baz\n====Gnidaeh====\n",

  324.                           "====Gnidaeh====\n"],

  325.                          page3.get_sections(include_lead=False))

  326. 

  327.         self.assertEqual([p4_IB1, p4_IIIA2], page4.get_sections(levels=[4]))

  328.         self.assertEqual([""], page2.get_sections(include_headings=False))

  329.         self.assertEqual(["\nSection I.B.1 body.\n\n&bull;Some content.\n\n",

  330.                           "\nEven more text.\n" + p4_IIIA2ai1],

  331.                          page4.get_sections(levels=[4],

  332.                                             include_headings=False))

  333. 

  334.         self.assertEqual([], page4.get_sections(matches=r"body"))

  335.         self.assertEqual([p4_IA, p4_I, p4_IB, p4_IB1],

  336.                          page4.get_sections(matches=r"Section\sI[.\s].*?"))

  337.         self.assertEqual([p4_IA, p4_IIIA1a, p4_IIIA, p4_IIIA2, p4_IIIA2ai1],

  338.                          page4.get_sections(matches=r".*?a.*?"))

  339.         self.assertEqual([p4_IIIA1a, p4_IIIA2ai1],

  340.                          page4.get_sections(matches=r".*?a.*?", flags=re.U))

  341.         self.assertEqual(["\nMore text.\n", "\nAn invalid section!"],

  342.                          page4.get_sections(matches=r".*?a.*?", flags=re.U,

  343.                                             include_headings=False))

  344. 

  345.         page5 = parse("X\n== Foo ==\nBar\n== Baz ==\nBuzz")

  346.         section = page5.get_sections(matches="Foo")[0]

  347.         section.replace("\nBar\n", "\nBarf ")

  348.         section.append("{{Haha}}\n")

  349.         self.assertEqual("== Foo ==\nBarf {{Haha}}\n", section)

  350.         self.assertEqual("X\n== Foo ==\nBarf {{Haha}}\n== Baz ==\nBuzz", page5)

  351. 

  352.     def test_strip_code(self):

  353.         """test Wikicode.strip_code()"""

  354.         # Since individual nodes have test cases for their __strip__ methods,

  355.         # we're only going to do an integration test:

  356.         code = parse("Foo [[bar]]\n\n{{baz}}\n\n[[a|b]] &Sigma;")

  357.         self.assertEqual("Foo bar\n\nb Σ",

  358.                          code.strip_code(normalize=True, collapse=True))

  359.         self.assertEqual("Foo bar\n\n\n\nb Σ",

  360.                          code.strip_code(normalize=True, collapse=False))

  361.         self.assertEqual("Foo bar\n\nb &Sigma;",

  362.                          code.strip_code(normalize=False, collapse=True))

  363.         self.assertEqual("Foo bar\n\n\n\nb &Sigma;",

  364.                          code.strip_code(normalize=False, collapse=False))

  365. 

  366.     def test_get_tree(self):

  367.         """test Wikicode.get_tree()"""

  368.         # Since individual nodes have test cases for their __showtree___

  369.         # methods, and the docstring covers all possibilities for the output of

  370.         # __showtree__, we'll test it only:

  371.         code = parse("Lorem ipsum {{foo|bar|{{baz}}|spam=eggs}}")

  372.         expected = "Lorem ipsum \n{{\n\t  foo\n\t| 1\n\t= bar\n\t| 2\n\t= " + \

  373.                    "{{\n\t\t\tbaz\n\t  }}\n\t| spam\n\t= eggs\n}}"

  374.         self.assertEqual(expected.expandtabs(4), code.get_tree())

  375. 

  376. if __name__ == "__main__":

  377.     unittest.main(verbosity=2)