Fix footnote parsing of footnote content (#536)

Fixes #412 and #493.  First we parse footnote content as
its own document avoid quirks with using li as a parent. Second, we
surround placeholders with STX and ETX to prevent them from interfering
with inline parsing; this is also consistent with how placeholders are
used everywhere else in Python Markdown.

3 files changed, 78 insertions, 13 deletions

diff --git a/markdown/extensions/footnotes.py b/markdown/extensions/footnotes.py
index 368334d..8bd8595 100644
--- a/markdown/extensions/footnotes.py
+++ b/markdown/extensions/footnotes.py
@@ -20,13 +20,13 @@ from ..preprocessors import Preprocessor
 from ..inlinepatterns import Pattern
 from ..treeprocessors import Treeprocessor
 from ..postprocessors import Postprocessor
-from ..util import etree, text_type
+from .. import util
 from ..odict import OrderedDict
 import re
 import copy
 
-FN_BACKLINK_TEXT = "zz1337820767766393qq"
-NBSP_PLACEHOLDER = "qq3936677670287331zz"
+FN_BACKLINK_TEXT = util.STX + "zz1337820767766393qq" + util.ETX
+NBSP_PLACEHOLDER = util.STX + "qq3936677670287331zz" + util.ETX
 DEF_RE = re.compile(r'[ ]{0,3}\[\^([^\]]*)\]:\s*(.*)')
 TABBED_RE = re.compile(r'((\t)|(    ))(.*)')
 RE_REF_ID = re.compile(r'(fnref)(\d+)')
@@ -169,16 +169,23 @@ class FootnoteExtension(Extension):
         if not list(self.footnotes.keys()):
             return None
 
-        div = etree.Element("div")
+        div = util.etree.Element("div")
         div.set('class', 'footnote')
-        etree.SubElement(div, "hr")
-        ol = etree.SubElement(div, "ol")
+        util.etree.SubElement(div, "hr")
+        ol = util.etree.SubElement(div, "ol")
+        surrogate_parent = util.etree.Element("div")
 
         for id in self.footnotes.keys():
-            li = etree.SubElement(ol, "li")
+            li = util.etree.SubElement(ol, "li")
             li.set("id", self.makeFootnoteId(id))
-            self.parser.parseChunk(li, self.footnotes[id])
-            backlink = etree.Element("a")
+            # Parse footnote with surrogate parent as li cannot be used.
+            # List block handlers have special logic to deal with li.
+            # When we are done parsing, we will copy everything over to li.
+            self.parser.parseChunk(surrogate_parent, self.footnotes[id])
+            for el in list(surrogate_parent):
+                li.append(el)
+                surrogate_parent.remove(el)
+            backlink = util.etree.Element("a")
             backlink.set("href", "#" + self.makeFootnoteRefId(id))
             if self.md.output_format not in ['html5', 'xhtml5']:
                 backlink.set("rev", "footnote")  # Invalid in HTML5
@@ -196,7 +203,7 @@ class FootnoteExtension(Extension):
                     node.text = node.text + NBSP_PLACEHOLDER
                     node.append(backlink)
                 else:
-                    p = etree.SubElement(li, "p")
+                    p = util.etree.SubElement(li, "p")
                     p.append(backlink)
         return div
 
@@ -303,14 +310,14 @@ class FootnotePattern(Pattern):
     def handleMatch(self, m):
         id = m.group(2)
         if id in self.footnotes.footnotes.keys():
-            sup = etree.Element("sup")
-            a = etree.SubElement(sup, "a")
+            sup = util.etree.Element("sup")
+            a = util.etree.SubElement(sup, "a")
             sup.set('id', self.footnotes.makeFootnoteRefId(id, found=True))
             a.set('href', '#' + self.footnotes.makeFootnoteId(id))
             if self.footnotes.md.output_format not in ['html5', 'xhtml5']:
                 a.set('rel', 'footnote')  # invalid in HTML5
             a.set('class', 'footnote-ref')
-            a.text = text_type(self.footnotes.footnotes.index(id) + 1)
+            a.text = util.text_type(self.footnotes.footnotes.index(id) + 1)
             return sup
         else:
             return None
diff --git a/tests/extensions/extra/footnote.html b/tests/extensions/extra/footnote.html
index 9cf42ed..4cca25c 100644
--- a/tests/extensions/extra/footnote.html
+++ b/tests/extensions/extra/footnote.html
@@ -3,6 +3,8 @@
 <p>Duplicate<sup id="fnref:a"><a class="footnote-ref" href="#fn:a" rel="footnote">6</a></sup> footnotes<sup id="fnref2:a"><a class="footnote-ref" href="#fn:a" rel="footnote">6</a></sup> test<sup id="fnref3:a"><a class="footnote-ref" href="#fn:a" rel="footnote">6</a></sup>.</p>
 <p>Duplicate<sup id="fnref:b"><a class="footnote-ref" href="#fn:b" rel="footnote">7</a></sup> footnotes<sup id="fnref2:b"><a class="footnote-ref" href="#fn:b" rel="footnote">7</a></sup> test<sup id="fnref3:b"><a class="footnote-ref" href="#fn:b" rel="footnote">7</a></sup>.</p>
 <p>Single after duplicates<sup id="fnref:c"><a class="footnote-ref" href="#fn:c" rel="footnote">8</a></sup>.</p>
+<p>Test emphasis at end of footnote<sup id="fnref:d"><a class="footnote-ref" href="#fn:d" rel="footnote">9</a></sup></p>
+<p>Complex footnote content<sup id="fnref:e"><a class="footnote-ref" href="#fn:e" rel="footnote">10</a></sup></p>
 <div class="footnote">
 <hr />
 <ol>
@@ -41,5 +43,29 @@ Nor is third...&#160;<a class="footnote-backref" href="#fnref:5" rev="footnote"
 <li id="fn:c">
 <p>3&#160;<a class="footnote-backref" href="#fnref:c" rev="footnote" title="Jump back to footnote 8 in the text">&#8617;</a></p>
 </li>
+<li id="fn:d">
+<p><em>emphasis works</em></p>
+<p><em>emphasis still works</em>&#160;<a class="footnote-backref" href="#fnref:d" rev="footnote" title="Jump back to footnote 9 in the text">&#8617;</a></p>
+</li>
+<li id="fn:e">
+<ol>
+<li>
+<p>The top couple half figure, contrary sides and hands across with bottom couple,</p>
+<p>Half figure back on your own sides, and turn partner to places,</p>
+<p>Swing partners with right hands into straight line long-ways, as in a reel, and</p>
+<p>Set,</p>
+<p>Hey and return to places,</p>
+<p>The other three couples do the same.</p>
+</li>
+<li>
+<p>Top and bottom couples meet and set,</p>
+<p>Then each gentleman leas the opposite lady to the couple on his left, and set,</p>
+<p>Aach four right and left,</p>
+<p>Swing side couples to places, and turn partners all eight,</p>
+<p>The other two couple o the same.</p>
+</li>
+</ol>
+<p><a class="footnote-backref" href="#fnref:e" rev="footnote" title="Jump back to footnote 10 in the text">&#8617;</a></p>
+</li>
 </ol>
 </div>
\ No newline at end of file
diff --git a/tests/extensions/extra/footnote.txt b/tests/extensions/extra/footnote.txt
index f95ff2c..93b5869 100644
--- a/tests/extensions/extra/footnote.txt
+++ b/tests/extensions/extra/footnote.txt
@@ -8,6 +8,10 @@ Duplicate[^b] footnotes[^b] test[^b].
 
 Single after duplicates[^c].
 
+Test emphasis at end of footnote[^d]
+
+Complex footnote content[^e]
+
 [^1]: Footnote that ends with a list:
 
     * item 1
@@ -28,3 +32,31 @@ Nor is third...
 [^a]: 1
 [^b]: 2
 [^c]: 3
+
+[^d]:
+    _emphasis works_
+
+    _emphasis still works_
+
+[^e]: 
+    1.  The top couple half figure, contrary sides and hands across with bottom couple,
+
+        Half figure back on your own sides, and turn partner to places,
+
+        Swing partners with right hands into straight line long-ways, as in a reel, and
+
+        Set,
+
+        Hey and return to places,
+
+        The other three couples do the same.
+
+    2.  Top and bottom couples meet and set,
+
+        Then each gentleman leas the opposite lady to the couple on his left, and set,
+
+        Aach four right and left,
+
+        Swing side couples to places, and turn partners all eight,
+
+        The other two couple o the same.