Fix double escaping of amp in attributes (#670)

Serializer should only escape & in attributes if not part of & 

Better regex avoid Unicode and `_` in amp detection.

In general, we don't want to escape already escaped content, but with code content, we want literal representations of escaped content, so have code content explicitly escape its content before placing in AtomicStrings.

Closes #669.

5 files changed, 21 insertions, 2 deletions

diff --git a/tests/misc/amp-in-url.html b/tests/misc/amp-in-url.html
deleted file mode 100644
index 2170a54..0000000
--- a/tests/misc/amp-in-url.html
+++ /dev/null
@@ -1 +0,0 @@
-<p><a href="http://www.freewisdom.org/this&amp;that">link</a></p>
\ No newline at end of file
diff --git a/tests/misc/amp-in-url.txt b/tests/misc/amp-in-url.txt
deleted file mode 100644
index 471106e..0000000
--- a/tests/misc/amp-in-url.txt
+++ /dev/null
@@ -1 +0,0 @@
-[link](http://www.freewisdom.org/this&that)
diff --git a/tests/test_apis.py b/tests/test_apis.py
index d9d520d..c813e56 100644
--- a/tests/test_apis.py
+++ b/tests/test_apis.py
@@ -666,6 +666,15 @@ class testSerializers(unittest.TestCase):
             '<div xmlns="&lt;&amp;&quot;test&#10;escaping&quot;&gt;"></div>'
         )
 
+    def testQNamePreEscaping(self):
+        """ Test QName that is already partially escaped. """
+        qname = markdown.util.etree.QName('&lt;&amp;"test&#10;escaping"&gt;', 'div')
+        el = markdown.util.etree.Element(qname)
+        self.assertEqual(
+            markdown.serializers.to_xhtml_string(el),
+            '<div xmlns="&lt;&amp;&quot;test&#10;escaping&quot;&gt;"></div>'
+        )
+
     def buildExtension(self):
         """ Build an extension which registers fakeSerializer. """
         def fakeSerializer(elem):
diff --git a/tests/test_syntax/inline/images.py b/tests/test_syntax/inline/test_images.py
index 52ce330..52ce330 100644
--- a/tests/test_syntax/inline/images.py
+++ b/tests/test_syntax/inline/test_images.py
diff --git a/tests/test_syntax/inline/links.py b/tests/test_syntax/inline/test_links.py
index d74bb75..3e8593f 100644
--- a/tests/test_syntax/inline/links.py
+++ b/tests/test_syntax/inline/test_links.py
@@ -118,3 +118,15 @@ class TestAdvancedLinks(TestCase):
             """<p><a href="http://link.com/with spaces '&quot;and quotes&quot;" title="and title">"""
             """Text</a> more text</p>"""
         )
+
+    def test_amp_in_url(self):
+        """Test amp in URLs."""
+
+        self.assertMarkdownRenders(
+            '[link](http://www.freewisdom.org/this&that)',
+            '<p><a href="http://www.freewisdom.org/this&amp;that">link</a></p>'
+        )
+        self.assertMarkdownRenders(
+            '[title](http://example.com/?a=1&amp;b=2)',
+            '<p><a href="http://example.com/?a=1&amp;b=2">title</a></p>'
+        )