Merge pull request #231 from mitya57/master

Add Smarty extension and use it for building docs.

10 files changed, 344 insertions, 5 deletions

diff --git a/docs/extensions/index.txt b/docs/extensions/index.txt
index 2137a22..b71c2ec 100644
--- a/docs/extensions/index.txt
+++ b/docs/extensions/index.txt
@@ -53,6 +53,7 @@ Extension                            | "Name"
 [Meta-Data]                          | `meta`
 [New Line to Break]                  | `nl2br`
 [Sane Lists]                         | `sane_lists`
+[SmartyPants]                        | `smarty`
 [Table of Contents]                  | `toc`
 [WikiLinks]                          | `wikilinks`
 
@@ -70,6 +71,7 @@ Extension                            | "Name"
 [Meta-Data]: meta_data.html
 [New Line to Break]: nl2br.html
 [Sane Lists]: sane_lists.html
+[SmartyPants]: smarty.html
 [Table of Contents]: toc.html
 [WikiLinks]: wikilinks.html
 
diff --git a/docs/extensions/sane_lists.txt b/docs/extensions/sane_lists.txt
index 4d24d17..7e67e1f 100644
--- a/docs/extensions/sane_lists.txt
+++ b/docs/extensions/sane_lists.txt
@@ -1,8 +1,8 @@
 title:      Sane Lists Extension
 prev_title: New Line to Break Extension
 prev_url:   nl2br.html
-next_title: Table of Contents Extension
-next_url:   toc.html
+next_title: SmartyPants Extension
+next_url:   smarty.html
 
 Sane Lists
 ==========
diff --git a/docs/extensions/smarty.txt b/docs/extensions/smarty.txt
new file mode 100644
index 0000000..84c6494
--- /dev/null
+++ b/docs/extensions/smarty.txt
@@ -0,0 +1,56 @@
+title: SmartyPants Extension
+prev_title: Sane Lists Extension
+prev_url:   sane_lists.html
+next_title: Table of Contents Extension
+next_url:   toc.html
+
+SmartyPants
+===========
+
+Summary
+-------
+
+The SmartyPants extension converts ASCII dashes, quotes and ellipses to
+their HTML entity equivalents.
+
+ASCII symbol | Unicode replacements
+------------ | --------------------
+'        | ‘ ’
+"       | “ ”
+\...         | …
+\--          | –
+-\--         | —
+
+Arguments
+---------
+
+All three arguments are set to `True` by default.
+
+Argument         | Description
+--------         | -----------
+`smart_dashes`   | whether to convert dashes
+`smart_quotes`   | whether to convert quotes
+`smart_ellipses` | whether to convert ellipses
+
+Usage
+-----
+
+Default configuration:
+
+    >>> html = markdown.markdown(text,
+    ...     extensions=['smarty']
+    ... )
+
+Disable quotes convertation:
+
+    >>> html = markdown.markdown(text,
+    ...     extensions=['smarty(smart_quotes=False)']
+    ... )
+
+Further reading
+---------------
+
+SmartyPants extension is based on the original SmartyPants implementation
+by John Gruber. Please read it's [documentation][1] for details.
+
+[1]: http://daringfireball.net/projects/smartypants/
diff --git a/docs/extensions/toc.txt b/docs/extensions/toc.txt
index af282c6..2a91bb6 100644
--- a/docs/extensions/toc.txt
+++ b/docs/extensions/toc.txt
@@ -1,6 +1,6 @@
 title:      Table of Contents Extension
-prev_title: Sane Lists Extension
-prev_url:   sane_lists.html
+prev_title: SmartyPants Extension
+prev_url:   smarty.html
 next_title: Wikilinks Extension
 next_url:   wikilinks.html
 
diff --git a/docs/siteindex.txt b/docs/siteindex.txt
index 45fdab2..6846015 100644
--- a/docs/siteindex.txt
+++ b/docs/siteindex.txt
@@ -44,6 +44,7 @@ Table of Contents
         * [Meta-Data](extensions/meta_data.html)
         * [New Line to Break](extensions/nl2br.html)
         * [Sane Lists](extensions/sane_lists.html)
+        * [SmartyPants](extensions/smarty.html)
         * [Table of Contents](extensions/toc.html)
         * [WikiLinks](extensions/wikilinks.html)
     * [Third Party Extensions](extensions/index.html#third-party-extensions)
diff --git a/markdown/extensions/smarty.py b/markdown/extensions/smarty.py
new file mode 100644
index 0000000..18f9217
--- /dev/null
+++ b/markdown/extensions/smarty.py
@@ -0,0 +1,233 @@
+# -*- coding: utf-8 -*-
+# Smarty extension for Python-Markdown
+# Author: 2013, Dmitry Shachnev <mitya57@gmail.com>
+
+# SmartyPants license:
+#
+#   Copyright (c) 2003 John Gruber <http://daringfireball.net/>
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions are
+#   met:
+#
+#   *  Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#
+#   *  Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#
+#   *  Neither the name "SmartyPants" nor the names of its contributors 
+#      may be used to endorse or promote products derived from this
+#      software without specific prior written permission.
+#
+#   This software is provided by the copyright holders and contributors "as
+#   is" and any express or implied warranties, including, but not limited
+#   to, the implied warranties of merchantability and fitness for a
+#   particular purpose are disclaimed. In no event shall the copyright
+#   owner or contributors be liable for any direct, indirect, incidental,
+#   special, exemplary, or consequential damages (including, but not
+#   limited to, procurement of substitute goods or services; loss of use,
+#   data, or profits; or business interruption) however caused and on any
+#   theory of liability, whether in contract, strict liability, or tort
+#   (including negligence or otherwise) arising in any way out of the use
+#   of this software, even if advised of the possibility of such damage.
+#
+#
+# smartypants.py license:
+#
+#   smartypants.py is a derivative work of SmartyPants.
+#   Copyright (c) 2004, 2007 Chad Miller <http://web.chad.org/>
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions are
+#   met:
+#
+#   *  Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#
+#   *  Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#
+#   This software is provided by the copyright holders and contributors "as
+#   is" and any express or implied warranties, including, but not limited
+#   to, the implied warranties of merchantability and fitness for a
+#   particular purpose are disclaimed. In no event shall the copyright
+#   owner or contributors be liable for any direct, indirect, incidental,
+#   special, exemplary, or consequential damages (including, but not
+#   limited to, procurement of substitute goods or services; loss of use,
+#   data, or profits; or business interruption) however caused and on any
+#   theory of liability, whether in contract, strict liability, or tort
+#   (including negligence or otherwise) arising in any way out of the use
+#   of this software, even if advised of the possibility of such damage.
+
+from __future__ import unicode_literals
+from . import Extension
+from ..inlinepatterns import HtmlPattern
+
+def canonicalize(regex):
+    """
+    Converts the regexp from the re.VERBOSE form to the canonical form,
+    i.e. remove all whitespace and ignore comments.
+    """
+    lines = regex.split('\n')
+    for i in range(len(lines)):
+        if ' #' in lines[i]:
+            lines[i] = lines[i][:lines[i].find(' #')]
+    return ''.join(lines).replace(' ', '')
+
+# Constants for quote education.
+punctClass = r"""[!"#\$\%'()*+,-.\/:;<=>?\@\[\\\]\^_`{|}~]"""
+endOfWordClass = r"[\s.,;:!?)]"
+closeClass = r"[^\ \t\r\n\[\{\(\-\u0002\u0003]"
+
+openingQuotesBase = r"""
+(
+    \s            | # a whitespace char, or
+    &nbsp;        | # a non-breaking space entity, or
+    --            | # dashes, or
+    –|—           | # unicode, or
+    &[mn]dash;    | # named dash entities, or
+    &#8211;|&#8212; # decimal entities
+)
+"""
+
+# Special case if the very first character is a quote
+# followed by punctuation at a non-word-break. Close the quotes by brute force:
+singleQuoteStartRe = r"^'(?=%s\\B)" % punctClass
+doubleQuoteStartRe = r'^"(?=%s\\B)' % punctClass
+
+# Special case for double sets of quotes, e.g.:
+#   <p>He said, "'Quoted' words in a larger quote."</p>
+doubleQuoteSetsRe = r""""'(?=\w)"""
+singleQuoteSetsRe = r"""'"(?=\w)"""
+
+# Get most opening double quotes:
+openingDoubleQuotesRegex = canonicalize("""
+%s      # symbols before the quote
+"       # the quote
+(?=\w)  # followed by a word character
+""" % openingQuotesBase)
+
+# Double closing quotes:
+closingDoubleQuotesRegex = canonicalize(r"""
+"
+(?=\s)
+""")
+
+closingDoubleQuotesRegex2 = canonicalize(r"""
+(?<=%s)  # character that indicates the quote should be closing
+"
+""" % closeClass)
+
+# Get most opening single quotes:
+openingSingleQuotesRegex = canonicalize(r"""
+%s      # symbols before the quote
+'       # the quote
+(?=\w)  # followed by a word character
+""" % openingQuotesBase)
+
+closingSingleQuotesRegex = canonicalize(r"""
+(?<=%s)
+'
+(?!\s | s\b | \d)
+""" % closeClass)
+
+closingSingleQuotesRegex2 = canonicalize(r"""
+(?<=%s)
+'
+(\s | s\b)
+""" % closeClass)
+
+# All remaining quotes should be opening ones
+remainingSingleQuotesRegex = "'"
+remainingDoubleQuotesRegex = '"'
+
+lsquo, rsquo, ldquo, rdquo = '&lsquo;', '&rsquo;', '&ldquo;', '&rdquo;'
+
+class SubstituteTextPattern(HtmlPattern):
+    def __init__(self, pattern, replace, markdown_instance):
+        """ Replaces matches with some text. """
+        HtmlPattern.__init__(self, pattern)
+        self.replace = replace
+        self.markdown = markdown_instance
+
+    def handleMatch(self, m):
+        result = ''
+        for part in self.replace:
+            if isinstance(part, int):
+                result += m.group(part)
+            else:
+                result += self.markdown.htmlStash.store(part, safe=True)
+        return result
+
+class SmartyExtension(Extension):
+    def __init__(self, configs):
+        self.config = {
+            'smart_quotes': [True, 'Educate quotes'],
+            'smart_dashes': [True, 'Educate dashes'],
+            'smart_ellipses': [True, 'Educate ellipses']
+        }
+        for key, value in configs:
+            if not isinstance(value, str):
+                value = bool(value)
+            elif value.lower() in ('true', 't', 'yes', 'y', '1'):
+                value = True
+            elif value.lower() in ('false', 'f', 'no', 'n', '0'):
+                value = False
+            else:
+                raise ValueError('Cannot parse bool value: %s' % value)
+            self.setConfig(key, value)
+
+    def _addPatterns(self, md, patterns, serie):
+        for ind, pattern in enumerate(patterns):
+            pattern += (md,)
+            pattern = SubstituteTextPattern(*pattern)
+            after = ('>smarty-%s-%d' % (serie, ind - 1) if ind else '>entity')
+            name = 'smarty-%s-%d' % (serie, ind)
+            md.inlinePatterns.add(name, pattern, after)
+
+    def educateDashes(self, md):
+        emDashesPattern = SubstituteTextPattern(r'(?<!-)---(?!-)', '&mdash;', md)
+        enDashesPattern = SubstituteTextPattern(r'(?<!-)--(?!-)', '&ndash;', md)
+        md.inlinePatterns.add('smarty-em-dashes', emDashesPattern, '>entity')
+        md.inlinePatterns.add('smarty-en-dashes', enDashesPattern,
+            '>smarty-em-dashes')
+
+    def educateEllipses(self, md):
+        ellipsesPattern = SubstituteTextPattern(r'(?<!\.)\.{3}(?!\.)', '&hellip;', md)
+        md.inlinePatterns.add('smarty-ellipses', ellipsesPattern, '>entity')
+
+    def educateQuotes(self, md):
+        patterns = (
+            (singleQuoteStartRe, (rsquo,)),
+            (doubleQuoteStartRe, (rdquo,)),
+            (doubleQuoteSetsRe, (ldquo + lsquo,)),
+            (singleQuoteSetsRe, (lsquo + ldquo,)),
+            (openingSingleQuotesRegex, (2, lsquo)),
+            (closingSingleQuotesRegex, (rsquo,)),
+            (closingSingleQuotesRegex2, (rsquo, 2)),
+            (remainingSingleQuotesRegex, (lsquo,)),
+            (openingDoubleQuotesRegex, (2, ldquo)),
+            (closingDoubleQuotesRegex, (rdquo,)),
+            (closingDoubleQuotesRegex2, (rdquo,)),
+            (remainingDoubleQuotesRegex, (ldquo,))
+        )
+        self._addPatterns(md, patterns, 'quotes')
+
+    def extendMarkdown(self, md, md_globals):
+        configs = self.getConfigs()
+        if configs['smart_quotes']:
+            self.educateQuotes(md)
+        if configs['smart_dashes']:
+            self.educateDashes(md)
+        if configs['smart_ellipses']:
+            self.educateEllipses(md)
+        md.ESCAPED_CHARS.extend(['"', "'"])
+
+def makeExtension(configs=None):
+    return SmartyExtension(configs)
diff --git a/setup.py b/setup.py
index a205c49..826de9d 100755
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@ class build_docs(Command):
         else:
             with codecs.open('docs/_template.html', encoding='utf-8') as f:
                 template = f.read()
-            self.md = markdown.Markdown(extensions=['extra', 'toc', 'meta', 'admonition'])
+            self.md = markdown.Markdown(extensions=['extra', 'toc', 'meta', 'admonition', 'smarty'])
             for infile in self.docs:
                 outfile, ext = os.path.splitext(infile)
                 if ext == '.txt':
diff --git a/tests/extensions/smarty.html b/tests/extensions/smarty.html
new file mode 100644
index 0000000..fbd15af
--- /dev/null
+++ b/tests/extensions/smarty.html
@@ -0,0 +1,20 @@
+<p>1440&ndash;80&rsquo;s<br />
+1440&ndash;&lsquo;80s<br />
+1440&mdash;&lsquo;80s<br />
+1960s<br />
+1960&rsquo;s<br />
+one two &lsquo;60s<br />
+&lsquo;60s</p>
+<p>&ldquo;Isn&rsquo;t this fun&rdquo;? &mdash; she said&hellip;<br />
+&ldquo;&lsquo;Quoted&rsquo; words in a larger quote.&rdquo;<br />
+&lsquo;Quoted &ldquo;words&rdquo; in a larger quote.&rsquo;<br />
+&ldquo;quoted&rdquo; text and <strong>bold &ldquo;quoted&rdquo; text</strong><br />
+&lsquo;quoted&rsquo; text and <strong>bold &lsquo;quoted&rsquo; text</strong><br />
+em-dashes (&mdash;) and ellipes (&hellip;)<br />
+&ldquo;<a href="http://example.com">Link</a>&rdquo; &mdash; she said.</p>
+<hr />
+<p>Escaped -- ndash<br />
+'Escaped' "quotes"<br />
+Escaped ellipsis...</p>
+<p>&lsquo;Escaped "quotes" in real ones&rsquo;<br />
+'&ldquo;Real&rdquo; quotes in escaped ones'</p>
\ No newline at end of file
diff --git a/tests/extensions/smarty.txt b/tests/extensions/smarty.txt
new file mode 100644
index 0000000..5b5ece7
--- /dev/null
+++ b/tests/extensions/smarty.txt
@@ -0,0 +1,24 @@
+1440--80's  
+1440--'80s  
+1440---'80s  
+1960s  
+1960's  
+one two '60s  
+'60s
+
+"Isn't this fun"? --- she said...  
+"'Quoted' words in a larger quote."  
+'Quoted "words" in a larger quote.'  
+"quoted" text and **bold "quoted" text**  
+'quoted' text and **bold 'quoted' text**  
+em-dashes (---) and ellipes (...)  
+"[Link](http://example.com)" --- she said.
+
+--- -- ---
+
+Escaped \-- ndash  
+\'Escaped\' \"quotes\"  
+Escaped ellipsis\...
+
+'Escaped \"quotes\" in real ones'  
+\'"Real" quotes in escaped ones\'
\ No newline at end of file
diff --git a/tests/extensions/test.cfg b/tests/extensions/test.cfg
index 42145c1..1a13b1c 100644
--- a/tests/extensions/test.cfg
+++ b/tests/extensions/test.cfg
@@ -38,3 +38,6 @@ extensions=nl2br,attr_list
 
 [admonition]
 extensions=admonition
+
+[smarty]
+extensions=smarty