aboutsummaryrefslogtreecommitdiffstats
path: root/markdown.py
diff options
context:
space:
mode:
Diffstat (limited to 'markdown.py')
-rw-r--r--[-rwxr-xr-x]markdown.py2251
1 files changed, 2 insertions, 2249 deletions
diff --git a/markdown.py b/markdown.py
index baf6567..7cba3a8 100755..100644
--- a/markdown.py
+++ b/markdown.py
@@ -1,2253 +1,6 @@
#!/usr/bin/env python
-"""
-Python Markdown
-===============
-Python Markdown converts Markdown to HTML and can be used as a library or
-called from the command line.
-
-## Basic usage as a module:
-
- import markdown
- md = Markdown()
- html = md.convert(your_text_string)
-
-## Basic use from the command line:
-
- python markdown.py source.txt > destination.html
-
-Run "python markdown.py --help" to see more options.
-
-## Extensions
-
-See <http://www.freewisdom.org/projects/python-markdown/> for more
-information and instructions on how to extend the functionality of
-Python Markdown. Read that before you try modifying this file.
-
-## Authors and License
-
-Started by [Manfred Stienstra](http://www.dwerg.net/). Continued and
-maintained by [Yuri Takhteyev](http://www.freewisdom.org), [Waylan
-Limberg](http://achinghead.com/) and [Artem Yunusov](http://blog.splyer.com).
-
-Contact: markdown@freewisdom.org
-
-Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later)
-Copyright 200? Django Software Foundation (OrderedDict implementation)
-Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b)
-Copyright 2004 Manfred Stienstra (the original version)
-
-License: BSD (see docs/LICENSE for details).
-"""
-
-version = "2.0-alpha"
-version_info = (2,0,0, "beta")
-
-import re
-import sys
-import codecs
-import htmlentitydefs
-import logging
-from logging import DEBUG, INFO, WARN, ERROR, CRITICAL
-from urlparse import urlparse, urlunparse
-
-
-"""
-CONSTANTS
-=============================================================================
-"""
-
-"""
-Constants you might want to modify
------------------------------------------------------------------------------
-"""
-
-# default logging level for command-line use
-COMMAND_LINE_LOGGING_LEVEL = CRITICAL
-TAB_LENGTH = 4 # expand tabs to this many spaces
-ENABLE_ATTRIBUTES = True # @id = xyz -> <... id="xyz">
-SMART_EMPHASIS = True # this_or_that does not become this<i>or</i>that
-HTML_REMOVED_TEXT = "[HTML_REMOVED]" # text used instead of HTML in safe mode
-BLOCK_LEVEL_ELEMENTS = re.compile("p|div|h[1-6]|blockquote|pre|table|dl|ol|ul"
- +"|script|noscript|form|fieldset|iframe|math"
- +"|ins|del|hr|hr/|style|li|dt|dd|tr")
-
-"""
-Constants you probably do not need to change
------------------------------------------------------------------------------
-"""
-
-RTL_BIDI_RANGES = ( (u'\u0590', u'\u07FF'),
- # Hebrew (0590-05FF), Arabic (0600-06FF),
- # Syriac (0700-074F), Arabic supplement (0750-077F),
- # Thaana (0780-07BF), Nko (07C0-07FF).
- (u'\u2D30', u'\u2D7F'), # Tifinagh
- )
-
-EXECUTABLE_NAME_FOR_USAGE = "python markdown.py"
-""" The name used in the usage statement displayed for python versions < 2.3.
-(With python 2.3 and higher the usage statement is generated by optparse
-and uses the actual name of the executable called.) """
-
-# Placeholders
-STX = u'\u0002' # Use STX ("Start of text") for start-of-placeholder
-ETX = u'\u0003' # Use ETX ("End of text") for end-of-placeholder
-HTML_PLACEHOLDER_PREFIX = STX+"wzxhzdk:"
-HTML_PLACEHOLDER = HTML_PLACEHOLDER_PREFIX + "%d" + ETX
-INLINE_PLACEHOLDER_PREFIX = STX+"klzzwxh:"
-INLINE_PLACEHOLDER = INLINE_PLACEHOLDER_PREFIX + "%s" + ETX
-AMP_SUBSTITUTE = STX+"amp"+ETX
-
-
-"""
-AUXILIARY GLOBAL FUNCTIONS
-=============================================================================
-"""
-
-def message(level, text):
- """ A wrapper method for logging debug messages. """
- logging.getLogger('MARKDOWN').log(level, text)
-
-def isString(s):
- """ Check if it's string """
- return isinstance(s, unicode) or isinstance(s, str)
-
-## Import
-def importETree():
- """Import the best implementation of ElementTree, return a module object."""
- etree_in_c = None
- try: # Is it Python 2.5+ with C implemenation of ElementTree installed?
- import xml.etree.cElementTree as etree_in_c
- except ImportError:
- try: # Is it Python 2.5+ with Python implementation of ElementTree?
- import xml.etree.ElementTree as etree
- except ImportError:
- try: # An earlier version of Python with cElementTree installed?
- import cElementTree as etree_in_c
- except ImportError:
- try: # An earlier version of Python with Python ElementTree?
- import elementtree.ElementTree as etree
- except ImportError:
- message(CRITICAL, "Failed to import ElementTree")
- sys.exit(1)
- if etree_in_c and etree_in_c.VERSION < "1.0":
- message(CRITICAL, "For cElementTree version 1.0 or higher is required.")
- sys.exit(1)
- elif etree_in_c :
- return etree_in_c
- elif etree.VERSION < "1.1":
- message(CRITICAL, "For ElementTree version 1.1 or higher is required")
- sys.exit(1)
- else :
- return etree
-
-def isBlockLevel(tag):
- """Check if the tag is a block level HTML tag."""
- return BLOCK_LEVEL_ELEMENTS.match(tag)
-
-ATTR_RE = re.compile("\{@([^\}]*)=([^\}]*)}") # {@id=123}
-
-def handleAttributes(text, parent):
- """Set values of an element based on attribute definitions ({@id=123})."""
- def attributeCallback(match):
- parent.set(match.group(1), match.group(2))
- return ATTR_RE.sub(attributeCallback, text)
-
-def dequote(string):
- """Remove quotes from around a string."""
- if ( ( string.startswith('"') and string.endswith('"'))
- or (string.startswith("'") and string.endswith("'")) ):
- return string[1:-1]
- else:
- return string
-
-
-"""
-OVERALL DESIGN
-=============================================================================
-
-Markdown processing takes place in four steps:
-
-1. A bunch of "preprocessors" munge the input text.
-2. BlockParser() parses the high-level structural elements of the
- pre-processed text into an ElementTree.
-3. A bunch of "treeprocessors" are run against the ElementTree. One such
- treeprocessor runs InlinePatterns against the ElementTree, detecting inline
- markup.
-4. Some post-processors are run against the text after the ElementTree has
- been serialized into text.
-5. The output is written to a string.
-
-Those steps are put together by the Markdown() class.
-
-The code below is organized as follows:
-
-1. BlockParser and it's BlockProcessors - does core block parsing.
-2. All the preprocessors, patterns, treeprocessors, and postprocessors.
-3. Markdown class - does the high-level wrapping.
-"""
-
-
-"""
-CORE MARKDOWN BLOCKPARSER
-=============================================================================
-
-This parser handles basic parsing of Markdown blocks. It doesn't concern itself
-with inline elements such as **bold** or *italics*, but rather just catches
-blocks, lists, quotes, etc.
-
-The BlockParser is made up of a bunch of BlockProssors, each handling a
-different type of block. Extensions may add/replace/remove BlockProcessors
-as they need to alter how markdown blocks are parsed.
-
-"""
-
-class BlockProcessor:
- """ Base class for block processors.
-
- Each subclass will provide the methods below to work with the source and
- tree. Each processor will need to define it's own ``test`` and ``run``
- methods. The ``test`` method should return True or False, to indicate
- whether the current block should be processed by this processor. If the
- test passes, the parser will call the processors ``run`` method.
-
- """
-
- def __init__(self, parser=None):
- self.parser = parser
-
- def lastChild(self, parent):
- """ Return the last child of an etree element. """
- if len(parent):
- return parent[-1]
- else:
- return None
-
- def detab(self, text):
- """ Remove a tab from the front of each line of the given text. """
- newtext = []
- lines = text.split('\n')
- for line in lines:
- if line.startswith(' '*TAB_LENGTH):
- newtext.append(line[TAB_LENGTH:])
- elif not line.strip():
- newtext.append('')
- else:
- break
- return '\n'.join(newtext), '\n'.join(lines[len(newtext):])
-
- def looseDetab(self, text):
- """ Remove a tab from front of lines but allowing dedented lines. """
- lines = text.split('\n')
- for i in range(len(lines)):
- if lines[i].startswith(' '*TAB_LENGTH):
- lines[i] = lines[i][TAB_LENGTH:]
- return '\n'.join(lines)
-
- def test(self, parent, block):
- """ Test for block type. Must be overridden by subclasses.
-
- As the parser loops through processors, it will call the ``test`` method
- on each to determine if the given block of text is of that type. This
- method must return a boolean ``True`` or ``False``. The actual method of
- testing is left to the needs of that particular block type. It could
- be as simple as ``block.startswith(some_string)`` or a complex regular
- expression. As the block type may be different depending on the parent
- of the block (i.e. inside a list), the parent etree element is also
- provided and may be used as part of the test.
-
- Keywords:
-
- * ``parent``: A etree element which will be the parent of the block.
- * ``block``: A block of text from the source which has been split at
- blank lines.
- """
- pass
-
- def run(self, parent, blocks):
- """ Run processor. Must be overridden by subclasses.
-
- When the parser determines the appropriate type of a block, the parser
- will call the corresponding processor's ``run`` method. This method
- should parse the individual lines of the block and append them to
- the etree.
-
- Note that both the ``parent`` and ``etree`` keywords are pointers
- to instances of the objects which should be edited in place. Each
- processor must make changes to the existing objects as there is no
- mechanism to return new/different objects to replace them.
-
- This means that this method should be adding SubElements or adding text
- to the parent, and should remove (``pop``) or add (``insert``) items to
- the list of blocks.
-
- Keywords:
-
- * ``parent``: A etree element which is the parent of the current block.
- * ``blocks``: A list of all remaining blocks of the document.
- """
- pass
-
-
-class ListIndentProcessor(BlockProcessor):
- """ Process children of list items.
-
- Example:
- * a list item
- process this part
-
- or this part
-
- """
-
- def test(self, parent, block):
- return block.startswith(' '*TAB_LENGTH) and \
- (parent.tag == "li" or \
- (len(parent) and parent[-1] and \
- (parent[-1].tag == "ul" or parent[-1].tag == "ol")
- )
- )
-
- def run(self, parent, blocks):
- block = self.looseDetab(blocks.pop(0))
- sibling = self.lastChild(parent)
- if parent.tag == 'li':
- # The parent is already a li. Just parse the child block.
- self.parser.parseBlocks(parent, [block])
- elif len(sibling) and sibling[-1].tag == 'li':
- # The parent is a list (``ol`` or ``ul``) which has children.
- # Assume the last child li is the parent of this block.
- if sibling[-1].text:
- # If the parent li has text, that text needs to be moved to a p
- block = '%s\n\n%s' % (sibling[-1].text, block)
- sibling[-1].text = ''
- self.parser.parseChunk(sibling[-1], block)
- else:
- # Create a new li and parse the block with it as the parent.
- li = etree.SubElement(sibling, 'li')
- self.parser.parseBlocks(li, [block])
-
-
-class CodeBlockProcessor(BlockProcessor):
- """ Process code blocks. """
-
- def test(self, parent, block):
- return block.startswith(' '*TAB_LENGTH)
-
- def run(self, parent, blocks):
- sibling = self.lastChild(parent)
- block = blocks.pop(0)
- theRest = ''
- if sibling and sibling.tag == "pre" and len(sibling) \
- and sibling[0].tag == "code":
- # The previous block was a code block. As blank lines do not start
- # new code blocks, append this block to the previous, adding back
- # linebreaks removed from the split into a list.
- code = sibling[0]
- block, theRest = self.detab(block)
- code.text = AtomicString('%s\n%s\n' % (code.text, block.rstrip()))
- else:
- # This is a new codeblock. Create the elements and insert text.
- pre = etree.SubElement(parent, 'pre')
- code = etree.SubElement(pre, 'code')
- block, theRest = self.detab(block)
- code.text = AtomicString('%s\n' % block.rstrip())
- if theRest:
- # This block contained unindented line(s) after the first indented
- # line. Insert these lines as the first block of the master blocks
- # list for future processing.
- blocks.insert(0, theRest)
-
-
-class BlockQuoteProcessor(BlockProcessor):
-
- RE = re.compile(r'(^|\n)[ ]{0,3}>[ ](.*)')
-
- def test(self, parent, block):
- return bool(self.RE.search(block))
-
- def run(self, parent, blocks):
- block = blocks.pop(0)
- m = self.RE.search(block)
- if m:
- before = block[:m.start()] # Lines before blockquote
- # Pass lines before blockquote in recursively for parsing forst.
- self.parser.parseBlocks(parent, [before])
- # Remove ``> `` from begining of each line.
- block = '\n'.join([self.clean(line) for line in
- block[m.start():].split('\n')])
- sibling = self.lastChild(parent)
- if sibling and sibling.tag == "blockquote":
- # Previous block was a blockquote so set that as this blocks parent
- quote = sibling
- else:
- # This is a new blockquote. Create a new parent element.
- quote = etree.SubElement(parent, 'blockquote')
- # Recursively parse block with blockquote as parent.
- self.parser.parseChunk(quote, block)
-
- def clean(self, line):
- """ Remove ``>`` from beginning of a line. """
- m = self.RE.match(line)
- if line.strip() == ">":
- return ""
- elif m:
- return m.group(2)
- else:
- return line
-
-class OListProcessor(BlockProcessor):
- """ Process ordered list blocks. """
-
- TAG = 'ol'
- # Detect an item (``1. item``). ``group(1)`` contains contents of item.
- RE = re.compile(r'^[ ]{0,3}\d+\.[ ](.*)')
- # Detect items on secondary lines. they can be of either list type.
- CHILD_RE = re.compile(r'^[ ]{0,3}((\d+\.)|[*+-])[ ](.*)')
- # Detect indented (nested) items of either type
- INDENT_RE = re.compile(r'^[ ]{4,7}((\d+\.)|[*+-])[ ].*')
-
- def test(self, parent, block):
- return bool(self.RE.match(block))
-
- def run(self, parent, blocks):
- # Check fr multiple items in one block.
- items = self.get_items(blocks.pop(0))
- sibling = self.lastChild(parent)
- if sibling and (sibling.tag == 'ol' or sibling.tag == 'ul'):
- # Previous block was a list item, so set that as parent
- lst = sibling
- # make sure previous item is in a p.
- if len(lst) and lst[-1].text and not len(lst[-1]):
- p = etree.SubElement(lst[-1], 'p')
- p.text = lst[-1].text
- lst[-1].text = ''
- # parse first block differently as it gets wrapped in a p.
- li = etree.SubElement(lst, 'li')
- self.parser.state.set('looselist')
- firstitem = items.pop(0)
- self.parser.parseBlocks(li, [firstitem])
- self.parser.state.reset()
- else:
- # This is a new list so create parent with appropriate tag.
- lst = etree.SubElement(parent, self.TAG)
- self.parser.state.set('list')
- # Loop through items in block, recursively parsing each with the
- # appropriate parent.
- for item in items:
- if item.startswith(' '*TAB_LENGTH):
- # Item is indented. Parse with last item as parent
- self.parser.parseBlocks(lst[-1], [item])
- else:
- # New item. Create li and parse with it as parent
- li = etree.SubElement(lst, 'li')
- self.parser.parseBlocks(li, [item])
- self.parser.state.reset()
-
- def get_items(self, block):
- """ Break a block into list items. """
- items = []
- for line in block.split('\n'):
- m = self.CHILD_RE.match(line)
- if m:
- # This is a new item. Append
- items.append(m.group(3))
- elif self.INDENT_RE.match(line):
- # This is an indented (possibly nested) item.
- if items[-1].startswith(' '*TAB_LENGTH):
- # Previous item was indented. Append to that item.
- items[-1] = '%s\n%s' % (items[-1], line)
- else:
- items.append(line)
- else:
- # This is another line of previous item. Append to that item.
- items[-1] = '%s\n%s' % (items[-1], line)
- return items
-
-
-class UListProcessor(OListProcessor):
- """ Process unordered list blocks. """
-
- TAG = 'ul'
- RE = re.compile(r'^[ ]{0,3}[*+-][ ](.*)')
-
-
-class HashHeaderProcessor(BlockProcessor):
- """ Process Hash Headers. """
-
- # Detect a header at start of any line in block
- RE = re.compile(r'(^|\n)(?P<level>#{1,6})(?P<header>.*?)#*(\n|$)')
-
- def test(self, parent, block):
- return bool(self.RE.search(block))
-
- def run(self, parent, blocks):
- block = blocks.pop(0)
- m = self.RE.search(block)
- if m:
- before = block[:m.start()] # All lines before header
- after = block[m.end():] # All lines after header
- if before:
- # As the header was not the first line of the block and the
- # lines before the header must be parsed first,
- # recursively parse this lines as a block.
- self.parser.parseBlocks(parent, [before])
- # Create header using named groups from RE
- h = etree.SubElement(parent, 'h%d' % len(m.group('level')))
- h.text = m.group('header').strip()
- if after:
- # Insert remaining lines as first block for future parsing.
- blocks.insert(0, after)
- else:
- # This should never happen, but just in case...
- message(CRITICAL, "We've got a problem header!")
-
-
-class SetextHeaderProcessor(BlockProcessor):
- """ Process Setext-style Headers. """
-
- # Detect Setext-style header. Must be first 2 lines of block.
- RE = re.compile(r'^.*?\n[=-]{3,}', re.MULTILINE)
-
- def test(self, parent, block):
- return bool(self.RE.match(block))
-
- def run(self, parent, blocks):
- lines = blocks.pop(0).split('\n')
- # Determine level. ``=`` is 1 and ``-`` is 2.
- if lines[1].startswith('='):
- level = 1
- else:
- level = 2
- h = etree.SubElement(parent, 'h%d' % level)
- h.text = lines[0].strip()
- if len(lines) > 2:
- # Block contains additional lines. Add to master blocks for later.
- blocks.insert(0, '\n'.join(lines[2:]))
-
-
-class HRProcessor(BlockProcessor):
- """ Process Horizontal Rules. """
-
- RE = r'[ ]{0,3}(?P<ch>[*_-])[ ]?((?P=ch)[ ]?){2,}[ ]*'
- # Detect hr on any line of a block.
- SEARCH_RE = re.compile(r'(^|\n)%s(\n|$)' % RE)
- # Match a hr on a single line of text.
- MATCH_RE = re.compile(r'^%s$' % RE)
-
- def test(self, parent, block):
- return bool(self.SEARCH_RE.search(block))
-
- def run(self, parent, blocks):
- lines = blocks.pop(0).split('\n')
- prelines = []
- # Check for lines in block before hr.
- for line in lines:
- m = self.MATCH_RE.match(line)
- if m:
- break
- else:
- prelines.append(line)
- if len(prelines):
- # Recursively parse lines before hr so they get parsed first.
- self.parser.parseBlocks(parent, ['\n'.join(prelines)])
- # create hr
- hr = etree.SubElement(parent, 'hr')
- # check for lines in block after hr.
- lines = lines[len(prelines)+1:]
- if len(lines):
- # Add lines after hr to master blocks for later parsing.
- blocks.insert(0, '\n'.join(lines))
-
-
-class EmptyBlockProcessor(BlockProcessor):
- """ Process blocks and start with an empty line. """
-
- # Detect a block that only contains whitespace
- # or only whitespace on the first line.
- RE = re.compile(r'^\s*\n')
-
- def test(self, parent, block):
- return bool(self.RE.match(block))
-
- def run(self, parent, blocks):
- block = blocks.pop(0)
- m = self.RE.match(block)
- if m:
- # Add remaining line to master blocks for later.
- blocks.insert(0, block[m.end():])
- sibling = self.lastChild(parent)
- if sibling and sibling.tag == 'pre' and sibling[0] and \
- sibling[0].tag == 'code':
- # Last block is a codeblock. Append to preserve whitespace.
- sibling[0].text = AtomicString('%s/n/n/n' % sibling[0].text )
-
-
-class ParagraphProcessor(BlockProcessor):
- """ Process Paragraph blocks. """
-
- def test(self, parent, block):
- return True
-
- def run(self, parent, blocks):
- block = blocks.pop(0)
- if block.strip():
- # Not a blank block. Add to parent, otherwise throw it away.
- if self.parser.state.isstate('list'):
- # The parent is a tight-list. Append to parent.text
- if parent.text:
- parent.text = '%s\n%s' % (parent.text, block)
- else:
- parent.text = block.lstrip()
- else:
- # Create a regular paragraph
- p = etree.SubElement(parent, 'p')
- p.text = block.lstrip()
-
-class State(list):
- """ Track the current and nested state of the parser.
-
- This utility class is used to track the state of the BlockParser and
- support multiple levels if nesting. It's just a simple API wrapped around
- a list. Each time a state is set, that state is appended to the end of the
- list. Each time a state is reset, that state is removed from the end of
- the list.
-
- Therefore, each time a state is set for a nested block, that state must be
- reset when we back out of that level of nesting or the state could be
- corrupted.
-
- While all the methods of a list object are available, only the three
- defined below need be used.
-
- """
-
- def set(self, state):
- """ Set a new state. """
- self.append(state)
-
- def reset(self):
- """ Step back one step in nested state. """
- self.pop()
-
- def isstate(self, state):
- """ Test that top (current) level is of given state. """
- if len(self):
- return self[-1] == state
- else:
- return False
-
-class BlockParser:
- """ Parse Markdown blocks into an ElementTree object.
-
- A wrapper class that stitches the various BlockProcessors together,
- looping through them and creating an ElementTree object.
- """
-
- def __init__(self):
- self.blockprocessors = OrderedDict()
- self.blockprocessors['empty'] = EmptyBlockProcessor(self)
- self.blockprocessors['indent'] = ListIndentProcessor(self)
- self.blockprocessors['code'] = CodeBlockProcessor(self)
- self.blockprocessors['hashheader'] = HashHeaderProcessor(self)
- self.blockprocessors['setextheader'] = SetextHeaderProcessor(self)
- self.blockprocessors['hr'] = HRProcessor(self)
- self.blockprocessors['olist'] = OListProcessor(self)
- self.blockprocessors['ulist'] = UListProcessor(self)
- self.blockprocessors['quote'] = BlockQuoteProcessor(self)
- self.blockprocessors['paragraph'] = ParagraphProcessor(self)
- self.state = State()
-
- def parseDocument(self, lines):
- """ Parse a markdown document into an ElementTree.
-
- Given a list of lines, an ElementTree object (not just a parent Element)
- is created and the root element is passed to the parser as the parent.
- The ElementTree object is returned.
-
- This should only be called on an entire document, not pieces.
-
- """
- # Create a ElementTree from the lines
- root = etree.Element("div")
- self.parseChunk(root, '\n'.join(lines))
- return etree.ElementTree(root)
-
- def parseChunk(self, parent, text):
- """ Parse a chunk of markdown text and attach to given etree node.
-
- While the ``text`` argument is generally assumed to contain multiple
- blocks which will be split on blank lines, it could contain only one
- block. Generally, this method would be called by extensions when
- block parsing is required.
-
- The ``parent`` etree Element passed in is altered in place.
- Nothing is returned.
-
- """
- self.parseBlocks(parent, text.split('\n\n'))
-
- def parseBlocks(self, parent, blocks):
- """ Process blocks of markdown text and attach to given etree node.
-
- Given a list of ``blocks``, each blockprocessor is stepped through
- until there are no blocks left. While an extension could potentially
- call this method directly, it's generally expected to be used internally.
-
- This is a public method as an extension may need to add/alter additional
- BlockProcessors which call this method to recursively parse a nested
- block.
-
- """
- while blocks:
- for processor in self.blockprocessors.values():
- if processor.test(parent, blocks[0]):
- processor.run(parent, blocks)
- break
-
-
-"""
-PRE-PROCESSORS
-=============================================================================
-
-Preprocessors work on source text before we start doing anything too
-complicated.
-"""
-
-class Processor:
- def __init__(self, markdown_instance=None):
- if markdown_instance:
- self.markdown = markdown_instance
-
-
-class Preprocessor (Processor):
- """
- Preprocessors are run after the text is broken into lines.
-
- Each preprocessor implements a "run" method that takes a pointer to a
- list of lines of the document, modifies it as necessary and returns
- either the same pointer or a pointer to a new list.
-
- Preprocessors must extend markdown.Preprocessor.
-
- """
- def run(self, lines):
- """
- Each subclass of Preprocessor should override the `run` method, which
- takes the document as a list of strings split by newlines and returns
- the (possibly modified) list of lines.
-
- """
- pass
-
-
-class HtmlBlockPreprocessor(Preprocessor):
- """Remove html blocks from the text and store them for later retrieval."""
-
- right_tag_patterns = ["</%s>", "%s>"]
-
- def _get_left_tag(self, block):
- return block[1:].replace(">", " ", 1).split()[0].lower()
-
- def _get_right_tag(self, left_tag, block):
- for p in self.right_tag_patterns:
- tag = p % left_tag
- i = block.rfind(tag)
- if i > 2:
- return tag.lstrip("<").rstrip(">"), i + len(p)-2 + len(left_tag)
- return block.rstrip()[-len(left_tag)-2:-1].lower(), len(block)
-
- def _equal_tags(self, left_tag, right_tag):
- if left_tag == 'div' or left_tag[0] in ['?', '@', '%']: # handle PHP, etc.
- return True
- if ("/" + left_tag) == right_tag:
- return True
- if (right_tag == "--" and left_tag == "--"):
- return True
- elif left_tag == right_tag[1:] \
- and right_tag[0] != "<":
- return True
- else:
- return False
-
- def _is_oneliner(self, tag):
- return (tag in ['hr', 'hr/'])
-
- def run(self, lines):
- text = "\n".join(lines)
- new_blocks = []
- text = text.split("\n\n")
- items = []
- left_tag = ''
- right_tag = ''
- in_tag = False # flag
-
- while text:
- block = text[0]
- if block.startswith("\n"):
- block = block[1:]
- text = text[1:]
-
- if block.startswith("\n"):
- block = block[1:]
-
- if not in_tag:
- if block.startswith("<"):
- left_tag = self._get_left_tag(block)
- right_tag, data_index = self._get_right_tag(left_tag, block)
-
- if data_index < len(block):
- text.insert(0, block[data_index:])
- block = block[:data_index]
-
- if not (isBlockLevel(left_tag) \
- or block[1] in ["!", "?", "@", "%"]):
- new_blocks.append(block)
- continue
-
- if self._is_oneliner(left_tag):
- new_blocks.append(block.strip())
- continue
-
- if block[1] == "!":
- # is a comment block
- left_tag = "--"
- right_tag, data_index = self._get_right_tag(left_tag, block)
- # keep checking conditions below and maybe just append
-
- if block.rstrip().endswith(">") \
- and self._equal_tags(left_tag, right_tag):
- new_blocks.append(
- self.markdown.htmlStash.store(block.strip()))
- continue
- else: #if not block[1] == "!":
- # if is block level tag and is not complete
-
- if isBlockLevel(left_tag) or left_tag == "--" \
- and not block.rstrip().endswith(">"):
- items.append(block.strip())
- in_tag = True
- else:
- new_blocks.append(
- self.markdown.htmlStash.store(block.strip()))
-
- continue
-
- new_blocks.append(block)
-
- else:
- items.append(block.strip())
-
- right_tag, data_index = self._get_right_tag(left_tag, block)
-
- if self._equal_tags(left_tag, right_tag):
- # if find closing tag
- in_tag = False
- new_blocks.append(
- self.markdown.htmlStash.store('\n\n'.join(items)))
- items = []
-
- if items:
- new_blocks.append(self.markdown.htmlStash.store('\n\n'.join(items)))
- new_blocks.append('\n')
-
- new_text = "\n\n".join(new_blocks)
- return new_text.split("\n")
-
-
-class ReferencePreprocessor(Preprocessor):
- """ Remove reference definitions from text and store for later use. """
-
- RE = re.compile(r'^(\ ?\ ?\ ?)\[([^\]]*)\]:\s*([^ ]*)(.*)$', re.DOTALL)
-
- def run (self, lines):
- new_text = [];
- for line in lines:
- m = self.RE.match(line)
- if m:
- id = m.group(2).strip().lower()
- t = m.group(4).strip() # potential title
- if not t:
- self.markdown.references[id] = (m.group(3), t)
- elif (len(t) >= 2
- and (t[0] == t[-1] == "\""
- or t[0] == t[-1] == "\'"
- or (t[0] == "(" and t[-1] == ")") ) ):
- self.markdown.references[id] = (m.group(3), t[1:-1])
- else:
- new_text.append(line)
- else:
- new_text.append(line)
-
- return new_text #+ "\n"
-
-
-"""
-INLINE PATTERNS
-=============================================================================
-
-Inline patterns such as *emphasis* are handled by means of auxiliary
-objects, one per pattern. Pattern objects must be instances of classes
-that extend markdown.Pattern. Each pattern object uses a single regular
-expression and needs support the following methods:
-
- pattern.getCompiledRegExp() # returns a regular expression
-
- pattern.handleMatch(m) # takes a match object and returns
- # an ElementTree element or just plain text
-
-All of python markdown's built-in patterns subclass from Pattern,
-but you can add additional patterns that don't.
-
-Also note that all the regular expressions used by inline must
-capture the whole block. For this reason, they all start with
-'^(.*)' and end with '(.*)!'. In case with built-in expression
-Pattern takes care of adding the "^(.*)" and "(.*)!".
-
-Finally, the order in which regular expressions are applied is very
-important - e.g. if we first replace http://.../ links with <a> tags
-and _then_ try to replace inline html, we would end up with a mess.
-So, we apply the expressions in the following order:
-
-* escape and backticks have to go before everything else, so
- that we can preempt any markdown patterns by escaping them.
-
-* then we handle auto-links (must be done before inline html)
-
-* then we handle inline HTML. At this point we will simply
- replace all inline HTML strings with a placeholder and add
- the actual HTML to a hash.
-
-* then inline images (must be done before links)
-
-* then bracketed links, first regular then reference-style
-
-* finally we apply strong and emphasis
-"""
-
-
-"""
-The actual regular expressions for patterns
------------------------------------------------------------------------------
-"""
-
-NOBRACKET = r'[^\]\[]*'
-BRK = ( r'\[('
- + (NOBRACKET + r'(\[')*6
- + (NOBRACKET+ r'\])*')*6
- + NOBRACKET + r')\]' )
-NOIMG = r'(?<!\!)'
-
-BACKTICK_RE = r'(?<!\\)(`+)(.+?)(?<!`)\2(?!`)' # `e=f()` or ``e=f("`")``
-ESCAPE_RE = r'\\(.)' # \<
-EMPHASIS_RE = r'(\*)([^\*]*)\2' # *emphasis*
-STRONG_RE = r'(\*{2}|_{2})(.*?)\2' # **strong**
-STRONG_EM_RE = r'(\*{3}|_{3})(.*?)\2' # ***strong***
-
-if SMART_EMPHASIS:
- EMPHASIS_2_RE = r'(?<!\S)(_)(\S.*?)\2' # _emphasis_
-else:
- EMPHASIS_2_RE = r'(_)(.*?)\2' # _emphasis_
-
-LINK_RE = NOIMG + BRK + \
-r'''\(\s*(<.*?>|((?:(?:\(.*?\))|[^\(\)]))*?)\s*((['"])(.*)\12)?\)'''
-# [text](url) or [text](<url>)
-
-IMAGE_LINK_RE = r'\!' + BRK + r'\s*\((<.*?>|([^\)]*))\)'
-# ![alttxt](http://x.com/) or ![alttxt](<http://x.com/>)
-REFERENCE_RE = NOIMG + BRK+ r'\s*\[([^\]]*)\]' # [Google][3]
-IMAGE_REFERENCE_RE = r'\!' + BRK + '\s*\[([^\]]*)\]' # ![alt text][2]
-NOT_STRONG_RE = r'( \* )' # stand-alone * or _
-AUTOLINK_RE = r'<((?:f|ht)tps?://[^>]*)>' # <http://www.123.com>
-AUTOMAIL_RE = r'<([^> \!]*@[^> ]*)>' # <me@example.com>
-
-HTML_RE = r'(\<([a-zA-Z/][^\>]*?|\!--.*?--)\>)' # <...>
-ENTITY_RE = r'(&[\#a-zA-Z0-9]*;)' # &amp;
-LINE_BREAK_RE = r' \n' # two spaces at end of line
-LINE_BREAK_2_RE = r' $' # two spaces at end of text
-
-
-"""
-The pattern classes
------------------------------------------------------------------------------
-"""
-
-class Pattern:
- """Base class that inline patterns subclass. """
-
- def __init__ (self, pattern, markdown_instance=None):
- """
- Create an instant of an inline pattern.
-
- Keyword arguments:
-
- * pattern: A regular expression that matches a pattern
-
- """
- self.pattern = pattern
- self.compiled_re = re.compile("^(.*?)%s(.*?)$" % pattern, re.DOTALL)
-
- # Api for Markdown to pass safe_mode into instance
- self.safe_mode = False
- if markdown_instance:
- self.markdown = markdown_instance
-
- def getCompiledRegExp (self):
- """ Return a compiled regular expression. """
- return self.compiled_re
-
- def handleMatch(self, m):
- """Return a ElementTree element from the given match.
-
- Subclasses should override this method.
-
- Keyword arguments:
-
- * m: A re match object containing a match of the pattern.
-
- """
- pass
-
- def type(self):
- """ Return class name, to define pattern type """
- return self.__class__.__name__
-
-BasePattern = Pattern # for backward compatibility
-
-class SimpleTextPattern (Pattern):
- """ Return a simple text of group(2) of a Pattern. """
- def handleMatch(self, m):
- text = m.group(2)
- if text == INLINE_PLACEHOLDER_PREFIX:
- return None
- return text
-
-class SimpleTagPattern (Pattern):
- """
- Return element of type `tag` with a text attribute of group(3)
- of a Pattern.
-
- """
- def __init__ (self, pattern, tag):
- Pattern.__init__(self, pattern)
- self.tag = tag
-
- def handleMatch(self, m):
- el = etree.Element(self.tag)
- el.text = m.group(3)
- return el
-
-
-class SubstituteTagPattern (SimpleTagPattern):
- """ Return a eLement of type `tag` with no children. """
- def handleMatch (self, m):
- return etree.Element(self.tag)
-
-
-class BacktickPattern (Pattern):
- """ Return a `<code>` element containing the matching text. """
- def __init__ (self, pattern):
- Pattern.__init__(self, pattern)
- self.tag = "code"
-
- def handleMatch(self, m):
- el = etree.Element(self.tag)
- el.text = AtomicString(m.group(3).strip())
- return el
-
-
-class DoubleTagPattern (SimpleTagPattern):
- """Return a ElementTree element nested in tag2 nested in tag1.
-
- Useful for strong emphasis etc.
-
- """
- def handleMatch(self, m):
- tag1, tag2 = self.tag.split(",")
- el1 = etree.Element(tag1)
- el2 = etree.SubElement(el1, tag2)
- el2.text = m.group(3)
- return el1
-
-
-class HtmlPattern (Pattern):
- """ Store raw inline html and return a placeholder. """
- def handleMatch (self, m):
- rawhtml = m.group(2)
- inline = True
- place_holder = self.markdown.htmlStash.store(rawhtml)
- return place_holder
-
-
-class LinkPattern (Pattern):
- """ Return a link element from the given match. """
- def handleMatch(self, m):
- el = etree.Element("a")
- el.text = m.group(2)
- title = m.group(11)
- href = m.group(9)
-
- if href:
- if href[0] == "<":
- href = href[1:-1]
- el.set("href", self.sanitize_url(href.strip()))
- else:
- el.set("href", "")
-
- if title:
- title = dequote(title) #.replace('"', "&quot;")
- el.set("title", title)
- return el
-
- def sanitize_url(self, url):
- """
- Sanitize a url against xss attacks in "safe_mode".
-
- Rather than specifically blacklisting `javascript:alert("XSS")` and all
- its aliases (see <http://ha.ckers.org/xss.html>), we whitelist known
- safe url formats. Most urls contain a network location, however some
- are known not to (i.e.: mailto links). Script urls do not contain a
- location. Additionally, for `javascript:...`, the scheme would be
- "javascript" but some aliases will appear to `urlparse()` to have no
- scheme. On top of that relative links (i.e.: "foo/bar.html") have no
- scheme. Therefore we must check "path", "parameters", "query" and
- "fragment" for any literal colons. We don't check "scheme" for colons
- because it *should* never have any and "netloc" must allow the form:
- `username:password@host:port`.
-
- """
- locless_schemes = ['', 'mailto', 'news']
- scheme, netloc, path, params, query, fragment = url = urlparse(url)
- safe_url = False
- if netloc != '' or scheme in locless_schemes:
- safe_url = True
-
- for part in url[2:]:
- if ":" in part:
- safe_url = False
-
- if self.markdown.safeMode and not safe_url:
- return ''
- else:
- return urlunparse(url)
-
-class ImagePattern(LinkPattern):
- """ Return a img element from the given match. """
- def handleMatch(self, m):
- el = etree.Element("img")
- src_parts = m.group(9).split()
- if src_parts:
- src = src_parts[0]
- if src[0] == "<" and src[-1] == ">":
- src = src[1:-1]
- el.set('src', self.sanitize_url(src))
- else:
- el.set('src', "")
- if len(src_parts) > 1:
- el.set('title', dequote(" ".join(src_parts[1:])))
-
- if ENABLE_ATTRIBUTES:
- truealt = handleAttributes(m.group(2), el)
- else:
- truealt = m.group(2)
-
- el.set('alt', truealt)
- return el
-
-class ReferencePattern(LinkPattern):
- """ Match to a stored reference and return link element. """
- def handleMatch(self, m):
- if m.group(9):
- id = m.group(9).lower()
- else:
- # if we got something like "[Google][]"
- # we'll use "google" as the id
- id = m.group(2).lower()
-
- if not self.markdown.references.has_key(id): # ignore undefined refs
- return None
- href, title = self.markdown.references[id]
-
- text = m.group(2)
- return self.makeTag(href, title, text)
-
- def makeTag(self, href, title, text):
- el = etree.Element('a')
-
- el.set('href', self.sanitize_url(href))
- if title:
- el.set('title', title)
-
- el.text = text
- return el
-
-
-class ImageReferencePattern (ReferencePattern):
- """ Match to a stored reference and return img element. """
- def makeTag(self, href, title, text):
- el = etree.Element("img")
- el.set("src", self.sanitize_url(href))
- if title:
- el.set("title", title)
- el.set("alt", text)
- return el
-
-
-class AutolinkPattern (Pattern):
- """ Return a link Element given an autolink (`<http://example/com>`). """
- def handleMatch(self, m):
- el = etree.Element("a")
- el.set('href', m.group(2))
- el.text = AtomicString(m.group(2))
- return el
-
-class AutomailPattern (Pattern):
- """
- Return a mailto link Element given an automail link (`<foo@example.com>`).
- """
- def handleMatch(self, m):
- el = etree.Element('a')
- email = m.group(2)
- if email.startswith("mailto:"):
- email = email[len("mailto:"):]
-
- def codepoint2name(code):
- """Return entity definition by code, or the code if not defined."""
- entity = htmlentitydefs.codepoint2name.get(code)
- if entity:
- return "%s%s;" % (AMP_SUBSTITUTE, entity)
- else:
- return "%s#%d;" % (AMP_SUBSTITUTE, code)
-
- letters = [codepoint2name(ord(letter)) for letter in email]
- el.text = AtomicString(''.join(letters))
-
- mailto = "mailto:" + email
- mailto = "".join([AMP_SUBSTITUTE + '#%d;' %
- ord(letter) for letter in mailto])
- el.set('href', mailto)
- return el
-
-
-"""
-POST-PROCESSORS
-=============================================================================
-
-Markdown also allows post-processors, which are similar to preprocessors in
-that they need to implement a "run" method. However, they are run after core
-processing.
-
-There are two types of post-processors: Treeprocessor and Postprocessor
-"""
-
-class Treeprocessor(Processor):
- """
- Treeprocessors are run on the ElementTree object before serialization.
-
- Each Treeprocessor implements a "run" method that takes a pointer to an
- ElementTree, modifies it as necessary and returns an ElementTree
- object.
-
- Treeprocessors must extend markdown.Treeprocessor.
-
- """
- def run(self, root):
- """
- Subclasses of Treeprocessor should implement a `run` method, which
- takes a root ElementTree. This method can return another ElementTree
- object, and the existing root ElementTree will be replaced, or it can
- modify the current tree and return None.
- """
- pass
-
-
-class InlineProcessor(Treeprocessor):
- """
- A Treeprocessor that traverses a tree, applying inline patterns.
- """
-
- def __init__ (self, md):
- self.__placeholder_prefix = INLINE_PLACEHOLDER_PREFIX
- self.__placeholder_suffix = ETX
- self.__placeholder_length = 4 + len(self.__placeholder_prefix) \
- + len(self.__placeholder_suffix)
- self.__placeholder_re = re.compile(INLINE_PLACEHOLDER % r'([0-9]{4})')
- self.markdown = md
-
- def __makePlaceholder(self, type):
- """ Generate a placeholder """
- id = "%04d" % len(self.stashed_nodes)
- hash = INLINE_PLACEHOLDER % id
- return hash, id
-
- def __findPlaceholder(self, data, index):
- """
- Extract id from data string, start from index
-
- Keyword arguments:
-
- * data: string
- * index: index, from which we start search
-
- Returns: placeholder id and string index, after the found placeholder.
- """
-
- m = self.__placeholder_re.search(data, index)
- if m:
- return m.group(1), m.end()
- else:
- return None, index + 1
-
- def __stashNode(self, node, type):
- """ Add node to stash """
- placeholder, id = self.__makePlaceholder(type)
- self.stashed_nodes[id] = node
- return placeholder
-
- def __handleInline(self, data, patternIndex=0):
- """
- Process string with inline patterns and replace it
- with placeholders
-
- Keyword arguments:
-
- * data: A line of Markdown text
- * patternIndex: The index of the inlinePattern to start with
-
- Returns: String with placeholders.
-
- """
- if not isinstance(data, AtomicString):
- startIndex = 0
- while patternIndex < len(self.markdown.inlinePatterns):
- data, matched, startIndex = self.__applyPattern(
- self.markdown.inlinePatterns.value_for_index(patternIndex),
- data, patternIndex, startIndex)
- if not matched:
- patternIndex += 1
- return data
-
- def __processElementText(self, node, subnode, isText=True):
- """
- Process placeholders in Element.text or Element.tail
- of Elements popped from self.stashed_nodes.
-
- Keywords arguments:
-
- * node: parent node
- * subnode: processing node
- * isText: bool variable, True - it's text, False - it's tail
-
- Returns: None
-
- """
- if isText:
- text = subnode.text
- subnode.text = None
- else:
- text = subnode.tail
- subnode.tail = None
-
- childResult = self.__processPlaceholders(text, subnode)
-
- if not isText and node is not subnode:
- pos = node.getchildren().index(subnode)
- node.remove(subnode)
- else:
- pos = 0
-
- childResult.reverse()
- for newChild in childResult:
- node.insert(pos, newChild)
-
- def __processPlaceholders(self, data, parent):
- """
- Process string with placeholders and generate ElementTree tree.
-
- Keyword arguments:
-
- * data: string with placeholders instead of ElementTree elements.
- * parent: Element, which contains processing inline data
-
- Returns: list with ElementTree elements with applied inline patterns.
- """
- def linkText(text):
- if text:
- if result:
- if result[-1].tail:
- result[-1].tail += text
- else:
- result[-1].tail = text
- else:
- if parent.text:
- parent.text += text
- else:
- parent.text = text
-
- result = []
- strartIndex = 0
- while data:
- index = data.find(self.__placeholder_prefix, strartIndex)
- if index != -1:
- id, phEndIndex = self.__findPlaceholder(data, index)
-
- if self.stashed_nodes.has_key(id):
- node = self.stashed_nodes.get(id)
-
- if index > 0:
- text = data[strartIndex:index]
- linkText(text)
-
- if not isString(node): # it's Element
- for child in [node] + node.getchildren():
- if child.tail:
- if child.tail.strip():
- self.__processElementText(node, child, False)
- if child.text:
- if child.text.strip():
- self.__processElementText(child, child)
- else: # it's just a string
- linkText(node)
- strartIndex = phEndIndex
- continue
-
- strartIndex = phEndIndex
- result.append(node)
-
- else: # wrong placeholder
- end = index + len(prefix)
- linkText(data[strartIndex:end])
- strartIndex = end
- else:
- text = data[strartIndex:]
- linkText(text)
- data = ""
-
- return result
-
- def __applyPattern(self, pattern, data, patternIndex, startIndex=0):
- """
- Check if the line fits the pattern, create the necessary
- elements, add it to stashed_nodes.
-
- Keyword arguments:
-
- * data: the text to be processed
- * pattern: the pattern to be checked
- * patternIndex: index of current pattern
- * startIndex: string index, from which we starting search
-
- Returns: String with placeholders instead of ElementTree elements.
-
- """
- match = pattern.getCompiledRegExp().match(data[startIndex:])
- leftData = data[:startIndex]
-
- if not match:
- return data, False, 0
-
- node = pattern.handleMatch(match)
-
- if node is None:
- return data, True, len(leftData) + match.span(len(match.groups()))[0]
-
- if not isString(node):
- if not isinstance(node.text, AtomicString):
- # We need to process current node too
- for child in [node] + node.getchildren():
- if not isString(node):
- if child.text:
- child.text = self.__handleInline(child.text,
- patternIndex + 1)
- if child.tail:
- child.tail = self.__handleInline(child.tail,
- patternIndex)
-
- placeholder = self.__stashNode(node, pattern.type())
-
- return "%s%s%s%s" % (leftData,
- match.group(1),
- placeholder, match.groups()[-1]), True, 0
-
- def run(self, tree):
- """Apply inline patterns to a parsed Markdown tree.
-
- Iterate over ElementTree, find elements with inline tag, apply inline
- patterns and append newly created Elements to tree. If you don't
- want process your data with inline paterns, instead of normal string,
- use subclass AtomicString:
-
- node.text = AtomicString("data won't be processed with inline patterns")
-
- Arguments:
-
- * markdownTree: ElementTree object, representing Markdown tree.
-
- Returns: ElementTree object with applied inline patterns.
-
- """
- self.stashed_nodes = {}
-
- stack = [tree]
-
- while stack:
- currElement = stack.pop()
- insertQueue = []
- for child in currElement.getchildren():
- if child.text and not isinstance(child.text, AtomicString):
- text = child.text
- child.text = None
- lst = self.__processPlaceholders(self.__handleInline(
- text), child)
- stack += lst
- insertQueue.append((child, lst))
-
- if child.getchildren():
- stack.append(child)
-
- for element, lst in insertQueue:
- if element.text:
- element.text = handleAttributes(element.text, element)
- i = 0
- for newChild in lst:
- # Processing attributes
- if newChild.tail:
- newChild.tail = handleAttributes(newChild.tail,
- element)
- if newChild.text:
- newChild.text = handleAttributes(newChild.text,
- newChild)
- element.insert(i, newChild)
- i += 1
-
- return tree
-
-
-class PrettifyTreeprocessor(Treeprocessor):
- """Add linebreaks to the html document."""
- def _prettifyETree(self, elem):
- """Recursively add linebreaks to ElementTree children."""
- i = "\n"
- if isBlockLevel(elem.tag) and elem.tag not in ['code', 'pre']:
- if (not elem.text or not elem.text.strip()) \
- and len(elem) and isBlockLevel(elem[0].tag):
- elem.text = i
- for e in elem:
- if isBlockLevel(e.tag):
- self._prettifyETree(e)
- if not elem.tail or not elem.tail.strip():
- elem.tail = i
- if not elem.tail or not elem.tail.strip():
- elem.tail = i
-
- def run(self, root):
- """.Add linebreaks to ElementTree root object."""
- self._prettifyETree(root)
- # Do <br />'s seperately as they are often in the middle of
- # inline content and missed by _prettifyETree.
- brs = root.getiterator('br')
- for br in brs:
- if not br.tail or not br.tail.strip():
- br.tail = '\n'
- else:
- br.tail = '\n%s' % br.tail
-
-
-class Postprocessor(Processor):
- """
- Postprocessors are run after the ElementTree it converted back into text.
-
- Each Postprocessor implements a "run" method that takes a pointer to a
- text string, modifies it as necessary and returns a text string.
-
- Postprocessors must extend markdown.Postprocessor.
-
- """
-
- def run(self, text):
- """
- Subclasses of Postprocessor should implement a `run` method, which
- takes the html document as a single text string and returns a
- (possibly modified) string.
-
- """
- pass
-
-
-
-class RawHtmlPostprocessor(Postprocessor):
- """ Restore raw html to the document. """
-
- def run(self, text):
- """ Iterate over html stash and restore "safe" html. """
- for i in range(self.markdown.htmlStash.html_counter):
- html, safe = self.markdown.htmlStash.rawHtmlBlocks[i]
- if self.markdown.safeMode and not safe:
- if str(self.markdown.safeMode).lower() == 'escape':
- html = self.escape(html)
- elif str(self.markdown.safeMode).lower() == 'remove':
- html = ''
- else:
- html = HTML_REMOVED_TEXT
- if safe or not self.markdown.safeMode:
- text = text.replace("<p>%s</p>" % (HTML_PLACEHOLDER % i),
- html + "\n")
- text = text.replace(HTML_PLACEHOLDER % i, html)
- return text
-
- def escape(self, html):
- """ Basic html escaping """
- html = html.replace('&', '&amp;')
- html = html.replace('<', '&lt;')
- html = html.replace('>', '&gt;')
- return html.replace('"', '&quot;')
-
-
-class AndSubstitutePostprocessor(Postprocessor):
- """ Restore valid entities """
- def __init__(self):
- pass
-
- def run(self, text):
- text = text.replace(AMP_SUBSTITUTE, "&")
- return text
-
-
-"""
-MISC AUXILIARY CLASSES
-=============================================================================
-"""
-
-class AtomicString(unicode):
- """A string which should not be further processed."""
- pass
-
-
-class HtmlStash:
- """
- This class is used for stashing HTML objects that we extract
- in the beginning and replace with place-holders.
- """
-
- def __init__ (self):
- """ Create a HtmlStash. """
- self.html_counter = 0 # for counting inline html segments
- self.rawHtmlBlocks=[]
-
- def store(self, html, safe=False):
- """
- Saves an HTML segment for later reinsertion. Returns a
- placeholder string that needs to be inserted into the
- document.
-
- Keyword arguments:
-
- * html: an html segment
- * safe: label an html segment as safe for safemode
-
- Returns : a placeholder string
-
- """
- self.rawHtmlBlocks.append((html, safe))
- placeholder = HTML_PLACEHOLDER % self.html_counter
- self.html_counter += 1
- return placeholder
-
- def reset(self):
- self.html_counter = 0
- self.rawHtmlBlocks = []
-
-class OrderedDict(dict):
- """
- A dictionary that keeps its keys in the order in which they're inserted.
-
- Copied from Django's SortedDict with some modifications.
-
- """
- def __new__(cls, *args, **kwargs):
- instance = super(OrderedDict, cls).__new__(cls, *args, **kwargs)
- instance.keyOrder = []
- return instance
-
- def __init__(self, data=None):
- if data is None:
- data = {}
- super(OrderedDict, self).__init__(data)
- if isinstance(data, dict):
- self.keyOrder = data.keys()
- else:
- self.keyOrder = []
- for key, value in data:
- if key not in self.keyOrder:
- self.keyOrder.append(key)
-
- def __deepcopy__(self, memo):
- from copy import deepcopy
- return self.__class__([(key, deepcopy(value, memo))
- for key, value in self.iteritems()])
-
- def __setitem__(self, key, value):
- super(OrderedDict, self).__setitem__(key, value)
- if key not in self.keyOrder:
- self.keyOrder.append(key)
-
- def __delitem__(self, key):
- super(OrderedDict, self).__delitem__(key)
- self.keyOrder.remove(key)
-
- def __iter__(self):
- for k in self.keyOrder:
- yield k
-
- def pop(self, k, *args):
- result = super(OrderedDict, self).pop(k, *args)
- try:
- self.keyOrder.remove(k)
- except ValueError:
- # Key wasn't in the dictionary in the first place. No problem.
- pass
- return result
-
- def popitem(self):
- result = super(OrderedDict, self).popitem()
- self.keyOrder.remove(result[0])
- return result
-
- def items(self):
- return zip(self.keyOrder, self.values())
-
- def iteritems(self):
- for key in self.keyOrder:
- yield key, super(OrderedDict, self).__getitem__(key)
-
- def keys(self):
- return self.keyOrder[:]
-
- def iterkeys(self):
- return iter(self.keyOrder)
-
- def values(self):
- return [super(OrderedDict, self).__getitem__(k) for k in self.keyOrder]
-
- def itervalues(self):
- for key in self.keyOrder:
- yield super(OrderedDict, self).__getitem__(key)
-
- def update(self, dict_):
- for k, v in dict_.items():
- self.__setitem__(k, v)
-
- def setdefault(self, key, default):
- if key not in self.keyOrder:
- self.keyOrder.append(key)
- return super(OrderedDict, self).setdefault(key, default)
-
- def value_for_index(self, index):
- """Return the value of the item at the given zero-based index."""
- return self[self.keyOrder[index]]
-
- def insert(self, index, key, value):
- """Insert the key, value pair before the item with the given index."""
- if key in self.keyOrder:
- n = self.keyOrder.index(key)
- del self.keyOrder[n]
- if n < index:
- index -= 1
- self.keyOrder.insert(index, key)
- super(OrderedDict, self).__setitem__(key, value)
-
- def copy(self):
- """Return a copy of this object."""
- # This way of initializing the copy means it works for subclasses, too.
- obj = self.__class__(self)
- obj.keyOrder = self.keyOrder[:]
- return obj
-
- def __repr__(self):
- """
- Replace the normal dict.__repr__ with a version that returns the keys
- in their sorted order.
- """
- return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()])
-
- def clear(self):
- super(OrderedDict, self).clear()
- self.keyOrder = []
-
- def index(self, key):
- """ Return the index of a given key. """
- return self.keyOrder.index(key)
-
- def index_for_location(self, location):
- """ Return index or None for a given location. """
- if location == '_begin':
- i = 0
- elif location == '_end':
- i = None
- elif location.startswith('<') or location.startswith('>'):
- i = self.index(location[1:])
- if location.startswith('>'):
- if i >= len(self):
- # last item
- i = None
- else:
- i += 1
- else:
- raise ValueError('Not a valid location: "%s". Location key '
- 'must start with a ">" or "<".' % location)
- return i
-
- def add(self, key, value, location):
- """ Insert by key location. """
- i = self.index_for_location(location)
- if i is not None:
- self.insert(i, key, value)
- else:
- self.__setitem__(key, value)
-
- def link(self, key, location):
- """ Change location of an existing item. """
- n = self.keyOrder.index(key)
- del self.keyOrder[n]
- i = self.index_for_location(location)
- try:
- if i is not None:
- self.keyOrder.insert(i, key)
- else:
- self.keyOrder.append(key)
- except Error:
- # restore to prevent data loss and reraise
- self.keyOrder.insert(n, key)
- raise Error
-
-
-"""
-Markdown
-=============================================================================
-"""
-
-class Markdown:
- """Convert Markdown to HTML."""
-
- def __init__(self,
- extensions=[],
- extension_configs={},
- safe_mode = False):
- """
- Creates a new Markdown instance.
-
- Keyword arguments:
-
- * extensions: A list of extensions.
- If they are of type string, the module mdx_name.py will be loaded.
- If they are a subclass of markdown.Extension, they will be used
- as-is.
- * extension-configs: Configuration setting for extensions.
- * safe_mode: Disallow raw html. One of "remove", "replace" or "escape".
-
- """
- self.parser = BlockParser()
- self.safeMode = safe_mode
- self.registeredExtensions = []
- self.docType = ""
- self.stripTopLevelTags = True
-
- self.preprocessors = OrderedDict()
- self.preprocessors["html_block"] = HtmlBlockPreprocessor(self)
- self.preprocessors["reference"] = ReferencePreprocessor(self)
- # footnote preprocessor will be inserted with "<reference"
-
- self.treeprocessors = OrderedDict()
- self.treeprocessors["inline"] = InlineProcessor(self)
- self.treeprocessors["prettify"] = PrettifyTreeprocessor(self)
-
- self.postprocessors = OrderedDict()
- self.postprocessors["raw_html"] = RawHtmlPostprocessor(self)
- self.postprocessors["amp_substitute"] = AndSubstitutePostprocessor()
- # footnote postprocessor will be inserted with ">amp_substitute"
-
- self.prePatterns = []
-
- self.inlinePatterns = OrderedDict()
- self.inlinePatterns["backtick"] = BacktickPattern(BACKTICK_RE)
- self.inlinePatterns["escape"] = SimpleTextPattern(ESCAPE_RE)
- self.inlinePatterns["reference"] = ReferencePattern(REFERENCE_RE, self)
- self.inlinePatterns["link"] = LinkPattern(LINK_RE, self)
- self.inlinePatterns["image_link"] = ImagePattern(IMAGE_LINK_RE, self)
- self.inlinePatterns["image_reference"] = \
- ImageReferencePattern(IMAGE_REFERENCE_RE, self)
- self.inlinePatterns["autolink"] = AutolinkPattern(AUTOLINK_RE, self)
- self.inlinePatterns["automail"] = AutomailPattern(AUTOMAIL_RE, self)
- self.inlinePatterns["linebreak2"] = \
- SubstituteTagPattern(LINE_BREAK_2_RE, 'br')
- self.inlinePatterns["linebreak"] = \
- SubstituteTagPattern(LINE_BREAK_RE, 'br')
- self.inlinePatterns["html"] = HtmlPattern(HTML_RE, self)
- self.inlinePatterns["entity"] = HtmlPattern(ENTITY_RE, self)
- self.inlinePatterns["not_strong"] = SimpleTextPattern(NOT_STRONG_RE)
- self.inlinePatterns["strong_em"] = \
- DoubleTagPattern(STRONG_EM_RE, 'strong,em')
- self.inlinePatterns["strong"] = SimpleTagPattern(STRONG_RE, 'strong')
- self.inlinePatterns["emphasis"] = SimpleTagPattern(EMPHASIS_RE, 'em')
- self.inlinePatterns["emphasis2"] = \
- SimpleTagPattern(EMPHASIS_2_RE, 'em')
- # The order of the handlers matters!!!
-
- self.references = {}
- self.htmlStash = HtmlStash()
- self.registerExtensions(extensions = extensions,
- configs = extension_configs)
- self.reset()
-
- def registerExtensions(self, extensions, configs):
- """
- Register extensions with this instance of Markdown.
-
- Keyword aurguments:
-
- * extensions: A list of extensions, which can either
- be strings or objects. See the docstring on Markdown.
- * configs: A dictionary mapping module names to config options.
-
- """
- for ext in extensions:
- if isinstance(ext, basestring):
- ext = load_extension(ext, configs.get(ext, []))
- elif hasattr(ext, 'extendMarkdown'):
- # Looks like an Extension.
- # Nothing to do here.
- pass
- else:
- message(ERROR, "Incorrect type! Extension '%s' is "
- "neither a string or an Extension." %(repr(ext)))
- continue
- ext.extendMarkdown(self, globals())
-
- def registerExtension(self, extension):
- """ This gets called by the extension """
- self.registeredExtensions.append(extension)
-
- def reset(self):
- """
- Resets all state variables so that we can start with a new text.
- """
- self.htmlStash.reset()
- self.references.clear()
-
- for extension in self.registeredExtensions:
- extension.reset()
-
- def convert (self, source):
- """Convert markdown to serialized XHTML."""
-
- # Fixup the source text
- if not source:
- return u"" # a blank unicode string
- try:
- source = unicode(source)
- except UnicodeDecodeError:
- message(CRITICAL, 'UnicodeDecodeError: Markdown only accepts unicode or ascii input.')
- return u""
-
- source = source.replace(STX, "").replace(ETX, "")
- source = source.replace("\r\n", "\n").replace("\r", "\n") + "\n\n"
- source = re.sub(r'\n\s+\n', '\n\n', source)
- source = source.expandtabs(TAB_LENGTH)
-
- # Split into lines and run the line preprocessors.
- self.lines = source.split("\n")
- for prep in self.preprocessors.values():
- self.lines = prep.run(self.lines)
-
- # Parse the high-level elements.
- root = self.parser.parseDocument(self.lines).getroot()
-
- # Run the tree-processors
- for treeprocessor in self.treeprocessors.values():
- newRoot = treeprocessor.run(root)
- if newRoot:
- root = newRoot
-
- # Serialize _properly_. Strip top-level tags.
- xml, length = codecs.utf_8_decode(etree.tostring(root, encoding="utf8"))
- if self.stripTopLevelTags:
- xml = xml.strip()[44:-7] + "\n"
-
- # Run the text post-processors
- for pp in self.postprocessors.values():
- xml = pp.run(xml)
-
- return xml.strip()
-
- def convertFile(self, input = None, output = None, encoding = None):
- """Converts a markdown file and returns the HTML as a unicode string.
-
- Decodes the file using the provided encoding (defaults to utf-8),
- passes the file content to markdown, and outputs the html to either
- the provided stream or the file with provided name, using the same
- encoding as the source file.
-
- **Note:** This is the only place that decoding and encoding of unicode
- takes place in Python-Markdown. (All other code is unicode-in /
- unicode-out.)
-
- Keyword arguments:
-
- * input: Name of source text file.
- * output: Name of output file. Writes to stdout if `None`.
- * extensions: A list of extension names (may contain config args).
- * encoding: Encoding of input and output files. Defaults to utf-8.
- * safe_mode: Disallow raw html. One of "remove", "replace" or "escape".
-
- """
-
- encoding = encoding or "utf-8"
-
- # Read the source
- input_file = codecs.open(input, mode="r", encoding=encoding)
- text = input_file.read()
- input_file.close()
- text = text.lstrip(u'\ufeff') # remove the byte-order mark
-
- # Convert
- html = self.convert(text)
-
- # Write to file or stdout
- if type(output) == type("string"):
- output_file = codecs.open(output, "w", encoding=encoding)
- output_file.write(html)
- output_file.close()
- else:
- output.write(html.encode(encoding))
-
-
-"""
-Extensions
------------------------------------------------------------------------------
-"""
-
-class Extension:
- """ Base class for extensions to subclass. """
- def __init__(self, configs = {}):
- """Create an instance of an Extention.
-
- Keyword arguments:
-
- * configs: A dict of configuration setting used by an Extension.
- """
- self.config = configs
-
- def getConfig(self, key):
- """ Return a setting for the given key or an empty string. """
- if self.config.has_key(key):
- return self.config[key][0]
- else:
- return ""
-
- def getConfigInfo(self):
- """ Return all config settings as a list of tuples. """
- return [(key, self.config[key][1]) for key in self.config.keys()]
-
- def setConfig(self, key, value):
- """ Set a config setting for `key` with the given `value`. """
- self.config[key][0] = value
-
- def extendMarkdown(self, md, md_globals):
- """
- Add the various proccesors and patterns to the Markdown Instance.
-
- This method must be overriden by every extension.
-
- Keyword arguments:
-
- * md: The Markdown instance.
-
- * md_globals: Global variables in the markdown module namespace.
-
- """
- pass
-
-def load_extension(ext_name, configs = []):
- """Load extension by name, then return the module.
-
- The extension name may contain arguments as part of the string in the
- following format: "extname(key1=value1,key2=value2)"
-
- """
-
- # Parse extensions config params (ignore the order)
- configs = dict(configs)
- pos = ext_name.find("(") # find the first "("
- if pos > 0:
- ext_args = ext_name[pos+1:-1]
- ext_name = ext_name[:pos]
- pairs = [x.split("=") for x in ext_args.split(",")]
- configs.update([(x.strip(), y.strip()) for (x, y) in pairs])
-
- # Setup the module names
- ext_module = 'markdown_extensions'
- module_name_new_style = '.'.join([ext_module, ext_name])
- module_name_old_style = '_'.join(['mdx', ext_name])
-
- # Try loading the extention first from one place, then another
- try: # New style (markdown_extensons.<extension>)
- module = __import__(module_name_new_style, {}, {}, [ext_module])
- except ImportError:
- try: # Old style (mdx.<extension>)
- module = __import__(module_name_old_style)
- except ImportError:
- message(CRITICAL, "Failed loading extension '%s' from '%s' or '%s'"
- % (ext_name, module_name_new_style, module_name_old_style))
-
- # If the module is loaded successfully, we expect it to define a
- # function called makeExtension()
- try:
- return module.makeExtension(configs.items())
- except:
- message(CRITICAL, "Failed to instantiate extension '%s'" % ext_name)
-
-def load_extensions(ext_names):
- """Loads multiple extensions"""
- extensions = []
- for ext_name in ext_names:
- extension = load_extension(ext_name)
- if extension:
- extensions.append(extension)
- return extensions
-
-# Extensions should use "markdown.etree" instead of "etree" (or do `from
-# markdown import etree`). Do not import it by yourself.
-
-etree = importETree()
-
-"""
-EXPORTED FUNCTIONS
-=============================================================================
-
-Those are the two functions we really mean to export: markdown() and
-markdownFromFile().
-"""
-
-def markdown(text,
- extensions = [],
- safe_mode = False):
- """Convert a markdown string to HTML and return HTML as a unicode string.
-
- This is a shortcut function for `Markdown` class to cover the most
- basic use case. It initializes an instance of Markdown, loads the
- necessary extensions and runs the parser on the given text.
-
- Keyword arguments:
-
- * text: Markdown formatted text as Unicode or ASCII string.
- * extensions: A list of extensions or extension names (may contain config args).
- * safe_mode: Disallow raw html. One of "remove", "replace" or "escape".
-
- Returns: An HTML document as a string.
-
- """
- md = Markdown(extensions=load_extensions(extensions),
- safe_mode = safe_mode)
- return md.convert(text)
-
-
-def markdownFromFile(input = None,
- output = None,
- extensions = [],
- encoding = None,
- safe = False):
- """Read markdown code from a file and write it to a file or a stream."""
- md = Markdown(extensions=load_extensions(extensions), safe_mode = safe)
- md.convertFile(input, output, encoding)
-
-
-"""
-COMMAND-LINE SPECIFIC STUFF
-=============================================================================
-
-The rest of the code is specifically for handling the case where Python
-Markdown is called from the command line.
-"""
-
-OPTPARSE_WARNING = """
-Python 2.3 or higher required for advanced command line options.
-For lower versions of Python use:
-
- %s INPUT_FILE > OUTPUT_FILE
-
-""" % EXECUTABLE_NAME_FOR_USAGE
-
-def parse_options():
- """
- Define and parse `optparse` options for command-line usage.
- """
-
- try:
- optparse = __import__("optparse")
- except:
- if len(sys.argv) == 2:
- return {'input': sys.argv[1],
- 'output': None,
- 'safe': False,
- 'extensions': [],
- 'encoding': None }, CRITICAL
- else:
- print OPTPARSE_WARNING
- return None, None
-
- parser = optparse.OptionParser(usage="%prog INPUTFILE [options]")
- parser.add_option("-f", "--file", dest="filename", default=sys.stdout,
- help="write output to OUTPUT_FILE",
- metavar="OUTPUT_FILE")
- parser.add_option("-e", "--encoding", dest="encoding",
- help="encoding for input and output files",)
- parser.add_option("-q", "--quiet", default = CRITICAL,
- action="store_const", const=CRITICAL+10, dest="verbose",
- help="suppress all messages")
- parser.add_option("-v", "--verbose",
- action="store_const", const=INFO, dest="verbose",
- help="print info messages")
- parser.add_option("-s", "--safe", dest="safe", default=False,
- metavar="SAFE_MODE",
- help="safe mode ('replace', 'remove' or 'escape' user's HTML tag)")
- parser.add_option("--noisy",
- action="store_const", const=DEBUG, dest="verbose",
- help="print debug messages")
- parser.add_option("-x", "--extension", action="append", dest="extensions",
- help = "load extension EXTENSION", metavar="EXTENSION")
-
- (options, args) = parser.parse_args()
-
- if not len(args) == 1:
- parser.print_help()
- return None, None
- else:
- input_file = args[0]
-
- if not options.extensions:
- options.extensions = []
-
- return {'input': input_file,
- 'output': options.filename,
- 'safe': options.safe,
- 'extensions': options.extensions,
- 'encoding': options.encoding }, options.verbose
-
-def command_line_run():
- """Run Markdown from the command line."""
-
- # Setup a logger manually for compatibility with Python 2.3
- logger = logging.getLogger('MARKDOWN')
- logger.setLevel(COMMAND_LINE_LOGGING_LEVEL)
- logger.addHandler(logging.StreamHandler())
-
- # Parse options and adjust logging level if necessary
- options, logging_level = parse_options()
- if not options: sys.exit(0)
- if logging_level: logging.getLogger('MARKDOWN').setLevel(logging_level)
-
- # Run
- markdownFromFile(**options)
+from markdown import commandline
if __name__ == '__main__':
- command_line_run()
+ commandline.run()