Combined the TextPreprocessors and Preprocessors into Preprocessors. Updated extensions and docs as well.

3 files changed, 13 insertions, 66 deletions

diff --git a/docs/writing_extensions.txt b/docs/writing_extensions.txt
index 617b92e..9d6fa42 100755
--- a/docs/writing_extensions.txt
+++ b/docs/writing_extensions.txt
@@ -19,8 +19,6 @@ helpful as well. For example, the [[Footnotes]] extension uses most of the
 features documented here.
 
 * [Preprocessors][]
-    * [TextPreprocessors][]
-    * [Line Preprocessors][]
 * [InlinePatterns][]
 * [Treeprocessors][] 
 * [Postprocessors][]
@@ -39,30 +37,7 @@ Preprocessors munge the source text before it is passed into the Markdown
 core. This is an excellent place to clean up bad syntax, extract things the 
 parser may otherwise choke on and perhaps even store it for later retrieval.
 
-There are two types of preprocessors: [TextPreprocessors][] and 
-[Line Preprocessors][].
-
-<h4 id="textpreprocessors">TextPreprocessors</h4>
-
-TextPreprocessors should inherit from ``markdown.TextPreprocessor`` and 
-implement a ``run`` method with one argument ``text``. The ``run`` method of 
-each TextPreprocessor will be passed the entire source text as a single Unicode
-string and should either return that single Unicode string, or an altered
-version of it.
-
-For example, a simple TextPreprocessor that normalizes newlines [^1] might look
-like this:
-
-    class NormalizePreprocessor(markdown.TextPreprocessor):
-        def run(self, text):
-            return text.replace("\r\n", "\n").replace("\r", "\n")
-
-[^1]: It should be noted that Markdown already normalizes newlines. This 
-example is for illustrative purposes only.
-
-<h4 id="linepreprocessors">Line Preprocessors</h4>
-
-Line Preprocessors should inherit from ``markdown.Preprocessor`` and implement 
+Preprocessors should inherit from ``markdown.Preprocessor`` and implement 
 a ``run`` method with one argument ``lines``. The ``run`` method of each Line
 Preprocessor will be passed the entire source text as a list of Unicode strings.
 Each string will contain one line of text. The ``run`` method should return
@@ -457,8 +432,6 @@ This is useful if you need to implement a large number of extensions with more
 than one residing in a module.
 
 [Preprocessors]: #preprocessors
-[TextPreprocessors]: #textpreprocessors
-[Line Preprocessors]: #linepreprocessors
 [InlinePatterns]: #inlinepatterns
 [Treeprocessors]: #treeprocessors
 [Postprocessors]: #postprocessors
diff --git a/markdown.py b/markdown.py
index c16b6d6..a8b4455 100755
--- a/markdown.py
+++ b/markdown.py
@@ -581,8 +581,7 @@ PRE-PROCESSORS
 =============================================================================
 
 Preprocessors work on source text before we start doing anything too
-complicated.  There are two types of preprocessors: TextPreprocessor and
-Preprocessor.
+complicated. 
 """
 
 class Processor:
@@ -590,27 +589,6 @@ class Processor:
         if markdown_instance:
             self.markdown = markdown_instance
 
-class TextPreprocessor (Processor):
-    """
-    TextPreprocessors are run before the text is broken into lines.
-
-    Each TextPreprocessor implements a "run" method that takes a pointer to a
-    text string of the document, modifies it as necessary and returns
-    either the same pointer or a pointer to a new string.
-
-    TextPreprocessors must extend markdown.TextPreprocessor.
-
-    """
-
-    def run(self, text):
-        """
-        Each subclass of TextPreprocessor should override the `run` method,
-        which takes the document text as a single string and returns the
-        (possibly modified) document as a single string.
-
-        """
-        pass
-
 
 class Preprocessor (Processor):
     """
@@ -633,7 +611,7 @@ class Preprocessor (Processor):
         pass
 
 
-class HtmlBlockPreprocessor(TextPreprocessor):
+class HtmlBlockPreprocessor(Preprocessor):
     """Remove html blocks from the text and store them for later retrieval."""
 
     right_tag_patterns = ["</%s>", "%s>"]
@@ -665,7 +643,8 @@ class HtmlBlockPreprocessor(TextPreprocessor):
     def _is_oneliner(self, tag):
         return (tag in ['hr', 'hr/'])
 
-    def run(self, text):
+    def run(self, lines):
+        text = "\n".join(lines)
         new_blocks = []
         text = text.split("\n\n")
         items = []
@@ -742,7 +721,8 @@ class HtmlBlockPreprocessor(TextPreprocessor):
             new_blocks.append(self.markdown.htmlStash.store('\n\n'.join(items)))
             new_blocks.append('\n')
 
-        return "\n\n".join(new_blocks)
+        new_text = "\n\n".join(new_blocks)
+        return new_text.split("\n")
 
 
 class HeaderPreprocessor(Preprocessor):
@@ -1781,12 +1761,9 @@ class Markdown:
         self.docType = ""
         self.stripTopLevelTags = True
 
-        self.textPreprocessors = Treap()
-        self.textPreprocessors.add("html_block",
-                                   HtmlBlockPreprocessor(self))
         self.preprocessors = Treap()
+        self.preprocessors.add("html_block", HtmlBlockPreprocessor(self))
         self.preprocessors.add("header", HeaderPreprocessor(self))
-        
         self.preprocessors.add("line", LinePreprocessor(self))
         self.preprocessors.add("reference", ReferencePreprocessor(self))
         # footnote preprocessor will be inserted with "<reference"
@@ -1890,10 +1867,6 @@ class Markdown:
         source = source.replace("\r\n", "\n").replace("\r", "\n") + "\n\n"
         source = source.expandtabs(TAB_LENGTH)
 
-        # Run the text preprocessors
-        for pp in self.textPreprocessors.heapsorted():
-            source = pp.run(source)
-
         # Split into lines and run the line preprocessors.
         self.lines = source.split("\n")
         for prep in self.preprocessors.heapsorted():
diff --git a/markdown_extensions/fenced_code.py b/markdown_extensions/fenced_code.py
index 1825106..fa768c0 100644
--- a/markdown_extensions/fenced_code.py
+++ b/markdown_extensions/fenced_code.py
@@ -76,15 +76,16 @@ class FencedCodeExtension(markdown.Extension):
     def extendMarkdown(self, md, md_globals):
         """ Add FencedBlockPreprocessor to the Markdown instance. """
 
-        md.textPreprocessors.add('fenced_code_block', 
+        md.preprocessors.add('fenced_code_block', 
                                  FencedBlockPreprocessor(md), 
                                  "_begin")
 
 
-class FencedBlockPreprocessor(markdown.TextPreprocessor):
+class FencedBlockPreprocessor(markdown.Preprocessor):
     
-    def run(self, text):
+    def run(self, lines):
         """ Match and store Fenced Code Blocks in the HtmlStash. """
+        text = "\n".join(lines)
         while 1:
             m = FENCED_BLOCK_RE.search(text)
             if m:
@@ -96,7 +97,7 @@ class FencedBlockPreprocessor(markdown.TextPreprocessor):
                 text = '%s\n%s\n%s'% (text[:m.start()], placeholder, text[m.end():])
             else:
                 break
-        return text
+        return text.split("\n")
 
     def _escape(self, txt):
         """ basic html escaping """