- changed default colors for syntax coloring

- disabled syntax coloring for texts larger than 20KB (too slow)
- added alot of comments to the syntax coloring code

3 files changed, 109 insertions, 65 deletions

diff --git a/Source/CMTextView.m b/Source/CMTextView.m
index 12b253da..cd00cafb 100644
--- a/Source/CMTextView.m
+++ b/Source/CMTextView.m
@@ -25,7 +25,7 @@
 #import "SPStringAdditions.h"
 
 /*
-all the extern variables and prototypes required for flex (syntax highlighting)
+all the extern variables and prototypes required for flex (used for syntax highlighting)
 */
 #import "tokens.h"
 extern int yylex();
@@ -84,7 +84,10 @@ YY_BUFFER_STATE yy_scan_string (const char *);
 }
 
 
-
+/*
+List of keywords for autocompletion. If you add a keyword here,
+it should also be added to the flex file tokens.l
+*/
 -(NSArray *)keywords {
 	return [NSArray arrayWithObjects:
 	@"ADD",
@@ -374,62 +377,77 @@ sets self as delegate for the textView's textStorage to enable syntax highlighti
 
 - (void)textStorageDidProcessEditing:(NSNotification *)notification
 /*
-performs syntax highlighting
-this method recolors the entire text on every keypress. for a single sql query,
-that should be no problem, but it could be a nuisance when editing large sql queries
-*/
+ performs syntax highlighting
+ This method recolors the entire text on every keypress. For performance reasons, this function does
+ nothing if the text is more than a few KB.
+ 
+ The main bottleneck is the [NSTextStorage addAttribute:value:range:] method - the parsing itself is really fast!
+ 
+ Some sample code from Andrew Choi ( http://members.shaw.ca/akochoi-old/blog/2003/11-09/index.html#3 ) has been reused.
+ */
 {
     NSTextStorage *textStore = [notification object];
     
     //make sure that the notification is from the correct textStorage object
-    if (textStore==[self textStorage])
-    {
+    if (textStore!=[self textStorage]) return;
     
-        NSColor *reservedColor = [NSColor blueColor];
-        NSColor *quoteColor    = [NSColor grayColor];
-        NSColor *commentColor  = [NSColor redColor ];
-        
-        NSString *string = [textStore string];
-        unsigned int length = [string length];
-        int token;
-        NSRange textRange, tokenRange;
-        
-        textRange = NSMakeRange(0, length);
-        
-        //first remove the old colors
-        [textStore removeAttribute:NSForegroundColorAttributeName range:textRange];
-
-        //initialise flex
-        yyuoffset = 0; yyuleng = 0;
-        yy_switch_to_buffer(yy_scan_string([[textStore string] UTF8String]));
-
-        //now loop through all the tokens
-        while (token=yylex()){
-            tokenRange = NSMakeRange(yyuoffset, yyuleng);  //convert the result from flex to an NSRange
-            tokenRange = NSIntersectionRange(tokenRange, textRange); // make sure that tokenRange is valid (and therefore within textRange)
-                                                                     // otherwise a bug in the lex code could cause the the TextView to crash
-            
-            switch (token) {
+    
+    NSColor *commentColor   = [NSColor colorWithDeviceRed:0.000 green:0.455 blue:0.000 alpha:1.000];
+    NSColor *quoteColor     = [NSColor colorWithDeviceRed:0.769 green:0.102 blue:0.086 alpha:1.000];
+    NSColor *keywordColor   = [NSColor colorWithDeviceRed:0.200 green:0.250 blue:1.000 alpha:1.000];
+    
+    NSColor *tokenColor;
+    
+    int token;
+    NSRange textRange, tokenRange;
+    
+    textRange = NSMakeRange(0, [textStore length]);
+    
+    //don't color texts longer than about 20KB. would be too slow
+    if (textRange.length > 20000) return; 
+    
+    //first remove the old colors
+    [textStore removeAttribute:NSForegroundColorAttributeName range:textRange];
+    
+    
+    //initialise flex
+    yyuoffset = 0; yyuleng = 0;
+    yy_switch_to_buffer(yy_scan_string([[textStore string] UTF8String]));
+    
+    //now loop through all the tokens
+    while (token=yylex()){
+        switch (token) {
             case SPT_SINGLE_QUOTED_TEXT:
             case SPT_DOUBLE_QUOTED_TEXT:
-                [textStore addAttribute: NSForegroundColorAttributeName
-                                  value: quoteColor
-                                  range: tokenRange ];
+                tokenColor = quoteColor;
                 break;
             case SPT_RESERVED_WORD:
-                [textStore addAttribute: NSForegroundColorAttributeName
-                                  value: reservedColor
-                                  range: tokenRange ];
+                tokenColor = keywordColor;
                 break;
             case SPT_COMMENT:
-                [textStore addAttribute: NSForegroundColorAttributeName
-                                  value: commentColor
-                                  range: tokenRange ];
+                tokenColor = commentColor;
                 break;
-        }}
+            default:
+                tokenColor = nil;
+        }
+        
+        if (!tokenColor) continue;
+        
+        tokenRange = NSMakeRange(yyuoffset, yyuleng);
+        
+        // make sure that tokenRange is valid (and therefore within textRange)
+        // otherwise a bug in the lex code could cause the the TextView to crash
+        tokenRange = NSIntersectionRange(tokenRange, textRange); 
+        if (!tokenRange.length) continue;
+        
+        [textStore addAttribute: NSForegroundColorAttributeName
+                          value: tokenColor
+                          range: tokenRange ];
         
     }
+    
 }
 
 
+
 @end
diff --git a/Source/tokens.h b/Source/tokens.h
index 81ea830f..ff4b70b0 100644
--- a/Source/tokens.h
+++ b/Source/tokens.h
@@ -12,4 +12,6 @@
 #define SPT_BACKTICK_QUOTED_TEXT 3
 #define SPT_RESERVED_WORD        4
 #define SPT_COMMENT              5
-#define SPT_OTHER                6
\ No newline at end of file
+#define SPT_WHITESPACE           6
+#define SPT_WORD                 7
+#define SPT_OTHER                8
\ No newline at end of file
diff --git a/Source/tokens.l b/Source/tokens.l
index 941378c3..e9c6ab88 100644
--- a/Source/tokens.l
+++ b/Source/tokens.l
@@ -1,31 +1,52 @@
 %{
+
+/*
+ * tokens.l - created by Jakob on 3/15/09 for Sequel Pro
+ *
+ * This is the lex file used for syntax coloring.
+ * To add new keywords, just add a line where the other
+ * keywords are and replace spaces with {s}
+ *
+ * If you're new to lex and interested what the code below does, I found
+ * "The Lex And Yacc Page" at http://dinosaur.compilertools.net/ to be
+ * very helpful. Keep in mind that Xcode actually uses flex, the GNU
+ * version of lex. There's a very thorough Texinfo manual for flex
+ * available. (type 'info flex' in the Terminal)
+ */
+ 
 #import "tokens.h"
 int utf8strlen(const char *s);
+int yyuoffset, yyuleng;
 
 #define YY_NO_UNPUT
 
-int yyuoffset, yyuleng;
-  
+//keep track of the current utf-8 character (not byte) offset and token length
 #define YY_USER_ACTION { yyuoffset += yyuleng; yyuleng = utf8strlen(yytext); }
 %}
 %option noyywrap
 %option case-insensitive
 s		[ \t\n]+
 word	[a-z_0-9]
-nonword	[^a-z_0-9#]
+nonword	[^a-z_0-9#\n\t]
 %x comment
 %%
-\"([^"\\]|\\(.|\n))*\"?				{ return SPT_DOUBLE_QUOTED_TEXT;   } /* double quoted strings */
-'([^'\\]|\\(.|\n))*'?				{ return SPT_SINGLE_QUOTED_TEXT;   } /* single quoted strings */
-`[^`]*`?						{ return SPT_BACKTICK_QUOTED_TEXT; } /* identifier quoting    */
-"/*"							{ BEGIN(comment); return SPT_COMMENT; }
-<comment>[^*]* 				    { return SPT_COMMENT; }
-<comment>"*"+                   { return SPT_COMMENT; }
-<comment>"*"+"/" 				{ BEGIN(INITIAL); return SPT_COMMENT; }
-							 
-#[^\n]*\n?				        |									 
---[ \t][^\n]*\n?						{ return SPT_COMMENT;  }		     /* -- Comments           */
-{s}								{ return SPT_OTHER;    } 								 /* ignore spaces         */
+\"([^"\\]|\\(.|\n))*\"?			{ return SPT_DOUBLE_QUOTED_TEXT;   }      /* double quoted strings          */
+'([^'\\]|\\(.|\n))*'?			{ return SPT_SINGLE_QUOTED_TEXT;   }      /* single quoted strings          */
+`[^`]*`?						{ return SPT_BACKTICK_QUOTED_TEXT; }      /* identifier quoting             */
+
+"/*"							{ BEGIN(comment); return SPT_COMMENT; }   /* beginning of a c style comment */
+<comment>[^*]* 				    { return SPT_COMMENT; }                   /* anything except * in a c cmnt  */
+<comment>"*"+                   { return SPT_COMMENT; }                   /* a range of *                   */
+<comment>"*"+"/" 				{ BEGIN(INITIAL); return SPT_COMMENT; }   /* a range of * with trailing /   
+                                                                             Thanks to John Dickinson for publishing 
+                                                                             this method of parsing C comments on
+                                                                             http://www.stillhq.com/pdfdb/000561/data.pdf
+                                                                                                            */
+                                                                                                            
+#[^\n]*\n?				        |									      /* # Comments                     */
+--[ \t][^\n]*\n?				{ return SPT_COMMENT;  }		          /* -- Comments                    */
+
+{s}								{ return SPT_WHITESPACE;    }		      /* ignore spaces                  */
 ADD                         	|
 ALL                         	|
 ALTER{s}TABLE               	|
@@ -296,12 +317,12 @@ WITH                        	|
 WRITE                       	|
 XOR                         	|
 YEAR_MONTH                  	|
-ZEROFILL            			{ return SPT_RESERVED_WORD;        } /* all the mysql reserved words  */
-{word}+							{ return SPT_OTHER;                } /* return any word  */
-{nonword}						{ return SPT_OTHER;                } /* return anything else  */
+ZEROFILL            			{ return SPT_RESERVED_WORD;        }   /* all the mysql reserved words  */
+{word}+							{ return SPT_WORD;                 }   /* return any word               */
+{nonword}						{ return SPT_OTHER;                }   /* return anything else          */
                     			
 <<EOF>>   						{
-                                    BEGIN(INITIAL);
+                                    BEGIN(INITIAL);   /* make sure we return to initial state when finished! */
             						yy_delete_buffer(YY_CURRENT_BUFFER);
             						return 0;
           						}
@@ -309,9 +330,12 @@ ZEROFILL            			{ return SPT_RESERVED_WORD;        } /* all the mysql res
 
 int utf8strlen(const char *s)
 /*
-This simple function calculates the length of an UTF8 string in characters (not bytes)
-It's not especially fast, but it's easy to comprehend
-*/
+ This simple function calculates the string length of an UTF-8 string
+ It's fast enough and easy to comprehend
+ 
+ Adapted from Kragen Javier Sitaker's my_strlen_utf8_c function as
+ found on http://canonical.org/~kragen/strlen-utf8.html
+ */
 {
     int j=0;
 	while (*s)