Move some duplicate code into it's own file

5 files changed, 140 insertions, 110 deletions

diff --git a/Source/SPEditorTokens.l b/Source/SPEditorTokens.l
index c10dbd60..a40105a1 100644
--- a/Source/SPEditorTokens.l
+++ b/Source/SPEditorTokens.l
@@ -43,8 +43,8 @@
  */
  
 #import "SPEditorTokens.h"
+#include "SPParserUtils.h"
 
-size_t utf8strlen(const char * _s);
 size_t yyuoffset, yyuleng;
 
 //keep track of the current utf-8 character (not byte) offset and token length
@@ -126,59 +126,6 @@ keywords	(X(OR|509|A)|S(MALLINT|SL|H(OW({s}(E(NGINE(S)?|RRORS)|M(ASTER|UTEX)|BIN
           						}
 %%
 
-#define ONEMASK ((size_t)(-1) / 0xFF)
-// adapted from http://www.daemonology.net/blog/2008-06-05-faster-utf8-strlen.html
-size_t utf8strlen(const char * _s)
-{
-	const char * s;
-	size_t count = 0;
-	size_t u;
-	unsigned char b;
-
-	/* Handle any initial misaligned bytes. */
-	for (s = _s; (uintptr_t)(s) & (sizeof(size_t) - 1); s++) {
-		b = *s;
-
-		/* Exit if we hit a zero byte. */
-		if (b == '\0')
-			goto done;
-
-		/* Is this byte NOT the first byte of a character? */
-		count += (b >> 7) & ((~b) >> 6);
-	}
-
-	/* Handle complete blocks. */
-	for (; ; s += sizeof(size_t)) {
-		/* Prefetch 256 bytes ahead. */
-		__builtin_prefetch(&s[256], 0, 0);
-
-		/* Grab 4 or 8 bytes of UTF-8 data. */
-		u = *(size_t *)(s);
-
-		/* Exit the loop if there are any zero bytes. */
-		if ((u - ONEMASK) & (~u) & (ONEMASK * 0x80))
-			break;
-
-		/* Count bytes which are NOT the first byte of a character. */
-		u = ((u & (ONEMASK * 0x80)) >> 7) & ((~u) >> 6);
-		count += (u * ONEMASK) >> ((sizeof(size_t) - 1) * 8);
-	}
-
-	/* Take care of any left-over bytes. */
-	for (; ; s++) {
-		b = *s;
-
-		/* Exit if we hit a zero byte. */
-		if (b == '\0')
-			break;
-
-		/* Is this byte NOT the first byte of a character? */
-		count += (b >> 7) & ((~b) >> 6);
-	}
-
-done:
-	return ((s - _s) - count);
-}
 
 /* un-optimized keywords:
 ACCESSIBLE
diff --git a/Source/SPParserUtils.c b/Source/SPParserUtils.c
new file mode 100644
index 00000000..b3b48945
--- /dev/null
+++ b/Source/SPParserUtils.c
@@ -0,0 +1,87 @@
+//
+//  SPParserUtils.c
+//  sequel-pro
+//
+//  Created by Max Lohrmann on 27.01.15.
+//  Relocated from existing files. Previous copyright applies.
+//
+//  Permission is hereby granted, free of charge, to any person
+//  obtaining a copy of this software and associated documentation
+//  files (the "Software"), to deal in the Software without
+//  restriction, including without limitation the rights to use,
+//  copy, modify, merge, publish, distribute, sublicense, and/or sell
+//  copies of the Software, and to permit persons to whom the
+//  Software is furnished to do so, subject to the following
+//  conditions:
+//
+//  The above copyright notice and this permission notice shall be
+//  included in all copies or substantial portions of the Software.
+//
+//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+//  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+//  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+//  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+//  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+//  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+//  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+//  OTHER DEALINGS IN THE SOFTWARE.
+//
+//  More info at <https://github.com/sequelpro/sequelpro>
+
+#include "SPParserUtils.h"
+#include <stdint.h>
+
+#define ONEMASK ((size_t)(-1) / 0xFF)
+
+// adapted from http://www.daemonology.net/blog/2008-06-05-faster-utf8-strlen.html
+size_t utf8strlen(const char * _s)
+{
+	const char * s;
+	size_t count = 0;
+	size_t u;
+	unsigned char b;
+	
+	/* Handle any initial misaligned bytes. */
+	for (s = _s; (uintptr_t)(s) & (sizeof(size_t) - 1); s++) {
+		b = *s;
+		
+		/* Exit if we hit a zero byte. */
+		if (b == '\0')
+			goto done;
+		
+		/* Is this byte NOT the first byte of a character? */
+		count += (b >> 7) & ((~b) >> 6);
+	}
+	
+	/* Handle complete blocks. */
+	for (; ; s += sizeof(size_t)) {
+		/* Prefetch 256 bytes ahead. */
+		__builtin_prefetch(&s[256], 0, 0);
+		
+		/* Grab 4 or 8 bytes of UTF-8 data. */
+		u = *(size_t *)(s);
+		
+		/* Exit the loop if there are any zero bytes. */
+		if ((u - ONEMASK) & (~u) & (ONEMASK * 0x80))
+			break;
+		
+		/* Count bytes which are NOT the first byte of a character. */
+		u = ((u & (ONEMASK * 0x80)) >> 7) & ((~u) >> 6);
+		count += (u * ONEMASK) >> ((sizeof(size_t) - 1) * 8);
+	}
+	
+	/* Take care of any left-over bytes. */
+	for (; ; s++) {
+		b = *s;
+		
+		/* Exit if we hit a zero byte. */
+		if (b == '\0')
+			break;
+		
+		/* Is this byte NOT the first byte of a character? */
+		count += (b >> 7) & ((~b) >> 6);
+	}
+	
+done:
+	return ((s - _s) - count);
+}
diff --git a/Source/SPParserUtils.h b/Source/SPParserUtils.h
new file mode 100644
index 00000000..487131e0
--- /dev/null
+++ b/Source/SPParserUtils.h
@@ -0,0 +1,41 @@
+//
+//  SPParserUtils.h
+//  sequel-pro
+//
+//  Created by Max Lohrmann on 27.01.15.
+//  Relocated from existing files. Previous copyright applies.
+//
+//  Permission is hereby granted, free of charge, to any person
+//  obtaining a copy of this software and associated documentation
+//  files (the "Software"), to deal in the Software without
+//  restriction, including without limitation the rights to use,
+//  copy, modify, merge, publish, distribute, sublicense, and/or sell
+//  copies of the Software, and to permit persons to whom the
+//  Software is furnished to do so, subject to the following
+//  conditions:
+//
+//  The above copyright notice and this permission notice shall be
+//  included in all copies or substantial portions of the Software.
+//
+//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+//  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+//  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+//  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+//  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+//  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+//  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+//  OTHER DEALINGS IN THE SOFTWARE.
+//
+//  More info at <https://github.com/sequelpro/sequelpro>
+
+#ifndef __SPParserUtils__
+#define __SPParserUtils__
+
+#include <stdio.h>
+
+/**
+ * Return number of characters (NOT bytes) in a given UTF-8 encoded C string.
+ */
+size_t utf8strlen(const char * _s);
+
+#endif /* defined(__SPParserUtils__) */
diff --git a/Source/SPSQLTokenizer.l b/Source/SPSQLTokenizer.l
index 24dac938..b9170edb 100644
--- a/Source/SPSQLTokenizer.l
+++ b/Source/SPSQLTokenizer.l
@@ -31,12 +31,12 @@
 //  More info at <https://github.com/sequelpro/sequelpro>
 
 #import "SPSQLTokenizer.h"
+#include "SPParserUtils.h"
 
-int utf8strlenfortoken(const char * _s);
-int yyuoffset, yyuleng;
+size_t yyuoffset, yyuleng;
 
 //keep track of the current utf-8 character (not byte) offset and token length
-#define YY_USER_ACTION { yyuoffset += yyuleng; yyuleng = utf8strlenfortoken(yytext); }
+#define YY_USER_ACTION { yyuoffset += yyuleng; yyuleng = utf8strlen(yytext); }
 //ignore the output of unmatched characters
 #define ECHO {}
 %}
@@ -90,56 +90,3 @@ compend		{s}"end"
             						return 0;
           						}
 %%
-#define ONEMASK ((size_t)(-1) / 0xFF)
-// adapted from http://www.daemonology.net/blog/2008-06-05-faster-utf8-strlen.html
-int utf8strlenfortoken(const char * _s)
-{
-	const char * s;
-	size_t count = 0;
-	size_t u;
-	unsigned char b;
-
-	/* Handle any initial misaligned bytes. */
-	for (s = _s; (uintptr_t)(s) & (sizeof(size_t) - 1); s++) {
-		b = *s;
-
-		/* Exit if we hit a zero byte. */
-		if (b == '\0')
-			goto done;
-
-		/* Is this byte NOT the first byte of a character? */
-		count += (b >> 7) & ((~b) >> 6);
-	}
-
-	/* Handle complete blocks. */
-	for (; ; s += sizeof(size_t)) {
-		/* Prefetch 256 bytes ahead. */
-		__builtin_prefetch(&s[256], 0, 0);
-
-		/* Grab 4 or 8 bytes of UTF-8 data. */
-		u = *(size_t *)(s);
-
-		/* Exit the loop if there are any zero bytes. */
-		if ((u - ONEMASK) & (~u) & (ONEMASK * 0x80))
-			break;
-
-		/* Count bytes which are NOT the first byte of a character. */
-		u = ((u & (ONEMASK * 0x80)) >> 7) & ((~u) >> 6);
-		count += (u * ONEMASK) >> ((sizeof(size_t) - 1) * 8);
-	}
-
-	/* Take care of any left-over bytes. */
-	for (; ; s++) {
-		b = *s;
-
-		/* Exit if we hit a zero byte. */
-		if (b == '\0')
-			break;
-
-		/* Is this byte NOT the first byte of a character? */
-		count += (b >> 7) & ((~b) >> 6);
-	}
-
-done:
-	return (int)((s - _s) - count);
-}
diff --git a/sequel-pro.xcodeproj/project.pbxproj b/sequel-pro.xcodeproj/project.pbxproj
index 87d779f8..7368bcf5 100644
--- a/sequel-pro.xcodeproj/project.pbxproj
+++ b/sequel-pro.xcodeproj/project.pbxproj
@@ -182,6 +182,8 @@
 		506CE9311A311C6C0039F736 /* SPTableContentFilterController.m in Sources */ = {isa = PBXBuildFile; fileRef = 506CE9301A311C6C0039F736 /* SPTableContentFilterController.m */; };
 		50A9F8B119EAD4B90053E571 /* SPGotoDatabaseController.m in Sources */ = {isa = PBXBuildFile; fileRef = 50A9F8B019EAD4B90053E571 /* SPGotoDatabaseController.m */; };
 		50D3C3491A75B8A800B5429C /* GotoDatabaseDialog.xib in Resources */ = {isa = PBXBuildFile; fileRef = 50D3C34B1A75B8A800B5429C /* GotoDatabaseDialog.xib */; };
+		50D3C3521A77135F00B5429C /* SPParserUtils.c in Sources */ = {isa = PBXBuildFile; fileRef = 50D3C3501A77135F00B5429C /* SPParserUtils.c */; };
+		50D3C3541A7715E600B5429C /* SPParserUtils.c in Sources */ = {isa = PBXBuildFile; fileRef = 50D3C3501A77135F00B5429C /* SPParserUtils.c */; };
 		50E217B318174246009D3580 /* SPColorSelectorView.m in Sources */ = {isa = PBXBuildFile; fileRef = 50E217B218174246009D3580 /* SPColorSelectorView.m */; };
 		50E217B618174280009D3580 /* SPFavoriteColorSupport.m in Sources */ = {isa = PBXBuildFile; fileRef = 50E217B518174280009D3580 /* SPFavoriteColorSupport.m */; };
 		5806B76411A991EC00813A88 /* SPDocumentController.m in Sources */ = {isa = PBXBuildFile; fileRef = 5806B76311A991EC00813A88 /* SPDocumentController.m */; };
@@ -887,6 +889,8 @@
 		50A9F8AF19EAD4B90053E571 /* SPGotoDatabaseController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPGotoDatabaseController.h; sourceTree = "<group>"; };
 		50A9F8B019EAD4B90053E571 /* SPGotoDatabaseController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SPGotoDatabaseController.m; sourceTree = "<group>"; };
 		50D3C34A1A75B8A800B5429C /* English */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = English; path = English.lproj/GotoDatabaseDialog.xib; sourceTree = "<group>"; };
+		50D3C3501A77135F00B5429C /* SPParserUtils.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = SPParserUtils.c; sourceTree = "<group>"; };
+		50D3C3511A77135F00B5429C /* SPParserUtils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPParserUtils.h; sourceTree = "<group>"; };
 		50E217B118174246009D3580 /* SPColorSelectorView.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPColorSelectorView.h; sourceTree = "<group>"; };
 		50E217B218174246009D3580 /* SPColorSelectorView.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SPColorSelectorView.m; sourceTree = "<group>"; };
 		50E217B418174280009D3580 /* SPFavoriteColorSupport.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPFavoriteColorSupport.h; sourceTree = "<group>"; };
@@ -2533,6 +2537,8 @@
 				BCD0AD4A0FBBFC480066EA5C /* SPSQLTokenizer.h */,
 				BCD0AD480FBBFC340066EA5C /* SPSQLTokenizer.l */,
 				1755A25C16B33BEA00B35787 /* SPSyntaxParser.h */,
+				50D3C3501A77135F00B5429C /* SPParserUtils.c */,
+				50D3C3511A77135F00B5429C /* SPParserUtils.h */,
 			);
 			name = Parsing;
 			sourceTree = "<group>";
@@ -3035,6 +3041,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				50D3C3541A7715E600B5429C /* SPParserUtils.c in Sources */,
 				BC34F3281292AD6F000DA1AA /* SPConstants.m in Sources */,
 				BC6D709E120C4C9F008027B5 /* SPEditorTokens.l in Sources */,
 				BC0E1487120AAB5C00E52E25 /* SPStringAdditions.m in Sources */,
@@ -3223,6 +3230,7 @@
 				17D3C66E128AD4710047709F /* SPFavoritesController.m in Sources */,
 				17D3C671128AD8160047709F /* SPSingleton.m in Sources */,
 				17D3C6D3128B1C900047709F /* SPFavoritesOutlineView.m in Sources */,
+				50D3C3521A77135F00B5429C /* SPParserUtils.c in Sources */,
 				BC68BFC7128D4EAE004907D9 /* SPBundleEditorController.m in Sources */,
 				BC1944D01297291800A236CD /* SPBundleCommandTextView.m in Sources */,
 				BC77C5E4129AA69E009AD832 /* SPBundleHTMLOutputController.m in Sources */,