Move some duplicate code into it's own file

1 files changed, 3 insertions, 56 deletions

diff --git a/Source/SPSQLTokenizer.l b/Source/SPSQLTokenizer.l
index 24dac938..b9170edb 100644
--- a/Source/SPSQLTokenizer.l
+++ b/Source/SPSQLTokenizer.l
@@ -31,12 +31,12 @@
 //  More info at <https://github.com/sequelpro/sequelpro>
 
 #import "SPSQLTokenizer.h"
+#include "SPParserUtils.h"
 
-int utf8strlenfortoken(const char * _s);
-int yyuoffset, yyuleng;
+size_t yyuoffset, yyuleng;
 
 //keep track of the current utf-8 character (not byte) offset and token length
-#define YY_USER_ACTION { yyuoffset += yyuleng; yyuleng = utf8strlenfortoken(yytext); }
+#define YY_USER_ACTION { yyuoffset += yyuleng; yyuleng = utf8strlen(yytext); }
 //ignore the output of unmatched characters
 #define ECHO {}
 %}
@@ -90,56 +90,3 @@ compend		{s}"end"
             						return 0;
           						}
 %%
-#define ONEMASK ((size_t)(-1) / 0xFF)
-// adapted from http://www.daemonology.net/blog/2008-06-05-faster-utf8-strlen.html
-int utf8strlenfortoken(const char * _s)
-{
-	const char * s;
-	size_t count = 0;
-	size_t u;
-	unsigned char b;
-
-	/* Handle any initial misaligned bytes. */
-	for (s = _s; (uintptr_t)(s) & (sizeof(size_t) - 1); s++) {
-		b = *s;
-
-		/* Exit if we hit a zero byte. */
-		if (b == '\0')
-			goto done;
-
-		/* Is this byte NOT the first byte of a character? */
-		count += (b >> 7) & ((~b) >> 6);
-	}
-
-	/* Handle complete blocks. */
-	for (; ; s += sizeof(size_t)) {
-		/* Prefetch 256 bytes ahead. */
-		__builtin_prefetch(&s[256], 0, 0);
-
-		/* Grab 4 or 8 bytes of UTF-8 data. */
-		u = *(size_t *)(s);
-
-		/* Exit the loop if there are any zero bytes. */
-		if ((u - ONEMASK) & (~u) & (ONEMASK * 0x80))
-			break;
-
-		/* Count bytes which are NOT the first byte of a character. */
-		u = ((u & (ONEMASK * 0x80)) >> 7) & ((~u) >> 6);
-		count += (u * ONEMASK) >> ((sizeof(size_t) - 1) * 8);
-	}
-
-	/* Take care of any left-over bytes. */
-	for (; ; s++) {
-		b = *s;
-
-		/* Exit if we hit a zero byte. */
-		if (b == '\0')
-			break;
-
-		/* Is this byte NOT the first byte of a character? */
-		count += (b >> 7) & ((~b) >> 6);
-	}
-
-done:
-	return (int)((s - _s) - count);
-}