aboutsummaryrefslogtreecommitdiffstats
path: root/Source/SPSQLTokenizer.l
diff options
context:
space:
mode:
Diffstat (limited to 'Source/SPSQLTokenizer.l')
-rw-r--r--Source/SPSQLTokenizer.l59
1 files changed, 3 insertions, 56 deletions
diff --git a/Source/SPSQLTokenizer.l b/Source/SPSQLTokenizer.l
index 24dac938..b9170edb 100644
--- a/Source/SPSQLTokenizer.l
+++ b/Source/SPSQLTokenizer.l
@@ -31,12 +31,12 @@
// More info at <https://github.com/sequelpro/sequelpro>
#import "SPSQLTokenizer.h"
+#include "SPParserUtils.h"
-int utf8strlenfortoken(const char * _s);
-int yyuoffset, yyuleng;
+size_t yyuoffset, yyuleng;
//keep track of the current utf-8 character (not byte) offset and token length
-#define YY_USER_ACTION { yyuoffset += yyuleng; yyuleng = utf8strlenfortoken(yytext); }
+#define YY_USER_ACTION { yyuoffset += yyuleng; yyuleng = utf8strlen(yytext); }
//ignore the output of unmatched characters
#define ECHO {}
%}
@@ -90,56 +90,3 @@ compend {s}"end"
return 0;
}
%%
-#define ONEMASK ((size_t)(-1) / 0xFF)
-// adapted from http://www.daemonology.net/blog/2008-06-05-faster-utf8-strlen.html
-int utf8strlenfortoken(const char * _s)
-{
- const char * s;
- size_t count = 0;
- size_t u;
- unsigned char b;
-
- /* Handle any initial misaligned bytes. */
- for (s = _s; (uintptr_t)(s) & (sizeof(size_t) - 1); s++) {
- b = *s;
-
- /* Exit if we hit a zero byte. */
- if (b == '\0')
- goto done;
-
- /* Is this byte NOT the first byte of a character? */
- count += (b >> 7) & ((~b) >> 6);
- }
-
- /* Handle complete blocks. */
- for (; ; s += sizeof(size_t)) {
- /* Prefetch 256 bytes ahead. */
- __builtin_prefetch(&s[256], 0, 0);
-
- /* Grab 4 or 8 bytes of UTF-8 data. */
- u = *(size_t *)(s);
-
- /* Exit the loop if there are any zero bytes. */
- if ((u - ONEMASK) & (~u) & (ONEMASK * 0x80))
- break;
-
- /* Count bytes which are NOT the first byte of a character. */
- u = ((u & (ONEMASK * 0x80)) >> 7) & ((~u) >> 6);
- count += (u * ONEMASK) >> ((sizeof(size_t) - 1) * 8);
- }
-
- /* Take care of any left-over bytes. */
- for (; ; s++) {
- b = *s;
-
- /* Exit if we hit a zero byte. */
- if (b == '\0')
- break;
-
- /* Is this byte NOT the first byte of a character? */
- count += (b >> 7) & ((~b) >> 6);
- }
-
-done:
- return (int)((s - _s) - count);
-}