diff options
| -rw-r--r-- | Source/SPEditorTokens.l | 55 | ||||
| -rw-r--r-- | Source/SPParserUtils.c | 87 | ||||
| -rw-r--r-- | Source/SPParserUtils.h | 41 | ||||
| -rw-r--r-- | Source/SPSQLTokenizer.l | 59 | ||||
| -rw-r--r-- | sequel-pro.xcodeproj/project.pbxproj | 8 |
5 files changed, 140 insertions, 110 deletions
diff --git a/Source/SPEditorTokens.l b/Source/SPEditorTokens.l index c10dbd60..a40105a1 100644 --- a/Source/SPEditorTokens.l +++ b/Source/SPEditorTokens.l @@ -43,8 +43,8 @@ */ #import "SPEditorTokens.h" +#include "SPParserUtils.h" -size_t utf8strlen(const char * _s); size_t yyuoffset, yyuleng; //keep track of the current utf-8 character (not byte) offset and token length @@ -126,59 +126,6 @@ keywords (X(OR|509|A)|S(MALLINT|SL|H(OW({s}(E(NGINE(S)?|RRORS)|M(ASTER|UTEX)|BIN } %% -#define ONEMASK ((size_t)(-1) / 0xFF) -// adapted from http://www.daemonology.net/blog/2008-06-05-faster-utf8-strlen.html -size_t utf8strlen(const char * _s) -{ - const char * s; - size_t count = 0; - size_t u; - unsigned char b; - - /* Handle any initial misaligned bytes. */ - for (s = _s; (uintptr_t)(s) & (sizeof(size_t) - 1); s++) { - b = *s; - - /* Exit if we hit a zero byte. */ - if (b == '\0') - goto done; - - /* Is this byte NOT the first byte of a character? */ - count += (b >> 7) & ((~b) >> 6); - } - - /* Handle complete blocks. */ - for (; ; s += sizeof(size_t)) { - /* Prefetch 256 bytes ahead. */ - __builtin_prefetch(&s[256], 0, 0); - - /* Grab 4 or 8 bytes of UTF-8 data. */ - u = *(size_t *)(s); - - /* Exit the loop if there are any zero bytes. */ - if ((u - ONEMASK) & (~u) & (ONEMASK * 0x80)) - break; - - /* Count bytes which are NOT the first byte of a character. */ - u = ((u & (ONEMASK * 0x80)) >> 7) & ((~u) >> 6); - count += (u * ONEMASK) >> ((sizeof(size_t) - 1) * 8); - } - - /* Take care of any left-over bytes. */ - for (; ; s++) { - b = *s; - - /* Exit if we hit a zero byte. */ - if (b == '\0') - break; - - /* Is this byte NOT the first byte of a character? */ - count += (b >> 7) & ((~b) >> 6); - } - -done: - return ((s - _s) - count); -} /* un-optimized keywords: ACCESSIBLE diff --git a/Source/SPParserUtils.c b/Source/SPParserUtils.c new file mode 100644 index 00000000..b3b48945 --- /dev/null +++ b/Source/SPParserUtils.c @@ -0,0 +1,87 @@ +// +// SPParserUtils.c +// sequel-pro +// +// Created by Max Lohrmann on 27.01.15. +// Relocated from existing files. Previous copyright applies. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// +// More info at <https://github.com/sequelpro/sequelpro> + +#include "SPParserUtils.h" +#include <stdint.h> + +#define ONEMASK ((size_t)(-1) / 0xFF) + +// adapted from http://www.daemonology.net/blog/2008-06-05-faster-utf8-strlen.html +size_t utf8strlen(const char * _s) +{ + const char * s; + size_t count = 0; + size_t u; + unsigned char b; + + /* Handle any initial misaligned bytes. */ + for (s = _s; (uintptr_t)(s) & (sizeof(size_t) - 1); s++) { + b = *s; + + /* Exit if we hit a zero byte. */ + if (b == '\0') + goto done; + + /* Is this byte NOT the first byte of a character? */ + count += (b >> 7) & ((~b) >> 6); + } + + /* Handle complete blocks. */ + for (; ; s += sizeof(size_t)) { + /* Prefetch 256 bytes ahead. */ + __builtin_prefetch(&s[256], 0, 0); + + /* Grab 4 or 8 bytes of UTF-8 data. */ + u = *(size_t *)(s); + + /* Exit the loop if there are any zero bytes. */ + if ((u - ONEMASK) & (~u) & (ONEMASK * 0x80)) + break; + + /* Count bytes which are NOT the first byte of a character. */ + u = ((u & (ONEMASK * 0x80)) >> 7) & ((~u) >> 6); + count += (u * ONEMASK) >> ((sizeof(size_t) - 1) * 8); + } + + /* Take care of any left-over bytes. */ + for (; ; s++) { + b = *s; + + /* Exit if we hit a zero byte. */ + if (b == '\0') + break; + + /* Is this byte NOT the first byte of a character? */ + count += (b >> 7) & ((~b) >> 6); + } + +done: + return ((s - _s) - count); +} diff --git a/Source/SPParserUtils.h b/Source/SPParserUtils.h new file mode 100644 index 00000000..487131e0 --- /dev/null +++ b/Source/SPParserUtils.h @@ -0,0 +1,41 @@ +// +// SPParserUtils.h +// sequel-pro +// +// Created by Max Lohrmann on 27.01.15. +// Relocated from existing files. Previous copyright applies. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// +// More info at <https://github.com/sequelpro/sequelpro> + +#ifndef __SPParserUtils__ +#define __SPParserUtils__ + +#include <stdio.h> + +/** + * Return number of characters (NOT bytes) in a given UTF-8 encoded C string. + */ +size_t utf8strlen(const char * _s); + +#endif /* defined(__SPParserUtils__) */ diff --git a/Source/SPSQLTokenizer.l b/Source/SPSQLTokenizer.l index 24dac938..b9170edb 100644 --- a/Source/SPSQLTokenizer.l +++ b/Source/SPSQLTokenizer.l @@ -31,12 +31,12 @@ // More info at <https://github.com/sequelpro/sequelpro> #import "SPSQLTokenizer.h" +#include "SPParserUtils.h" -int utf8strlenfortoken(const char * _s); -int yyuoffset, yyuleng; +size_t yyuoffset, yyuleng; //keep track of the current utf-8 character (not byte) offset and token length -#define YY_USER_ACTION { yyuoffset += yyuleng; yyuleng = utf8strlenfortoken(yytext); } +#define YY_USER_ACTION { yyuoffset += yyuleng; yyuleng = utf8strlen(yytext); } //ignore the output of unmatched characters #define ECHO {} %} @@ -90,56 +90,3 @@ compend {s}"end" return 0; } %% -#define ONEMASK ((size_t)(-1) / 0xFF) -// adapted from http://www.daemonology.net/blog/2008-06-05-faster-utf8-strlen.html -int utf8strlenfortoken(const char * _s) -{ - const char * s; - size_t count = 0; - size_t u; - unsigned char b; - - /* Handle any initial misaligned bytes. */ - for (s = _s; (uintptr_t)(s) & (sizeof(size_t) - 1); s++) { - b = *s; - - /* Exit if we hit a zero byte. */ - if (b == '\0') - goto done; - - /* Is this byte NOT the first byte of a character? */ - count += (b >> 7) & ((~b) >> 6); - } - - /* Handle complete blocks. */ - for (; ; s += sizeof(size_t)) { - /* Prefetch 256 bytes ahead. */ - __builtin_prefetch(&s[256], 0, 0); - - /* Grab 4 or 8 bytes of UTF-8 data. */ - u = *(size_t *)(s); - - /* Exit the loop if there are any zero bytes. */ - if ((u - ONEMASK) & (~u) & (ONEMASK * 0x80)) - break; - - /* Count bytes which are NOT the first byte of a character. */ - u = ((u & (ONEMASK * 0x80)) >> 7) & ((~u) >> 6); - count += (u * ONEMASK) >> ((sizeof(size_t) - 1) * 8); - } - - /* Take care of any left-over bytes. */ - for (; ; s++) { - b = *s; - - /* Exit if we hit a zero byte. */ - if (b == '\0') - break; - - /* Is this byte NOT the first byte of a character? */ - count += (b >> 7) & ((~b) >> 6); - } - -done: - return (int)((s - _s) - count); -} diff --git a/sequel-pro.xcodeproj/project.pbxproj b/sequel-pro.xcodeproj/project.pbxproj index 87d779f8..7368bcf5 100644 --- a/sequel-pro.xcodeproj/project.pbxproj +++ b/sequel-pro.xcodeproj/project.pbxproj @@ -182,6 +182,8 @@ 506CE9311A311C6C0039F736 /* SPTableContentFilterController.m in Sources */ = {isa = PBXBuildFile; fileRef = 506CE9301A311C6C0039F736 /* SPTableContentFilterController.m */; }; 50A9F8B119EAD4B90053E571 /* SPGotoDatabaseController.m in Sources */ = {isa = PBXBuildFile; fileRef = 50A9F8B019EAD4B90053E571 /* SPGotoDatabaseController.m */; }; 50D3C3491A75B8A800B5429C /* GotoDatabaseDialog.xib in Resources */ = {isa = PBXBuildFile; fileRef = 50D3C34B1A75B8A800B5429C /* GotoDatabaseDialog.xib */; }; + 50D3C3521A77135F00B5429C /* SPParserUtils.c in Sources */ = {isa = PBXBuildFile; fileRef = 50D3C3501A77135F00B5429C /* SPParserUtils.c */; }; + 50D3C3541A7715E600B5429C /* SPParserUtils.c in Sources */ = {isa = PBXBuildFile; fileRef = 50D3C3501A77135F00B5429C /* SPParserUtils.c */; }; 50E217B318174246009D3580 /* SPColorSelectorView.m in Sources */ = {isa = PBXBuildFile; fileRef = 50E217B218174246009D3580 /* SPColorSelectorView.m */; }; 50E217B618174280009D3580 /* SPFavoriteColorSupport.m in Sources */ = {isa = PBXBuildFile; fileRef = 50E217B518174280009D3580 /* SPFavoriteColorSupport.m */; }; 5806B76411A991EC00813A88 /* SPDocumentController.m in Sources */ = {isa = PBXBuildFile; fileRef = 5806B76311A991EC00813A88 /* SPDocumentController.m */; }; @@ -887,6 +889,8 @@ 50A9F8AF19EAD4B90053E571 /* SPGotoDatabaseController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPGotoDatabaseController.h; sourceTree = "<group>"; }; 50A9F8B019EAD4B90053E571 /* SPGotoDatabaseController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SPGotoDatabaseController.m; sourceTree = "<group>"; }; 50D3C34A1A75B8A800B5429C /* English */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = English; path = English.lproj/GotoDatabaseDialog.xib; sourceTree = "<group>"; }; + 50D3C3501A77135F00B5429C /* SPParserUtils.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = SPParserUtils.c; sourceTree = "<group>"; }; + 50D3C3511A77135F00B5429C /* SPParserUtils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPParserUtils.h; sourceTree = "<group>"; }; 50E217B118174246009D3580 /* SPColorSelectorView.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPColorSelectorView.h; sourceTree = "<group>"; }; 50E217B218174246009D3580 /* SPColorSelectorView.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SPColorSelectorView.m; sourceTree = "<group>"; }; 50E217B418174280009D3580 /* SPFavoriteColorSupport.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPFavoriteColorSupport.h; sourceTree = "<group>"; }; @@ -2533,6 +2537,8 @@ BCD0AD4A0FBBFC480066EA5C /* SPSQLTokenizer.h */, BCD0AD480FBBFC340066EA5C /* SPSQLTokenizer.l */, 1755A25C16B33BEA00B35787 /* SPSyntaxParser.h */, + 50D3C3501A77135F00B5429C /* SPParserUtils.c */, + 50D3C3511A77135F00B5429C /* SPParserUtils.h */, ); name = Parsing; sourceTree = "<group>"; @@ -3035,6 +3041,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 50D3C3541A7715E600B5429C /* SPParserUtils.c in Sources */, BC34F3281292AD6F000DA1AA /* SPConstants.m in Sources */, BC6D709E120C4C9F008027B5 /* SPEditorTokens.l in Sources */, BC0E1487120AAB5C00E52E25 /* SPStringAdditions.m in Sources */, @@ -3223,6 +3230,7 @@ 17D3C66E128AD4710047709F /* SPFavoritesController.m in Sources */, 17D3C671128AD8160047709F /* SPSingleton.m in Sources */, 17D3C6D3128B1C900047709F /* SPFavoritesOutlineView.m in Sources */, + 50D3C3521A77135F00B5429C /* SPParserUtils.c in Sources */, BC68BFC7128D4EAE004907D9 /* SPBundleEditorController.m in Sources */, BC1944D01297291800A236CD /* SPBundleCommandTextView.m in Sources */, BC77C5E4129AA69E009AD832 /* SPBundleHTMLOutputController.m in Sources */, |