From cc0c0a7842e3bff325fa29c71f5115361981797d Mon Sep 17 00:00:00 2001 From: rowanbeentje Date: Mon, 28 Sep 2009 00:43:41 +0000 Subject: Rewrite CSV import: - Replace the CSV parsing function (arrayForCSV:) with a new SPCSVParser class - Make speed improvements to SPCSVParser to achieve 1.9x faster parsing than the old arrayForCSV: function - Rewrite CSV imports to be performed as a streaming import, keeping memory usage much much lower - CSV field mapping preview is now shown very early on in the import process, as soon as the first hundred rwos are available for a preview - Progress bars are more consistent and accurate - CSV rows are grouped into batches of up to 50 (depending on line length) for import, falling back to one-query-per-row if errors occur. The current error reporting level is therefore maintained, but imports of non-erroring data are much much faster. - Improve processing speed slightly - Fix some odd edge cases in CSV parsing This addresses issue #389. --- sequel-pro.xcodeproj/project.pbxproj | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'sequel-pro.xcodeproj/project.pbxproj') diff --git a/sequel-pro.xcodeproj/project.pbxproj b/sequel-pro.xcodeproj/project.pbxproj index 9b886ce9..8ebc1976 100644 --- a/sequel-pro.xcodeproj/project.pbxproj +++ b/sequel-pro.xcodeproj/project.pbxproj @@ -139,6 +139,7 @@ 58186D210F4CB38900851FE9 /* ConnectionErrorDialog.xib in Resources */ = {isa = PBXBuildFile; fileRef = 58186D1F0F4CB38900851FE9 /* ConnectionErrorDialog.xib */; }; 5822C9B51000DB2400DCC3D6 /* SPConnectionController.m in Sources */ = {isa = PBXBuildFile; fileRef = 5822C9B41000DB2400DCC3D6 /* SPConnectionController.m */; }; 5822CAE110011C8000DCC3D6 /* ConnectionView.xib in Resources */ = {isa = PBXBuildFile; fileRef = 5822CADF10011C8000DCC3D6 /* ConnectionView.xib */; }; + 5822D3091061833C00CE2157 /* SPCSVParser.m in Sources */ = {isa = PBXBuildFile; fileRef = 5822D3081061833C00CE2157 /* SPCSVParser.m */; }; 583B77D4103870C800B21F7E /* MCPStreamingResult.m in Sources */ = {isa = PBXBuildFile; fileRef = 583B779810386B0200B21F7E /* MCPStreamingResult.m */; }; 5841423F0F97E11000A34B47 /* NoodleLineNumberView.m in Sources */ = {isa = PBXBuildFile; fileRef = 5841423E0F97E11000A34B47 /* NoodleLineNumberView.m */; }; 584192A1101E57BB0089807F /* NSMutableArray-MultipleSort.m in Sources */ = {isa = PBXBuildFile; fileRef = 584192A0101E57BB0089807F /* NSMutableArray-MultipleSort.m */; }; @@ -515,6 +516,8 @@ 5822C9B31000DB2400DCC3D6 /* SPConnectionController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPConnectionController.h; sourceTree = ""; }; 5822C9B41000DB2400DCC3D6 /* SPConnectionController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SPConnectionController.m; sourceTree = ""; }; 5822CAE010011C8000DCC3D6 /* English */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = English; path = English.lproj/ConnectionView.xib; sourceTree = ""; }; + 5822D3071061833C00CE2157 /* SPCSVParser.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPCSVParser.h; sourceTree = ""; }; + 5822D3081061833C00CE2157 /* SPCSVParser.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SPCSVParser.m; sourceTree = ""; }; 583B779710386B0200B21F7E /* MCPStreamingResult.h */ = {isa = PBXFileReference; explicitFileType = sourcecode.c.h; fileEncoding = 4; path = MCPStreamingResult.h; sourceTree = ""; }; 583B779810386B0200B21F7E /* MCPStreamingResult.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MCPStreamingResult.m; sourceTree = ""; }; 5841423D0F97E11000A34B47 /* NoodleLineNumberView.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NoodleLineNumberView.h; sourceTree = ""; }; @@ -1258,6 +1261,8 @@ children = ( 58FEF16B0F23D66600518E8E /* SPSQLParser.h */, 58FEF16C0F23D66600518E8E /* SPSQLParser.m */, + 5822D3071061833C00CE2157 /* SPCSVParser.h */, + 5822D3081061833C00CE2157 /* SPCSVParser.m */, 179F15040F7C433C00579954 /* SPEditorTokens.h */, 179F15050F7C433C00579954 /* SPEditorTokens.l */, BCD0AD480FBBFC340066EA5C /* SPSQLTokenizer.l */, @@ -1709,6 +1714,7 @@ 17F5B1541048C50D00FC794F /* SPExporter.m in Sources */, 17F5B39C1049B96A00FC794F /* SPSQLExporter.m in Sources */, BC29C37F10501EFD00DD6C6E /* SPQueryController.m in Sources */, + 5822D3091061833C00CE2157 /* SPCSVParser.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; -- cgit v1.2.3