From cc0c0a7842e3bff325fa29c71f5115361981797d Mon Sep 17 00:00:00 2001 From: rowanbeentje Date: Mon, 28 Sep 2009 00:43:41 +0000 Subject: Rewrite CSV import: - Replace the CSV parsing function (arrayForCSV:) with a new SPCSVParser class - Make speed improvements to SPCSVParser to achieve 1.9x faster parsing than the old arrayForCSV: function - Rewrite CSV imports to be performed as a streaming import, keeping memory usage much much lower - CSV field mapping preview is now shown very early on in the import process, as soon as the first hundred rwos are available for a preview - Progress bars are more consistent and accurate - CSV rows are grouped into batches of up to 50 (depending on line length) for import, falling back to one-query-per-row if errors occur. The current error reporting level is therefore maintained, but imports of non-erroring data are much much faster. - Improve processing speed slightly - Fix some odd edge cases in CSV parsing This addresses issue #389. --- Source/TableDump.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'Source/TableDump.h') diff --git a/Source/TableDump.h b/Source/TableDump.h index dd9ffd6c..28d8107d 100644 --- a/Source/TableDump.h +++ b/Source/TableDump.h @@ -97,10 +97,11 @@ MCPConnection *mySQLConnection; NSMutableArray *tables; - NSArray *importArray; + NSArray *fieldMappingImportArray; + BOOL fieldMappingImportArrayIsPreview; NSMutableArray *fieldMappingArray; NSMutableArray *fieldMappingButtonOptions; - int currentRow; + int fieldMappingCurrentRow; NSUserDefaults *prefs; BOOL progressCancelled; } @@ -125,12 +126,10 @@ - (IBAction)changeFormat:(id)sender; - (IBAction)changeTable:(id)sender; - (void)openPanelDidEnd:(NSOpenPanel *)sheet returnCode:(int)returnCode contextInfo:(NSString *)contextInfo; +- (BOOL) buildFieldMappingArrayWithData:(NSArray *)importData isPreview:(BOOL)dataIsPreviewData; - (void)setupFieldMappingArray; - (void)updateFieldMappingButtonCell; -- (NSArray *)arrayForCSV:(NSString *)csv terminatedBy:(NSString *)terminated - enclosedBy:(NSString *)enclosed escapedBy:(NSString *)escaped lineEnds:(NSString *)lineEnds; -- (NSArray *)arrayForString:(NSString *)string enclosed:(NSString *)enclosed - escaped:(NSString *)escaped terminated:(NSString *)terminated; +- (NSString *) mappedValueStringForRowArray:(NSArray *)csvRowArray; // Export methods - (BOOL)dumpSelectedTablesAsSqlToFileHandle:(NSFileHandle *)fileHandle; -- cgit v1.2.3