diff options
Diffstat (limited to 'Source')
| -rw-r--r-- | Source/SPCSVExporter.h | 14 | ||||
| -rw-r--r-- | Source/SPCSVExporter.m | 183 | ||||
| -rw-r--r-- | Source/SPExportController.h | 5 | ||||
| -rw-r--r-- | Source/SPExportController.m | 160 |
4 files changed, 183 insertions, 179 deletions
diff --git a/Source/SPCSVExporter.h b/Source/SPCSVExporter.h index c0cc75c4..b0edc04e 100644 --- a/Source/SPCSVExporter.h +++ b/Source/SPCSVExporter.h @@ -28,21 +28,11 @@ #import "MCPKit.h" #import "SPExporter.h" -/** - * - */ -@interface SPCSVExporterDelegate - -- (void)csvDataAvailable:(NSString *)data; - -@end - - @interface SPCSVExporter : SPExporter { // CSV data NSArray *csvDataArray; - MCPResult *csvDataResult; + MCPStreamingResult *csvDataResult; // CSV options BOOL csvOutputFieldNames; @@ -55,7 +45,7 @@ } @property (readwrite, retain) NSArray *csvDataArray; -@property (readwrite, retain) MCPResult *csvDataResult; +@property (readwrite, retain) MCPStreamingResult *csvDataResult; @property (readwrite, assign) BOOL csvOutputFieldNames; @property (readwrite, retain) NSString *csvFieldSeparatorString; diff --git a/Source/SPCSVExporter.m b/Source/SPCSVExporter.m index 9bbbb4af..4937e7c9 100644 --- a/Source/SPCSVExporter.m +++ b/Source/SPCSVExporter.m @@ -48,18 +48,19 @@ @try { NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init]; - NSMutableArray *csvRow = [NSMutableArray array]; - NSMutableString *csvCell = [NSMutableString string]; - NSMutableString *csvString = [NSMutableString string]; - NSMutableString *csvData = [NSMutableString string]; - + NSMutableString *csvString = [NSMutableString string]; + NSMutableString *csvData = [NSMutableString string]; + NSMutableString *csvCellString = [NSMutableString string]; + + NSArray *csvRow; NSScanner *csvNumericTester; NSString *escapedEscapeString, *escapedFieldSeparatorString, *escapedEnclosingString, *escapedLineEndString, *dataConversionString; - + + id csvCell; BOOL csvCellIsNumeric; BOOL quoteFieldSeparators = [[self csvEnclosingCharacterString] isEqualToString:@""]; - - NSUInteger i, j, startingRow, totalRows; + + NSUInteger i, totalRows, csvCellCount = 0; // Check that we have all the required info before starting the export if ((![self csvOutputFieldNames]) || @@ -82,12 +83,10 @@ // Check that we have at least some data to export if ((![self csvDataArray]) && (![self csvDataResult])) return; - + // Mark the process as running [self setExportProcessIsRunning:YES]; - if ([self csvDataResult] != nil && [[self csvDataResult] numOfRows]) [[self csvDataResult] dataSeek:0]; - // Detect and restore special characters being used as terminating or line end strings NSMutableString *tempSeparatorString = [NSMutableString stringWithString:[self csvFieldSeparatorString]]; @@ -132,143 +131,165 @@ escapedEnclosingString = [[self csvEscapeString] stringByAppendingString:[self csvEnclosingCharacterString]]; escapedLineEndString = [[self csvEscapeString] stringByAppendingString:[self csvLineEndingString]]; - // Determine the total number of rows and starting row depending on supplied data format - if ([self csvDataArray] == nil) { - startingRow = [self csvOutputFieldNames] ? 1 : 0; - totalRows = [[self csvDataResult] numOfRows]; - } - else { - startingRow = [self csvOutputFieldNames] ? 0 : 1; - totalRows = [[self csvDataArray] count]; - } + // Set up the starting row; for supplied arrays, which include the column + // headers as the first row, decide whether to skip the first row. + NSUInteger currentRowIndex = 0; [csvData setString:@""]; - - // Walk through the supplied data constructing the CSV string - for (i = startingRow; i < totalRows; i++) + + if (([self csvDataArray]) && (![self csvOutputFieldNames])) currentRowIndex++; + + // Drop into the processing loop + NSAutoreleasePool *csvExportPool = [[NSAutoreleasePool alloc] init]; + + NSUInteger currentPoolDataLength = 0; + + while (1) { - // Check to see if the operation has been cancelled. If so exit the loop. - if ([self isCancelled]) { - break; - } - - // Update the progress value - if (totalRows) [self setExportProgressValue:(((i + 1) * 100) / totalRows)]; - - // Retrieve the row from the supplied data - if ([self csvDataArray] == nil) { - // Header row - [csvRow setArray:(i == -1) ? [[self csvDataResult] fetchFieldNames] : [[self csvDataResult] fetchRowAsArray]]; + // Retrieve the next row from the supplied data, either directly from the array... + if ([self csvDataArray]) { + csvRow = NSArrayObjectAtIndex([self csvDataArray], currentRowIndex); } + // Or by reading an appropriate row from the streaming result else { - [csvRow setArray:NSArrayObjectAtIndex([self csvDataArray], i)]; + // If still requested to read the field names, get the field names + if ([self csvOutputFieldNames]) { + csvRow = [[self csvDataResult] fetchFieldNames]; + [self setCsvOutputFieldNames:NO]; + } + else { + csvRow = [[self csvDataResult] fetchNextRowAsArray]; + + if (!csvRow) break; + } } + // Get the cell count if we don't already have it stored + if (!csvCellCount) csvCellCount = [csvRow count]; + [csvString setString:@""]; - for (j = 0; j < [csvRow count]; j++) + for (i = 0 ; i < csvCellCount; i++) { + csvCell = NSArrayObjectAtIndex(csvRow, i); + // For NULL objects supplied from a queryResult, add an unenclosed null string as per prefs - if ([[csvRow objectAtIndex:j] isKindOfClass:[NSNull class]]) { + if ([csvCell isKindOfClass:[NSNull class]]) { [csvString appendString:[self csvNULLString]]; - if (j < [csvRow count] - 1) [csvString appendString:[self csvFieldSeparatorString]]; + if (i < (csvCellCount - 1)) [csvString appendString:[self csvFieldSeparatorString]]; continue; } // Retrieve the contents of this cell - if ([NSArrayObjectAtIndex(csvRow, j) isKindOfClass:[NSData class]]) { - dataConversionString = [[NSString alloc] initWithData:NSArrayObjectAtIndex(csvRow, j) encoding:[self exportOutputEncoding]]; + if ([csvCell isKindOfClass:[NSData class]]) { + dataConversionString = [[NSString alloc] initWithData:csvCell encoding:[self exportOutputEncoding]]; if (dataConversionString == nil) { - dataConversionString = [[NSString alloc] initWithData:NSArrayObjectAtIndex(csvRow, j) encoding:NSASCIIStringEncoding]; + dataConversionString = [[NSString alloc] initWithData:csvCell encoding:NSASCIIStringEncoding]; } - [csvCell setString:[NSString stringWithString:dataConversionString]]; + [csvCellString setString:[NSString stringWithString:dataConversionString]]; [dataConversionString release]; } else { - [csvCell setString:[NSArrayObjectAtIndex(csvRow, j) description]]; + [csvCellString setString:[csvCell description]]; } // For NULL values supplied via an array add the unenclosed null string as set in preferences - if ([csvCell isEqualToString:[self csvNULLString]]) { + if ([csvCellString isEqualToString:[self csvNULLString]]) { [csvString appendString:[self csvNULLString]]; } // Add empty strings as a pair of enclosing characters. - else if ([csvCell length] == 0) { + else if ([csvCellString length] == 0) { [csvString appendString:[self csvEnclosingCharacterString]]; [csvString appendString:[self csvEnclosingCharacterString]]; - - } + } else { - // Test whether this cell contains a number - if ([NSArrayObjectAtIndex(csvRow, j) isKindOfClass:[NSData class]]) { + // If an array of bools supplying information as to whether the column is numeric has been supplied, use it. + if ([self csvTableColumnNumericStatus] != nil) { + csvCellIsNumeric = [NSArrayObjectAtIndex([self csvTableColumnNumericStatus], i) boolValue]; + } + // Otherwise, first test whether this cell contains data + else if ([NSArrayObjectAtIndex(csvRow, i) isKindOfClass:[NSData class]]) { csvCellIsNumeric = NO; } - // If an array of bools supplying information as to whether the column is numeric has been supplied, use it. - else if ([self csvTableColumnNumericStatus] != nil) { - csvCellIsNumeric = [NSArrayObjectAtIndex([self csvTableColumnNumericStatus], j) boolValue]; - } // Or fall back to testing numeric content via an NSScanner. else { - csvNumericTester = [NSScanner scannerWithString:csvCell]; + csvNumericTester = [NSScanner scannerWithString:csvCellString]; + csvCellIsNumeric = [csvNumericTester scanFloat:nil] && [csvNumericTester isAtEnd] && - ([csvCell characterAtIndex:0] != '0' || - [csvCell length] == 1 || - ([csvCell length] > 1 && - [csvCell characterAtIndex:1] == '.')); + ([csvCellString characterAtIndex:0] != '0' || + [csvCellString length] == 1 || + ([csvCellString length] > 1 && + [csvCellString characterAtIndex:1] == '.')); } // Escape any occurrences of the escaping character - [csvCell replaceOccurrencesOfString:[self csvEscapeString] - withString:escapedEscapeString - options:NSLiteralSearch - range:NSMakeRange(0, [csvCell length])]; + [csvCellString replaceOccurrencesOfString:[self csvEscapeString] + withString:escapedEscapeString + options:NSLiteralSearch + range:NSMakeRange(0, [csvCellString length])]; // Escape any occurrences of the enclosure string if (![[self csvEscapeString] isEqualToString:[self csvEnclosingCharacterString]]) { - [csvCell replaceOccurrencesOfString:[self csvEnclosingCharacterString] - withString:escapedEnclosingString - options:NSLiteralSearch - range:NSMakeRange(0, [csvCell length])]; + [csvCellString replaceOccurrencesOfString:[self csvEnclosingCharacterString] + withString:escapedEnclosingString + options:NSLiteralSearch + range:NSMakeRange(0, [csvCellString length])]; } // Escape occurrences of the line end character - [csvCell replaceOccurrencesOfString:[self csvLineEndingString] - withString:escapedLineEndString - options:NSLiteralSearch - range:NSMakeRange(0, [csvCell length])]; + [csvCellString replaceOccurrencesOfString:[self csvLineEndingString] + withString:escapedLineEndString + options:NSLiteralSearch + range:NSMakeRange(0, [csvCellString length])]; // If the string isn't quoted or otherwise enclosed, escape occurrences of the field separators if (quoteFieldSeparators || csvCellIsNumeric) { - [csvCell replaceOccurrencesOfString:[self csvFieldSeparatorString] - withString:escapedFieldSeparatorString - options:NSLiteralSearch - range:NSMakeRange(0, [csvCell length])]; + [csvCellString replaceOccurrencesOfString:[self csvFieldSeparatorString] + withString:escapedFieldSeparatorString + options:NSLiteralSearch + range:NSMakeRange(0, [csvCellString length])]; } // Write out the cell data by appending strings - this is significantly faster than stringWithFormat. if (csvCellIsNumeric) { - [csvString appendString:csvCell]; + [csvString appendString:csvCellString]; } else { [csvString appendString:[self csvEnclosingCharacterString]]; - [csvString appendString:csvCell]; + [csvString appendString:csvCellString]; [csvString appendString:[self csvEnclosingCharacterString]]; } } - if (j < ([csvRow count] - 1)) [csvString appendString:[self csvFieldSeparatorString]]; + if (i < ([csvRow count] - 1)) [csvString appendString:[self csvFieldSeparatorString]]; } - // Append the line ending to the string for this row + // Append the line ending to the string for this row, and record the length processed for pool flushing [csvString appendString:[self csvLineEndingString]]; [csvData appendString:csvString]; + + currentPoolDataLength += [csvString length]; + + currentRowIndex++; + + // Update the progress value + if (totalRows) [self setExportProgressValue:(((i + 1) * 100) / totalRows)]; + + // If an array was supplied and we've processed all rows, break + if ([self csvDataArray] && (totalRows == currentRowIndex)) break; + + // Drain the autorelease pool as required to keep memory usage low + if (currentPoolDataLength > 250000) { + [csvExportPool drain]; + csvExportPool = [[NSAutoreleasePool alloc] init]; + } } - + // Assign the resulting CSV data to the expoter's export data [self setExportData:csvData]; @@ -276,7 +297,7 @@ [self setExportProcessIsRunning:NO]; // Call the delegate's didEndSelector while passing this exporter to it - [[self delegate] performSelectorOnMainThread:[self didEndSelector] withObject:self waitUntilDone:YES]; + [[self delegate] performSelectorOnMainThread:[self didEndSelector] withObject:self waitUntilDone:NO]; [pool release]; } diff --git a/Source/SPExportController.h b/Source/SPExportController.h index eec8ce63..294f5851 100644 --- a/Source/SPExportController.h +++ b/Source/SPExportController.h @@ -27,6 +27,8 @@ #import "SPExporterDataAccess.h" +#import "SPLogger.h" + // Export type constants enum { SP_SQL_EXPORT = 1, @@ -67,6 +69,7 @@ typedef NSUInteger SPExportSource; IBOutlet id exportInputMatrix; IBOutlet id exportFilePerTableCheck; IBOutlet id exportFilePerTableNote; + IBOutlet id exportProcessLowMemory; // Export progress sheet IBOutlet id exportProgressWindow; @@ -116,6 +119,8 @@ typedef NSUInteger SPExportSource; // Concurrent operation queue NSOperationQueue *operationQueue; + + SPLogger *log; } @property (readwrite, assign) BOOL exportCancelled; diff --git a/Source/SPExportController.m b/Source/SPExportController.m index b6adc58a..f7b1878f 100644 --- a/Source/SPExportController.m +++ b/Source/SPExportController.m @@ -32,9 +32,8 @@ @interface SPExportController (PrivateAPI) -- (NSString *)_htmlEscapeString:(NSString *)string; - (void)_initializeExportUsingSelectedOptions; -- (BOOL)_exportTablesAsCSV:(NSArray *)exportTables usingDataExporter:(SPExporter *)exporter; +- (BOOL)_exportTables:(NSArray *)exportTables asType:(SPExportType)type; @end @@ -53,6 +52,8 @@ tables = [[NSMutableArray alloc] init]; operationQueue = [[NSOperationQueue alloc] init]; + + log = [SPLogger logger]; } return self; @@ -163,7 +164,12 @@ [panel setCanChooseDirectories:YES]; [panel setCanCreateDirectories:YES]; - [panel beginSheetForDirectory:NSHomeDirectory() file:nil modalForWindow:exportWindow modalDelegate:self didEndSelector:@selector(savePanelDidEnd:returnCode:contextInfo:) contextInfo:nil]; + [panel beginSheetForDirectory:NSHomeDirectory() + file:nil + modalForWindow:exportWindow + modalDelegate:self + didEndSelector:@selector(savePanelDidEnd:returnCode:contextInfo:) + contextInfo:nil]; } #pragma mark - @@ -227,6 +233,8 @@ */ - (void)exporterDataConversionProcessComplete:(SPExporter *)exporter { + // Do something with the data... + // If there are no more operations in the queue, close the progress sheet if ([[operationQueue operations] count] == 0) { [NSApp endSheet:exportProgressWindow returnCode:0]; @@ -278,32 +286,6 @@ @implementation SPExportController (PrivateAPI) /** - * Escapes the supplied HTML string - */ -- (NSString *)_htmlEscapeString:(NSString *)string -{ - NSMutableString *mutableString = [NSMutableString stringWithString:string]; - - [mutableString replaceOccurrencesOfString:@"&" withString:@"&" - options:NSLiteralSearch - range:NSMakeRange(0, [mutableString length])]; - - [mutableString replaceOccurrencesOfString:@"<" withString:@"<" - options:NSLiteralSearch - range:NSMakeRange(0, [mutableString length])]; - - [mutableString replaceOccurrencesOfString:@">" withString:@">" - options:NSLiteralSearch - range:NSMakeRange(0, [mutableString length])]; - - [mutableString replaceOccurrencesOfString:@"\"" withString:@""" - options:NSLiteralSearch - range:NSMakeRange(0, [mutableString length])]; - - return [NSString stringWithString:mutableString]; -} - -/** * */ - (void)_initializeExportUsingSelectedOptions @@ -319,7 +301,7 @@ } } - // Determine what data to use (filtered result, custom query result or selected tables) for the export operation + // Determine what data to use (filtered result, custom query result or selected table(s)) for the export operation SPExportSource exportSource = ([exportInputMatrix selectedRow] + 1); NSMutableArray *exportTables = [NSMutableArray array]; @@ -345,43 +327,7 @@ break; } - SPExporter *exporter; - SPCSVExporter *csvExporter; - - // Based on the type of export create a new instance of the corresponding exporter and set it's specific options - switch (exportType) - { - case SP_SQL_EXPORT: - - break; - case SP_CSV_EXPORT: - csvExporter = [[SPCSVExporter alloc] initWithDelegate:self]; - - [csvExporter setCsvOutputFieldNames:[exportCSVIncludeFieldNamesCheck state]]; - [csvExporter setCsvFieldSeparatorString:[exportCSVFieldsTerminatedField stringValue]]; - [csvExporter setCsvEnclosingCharacterString:[exportCSVFieldsWrappedField stringValue]]; - [csvExporter setCsvLineEndingString:[exportCSVLinesTerminatedField stringValue]]; - [csvExporter setCsvEscapeString:[exportCSVFieldsEscapedField stringValue]]; - - [csvExporter setExportOutputEncoding:[MCPConnection encodingForMySQLEncoding:[[tableDocumentInstance connectionEncoding] UTF8String]]]; - [csvExporter setCsvNULLString:[[NSUserDefaults standardUserDefaults] objectForKey:@"NullValue"]]; - - exporter = csvExporter; - break; - case SP_XML_EXPORT: - - break; - case SP_PDF_EXPORT: - - break; - case SP_HTML_EXPORT: - - break; - case SP_EXCEL_EXPORT: - - break; - } - + // Begin the export based on the type switch (exportSource) { case SP_FILTERED_EXPORT: @@ -391,24 +337,23 @@ break; case SP_TABLE_EXPORT: - [self _exportTablesAsCSV:exportTables usingDataExporter:exporter]; + [self _exportTables:exportTables asType:exportType]; break; } } /** - * Exports the contents' of the supplied array of tables using the supplied exporter and export type. Note that - * this method currently only supports exporting in CSV and XML formats. + * Exports the contents' of the supplied array of tables. Note that this method currently only supports + * exporting in CSV and XML formats. */ -- (BOOL)_exportTablesAsCSV:(NSArray *)exportTables usingDataExporter:(SPCSVExporter *)exporter +- (BOOL)_exportTables:(NSArray *)exportTables asType:(SPExportType)type { - NSUInteger tableCount, i; + NSUInteger i; NSMutableString *errors = [NSMutableString string]; NSMutableString *infoString = [NSMutableString string]; NSDictionary *tableDetails; - NSMutableArray *tableColumnNumericStatus; NSStringEncoding encoding = [[self connection] encoding]; // Reset the interface @@ -440,19 +385,21 @@ options:NSLiteralSearch range:NSMakeRange(0, [csvLineEnd length])]; - if ([exportTables count] > 1) { + /*if ([exportTables count] > 1) { [infoString setString:[NSString stringWithFormat:@"Host: %@ Database: %@ Generation Time: %@%@%@", [tableDocumentInstance host], [tableDocumentInstance database], [NSDate date], csvLineEnd, csvLineEnd]]; - } + }*/ - tableCount = [exportTables count]; + NSUInteger tableCount = [exportTables count]; // Loop through the tables - for (i = 0 ; i < ((tableCount) && (![self exportCancelled])); i++) + for (i = 0 ; i < tableCount; i++) { + if ([self exportCancelled]) break; + // Update the progress text and reset the progress bar to indeterminate status NSString *tableName = [exportTables objectAtIndex:i]; - + [exportProgressText setStringValue:[NSString stringWithFormat:NSLocalizedString(@"Table %d of %d (%@): fetching data...", @"text showing that app is fetching data for table dump"), (i + 1), tableCount, tableName]]; [exportProgressText displayIfNeeded]; @@ -475,7 +422,7 @@ } // Retrieve the table details via the data class, and use it to build an array containing column numeric status - tableColumnNumericStatus = [NSMutableArray array]; + NSMutableArray *tableColumnNumericStatus = [NSMutableArray array]; for (NSDictionary *column in [tableDetails objectForKey:@"columns"]) { @@ -485,19 +432,60 @@ [tableColumnTypeGrouping isEqualToString:@"integer"] || [tableColumnTypeGrouping isEqualToString:@"float"])]]; } - - [exporter setCsvTableColumnNumericStatus:tableColumnNumericStatus]; - // Retrieve all the content within this table - queryResult = [connection queryString:[NSString stringWithFormat:@"SELECT * FROM %@", [tableName backtickQuotedString]]]; - + // Use low memory export? + BOOL useLowMemoryBlockingStreaming = ([exportProcessLowMemory state] == NSOnState); + + // Make a streaming request for the data + MCPStreamingResult *queryResultStreaming = [connection streamingQueryString:[NSString stringWithFormat:@"SELECT * FROM %@", [tableName backtickQuotedString]] useLowMemoryBlockingStreaming:useLowMemoryBlockingStreaming]; + // Note any errors during retrieval if (![[connection getLastErrorMessage] isEqualToString:@""]) { [errors appendString:[NSString stringWithFormat:@"%@\n", [connection getLastErrorMessage]]]; } - // Assign the data to the exporter - [exporter setCsvDataResult:queryResult]; + SPExporter *exporter; + SPCSVExporter *csvExporter; + + // Based on the type of export create a new instance of the corresponding exporter and set it's specific options + switch (type) + { + case SP_SQL_EXPORT: + + break; + case SP_CSV_EXPORT: + csvExporter = [[SPCSVExporter alloc] initWithDelegate:self]; + + [csvExporter setCsvOutputFieldNames:[exportCSVIncludeFieldNamesCheck state]]; + [csvExporter setCsvFieldSeparatorString:[exportCSVFieldsTerminatedField stringValue]]; + [csvExporter setCsvEnclosingCharacterString:[exportCSVFieldsWrappedField stringValue]]; + [csvExporter setCsvLineEndingString:[exportCSVLinesTerminatedField stringValue]]; + [csvExporter setCsvEscapeString:[exportCSVFieldsEscapedField stringValue]]; + + [csvExporter setExportOutputEncoding:[MCPConnection encodingForMySQLEncoding:[[tableDocumentInstance connectionEncoding] UTF8String]]]; + [csvExporter setCsvNULLString:[[NSUserDefaults standardUserDefaults] objectForKey:@"NullValue"]]; + + [csvExporter setCsvTableColumnNumericStatus:tableColumnNumericStatus]; + + // Assign the data to the exporter + [csvExporter setCsvDataResult:queryResultStreaming]; + + exporter = csvExporter; + + break; + case SP_XML_EXPORT: + + break; + case SP_PDF_EXPORT: + + break; + case SP_HTML_EXPORT: + + break; + case SP_EXCEL_EXPORT: + + break; + } // Update the progress text and set the progress bar back to determinate [exportProgressText setStringValue:[NSString stringWithFormat:NSLocalizedString(@"Table %d of %d (%@): Writing...", @"text showing that app is writing data for table export"), (i + 1), tableCount, tableName]]; |