aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Source/SPCSVParser.h117
-rw-r--r--Source/SPCSVParser.m642
-rw-r--r--Source/TableDump.h11
-rw-r--r--Source/TableDump.m1006
-rw-r--r--sequel-pro.xcodeproj/project.pbxproj6
5 files changed, 1205 insertions, 577 deletions
diff --git a/Source/SPCSVParser.h b/Source/SPCSVParser.h
new file mode 100644
index 00000000..0f1a8a20
--- /dev/null
+++ b/Source/SPCSVParser.h
@@ -0,0 +1,117 @@
+//
+// $Id$
+//
+// SPCSVParser.h
+// sequel-pro
+//
+// Created by Rowan Beentje on 16/09/2009.
+// Copyright 2009 Rowan Beentje. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//
+// More info at <http://code.google.com/p/sequel-pro/>
+
+#import <Cocoa/Cocoa.h>
+
+/*
+ * This class provides a string class intended for CSV parsing. Unlike SPSQLParser, this
+ * does not extend NSMutableString and instead provides only a subset of similar methods.
+ * Internally, an approach similar to NSScanner is used to support multi-character strings.
+ * The methods are designed with the intention that as a string is parsed the parsed content
+ * is removed. This also allows parsing to occur in "streaming" mode, with parseable content
+ * being pulled off the start of the string as additional content is appended onto the end of
+ * the string, eg from a file.
+ *
+ * Supports:
+ * - Control of field terminator, line terminator, string enclosures and escape characters.
+ * - Multi-character field terminator, line terminator, string enclosures, and escape strings.
+ * - Stream-based processing (recommended that strings split by \n or \r are used when streaming
+ * to minimise multibyte issues)
+ * - Correct treatment of line terminators within quoted strings and proper escape support
+ * including escape characters matching the quote characters in Excel style
+ *
+ * The internal usage of string range finding, similar to the NSScanner approach, means this
+ * could be significantly sped up for single-character terminators.
+ */
+
+#define SPCSVPARSER_TRIM_ENACT_LENGTH 250000
+
+@interface SPCSVParser : NSObject
+{
+ NSMutableString *csvString;
+
+ long trimPosition;
+ long parserPosition;
+ long totalLengthParsed;
+ long csvStringLength;
+ int fieldCount;
+
+ NSString *nullReplacementString;
+ NSString *fieldEndString;
+ NSString *lineEndString;
+ NSString *fieldQuoteString;
+ NSString *escapeString;
+ NSString *escapedFieldEndString;
+ NSString *escapedLineEndString;
+ NSString *escapedFieldQuoteString;
+ NSString *escapedEscapeString;
+ int fieldEndLength;
+ int lineEndLength;
+ int fieldQuoteLength;
+ int escapeLength;
+ NSCharacterSet *skipCharacterSet;
+ NSScanner *csvScanner;
+
+ BOOL escapeStringIsFieldQuoteString;
+}
+
+/* Retrieving data from the CSV string */
+- (NSArray *) array;
+- (NSArray *) getRowAsArray;
+- (NSArray *) getRowAsArrayAndTrimString:(BOOL)trimString stringIsComplete:(BOOL)stringComplete;
+
+/* Adding new data to the string */
+- (void) appendString:(NSString *)aString;
+- (void) setString:(NSString *)aString;
+
+/* Basic information */
+- (NSUInteger) length;
+- (NSString *) string;
+- (long) parserPosition;
+- (long) totalLengthParsed;
+
+/* Setting the terminator, quote, escape and null character replacement strings */
+- (void) setFieldTerminatorString:(NSString *)theString convertDisplayStrings:(BOOL)convertString;
+- (void) setLineTerminatorString:(NSString *)theString convertDisplayStrings:(BOOL)convertString;
+- (void) setFieldQuoteString:(NSString *)theString convertDisplayStrings:(BOOL)convertString;
+- (void) setEscapeString:(NSString *)theString convertDisplayStrings:(BOOL)convertString;
+- (void) setNullReplacementString:(NSString *)nullString;
+
+/* Init and internal update methods */
+- (void) _initialiseCSVParserDefaults;
+- (void) _moveParserPastSkippableCharacters;
+- (long) _getDistanceToString:(NSString *)theString;
+- (void) _updateState;
+- (NSString *) _convertDisplayString:(NSString *)theString;
+- (void) _updateSkipCharacterSet;
+
+/* Initialisation and teardown */
+#pragma mark -
+- (id) init;
+- (id) initWithString:(NSString *)aString;
+- (id) initWithContentsOfFile:(NSString *)path encoding:(NSStringEncoding)enc error:(NSError **)error;
+- (void) dealloc;
+
+@end
diff --git a/Source/SPCSVParser.m b/Source/SPCSVParser.m
new file mode 100644
index 00000000..b141efcd
--- /dev/null
+++ b/Source/SPCSVParser.m
@@ -0,0 +1,642 @@
+//
+// $Id$
+//
+// SPCSVParser.m
+// sequel-pro
+//
+// Created by Rowan Beentje on 16/09/2009.
+// Copyright 2009 Rowan Beentje. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//
+// More info at <http://code.google.com/p/sequel-pro/>
+
+#import "SPCSVParser.h"
+
+/**
+ * Please see the header files for a general description of the purpose of this class.
+ */
+
+@implementation SPCSVParser : NSObject
+
+#pragma mark -
+#pragma mark Retrieving data from the CSV string
+
+/**
+ * Retrieve the entire two-dimensional array represented by the current string.
+ * Serves as a concenience method and also an example of how to use getRow:.
+ */
+- (NSArray *) array
+{
+ NSMutableArray *csvArray = [[NSMutableArray alloc] init];
+ NSArray *csvRowArray;
+
+ // Ensure that the full string is being parsed by resetting the parser position
+ parserPosition = trimPosition;
+ totalLengthParsed = 0;
+
+ // Loop through the results fetching process
+ while (csvRowArray = [self getRowAsArrayAndTrimString:NO stringIsComplete:YES])
+ {
+ CFArrayAppendValue((CFMutableArrayRef)csvArray, csvRowArray);
+ }
+
+ // Return the array
+ [csvArray autorelease];
+ return [NSArray arrayWithArray:csvArray];
+}
+
+/**
+ * Convenience method to retrieve the next row from the CSV string, without trimming, and
+ * assuming the CSV string is fully set.
+ * Returns nil if no more rows can be returned.
+ */
+- (NSArray *) getRowAsArray
+{
+ return [self getRowAsArrayAndTrimString:NO stringIsComplete:YES];
+}
+
+/**
+ * Allow retrieving a row from the result set - walks along the CSV, parsing until
+ * a row has been found. The resulting row is padding with NSNulls to match the first
+ * row encountered if necessary.
+ * Takes two arguments: whether to trim the string, useful when processing a CSV in
+ * streaming mode, and whether the current string is known to be complete. If in streaming
+ * mode, and the entire string has not yet been supplied, this should be set to NO; this
+ * prevents the final row (possibly without a trailing line terminator) from being returned
+ * prematurely.
+ * Returns nil if no more rows can be returned.
+ */
+- (NSArray *) getRowAsArrayAndTrimString:(BOOL)trimString stringIsComplete:(BOOL)stringComplete
+{
+ NSMutableArray *csvRowArray;
+ NSMutableString *csvCellString = [NSMutableString string];
+ long startingParserPosition, nextQuoteDistance, nextFieldEndDistance, nextLineEndDistance;
+ int skipLength, j;
+ BOOL fieldIsQuoted, isEscaped;
+ BOOL lineEndingEncountered = NO;
+
+ if (fieldCount == NSNotFound)
+ csvRowArray = [NSMutableArray array];
+ else
+ csvRowArray = [NSMutableArray arrayWithCapacity:fieldCount];
+
+ // Store the starting parser position so it can be restored if necessary
+ startingParserPosition = parserPosition;
+
+ // Loop along the CSV string, parsing.
+ while (parserPosition < csvStringLength && !lineEndingEncountered) {
+ [csvCellString setString:@""];
+ fieldIsQuoted = NO;
+
+ // Skip unescaped, unquoted whitespace where possible
+ [self _moveParserPastSkippableCharacters];
+
+ // Check the start of the string for the quote character, and loop along the string
+ // if so to capture the entire quoted string.
+ if (fieldQuoteLength && parserPosition + fieldQuoteLength <= csvStringLength
+ && [[csvString substringWithRange:NSMakeRange(parserPosition, fieldQuoteLength)] isEqualToString:fieldQuoteString])
+ {
+ parserPosition += fieldQuoteLength;
+ fieldIsQuoted = YES;
+
+ while (parserPosition < csvStringLength) {
+
+ // Find the next quote string
+ nextQuoteDistance = [self _getDistanceToString:fieldQuoteString];
+
+ // Check to see if the quote string encountered was escaped... or an escaper
+ if (escapeLength && nextQuoteDistance != NSNotFound) {
+ j = 1;
+ isEscaped = NO;
+ if (!escapeStringIsFieldQuoteString) {
+ while (j * escapeLength <= nextQuoteDistance
+ && ([[csvString substringWithRange:NSMakeRange((parserPosition + nextQuoteDistance - (j*escapeLength)), escapeLength)] isEqualToString:escapeString]))
+ {
+ isEscaped = !isEscaped;
+ j++;
+ }
+ skipLength = fieldQuoteLength;
+ } else {
+ if (parserPosition + nextQuoteDistance + (2 * fieldQuoteLength) <= csvStringLength
+ && [[csvString substringWithRange:NSMakeRange(parserPosition + nextQuoteDistance + fieldQuoteLength, fieldQuoteLength)] isEqualToString:fieldQuoteString])
+ {
+ isEscaped = YES;
+ skipLength = 2 * fieldQuoteLength;
+ }
+ }
+
+ // If it was escaped, keep processing the field.
+ if (isEscaped) {
+
+ // Append the matched string, together with the field quote character
+ // which has been determined to be within the string - but append the
+ // field end character unescaped to avoid later processing.
+ if (escapeStringIsFieldQuoteString) {
+ [csvCellString appendString:[csvString substringWithRange:NSMakeRange(parserPosition, nextQuoteDistance+fieldQuoteLength)]];
+ } else {
+ [csvCellString appendString:[csvString substringWithRange:NSMakeRange(parserPosition, nextQuoteDistance - escapeLength)]];
+ [csvCellString appendString:fieldQuoteString];
+ }
+
+ // Move the parser location to beyond the field end character[s]
+ parserPosition += nextQuoteDistance + skipLength;
+ continue;
+ }
+ }
+
+ // Add on the scanned string up to the terminating quote character.
+ if (nextQuoteDistance != NSNotFound) {
+ [csvCellString appendString:[csvString substringWithRange:NSMakeRange(parserPosition, nextQuoteDistance)]];
+ parserPosition += nextQuoteDistance + fieldQuoteLength;
+ } else {
+ [csvCellString appendString:[csvString substringWithRange:NSMakeRange(parserPosition, csvStringLength - parserPosition)]];
+ parserPosition = csvStringLength;
+ }
+
+ // We should now be at the end of the field - continue on past the quote,
+ // and remove whitespace if possible.
+ if (parserPosition < csvStringLength) {
+ [self _moveParserPastSkippableCharacters];
+ }
+
+ // Break out of the quoted field processing loop.
+ break;
+ }
+ }
+
+ // With quoted strings processed, now process the field until the next field end
+ // character, or the next line end character, both of which may terminate the current
+ // field. This also handles unquoted strings/numbers.
+ while (parserPosition < csvStringLength) {
+
+ // Determine whether a line end or a field end occurs first
+ nextFieldEndDistance = [self _getDistanceToString:fieldEndString];
+ nextLineEndDistance = [self _getDistanceToString:lineEndString];
+ if (nextLineEndDistance != NSNotFound
+ && (nextLineEndDistance < nextFieldEndDistance
+ || nextFieldEndDistance == NSNotFound))
+ {
+ nextFieldEndDistance = nextLineEndDistance;
+ lineEndingEncountered = YES;
+ skipLength = lineEndLength;
+ } else if (nextFieldEndDistance != NSNotFound) {
+ skipLength = fieldEndLength;
+ } else {
+ [csvCellString appendString:[csvString substringWithRange:NSMakeRange(parserPosition, csvStringLength - parserPosition)]];
+ parserPosition = csvStringLength;
+ break;
+ }
+
+ // Check to see if the termination character was escaped
+ if (escapeLength) {
+ j = 1;
+ isEscaped = NO;
+ while (j * escapeLength <= nextFieldEndDistance
+ && ([[csvString substringWithRange:NSMakeRange((parserPosition + nextFieldEndDistance - (j*escapeLength)), escapeLength)] isEqualToString:escapeString]))
+ {
+ isEscaped = !isEscaped;
+ j++;
+ }
+
+ // If it was, continue processing the field
+ if (isEscaped) {
+
+ // Append the matched string, together with the field/line character
+ // which was encountered - but append the string unescaped to avoid
+ // later processing.
+ [csvCellString appendString:[csvString substringWithRange:NSMakeRange(parserPosition, nextFieldEndDistance - escapeLength)]];
+ if (lineEndingEncountered) {
+ [csvCellString appendString:lineEndString];
+ lineEndingEncountered = NO;
+ } else {
+ [csvCellString appendString:fieldEndString];
+ }
+
+ // Update the parser location as appropriate
+ parserPosition += nextFieldEndDistance + skipLength;
+ continue;
+ }
+ }
+
+ // Add on the scanned string up to the terminating character
+ [csvCellString appendString:[csvString substringWithRange:NSMakeRange(parserPosition, nextFieldEndDistance)]];
+ parserPosition += nextFieldEndDistance + skipLength;
+
+ break;
+ }
+
+ // We now have a field content string.
+ // Insert a NSNull object if the cell contains an unescaped null character or
+ // an unquoted string which matches the set null replacement string.
+ if ([csvCellString isEqualToString:@"\\N"]
+ || (!fieldIsQuoted && nullReplacementString && [csvCellString isEqualToString:nullReplacementString]))
+ {
+ [csvRowArray addObject:[NSNull null]];
+ } else {
+
+ // Clean up escaped characters
+ if (escapeLength) {
+ if (fieldIsQuoted && fieldEndLength)
+ [csvCellString replaceOccurrencesOfString:escapedFieldEndString withString:fieldEndString options:NSLiteralSearch range:NSMakeRange(0, [csvCellString length])];
+ if (!fieldIsQuoted && fieldQuoteLength)
+ [csvCellString replaceOccurrencesOfString:escapedFieldQuoteString withString:fieldQuoteString options:NSLiteralSearch range:NSMakeRange(0, [csvCellString length])];
+ if (fieldIsQuoted && lineEndLength)
+ [csvCellString replaceOccurrencesOfString:escapedLineEndString withString:lineEndString options:NSLiteralSearch range:NSMakeRange(0, [csvCellString length])];
+ if (!escapeStringIsFieldQuoteString)
+ [csvCellString replaceOccurrencesOfString:escapedEscapeString withString:escapeString options:NSLiteralSearch range:NSMakeRange(0, [csvCellString length])];
+ }
+
+ // Add the field to the row array
+ [csvRowArray addObject:[NSString stringWithString:csvCellString]];
+ }
+ }
+
+ // If no line ending was encountered, as stringIsComplete is set to NO, return nil
+ // to ensure we don't return a "row" which is incomplete
+ if (!lineEndingEncountered && !stringComplete) {
+ parserPosition = startingParserPosition;
+ return nil;
+ }
+
+ // Update the total parsed length (differs from parserPosition following trims)
+ totalLengthParsed += parserPosition - startingParserPosition;
+
+ // Skip empty rows
+ if ([csvRowArray count] == 0
+ || ([csvRowArray count] == 1
+ && ([[csvRowArray objectAtIndex:0] isMemberOfClass:[NSNull class]]
+ || ![[csvRowArray objectAtIndex:0] length])))
+ {
+
+ // If the parser is at the end of the string, return nil
+ if (parserPosition == csvStringLength) return nil;
+
+ // Otherwise, retrieve the next row and return that instead
+ return [self getRowAsArrayAndTrimString:trimString stringIsComplete:stringComplete];
+ }
+
+ // Update the string trim state if appropriate, and lazily trigger trims
+ if (trimString) {
+ trimPosition = parserPosition;
+ [self _updateState];
+ }
+
+ // Capture the length of the first row when processing, and ensure that all
+ // subsequent rows contain that many cells
+ if (fieldCount == NSNotFound) {
+ fieldCount = [csvRowArray count];
+ } else if ([csvRowArray count] < fieldCount) {
+ for (j = [csvRowArray count]; j < fieldCount; j++) [csvRowArray addObject:[NSNull null]];
+ }
+
+ // Return the row
+ return csvRowArray;
+}
+
+#pragma mark -
+#pragma mark Adding new data to the string
+
+/**
+ * Append additional data to the CSV string, for example to allow streaming parsing.
+ */
+- (void) appendString:(NSString *)aString
+{
+ [csvString appendString:aString];
+ csvStringLength += [aString length];
+}
+
+/**
+ * Completely replace the underlying CSV string.
+ */
+- (void) setString:(NSString *)aString
+{
+ trimPosition = 0;
+ totalLengthParsed = 0;
+ [csvString setString:aString];
+ csvStringLength = [csvString length];
+}
+
+#pragma mark -
+#pragma mark Basic information
+
+/**
+ * Retrieve the string length.
+ */
+- (NSUInteger) length
+{
+ return csvStringLength - trimPosition;
+}
+
+/**
+ * Retrieve the underlying CSV string.
+ */
+- (NSString *) string
+{
+ return [csvString substringWithRange:NSMakeRange(trimPosition, csvStringLength - trimPosition)];
+}
+
+/**
+ * Return the parser position
+ */
+- (long) parserPosition
+{
+ return parserPosition;
+}
+
+/**
+ * Return the total length of CSV parsed so far - differs from the parser position in
+ * streaming/trimming situations
+ */
+- (long) totalLengthParsed
+{
+ return totalLengthParsed;
+}
+
+#pragma mark -
+#pragma mark Setting the terminator, quote, escape and null character replacement strings
+
+/**
+ * Allow setting the field terminator string.
+ * If passing in fields from a user interface which are representative, eg the user typing in
+ * \ and t for a tab character, pass in YES as the second character. (eg "\t" => \t)
+ */
+- (void) setFieldTerminatorString:(NSString *)theString convertDisplayStrings:(BOOL)convertString
+{
+ if (convertString) {
+ theString = [self _convertDisplayString:theString];
+ }
+
+ [fieldEndString release];
+ fieldEndString = [[NSString alloc] initWithString:theString];
+ fieldEndLength = [fieldEndString length];
+ [escapedFieldEndString release];
+ escapedFieldEndString = [[NSString alloc] initWithFormat:@"%@%@", escapeString, fieldEndString];
+
+ [self _updateSkipCharacterSet];
+}
+
+/**
+ * Allow setting the field terminator string.
+ * If passing in fields from a user interface which are representative, eg the user typing in
+ * \ and t for a tab character, pass in YES as the second character. (eg "\t" => \t)
+ */
+- (void) setLineTerminatorString:(NSString *)theString convertDisplayStrings:(BOOL)convertString
+{
+ if (convertString) {
+ theString = [self _convertDisplayString:theString];
+ }
+
+ [lineEndString release];
+ lineEndString = [[NSString alloc] initWithString:theString];
+ lineEndLength = [lineEndString length];
+ [escapedLineEndString release];
+ escapedLineEndString = [[NSString alloc] initWithFormat:@"%@%@", escapeString, lineEndString];
+
+ [self _updateSkipCharacterSet];
+}
+
+/**
+ * Allow setting the field terminator string.
+ * If passing in fields from a user interface which are representative, eg the user typing in
+ * \ and t for a tab character, pass in YES as the second character. (eg "\t" => \t)
+ */
+- (void) setFieldQuoteString:(NSString *)theString convertDisplayStrings:(BOOL)convertString
+{
+ if (convertString) {
+ theString = [self _convertDisplayString:theString];
+ }
+
+ [fieldQuoteString release];
+ fieldQuoteString = [[NSString alloc] initWithString:theString];
+ fieldQuoteLength = [fieldQuoteString length];
+ [escapedFieldQuoteString release];
+ escapedFieldQuoteString = [[NSString alloc] initWithFormat:@"%@%@", escapeString, fieldQuoteString];
+ escapeStringIsFieldQuoteString = [fieldQuoteString isEqualToString:escapeString];
+
+ [self _updateSkipCharacterSet];
+}
+
+/**
+ * Allow setting the field terminator string.
+ * If passing in fields from a user interface which are representative, eg the user typing in
+ * \ and t for a tab character, pass in YES as the second character. (eg "\t" => \t)
+ */
+- (void) setEscapeString:(NSString *)theString convertDisplayStrings:(BOOL)convertString
+{
+ if (convertString) {
+ theString = [self _convertDisplayString:theString];
+ }
+
+ [escapeString release];
+ escapeString = [[NSString alloc] initWithString:theString];
+ escapeLength = [escapeString length];
+ [escapedEscapeString release];
+ escapedEscapeString = [[NSString alloc] initWithFormat:@"%@%@", escapeString, escapeString];
+ escapeStringIsFieldQuoteString = [fieldQuoteString isEqualToString:escapeString];
+
+ [self _updateSkipCharacterSet];
+}
+
+/**
+ * Allow setting a string to be replaced by NSNull if it is encountered *unquoted*
+ * during import. Defaults to nil, so no strings are replaced.
+ */
+- (void) setNullReplacementString:(NSString *)nullString
+{
+ if (nullReplacementString) [nullReplacementString release], nullReplacementString = nil;
+
+ if (nullString) nullReplacementString = [[NSString alloc] initWithString:nullString];
+}
+
+#pragma mark -
+#pragma mark Init and internal update methods
+
+/**
+ * Set up the string for CSV parsing, together with class defaults.
+ */
+- (void) _initialiseCSVParserDefaults
+{
+ trimPosition = 0;
+ fieldCount = NSNotFound;
+ parserPosition = 0;
+ totalLengthParsed = 0;
+ csvStringLength = [csvString length];
+
+ // Set up the default field and line separators, together with quote
+ // and escape strings
+ fieldEndString = [[NSString alloc] initWithString:@","];
+ lineEndString = [[NSString alloc] initWithString:@"\n"];
+ fieldQuoteString = [[NSString alloc] initWithString:@"\""];
+ escapeString = [[NSString alloc] initWithString:@"\\"];
+ escapeStringIsFieldQuoteString = NO;
+ escapedFieldEndString = [[NSString alloc] initWithString:@"\\,"];
+ escapedLineEndString = [[NSString alloc] initWithString:@"\\\n"];
+ escapedFieldQuoteString = [[NSString alloc] initWithString:@"\\\""];
+ escapedEscapeString = [[NSString alloc] initWithString:@"\\\\"];
+ fieldEndLength = [fieldEndString length];
+ lineEndLength = [lineEndString length];
+ fieldQuoteLength = [fieldQuoteString length];
+ escapeLength = [escapeString length];
+
+ // Set up the default null replacement character string as nil
+ nullReplacementString = nil;
+
+ // With the default field and line separators, it's possible to skip
+ // a few characters - reset the character set that can be skipped
+ skipCharacterSet = nil;
+ [self _updateSkipCharacterSet];
+}
+
+/**
+ * Update the string state, enacting trims lazily to trade-off memory usage and the
+ * speed hit with constant string updates and NSScanner resets.
+ */
+- (void) _updateState
+{
+
+ // If the trim position is still before the trim enact point, do nothing.
+ if (trimPosition < SPCSVPARSER_TRIM_ENACT_LENGTH) return;
+
+ // Trim the string
+ [csvString deleteCharactersInRange:NSMakeRange(0, trimPosition)];
+
+ // Update the parse position and stored string length
+ parserPosition -= trimPosition;
+ csvStringLength -= trimPosition;
+
+ // Reset the trim position
+ trimPosition = 0;
+}
+
+/**
+ * Takes a display string and converts representations of special characters
+ * to the represented characters; for example, the string with characters \ and t
+ * to the tab character.
+ */
+- (NSString *) _convertDisplayString:(NSString *)theString
+{
+ NSMutableString *conversionString = [NSMutableString stringWithString:theString];
+
+ [conversionString replaceOccurrencesOfString:@"\\t" withString:@"\t"
+ options:NSLiteralSearch
+ range:NSMakeRange(0, [conversionString length])];
+ [conversionString replaceOccurrencesOfString:@"\\n" withString:@"\n"
+ options:NSLiteralSearch
+ range:NSMakeRange(0, [conversionString length])];
+ [conversionString replaceOccurrencesOfString:@"\\r" withString:@"\r"
+ options:NSLiteralSearch
+ range:NSMakeRange(0, [conversionString length])];
+
+ return [NSString stringWithString:conversionString];
+}
+
+/**
+ * Reset the character set that can be skipped when processing the CSV.
+ * This is called whenever the delimiters, quotes and escapes are updated.
+ */
+- (void) _updateSkipCharacterSet
+{
+ NSMutableString *charactersToSkip;
+
+ if (skipCharacterSet) [skipCharacterSet release], skipCharacterSet = nil;
+
+ charactersToSkip = [[NSMutableString alloc] init];
+ if (![fieldEndString isEqualToString:@" "] && ![fieldQuoteString isEqualToString:@" "] && ![escapeString isEqualToString:@" "] && ![lineEndString isEqualToString:@" "])
+ [charactersToSkip appendString:@" "];
+ if (![fieldEndString isEqualToString:@"\t"] && ![fieldQuoteString isEqualToString:@"\t"] && ![escapeString isEqualToString:@"\t"] && ![lineEndString isEqualToString:@"\t"])
+ [charactersToSkip appendString:@"\t"];
+
+ if ([charactersToSkip length])
+ skipCharacterSet = [[NSCharacterSet characterSetWithCharactersInString:charactersToSkip] retain];
+
+ [charactersToSkip release];
+}
+
+/**
+ * Get the distance to the next occurence of the specified string, from the current
+ * parser location. Returns NSNotFound if the string could not be found before the end
+ * of the string.
+ */
+- (long) _getDistanceToString:(NSString *)theString
+{
+ NSRange stringRange = [csvString rangeOfString:theString options:NSLiteralSearch range:NSMakeRange(parserPosition, csvStringLength - parserPosition)];
+
+ if (stringRange.location == NSNotFound) return NSNotFound;
+
+ return stringRange.location - parserPosition;
+}
+
+/**
+ * Move the parser past any skippable characters - this should be called to effectively trim
+ * whitespace from the starts and ends of cells, unless that whitespace is quoted. By
+ * maintaining a list of skippable characters, any whitespace used in quote/line end/field
+ * end chars is preserved safely.
+ */
+- (void) _moveParserPastSkippableCharacters
+{
+ if (!skipCharacterSet) return;
+
+ int i = 0;
+ while (parserPosition + i < csvStringLength) {
+ if (![skipCharacterSet characterIsMember:[csvString characterAtIndex:parserPosition+i]]) break;
+ i++;
+ }
+ if (i) parserPosition += i;
+}
+
+/**
+ * Required and primitive methods to allow subclassing class cluster
+ */
+#pragma mark -
+
+- (id) init {
+ if (self = [super init]) {
+ csvString = [[NSMutableString alloc] init];
+ [self _initialiseCSVParserDefaults];
+ }
+ return self;
+}
+- (id) initWithString:(NSString *)aString
+{
+ if (self = [super init]) {
+ csvString = [[NSMutableString alloc] initWithString:aString];
+ [self _initialiseCSVParserDefaults];
+ }
+ return self;
+}
+- (id) initWithContentsOfFile:(NSString *)path encoding:(NSStringEncoding)encoding error:(NSError **)error {
+ if (self = [super init]) {
+ csvString = [[NSMutableString alloc] initWithContentsOfFile:path encoding:encoding error:error];
+ [self _initialiseCSVParserDefaults];
+ }
+ return self;
+}
+- (void) dealloc {
+ [csvString release];
+ [fieldEndString release];
+ [lineEndString release];
+ [fieldQuoteString release];
+ [escapeString release];
+ [escapedFieldEndString release];
+ [escapedLineEndString release];
+ [escapedFieldQuoteString release];
+ [escapedEscapeString release];
+ if (nullReplacementString) [nullReplacementString release];
+ if (skipCharacterSet) [skipCharacterSet release];
+ [super dealloc];
+}
+
+@end \ No newline at end of file
diff --git a/Source/TableDump.h b/Source/TableDump.h
index dd9ffd6c..28d8107d 100644
--- a/Source/TableDump.h
+++ b/Source/TableDump.h
@@ -97,10 +97,11 @@
MCPConnection *mySQLConnection;
NSMutableArray *tables;
- NSArray *importArray;
+ NSArray *fieldMappingImportArray;
+ BOOL fieldMappingImportArrayIsPreview;
NSMutableArray *fieldMappingArray;
NSMutableArray *fieldMappingButtonOptions;
- int currentRow;
+ int fieldMappingCurrentRow;
NSUserDefaults *prefs;
BOOL progressCancelled;
}
@@ -125,12 +126,10 @@
- (IBAction)changeFormat:(id)sender;
- (IBAction)changeTable:(id)sender;
- (void)openPanelDidEnd:(NSOpenPanel *)sheet returnCode:(int)returnCode contextInfo:(NSString *)contextInfo;
+- (BOOL) buildFieldMappingArrayWithData:(NSArray *)importData isPreview:(BOOL)dataIsPreviewData;
- (void)setupFieldMappingArray;
- (void)updateFieldMappingButtonCell;
-- (NSArray *)arrayForCSV:(NSString *)csv terminatedBy:(NSString *)terminated
- enclosedBy:(NSString *)enclosed escapedBy:(NSString *)escaped lineEnds:(NSString *)lineEnds;
-- (NSArray *)arrayForString:(NSString *)string enclosed:(NSString *)enclosed
- escaped:(NSString *)escaped terminated:(NSString *)terminated;
+- (NSString *) mappedValueStringForRowArray:(NSArray *)csvRowArray;
// Export methods
- (BOOL)dumpSelectedTablesAsSqlToFileHandle:(NSFileHandle *)fileHandle;
diff --git a/Source/TableDump.m b/Source/TableDump.m
index 6a55610d..3f518214 100644
--- a/Source/TableDump.m
+++ b/Source/TableDump.m
@@ -31,6 +31,7 @@
#import "CustomQuery.h"
#import "SPGrowlController.h"
#import "SPSQLParser.h"
+#import "SPCSVParser.h"
#import "SPTableData.h"
#import "SPStringAdditions.h"
#import "SPArrayAdditions.h"
@@ -108,6 +109,17 @@
[NSApp stopModalWithCode:[sender tag]];
}
+/**
+ * Convenience method for closing and restoring the progress sheet to default state.
+ */
+- (void) closeAndStopProgressSheet
+{
+ [NSApp endSheet:singleProgressSheet];
+ [singleProgressSheet orderOut:nil];
+ [singleProgressBar stopAnimation:self];
+ [singleProgressBar setMaxValue:100];
+}
+
#pragma mark -
#pragma mark Export methods
@@ -316,9 +328,7 @@
}
// Close the progress sheet
- [singleProgressBar stopAnimation:self];
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
+ [self closeAndStopProgressSheet];
// Export the current custom query result set to a file in CSV or XML format
} else if ( [contextInfo isEqualToString:@"exportCustomResultAsCSV"]
@@ -356,9 +366,7 @@
}
// Close the progress sheet
- [singleProgressBar stopAnimation:self];
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
+ [self closeAndStopProgressSheet];
// Export multiple tables to a file in CSV format
} else if ( [contextInfo isEqualToString:@"exportMultipleTablesAsCSV"] ) {
@@ -431,12 +439,12 @@
[tablesListInstance selectTableOrViewWithName:[fieldMappingPopup titleOfSelectedItem]];
//set up tableView
- currentRow = 0;
+ fieldMappingCurrentRow = 0;
if (fieldMappingArray) [fieldMappingArray release], fieldMappingArray = nil;
[self setupFieldMappingArray];
[rowDownButton setEnabled:NO];
- [rowUpButton setEnabled:([importArray count] > 1)];
- [recordCountLabel setStringValue:[NSString stringWithFormat:@"%i of %i records", currentRow+1, [importArray count]]];
+ [rowUpButton setEnabled:([fieldMappingImportArray count] > 1)];
+ [recordCountLabel setStringValue:[NSString stringWithFormat:@"%i of %@%i records", fieldMappingCurrentRow+1, fieldMappingImportArrayIsPreview?@"first ":@"", [fieldMappingImportArray count]]];
[self updateFieldMappingButtonCell];
[fieldMappingTableView reloadData];
@@ -500,6 +508,7 @@
[singleProgressBar setIndeterminate:NO];
[singleProgressBar setMaxValue:fileTotalLength];
[singleProgressBar setUsesThreadedAnimation:YES];
+ [singleProgressBar startAnimation:self];
// Open the progress sheet
[NSApp beginSheet:singleProgressSheet modalForWindow:tableWindow modalDelegate:self didEndSelector:nil contextInfo:nil];
@@ -512,12 +521,15 @@
sqlDataBuffer = [[NSMutableData alloc] init];
importPool = [[NSAutoreleasePool alloc] init];
while (1) {
+ if (progressCancelled) break;
+
@try {
fileChunk = [sqlFileHandle readDataOfLength:fileChunkMaxLength];
}
// Report file read errors, and bail
@catch (NSException *exception) {
+ [self closeAndStopProgressSheet];
NSBeginAlertSheet(NSLocalizedString(@"SQL read error title", @"File read error"),
NSLocalizedString(@"OK", @"OK button"),
nil, nil, tableWindow, self, nil, nil, nil,
@@ -565,6 +577,7 @@
sqlString = [[NSString alloc] initWithData:[sqlDataBuffer subdataWithRange:NSMakeRange(dataBufferLastQueryEndPosition, dataBufferPosition - dataBufferLastQueryEndPosition)]
encoding:[MCPConnection encodingForMySQLEncoding:[[tableDocumentInstance connectionEncoding] UTF8String]]];
if (!sqlString) {
+ [self closeAndStopProgressSheet];
NSBeginAlertSheet(NSLocalizedString(@"SQL read error title", @"File read error"),
NSLocalizedString(@"OK", @"OK button"),
nil, nil, tableWindow, self, nil, nil, nil,
@@ -630,7 +643,7 @@
// If any text remains in the SQL parser, it's an unterminated query - execute it.
query = [sqlParser stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]];
- if ([query length]) {
+ if ([query length] && !progressCancelled) {
// Run the query
[mySQLConnection queryString:query usingEncoding:sqlEncoding streamingResult:NO];
@@ -651,10 +664,8 @@
[tableDocumentInstance setQueryMode:SP_QUERYMODE_INTERFACE];
// Close progress sheet
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
- [singleProgressBar setMaxValue:100];
-
+ [self closeAndStopProgressSheet];
+
// Display any errors
if ([errors length]) {
[errorsView setString:errors];
@@ -681,256 +692,286 @@
- (void) importCSVFile:(NSString *)filename
{
- NSString *dumpFile = nil;
- NSError *errorStr = nil;
+ NSAutoreleasePool *importPool;
+ NSFileHandle *csvFileHandle;
+ NSMutableData *csvDataBuffer;
+ const unsigned char *csvDataBufferBytes;
+ NSData *fileChunk;
+ NSString *csvString;
+ SPCSVParser *csvParser;
+ NSMutableString *query;
NSMutableString *errors = [NSMutableString string];
+ NSMutableString *insertBaseString = [NSMutableString string];
+ NSMutableArray *parsedRows = [[NSMutableArray alloc] init];
+ NSMutableArray *parsePositions = [[NSMutableArray alloc] init];
+ NSArray *csvRowArray;
+ NSInteger fileChunkMaxLength = 256 * 1024;
+ NSInteger csvRowsPerQuery = 50;
+ NSInteger csvRowsThisQuery;
+ NSInteger fileTotalLength = 0;
+ NSInteger rowsImported = 0;
+ NSInteger dataBufferLength = 0;
+ NSInteger dataBufferPosition = 0;
+ NSInteger dataBufferLastQueryEndPosition = 0;
+ int i;
+ BOOL allDataRead = NO;
+ NSStringEncoding csvEncoding = [MCPConnection encodingForMySQLEncoding:[[tableDocumentInstance connectionEncoding] UTF8String]];
+ if (fieldMappingArray) [fieldMappingArray release], fieldMappingArray = nil;
+
+ // Open a filehandle for the CSV file
+ csvFileHandle = [NSFileHandle fileHandleForReadingAtPath:filename];
+ if (!csvFileHandle) {
+ NSBeginAlertSheet(NSLocalizedString(@"Import Error title", @"Import Error"),
+ NSLocalizedString(@"OK button label", @"OK button"),
+ nil, nil, tableWindow, self, nil, nil, nil,
+ NSLocalizedString(@"CSV file open error", @"The CSV file you selected could not be found or read."));
+ return;
+ }
+
+ // Grab the file length
+ fileTotalLength = [[[[NSFileManager defaultManager] fileAttributesAtPath:filename traverseLink:YES] objectForKey:NSFileSize] integerValue];
+ if (!fileTotalLength) fileTotalLength = 1;
// Reset progress interface
[errorsView setString:@""];
- [errorsView displayIfNeeded];
[singleProgressTitle setStringValue:NSLocalizedString(@"Importing CSV", @"text showing that the application is importing CSV")];
- [singleProgressTitle displayIfNeeded];
- [singleProgressText setStringValue:NSLocalizedString(@"Reading...", @"text showing that app is reading dump")];
- [singleProgressText displayIfNeeded];
+ [singleProgressText setStringValue:NSLocalizedString(@"Reading...", @"text showing that app is reading CSV file")];
[singleProgressBar setIndeterminate:YES];
[singleProgressBar setUsesThreadedAnimation:YES];
[singleProgressBar startAnimation:self];
-
- int code;
-
- //open progress sheet
- [NSApp beginSheet:singleProgressSheet
- modalForWindow:tableWindow
- modalDelegate:self
- didEndSelector:nil
- contextInfo:nil];
+
+ // Open the progress sheet
+ [NSApp beginSheet:singleProgressSheet modalForWindow:tableWindow modalDelegate:self didEndSelector:nil contextInfo:nil];
[singleProgressSheet makeKeyWindow];
[tableDocumentInstance setQueryMode:SP_QUERYMODE_IMPORTEXPORT];
-
- // Read the file with the current connection encoding.
- dumpFile = [NSString stringWithContentsOfFile:filename
- encoding:[MCPConnection encodingForMySQLEncoding:[[tableDocumentInstance connectionEncoding] UTF8String]]
- error:&errorStr];
- if (errorStr) {
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
- NSBeginAlertSheet(NSLocalizedString(@"Error", @"Error"),
- NSLocalizedString(@"OK", @"OK button"),
- nil, nil,
- tableWindow, self,
- nil, nil, nil,
- [errorStr localizedDescription]
- );
- [tableDocumentInstance setQueryMode:SP_QUERYMODE_INTERFACE];
- return;
- }
-
-
- //put file in array
- if (importArray)
- [importArray release];
-
- importArray = [[self arrayForCSV:dumpFile
- terminatedBy:[importFieldsTerminatedField stringValue]
- enclosedBy:[importFieldsEnclosedField stringValue]
- escapedBy:[importFieldsEscapedField stringValue]
- lineEnds:[importLinesTerminatedField stringValue]] retain];
-
- long importArrayCount = [importArray count];
-
- //close progress sheet
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
- [singleProgressBar stopAnimation:self];
- [singleProgressBar setUsesThreadedAnimation:NO];
- [singleProgressBar setIndeterminate:NO];
-
- if(importArrayCount == 0){
- NSBeginAlertSheet(NSLocalizedString(@"Error", @"Error"),
- NSLocalizedString(@"OK", @"OK button"),
- nil, nil,
- tableWindow, self,
- nil, nil, nil,
- NSLocalizedString(@"Could not parse file as CSV", @"Error when we can't parse/split file as CSV")
- );
- [importArray release], importArray = nil;
- [tableDocumentInstance setQueryMode:SP_QUERYMODE_INTERFACE];
- return;
- }
-
- if (progressCancelled) {
- progressCancelled = NO;
- [importArray release], importArray = nil;
- [tableDocumentInstance setQueryMode:SP_QUERYMODE_INTERFACE];
- return;
- }
- MCPResult *theResult;
- int i;
- theResult = (MCPResult *) [mySQLConnection listTables];
- if ([theResult numOfRows]) [theResult dataSeek:0];
- [fieldMappingPopup removeAllItems];
- for ( i = 0 ; i < [theResult numOfRows] ; i++ ) {
- [fieldMappingPopup addItemWithTitle:NSArrayObjectAtIndex([theResult fetchRowAsArray], 0)];
- }
-
- if ([tableDocumentInstance table] != nil && ![(NSString *)[tableDocumentInstance table] isEqualToString:@""]) {
- [fieldMappingPopup selectItemWithTitle:[(TableDocument *)tableDocumentInstance table]];
- } else {
- [fieldMappingPopup selectItemAtIndex:0];
- }
-
- if( ![tablesListInstance selectTableOrViewWithName:[fieldMappingPopup titleOfSelectedItem]] ) {
- [errors appendString:[NSString stringWithFormat:NSLocalizedString(@"[ERROR] %@\n", @"error text when trying to import csv data, but we have no tables in the db"), @"Can't import CSV data into a database without any tables!"]];
- } else {
-
- //set up tableView
- currentRow = 0;
+ // Read in the file in a loop. The loop actually needs to perform three tasks: read in
+ // CSV data and parse them into row arrays; present the field mapping interface once it
+ // has some data to show within the interface; and use the field mapping data to construct
+ // and send queries to the server. The loop is mainly to perform the first of these; the
+ // other two must therefore be performed where possible.
+ csvParser = [[SPCSVParser alloc] init];
+ csvDataBuffer = [[NSMutableData alloc] init];
+ importPool = [[NSAutoreleasePool alloc] init];
+ while (1) {
+ if (progressCancelled) break;
- // Sanity check the first row of the CSV to prevent hang loops caused by wrong line ending entry
- if ([[importArray objectAtIndex:currentRow] count] > 512) {
- NSBeginAlertSheet(NSLocalizedString(@"Error", @"error"),
+ @try {
+ fileChunk = [csvFileHandle readDataOfLength:fileChunkMaxLength];
+ }
+
+ // Report file read errors, and bail
+ @catch (NSException *exception) {
+ [self closeAndStopProgressSheet];
+ NSBeginAlertSheet(NSLocalizedString(@"CSV read error title", @"File read error"),
NSLocalizedString(@"OK", @"OK button"),
- nil, nil,
- tableWindow, self,
- nil, nil, nil,
- NSLocalizedString(@"The CSV was read as containing more than 512 columns, more than the maximum columns permitted for speed reasons by Sequel Pro.\n\nThis usually happens due to errors reading the CSV; please double-check the CSV to be imported and the line endings and escape characters at the bottom of the CSV selection dialog.", @"Error when CSV appears to have too many columns to import, probably due to line ending mismatch")
- );
- [importArray release], importArray = nil;
+ nil, nil, tableWindow, self, nil, nil, nil,
+ [NSString stringWithFormat:NSLocalizedString(@"CSV read error", @"An error occurred when reading the file.\n\nOnly %i rows were imported.\n\n(%@)"), rowsImported, [exception reason]]);
+ [csvParser release];
+ [csvDataBuffer release];
+ [parsedRows release];
+ [parsePositions release];
+ [importPool drain];
[tableDocumentInstance setQueryMode:SP_QUERYMODE_INTERFACE];
return;
}
-
- if (fieldMappingArray) [fieldMappingArray release], fieldMappingArray = nil;
- [self setupFieldMappingArray];
- [rowDownButton setEnabled:NO];
- [rowUpButton setEnabled:(importArrayCount > 1)];
- [recordCountLabel setStringValue:[NSString stringWithFormat:@"%i of %i records", currentRow+1, importArrayCount]];
-
- //set up tableView buttons
- NSPopUpButtonCell *buttonCell = [[NSPopUpButtonCell alloc] init];
- [buttonCell setControlSize:NSSmallControlSize];
- [buttonCell setFont:[NSFont labelFontOfSize:[NSFont smallSystemFontSize]]];
- [buttonCell setBordered:NO];
- [[fieldMappingTableView tableColumnWithIdentifier:@"value"] setDataCell:buttonCell];
- [self updateFieldMappingButtonCell];
- [fieldMappingTableView reloadData];
- [buttonCell release];
-
- // show fieldMapping sheet
- [NSApp beginSheet:fieldMappingSheet
- modalForWindow:tableWindow
- modalDelegate:self
- didEndSelector:nil
- contextInfo:nil];
-
- code = [NSApp runModalForWindow:fieldMappingSheet];
- [NSApp endSheet:fieldMappingSheet];
- [fieldMappingSheet orderOut:nil];
-
- if ( code ) {
- //import array into db
- NSMutableString *fNames = [NSMutableString string];
- //NSMutableArray *fValuesIndexes = [NSMutableArray array];
- NSMutableString *fValues = [NSMutableString string];
- NSString *insertFormatString = nil;
- int i,j;
-
- //open progress sheet
- [NSApp beginSheet:singleProgressSheet
- modalForWindow:tableWindow
- modalDelegate:self
- didEndSelector:nil
- contextInfo:nil];
-
- [singleProgressBar setUsesThreadedAnimation:NO];
- [singleProgressSheet makeKeyWindow];
- [singleProgressText setStringValue:NSLocalizedString(@"Creating rows...", @"text showing that app is importing rows from CSV")];
- [singleProgressText displayIfNeeded];
-
- // get fields to be imported
- for (i = 0; i < [fieldMappingArray count] ; i++ ) {
- if ([NSArrayObjectAtIndex(fieldMappingArray, i) intValue] > 0) {
- if ( [fNames length] )
- [fNames appendString:@","];
-
- [fNames appendString:[NSArrayObjectAtIndex([tableSourceInstance fieldNames], i) backtickQuotedString]];
+
+ // If no data returned, end of file - set a marker to ensure full processing
+ if (!fileChunk || ![fileChunk length]) {
+ allDataRead = YES;
+
+ // Otherwise add the data to the read/parse buffer
+ } else {
+ [csvDataBuffer appendData:fileChunk];
+ }
+
+ // Step through the data buffer, identifying line endings to parse the data with
+ csvDataBufferBytes = [csvDataBuffer bytes];
+ dataBufferLength = [csvDataBuffer length];
+ for ( ; dataBufferPosition < dataBufferLength || allDataRead; dataBufferPosition++) {
+ if (csvDataBufferBytes[dataBufferPosition] == 0x0A || csvDataBufferBytes[dataBufferPosition] == 0x0D || allDataRead) {
+
+ // Keep reading through any other line endings
+ while (dataBufferPosition + 1 < dataBufferLength
+ && (csvDataBufferBytes[dataBufferPosition+1] == 0x0A
+ || csvDataBufferBytes[dataBufferPosition+1] == 0x0D))
+ {
+ dataBufferPosition++;
+ }
+
+ // Try to generate a NSString with the resulting data
+ csvString = [[NSString alloc] initWithData:[csvDataBuffer subdataWithRange:NSMakeRange(dataBufferLastQueryEndPosition, dataBufferPosition - dataBufferLastQueryEndPosition)] encoding:csvEncoding];
+ if (!csvString) {
+ [self closeAndStopProgressSheet];
+ NSBeginAlertSheet(NSLocalizedString(@"CSV read error title", @"File read error"),
+ NSLocalizedString(@"OK", @"OK button"),
+ nil, nil, tableWindow, self, nil, nil, nil,
+ [NSString stringWithFormat:NSLocalizedString(@"CSV encoding read error", @"An error occurred when reading the file, as it could not be read using %@.\n\nOnly %i rows were imported."), [[tableDocumentInstance connectionEncoding] UTF8String], rowsImported]);
+ [csvParser release];
+ [csvDataBuffer release];
+ [parsedRows release];
+ [parsePositions release];
+ [importPool drain];
+ [tableDocumentInstance setQueryMode:SP_QUERYMODE_INTERFACE];
+ return;
}
+
+ // Add the NSString segment to the CSV parser and release it
+ [csvParser appendString:csvString];
+ [csvString release];
+
+ if (allDataRead) break;
+
+ // Increment the buffer end position marker
+ dataBufferLastQueryEndPosition = dataBufferPosition;
}
-
- // import array
- long fieldMappingArrayCount = [fieldMappingArray count];
- insertFormatString = [NSString stringWithFormat:@"INSERT INTO %@ (%@) VALUES (%%@)",
- [[fieldMappingPopup titleOfSelectedItem] backtickQuotedString], fNames];
- int fieldMappingIntValue;
- Class nullClass = [NSNull class];
-
- for ( i = 0 ; i < importArrayCount ; i++ ) {
- //show progress bar
- [singleProgressBar setDoubleValue:((i+1)*100/importArrayCount)];
-
- if ( !([importFieldNamesSwitch state] && (i == 0)) ) {
- //put values in string
- [fValues setString:@""];
-
- for ( j = 0 ; j < fieldMappingArrayCount ; j++ ) {
- fieldMappingIntValue = [NSArrayObjectAtIndex(fieldMappingArray,j) intValue];
- if ( fieldMappingIntValue > 0 ) {
-
- if ( [fValues length] )
- [fValues appendString:@","];
+ }
+
+ // Trim the data buffer if part of it was used
+ if (dataBufferLastQueryEndPosition) {
+ [csvDataBuffer setData:[csvDataBuffer subdataWithRange:NSMakeRange(dataBufferLastQueryEndPosition, dataBufferLength - dataBufferLastQueryEndPosition)]];
+ dataBufferPosition -= dataBufferLastQueryEndPosition;
+ dataBufferLastQueryEndPosition = 0;
+ }
- id c = NSArrayObjectAtIndex(NSArrayObjectAtIndex(importArray, i), (fieldMappingIntValue - 1));
+ // Extract and process any full CSV rows found so far. Also trigger processing if all
+ // rows have been read, in order to ensure short files are still processed.
+ while ((csvRowArray = [csvParser getRowAsArrayAndTrimString:YES stringIsComplete:allDataRead]) || (allDataRead && !fieldMappingArray)) {
- [fValues appendString: ([c isMemberOfClass:nullClass]) ?
- @"NULL" : [NSString stringWithFormat:@"'%@'", [mySQLConnection prepareString:c]]];
- }
+ // If valid, add the row array and length to local storage
+ if (csvRowArray) {
+ [parsedRows addObject:csvRowArray];
+ [parsePositions addObject:[NSNumber numberWithLong:[csvParser totalLengthParsed]]];
+ }
+
+ // If we have no field mapping array, and either the first hundred rows or all
+ // the rows, request the field mapping from the user.
+ if (!fieldMappingArray
+ && ([parsedRows count] >= 100 || (!csvRowArray && allDataRead)))
+ {
+ [self closeAndStopProgressSheet];
+ if (![self buildFieldMappingArrayWithData:parsedRows isPreview:!allDataRead]) {
+ [csvParser release];
+ [csvDataBuffer release];
+ [parsedRows release];
+ [parsePositions release];
+ [importPool drain];
+ [tableDocumentInstance setQueryMode:SP_QUERYMODE_INTERFACE];
+ return;
+ }
+
+ // Reset progress interface and open the progress sheet
+ [singleProgressBar setIndeterminate:NO];
+ [singleProgressBar setMaxValue:fileTotalLength];
+ [singleProgressBar startAnimation:self];
+ [NSApp beginSheet:singleProgressSheet modalForWindow:tableWindow modalDelegate:self didEndSelector:nil contextInfo:nil];
+ [singleProgressSheet makeKeyWindow];
+
+ // Set up the field names import string
+ [insertBaseString appendString:@"INSERT INTO "];
+ [insertBaseString appendString:[[fieldMappingPopup titleOfSelectedItem] backtickQuotedString]];
+ [insertBaseString appendString:@" ("];
+ for (i = 0; i < [fieldMappingArray count]; i++) {
+ if ([NSArrayObjectAtIndex(fieldMappingArray, i) intValue] > 0) {
+ if (i > 0) [insertBaseString appendString:@","];
+ [insertBaseString appendString:[NSArrayObjectAtIndex([tableSourceInstance fieldNames], i) backtickQuotedString]];
}
-
- //perform query
- [mySQLConnection queryString:[NSString stringWithFormat:insertFormatString, fValues]];
-
- if ( ![[mySQLConnection getLastErrorMessage] isEqualToString:@""] ) {
- [errors appendString:[NSString stringWithFormat:
- NSLocalizedString(@"[ERROR in line %d] %@\n", @"error text when reading of csv file gave errors"),
- (i+1),[mySQLConnection getLastErrorMessage]]];
+ }
+ [insertBaseString appendString:@") VALUES\n"];
+
+ // Remove the header row from the data set if appropriate
+ if ([importFieldNamesSwitch state] == NSOnState) {
+ [parsedRows removeObjectAtIndex:0];
+ [parsePositions removeObjectAtIndex:0];
+ }
+ }
+ if (!fieldMappingArray) continue;
+
+ // If we have more than the csvRowsPerQuery amount, or if we're at the end of the
+ // available data, construct and run a query.
+ while ([parsedRows count] >= csvRowsPerQuery
+ || (!csvRowArray && allDataRead && [parsedRows count]))
+ {
+ query = [[NSMutableString alloc] initWithString:insertBaseString];
+ csvRowsThisQuery = 0;
+ for (i = 0; i < csvRowsPerQuery && i < [parsedRows count]; i++) {
+ if (i > 0) [query appendString:@",\n"];
+ [query appendString:[self mappedValueStringForRowArray:[parsedRows objectAtIndex:i]]];
+ csvRowsThisQuery++;
+ if ([query length] > 250000) break;
+ }
+
+ // Perform the query
+ [mySQLConnection queryString:query];
+ [query release];
+
+ // If an error occurred, run the queries individually to get exact line errors
+ if (![[mySQLConnection getLastErrorMessage] isEqualToString:@""]) {
+ for (i = 0; i < csvRowsThisQuery; i++) {
+ query = [[NSMutableString alloc] initWithString:insertBaseString];
+ [query appendString:[self mappedValueStringForRowArray:[parsedRows objectAtIndex:i]]];
+ [mySQLConnection queryString:query];
+ [query release];
+ if ( ![[mySQLConnection getLastErrorMessage] isEqualToString:@""] ) {
+ [errors appendString:[NSString stringWithFormat:
+ NSLocalizedString(@"[ERROR in row %d] %@\n", @"error text when reading of csv file gave errors"),
+ (rowsImported+1),[mySQLConnection getLastErrorMessage]]];
+ }
+ rowsImported++;
+ [singleProgressBar setDoubleValue:[[parsePositions objectAtIndex:i] doubleValue]];
+ [singleProgressText setStringValue:[NSString stringWithFormat:NSLocalizedString(@"Imported %@ of %@", @"SQL import progress text"),
+ [NSString stringForByteSize:[[parsePositions objectAtIndex:i] longValue]], [NSString stringForByteSize:fileTotalLength]]];
}
+ } else {
+ rowsImported += csvRowsThisQuery;
+ [singleProgressBar setDoubleValue:[[parsePositions objectAtIndex:csvRowsThisQuery-1] doubleValue]];
+ [singleProgressText setStringValue:[NSString stringWithFormat:NSLocalizedString(@"Imported %@ of %@", @"SQL import progress text"),
+ [NSString stringForByteSize:[[parsePositions objectAtIndex:csvRowsThisQuery-1] longValue]], [NSString stringForByteSize:fileTotalLength]]];
}
+
+ // Update the arrays
+ [parsedRows removeObjectsInRange:NSMakeRange(0, csvRowsThisQuery)];
+ [parsePositions removeObjectsInRange:NSMakeRange(0, csvRowsThisQuery)];
}
-
- //close progress sheet
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
}
- [tableContentInstance loadTableValues];
+ // If all the data has been read, break out of the processing loop
+ if (allDataRead) break;
+
+ // Reset the autorelease pool
+ [importPool drain];
+ importPool = [[NSAutoreleasePool alloc] init];
}
-
- //display errors
- if ( [errors length] ) {
+
+ // Clean up
+ [csvParser release];
+ [csvDataBuffer release];
+ [parsedRows release];
+ [parsePositions release];
+ if (fieldMappingArray) [fieldMappingArray release], fieldMappingArray = nil;
+ [importPool drain];
+ [tableDocumentInstance setQueryMode:SP_QUERYMODE_INTERFACE];
+
+ // Close progress sheet
+ [self closeAndStopProgressSheet];
+
+ // Display any errors
+ if ([errors length]) {
[errorsView setString:errors];
- [NSApp beginSheet:errorsSheet
- modalForWindow:tableWindow
- modalDelegate:self
- didEndSelector:nil
- contextInfo:nil];
-
- [NSApp runModalForWindow:errorsSheet];
+ [NSApp beginSheet:errorsSheet modalForWindow:tableWindow modalDelegate:self didEndSelector:nil contextInfo:nil];
+ [NSApp runModalForWindow:errorsSheet];
[NSApp endSheet:errorsSheet];
[errorsSheet orderOut:nil];
}
- //free arrays
- if (fieldMappingArray) [fieldMappingArray release], fieldMappingArray = nil;
- [importArray release], importArray = nil;
-
- [tableDocumentInstance setQueryMode:SP_QUERYMODE_INTERFACE];
-
// Import finished Growl notification
[[SPGrowlController sharedGrowlController] notifyWithTitle:@"Import Finished"
description:[NSString stringWithFormat:NSLocalizedString(@"Finished importing %@",@"description for finished importing growl notification"), [filename lastPathComponent]]
notificationName:@"Import Finished"];
+
+ // Update the content view
+ [tableContentInstance reloadTable:self];
}
- (void)openPanelDidEnd:(NSOpenPanel *)sheet returnCode:(int)returnCode contextInfo:(NSString *)contextInfo
@@ -954,6 +995,112 @@
[NSThread detachNewThreadSelector:@selector(importBackgroundProcess:) toTarget:self withObject:filename];
}
/*
+ * Sets up the field mapping array, and asks the user to provide a field mapping to an
+ * appropriate table; on success, constructs the field mapping array into the global variable,
+ * and returns true. On failure, displays error messages itself, and returns false.
+ * Takes an array of data to show when selecting the field mapping, and an indicator of whether
+ * that dataset is complete or a preview of the full data set.
+ */
+- (BOOL) buildFieldMappingArrayWithData:(NSArray *)importData isPreview:(BOOL)dataIsPreviewData
+{
+
+ // Ensure data was provided, or alert than an import error occurred and return false.
+ if (![importData count]) {
+ NSBeginAlertSheet(NSLocalizedString(@"Error", @"Error"),
+ NSLocalizedString(@"OK", @"OK button"),
+ nil, nil,
+ tableWindow, self,
+ nil, nil, nil,
+ NSLocalizedString(@"Could not parse file as CSV", @"Error when we can't parse/split file as CSV")
+ );
+ return FALSE;
+ }
+
+ // Sanity check the first row of the CSV to prevent hang loops caused by wrong line ending entry
+ if ([[importData objectAtIndex:0] count] > 512) {
+ NSBeginAlertSheet(NSLocalizedString(@"Error", @"error"),
+ NSLocalizedString(@"OK", @"OK button"),
+ nil, nil,
+ tableWindow, self,
+ nil, nil, nil,
+ NSLocalizedString(@"The CSV was read as containing more than 512 columns, more than the maximum columns permitted for speed reasons by Sequel Pro.\n\nThis usually happens due to errors reading the CSV; please double-check the CSV to be imported and the line endings and escape characters at the bottom of the CSV selection dialog.", @"Error when CSV appears to have too many columns to import, probably due to line ending mismatch")
+ );
+ return FALSE;
+ }
+ fieldMappingImportArrayIsPreview = dataIsPreviewData;
+
+ // Get the list of tables to display in the field mapping interface
+ MCPResult *theResult;
+ int i;
+ theResult = (MCPResult *) [mySQLConnection listTables];
+ if ([theResult numOfRows]) [theResult dataSeek:0];
+ [fieldMappingPopup removeAllItems];
+ for ( i = 0 ; i < [theResult numOfRows] ; i++ ) {
+ [fieldMappingPopup addItemWithTitle:NSArrayObjectAtIndex([theResult fetchRowAsArray], 0)];
+ }
+
+ // Select either the currently selected table, or the first item in the list
+ if ([tableDocumentInstance table] != nil && ![[tablesListInstance tableName] isEqualToString:@""]) {
+ [fieldMappingPopup selectItemWithTitle:[tablesListInstance tableName]];
+ } else {
+ [fieldMappingPopup selectItemAtIndex:0];
+ }
+
+ // Error if table selection occurred - usually because there's no tables to select.
+ if( ![tablesListInstance selectTableOrViewWithName:[fieldMappingPopup titleOfSelectedItem]] ) {
+ NSBeginAlertSheet(NSLocalizedString(@"Error", @"error"),
+ NSLocalizedString(@"OK", @"OK button"),
+ nil, nil,
+ tableWindow, self,
+ nil, nil, nil,
+ NSLocalizedString(@"Can't import CSV data into a database without any tables!", @"error text when trying to import csv data, but we have no tables in the db")
+ );
+ return FALSE;
+ }
+
+ // Set the import array
+ if (fieldMappingImportArray) [fieldMappingImportArray release];
+ fieldMappingImportArray = [[NSArray alloc] initWithArray:importData];
+
+ // Set up tableView
+ fieldMappingCurrentRow = 0;
+
+ if (fieldMappingArray) [fieldMappingArray release], fieldMappingArray = nil;
+ [self setupFieldMappingArray];
+ [rowDownButton setEnabled:NO];
+ [rowUpButton setEnabled:([fieldMappingImportArray count] > 1)];
+ [recordCountLabel setStringValue:[NSString stringWithFormat:@"%i of %@%i records", fieldMappingCurrentRow+1, fieldMappingImportArrayIsPreview?@"first ":@"", [fieldMappingImportArray count]]];
+
+ // Set up tableView buttons
+ NSPopUpButtonCell *buttonCell = [[NSPopUpButtonCell alloc] init];
+ [buttonCell setControlSize:NSSmallControlSize];
+ [buttonCell setFont:[NSFont labelFontOfSize:[NSFont smallSystemFontSize]]];
+ [buttonCell setBordered:NO];
+ [[fieldMappingTableView tableColumnWithIdentifier:@"value"] setDataCell:buttonCell];
+ [self updateFieldMappingButtonCell];
+ [fieldMappingTableView reloadData];
+ [buttonCell release];
+
+ // Show fieldMapping sheet
+ [NSApp beginSheet:fieldMappingSheet
+ modalForWindow:tableWindow
+ modalDelegate:self
+ didEndSelector:nil
+ contextInfo:nil];
+
+ int code = [NSApp runModalForWindow:fieldMappingSheet];
+ [NSApp endSheet:fieldMappingSheet];
+ [fieldMappingSheet orderOut:nil];
+
+ // Return success or failure based on confirmation or cancellation
+ if (code) {
+ return TRUE;
+ } else {
+ return FALSE;
+ }
+}
+
+/*
* Sets up the fieldMapping array to be shown in the tableView
*/
- (void)setupFieldMappingArray
@@ -964,7 +1111,7 @@
fieldMappingArray = [[NSMutableArray alloc] init];
for (i = 0; i < [[tableSourceInstance fieldNames] count]; i++) {
- if (i < [NSArrayObjectAtIndex(importArray, currentRow) count] && ![NSArrayObjectAtIndex(NSArrayObjectAtIndex(importArray, currentRow), i) isKindOfClass:[NSNull class]]) {
+ if (i < [NSArrayObjectAtIndex(fieldMappingImportArray, fieldMappingCurrentRow) count] && ![NSArrayObjectAtIndex(NSArrayObjectAtIndex(fieldMappingImportArray, fieldMappingCurrentRow), i) isKindOfClass:[NSNull class]]) {
value = i + 1;
} else {
value = 0;
@@ -984,7 +1131,7 @@
{
int i;
- [fieldMappingButtonOptions setArray:[importArray objectAtIndex:currentRow]];
+ [fieldMappingButtonOptions setArray:[fieldMappingImportArray objectAtIndex:fieldMappingCurrentRow]];
for (i = 0; i < [fieldMappingButtonOptions count]; i++) {
if ([[fieldMappingButtonOptions objectAtIndex:i] isNSNull]) {
[fieldMappingButtonOptions replaceObjectAtIndex:i withObject:[NSString stringWithFormat:@"%i. %@", i+1, [prefs objectForKey:@"NullValue"]]];
@@ -1000,20 +1147,56 @@
*/
{
if ( [sender tag] == 0 ) {
- currentRow--;
+ fieldMappingCurrentRow--;
} else {
- currentRow++;
+ fieldMappingCurrentRow++;
}
[self updateFieldMappingButtonCell];
//-----------[self setupFieldMappingArray];
[fieldMappingTableView reloadData];
- [recordCountLabel setStringValue:[NSString stringWithFormat:@"%i of %i records", currentRow+1, [importArray count]]];
+ [recordCountLabel setStringValue:[NSString stringWithFormat:@"%i of %@%i records", fieldMappingCurrentRow+1, fieldMappingImportArrayIsPreview?@"first ":@"", [fieldMappingImportArray count]]];
// enable/disable buttons
- [rowDownButton setEnabled:(currentRow != 0)];
- [rowUpButton setEnabled:(currentRow != ([importArray count]-1))];
+ [rowDownButton setEnabled:(fieldMappingCurrentRow != 0)];
+ [rowUpButton setEnabled:(fieldMappingCurrentRow != ([fieldMappingImportArray count]-1))];
+}
+
+/*
+ * Construct the VALUES string for a CSV row, based on the field mapping array - including
+ * surrounding brackets but not including the VALUES keyword.
+ */
+- (NSString *) mappedValueStringForRowArray:(NSArray *)csvRowArray
+{
+ NSMutableString *valueString = [NSMutableString stringWithString:@"("];
+ int i;
+ int mapColumn;
+ id cellData;
+ int mappingArrayCount = [fieldMappingArray count];
+
+ for (i = 0; i < mappingArrayCount; i++) {
+ mapColumn = [NSArrayObjectAtIndex(fieldMappingArray, i) intValue];
+
+ // Skip unmapped columns
+ if (!mapColumn) continue;
+
+ if ([valueString length] > 1) [valueString appendString:@","];
+
+ // Append the data
+ cellData = NSArrayObjectAtIndex(csvRowArray, mapColumn - 1);
+
+ if (cellData == [NSNull null]) {
+ [valueString appendString:@"NULL"];
+ } else {
+ [valueString appendString:@"'"];
+ [valueString appendString:[mySQLConnection prepareString:cellData]];
+ [valueString appendString:@"'"];
+ }
+ }
+
+ [valueString appendString:@")"];
+ return valueString;
}
#pragma mark -
@@ -1343,11 +1526,7 @@
[previousConnectionEncoding release];
// Close the progress sheet
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
-
- // Restore the default maximum of the progress bar
- [singleProgressBar setMaxValue:100];
+ [self closeAndStopProgressSheet];
// Show errors sheet if there have been errors
if ( [errors length] ) {
@@ -1505,10 +1684,9 @@
// Close the progress sheet
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
-
-
+ [self closeAndStopProgressSheet];
+
+
return TRUE;
}
@@ -1754,245 +1932,16 @@
// Close the progress sheet if it's present
if ( !silently ) {
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
- }
-
- // Restore the progress bar to a normal maximum
- [singleProgressBar setMaxValue:100];
-
- return TRUE;
-}
-
-
-- (NSArray *)arrayForCSV:(NSString *)csv terminatedBy:(NSString *)fieldEndString
- enclosedBy:(NSString *)fieldQuoteString escapedBy:(NSString *)escapeString lineEnds:(NSString *)lineEndString
-/*
- loads a csv string into an array
- */
-{
- NSMutableString *tempInputString = [NSMutableString string];
- NSMutableArray *tempArray = [NSMutableArray array];
- NSMutableArray *tempRowArray = [NSMutableArray array];
- NSMutableString *mutableField = [NSMutableString string];
- NSScanner *scanner;
- NSString *scanString, *stringToLineEnd, *stringToFieldEnd;
- NSString *escapedFieldEndString, *escapedFieldQuoteString, *escapedEscapeString, *escapedLineEndString;
- NSString *nullString = [prefs objectForKey:@"NullValue"];
- NSCharacterSet *whitespaceSet = nil;
- BOOL isEscaped, escapeStringIsFieldQuoteString, processingLine, processingField, fieldWasQuoted;
- int fieldCount = NSNotFound;
- int i,j,csvLength,fieldEndLength,fieldQuoteLength,escapeLength,lineEndLength,skipLength;
-
- // Fix tabs and line endings in the inputs
- for (i = 0; i < 4; i++) {
- switch (i) {
- case 0: [tempInputString setString:fieldEndString]; break;
- case 1: [tempInputString setString:fieldQuoteString]; break;
- case 2: [tempInputString setString:escapeString]; break;
- case 3: [tempInputString setString:lineEndString]; break;
- }
- [tempInputString replaceOccurrencesOfString:@"\\t" withString:@"\t"
- options:NSLiteralSearch
- range:NSMakeRange(0, [tempInputString length])];
- [tempInputString replaceOccurrencesOfString:@"\\n" withString:@"\n"
- options:NSLiteralSearch
- range:NSMakeRange(0, [tempInputString length])];
- [tempInputString replaceOccurrencesOfString:@"\\r" withString:@"\r"
- options:NSLiteralSearch
- range:NSMakeRange(0, [tempInputString length])];
- switch (i) {
- case 0: fieldEndString = [NSString stringWithString:tempInputString]; break;
- case 1: fieldQuoteString = [NSString stringWithString:tempInputString]; break;
- case 2: escapeString = [NSString stringWithString:tempInputString]; break;
- case 3: lineEndString = [NSString stringWithString:tempInputString]; break;
- }
- }
- fieldEndLength = [fieldEndString length];
- fieldQuoteLength = [fieldQuoteString length];
- escapeLength = [escapeString length];
- lineEndLength = [lineEndString length];
- csvLength = [csv length];
- escapeStringIsFieldQuoteString = [fieldQuoteString isEqualToString:escapeString];
- escapedFieldEndString = [NSString stringWithFormat:@"%@%@", escapeString, fieldEndString];
- escapedFieldQuoteString = [NSString stringWithFormat:@"%@%@", escapeString, fieldQuoteString];
- escapedEscapeString = [NSString stringWithFormat:@"%@%@", escapeString, escapeString];
- escapedLineEndString = [NSString stringWithFormat:@"%@%@", escapeString, lineEndString];
-
- // Set up characters it should be possible to trim
- [tempInputString setString:@""];
- if (![fieldEndString isEqualToString:@" "] && ![fieldQuoteString isEqualToString:@" "] && ![escapeString isEqualToString:@" "] && ![lineEndString isEqualToString:@" "])
- [tempInputString appendString:@" "];
- if (![fieldEndString isEqualToString:@"\t"] && ![fieldQuoteString isEqualToString:@"\t"] && ![escapeString isEqualToString:@"\t"] && ![lineEndString isEqualToString:@"\t"])
- [tempInputString appendString:@"\t"];
- if ([tempInputString length]) whitespaceSet = [NSCharacterSet characterSetWithCharactersInString:tempInputString];
-
- // Set up the scanner to process the CSV
- scanner = [[NSScanner alloc] initWithString:csv];
- [scanner setCharactersToBeSkipped:nil];
-
- while ( ![scanner isAtEnd] && !progressCancelled) {
-
- // Scan the string line by line into an array for each row.
- processingLine = YES;
- [tempRowArray removeAllObjects];
- while (![scanner isAtEnd] && processingLine) {
- [mutableField setString:@""];
- processingField = YES;
- fieldWasQuoted = NO;
-
- // Skip unescaped, unquoted whitespace where possible
- if (whitespaceSet) [scanner scanCharactersFromSet:whitespaceSet intoString:nil];
-
- i = [scanner scanLocation];
-
- // Look at the next section of the string, and determine whether it's enclosed in the field quote string
- if (fieldQuoteLength && i + fieldQuoteLength <= csvLength
- && [[csv substringWithRange:NSMakeRange(i, fieldQuoteLength)] isEqualToString:fieldQuoteString])
- {
- [scanner setScanLocation:i+fieldQuoteLength];
- fieldWasQuoted = YES;
-
- while (![scanner isAtEnd] && processingField) {
-
- // Process the field until the next quote string
- if (![scanner scanUpToString:fieldQuoteString intoString:&scanString]) scanString = @"";
- [mutableField appendString:scanString];
-
- // Check to see if the quote string encountered was escaped... or an escaper
- if (escapeLength) {
- j = 1;
- isEscaped = NO;
- if (!escapeStringIsFieldQuoteString) {
- while (j * escapeLength <= [scanString length]
- && ([[mutableField substringWithRange:NSMakeRange(([mutableField length] - (j*escapeLength)), escapeLength)] isEqualToString:escapeString]))
- {
- isEscaped = !isEscaped;
- j++;
- }
- skipLength = fieldQuoteLength;
- } else {
- if ([scanner scanLocation] + (2 * fieldQuoteLength) <= csvLength
- && [[csv substringWithRange:NSMakeRange([scanner scanLocation] + fieldQuoteLength, fieldQuoteLength)] isEqualToString:fieldQuoteString])
- {
- isEscaped = YES;
- skipLength = 2 * fieldQuoteLength;
- }
- }
-
- // If it was escaped, keep processing the field
- if (isEscaped) {
- if (![scanner isAtEnd]) {
- [mutableField appendString:[csv substringWithRange:NSMakeRange([scanner scanLocation], skipLength)]];
- [scanner setScanLocation:[scanner scanLocation] + skipLength];
- }
- continue;
- }
- }
-
- // We should now be at the end of the field - but let the code below keep going until
- // the field end character is actually reached.
- if (![scanner isAtEnd]) {
- [scanner setScanLocation:[scanner scanLocation] + fieldQuoteLength];
- if (whitespaceSet) [scanner scanCharactersFromSet:whitespaceSet intoString:nil];
- }
- processingField = NO;
- }
- }
-
- // Process until the next field end string *or* line end string, ugh!
- processingField = YES;
- while (![scanner isAtEnd] && processingField) {
- i = [scanner scanLocation];
- if (![scanner scanUpToString:lineEndString intoString:&stringToLineEnd]) stringToLineEnd = @"";
- [scanner setScanLocation:i];
- if (![scanner scanUpToString:fieldEndString intoString:&stringToFieldEnd]) stringToFieldEnd = @"";
- if ([stringToFieldEnd length] < [stringToLineEnd length]) {
- scanString = stringToFieldEnd;
- skipLength = fieldEndLength;
- } else {
- [scanner setScanLocation:i + [stringToLineEnd length]];
- scanString = stringToLineEnd;
- processingLine = NO;
- skipLength = lineEndLength;
- }
- [mutableField appendString:scanString];
-
- // Check to see if the termination character was escaped
- if (escapeLength) {
- j = 1;
- isEscaped = NO;
- while (j * escapeLength <= [scanString length]
- && ([[mutableField substringWithRange:NSMakeRange(([mutableField length] - (j*escapeLength)), escapeLength)] isEqualToString:escapeString]))
- {
- isEscaped = !isEscaped;
- j++;
- }
-
- // If it was, continue processing the field
- if (isEscaped) {
- if (![scanner isAtEnd]) {
- [mutableField appendString:[csv substringWithRange:NSMakeRange([scanner scanLocation], skipLength)]];
- [scanner setScanLocation:[scanner scanLocation] + skipLength];
- }
- continue;
- }
- }
-
- // We should be at the end of the field.
- if (![scanner isAtEnd]) [scanner setScanLocation:[scanner scanLocation] + skipLength];
- processingField = NO;
- }
-
- // We now have a field content string.
- // Insert a NSNull object if the cell contains an unescaped null character or an unescaped string
- // which matches the NULL string set in preferences.
- if ([mutableField isEqualToString:@"\\N"]
- || (!fieldWasQuoted && [mutableField isEqualToString:nullString]))
- {
- [tempRowArray addObject:[NSNull null]];
- } else {
-
- // Clean up escaped characters
- if (escapeLength) {
- if (fieldEndLength)
- [mutableField replaceOccurrencesOfString:escapedFieldEndString withString:fieldEndString options:NSLiteralSearch range:NSMakeRange(0, [mutableField length])];
- if (fieldQuoteLength)
- [mutableField replaceOccurrencesOfString:escapedFieldQuoteString withString:fieldQuoteString options:NSLiteralSearch range:NSMakeRange(0, [mutableField length])];
- if (lineEndLength)
- [mutableField replaceOccurrencesOfString:escapedLineEndString withString:lineEndString options:NSLiteralSearch range:NSMakeRange(0, [mutableField length])];
- [mutableField replaceOccurrencesOfString:escapedEscapeString withString:escapeString options:NSLiteralSearch range:NSMakeRange(0, [mutableField length])];
- }
-
- // Add the field to the row array
- [tempRowArray addObject:[NSString stringWithString:mutableField]];
- }
- }
-
- // Capture the length of the first row and ensure all other rows contain that many items
- if (fieldCount == NSNotFound) {
- fieldCount = [tempRowArray count];
- } else if ([tempRowArray count] < fieldCount) {
-
- // Skip empty rows
- if ([tempRowArray count] == 0
- || ([tempRowArray count] == 1 && ([[tempRowArray objectAtIndex:0] isNSNull] || ![[tempRowArray objectAtIndex:0] length])))
- {
- continue;
- }
-
- for (j = [tempRowArray count]; j < fieldCount; j++) [tempRowArray addObject:[NSNull null]];
- }
+ [self closeAndStopProgressSheet];
+ } else {
- // Add the row to the master output array
- [tempArray addObject:[NSArray arrayWithArray:tempRowArray]];
+ // Restore the progress bar to a normal maximum
+ [singleProgressBar setMaxValue:100];
}
- [scanner release];
- return [NSArray arrayWithArray:tempArray];
+ return TRUE;
}
-
/*
* Takes an array, or streaming result reference, and writes it in XML
* format to the supplied NSFileHandle. For output, also takes a table
@@ -2147,12 +2096,12 @@
// Close the progress sheet if appropriate
if ( !silently ) {
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
- }
+ [self closeAndStopProgressSheet];
+ } else {
- // Restore the progress bar to a normal maximum
- [singleProgressBar setMaxValue:100];
+ // Restore the progress bar to a normal maximum
+ [singleProgressBar setMaxValue:100];
+ }
return TRUE;
}
@@ -2368,8 +2317,7 @@
}
// Close the progress sheet
- [NSApp endSheet:singleProgressSheet];
- [singleProgressSheet orderOut:nil];
+ [self closeAndStopProgressSheet];
// Show the errors sheet if there have been errors
if ( [errors length] ) {
@@ -2484,91 +2432,6 @@
[fieldString release];
return [placeholderSyntax autorelease];
}
-
-/*
- * Split a string by the terminated-character if this is not escaped
- * if enclosed-character is given, ignores characters inside enclosed-characters
- */
-- (NSArray *)arrayForString:(NSString *)string enclosed:(NSString *)enclosed
- escaped:(NSString *)escaped terminated:(NSString *)terminated
-{
- NSMutableArray *tempArray = [NSMutableArray array];
- BOOL inString = NO;
- BOOL isEscaped = NO;
- BOOL br = NO;
- unsigned i, j, start;
- unichar enc;
- unichar esc;
- unichar ter;
-
- //we take only first character by now (too complicated otherwise)
- if ( [enclosed length] ) {
- enc = [enclosed characterAtIndex:0];
- }
- if ( [escaped length] ) {
- esc = [escaped characterAtIndex:0];
- }
- if ( [terminated length] ) {
- ter = [terminated characterAtIndex:0];
- }
-
- start = 0;
-
- for ( i = 0 ; i < [string length] ; i++ ) {
- if ( inString ) {
- //we are in a string
- br = NO;
- while ( !br ) {
- if ( i >= [string length] ) {
- //end of string -> no second enclose character found
- br = YES;
- } else if ( [string characterAtIndex:i] == enc ) {
- //second enclose-character found
- //enclose-character escaped?
- isEscaped = NO;
- j = 1;
- while ( (i-j>0) && ([string characterAtIndex:(i-j)] == esc) ) {
- isEscaped = !isEscaped;
- j++;
- }
- if ( !isEscaped ) {
- inString = NO;
- br = YES;
- }
- }
- if ( !br )
- i++;
- }
- } else if ( [string characterAtIndex:i] == ter ) {
- //terminated-character found
- if ( [enclosed isEqualToString:@""] ) {
- //check if terminated character is escaped
- isEscaped = NO;
- j = 1;
- while ( (i-j>0) && ([string characterAtIndex:(i-j)] == esc) ) {
- isEscaped = !isEscaped;
- j++;
- }
- if ( !isEscaped ) {
- [tempArray addObject:[string substringWithRange:NSMakeRange(start,(i-start))]];
- start = i + 1;
- }
- } else {
- //add object to array
- [tempArray addObject:[string substringWithRange:NSMakeRange(start,(i-start))]];
- start = i + 1;
- }
- } else if ( [string characterAtIndex:i] == enc ) {
- //enclosed-character found
- inString = YES;
- }
- }
-
- //add rest of string to array
- [tempArray addObject:[string substringWithRange:NSMakeRange(start,([string length]-start))]];
-
- return [NSArray arrayWithArray:tempArray];
-}
//additional methods
- (void)setConnection:(MCPConnection *)theConnection
@@ -2747,7 +2610,8 @@ objectValueForTableColumn:(NSTableColumn *)aTableColumn
tables = [[NSMutableArray alloc] init];
fieldMappingButtonOptions = [[NSMutableArray alloc] init];
fieldMappingArray = nil;
- importArray = nil;
+ fieldMappingImportArray = nil;
+ fieldMappingImportArrayIsPreview = NO;
prefs = nil;
return self;
@@ -2757,7 +2621,7 @@ objectValueForTableColumn:(NSTableColumn *)aTableColumn
{
[tables release];
[fieldMappingButtonOptions release];
- if (importArray) [importArray release];
+ if (fieldMappingImportArray) [fieldMappingImportArray release];
if (fieldMappingArray) [fieldMappingArray release];
if (prefs) [prefs release];
diff --git a/sequel-pro.xcodeproj/project.pbxproj b/sequel-pro.xcodeproj/project.pbxproj
index 9b886ce9..8ebc1976 100644
--- a/sequel-pro.xcodeproj/project.pbxproj
+++ b/sequel-pro.xcodeproj/project.pbxproj
@@ -139,6 +139,7 @@
58186D210F4CB38900851FE9 /* ConnectionErrorDialog.xib in Resources */ = {isa = PBXBuildFile; fileRef = 58186D1F0F4CB38900851FE9 /* ConnectionErrorDialog.xib */; };
5822C9B51000DB2400DCC3D6 /* SPConnectionController.m in Sources */ = {isa = PBXBuildFile; fileRef = 5822C9B41000DB2400DCC3D6 /* SPConnectionController.m */; };
5822CAE110011C8000DCC3D6 /* ConnectionView.xib in Resources */ = {isa = PBXBuildFile; fileRef = 5822CADF10011C8000DCC3D6 /* ConnectionView.xib */; };
+ 5822D3091061833C00CE2157 /* SPCSVParser.m in Sources */ = {isa = PBXBuildFile; fileRef = 5822D3081061833C00CE2157 /* SPCSVParser.m */; };
583B77D4103870C800B21F7E /* MCPStreamingResult.m in Sources */ = {isa = PBXBuildFile; fileRef = 583B779810386B0200B21F7E /* MCPStreamingResult.m */; };
5841423F0F97E11000A34B47 /* NoodleLineNumberView.m in Sources */ = {isa = PBXBuildFile; fileRef = 5841423E0F97E11000A34B47 /* NoodleLineNumberView.m */; };
584192A1101E57BB0089807F /* NSMutableArray-MultipleSort.m in Sources */ = {isa = PBXBuildFile; fileRef = 584192A0101E57BB0089807F /* NSMutableArray-MultipleSort.m */; };
@@ -515,6 +516,8 @@
5822C9B31000DB2400DCC3D6 /* SPConnectionController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPConnectionController.h; sourceTree = "<group>"; };
5822C9B41000DB2400DCC3D6 /* SPConnectionController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SPConnectionController.m; sourceTree = "<group>"; };
5822CAE010011C8000DCC3D6 /* English */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = English; path = English.lproj/ConnectionView.xib; sourceTree = "<group>"; };
+ 5822D3071061833C00CE2157 /* SPCSVParser.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPCSVParser.h; sourceTree = "<group>"; };
+ 5822D3081061833C00CE2157 /* SPCSVParser.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SPCSVParser.m; sourceTree = "<group>"; };
583B779710386B0200B21F7E /* MCPStreamingResult.h */ = {isa = PBXFileReference; explicitFileType = sourcecode.c.h; fileEncoding = 4; path = MCPStreamingResult.h; sourceTree = "<group>"; };
583B779810386B0200B21F7E /* MCPStreamingResult.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MCPStreamingResult.m; sourceTree = "<group>"; };
5841423D0F97E11000A34B47 /* NoodleLineNumberView.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NoodleLineNumberView.h; sourceTree = "<group>"; };
@@ -1258,6 +1261,8 @@
children = (
58FEF16B0F23D66600518E8E /* SPSQLParser.h */,
58FEF16C0F23D66600518E8E /* SPSQLParser.m */,
+ 5822D3071061833C00CE2157 /* SPCSVParser.h */,
+ 5822D3081061833C00CE2157 /* SPCSVParser.m */,
179F15040F7C433C00579954 /* SPEditorTokens.h */,
179F15050F7C433C00579954 /* SPEditorTokens.l */,
BCD0AD480FBBFC340066EA5C /* SPSQLTokenizer.l */,
@@ -1709,6 +1714,7 @@
17F5B1541048C50D00FC794F /* SPExporter.m in Sources */,
17F5B39C1049B96A00FC794F /* SPSQLExporter.m in Sources */,
BC29C37F10501EFD00DD6C6E /* SPQueryController.m in Sources */,
+ 5822D3091061833C00CE2157 /* SPCSVParser.m in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};