- Second part of r498: ensure that SQL imports are read in as UTF8, and queries are sent as UTF8, if possible; this correctly preserves encoding data when working with files export from Sequel Pro or via mysqldump, and attempts to fall back to the current connection encodings for other files. This should resolve Issue #116.

3 files changed, 62 insertions, 8 deletions

diff --git a/Source/CMMCPConnection.h b/Source/CMMCPConnection.h
index 70b460f1..eece0d3c 100644
--- a/Source/CMMCPConnection.h
+++ b/Source/CMMCPConnection.h
@@ -64,6 +64,7 @@
 - (void) setParentWindow:(NSWindow *)theWindow;
 - (BOOL) selectDB:(NSString *) dbName;
 - (CMMCPResult *) queryString:(NSString *) query;
+- (CMMCPResult *) queryString:(NSString *) query usingEncoding:(NSStringEncoding) encoding;
 - (float) lastQueryExecutionTime;
 - (MCPResult *) listDBsLike:(NSString *) dbsName;
 - (BOOL) checkConnection;
@@ -74,5 +75,6 @@
 - (void) stopKeepAliveTimer;
 - (void) keepAlive:(NSTimer *)theTimer;
 - (void) threadedKeepAlive;
+- (const char *) cStringFromString:(NSString *) theString usingEncoding:(NSStringEncoding) encoding;
 
 @end
diff --git a/Source/CMMCPConnection.m b/Source/CMMCPConnection.m
index ac75819f..41a828e2 100644
--- a/Source/CMMCPConnection.m
+++ b/Source/CMMCPConnection.m
@@ -345,15 +345,25 @@ static void forcePingTimeout(int signalNumber);
 
 
 /*
+ * Override the standard queryString: method to default to the connection encoding, as before,
+ * before pssing on to queryString: usingEncoding:.
+ */
+- (CMMCPResult *)queryString:(NSString *) query
+{
+	return [self queryString:query usingEncoding:mEncoding];
+}
+
+
+/*
  * Modified version of queryString to be used in Sequel Pro.
  * Error checks extensively - if this method fails, it will ask how to proceed and loop depending
  * on the status, not returning control until either the query has been executed and the result can
  * be returned or the connection and document have been closed.
  */
-- (CMMCPResult *)queryString:(NSString *) query
+- (CMMCPResult *)queryString:(NSString *) query usingEncoding:(NSStringEncoding) encoding
 {
 	CMMCPResult	*theResult;
-	const char	*theCQuery = [self cStringFromString:query];
+	const char	*theCQuery;
 	int			theQueryCode;
 	NSDate		*queryStartDate;
 
@@ -362,6 +372,9 @@ static void forcePingTimeout(int signalNumber);
 
 	[self stopKeepAliveTimer];
 
+	// Generate the cString as appropriate
+	theCQuery = [self cStringFromString:query usingEncoding:encoding];
+
 	// Check the connection.  This triggers reconnects as necessary, and should only return false if a disconnection
 	// has been requested - in which case return nil
 	if (![self checkConnection]) return nil;
@@ -662,4 +675,23 @@ static void forcePingTimeout(int signalNumber)
 	}
 	lastKeepAliveSuccess = [[NSDate alloc] initWithTimeIntervalSinceNow:0];
 }
-@end
+
+
+/*
+ * Modified version of the original to support a supplied encoding.
+ * For internal use only. Transforms a NSString to a C type string (ending with \0).
+ * Lossy conversions are enabled.
+ */
+- (const char *) cStringFromString:(NSString *) theString usingEncoding:(NSStringEncoding) encoding
+{
+	NSMutableData	*theData;
+	
+	if (! theString) {
+		return (const char *)NULL;
+	}
+	
+	theData = [NSMutableData dataWithData:[theString dataUsingEncoding:encoding allowLossyConversion:YES]];
+	[theData increaseLengthBy:1];
+	return (const char *)[theData bytes];
+}
+@end
\ No newline at end of file
diff --git a/Source/TableDump.m b/Source/TableDump.m
index 8dce751f..3f543d4b 100644
--- a/Source/TableDump.m
+++ b/Source/TableDump.m
@@ -375,11 +375,27 @@
 	NSError *errorStr = nil;
 	NSMutableString *errors = [NSMutableString string];
 	NSString *fileType = [[importFormatPopup selectedItem] title];
+	BOOL importSQLAsUTF8 = YES; 
+
+	// Load file into string.  For SQL imports, try UTF8 file encoding before the current encoding.
+	if ([fileType isEqualToString:@"SQL"]) {
+		NSLog(@"Reading as utf8");
+		dumpFile = [SPSQLParser stringWithContentsOfFile:filename
+											 encoding:NSUTF8StringEncoding
+												error:&errorStr];
+												NSLog(dumpFile);
+		if (errorStr) {
+			importSQLAsUTF8 = NO;
+			errorStr = nil;
+		}
+	}
 
-	//load file into string
-	dumpFile = [SPSQLParser stringWithContentsOfFile:filename
-										 encoding:[CMMCPConnection encodingForMySQLEncoding:[[tableDocumentInstance connectionEncoding] UTF8String]]
-											error:&errorStr];
+	// If the SQL-as-UTF8 read failed, and for CSVs, use the current connection encoding.
+	if (!importSQLAsUTF8 || [fileType isEqualToString:@"CSV"]) {
+		dumpFile = [SPSQLParser stringWithContentsOfFile:filename
+											 encoding:[CMMCPConnection encodingForMySQLEncoding:[[tableDocumentInstance connectionEncoding] UTF8String]]
+												error:&errorStr];
+	}
 
 	if (errorStr) {
 		NSBeginAlertSheet(NSLocalizedString(@"Error", @"Title of error alert"),
@@ -435,7 +451,11 @@
 			if ([[[queries objectAtIndex:i] stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] length] == 0)
 				continue;
 
-			[mySQLConnection queryString:[queries objectAtIndex:i]];
+			if (importSQLAsUTF8) {
+				[mySQLConnection queryString:[queries objectAtIndex:i] usingEncoding:NSUTF8StringEncoding];			
+			} else {
+				[mySQLConnection queryString:[queries objectAtIndex:i]];
+			}
 			
 			if (![[mySQLConnection getLastErrorMessage] isEqualToString:@""] && ![[mySQLConnection getLastErrorMessage] isEqualToString:@"Query was empty"]) {
 				[errors appendString:[NSString stringWithFormat:NSLocalizedString(@"[ERROR in query %d] %@\n", @"error text when multiple custom query failed"), (i+1),[mySQLConnection getLastErrorMessage]]];