- Change MCPStreamingResult to use a safer streaming mode by default - download all results as fast as possible from the server, to avoid blocking, but do so in a background thread to allow results processing to start as soon as data is available. Many thanks to Hans-Jörg Bibiko for assistance with this.

 - Add an option to the SQL export dialog to allow selection of the full-streaming method, with a warning that it may block table UPDATES/INSERTS.

9 files changed, 483 insertions, 175 deletions

diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.h b/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.h
index 932237d8..5b8aefba 100644
--- a/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.h
+++ b/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.h
@@ -32,6 +32,13 @@
 
 #import "mysql.h"
 
+enum mcp_query_streaming_types
+{
+	MCP_NO_STREAMING = 0,
+	MCP_FAST_STREAMING = 1,
+	MCP_LOWMEM_STREAMING = 2
+};
+
 @class MCPResult, MCPStreamingResult;
 @protocol MCPConnectionProxy;
 
@@ -186,7 +193,8 @@ static inline NSData* NSStringDataUsingLossyEncoding(NSString* self, int encodin
 - (NSString *)quoteObject:(id)theObject;
 - (MCPResult *)queryString:(NSString *)query;
 - (MCPStreamingResult *)streamingQueryString:(NSString *)query;
-- (id)queryString:(NSString *) query usingEncoding:(NSStringEncoding) encoding streamingResult:(BOOL) streamResult;
+- (MCPStreamingResult *)streamingQueryString:(NSString *)query useLowMemoryBlockingStreaming:(BOOL)fullStream;
+- (id)queryString:(NSString *) query usingEncoding:(NSStringEncoding) encoding streamingResult:(int) streamResult;
 - (double)lastQueryExecutionTime;
 - (my_ulonglong)affectedRows;
 - (my_ulonglong)insertId;
diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m b/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m
index b311ac14..a61f3223 100644
--- a/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m
+++ b/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m
@@ -1231,17 +1231,31 @@ static void forcePingTimeout(int signalNumber)
  */
 - (MCPResult *)queryString:(NSString *)query
 {
-	return [self queryString:query usingEncoding:mEncoding streamingResult:NO];
+	return [self queryString:query usingEncoding:mEncoding streamingResult:MCP_NO_STREAMING];
 }
 
 /**
  * Takes a query string and returns an MCPStreamingResult representing the result of the query.
  * The returned MCPStreamingResult is retained and the client is responsible for releasing it.
  * If no fields are present in the result, nil will be returned.
+ * Uses safe/fast mode, which may use more memory as results are downloaded.
  */
 - (MCPStreamingResult *)streamingQueryString:(NSString *)query
 {
-	return [self queryString:query usingEncoding:mEncoding streamingResult:YES];
+	return [self queryString:query usingEncoding:mEncoding streamingResult:MCP_FAST_STREAMING];
+}
+
+/**
+ * Takes a query string and returns an MCPStreamingResult representing the result of the query.
+ * The returned MCPStreamingResult is retained and the client is responsible for releasing it.
+ * If no fields are present in the result, nil will be returned.
+ * Can be used in either fast/safe mode, where data is downloaded as fast as possible to avoid
+ * blocking the server, or in full streaming mode for lowest memory usage but potentially blocking
+ * the table.
+ */
+- (MCPStreamingResult *)streamingQueryString:(NSString *)query useLowMemoryBlockingStreaming:(BOOL)fullStream
+{
+	return [self queryString:query usingEncoding:mEncoding streamingResult:(fullStream?MCP_LOWMEM_STREAMING:MCP_FAST_STREAMING)];
 }
 
 /**
@@ -1250,7 +1264,7 @@ static void forcePingTimeout(int signalNumber)
  * and the result can be returned or the connection and document have been closed.
  * If using streamingResult, the caller is responsible for releasing the result set.
  */
-- (id)queryString:(NSString *) query usingEncoding:(NSStringEncoding) encoding streamingResult:(BOOL) streamResult
+- (id)queryString:(NSString *) query usingEncoding:(NSStringEncoding) encoding streamingResult:(int) streamResultType
 {
 	MCPResult		*theResult = nil;
 	double			queryStartTime, queryExecutionTime;
@@ -1354,13 +1368,15 @@ static void forcePingTimeout(int signalNumber)
 			if (mysql_field_count(mConnection) != 0) {
 				
 				// For normal result sets, fetch the results and unlock the connection
-				if (!streamResult) {
+				if (streamResultType == MCP_NO_STREAMING) {
 					theResult = [[MCPResult alloc] initWithMySQLPtr:mConnection encoding:mEncoding timeZone:mTimeZone];
 					[queryLock unlock];
 				
 				// For streaming result sets, fetch the result pointer and leave the connection locked
-				} else {
-					theResult = [[MCPStreamingResult alloc] initWithMySQLPtr:mConnection encoding:mEncoding timeZone:mTimeZone connection:self];
+				} else if (streamResultType == MCP_FAST_STREAMING) {
+					theResult = [[MCPStreamingResult alloc] initWithMySQLPtr:mConnection encoding:mEncoding timeZone:mTimeZone connection:self withFullStreaming:NO];
+				} else if (streamResultType == MCP_LOWMEM_STREAMING) {
+					theResult = [[MCPStreamingResult alloc] initWithMySQLPtr:mConnection encoding:mEncoding timeZone:mTimeZone connection:self withFullStreaming:YES];
 				}
 				
 				// Ensure no problem occurred during the result fetch
@@ -1375,7 +1391,7 @@ static void forcePingTimeout(int signalNumber)
 			
 			queryErrorMessage = [[NSString alloc] initWithString:@""];
 			queryErrorId = 0;
-			if (!streamResult) {
+			if (streamResultType == MCP_NO_STREAMING) {
 				queryAffectedRows = mysql_affected_rows(mConnection);
 			} else {
 				queryAffectedRows = 0;
@@ -1398,7 +1414,7 @@ static void forcePingTimeout(int signalNumber)
 		break;
 	}
 	
-	if (!streamResult) {
+	if (streamResultType == MCP_NO_STREAMING) {
 		
 		// If the mysql thread id has changed as a result of a connection error,
 		// ensure connection details are still correct
@@ -1423,7 +1439,7 @@ static void forcePingTimeout(int signalNumber)
 	(void)(*startKeepAliveTimerResettingStatePtr)(self, startKeepAliveTimerResettingStateSEL, YES);
 	
 	if (!theResult) return nil;
-	if (streamResult) return theResult;
+	if (streamResultType != MCP_NO_STREAMING) return theResult;
 	return [theResult autorelease];
 }
 
diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPResult.h b/Frameworks/MCPKit/MCPFoundationKit/MCPResult.h
index 1dd7404f..cb66a6cf 100644
--- a/Frameworks/MCPKit/MCPFoundationKit/MCPResult.h
+++ b/Frameworks/MCPKit/MCPFoundationKit/MCPResult.h
@@ -37,7 +37,6 @@
 {
 	MYSQL_RES		 *mResult;		 /* The MYSQL_RES structure of the C API. */
 	NSArray			 *mNames;		 /* An NSArray holding the name of the columns. */
-	NSDictionary	 *mMySQLLocales; /* A Locales dictionary to define the locales of MySQL. */
 	NSStringEncoding mEncoding;		 /* The encoding used by MySQL server, to ISO-1 default. */
 	unsigned int	 mNumOfFields;	 /* The number of fields in the result. */
 	NSTimeZone		 *mTimeZone;	 /* The time zone of the connection when the query was made. */
diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPResult.m b/Frameworks/MCPKit/MCPFoundationKit/MCPResult.m
index 158d01c8..f3edd9a5 100644
--- a/Frameworks/MCPKit/MCPFoundationKit/MCPResult.m
+++ b/Frameworks/MCPKit/MCPFoundationKit/MCPResult.m
@@ -253,11 +253,7 @@ const OUR_CHARSET our_charsets60[] =
 			[mNames release];
 			mNames = NULL;
 		}
-		
-		if (mMySQLLocales == NULL) {
-			mMySQLLocales = [[MCPConnection getMySQLLocales] retain];
-		}
-		
+
 		mNumOfFields = 0;
 	}
 	
@@ -292,10 +288,6 @@ const OUR_CHARSET our_charsets60[] =
 		else {
 			mNumOfFields = 0;
 		}
-		
-		if (mMySQLLocales == NULL) {
-			mMySQLLocales = [[MCPConnection getMySQLLocales] retain];
-		}
 	}
 	
 	return self;
@@ -330,10 +322,6 @@ const OUR_CHARSET our_charsets60[] =
 		else {
 			mNumOfFields = 0;
 		}
-
-		if (mMySQLLocales == NULL) {
-			mMySQLLocales = [[MCPConnection getMySQLLocales] retain];
-		}
 	}
 	
 	return self;    
@@ -1341,10 +1329,6 @@ const OUR_CHARSET our_charsets60[] =
 		[mNames autorelease];
 	}
 	
-	if (mMySQLLocales) {
-		[mMySQLLocales autorelease];
-	}
-	
 	[super dealloc];
 }
 
diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h
index 168f85d3..ae296192 100644
--- a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h
+++ b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h
@@ -1,5 +1,5 @@
 //
-//	$Id$
+//  $Id$
 //
 //  MCPStreamingResult.h
 //  sequel-pro
@@ -28,14 +28,34 @@
 
 @class MCPConnection;
 
+typedef struct SP_MYSQL_ROWS {
+    char *data;
+    unsigned long *dataLengths;
+    struct SP_MYSQL_ROWS *nextRow;
+} LOCAL_ROW_DATA;
+
 @interface MCPStreamingResult : MCPResult
 {
-	MCPConnection *parentConnection;
+    MCPConnection *parentConnection;
 
-	MYSQL_FIELD		*fieldDefinitions;
+    MYSQL_FIELD *fieldDefinitions;
+    BOOL fullyStreaming;
+    BOOL dataDownloaded;
+    BOOL dataFreed;
+    LOCAL_ROW_DATA *localDataStore;
+	LOCAL_ROW_DATA *currentDataStoreEntry;
+    LOCAL_ROW_DATA *localDataStoreLastEntry;
+    unsigned long localDataRows;
+    unsigned long localDataAllocated;
+    unsigned long downloadedRowCount;
+    unsigned long processedRowCount;
+    unsigned long freedRowCount;
 }
 
 - (id)initWithMySQLPtr:(MYSQL *)mySQLPtr encoding:(NSStringEncoding)theEncoding timeZone:(NSTimeZone *)theTimeZone connection:(MCPConnection *)theConnection;
+- (id)initWithMySQLPtr:(MYSQL *)mySQLPtr encoding:(NSStringEncoding)theEncoding timeZone:(NSTimeZone *)theTimeZone connection:(MCPConnection *)theConnection withFullStreaming:(BOOL)useFullStreaming;
+
+// Results fetching
 - (NSArray *)fetchNextRowAsArray;
 
-@end
+@end
\ No newline at end of file
diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m
index 5eb147d3..63353056 100644
--- a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m
+++ b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m
@@ -1,5 +1,5 @@
 //
-//	$Id$
+//  $Id$
 //
 //  MCPStreamingResult.m
 //  sequel-pro
@@ -31,11 +31,24 @@
 /**
  * IMPORTANT NOTE
  *
- * MCPStreamingResult can produce fast and low-memory result reads, but should not
- * be widely used for reads as it can result in MySQL thread or table blocking.
+ * MCPStreamingResult can operate in two modes.  The default mode is a safe implementation,
+ * which operates in a multithreaded fashion - a worker thread is set up to download the results as
+ * fast as possible in the background, while the results are made available via a blocking (and so
+ * single-thread-compatible) fetchNextRowAsArray call.  This provides the benefit of allowing a progress
+ * bar to be shown during downloads, and threaded processing, but still has reasonable memory usage for
+ * the downloaded result - and won't block the server.
+ * Alternatively, withFullStreaming: can be set to YES, in which case each row will be accessed on-demand;
+ * this can be dangerous as it means a SELECT will tie up the server for longer, as for MyISAM tables
+ * updates (and subsequent reads) must block while a SELECT is still running.  However this can be useful
+ * for certain processes such as working with very large tables to keep memory usage low.
  */
 
 
+@interface MCPStreamingResult (PrivateAPI)
+- (void) _downloadAllData;
+- (void) _freeAllDataWhenDone;
+@end
+
 @implementation MCPStreamingResult : MCPResult
 
 #pragma mark -
@@ -47,36 +60,64 @@
  */
 - (id)initWithMySQLPtr:(MYSQL *)mySQLPtr encoding:(NSStringEncoding)theEncoding timeZone:(NSTimeZone *)theTimeZone connection:(MCPConnection *)theConnection
 {
-	if ((self = [super init])) {
-		mEncoding = theEncoding;
-		mTimeZone = [theTimeZone retain];
-		parentConnection = theConnection;
-		
-		if (mResult) {
-			mysql_free_result(mResult);
-			mResult = NULL;
-		}
-		
-		if (mNames) {
-			[mNames release];
-			mNames = NULL;
-		}
-		
-		mResult = mysql_use_result(mySQLPtr);
-		
-		if (mResult) {
-			mNumOfFields = mysql_num_fields(mResult);
-			fieldDefinitions = mysql_fetch_fields(mResult);
-		} else {
-			mNumOfFields = 0;
-		}
-		
-		if (mMySQLLocales == NULL) {
-			mMySQLLocales = [[MCPConnection getMySQLLocales] retain];
-		}
-	}
-	
-	return self;
+    return [self initWithMySQLPtr:mySQLPtr encoding:theEncoding timeZone:theTimeZone connection:theConnection withFullStreaming:NO];
+
+}
+
+/**
+ * Master initialisation method, allowing selection of either full streaming or safe streaming
+ * (see "important note" above)
+ */
+- (id)initWithMySQLPtr:(MYSQL *)mySQLPtr encoding:(NSStringEncoding)theEncoding timeZone:(NSTimeZone *)theTimeZone connection:(MCPConnection *)theConnection withFullStreaming:(BOOL)useFullStreaming
+{
+    if ((self = [super init])) {
+        mEncoding = theEncoding;
+        mTimeZone = [theTimeZone retain];
+        parentConnection = theConnection;
+        fullyStreaming = useFullStreaming;
+        
+        if (mResult) {
+            mysql_free_result(mResult);
+            mResult = NULL;
+        }
+        
+        if (mNames) {
+            [mNames release];
+            mNames = NULL;
+        }
+        
+        mResult = mysql_use_result(mySQLPtr);
+        
+        if (mResult) {
+            mNumOfFields = mysql_num_fields(mResult);
+            fieldDefinitions = mysql_fetch_fields(mResult);
+        } else {
+            mNumOfFields = 0;
+        }
+
+		// If the result is opened in download-data-fast safe mode, set up the additional variables
+		// and threads required.
+        if (!fullyStreaming) {
+			dataDownloaded = NO;
+			dataFreed = NO;
+            localDataStore = NULL;
+			currentDataStoreEntry = NULL;
+            localDataStoreLastEntry = NULL;
+            localDataRows = 0;
+            localDataAllocated = 0;
+            downloadedRowCount = 0;
+            processedRowCount = 0;
+			freedRowCount = 0;
+
+            // Start the data download thread
+            [NSThread detachNewThreadSelector:@selector(_downloadAllData) toTarget:self withObject:nil];
+			
+			// Start the data freeing thread
+			[NSThread detachNewThreadSelector:@selector(_freeAllDataWhenDone) toTarget:self withObject:nil];
+        }
+    }
+    
+    return self;
 }
 
 /**
@@ -84,9 +125,9 @@
  */
 - (void) dealloc
 {
-	[parentConnection unlockConnection];
-	
-	[super dealloc];
+    [parentConnection unlockConnection];
+    
+    [super dealloc];
 }
 
 #pragma mark -
@@ -98,100 +139,149 @@
  */
 - (NSArray *)fetchNextRowAsArray
 {
-	MYSQL_ROW theRow;
-	unsigned long *fieldLengths;
-	int i;
-	NSMutableArray *returnArray;
+    MYSQL_ROW theRow;
+	char *theRowData;
+    unsigned long *fieldLengths;
+    int i, copiedDataLength;
+    NSMutableArray *returnArray;
 
-	// Retrieve the next row
-	theRow = mysql_fetch_row(mResult);
+    // Retrieve the next row according to the mode this result set is in.
+	// If fully streaming, retrieve the MYSQL_ROW
+    if (fullyStreaming) {
+        theRow = mysql_fetch_row(mResult);
 
-	// If no data was returned, we're at the end of the result set - return nil.
-	if (theRow == NULL) return nil;
+        // If no data was returned, we're at the end of the result set - return nil.
+        if (theRow == NULL) return nil;
 
-	// Retrieve the lengths of the returned data
-	fieldLengths = mysql_fetch_lengths(mResult);
+        // Retrieve the lengths of the returned data
+        fieldLengths = mysql_fetch_lengths(mResult);
 
-	// Initialise the array to return
-	returnArray = [NSMutableArray arrayWithCapacity:mNumOfFields];
-	for (i = 0; i < mNumOfFields; i++) {
-		id cellData;
+	// If in cached-streaming/fast download mode, get a reference to the data for the current row
+    } else {
+		copiedDataLength = 0;
+        
+        // Check to see whether we need to wait for the data to be availabe
+		// - if so, wait 1ms before checking again
+        while (!dataDownloaded && processedRowCount == downloadedRowCount) {
+            usleep(1000);
+        }
 
-		// Use NSNulls for the NULL data type
-		if (theRow[i] == NULL) {
-			cellData = [NSNull null];
+        // If all rows have been processed, we're at the end of the result set - return nil
+		// once all memory has been freed
+        if (processedRowCount == downloadedRowCount) {
+			while (!dataFreed) usleep(1000);
+			return nil;
+		}
 
-		// Otherwise, switch by data type
-		} else {
+        // Retrieve a reference to the data and the associated lengths
+        theRowData = currentDataStoreEntry->data;
+        fieldLengths = currentDataStoreEntry->dataLengths;
+    }
 
-			// Create a null-terminated data string for processing
-			char *theData = calloc(sizeof(char), fieldLengths[i]+1);
-			memcpy(theData, theRow[i], fieldLengths[i]);
-			theData[fieldLengths[i]] = '\0';
-
-			switch (fieldDefinitions[i].type) {
-				case FIELD_TYPE_TINY:
-				case FIELD_TYPE_SHORT:
-				case FIELD_TYPE_INT24:
-				case FIELD_TYPE_LONG:
-				case FIELD_TYPE_LONGLONG:
-				case FIELD_TYPE_DECIMAL:
-				case FIELD_TYPE_NEWDECIMAL:
-				case FIELD_TYPE_FLOAT:
-				case FIELD_TYPE_DOUBLE:
-				case FIELD_TYPE_TIMESTAMP:
-				case FIELD_TYPE_DATE:
-				case FIELD_TYPE_TIME:
-				case FIELD_TYPE_DATETIME:
-				case FIELD_TYPE_YEAR:
-				case FIELD_TYPE_VAR_STRING:
-				case FIELD_TYPE_STRING:
-				case FIELD_TYPE_SET:
-				case FIELD_TYPE_ENUM:
-				case FIELD_TYPE_NEWDATE: // Don't know what the format for this type is...
-					cellData = [NSString stringWithCString:theData encoding:mEncoding];
-					break;
-					
-				case FIELD_TYPE_BIT:
-					cellData = [NSString stringWithFormat:@"%u", theData[0]];
-					break;
-					
-				case FIELD_TYPE_TINY_BLOB:
-				case FIELD_TYPE_BLOB:
-				case FIELD_TYPE_MEDIUM_BLOB:
-				case FIELD_TYPE_LONG_BLOB:
-					
-					// For binary data, return the data
-					if (fieldDefinitions[i].flags & BINARY_FLAG) {
-						cellData = [NSData dataWithBytes:theData length:fieldLengths[i]];
-
-					// For string data, convert to text
-					} else {
-						cellData = [[NSString alloc] initWithBytes:theData length:fieldLengths[i] encoding:mEncoding];
-						if (cellData) [cellData autorelease];
-					}
-					break;
-					
-				case FIELD_TYPE_NULL:
-					cellData = [NSNull null];
-					break;
-					
-				default:
-					NSLog(@"in fetchNextRowAsArray : Unknown type : %d for column %d, sending back a NSData object", (int)fieldDefinitions[i].type, (int)i);
-					cellData = [NSData dataWithBytes:theData length:fieldLengths[i]];
-					break;
+    // Initialise the array to return
+    returnArray = [NSMutableArray arrayWithCapacity:mNumOfFields];
+    for (i = 0; i < mNumOfFields; i++) {
+        id cellData = nil;
+		char *theData;
+
+		// In fully streaming mode, copy across the data for the MYSQL_ROW
+		if (fullyStreaming) {
+			if (theRow[i] == NULL) {
+				cellData = [NSNull null];
+			} else {
+				theData = calloc(sizeof(char), fieldLengths[i]+1);
+				memcpy(theData, theRow[i], fieldLengths[i]);
+				theData[fieldLengths[i]] = '\0';
 			}
+		
+		// In cached-streaming mode, use a reference to the downloaded data
+		} else {
+			if (fieldLengths[i] == NSNotFound) {
+				cellData = [NSNull null];
+			} else {
+				theData = theRowData+copiedDataLength;
+				copiedDataLength += fieldLengths[i] + 1;
+			}
+		}
 
-			free(theData);
+        // If the data hasn't already been detected as NULL - in which case it will have been
+		// set to NSNull - process the data by type
+        if (cellData == nil) {
 
-			// If a creator returned a nil object, replace with NSNull
-			if (cellData == nil) cellData = [NSNull null];
-		}
+            switch (fieldDefinitions[i].type) {
+                case FIELD_TYPE_TINY:
+                case FIELD_TYPE_SHORT:
+                case FIELD_TYPE_INT24:
+                case FIELD_TYPE_LONG:
+                case FIELD_TYPE_LONGLONG:
+                case FIELD_TYPE_DECIMAL:
+                case FIELD_TYPE_NEWDECIMAL:
+                case FIELD_TYPE_FLOAT:
+                case FIELD_TYPE_DOUBLE:
+                case FIELD_TYPE_TIMESTAMP:
+                case FIELD_TYPE_DATE:
+                case FIELD_TYPE_TIME:
+                case FIELD_TYPE_DATETIME:
+                case FIELD_TYPE_YEAR:
+                case FIELD_TYPE_VAR_STRING:
+                case FIELD_TYPE_STRING:
+                case FIELD_TYPE_SET:
+                case FIELD_TYPE_ENUM:
+                case FIELD_TYPE_NEWDATE: // Don't know what the format for this type is...
+                    cellData = [NSString stringWithCString:theData encoding:mEncoding];
+                    break;
+                    
+                case FIELD_TYPE_BIT:
+                    cellData = [NSString stringWithFormat:@"%u", theData[0]];
+                    break;
+                    
+                case FIELD_TYPE_TINY_BLOB:
+                case FIELD_TYPE_BLOB:
+                case FIELD_TYPE_MEDIUM_BLOB:
+                case FIELD_TYPE_LONG_BLOB:
+                    
+                    // For binary data, return the data
+                    if (fieldDefinitions[i].flags & BINARY_FLAG) {
+                        cellData = [NSData dataWithBytes:theData length:fieldLengths[i]];
+
+                    // For string data, convert to text
+                    } else {
+                        cellData = [[NSString alloc] initWithBytes:theData length:fieldLengths[i] encoding:mEncoding];
+                        if (cellData) [cellData autorelease];
+                    }
+                    break;
+                    
+                case FIELD_TYPE_NULL:
+                    cellData = [NSNull null];
+                    break;
+                    
+                default:
+                    NSLog(@"in fetchNextRowAsArray : Unknown type : %d for column %d, sending back a NSData object", (int)fieldDefinitions[i].type, (int)i);
+                    cellData = [NSData dataWithBytes:theData length:fieldLengths[i]];
+                    break;
+            }
+
+			// Free the data if it was originally allocated
+			if (fullyStreaming) free(theData);
+
+            // If a creator returned a nil object, replace with NSNull
+            if (cellData == nil) cellData = [NSNull null];
+        }
+
+        [returnArray insertObject:cellData atIndex:i];
+    }
 
-		[returnArray insertObject:cellData atIndex:i];
+	// If in cached-streaming mode, update the current entry processed count
+	if (!fullyStreaming) {
+
+        // Update the active-data pointer to the next item in the list, or set to NULL if no more items
+		currentDataStoreEntry = currentDataStoreEntry->nextRow;
+
+		// Increment counter
+        processedRowCount++;
 	}
 
-	return returnArray;
+    return returnArray;
 }
 
 #pragma mark -
@@ -199,13 +289,117 @@
 
 - (my_ulonglong) numOfRows
 {
-	NSLog(@"numOfRows cannot be used with streaming results");
-	return 0;
+    NSLog(@"numOfRows cannot be used with streaming results");
+    return 0;
 }
 
 - (void) dataSeek:(my_ulonglong) row
 {
-	NSLog(@"dataSeek cannot be used with streaming results");
+    NSLog(@"dataSeek cannot be used with streaming results");
+}
+
+@end
+
+@implementation MCPStreamingResult (PrivateAPI)
+
+/**
+ * Used internally to download results in a background thread
+ */
+- (void) _downloadAllData
+{
+	NSAutoreleasePool *downloadPool = [[NSAutoreleasePool alloc] init];
+    MYSQL_ROW theRow;
+    unsigned long *fieldLengths;
+    int i, dataCopiedLength, rowDataLength;
+    LOCAL_ROW_DATA *newRowStore;
+
+    size_t sizeOfLocalRowData = sizeof(LOCAL_ROW_DATA);
+    size_t sizeOfDataLengths = (size_t)(sizeof(unsigned long) * mNumOfFields);
+
+    // Loop through the rows until the end of the data is reached - indicated via a NULL
+    while (theRow = mysql_fetch_row(mResult)) {
+
+        // Retrieve the lengths of the returned data
+        fieldLengths = mysql_fetch_lengths(mResult);
+		rowDataLength = 0;
+		dataCopiedLength = 0;
+		for (i = 0; i < mNumOfFields; i++)
+			rowDataLength += fieldLengths[i];
+
+        // Initialise memory for the row and set a NULL pointer for the next item
+        newRowStore = malloc(sizeOfLocalRowData);
+        newRowStore->nextRow = NULL;
+
+		// Set up the row data store - a char* - and copy in the data if there is any,
+		// using a null terminator for each field boundary for easier data processing later
+		if (rowDataLength) {
+			newRowStore->data = malloc(sizeof(char) * (rowDataLength + mNumOfFields));
+			for (i = 0; i < mNumOfFields; i++) {
+				if (theRow[i] != NULL) {
+					memcpy(newRowStore->data+dataCopiedLength, theRow[i], fieldLengths[i]);
+					newRowStore->data[dataCopiedLength+fieldLengths[i]] = '\0';
+					dataCopiedLength += fieldLengths[i] + 1;
+				} else {
+					fieldLengths[i] = NSNotFound;
+				}
+			}
+		} else {
+			newRowStore->data = NULL;
+		}
+		
+		// Set up and copy in the field lengths
+		newRowStore->dataLengths = memcpy(malloc(sizeOfDataLengths), fieldLengths, sizeOfDataLengths);
+
+		// Add the newly allocated row to end of the storage linked list
+        if (localDataStore) {
+            localDataStoreLastEntry->nextRow = newRowStore;
+        } else {
+            localDataStore = newRowStore;
+        }
+        localDataStoreLastEntry = newRowStore;
+		if (!currentDataStoreEntry) currentDataStoreEntry = newRowStore;
+
+		// Update the downloaded row count
+        downloadedRowCount++;
+    }
+
+    dataDownloaded = YES;
+	[downloadPool drain];
+}
+
+/**
+ * Used internally to free data which has been fully processed; done in a thread to allow
+ * fetchNextRowAsArray to be faster.
+ */
+- (void) _freeAllDataWhenDone
+{
+	NSAutoreleasePool *dataFreeingPool = [[NSAutoreleasePool alloc] init];
+
+	while (!dataDownloaded || freedRowCount != downloadedRowCount) {
+	
+		// If the freed row count matches the processed row count, wait before retrying
+		if (freedRowCount == processedRowCount) {
+			usleep(1000);
+			continue;
+		}
+
+		// Free a single item off the bottom of the list
+        // Update the data pointer to the next item in the list, or set to NULL if no more items
+        LOCAL_ROW_DATA *rowToRemove = localDataStore;
+		localDataStore = localDataStore->nextRow;
+
+		// Free memory for the first row
+        rowToRemove->nextRow = NULL;
+        free(rowToRemove->dataLengths);
+		if (rowToRemove->data != NULL) free(rowToRemove->data);
+		free(rowToRemove);
+		
+		// Increment the counter
+		freedRowCount++;
+	}
+	
+	dataFreed = YES;
+	[dataFreeingPool drain];
 }
 
 @end
\ No newline at end of file
diff --git a/Interfaces/English.lproj/DBView.xib b/Interfaces/English.lproj/DBView.xib
index 105be269..48775ba0 100644
--- a/Interfaces/English.lproj/DBView.xib
+++ b/Interfaces/English.lproj/DBView.xib
@@ -2,13 +2,12 @@
 <archive type="com.apple.InterfaceBuilder3.Cocoa.XIB" version="7.03">
 	<data>
 		<int key="IBDocument.SystemTarget">1050</int>
-		<string key="IBDocument.SystemVersion">9L30</string>
+		<string key="IBDocument.SystemVersion">9J61</string>
 		<string key="IBDocument.InterfaceBuilderVersion">677</string>
-		<string key="IBDocument.AppKitVersion">949.54</string>
+		<string key="IBDocument.AppKitVersion">949.46</string>
 		<string key="IBDocument.HIToolboxVersion">353.00</string>
 		<object class="NSMutableArray" key="IBDocument.EditedObjectIDs">
 			<bool key="EncodedWithXMLCoder">YES</bool>
-			<integer value="500"/>
 		</object>
 		<object class="NSArray" key="IBDocument.PluginDependencies">
 			<bool key="EncodedWithXMLCoder">YES</bool>
@@ -5016,7 +5015,7 @@
 				<string key="NSWindowContentMaxSize">{600, 142}</string>
 				<string key="NSWindowContentMinSize">{269, 142}</string>
 				<object class="NSView" key="NSWindowView" id="846512394">
-					<reference key="NSNextResponder"/>
+					<nil key="NSNextResponder"/>
 					<int key="NSvFlags">256</int>
 					<object class="NSMutableArray" key="NSSubviews">
 						<bool key="EncodedWithXMLCoder">YES</bool>
@@ -5025,7 +5024,6 @@
 							<int key="NSvFlags">256</int>
 							<string key="NSFrame">{{36, 105}, {71, 14}}</string>
 							<reference key="NSSuperview" ref="846512394"/>
-							<reference key="NSWindow"/>
 							<bool key="NSEnabled">YES</bool>
 							<object class="NSTextFieldCell" key="NSCell" id="967312779">
 								<int key="NSCellFlags">68288064</int>
@@ -5042,7 +5040,6 @@
 							<int key="NSvFlags">256</int>
 							<string key="NSFrame">{{17, 80}, {90, 14}}</string>
 							<reference key="NSSuperview" ref="846512394"/>
-							<reference key="NSWindow"/>
 							<bool key="NSEnabled">YES</bool>
 							<object class="NSTextFieldCell" key="NSCell" id="308586219">
 								<int key="NSCellFlags">68288064</int>
@@ -5059,7 +5056,6 @@
 							<int key="NSvFlags">258</int>
 							<string key="NSFrame">{{112, 104}, {137, 18}}</string>
 							<reference key="NSSuperview" ref="846512394"/>
-							<reference key="NSWindow"/>
 							<bool key="NSEnabled">YES</bool>
 							<object class="NSTextFieldCell" key="NSCell" id="39963071">
 								<int key="NSCellFlags">-1804468671</int>
@@ -5077,7 +5073,6 @@
 							<int key="NSvFlags">257</int>
 							<string key="NSFrame">{{184, 13}, {70, 28}}</string>
 							<reference key="NSSuperview" ref="846512394"/>
-							<reference key="NSWindow"/>
 							<int key="NSTag">1</int>
 							<bool key="NSEnabled">YES</bool>
 							<object class="NSButtonCell" key="NSCell" id="805893132">
@@ -5101,7 +5096,6 @@
 							<int key="NSvFlags">256</int>
 							<string key="NSFrame">{{109, 75}, {143, 22}}</string>
 							<reference key="NSSuperview" ref="846512394"/>
-							<reference key="NSWindow"/>
 							<bool key="NSEnabled">YES</bool>
 							<object class="NSPopUpButtonCell" key="NSCell" id="285868883">
 								<int key="NSCellFlags">-2076049856</int>
@@ -5425,7 +5419,6 @@
 							<int key="NSvFlags">256</int>
 							<string key="NSFrame">{{41, 54}, {66, 14}}</string>
 							<reference key="NSSuperview" ref="846512394"/>
-							<reference key="NSWindow"/>
 							<bool key="NSEnabled">YES</bool>
 							<object class="NSTextFieldCell" key="NSCell" id="193977111">
 								<int key="NSCellFlags">68288064</int>
@@ -5442,7 +5435,6 @@
 							<int key="NSvFlags">256</int>
 							<string key="NSFrame">{{109, 50}, {143, 22}}</string>
 							<reference key="NSSuperview" ref="846512394"/>
-							<reference key="NSWindow"/>
 							<bool key="NSEnabled">YES</bool>
 							<object class="NSPopUpButtonCell" key="NSCell" id="285541065">
 								<int key="NSCellFlags">-2076049856</int>
@@ -5478,7 +5470,6 @@
 							<int key="NSvFlags">257</int>
 							<string key="NSFrame">{{116, 13}, {70, 28}}</string>
 							<reference key="NSSuperview" ref="846512394"/>
-							<reference key="NSWindow"/>
 							<bool key="NSEnabled">YES</bool>
 							<object class="NSButtonCell" key="NSCell" id="783843620">
 								<int key="NSCellFlags">67239424</int>
@@ -5497,8 +5488,6 @@
 						</object>
 					</object>
 					<string key="NSFrameSize">{269, 142}</string>
-					<reference key="NSSuperview"/>
-					<reference key="NSWindow"/>
 				</object>
 				<string key="NSScreenRect">{{0, 0}, {1680, 1028}}</string>
 				<string key="NSMinSize">{269, 164}</string>
@@ -5517,7 +5506,7 @@
 				<string key="NSWindowContentMaxSize">{600, 127}</string>
 				<string key="NSWindowContentMinSize">{260, 127}</string>
 				<object class="NSView" key="NSWindowView" id="653204527">
-					<reference key="NSNextResponder"/>
+					<nil key="NSNextResponder"/>
 					<int key="NSvFlags">256</int>
 					<object class="NSMutableArray" key="NSSubviews">
 						<bool key="EncodedWithXMLCoder">YES</bool>
@@ -5621,7 +5610,6 @@
 						</object>
 					</object>
 					<string key="NSFrameSize">{260, 127}</string>
-					<reference key="NSSuperview"/>
 				</object>
 				<string key="NSScreenRect">{{0, 0}, {1440, 878}}</string>
 				<string key="NSMinSize">{260, 149}</string>
@@ -9459,7 +9447,6 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string>
 								</object>
 								<string key="NSFrame">{{1, 1}, {198, 107}}</string>
 								<reference key="NSSuperview" ref="27548452"/>
-								<reference key="NSNextKeyView" ref="29661959"/>
 								<reference key="NSDocView" ref="29661959"/>
 								<reference key="NSBGColor" ref="1024678221"/>
 								<int key="NScvFlags">4</int>
@@ -9487,7 +9474,6 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string>
 						</object>
 						<string key="NSFrame">{{20, 20}, {211, 109}}</string>
 						<reference key="NSSuperview" ref="774289419"/>
-						<reference key="NSNextKeyView" ref="383807970"/>
 						<int key="NSsFlags">18</int>
 						<reference key="NSVScroller" ref="438736883"/>
 						<reference key="NSHScroller" ref="721548430"/>
@@ -9588,6 +9574,27 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string>
 							<int key="NSPeriodicInterval">25</int>
 						</object>
 					</object>
+					<object class="NSButton" id="380750376">
+						<reference key="NSNextResponder" ref="774289419"/>
+						<int key="NSvFlags">265</int>
+						<string key="NSFrame">{{288, 32}, {151, 18}}</string>
+						<reference key="NSSuperview" ref="774289419"/>
+						<bool key="NSEnabled">YES</bool>
+						<object class="NSButtonCell" key="NSCell" id="190493018">
+							<int key="NSCellFlags">67239424</int>
+							<int key="NSCellFlags2">131072</int>
+							<string key="NSContents">Low-memory export</string>
+							<reference key="NSSupport" ref="26"/>
+							<reference key="NSControlView" ref="380750376"/>
+							<int key="NSButtonFlags">1211912703</int>
+							<int key="NSButtonFlags2">2</int>
+							<reference key="NSAlternateImage" ref="386686735"/>
+							<string key="NSAlternateContents"/>
+							<string key="NSKeyEquivalent"/>
+							<int key="NSPeriodicDelay">200</int>
+							<int key="NSPeriodicInterval">25</int>
+						</object>
+					</object>
 					<object class="NSButton" id="7792134">
 						<reference key="NSNextResponder" ref="774289419"/>
 						<int key="NSvFlags">265</int>
@@ -9651,6 +9658,26 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string>
 							<int key="NSPeriodicInterval">25</int>
 						</object>
 					</object>
+					<object class="NSTextField" id="953478607">
+						<reference key="NSNextResponder" ref="774289419"/>
+						<int key="NSvFlags">268</int>
+						<string key="NSFrame">{{306, 22}, {141, 11}}</string>
+						<reference key="NSSuperview" ref="774289419"/>
+						<bool key="NSEnabled">YES</bool>
+						<object class="NSTextFieldCell" key="NSCell" id="257935364">
+							<int key="NSCellFlags">68288064</int>
+							<int key="NSCellFlags2">272892928</int>
+							<string key="NSContents">May block server during export</string>
+							<object class="NSFont" key="NSSupport">
+								<string key="NSName">LucidaGrande</string>
+								<double key="NSSize">9.000000e+00</double>
+								<int key="NSfFlags">3614</int>
+							</object>
+							<reference key="NSControlView" ref="953478607"/>
+							<reference key="NSBackgroundColor" ref="62854682"/>
+							<reference key="NSTextColor" ref="454249633"/>
+						</object>
+					</object>
 				</object>
 				<string key="NSFrameSize">{457, 149}</string>
 				<string key="NSClassName">NSView</string>
@@ -14956,6 +14983,14 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string>
 					</object>
 					<int key="connectionID">6354</int>
 				</object>
+				<object class="IBConnectionRecord">
+					<object class="IBOutletConnection" key="connection">
+						<string key="label">sqlFullStreamingSwitch</string>
+						<reference key="source" ref="225526897"/>
+						<reference key="destination" ref="380750376"/>
+					</object>
+					<int key="connectionID">6357</int>
+				</object>
 			</object>
 			<object class="IBMutableOrderedSet" key="objectRecords">
 				<object class="NSArray" key="orderedObjects">
@@ -15590,6 +15625,8 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string>
 							<reference ref="7792134"/>
 							<reference ref="178833443"/>
 							<reference ref="339042383"/>
+							<reference ref="380750376"/>
+							<reference ref="953478607"/>
 						</object>
 						<reference key="parent" ref="1043842561"/>
 						<string key="objectName">exportDumpView</string>
@@ -21103,6 +21140,34 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string>
 						<reference key="object" ref="882848749"/>
 						<reference key="parent" ref="1012516136"/>
 					</object>
+					<object class="IBObjectRecord">
+						<int key="objectID">6355</int>
+						<reference key="object" ref="380750376"/>
+						<object class="NSMutableArray" key="children">
+							<bool key="EncodedWithXMLCoder">YES</bool>
+							<reference ref="190493018"/>
+						</object>
+						<reference key="parent" ref="774289419"/>
+					</object>
+					<object class="IBObjectRecord">
+						<int key="objectID">6356</int>
+						<reference key="object" ref="190493018"/>
+						<reference key="parent" ref="380750376"/>
+					</object>
+					<object class="IBObjectRecord">
+						<int key="objectID">6358</int>
+						<reference key="object" ref="953478607"/>
+						<object class="NSMutableArray" key="children">
+							<bool key="EncodedWithXMLCoder">YES</bool>
+							<reference ref="257935364"/>
+						</object>
+						<reference key="parent" ref="774289419"/>
+					</object>
+					<object class="IBObjectRecord">
+						<int key="objectID">6359</int>
+						<reference key="object" ref="257935364"/>
+						<reference key="parent" ref="953478607"/>
+					</object>
 				</object>
 			</object>
 			<object class="NSMutableDictionary" key="flattenedProperties">
@@ -22362,6 +22427,12 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string>
 					<string>6344.ImportedFromIB2</string>
 					<string>6347.IBPluginDependency</string>
 					<string>6348.IBPluginDependency</string>
+					<string>6355.IBAttributePlaceholdersKey</string>
+					<string>6355.IBPluginDependency</string>
+					<string>6355.ImportedFromIB2</string>
+					<string>6356.IBPluginDependency</string>
+					<string>6358.IBPluginDependency</string>
+					<string>6359.IBPluginDependency</string>
 					<string>654.IBPluginDependency</string>
 					<string>654.ImportedFromIB2</string>
 					<string>655.IBPluginDependency</string>
@@ -23402,8 +23473,8 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string>
 					<string>{780, 480}</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<reference ref="9"/>
-					<string>{{697, 485}, {260, 127}}</string>
-					<string>{{697, 485}, {260, 127}}</string>
+					<string>{{628, 485}, {260, 127}}</string>
+					<string>{{628, 485}, {260, 127}}</string>
 					<reference ref="9"/>
 					<reference ref="5"/>
 					<reference ref="9"/>
@@ -24168,6 +24239,19 @@ aGUgYWN0aXZlIHNlbGVjdGlvbiAo4oyl4oyYUik</string>
 					<reference ref="9"/>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+					<object class="NSMutableDictionary">
+						<string key="NS.key.0">ToolTip</string>
+						<object class="IBToolTipAttribute" key="NS.object.0">
+							<string key="name">ToolTip</string>
+							<reference key="object" ref="380750376"/>
+							<string key="toolTip">Use very low memory export, however large the table.  With some table types like MyISAM this may prevent UPDATES/INSERTS until the table is exported.</string>
+						</object>
+					</object>
+					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+					<reference ref="9"/>
+					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<reference ref="9"/>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
@@ -24268,7 +24352,7 @@ Y2hhbmdlIHRoZSBvcmRlcg</string>
 					<reference ref="9"/>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<reference ref="9"/>
-					<string>{{193, 780}, {457, 149}}</string>
+					<string>{{193, 707}, {457, 149}}</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<reference ref="9"/>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
@@ -24388,7 +24472,7 @@ Y2hhbmdlIHRoZSBvcmRlcg</string>
 				</object>
 			</object>
 			<nil key="sourceID"/>
-			<int key="maxID">6354</int>
+			<int key="maxID">6359</int>
 		</object>
 		<object class="IBClassDescriber" key="IBDocument.Classes">
 			<object class="NSMutableArray" key="referencedPartialClassDescriptions">
@@ -25385,6 +25469,7 @@ Y2hhbmdlIHRoZSBvcmRlcg</string>
 							<string>singleProgressSheet</string>
 							<string>singleProgressText</string>
 							<string>singleProgressTitle</string>
+							<string>sqlFullStreamingSwitch</string>
 							<string>tableContentInstance</string>
 							<string>tableDataInstance</string>
 							<string>tableDocumentInstance</string>
@@ -25446,6 +25531,7 @@ Y2hhbmdlIHRoZSBvcmRlcg</string>
 							<string>id</string>
 							<string>id</string>
 							<string>id</string>
+							<string>id</string>
 						</object>
 					</object>
 					<object class="IBClassDescriptionSource" key="sourceIdentifier">
diff --git a/Source/TableDump.h b/Source/TableDump.h
index d32cddea..dc99d4bb 100644
--- a/Source/TableDump.h
+++ b/Source/TableDump.h
@@ -75,6 +75,7 @@
     IBOutlet id addCreateTableSwitch;
     IBOutlet id addTableContentSwitch;
     IBOutlet id addErrorsSwitch;
+    IBOutlet id sqlFullStreamingSwitch;
     IBOutlet id errorsSheet;
     IBOutlet id errorsView;
     IBOutlet id singleProgressSheet;
diff --git a/Source/TableDump.m b/Source/TableDump.m
index 4e53afb9..e07e4c86 100644
--- a/Source/TableDump.m
+++ b/Source/TableDump.m
@@ -1001,7 +1001,7 @@
 			rowCount = [[[[mySQLConnection queryString:[NSString stringWithFormat:@"SELECT COUNT(1) FROM %@", [tableName backtickQuotedString]]] fetchRowAsArray] objectAtIndex:0] intValue];
 			
 			// Set up a result set in streaming mode
-			streamingResult = [mySQLConnection streamingQueryString:[NSString stringWithFormat:@"SELECT * FROM %@", [tableName backtickQuotedString]]];
+			streamingResult = [mySQLConnection streamingQueryString:[NSString stringWithFormat:@"SELECT * FROM %@", [tableName backtickQuotedString]] useLowMemoryBlockingStreaming:([sqlFullStreamingSwitch state] == NSOnState)];
 			fieldNames = [streamingResult fetchFieldNames];			
 			
 			// Update the progress text and set the progress bar back to determinate