- Fix NSLock console lines

 - Speed up table content processing a bit
 - Make the table content download/processing determinate where an approximate row count is available
 - Clean up table content source, assuming MCPStreamingResult will remain in use

6 files changed, 99 insertions, 88 deletions

diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m b/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m
index a61f3223..4413e1c0 100644
--- a/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m
+++ b/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m
@@ -1485,7 +1485,7 @@ static void forcePingTimeout(int signalNumber)
  */
 - (void)unlockConnection
 {
-	[queryLock unlock];
+	[queryLock performSelectorOnMainThread:@selector(unlock) withObject:nil waitUntilDone:YES];
 }
 
 #pragma mark -
diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h
index ae296192..146f81e7 100644
--- a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h
+++ b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h
@@ -29,27 +29,28 @@
 @class MCPConnection;
 
 typedef struct SP_MYSQL_ROWS {
-    char *data;
-    unsigned long *dataLengths;
-    struct SP_MYSQL_ROWS *nextRow;
+	char *data;
+	unsigned long *dataLengths;
+	struct SP_MYSQL_ROWS *nextRow;
 } LOCAL_ROW_DATA;
 
 @interface MCPStreamingResult : MCPResult
 {
-    MCPConnection *parentConnection;
-
-    MYSQL_FIELD *fieldDefinitions;
-    BOOL fullyStreaming;
-    BOOL dataDownloaded;
-    BOOL dataFreed;
-    LOCAL_ROW_DATA *localDataStore;
+	MCPConnection *parentConnection;
+
+	MYSQL_FIELD *fieldDefinitions;
+	BOOL fullyStreaming;
+	BOOL connectionUnlocked;
+	BOOL dataDownloaded;
+	BOOL dataFreed;
+	LOCAL_ROW_DATA *localDataStore;
 	LOCAL_ROW_DATA *currentDataStoreEntry;
-    LOCAL_ROW_DATA *localDataStoreLastEntry;
-    unsigned long localDataRows;
-    unsigned long localDataAllocated;
-    unsigned long downloadedRowCount;
-    unsigned long processedRowCount;
-    unsigned long freedRowCount;
+	LOCAL_ROW_DATA *localDataStoreLastEntry;
+	unsigned long localDataRows;
+	unsigned long localDataAllocated;
+	unsigned long downloadedRowCount;
+	unsigned long processedRowCount;
+	unsigned long freedRowCount;
 }
 
 - (id)initWithMySQLPtr:(MYSQL *)mySQLPtr encoding:(NSStringEncoding)theEncoding timeZone:(NSTimeZone *)theTimeZone connection:(MCPConnection *)theConnection;
diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m
index e51785f8..3f3e2073 100644
--- a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m
+++ b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m
@@ -74,6 +74,7 @@
 		mTimeZone = [theTimeZone retain];
 		parentConnection = theConnection;
 		fullyStreaming = useFullStreaming;
+		connectionUnlocked = NO;
 
 		if (mResult) {
 			mysql_free_result(mResult);
@@ -122,9 +123,9 @@
 /**
  * Deallocate the result and unlock the parent connection for further use
  */
-	- (void) dealloc
+- (void) dealloc
 {
-	[parentConnection unlockConnection];
+	if (!connectionUnlocked) [parentConnection unlockConnection];
 
 	[super dealloc];
 }
@@ -169,6 +170,8 @@
 		// once all memory has been freed
 		if (processedRowCount == downloadedRowCount) {
 			while (!dataFreed) usleep(1000);
+			[parentConnection unlockConnection];
+			connectionUnlocked = YES;
 			return nil;
 		}
 
diff --git a/Source/SPArrayAdditions.h b/Source/SPArrayAdditions.h
index 473740fc..31493f22 100644
--- a/Source/SPArrayAdditions.h
+++ b/Source/SPArrayAdditions.h
@@ -28,6 +28,13 @@ static inline id NSArrayObjectAtIndex(NSArray* self, NSUInteger i) {
 	return (id)CFArrayGetValueAtIndex((CFArrayRef)self, i);
 }
 
+static inline void NSMutableArrayAddObject(NSArray* self, id anObject) {
+	typedef void (*SPMutableArrayAddObjectMethodPtr)(NSArray*, SEL, id);
+	static SPMutableArrayAddObjectMethodPtr SPNSMutableArrayAddObject;
+	if (!SPNSMutableArrayAddObject) SPNSMutableArrayAddObject = (SPMutableArrayAddObjectMethodPtr)[self methodForSelector:@selector(addObject:)];
+	SPNSMutableArrayAddObject(self, @selector(addObject:), anObject);
+}
+
 @interface NSArray (SPArrayAdditions)
 
 - (NSString *)componentsJoinedAndBacktickQuoted;
diff --git a/Source/TableContent.h b/Source/TableContent.h
index 91ffda7e..0ab7c239 100644
--- a/Source/TableContent.h
+++ b/Source/TableContent.h
@@ -103,7 +103,7 @@
 - (void)clickLinkArrow:(SPTextAndLinkCell *)theArrowCell;
 - (IBAction)setCompareTypes:(id)sender;
 - (IBAction)stepLimitRows:(id)sender;
-- (NSArray *)fetchResultAsArray:(MCPResult *)theResult;
+- (void)processResultIntoDataStorage:(MCPStreamingResult *)theResult approximateRowCount:(long)targetRowCount;
 - (BOOL)addRowToDB;
 - (NSString *)argumentForRow:(int)row;
 - (BOOL)tableContainsBlobOrTextColumns;
diff --git a/Source/TableContent.m b/Source/TableContent.m
index 3d737de5..e61bf5ca 100644
--- a/Source/TableContent.m
+++ b/Source/TableContent.m
@@ -402,16 +402,8 @@
 	NSMutableString *queryString;
 	NSString *queryStringBeforeLimit = nil;
 	NSString *filterString;
-	MCPResult *queryResult;
 	MCPStreamingResult *streamingResult;
-	NSArray *tempRow;
-	NSMutableArray *modifiedRow = [NSMutableArray array];
-	NSMutableArray *columnBlobStatuses = [NSMutableArray array];
-	int i;
-	Class nullClass = [NSNull class];
-	id prefsNullValue = [prefs objectForKey:@"NullValue"];
-	BOOL prefsLoadBlobsAsNeeded = [prefs boolForKey:@"LoadBlobsAsNeeded"];
-	long columnsCount = [dataColumns count];
+	int rowsToLoad = [[tableDataInstance statusValueForKey:@"Rows"] intValue];
 	
 	// Notify any listeners that a query has started
 	[[NSNotificationCenter defaultCenter] postNotificationName:@"SMySQLQueryWillBePerformed" object:self];
@@ -451,40 +443,16 @@
 
 		// Append the limit settings
 		[queryString appendFormat:@" LIMIT %d,%d", [limitRowsField intValue]-1, [prefs integerForKey:@"LimitResultsValue"]];
+
+		// Update the approximate count of the rows to load
+		rowsToLoad = rowsToLoad - ([limitRowsField intValue]-1);
+		if (rowsToLoad > [prefs integerForKey:@"LimitResultsValue"]) rowsToLoad = [prefs integerForKey:@"LimitResultsValue"];
 	}
 	
+	// Perform and process the query
 	[self setUsedQuery:queryString];
-	
-	// Run the query and capture the result
-
-	// Build up an array of which columns are blobs for faster iteration
-	for ( i = 0; i < columnsCount ; i++ ) {
-		[columnBlobStatuses addObject:[NSNumber numberWithBool:[tableDataInstance columnIsBlobOrText:[NSArrayObjectAtIndex(dataColumns, i) objectForKey:@"name"] ]]];
-	}
-
-	[tableValues removeAllObjects];
 	streamingResult = [mySQLConnection streamingQueryString:queryString];
-	while (tempRow = [streamingResult fetchNextRowAsArray]) {
-		[modifiedRow removeAllObjects];
-		for ( i = 0; i < columnsCount; i++ ) {
-			if ( [NSArrayObjectAtIndex(tempRow, i) isMemberOfClass:nullClass] ) {
-				[modifiedRow addObject:prefsNullValue];
-			} else {
-				[modifiedRow addObject:NSArrayObjectAtIndex(tempRow, i)];
-			}
-		}
-
-		// Add values for hidden blob and text fields if appropriate
-		if ( prefsLoadBlobsAsNeeded ) {
-			for ( i = 0 ; i < columnsCount ; i++ ) {
-				if ( [NSArrayObjectAtIndex(columnBlobStatuses, i) boolValue] ) {
-					[modifiedRow replaceObjectAtIndex:i withObject:NSLocalizedString(@"(not loaded)", @"value shown for hidden blob and text fields")];
-				}
-			}
-		}
-
-		[tableValues addObject:[NSMutableArray arrayWithArray:modifiedRow]];
-	}
+	[self processResultIntoDataStorage:streamingResult approximateRowCount:rowsToLoad];
 	[streamingResult release];
 
 	// If the result is empty, and a limit is active, reset the limit
@@ -492,8 +460,9 @@
 		[limitRowsField setStringValue:@"1"];
 		queryString = [NSMutableString stringWithFormat:@"%@ LIMIT 0,%d", queryStringBeforeLimit, [prefs integerForKey:@"LimitResultsValue"]];
 		[self setUsedQuery:queryString];
-		queryResult = [mySQLConnection queryString:queryString];
-		[tableValues setArray:[self fetchResultAsArray:queryResult]];
+		streamingResult = [mySQLConnection streamingQueryString:queryString];
+		[self processResultIntoDataStorage:streamingResult approximateRowCount:[prefs integerForKey:@"LimitResultsValue"]];
+		[streamingResult release];
 	}
 
 	if ([prefs boolForKey:@"LimitResults"]
@@ -1225,54 +1194,85 @@
 }
 
 /*
- * Fetches the result as an array, with an array for each row in it
+ * Processes a supplied streaming result set, loading it into the data array.
  */
-- (NSArray *)fetchResultAsArray:(MCPResult *)theResult
+- (void)processResultIntoDataStorage:(MCPStreamingResult *)theResult approximateRowCount:(long)targetRowCount
 {
-	unsigned long numOfRows = [theResult numOfRows];
-	NSMutableArray *tempResult = [NSMutableArray arrayWithCapacity:numOfRows];
-
 	NSArray *tempRow;
-	NSMutableArray *modifiedRow = [NSMutableArray array];
-	NSMutableArray *columnBlobStatuses = [NSMutableArray array];
-	int i, j;
-	Class nullClass = [NSNull class];
-	id prefsNullValue = [prefs objectForKey:@"NullValue"];
-	BOOL prefsLoadBlobsAsNeeded = [prefs boolForKey:@"LoadBlobsAsNeeded"];
-
+	NSMutableArray *newRow;
+	NSMutableArray *columnBlobStatuses = [[NSMutableArray alloc] init];
+	int i;
+	int lastProgressValue = 0;
+	long rowsProcessed = 0;
 	long columnsCount = [dataColumns count];
+	NSAutoreleasePool *dataLoadingPool;
+	NSProgressIndicator *dataLoadingIndicator = [tableDocumentInstance valueForKey:@"queryProgressBar"];
+	id prefsNullValue = [[prefs objectForKey:@"NullValue"] retain];
+	BOOL prefsLoadBlobsAsNeeded = [prefs boolForKey:@"LoadBlobsAsNeeded"];
+	Class nullClass = [NSNull class];
 
 	// Build up an array of which columns are blobs for faster iteration
 	for ( i = 0; i < columnsCount ; i++ ) {
 		[columnBlobStatuses addObject:[NSNumber numberWithBool:[tableDataInstance columnIsBlobOrText:[NSArrayObjectAtIndex(dataColumns, i) objectForKey:@"name"] ]]];
 	}
-	
-	if (numOfRows) [theResult dataSeek:0];
-	for ( i = 0 ; i < numOfRows ; i++ ) {
-		[modifiedRow removeAllObjects];
-		tempRow = [theResult fetchRowAsArray];
 
-		for ( j = 0; j < columnsCount; j++ ) {
-			if ( [NSArrayObjectAtIndex(tempRow, j) isMemberOfClass:nullClass] ) {
-				[modifiedRow addObject:prefsNullValue];
+	// Remove all items from the table and reset the progress indicator
+	[tableValues removeAllObjects];
+	if (targetRowCount) [dataLoadingIndicator setIndeterminate:NO];
+	[dataLoadingIndicator setDoubleValue:0];
+	[dataLoadingIndicator display];
+
+	// Set up an autorelease pool for row processing
+	dataLoadingPool = [[NSAutoreleasePool alloc] init];
+
+	// Loop through the result rows as they become available
+	while (tempRow = [theResult fetchNextRowAsArray]) {
+		NSMutableArrayAddObject(tableValues, [NSMutableArray arrayWithCapacity:columnsCount]);
+		newRow = NSArrayObjectAtIndex(tableValues, rowsProcessed);
+
+		// Process the retrieved row
+		for ( i = 0; i < columnsCount; i++ ) {
+			if ( [NSArrayObjectAtIndex(tempRow, i) isMemberOfClass:nullClass] ) {
+				NSMutableArrayAddObject(newRow, prefsNullValue);
 			} else {
-				[modifiedRow addObject:NSArrayObjectAtIndex(tempRow, j)];
+				NSMutableArrayAddObject(newRow, NSArrayObjectAtIndex(tempRow, i));
 			}
 		}
 
 		// Add values for hidden blob and text fields if appropriate
 		if ( prefsLoadBlobsAsNeeded ) {
-			for ( j = 0 ; j < columnsCount ; j++ ) {
-				if ( [NSArrayObjectAtIndex(columnBlobStatuses, j) boolValue] ) {
-					[modifiedRow replaceObjectAtIndex:j withObject:NSLocalizedString(@"(not loaded)", @"value shown for hidden blob and text fields")];
+			for ( i = 0 ; i < columnsCount ; i++ ) {
+				if ( [NSArrayObjectAtIndex(columnBlobStatuses, i) boolValue] ) {
+					[newRow replaceObjectAtIndex:i withObject:NSLocalizedString(@"(not loaded)", @"value shown for hidden blob and text fields")];
 				}
 			}
 		}
 
-		[tempResult addObject:[NSMutableArray arrayWithArray:modifiedRow]];
-	}
+		// Update the progress bar as necessary, minimising updates
+		rowsProcessed++;
+		if (rowsProcessed == targetRowCount) {
+			[dataLoadingIndicator setIndeterminate:YES];
+		} else if (rowsProcessed < targetRowCount) {
+			[dataLoadingIndicator setDoubleValue:(rowsProcessed*100/targetRowCount)];
+			if ((int)[dataLoadingIndicator doubleValue] > lastProgressValue) {
+				[dataLoadingIndicator display];
+				lastProgressValue = (int)[dataLoadingIndicator doubleValue];
+			}
+		}
 
-	return tempResult;
+		// Drain and reset the autorelease pool every ~1024 rows
+		if (!(rowsProcessed % 1024)) {
+			[dataLoadingPool drain];
+			dataLoadingPool = [[NSAutoreleasePool alloc] init];
+		}
+	}
+	
+	// Clean up the autorelease pool and reset the progress indicator
+	[dataLoadingPool drain];
+	[dataLoadingIndicator setIndeterminate:YES];
+	
+	[columnBlobStatuses release];
+	[prefsNullValue release];
 }