Rework table content and custom query data loading and storage for speed increases and lower memory usage:

 - Add a new SPMySQLStreamingResultStore class to SPMySQL.framework.  This class acts as both a result set and a data store for the accompanying data, storing the row information in a custom format in a custom malloc zone.
 - Amend SPDataStorage to wrap the new class, so original result information is stored in the one location in the custom format.  Any edited information is handled by SPDataStorage for clean separation
 - Rework table content and custom query data data stores to use the new class.  This significantly speeds up data loading, resulting in faster data loads if they weren't previously network constrained, or lower CPU usage otherwise.  The memory usage is also lowered, with the memory overhead for many small cells being enormously reduced.

1 files changed, 3 insertions, 158 deletions

diff --git a/Frameworks/SPMySQLFramework/Source/SPMySQLResult.m b/Frameworks/SPMySQLFramework/Source/SPMySQLResult.m
index fdc83332..8f51118b 100644
--- a/Frameworks/SPMySQLFramework/Source/SPMySQLResult.m
+++ b/Frameworks/SPMySQLFramework/Source/SPMySQLResult.m
@@ -34,7 +34,6 @@
 #import "SPMySQL Private APIs.h"
 #import "SPMySQLArrayAdditions.h"
 
-static SPMySQLResultFieldProcessor fieldProcessingMap[256];
 static id NSNullPointer;
 
 @implementation SPMySQLResult
@@ -48,45 +47,14 @@ static id NSNullPointer;
 #pragma mark -
 #pragma mark Setup and teardown
 
-/**
- * In the one-off class initialisation, set up the result processing map
- */
 + (void)initialize
 {
 
 	// Cached NSNull singleton reference
 	if (!NSNullPointer) NSNullPointer = [NSNull null];
 
-	// Go through the list of enum_field_types in mysql_com.h, mapping each to the method for
-	// processing that result set.
-	fieldProcessingMap[MYSQL_TYPE_DECIMAL] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_TINY] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_SHORT] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_LONG] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_FLOAT] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_DOUBLE] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_NULL] = SPMySQLResultFieldAsNull;
-	fieldProcessingMap[MYSQL_TYPE_TIMESTAMP] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_LONGLONG] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_INT24] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_DATE] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_TIME] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_DATETIME] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_YEAR] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_NEWDATE] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_VARCHAR] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_BIT] = SPMySQLResultFieldAsBit;
-	fieldProcessingMap[MYSQL_TYPE_NEWDECIMAL] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_ENUM] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_SET] = SPMySQLResultFieldAsString;
-	fieldProcessingMap[MYSQL_TYPE_TINY_BLOB] = SPMySQLResultFieldAsBlob;
-	fieldProcessingMap[MYSQL_TYPE_MEDIUM_BLOB] = SPMySQLResultFieldAsBlob;
-	fieldProcessingMap[MYSQL_TYPE_LONG_BLOB] = SPMySQLResultFieldAsBlob;
-	fieldProcessingMap[MYSQL_TYPE_BLOB] = SPMySQLResultFieldAsBlob;
-	fieldProcessingMap[MYSQL_TYPE_VAR_STRING] = SPMySQLResultFieldAsStringOrBlob;
-	fieldProcessingMap[MYSQL_TYPE_STRING] = SPMySQLResultFieldAsStringOrBlob;
-	fieldProcessingMap[MYSQL_TYPE_GEOMETRY] = SPMySQLResultFieldAsGeometry;
-	fieldProcessingMap[MYSQL_TYPE_DECIMAL] = SPMySQLResultFieldAsString;
+	// Set up data conversion details
+	[self _initializeDataConversion];
 }
 
 /**
@@ -105,7 +73,6 @@ static id NSNullPointer;
 
 		fieldDefinitions = NULL;
 		fieldNames = NULL;
-		fieldTypes = NULL;
 
 		defaultRowReturnType = SPMySQLResultRowAsDictionary;
 	}
@@ -134,11 +101,9 @@ static id NSNullPointer;
 		// Cache the field definitions and build up an array of cached field names and types
 		fieldDefinitions = mysql_fetch_fields(resultSet);
 		fieldNames = malloc(sizeof(NSString *) * numberOfFields);
-		fieldTypes = malloc(sizeof(unsigned int) * numberOfFields);
 		for (NSUInteger i = 0; i < numberOfFields; i++) {
 			MYSQL_FIELD aField = fieldDefinitions[i];
 			fieldNames[i] = [[self _stringWithBytes:aField.name length:aField.name_length] retain];
-			fieldTypes[i] = aField.type;
 		}
 	}
 
@@ -154,7 +119,6 @@ static id NSNullPointer;
 			[fieldNames[i] release];
 		}
 		free(fieldNames);
-		free(fieldTypes);
 	}
 
 	[super dealloc];
@@ -281,7 +245,7 @@ static id NSNullPointer;
 
 	// Convert each of the cells in the row in turn
 	for (NSUInteger i = 0; i < numberOfFields; i++) {
-		id cellData = SPMySQLResultGetObject(self, theRow[i], theRowDataLengths[i], fieldTypes[i], i);
+		id cellData = SPMySQLResultGetObject(self, theRow[i], theRowDataLengths[i], i, NSNotFound);
 
 		// If object creation failed, display a null
 		if (!cellData) cellData = NSNullPointer;
@@ -339,47 +303,6 @@ static id NSNullPointer;
 	return itemsToReturn;
 }
 
-#pragma mark -
-#pragma mark Data conversion
-
-/**
- * Provides a binary representation of the supplied bytes as a returned NSString.
- * The resulting binary representation will be zero-padded according to the supplied
- * field length.
- * MySQL stores bit data as string data stored in an 8-bit wide character set.
- */
-+ (NSString *)bitStringWithBytes:(const char *)bytes length:(NSUInteger)length padToLength:(NSUInteger)padLength
-{
-	NSUInteger i = 0;
-	NSUInteger bitLength = length << 3;
-
-	if (bytes == NULL) {
-		return nil;
-	}
-
-	// Ensure padLength is never lower than the length
-	if (padLength < bitLength) {
-		padLength = bitLength;
-	}
-
-	// Generate a nul-terminated C string representation of the binary data
-	char *cStringBuffer = malloc(padLength + 1);
-	cStringBuffer[padLength] = '\0';
-	while (i < bitLength) {
-		cStringBuffer[padLength - ++i] = ( (bytes[length - 1 - (i >> 3)] >> (i & 0x7)) & 1 ) ? '1' : '0';
-	}
-	while (i++ < padLength) {
-		cStringBuffer[padLength - i] = '0';
-	}
-
-	// Convert to a string
-	NSString *returnString = [NSString stringWithUTF8String:cStringBuffer];
-
-	// Free up memory and return
-	free(cStringBuffer);
-	return returnString;
-}
-
 @end
 
 #pragma mark -
@@ -406,82 +329,4 @@ static id NSNullPointer;
 	queryExecutionTime = theExecutionTime;
 }
 
-/**
- * Core data conversion function, taking C data provided by MySQL and converting
- * to an appropriate return type.
- * Note that the data passed in currently is *not* nul-terminated for fast
- * streaming results, which is safe for the current implementation but should be
- * kept in mind for future changes.
- */
-- (id)_getObjectFromBytes:(char *)bytes ofLength:(NSUInteger)length fieldType:(unsigned int)fieldType fieldDefinitionIndex:(NSUInteger)fieldIndex
-{
-
-	// A NULL pointer for the data indicates a null value; return a NSNull object.
-	if (bytes == NULL) return NSNullPointer;
-
-	// Determine the field processor to use
-	SPMySQLResultFieldProcessor dataProcessor = fieldProcessingMap[fieldType];
-
-	// Switch the method to process the cell data based on the field type mapping.
-	// Do this in two passes: the first as logic may cause a change in processor required.
-	switch (dataProcessor) {
-
-		// STRING and VAR_STRING types may be strings or binary types; check the binary flag
-		case SPMySQLResultFieldAsStringOrBlob:
-			if (fieldDefinitions[fieldIndex].flags & BINARY_FLAG) {
-				dataProcessor = SPMySQLResultFieldAsBlob;
-			}
-			break;
-
-		// Blob types may be automatically be converted to strings, or may be non-binary
-		case SPMySQLResultFieldAsBlob:
-			if (!(fieldDefinitions[fieldIndex].flags & BINARY_FLAG)) {
-				dataProcessor = SPMySQLResultFieldAsString;
-			}
-			break;
-
-		// In most cases, use the original data processor.
-		default:
-			break;
-	}
-
-	// If this instance is set to convert all data as strings, alter the processor.
-	if (returnDataAsStrings && dataProcessor == SPMySQLResultFieldAsBlob) {
-		dataProcessor = SPMySQLResultFieldAsString;
-	}
-
-	// Now switch the processing method again to actually process the data.
-	switch (dataProcessor) {
-
-		// Convert string types using a method that will preserve any nul characters
-		// within the string
-		case SPMySQLResultFieldAsString:
-		case SPMySQLResultFieldAsStringOrBlob:
-			return [[[NSString alloc] initWithBytes:bytes length:length encoding:stringEncoding] autorelease];
-
-		// Convert BLOB types to NSData
-		case SPMySQLResultFieldAsBlob:
-			return [NSData dataWithBytes:bytes length:length];
-		
-		// For Geometry types, use a special Geometry object to handle their complexity
-		case SPMySQLResultFieldAsGeometry:
-			return [SPMySQLGeometryData dataWithBytes:bytes length:length];
-
-		// For bit fields, get a zero-padded representation of the data
-		case SPMySQLResultFieldAsBit:
-			return [SPMySQLResult bitStringWithBytes:bytes length:length padToLength:fieldDefinitions[fieldIndex].length];
-
-		// Convert null types to NSNulls
-		case SPMySQLResultFieldAsNull:
-			return NSNullPointer;
-
-		case SPMySQLResultFieldAsUnhandled:
-			NSLog(@"SPMySQLResult processing encountered an unknown field type (%d), falling back to NSData handling", fieldType);
-			return [NSData dataWithBytes:bytes length:length];
-	}
-
-	[NSException raise:NSInternalInconsistencyException format:@"Unhandled field type when processing SPMySQLResults"];
-	return nil;
-}
-
 @end