diff options
| -rw-r--r-- | Frameworks/MCPKit/MCPFoundationKit/MCPConnection.h | 10 | ||||
| -rw-r--r-- | Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m | 34 | ||||
| -rw-r--r-- | Frameworks/MCPKit/MCPFoundationKit/MCPResult.h | 1 | ||||
| -rw-r--r-- | Frameworks/MCPKit/MCPFoundationKit/MCPResult.m | 18 | ||||
| -rw-r--r-- | Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h | 28 | ||||
| -rw-r--r-- | Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m | 434 | ||||
| -rw-r--r-- | Interfaces/English.lproj/DBView.xib | 130 | ||||
| -rw-r--r-- | Source/TableDump.h | 1 | ||||
| -rw-r--r-- | Source/TableDump.m | 2 |
9 files changed, 483 insertions, 175 deletions
diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.h b/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.h index 932237d8..5b8aefba 100644 --- a/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.h +++ b/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.h @@ -32,6 +32,13 @@ #import "mysql.h" +enum mcp_query_streaming_types +{ + MCP_NO_STREAMING = 0, + MCP_FAST_STREAMING = 1, + MCP_LOWMEM_STREAMING = 2 +}; + @class MCPResult, MCPStreamingResult; @protocol MCPConnectionProxy; @@ -186,7 +193,8 @@ static inline NSData* NSStringDataUsingLossyEncoding(NSString* self, int encodin - (NSString *)quoteObject:(id)theObject; - (MCPResult *)queryString:(NSString *)query; - (MCPStreamingResult *)streamingQueryString:(NSString *)query; -- (id)queryString:(NSString *) query usingEncoding:(NSStringEncoding) encoding streamingResult:(BOOL) streamResult; +- (MCPStreamingResult *)streamingQueryString:(NSString *)query useLowMemoryBlockingStreaming:(BOOL)fullStream; +- (id)queryString:(NSString *) query usingEncoding:(NSStringEncoding) encoding streamingResult:(int) streamResult; - (double)lastQueryExecutionTime; - (my_ulonglong)affectedRows; - (my_ulonglong)insertId; diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m b/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m index b311ac14..a61f3223 100644 --- a/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m +++ b/Frameworks/MCPKit/MCPFoundationKit/MCPConnection.m @@ -1231,17 +1231,31 @@ static void forcePingTimeout(int signalNumber) */ - (MCPResult *)queryString:(NSString *)query { - return [self queryString:query usingEncoding:mEncoding streamingResult:NO]; + return [self queryString:query usingEncoding:mEncoding streamingResult:MCP_NO_STREAMING]; } /** * Takes a query string and returns an MCPStreamingResult representing the result of the query. * The returned MCPStreamingResult is retained and the client is responsible for releasing it. * If no fields are present in the result, nil will be returned. + * Uses safe/fast mode, which may use more memory as results are downloaded. */ - (MCPStreamingResult *)streamingQueryString:(NSString *)query { - return [self queryString:query usingEncoding:mEncoding streamingResult:YES]; + return [self queryString:query usingEncoding:mEncoding streamingResult:MCP_FAST_STREAMING]; +} + +/** + * Takes a query string and returns an MCPStreamingResult representing the result of the query. + * The returned MCPStreamingResult is retained and the client is responsible for releasing it. + * If no fields are present in the result, nil will be returned. + * Can be used in either fast/safe mode, where data is downloaded as fast as possible to avoid + * blocking the server, or in full streaming mode for lowest memory usage but potentially blocking + * the table. + */ +- (MCPStreamingResult *)streamingQueryString:(NSString *)query useLowMemoryBlockingStreaming:(BOOL)fullStream +{ + return [self queryString:query usingEncoding:mEncoding streamingResult:(fullStream?MCP_LOWMEM_STREAMING:MCP_FAST_STREAMING)]; } /** @@ -1250,7 +1264,7 @@ static void forcePingTimeout(int signalNumber) * and the result can be returned or the connection and document have been closed. * If using streamingResult, the caller is responsible for releasing the result set. */ -- (id)queryString:(NSString *) query usingEncoding:(NSStringEncoding) encoding streamingResult:(BOOL) streamResult +- (id)queryString:(NSString *) query usingEncoding:(NSStringEncoding) encoding streamingResult:(int) streamResultType { MCPResult *theResult = nil; double queryStartTime, queryExecutionTime; @@ -1354,13 +1368,15 @@ static void forcePingTimeout(int signalNumber) if (mysql_field_count(mConnection) != 0) { // For normal result sets, fetch the results and unlock the connection - if (!streamResult) { + if (streamResultType == MCP_NO_STREAMING) { theResult = [[MCPResult alloc] initWithMySQLPtr:mConnection encoding:mEncoding timeZone:mTimeZone]; [queryLock unlock]; // For streaming result sets, fetch the result pointer and leave the connection locked - } else { - theResult = [[MCPStreamingResult alloc] initWithMySQLPtr:mConnection encoding:mEncoding timeZone:mTimeZone connection:self]; + } else if (streamResultType == MCP_FAST_STREAMING) { + theResult = [[MCPStreamingResult alloc] initWithMySQLPtr:mConnection encoding:mEncoding timeZone:mTimeZone connection:self withFullStreaming:NO]; + } else if (streamResultType == MCP_LOWMEM_STREAMING) { + theResult = [[MCPStreamingResult alloc] initWithMySQLPtr:mConnection encoding:mEncoding timeZone:mTimeZone connection:self withFullStreaming:YES]; } // Ensure no problem occurred during the result fetch @@ -1375,7 +1391,7 @@ static void forcePingTimeout(int signalNumber) queryErrorMessage = [[NSString alloc] initWithString:@""]; queryErrorId = 0; - if (!streamResult) { + if (streamResultType == MCP_NO_STREAMING) { queryAffectedRows = mysql_affected_rows(mConnection); } else { queryAffectedRows = 0; @@ -1398,7 +1414,7 @@ static void forcePingTimeout(int signalNumber) break; } - if (!streamResult) { + if (streamResultType == MCP_NO_STREAMING) { // If the mysql thread id has changed as a result of a connection error, // ensure connection details are still correct @@ -1423,7 +1439,7 @@ static void forcePingTimeout(int signalNumber) (void)(*startKeepAliveTimerResettingStatePtr)(self, startKeepAliveTimerResettingStateSEL, YES); if (!theResult) return nil; - if (streamResult) return theResult; + if (streamResultType != MCP_NO_STREAMING) return theResult; return [theResult autorelease]; } diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPResult.h b/Frameworks/MCPKit/MCPFoundationKit/MCPResult.h index 1dd7404f..cb66a6cf 100644 --- a/Frameworks/MCPKit/MCPFoundationKit/MCPResult.h +++ b/Frameworks/MCPKit/MCPFoundationKit/MCPResult.h @@ -37,7 +37,6 @@ { MYSQL_RES *mResult; /* The MYSQL_RES structure of the C API. */ NSArray *mNames; /* An NSArray holding the name of the columns. */ - NSDictionary *mMySQLLocales; /* A Locales dictionary to define the locales of MySQL. */ NSStringEncoding mEncoding; /* The encoding used by MySQL server, to ISO-1 default. */ unsigned int mNumOfFields; /* The number of fields in the result. */ NSTimeZone *mTimeZone; /* The time zone of the connection when the query was made. */ diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPResult.m b/Frameworks/MCPKit/MCPFoundationKit/MCPResult.m index 158d01c8..f3edd9a5 100644 --- a/Frameworks/MCPKit/MCPFoundationKit/MCPResult.m +++ b/Frameworks/MCPKit/MCPFoundationKit/MCPResult.m @@ -253,11 +253,7 @@ const OUR_CHARSET our_charsets60[] = [mNames release]; mNames = NULL; } - - if (mMySQLLocales == NULL) { - mMySQLLocales = [[MCPConnection getMySQLLocales] retain]; - } - + mNumOfFields = 0; } @@ -292,10 +288,6 @@ const OUR_CHARSET our_charsets60[] = else { mNumOfFields = 0; } - - if (mMySQLLocales == NULL) { - mMySQLLocales = [[MCPConnection getMySQLLocales] retain]; - } } return self; @@ -330,10 +322,6 @@ const OUR_CHARSET our_charsets60[] = else { mNumOfFields = 0; } - - if (mMySQLLocales == NULL) { - mMySQLLocales = [[MCPConnection getMySQLLocales] retain]; - } } return self; @@ -1341,10 +1329,6 @@ const OUR_CHARSET our_charsets60[] = [mNames autorelease]; } - if (mMySQLLocales) { - [mMySQLLocales autorelease]; - } - [super dealloc]; } diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h index 168f85d3..ae296192 100644 --- a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h +++ b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.h @@ -1,5 +1,5 @@ // -// $Id$ +// $Id$ // // MCPStreamingResult.h // sequel-pro @@ -28,14 +28,34 @@ @class MCPConnection; +typedef struct SP_MYSQL_ROWS { + char *data; + unsigned long *dataLengths; + struct SP_MYSQL_ROWS *nextRow; +} LOCAL_ROW_DATA; + @interface MCPStreamingResult : MCPResult { - MCPConnection *parentConnection; + MCPConnection *parentConnection; - MYSQL_FIELD *fieldDefinitions; + MYSQL_FIELD *fieldDefinitions; + BOOL fullyStreaming; + BOOL dataDownloaded; + BOOL dataFreed; + LOCAL_ROW_DATA *localDataStore; + LOCAL_ROW_DATA *currentDataStoreEntry; + LOCAL_ROW_DATA *localDataStoreLastEntry; + unsigned long localDataRows; + unsigned long localDataAllocated; + unsigned long downloadedRowCount; + unsigned long processedRowCount; + unsigned long freedRowCount; } - (id)initWithMySQLPtr:(MYSQL *)mySQLPtr encoding:(NSStringEncoding)theEncoding timeZone:(NSTimeZone *)theTimeZone connection:(MCPConnection *)theConnection; +- (id)initWithMySQLPtr:(MYSQL *)mySQLPtr encoding:(NSStringEncoding)theEncoding timeZone:(NSTimeZone *)theTimeZone connection:(MCPConnection *)theConnection withFullStreaming:(BOOL)useFullStreaming; + +// Results fetching - (NSArray *)fetchNextRowAsArray; -@end +@end
\ No newline at end of file diff --git a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m index 5eb147d3..63353056 100644 --- a/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m +++ b/Frameworks/MCPKit/MCPFoundationKit/MCPStreamingResult.m @@ -1,5 +1,5 @@ // -// $Id$ +// $Id$ // // MCPStreamingResult.m // sequel-pro @@ -31,11 +31,24 @@ /** * IMPORTANT NOTE * - * MCPStreamingResult can produce fast and low-memory result reads, but should not - * be widely used for reads as it can result in MySQL thread or table blocking. + * MCPStreamingResult can operate in two modes. The default mode is a safe implementation, + * which operates in a multithreaded fashion - a worker thread is set up to download the results as + * fast as possible in the background, while the results are made available via a blocking (and so + * single-thread-compatible) fetchNextRowAsArray call. This provides the benefit of allowing a progress + * bar to be shown during downloads, and threaded processing, but still has reasonable memory usage for + * the downloaded result - and won't block the server. + * Alternatively, withFullStreaming: can be set to YES, in which case each row will be accessed on-demand; + * this can be dangerous as it means a SELECT will tie up the server for longer, as for MyISAM tables + * updates (and subsequent reads) must block while a SELECT is still running. However this can be useful + * for certain processes such as working with very large tables to keep memory usage low. */ +@interface MCPStreamingResult (PrivateAPI) +- (void) _downloadAllData; +- (void) _freeAllDataWhenDone; +@end + @implementation MCPStreamingResult : MCPResult #pragma mark - @@ -47,36 +60,64 @@ */ - (id)initWithMySQLPtr:(MYSQL *)mySQLPtr encoding:(NSStringEncoding)theEncoding timeZone:(NSTimeZone *)theTimeZone connection:(MCPConnection *)theConnection { - if ((self = [super init])) { - mEncoding = theEncoding; - mTimeZone = [theTimeZone retain]; - parentConnection = theConnection; - - if (mResult) { - mysql_free_result(mResult); - mResult = NULL; - } - - if (mNames) { - [mNames release]; - mNames = NULL; - } - - mResult = mysql_use_result(mySQLPtr); - - if (mResult) { - mNumOfFields = mysql_num_fields(mResult); - fieldDefinitions = mysql_fetch_fields(mResult); - } else { - mNumOfFields = 0; - } - - if (mMySQLLocales == NULL) { - mMySQLLocales = [[MCPConnection getMySQLLocales] retain]; - } - } - - return self; + return [self initWithMySQLPtr:mySQLPtr encoding:theEncoding timeZone:theTimeZone connection:theConnection withFullStreaming:NO]; + +} + +/** + * Master initialisation method, allowing selection of either full streaming or safe streaming + * (see "important note" above) + */ +- (id)initWithMySQLPtr:(MYSQL *)mySQLPtr encoding:(NSStringEncoding)theEncoding timeZone:(NSTimeZone *)theTimeZone connection:(MCPConnection *)theConnection withFullStreaming:(BOOL)useFullStreaming +{ + if ((self = [super init])) { + mEncoding = theEncoding; + mTimeZone = [theTimeZone retain]; + parentConnection = theConnection; + fullyStreaming = useFullStreaming; + + if (mResult) { + mysql_free_result(mResult); + mResult = NULL; + } + + if (mNames) { + [mNames release]; + mNames = NULL; + } + + mResult = mysql_use_result(mySQLPtr); + + if (mResult) { + mNumOfFields = mysql_num_fields(mResult); + fieldDefinitions = mysql_fetch_fields(mResult); + } else { + mNumOfFields = 0; + } + + // If the result is opened in download-data-fast safe mode, set up the additional variables + // and threads required. + if (!fullyStreaming) { + dataDownloaded = NO; + dataFreed = NO; + localDataStore = NULL; + currentDataStoreEntry = NULL; + localDataStoreLastEntry = NULL; + localDataRows = 0; + localDataAllocated = 0; + downloadedRowCount = 0; + processedRowCount = 0; + freedRowCount = 0; + + // Start the data download thread + [NSThread detachNewThreadSelector:@selector(_downloadAllData) toTarget:self withObject:nil]; + + // Start the data freeing thread + [NSThread detachNewThreadSelector:@selector(_freeAllDataWhenDone) toTarget:self withObject:nil]; + } + } + + return self; } /** @@ -84,9 +125,9 @@ */ - (void) dealloc { - [parentConnection unlockConnection]; - - [super dealloc]; + [parentConnection unlockConnection]; + + [super dealloc]; } #pragma mark - @@ -98,100 +139,149 @@ */ - (NSArray *)fetchNextRowAsArray { - MYSQL_ROW theRow; - unsigned long *fieldLengths; - int i; - NSMutableArray *returnArray; + MYSQL_ROW theRow; + char *theRowData; + unsigned long *fieldLengths; + int i, copiedDataLength; + NSMutableArray *returnArray; - // Retrieve the next row - theRow = mysql_fetch_row(mResult); + // Retrieve the next row according to the mode this result set is in. + // If fully streaming, retrieve the MYSQL_ROW + if (fullyStreaming) { + theRow = mysql_fetch_row(mResult); - // If no data was returned, we're at the end of the result set - return nil. - if (theRow == NULL) return nil; + // If no data was returned, we're at the end of the result set - return nil. + if (theRow == NULL) return nil; - // Retrieve the lengths of the returned data - fieldLengths = mysql_fetch_lengths(mResult); + // Retrieve the lengths of the returned data + fieldLengths = mysql_fetch_lengths(mResult); - // Initialise the array to return - returnArray = [NSMutableArray arrayWithCapacity:mNumOfFields]; - for (i = 0; i < mNumOfFields; i++) { - id cellData; + // If in cached-streaming/fast download mode, get a reference to the data for the current row + } else { + copiedDataLength = 0; + + // Check to see whether we need to wait for the data to be availabe + // - if so, wait 1ms before checking again + while (!dataDownloaded && processedRowCount == downloadedRowCount) { + usleep(1000); + } - // Use NSNulls for the NULL data type - if (theRow[i] == NULL) { - cellData = [NSNull null]; + // If all rows have been processed, we're at the end of the result set - return nil + // once all memory has been freed + if (processedRowCount == downloadedRowCount) { + while (!dataFreed) usleep(1000); + return nil; + } - // Otherwise, switch by data type - } else { + // Retrieve a reference to the data and the associated lengths + theRowData = currentDataStoreEntry->data; + fieldLengths = currentDataStoreEntry->dataLengths; + } - // Create a null-terminated data string for processing - char *theData = calloc(sizeof(char), fieldLengths[i]+1); - memcpy(theData, theRow[i], fieldLengths[i]); - theData[fieldLengths[i]] = '\0'; - - switch (fieldDefinitions[i].type) { - case FIELD_TYPE_TINY: - case FIELD_TYPE_SHORT: - case FIELD_TYPE_INT24: - case FIELD_TYPE_LONG: - case FIELD_TYPE_LONGLONG: - case FIELD_TYPE_DECIMAL: - case FIELD_TYPE_NEWDECIMAL: - case FIELD_TYPE_FLOAT: - case FIELD_TYPE_DOUBLE: - case FIELD_TYPE_TIMESTAMP: - case FIELD_TYPE_DATE: - case FIELD_TYPE_TIME: - case FIELD_TYPE_DATETIME: - case FIELD_TYPE_YEAR: - case FIELD_TYPE_VAR_STRING: - case FIELD_TYPE_STRING: - case FIELD_TYPE_SET: - case FIELD_TYPE_ENUM: - case FIELD_TYPE_NEWDATE: // Don't know what the format for this type is... - cellData = [NSString stringWithCString:theData encoding:mEncoding]; - break; - - case FIELD_TYPE_BIT: - cellData = [NSString stringWithFormat:@"%u", theData[0]]; - break; - - case FIELD_TYPE_TINY_BLOB: - case FIELD_TYPE_BLOB: - case FIELD_TYPE_MEDIUM_BLOB: - case FIELD_TYPE_LONG_BLOB: - - // For binary data, return the data - if (fieldDefinitions[i].flags & BINARY_FLAG) { - cellData = [NSData dataWithBytes:theData length:fieldLengths[i]]; - - // For string data, convert to text - } else { - cellData = [[NSString alloc] initWithBytes:theData length:fieldLengths[i] encoding:mEncoding]; - if (cellData) [cellData autorelease]; - } - break; - - case FIELD_TYPE_NULL: - cellData = [NSNull null]; - break; - - default: - NSLog(@"in fetchNextRowAsArray : Unknown type : %d for column %d, sending back a NSData object", (int)fieldDefinitions[i].type, (int)i); - cellData = [NSData dataWithBytes:theData length:fieldLengths[i]]; - break; + // Initialise the array to return + returnArray = [NSMutableArray arrayWithCapacity:mNumOfFields]; + for (i = 0; i < mNumOfFields; i++) { + id cellData = nil; + char *theData; + + // In fully streaming mode, copy across the data for the MYSQL_ROW + if (fullyStreaming) { + if (theRow[i] == NULL) { + cellData = [NSNull null]; + } else { + theData = calloc(sizeof(char), fieldLengths[i]+1); + memcpy(theData, theRow[i], fieldLengths[i]); + theData[fieldLengths[i]] = '\0'; } + + // In cached-streaming mode, use a reference to the downloaded data + } else { + if (fieldLengths[i] == NSNotFound) { + cellData = [NSNull null]; + } else { + theData = theRowData+copiedDataLength; + copiedDataLength += fieldLengths[i] + 1; + } + } - free(theData); + // If the data hasn't already been detected as NULL - in which case it will have been + // set to NSNull - process the data by type + if (cellData == nil) { - // If a creator returned a nil object, replace with NSNull - if (cellData == nil) cellData = [NSNull null]; - } + switch (fieldDefinitions[i].type) { + case FIELD_TYPE_TINY: + case FIELD_TYPE_SHORT: + case FIELD_TYPE_INT24: + case FIELD_TYPE_LONG: + case FIELD_TYPE_LONGLONG: + case FIELD_TYPE_DECIMAL: + case FIELD_TYPE_NEWDECIMAL: + case FIELD_TYPE_FLOAT: + case FIELD_TYPE_DOUBLE: + case FIELD_TYPE_TIMESTAMP: + case FIELD_TYPE_DATE: + case FIELD_TYPE_TIME: + case FIELD_TYPE_DATETIME: + case FIELD_TYPE_YEAR: + case FIELD_TYPE_VAR_STRING: + case FIELD_TYPE_STRING: + case FIELD_TYPE_SET: + case FIELD_TYPE_ENUM: + case FIELD_TYPE_NEWDATE: // Don't know what the format for this type is... + cellData = [NSString stringWithCString:theData encoding:mEncoding]; + break; + + case FIELD_TYPE_BIT: + cellData = [NSString stringWithFormat:@"%u", theData[0]]; + break; + + case FIELD_TYPE_TINY_BLOB: + case FIELD_TYPE_BLOB: + case FIELD_TYPE_MEDIUM_BLOB: + case FIELD_TYPE_LONG_BLOB: + + // For binary data, return the data + if (fieldDefinitions[i].flags & BINARY_FLAG) { + cellData = [NSData dataWithBytes:theData length:fieldLengths[i]]; + + // For string data, convert to text + } else { + cellData = [[NSString alloc] initWithBytes:theData length:fieldLengths[i] encoding:mEncoding]; + if (cellData) [cellData autorelease]; + } + break; + + case FIELD_TYPE_NULL: + cellData = [NSNull null]; + break; + + default: + NSLog(@"in fetchNextRowAsArray : Unknown type : %d for column %d, sending back a NSData object", (int)fieldDefinitions[i].type, (int)i); + cellData = [NSData dataWithBytes:theData length:fieldLengths[i]]; + break; + } + + // Free the data if it was originally allocated + if (fullyStreaming) free(theData); + + // If a creator returned a nil object, replace with NSNull + if (cellData == nil) cellData = [NSNull null]; + } + + [returnArray insertObject:cellData atIndex:i]; + } - [returnArray insertObject:cellData atIndex:i]; + // If in cached-streaming mode, update the current entry processed count + if (!fullyStreaming) { + + // Update the active-data pointer to the next item in the list, or set to NULL if no more items + currentDataStoreEntry = currentDataStoreEntry->nextRow; + + // Increment counter + processedRowCount++; } - return returnArray; + return returnArray; } #pragma mark - @@ -199,13 +289,117 @@ - (my_ulonglong) numOfRows { - NSLog(@"numOfRows cannot be used with streaming results"); - return 0; + NSLog(@"numOfRows cannot be used with streaming results"); + return 0; } - (void) dataSeek:(my_ulonglong) row { - NSLog(@"dataSeek cannot be used with streaming results"); + NSLog(@"dataSeek cannot be used with streaming results"); +} + +@end + +@implementation MCPStreamingResult (PrivateAPI) + +/** + * Used internally to download results in a background thread + */ +- (void) _downloadAllData +{ + NSAutoreleasePool *downloadPool = [[NSAutoreleasePool alloc] init]; + MYSQL_ROW theRow; + unsigned long *fieldLengths; + int i, dataCopiedLength, rowDataLength; + LOCAL_ROW_DATA *newRowStore; + + size_t sizeOfLocalRowData = sizeof(LOCAL_ROW_DATA); + size_t sizeOfDataLengths = (size_t)(sizeof(unsigned long) * mNumOfFields); + + // Loop through the rows until the end of the data is reached - indicated via a NULL + while (theRow = mysql_fetch_row(mResult)) { + + // Retrieve the lengths of the returned data + fieldLengths = mysql_fetch_lengths(mResult); + rowDataLength = 0; + dataCopiedLength = 0; + for (i = 0; i < mNumOfFields; i++) + rowDataLength += fieldLengths[i]; + + // Initialise memory for the row and set a NULL pointer for the next item + newRowStore = malloc(sizeOfLocalRowData); + newRowStore->nextRow = NULL; + + // Set up the row data store - a char* - and copy in the data if there is any, + // using a null terminator for each field boundary for easier data processing later + if (rowDataLength) { + newRowStore->data = malloc(sizeof(char) * (rowDataLength + mNumOfFields)); + for (i = 0; i < mNumOfFields; i++) { + if (theRow[i] != NULL) { + memcpy(newRowStore->data+dataCopiedLength, theRow[i], fieldLengths[i]); + newRowStore->data[dataCopiedLength+fieldLengths[i]] = '\0'; + dataCopiedLength += fieldLengths[i] + 1; + } else { + fieldLengths[i] = NSNotFound; + } + } + } else { + newRowStore->data = NULL; + } + + // Set up and copy in the field lengths + newRowStore->dataLengths = memcpy(malloc(sizeOfDataLengths), fieldLengths, sizeOfDataLengths); + + // Add the newly allocated row to end of the storage linked list + if (localDataStore) { + localDataStoreLastEntry->nextRow = newRowStore; + } else { + localDataStore = newRowStore; + } + localDataStoreLastEntry = newRowStore; + if (!currentDataStoreEntry) currentDataStoreEntry = newRowStore; + + // Update the downloaded row count + downloadedRowCount++; + } + + dataDownloaded = YES; + [downloadPool drain]; +} + +/** + * Used internally to free data which has been fully processed; done in a thread to allow + * fetchNextRowAsArray to be faster. + */ +- (void) _freeAllDataWhenDone +{ + NSAutoreleasePool *dataFreeingPool = [[NSAutoreleasePool alloc] init]; + + while (!dataDownloaded || freedRowCount != downloadedRowCount) { + + // If the freed row count matches the processed row count, wait before retrying + if (freedRowCount == processedRowCount) { + usleep(1000); + continue; + } + + // Free a single item off the bottom of the list + // Update the data pointer to the next item in the list, or set to NULL if no more items + LOCAL_ROW_DATA *rowToRemove = localDataStore; + localDataStore = localDataStore->nextRow; + + // Free memory for the first row + rowToRemove->nextRow = NULL; + free(rowToRemove->dataLengths); + if (rowToRemove->data != NULL) free(rowToRemove->data); + free(rowToRemove); + + // Increment the counter + freedRowCount++; + } + + dataFreed = YES; + [dataFreeingPool drain]; } @end
\ No newline at end of file diff --git a/Interfaces/English.lproj/DBView.xib b/Interfaces/English.lproj/DBView.xib index 105be269..48775ba0 100644 --- a/Interfaces/English.lproj/DBView.xib +++ b/Interfaces/English.lproj/DBView.xib @@ -2,13 +2,12 @@ <archive type="com.apple.InterfaceBuilder3.Cocoa.XIB" version="7.03"> <data> <int key="IBDocument.SystemTarget">1050</int> - <string key="IBDocument.SystemVersion">9L30</string> + <string key="IBDocument.SystemVersion">9J61</string> <string key="IBDocument.InterfaceBuilderVersion">677</string> - <string key="IBDocument.AppKitVersion">949.54</string> + <string key="IBDocument.AppKitVersion">949.46</string> <string key="IBDocument.HIToolboxVersion">353.00</string> <object class="NSMutableArray" key="IBDocument.EditedObjectIDs"> <bool key="EncodedWithXMLCoder">YES</bool> - <integer value="500"/> </object> <object class="NSArray" key="IBDocument.PluginDependencies"> <bool key="EncodedWithXMLCoder">YES</bool> @@ -5016,7 +5015,7 @@ <string key="NSWindowContentMaxSize">{600, 142}</string> <string key="NSWindowContentMinSize">{269, 142}</string> <object class="NSView" key="NSWindowView" id="846512394"> - <reference key="NSNextResponder"/> + <nil key="NSNextResponder"/> <int key="NSvFlags">256</int> <object class="NSMutableArray" key="NSSubviews"> <bool key="EncodedWithXMLCoder">YES</bool> @@ -5025,7 +5024,6 @@ <int key="NSvFlags">256</int> <string key="NSFrame">{{36, 105}, {71, 14}}</string> <reference key="NSSuperview" ref="846512394"/> - <reference key="NSWindow"/> <bool key="NSEnabled">YES</bool> <object class="NSTextFieldCell" key="NSCell" id="967312779"> <int key="NSCellFlags">68288064</int> @@ -5042,7 +5040,6 @@ <int key="NSvFlags">256</int> <string key="NSFrame">{{17, 80}, {90, 14}}</string> <reference key="NSSuperview" ref="846512394"/> - <reference key="NSWindow"/> <bool key="NSEnabled">YES</bool> <object class="NSTextFieldCell" key="NSCell" id="308586219"> <int key="NSCellFlags">68288064</int> @@ -5059,7 +5056,6 @@ <int key="NSvFlags">258</int> <string key="NSFrame">{{112, 104}, {137, 18}}</string> <reference key="NSSuperview" ref="846512394"/> - <reference key="NSWindow"/> <bool key="NSEnabled">YES</bool> <object class="NSTextFieldCell" key="NSCell" id="39963071"> <int key="NSCellFlags">-1804468671</int> @@ -5077,7 +5073,6 @@ <int key="NSvFlags">257</int> <string key="NSFrame">{{184, 13}, {70, 28}}</string> <reference key="NSSuperview" ref="846512394"/> - <reference key="NSWindow"/> <int key="NSTag">1</int> <bool key="NSEnabled">YES</bool> <object class="NSButtonCell" key="NSCell" id="805893132"> @@ -5101,7 +5096,6 @@ <int key="NSvFlags">256</int> <string key="NSFrame">{{109, 75}, {143, 22}}</string> <reference key="NSSuperview" ref="846512394"/> - <reference key="NSWindow"/> <bool key="NSEnabled">YES</bool> <object class="NSPopUpButtonCell" key="NSCell" id="285868883"> <int key="NSCellFlags">-2076049856</int> @@ -5425,7 +5419,6 @@ <int key="NSvFlags">256</int> <string key="NSFrame">{{41, 54}, {66, 14}}</string> <reference key="NSSuperview" ref="846512394"/> - <reference key="NSWindow"/> <bool key="NSEnabled">YES</bool> <object class="NSTextFieldCell" key="NSCell" id="193977111"> <int key="NSCellFlags">68288064</int> @@ -5442,7 +5435,6 @@ <int key="NSvFlags">256</int> <string key="NSFrame">{{109, 50}, {143, 22}}</string> <reference key="NSSuperview" ref="846512394"/> - <reference key="NSWindow"/> <bool key="NSEnabled">YES</bool> <object class="NSPopUpButtonCell" key="NSCell" id="285541065"> <int key="NSCellFlags">-2076049856</int> @@ -5478,7 +5470,6 @@ <int key="NSvFlags">257</int> <string key="NSFrame">{{116, 13}, {70, 28}}</string> <reference key="NSSuperview" ref="846512394"/> - <reference key="NSWindow"/> <bool key="NSEnabled">YES</bool> <object class="NSButtonCell" key="NSCell" id="783843620"> <int key="NSCellFlags">67239424</int> @@ -5497,8 +5488,6 @@ </object> </object> <string key="NSFrameSize">{269, 142}</string> - <reference key="NSSuperview"/> - <reference key="NSWindow"/> </object> <string key="NSScreenRect">{{0, 0}, {1680, 1028}}</string> <string key="NSMinSize">{269, 164}</string> @@ -5517,7 +5506,7 @@ <string key="NSWindowContentMaxSize">{600, 127}</string> <string key="NSWindowContentMinSize">{260, 127}</string> <object class="NSView" key="NSWindowView" id="653204527"> - <reference key="NSNextResponder"/> + <nil key="NSNextResponder"/> <int key="NSvFlags">256</int> <object class="NSMutableArray" key="NSSubviews"> <bool key="EncodedWithXMLCoder">YES</bool> @@ -5621,7 +5610,6 @@ </object> </object> <string key="NSFrameSize">{260, 127}</string> - <reference key="NSSuperview"/> </object> <string key="NSScreenRect">{{0, 0}, {1440, 878}}</string> <string key="NSMinSize">{260, 149}</string> @@ -9459,7 +9447,6 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string> </object> <string key="NSFrame">{{1, 1}, {198, 107}}</string> <reference key="NSSuperview" ref="27548452"/> - <reference key="NSNextKeyView" ref="29661959"/> <reference key="NSDocView" ref="29661959"/> <reference key="NSBGColor" ref="1024678221"/> <int key="NScvFlags">4</int> @@ -9487,7 +9474,6 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string> </object> <string key="NSFrame">{{20, 20}, {211, 109}}</string> <reference key="NSSuperview" ref="774289419"/> - <reference key="NSNextKeyView" ref="383807970"/> <int key="NSsFlags">18</int> <reference key="NSVScroller" ref="438736883"/> <reference key="NSHScroller" ref="721548430"/> @@ -9588,6 +9574,27 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string> <int key="NSPeriodicInterval">25</int> </object> </object> + <object class="NSButton" id="380750376"> + <reference key="NSNextResponder" ref="774289419"/> + <int key="NSvFlags">265</int> + <string key="NSFrame">{{288, 32}, {151, 18}}</string> + <reference key="NSSuperview" ref="774289419"/> + <bool key="NSEnabled">YES</bool> + <object class="NSButtonCell" key="NSCell" id="190493018"> + <int key="NSCellFlags">67239424</int> + <int key="NSCellFlags2">131072</int> + <string key="NSContents">Low-memory export</string> + <reference key="NSSupport" ref="26"/> + <reference key="NSControlView" ref="380750376"/> + <int key="NSButtonFlags">1211912703</int> + <int key="NSButtonFlags2">2</int> + <reference key="NSAlternateImage" ref="386686735"/> + <string key="NSAlternateContents"/> + <string key="NSKeyEquivalent"/> + <int key="NSPeriodicDelay">200</int> + <int key="NSPeriodicInterval">25</int> + </object> + </object> <object class="NSButton" id="7792134"> <reference key="NSNextResponder" ref="774289419"/> <int key="NSvFlags">265</int> @@ -9651,6 +9658,26 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string> <int key="NSPeriodicInterval">25</int> </object> </object> + <object class="NSTextField" id="953478607"> + <reference key="NSNextResponder" ref="774289419"/> + <int key="NSvFlags">268</int> + <string key="NSFrame">{{306, 22}, {141, 11}}</string> + <reference key="NSSuperview" ref="774289419"/> + <bool key="NSEnabled">YES</bool> + <object class="NSTextFieldCell" key="NSCell" id="257935364"> + <int key="NSCellFlags">68288064</int> + <int key="NSCellFlags2">272892928</int> + <string key="NSContents">May block server during export</string> + <object class="NSFont" key="NSSupport"> + <string key="NSName">LucidaGrande</string> + <double key="NSSize">9.000000e+00</double> + <int key="NSfFlags">3614</int> + </object> + <reference key="NSControlView" ref="953478607"/> + <reference key="NSBackgroundColor" ref="62854682"/> + <reference key="NSTextColor" ref="454249633"/> + </object> + </object> </object> <string key="NSFrameSize">{457, 149}</string> <string key="NSClassName">NSView</string> @@ -14956,6 +14983,14 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string> </object> <int key="connectionID">6354</int> </object> + <object class="IBConnectionRecord"> + <object class="IBOutletConnection" key="connection"> + <string key="label">sqlFullStreamingSwitch</string> + <reference key="source" ref="225526897"/> + <reference key="destination" ref="380750376"/> + </object> + <int key="connectionID">6357</int> + </object> </object> <object class="IBMutableOrderedSet" key="objectRecords"> <object class="NSArray" key="orderedObjects"> @@ -15590,6 +15625,8 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string> <reference ref="7792134"/> <reference ref="178833443"/> <reference ref="339042383"/> + <reference ref="380750376"/> + <reference ref="953478607"/> </object> <reference key="parent" ref="1043842561"/> <string key="objectName">exportDumpView</string> @@ -21103,6 +21140,34 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string> <reference key="object" ref="882848749"/> <reference key="parent" ref="1012516136"/> </object> + <object class="IBObjectRecord"> + <int key="objectID">6355</int> + <reference key="object" ref="380750376"/> + <object class="NSMutableArray" key="children"> + <bool key="EncodedWithXMLCoder">YES</bool> + <reference ref="190493018"/> + </object> + <reference key="parent" ref="774289419"/> + </object> + <object class="IBObjectRecord"> + <int key="objectID">6356</int> + <reference key="object" ref="190493018"/> + <reference key="parent" ref="380750376"/> + </object> + <object class="IBObjectRecord"> + <int key="objectID">6358</int> + <reference key="object" ref="953478607"/> + <object class="NSMutableArray" key="children"> + <bool key="EncodedWithXMLCoder">YES</bool> + <reference ref="257935364"/> + </object> + <reference key="parent" ref="774289419"/> + </object> + <object class="IBObjectRecord"> + <int key="objectID">6359</int> + <reference key="object" ref="257935364"/> + <reference key="parent" ref="953478607"/> + </object> </object> </object> <object class="NSMutableDictionary" key="flattenedProperties"> @@ -22362,6 +22427,12 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string> <string>6344.ImportedFromIB2</string> <string>6347.IBPluginDependency</string> <string>6348.IBPluginDependency</string> + <string>6355.IBAttributePlaceholdersKey</string> + <string>6355.IBPluginDependency</string> + <string>6355.ImportedFromIB2</string> + <string>6356.IBPluginDependency</string> + <string>6358.IBPluginDependency</string> + <string>6359.IBPluginDependency</string> <string>654.IBPluginDependency</string> <string>654.ImportedFromIB2</string> <string>655.IBPluginDependency</string> @@ -23402,8 +23473,8 @@ IGRvIHlvdSB3YW50IHRvIGFkZCBmb3IgdGhpcyBmaWVsZD8</string> <string>{780, 480}</string> <string>com.apple.InterfaceBuilder.CocoaPlugin</string> <reference ref="9"/> - <string>{{697, 485}, {260, 127}}</string> - <string>{{697, 485}, {260, 127}}</string> + <string>{{628, 485}, {260, 127}}</string> + <string>{{628, 485}, {260, 127}}</string> <reference ref="9"/> <reference ref="5"/> <reference ref="9"/> @@ -24168,6 +24239,19 @@ aGUgYWN0aXZlIHNlbGVjdGlvbiAo4oyl4oyYUik</string> <reference ref="9"/> <string>com.apple.InterfaceBuilder.CocoaPlugin</string> <string>com.apple.InterfaceBuilder.CocoaPlugin</string> + <object class="NSMutableDictionary"> + <string key="NS.key.0">ToolTip</string> + <object class="IBToolTipAttribute" key="NS.object.0"> + <string key="name">ToolTip</string> + <reference key="object" ref="380750376"/> + <string key="toolTip">Use very low memory export, however large the table. With some table types like MyISAM this may prevent UPDATES/INSERTS until the table is exported.</string> + </object> + </object> + <string>com.apple.InterfaceBuilder.CocoaPlugin</string> + <reference ref="9"/> + <string>com.apple.InterfaceBuilder.CocoaPlugin</string> + <string>com.apple.InterfaceBuilder.CocoaPlugin</string> + <string>com.apple.InterfaceBuilder.CocoaPlugin</string> <string>com.apple.InterfaceBuilder.CocoaPlugin</string> <reference ref="9"/> <string>com.apple.InterfaceBuilder.CocoaPlugin</string> @@ -24268,7 +24352,7 @@ Y2hhbmdlIHRoZSBvcmRlcg</string> <reference ref="9"/> <string>com.apple.InterfaceBuilder.CocoaPlugin</string> <reference ref="9"/> - <string>{{193, 780}, {457, 149}}</string> + <string>{{193, 707}, {457, 149}}</string> <string>com.apple.InterfaceBuilder.CocoaPlugin</string> <reference ref="9"/> <string>com.apple.InterfaceBuilder.CocoaPlugin</string> @@ -24388,7 +24472,7 @@ Y2hhbmdlIHRoZSBvcmRlcg</string> </object> </object> <nil key="sourceID"/> - <int key="maxID">6354</int> + <int key="maxID">6359</int> </object> <object class="IBClassDescriber" key="IBDocument.Classes"> <object class="NSMutableArray" key="referencedPartialClassDescriptions"> @@ -25385,6 +25469,7 @@ Y2hhbmdlIHRoZSBvcmRlcg</string> <string>singleProgressSheet</string> <string>singleProgressText</string> <string>singleProgressTitle</string> + <string>sqlFullStreamingSwitch</string> <string>tableContentInstance</string> <string>tableDataInstance</string> <string>tableDocumentInstance</string> @@ -25446,6 +25531,7 @@ Y2hhbmdlIHRoZSBvcmRlcg</string> <string>id</string> <string>id</string> <string>id</string> + <string>id</string> </object> </object> <object class="IBClassDescriptionSource" key="sourceIdentifier"> diff --git a/Source/TableDump.h b/Source/TableDump.h index d32cddea..dc99d4bb 100644 --- a/Source/TableDump.h +++ b/Source/TableDump.h @@ -75,6 +75,7 @@ IBOutlet id addCreateTableSwitch; IBOutlet id addTableContentSwitch; IBOutlet id addErrorsSwitch; + IBOutlet id sqlFullStreamingSwitch; IBOutlet id errorsSheet; IBOutlet id errorsView; IBOutlet id singleProgressSheet; diff --git a/Source/TableDump.m b/Source/TableDump.m index 4e53afb9..e07e4c86 100644 --- a/Source/TableDump.m +++ b/Source/TableDump.m @@ -1001,7 +1001,7 @@ rowCount = [[[[mySQLConnection queryString:[NSString stringWithFormat:@"SELECT COUNT(1) FROM %@", [tableName backtickQuotedString]]] fetchRowAsArray] objectAtIndex:0] intValue]; // Set up a result set in streaming mode - streamingResult = [mySQLConnection streamingQueryString:[NSString stringWithFormat:@"SELECT * FROM %@", [tableName backtickQuotedString]]]; + streamingResult = [mySQLConnection streamingQueryString:[NSString stringWithFormat:@"SELECT * FROM %@", [tableName backtickQuotedString]] useLowMemoryBlockingStreaming:([sqlFullStreamingSwitch state] == NSOnState)]; fieldNames = [streamingResult fetchFieldNames]; // Update the progress text and set the progress bar back to determinate |