Visible improvements in this build:

 - Significantly reduce the queries that have to be performed, improving lag - especially over slow connections (Issue #118; see new controller info under headline code changes).
 - Fix Issue #117 properly (export numeric quoting - we now have access to column types and so can quote appropriately).
 - Fix Issue #145 (loss of unsigned/null/default attributes when reordering columns).
 - Fixes Issue #90 (support for filtering DECIMAL column types)
 - Improve table scrolling speed when the table contains long items. (Added a NSFormatter to automatically truncate strings > 150 chars for display purposes only)
 - Improved SQL compatibility - for example /* C style comments */ are now correctly ignored in imports and custom queries.
 - Add text and symbols emphasising that the table info pane / status view row count is an approximation (partially addresses Issue #141)
 - Fixes a major memory leak whenever opening or scrolling tables containing text/blob data.
 - SQL import is now faster (SQL parsing part is 3x faster).
 - Speed up SQL export (1.5x faster for numeric data; 1.1x faster for string data) and slightly speed up CSV export (~1.1x faster).
 - Display sizes on the status view using the byte size formatter, as per table info pane.
 

Headline code changes:
 - Add a new NSMutableString subclass, SPSQLParser.  See the header file for documentation and overview, but in short it's a centralised place for SQL parsing.  Centralises and improves parsing, improves comment support, improves quoting support.  Despite the improved featureset this is also faster than the previous distributed implementations - for example, when used to replace the old splitQueries:, > 3x speedup.
 - Implement a new controller which handles a structure and status cache for the current table, and provides structure parsing for specified tables.  This cache is now used throughout the code, reducing the queries that have to be performed and providing additional information about the table structure for use; I think it also improves column type format slightly.
 - The table info pane and the status view now draw all their data from the cache.


Tweaks:
 - Table encoding is now detected directly instead of being derived from the collation - increased accuracy and cope with the DEFAULT encoding.
 - Comments and formatting cleaned up in bits I was working on, obviously.
 - A couple of methods - particularly [tablesListInstance table] and [tableDocument encoding] - have been renamed to avoid conflicts and fix code warnings.


Future improvements now possible:
 - As we now have access to column types and other information, we can provide per-type behaviour where desired.
 - The table parsing doesn't currently pull out comments or table indices, together with one or two other attributes.  Some of this would be useful for display; some, such as indices, could be used to draw the table structure view as long as we're happy discarding a couple of columns (ie cardinality!)

1 files changed, 26 insertions, 43 deletions

diff --git a/Source/TableDocument.m b/Source/TableDocument.m
index 6969e561..b9cb66a6 100644
--- a/Source/TableDocument.m
+++ b/Source/TableDocument.m
@@ -33,6 +33,8 @@
 #import "TableStatus.h"
 #import "ImageAndTextCell.h"
 #import "SPGrowlController.h"
+#import "SPSQLParser.h"
+#import "SPTableData.h"
 
 NSString *TableDocumentFavoritesControllerSelectionIndexDidChange = @"TableDocumentFavoritesControllerSelectionIndexDidChange";
 NSString *TableDocumentFavoritesControllerFavoritesDidChange = @"TableDocumentFavoritesControllerFavoritesDidChange";
@@ -190,9 +192,9 @@ NSString *TableDocumentFavoritesControllerFavoritesDidChange = @"TableDocumentFa
 		// set encoding
 		NSString *encodingName = [prefs objectForKey:@"encoding"];
 		if ( [encodingName isEqualToString:@"Autodetect"] ) {
-			[self detectDatabaseEncoding];
+			[self setConnectionEncoding:[self databaseEncoding] reloadingViews:NO];
 		} else {
-			[self setEncoding:[self mysqlEncodingFromDisplayEncoding:encodingName]];
+			[self setConnectionEncoding:[self mysqlEncodingFromDisplayEncoding:encodingName] reloadingViews:NO];
 		}
 		//get mysql version
 		theResult = [mySQLConnection queryString:@"SHOW VARIABLES LIKE 'version'"];
@@ -210,6 +212,7 @@ NSString *TableDocumentFavoritesControllerFavoritesDidChange = @"TableDocumentFa
 		[customQueryInstance setConnection:mySQLConnection];
 		[tableDumpInstance setConnection:mySQLConnection];
 		[tableStatusInstance setConnection:mySQLConnection];
+		[tableDataInstance setConnection:mySQLConnection];
 		[self setFileName:[NSString stringWithFormat:@"(MySQL %@) %@@%@ %@", mySQLVersion, [userField stringValue],
 						   [hostField stringValue], [databaseField stringValue]]];
 		[tableWindow setTitle:[NSString stringWithFormat:@"(MySQL %@) %@@%@/%@", mySQLVersion, [userField stringValue],
@@ -642,7 +645,7 @@ NSString *TableDocumentFavoritesControllerFavoritesDidChange = @"TableDocumentFa
 /**
  * Set the encoding for the database connection
  */
-- (void)setEncoding:(NSString *)mysqlEncoding
+- (void)setConnectionEncoding:(NSString *)mysqlEncoding reloadingViews:(BOOL)reloadViews
 {
 	// set encoding of connection and client
 	[mySQLConnection queryString:[NSString stringWithFormat:@"SET NAMES '%@'", mysqlEncoding]];
@@ -652,22 +655,29 @@ NSString *TableDocumentFavoritesControllerFavoritesDidChange = @"TableDocumentFa
 		[_encoding autorelease];
 		_encoding = [mysqlEncoding retain];
 	} else {
-		[self detectDatabaseEncoding];
+		[mySQLConnection queryString:[NSString stringWithFormat:@"SET NAMES '%@'", [self databaseEncoding]]];
+		if ( ![[mySQLConnection getLastErrorMessage] isEqualToString:@""] ) {
+			NSLog(@"Error: could not set encoding to %@ nor fall back to database encoding on MySQL %@", mysqlEncoding, [self mySQLVersion]);
+			return;
+		}
 	}
 		
 	// update the selected menu item
 	[self updateEncodingMenuWithSelectedEncoding:[self encodingNameFromMySQLEncoding:mysqlEncoding]];
 	
-	// reload stuff
-	[tableSourceInstance reloadTable:self];
-	[tableContentInstance reloadTable:self];
-	[tableStatusInstance reloadTable:self];
+	// Reload stuff as appropriate
+	[tableDataInstance resetAllData];
+	if (reloadViews) {
+		if ([tablesListInstance structureLoaded]) [tableSourceInstance reloadTable:self];
+		if ([tablesListInstance contentLoaded]) [tableContentInstance reloadTable:self];
+		if ([tablesListInstance statusLoaded]) [tableStatusInstance reloadTable:self];
+	}
 }
 
 /**
  * returns the current mysql encoding for this object
  */
-- (NSString *)encoding
+- (NSString *)connectionEncoding
 {
 	return _encoding;
 }
@@ -755,9 +765,10 @@ NSString *TableDocumentFavoritesControllerFavoritesDidChange = @"TableDocumentFa
 }
 
 /**
- * Autodetect the connection encoding and select the relevant encoding menu item in Database -> Database Encoding
+ * Detect and return the database connection encoding.
+ * TODO: See http://code.google.com/p/sequel-pro/issues/detail?id=134 - some question over why this [historically] uses _connection not _database...
  */
-- (void)detectDatabaseEncoding
+- (NSString *)databaseEncoding
 {
 	// MySQL > 4.0
 	id mysqlEncoding = [[[mySQLConnection queryString:@"SHOW VARIABLES LIKE 'character_set_connection'"] fetchRowAsDictionary] objectForKey:@"Value"];
@@ -774,36 +785,7 @@ NSString *TableDocumentFavoritesControllerFavoritesDidChange = @"TableDocumentFa
 		mysqlEncoding = @"latin1";
 	}
 	
-	[mySQLConnection setEncoding:[CMMCPConnection encodingForMySQLEncoding:[mysqlEncoding cString]]];
-	
-	// save the encoding
-	[_encoding autorelease];
-	_encoding = [mysqlEncoding retain];
-	
-	// update the selected menu item
-	[self updateEncodingMenuWithSelectedEncoding:[self encodingNameFromMySQLEncoding:mysqlEncoding]];
-}
-
-/**
- * Detects and sets the character set encoding of the supplied table name.
- */
-- (void)detectTableEncodingForTable:(NSString *)table;
-{
-	NSString *tableCollation = [[[mySQLConnection queryString:[NSString stringWithFormat:@"SHOW TABLE STATUS LIKE '%@'", table]] fetchRowAsDictionary] objectForKey:@"Collation"];
-
-	if (tableCollation != nil) {
-		// Split up the collation string so we can get the encoding
-		NSArray *encodingComponents = [tableCollation componentsSeparatedByString:@"_"];
-		
-		if ([encodingComponents count] > 0) {
-			NSString *tableEncoding = [encodingComponents objectAtIndex:0];
-			
-			[self setEncoding:tableEncoding];
-		}
-	}
-	else {
-		NSLog(@"Error: unable to determine table collation and thus character encoding for table '%@'", table);
-	}
+	return mysqlEncoding;
 }
 
 /**
@@ -811,7 +793,7 @@ NSString *TableDocumentFavoritesControllerFavoritesDidChange = @"TableDocumentFa
  */
 - (IBAction)chooseEncoding:(id)sender
 {
-	[self setEncoding:[self mysqlEncodingFromDisplayEncoding:[(NSMenuItem *)sender title]]];
+	[self setConnectionEncoding:[self mysqlEncodingFromDisplayEncoding:[(NSMenuItem *)sender title]] reloadingViews:YES];
 }
 
 /**
@@ -995,6 +977,7 @@ NSString *TableDocumentFavoritesControllerFavoritesDidChange = @"TableDocumentFa
 	NSRunInformationalAlertPanel([NSString stringWithFormat:@"Checksum '%@' table", [self table]], [NSString stringWithFormat:@"Checksum: %@", [theRow objectForKey:@"Checksum"]], @"OK", nil, nil);
 }
 
+
 #pragma mark Other Methods
 /**
  * returns the host
@@ -1086,7 +1069,7 @@ NSString *TableDocumentFavoritesControllerFavoritesDidChange = @"TableDocumentFa
  returns the currently selected table (passing the request to TablesList)
  */
 {
-	return (NSString *)[tablesListInstance table];
+	return [tablesListInstance tableName];
 }
 
 - (NSString *)mySQLVersion