Experiment with #2979

* Updated libmysqlclient from 5.5.56 -> 5.5.59
* Changed the way the connection password is passed between SPMySQL and libmysqlclient

9 files changed, 197 insertions, 20 deletions

diff --git a/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/001-cpp-dependency.diff b/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/001-cpp-dependency.diff
index 06c20001..e3ce848a 100644
--- a/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/001-cpp-dependency.diff
+++ b/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/001-cpp-dependency.diff
@@ -1,3 +1,11 @@
+This patch is neccesary to remove a linker error when trying to link SPMySQL with libmysqlclient.a.
+
+To apply:
+	cd mysql-source-root
+	patch -p1 < this-file
+
+(patch created with `diff -Naur`)
+
 --- mysql-5.5.56-dist/extra/yassl/taocrypt/include/runtime.hpp	2017-04-27 09:12:30.000000000 +0200
 +++ mysql-5.5.56/extra/yassl/taocrypt/include/runtime.hpp	2017-05-20 23:27:14.000000000 +0200
 @@ -53,8 +53,8 @@
diff --git a/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/002-new-types.diff b/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/002-new-types.diff
index bb42f9d9..47aa9c6d 100644
--- a/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/002-new-types.diff
+++ b/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/002-new-types.diff
@@ -1,3 +1,12 @@
+This patch backports field types that were added after MySQL 5.5,
+but are technically still compatible to the old client libs.
+
+To apply:
+	cd mysql-source-root
+	patch -p1 < this-file
+
+(patch created with `diff -Naur`)
+
 --- mysql-5.5.56-dist/include/mysql_com.h	2017-04-27 09:12:30.000000000 +0200
 +++ mysql-5.5.56/include/mysql_com.h	2017-05-21 01:46:44.000000000 +0200
 @@ -349,7 +349,11 @@
diff --git a/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/003-callback-password-auth.diff b/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/003-callback-password-auth.diff
new file mode 100644
index 00000000..78fa873d
--- /dev/null
+++ b/Frameworks/SPMySQLFramework/MySQL Client Libraries/Patches/003-callback-password-auth.diff
@@ -0,0 +1,96 @@
+This patch changes the way libmysqlclient receives the connection password.
+Usually it will get the password by trying in order:
+	1) The passwd that is passed as a parameter to mysql_real_connect()
+	2) The password that was set on MYSQL->options.password
+	3) The contents of the environment variable MYSQL_PWD (compile time setting)
+	4) An empty string
+
+If a connection could be made (not yet authenticathed) the password will be stored
+in MYSQL->passwd for the whole lifetime of the struct.
+
+We don't want that for two reasons:
+	1) That way the password stays in plaintext memory for possibly a long time (and 
+	   may even get swapped to disk)
+	2) MySQL uses plugins for auth (negotiated with the server) and some of them may
+	   transmit the password in plaintext over an unsecure connection.
+	   Since we have no control over that we would have to decide beforehand if that
+	   COULD happen and flat out always deny or allow Keychain access (since e.g.
+	   the AVAILABILITY of the cleartext plugin can be controlled by an envvar).
+
+So with this patch we change the flow of information:
+Now mysql doesn't receive the password up front, but instead it has to ask the user (ie. SPMySQL)
+to get the password precisely then when it needs it and mysql will also tell us
+which auth plugin it negotiated with the server, so we can decide on a per situation
+basis whether to request manual input or fetch it from Keychain.
+
+To apply:
+	cd mysql-source-root
+	patch -p1 < this-file
+
+(patch created with `diff -Naur`)
+
+diff -Naur mysql-5.5.59-dist/include/mysql.h mysql-5.5.59/include/mysql.h
+--- mysql-5.5.59-dist/include/mysql.h	2017-11-27 13:03:17.000000000 +0100
++++ mysql-5.5.59/include/mysql.h	2018-02-14 00:28:26.000000000 +0100
+@@ -288,6 +288,18 @@
+   /* needed for embedded server - no net buffer to store the 'info' */
+   char *info_buffer;
+   void *extension;
++
++  /* SPMySQL patch: 
++   *   Set this to a callback function that will be invoked when mysql wants to do authentication.
++   *   @param mysql          The MYSQL struct
++   *   @param plugin         The name of the auth plugin that will be used (usually either 
++   *                         "mysql_native_password", "mysql_old_password" or "mysql_clear_password")
++   *   @param with_password  A block function you must invoke, during which mysql can use the password you provide via the passwd parameter.
++   *                         After the block you should immediately clear the password from memory again.
++   */
++  void (*passwd_callback)(struct st_mysql *mysql, const char *plugin, void (^with_password)(const char *passwd));
++  /* SPMySQL patch: This is used with passwd_callback to bridge back to OOP land */
++  void *sp_context;
+ } MYSQL;
+ 
+ 
+diff -Naur mysql-5.5.59-dist/sql-common/client.c mysql-5.5.59/sql-common/client.c
+--- mysql-5.5.59-dist/sql-common/client.c	2017-11-27 13:03:17.000000000 +0100
++++ mysql-5.5.59/sql-common/client.c	2018-02-14 00:34:26.000000000 +0100
+@@ -2952,7 +2952,7 @@
+   auth_plugin_t *auth_plugin;
+   MCPVIO_EXT    mpvio;
+   ulong		pkt_length;
+-  int           res;
++  __block int   res;
+ 
+   DBUG_ENTER ("run_plugin_auth");
+   /* determine the default/initial plugin to use */
+@@ -2996,7 +2996,29 @@
+   mpvio.db= db;
+   mpvio.plugin= auth_plugin;
+ 
+-  res= auth_plugin->authenticate_user((struct st_plugin_vio *)&mpvio, mysql);
++  /*
++   * SPMySQL Patch to inverse the password flow
++   */
++  if(mysql->passwd_callback)
++  {
++    res = CR_ERROR; //fallback, if block is never invoked
++    mysql->passwd_callback(mysql, auth_plugin_name, ^(const char *passwd) {
++      char *saved_passwd = mysql->passwd;
++      mysql->passwd = (char *)(passwd ? passwd : ""); // see mysql_change_user
++      res= auth_plugin->authenticate_user((struct st_plugin_vio *)&mpvio, mysql);
++      mysql->passwd = saved_passwd;
++    });
++  }
++  else 
++  {
++    set_mysql_extended_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, 
++                             unknown_sqlstate, 
++                             ER(CR_AUTH_PLUGIN_CANNOT_LOAD), 
++                             auth_plugin_name, 
++                             "passwd_callback not set!");
++    DBUG_RETURN (1);
++  }
++
+   DBUG_PRINT ("info", ("authenticate_user returned %s", 
+                        res == CR_OK ? "CR_OK" : 
+                        res == CR_ERROR ? "CR_ERROR" :
diff --git a/Frameworks/SPMySQLFramework/MySQL Client Libraries/include/mysql.h b/Frameworks/SPMySQLFramework/MySQL Client Libraries/include/mysql.h
index 3a27ab41..353267aa 100644
--- a/Frameworks/SPMySQLFramework/MySQL Client Libraries/include/mysql.h
+++ b/Frameworks/SPMySQLFramework/MySQL Client Libraries/include/mysql.h
@@ -288,6 +288,18 @@ typedef struct st_mysql
   /* needed for embedded server - no net buffer to store the 'info' */
   char *info_buffer;
   void *extension;
+
+  /* SPMySQL patch: 
+   *   Set this to a callback function that will be invoked when mysql wants to do authentication.
+   *   @param mysql          The MYSQL struct
+   *   @param plugin         The name of the auth plugin that will be used (usually either 
+   *                         "mysql_native_password", "mysql_old_password" or "mysql_clear_password")
+   *   @param with_password  A block function you must invoke, during which mysql can use the password you provide via the passwd parameter.
+   *                         After the block you should immediately clear the password from memory again.
+   */
+  void (*passwd_callback)(struct st_mysql *mysql, const char *plugin, void (^with_password)(const char *passwd));
+  /* SPMySQL patch: This is used with passwd_callback to bridge back to OOP land */
+  void *sp_context;
 } MYSQL;
 
 
diff --git a/Frameworks/SPMySQLFramework/MySQL Client Libraries/include/mysql_version.h b/Frameworks/SPMySQLFramework/MySQL Client Libraries/include/mysql_version.h
index 8c18116a..77ea7e07 100644
--- a/Frameworks/SPMySQLFramework/MySQL Client Libraries/include/mysql_version.h
+++ b/Frameworks/SPMySQLFramework/MySQL Client Libraries/include/mysql_version.h
@@ -11,11 +11,11 @@
 #include <custom_conf.h>
 #else
 #define PROTOCOL_VERSION		10
-#define MYSQL_SERVER_VERSION		"5.5.56"
+#define MYSQL_SERVER_VERSION		"5.5.59"
 #define MYSQL_BASE_VERSION		"mysqld-5.5"
 #define MYSQL_SERVER_SUFFIX_DEF		""
 #define FRM_VER				6
-#define MYSQL_VERSION_ID		50556
+#define MYSQL_VERSION_ID		50559
 #define MYSQL_PORT			3306
 #define MYSQL_PORT_DEFAULT		0
 #define MYSQL_UNIX_ADDR			"/tmp/mysql.sock"
diff --git a/Frameworks/SPMySQLFramework/MySQL Client Libraries/lib/libmysqlclient.a b/Frameworks/SPMySQLFramework/MySQL Client Libraries/lib/libmysqlclient.a
index 0fccae22..27e9e15c 100644
--- a/Frameworks/SPMySQLFramework/MySQL Client Libraries/lib/libmysqlclient.a
+++ b/Frameworks/SPMySQLFramework/MySQL Client Libraries/lib/libmysqlclient.a
diff --git a/Frameworks/SPMySQLFramework/Source/SPMySQL Private APIs.h b/Frameworks/SPMySQLFramework/Source/SPMySQL Private APIs.h
index 8fdf4e7e..b2fa9f5d 100644
--- a/Frameworks/SPMySQLFramework/Source/SPMySQL Private APIs.h
+++ b/Frameworks/SPMySQLFramework/Source/SPMySQL Private APIs.h
@@ -41,6 +41,7 @@
 
 - (BOOL)_connect;
 - (MYSQL *)_makeRawMySQLConnectionWithEncoding:(NSString *)encodingName isMasterConnection:(BOOL)isMaster;
+- (void)_mysqlConnection:(MYSQL *)connection wantsPassword:(void (^)(const char *passwd))inBlock withPlugin:(const char *)pluginName;
 - (BOOL)_reconnectAllowingRetries:(BOOL)canRetry;
 - (BOOL)_reconnectAfterBackgroundConnectionLoss;
 - (BOOL)_waitForNetworkConnectionWithTimeout:(double)timeoutSeconds;
diff --git a/Frameworks/SPMySQLFramework/Source/SPMySQLConnection.m b/Frameworks/SPMySQLFramework/Source/SPMySQLConnection.m
index d50375e2..f6755f60 100644
--- a/Frameworks/SPMySQLFramework/Source/SPMySQLConnection.m
+++ b/Frameworks/SPMySQLFramework/Source/SPMySQLConnection.m
@@ -33,6 +33,10 @@
 #include <mach/mach_time.h>
 #include <pthread.h>
 #include <SystemConfiguration/SCNetworkReachability.h>
+#include <errno.h>
+#define __STDC_WANT_LIB_EXT1__ 1
+#include <string.h>
+#include <stdlib.h>
 
 // Thread flag constant
 static pthread_key_t mySQLThreadInitFlagKey;
@@ -49,6 +53,7 @@ const SPMySQLClientFlags SPMySQLConnectionOptions =
 // List of permissible ciphers to use for SSL connections
 const char *SPMySQLSSLPermissibleCiphers = "DHE-RSA-AES256-SHA:AES256-SHA:DHE-RSA-AES128-SHA:AES128-SHA:AES256-RMD:AES128-RMD:DES-CBC3-RMD:DHE-RSA-AES256-RMD:DHE-RSA-AES128-RMD:DHE-RSA-DES-CBC3-RMD:RC4-SHA:RC4-MD5:DES-CBC3-SHA:DES-CBC-SHA:EDH-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC-SHA";
 
+static void PasswordCallback(MYSQL *mysql, const char *plugin, void (^with_password)(const char *passwd));
 
 @implementation SPMySQLConnection
 
@@ -612,27 +617,11 @@ static uint64_t _elapsedMicroSecondsSinceAbsoluteTime(uint64_t comparisonTime)
 	// Set up the connection variables in the format MySQL needs, from the class-wide variables
 	const char *theHost = NULL;
 	const char *theUsername = "";
-	const char *thePassword = NULL;
 	const char *theSocket = NULL;
 
 	if (host) theHost = [host UTF8String]; //mysql calls getaddrinfo on the hostname. Apples code uses -UTF8String in that situation.
 	if (username) theUsername = _cStringForStringWithEncoding(username, connectEncodingNS, NULL); //during connect this is in MYSQL_SET_CHARSET_NAME encoding
 
-	// If a password was supplied, use it; otherwise ask the delegate if appropriate.
-	//
-	// Note that password has no charset in mysql: If a user password is set to 'ü' on a latin1 connection
-	// and you later try to connect on an UTF-8 terminal (or vice versa) it will fail. The MySQL (5.5) manual wrongly states that
-	// MYSQL_SET_CHARSET_NAME has influence over that, but it does not and could not, since the password is hashed by the client
-	// before transmitting it to the server and the (5.5) client has no charset support, effectively treating password as
-	// a NUL-terminated byte array.
-	// There is one exception, though: The "mysql_clear_password" auth plugin sends the password in plaintext and the server side
-	// MAY choose to do a charset conversion as appropriate before handing it to whatever backend is used.
-	// Since we don't know which auth plugin server and client will agree upon, we'll do as the manual says...
-	if (password) {
-		thePassword = _cStringForStringWithEncoding(password, connectEncodingNS, NULL);
-	} else if ([delegate respondsToSelector:@selector(keychainPasswordForConnection:)]) {
-		thePassword = _cStringForStringWithEncoding([delegate keychainPasswordForConnection:self], connectEncodingNS, NULL);
-	}
 
 	// If set to use a socket and a socket was supplied, use it; otherwise, search for a socket to use
 	if (useSocket) {
@@ -690,7 +679,11 @@ static uint64_t _elapsedMicroSecondsSinceAbsoluteTime(uint64_t comparisonTime)
 		}
 	}
 
-	MYSQL *connectionStatus = mysql_real_connect(theConnection, theHost, theUsername, thePassword, NULL, (unsigned int)port, theSocket, [self clientFlags]);
+	// we will provide the password via this callback. the mysql_real_connect parameter is a dummy and won't work
+	theConnection->passwd_callback = &PasswordCallback;
+	theConnection->sp_context = self;
+	
+	MYSQL *connectionStatus = mysql_real_connect(theConnection, theHost, theUsername, "", NULL, (unsigned int)port, theSocket, [self clientFlags]);
 
 	// If the connection failed, return NULL
 	if (theConnection != connectionStatus) {
@@ -733,6 +726,58 @@ static uint64_t _elapsedMicroSecondsSinceAbsoluteTime(uint64_t comparisonTime)
 	return theConnection;
 }
 
+- (void)_mysqlConnection:(MYSQL *)connection wantsPassword:(void (^)(const char *passwd))inBlock withPlugin:(const char *)pluginName
+{
+	// If a password was supplied, use it; otherwise ask the delegate if appropriate.
+	//
+	// Note that password has no charset in mysql: If a user password is set to 'ü' on a latin1 connection
+	// and you later try to connect on an UTF-8 terminal (or vice versa) it will fail. The MySQL (5.5) manual wrongly states that
+	// MYSQL_SET_CHARSET_NAME has influence over that, but it does not and could not, since the password is hashed by the client
+	// before transmitting it to the server and the (5.5) client has no charset support, effectively treating password as
+	// a NUL-terminated byte array.
+	// There is one exception, though: The "mysql_clear_password" auth plugin sends the password in plaintext and the server side
+	// MAY choose to do a charset conversion as appropriate before handing it to whatever backend is used.
+	// Since we don't know which auth plugin server and client will agree upon, we'll do as the manual says...
+	NSString *passwd = nil;
+	
+	if (password) {
+		passwd = password;
+	} else if ([delegate respondsToSelector:@selector(keychainPasswordForConnection:)]) {
+		passwd = [delegate keychainPasswordForConnection:self]; //TODO pass pluginName to client
+	}
+	
+	// shortcut for empty/nil password
+	if(![passwd length]) {
+		inBlock(NULL);
+		return;
+	}
+
+	NSStringEncoding connectEncodingNS = [SPMySQLConnection stringEncodingForMySQLCharset:connection->options.charset_name];
+	NSInteger cLength = [passwd lengthOfBytesUsingEncoding:connectEncodingNS];
+	
+	if(!cLength || cLength == NSIntegerMax) {
+		NSLog(@"%s: -lengthOfBytesUsingEncoding: returned 0 or NSIntegerMax for encoding %lu (mysql: %s)", __PRETTY_FUNCTION__, connectEncodingNS, connection->options.charset_name);
+		return;
+	}
+	
+	char *cBuffer = malloc(++cLength);
+	
+	if(!cBuffer) {
+		NSLog(@"%s: malloc(%ld) failed: %s", __PRETTY_FUNCTION__, (long)cLength, strerror(errno));
+		return;
+	}
+	
+	if([passwd getCString:cBuffer maxLength:cLength encoding:connectEncodingNS]) {
+		inBlock(cBuffer);
+	}
+	else {
+		NSLog(@"%s: -getCString:maxLength:encoding: failed for password!", __PRETTY_FUNCTION__);
+	}
+
+	memset_s(cBuffer, cLength, '\0', cLength); //clear password from memory
+	free(cBuffer);
+}
+
 /**
  * Perform a reconnection task, either once-only or looping as requested.  If looping is
  * permitted and this method fails, it will ask how to proceed and loop depending on
@@ -1162,3 +1207,9 @@ static uint64_t _elapsedMicroSecondsSinceAbsoluteTime(uint64_t comparisonTime)
 }
 
 @end
+
+void PasswordCallback(MYSQL *mysql, const char *plugin, void (^with_password)(const char *passwd))
+{
+	assert(mysql && mysql->sp_context);
+	[(SPMySQLConnection *)mysql->sp_context _mysqlConnection:mysql wantsPassword:with_password withPlugin:plugin];
+}
diff --git a/Frameworks/SPMySQLFramework/build-mysql-client.sh b/Frameworks/SPMySQLFramework/build-mysql-client.sh
index 5475b0a0..ca43eab2 100755
--- a/Frameworks/SPMySQLFramework/build-mysql-client.sh
+++ b/Frameworks/SPMySQLFramework/build-mysql-client.sh
@@ -47,7 +47,7 @@ CLEAN='NO'
 MIN_OS_X_VERSION='10.6'
 ARCHITECTURES='-arch i386 -arch x86_64'
 
-CONFIGURE_OPTIONS='-DBUILD_CONFIG=mysql_release -DENABLED_LOCAL_INFILE=1 -DWITH_SSL=bundled -DWITH_MYSQLD_LDFLAGS="-all-static --disable-shared" -DWITHOUT_SERVER=1 -DWITH_ZLIB=system'
+CONFIGURE_OPTIONS='-DBUILD_CONFIG=mysql_release -DENABLED_LOCAL_INFILE=1 -DWITH_SSL=bundled -DWITH_MYSQLD_LDFLAGS="-all-static --disable-shared" -DWITHOUT_SERVER=1 -DWITH_ZLIB=system -DWITH_UNIT_TESTS=0'
 OUTPUT_DIR='SPMySQLFiles.build'
 
 ESC=`printf '\033'`