Merge pull request #275 from terrycain/docker_test_fixtures

Docker test fixtures

.travis.yml

@@ -9,6 +9,9 @@ env:
         - PYTHONASYNCIODEBUG=1
         - PYTHONASYNCIODEBUG=
 
+services:
+    - docker
+
 matrix:
    include:
         - python: 3.6

requirements-dev.txt

@@ -5,6 +5,8 @@ ipython==6.2.1
 pytest==3.4.2
 pytest-cov==2.5.1
 pytest-sugar==0.9.1
+PyMySQL>=0.7.5
+docker==3.1.4
 sphinx==1.7.1
 sphinxcontrib-asyncio==0.2.0
 sqlalchemy==1.2.5

tests/base.py

@@ -30,7 +30,7 @@ def _connect_all(self):
     def setUp(self):
         super(AIOPyMySQLTestCase, self).setUp()
         self.host = os.environ.get('MYSQL_HOST', 'localhost')
-        self.port = os.environ.get('MYSQL_PORT', 3306)
+        self.port = int(os.environ.get('MYSQL_PORT', 3306))
         self.user = os.environ.get('MYSQL_USER', 'root')
         self.db = os.environ.get('MYSQL_DB', 'test_pymysql')
         self.other_db = os.environ.get('OTHER_MYSQL_DB', 'test_pymysql2')
@@ -47,7 +47,7 @@ def tearDown(self):
 
     @asyncio.coroutine
     def connect(self, host=None, user=None, password=None,
-                db=None, use_unicode=True, no_delay=None, **kwargs):
+                db=None, use_unicode=True, no_delay=None, port=None, **kwargs):
         if host is None:
             host = self.host
         if user is None:
@@ -56,16 +56,20 @@ def connect(self, host=None, user=None, password=None,
             password = self.password
         if db is None:
             db = self.db
+        if port is None:
+            port = self.port
         conn = yield from aiomysql.connect(loop=self.loop, host=host,
                                            user=user, password=password,
                                            db=db, use_unicode=use_unicode,
-                                           no_delay=no_delay, **kwargs)
+                                           no_delay=no_delay, port=port,
+                                           **kwargs)
         self.addCleanup(conn.close)
         return conn
 
     @asyncio.coroutine
     def create_pool(self, host=None, user=None, password=None,
-                    db=None, use_unicode=True, no_delay=None, **kwargs):
+                    db=None, use_unicode=True, no_delay=None,
+                    port=None, **kwargs):
         if host is None:
             host = self.host
         if user is None:
@@ -74,9 +78,12 @@ def create_pool(self, host=None, user=None, password=None,
             password = self.password
         if db is None:
             db = self.db
+        if port is None:
+            port = self.port
         pool = yield from aiomysql.create_pool(loop=self.loop, host=host,
                                                user=user, password=password,
                                                db=db, use_unicode=use_unicode,
-                                               no_delay=no_delay, **kwargs)
+                                               no_delay=no_delay, port=port,
+                                               **kwargs)
         self.addCleanup(pool.close)
         return pool

tests/conftest.py

@@ -1,9 +1,16 @@
 import asyncio
 import gc
 import os
+import ssl
+import socket
 import sys
+import time
+import uuid
+
+from docker import APIClient
 
 import aiomysql
+import pymysql
 import pytest
 
 
@@ -14,11 +21,36 @@
     uvloop = None
 
 
+@pytest.fixture(scope='session')
+def unused_port():
+    def f():
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind(('127.0.0.1', 0))
+            return s.getsockname()[1]
+    return f
+
+
 def pytest_generate_tests(metafunc):
     if 'loop_type' in metafunc.fixturenames:
         loop_type = ['asyncio', 'uvloop'] if uvloop else ['asyncio']
         metafunc.parametrize("loop_type", loop_type)
 
+    # if 'mysql_tag' in metafunc.fixturenames:
+    #     tags = set(metafunc.config.option.mysql_tag)
+    #     if not tags:
+    #         tags = ['5.7']
+    #     elif 'all' in tags:
+    #         tags = ['5.6', '5.7', '8.0']
+    #     else:
+    #         tags = list(tags)
+    #     metafunc.parametrize("mysql_tag", tags, scope='session')
+
+
+# This is here unless someone fixes the generate_tests bit
+@pytest.yield_fixture(scope='session')
+def mysql_tag():
+    return '5.6'
+
 
 @pytest.yield_fixture
 def loop(request, loop_type):
@@ -77,10 +109,19 @@ def pytest_ignore_collect(path, config):
             return True
 
 
+def pytest_addoption(parser):
+    parser.addoption("--mysql_tag", action="append", default=[],
+                     help=("MySQL server versions. "
+                           "May be used several times. "
+                           "Available values: 5.6, 5.7, 8.0, all"))
+    parser.addoption("--no-pull", action="store_true", default=False,
+                     help="Don't perform docker images pulling")
+
+
 @pytest.fixture
 def mysql_params():
     params = {"host": os.environ.get('MYSQL_HOST', 'localhost'),
-              "port": os.environ.get('MYSQL_PORT', 3306),
+              "port": int(os.environ.get('MYSQL_PORT', 3306)),
               "user": os.environ.get('MYSQL_USER', 'root'),
               "db": os.environ.get('MYSQL_DB', 'test_pymysql'),
               "password": os.environ.get('MYSQL_PASSWORD', ''),
@@ -164,3 +205,108 @@ def _register_table(table_name):
         # TODO: probably this is not safe code
         sql = "DROP TABLE IF EXISTS {};".format(t)
         loop.run_until_complete(cursor.execute(sql))
+
+
+@pytest.fixture(scope='session')
+def session_id():
+    """Unique session identifier, random string."""
+    return str(uuid.uuid4())
+
+
+@pytest.fixture(scope='session')
+def docker():
+    return APIClient(version='auto')
+
+
+@pytest.fixture(scope='session')
+def mysql_server(unused_port, docker, session_id, mysql_tag, request):
+    if not request.config.option.no_pull:
+        docker.pull('mysql:{}'.format(mysql_tag))
+
+    # bound IPs do not work on OSX
+    host = "127.0.0.1"
+    host_port = unused_port()
+
+    # As TLS is optional, might as well always configure it
+    ssl_directory = os.path.join(os.path.dirname(__file__),
+                                 'ssl_resources', 'ssl')
+    ca_file = os.path.join(ssl_directory, 'ca.pem')
+    tls_cnf = os.path.join(os.path.dirname(__file__),
+                           'ssl_resources', 'tls.cnf')
+
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+    ctx.check_hostname = False
+    ctx.load_verify_locations(cafile=ca_file)
+    # ctx.verify_mode = ssl.CERT_NONE
+
+    container_args = dict(
+        image='mysql:{}'.format(mysql_tag),
+        name='aiomysql-test-server-{}-{}'.format(mysql_tag, session_id),
+        ports=[3306],
+        detach=True,
+        host_config=docker.create_host_config(
+            port_bindings={3306: (host, host_port)},
+            binds={
+                ssl_directory: {'bind': '/etc/mysql/ssl', 'mode': 'ro'},
+                tls_cnf: {'bind': '/etc/mysql/conf.d/tls.cnf', 'mode': 'ro'},
+            }
+        ),
+        environment={'MYSQL_ROOT_PASSWORD': 'rootpw'}
+    )
+
+    container = docker.create_container(**container_args)
+
+    try:
+        docker.start(container=container['Id'])
+
+        # MySQL restarts at least 4 times in the container before its ready
+        time.sleep(10)
+
+        server_params = {
+            'host': host,
+            'port': host_port,
+            'user': 'root',
+            'password': 'rootpw',
+            'ssl': ctx
+        }
+        delay = 0.001
+        for i in range(100):
+            try:
+                connection = pymysql.connect(
+                    db='mysql',
+                    charset='utf8mb4',
+                    cursorclass=pymysql.cursors.DictCursor,
+                    **server_params)
+
+                with connection.cursor() as cursor:
+                    cursor.execute("SHOW VARIABLES LIKE '%ssl%';")
+
+                    result = cursor.fetchall()
+                    result = {item['Variable_name']:
+                              item['Value'] for item in result}
+
+                    assert result['have_ssl'] == "YES", \
+                        "SSL Not Enabled on docker'd MySQL"
+
+                    cursor.execute("SHOW STATUS LIKE '%Ssl_version%'")
+
+                    result = cursor.fetchone()
+                    # As we connected with TLS, it should start with that :D
+                    assert result['Value'].startswith('TLS'), \
+                        "Not connected to the database with TLS"
+
+                break
+            except Exception as err:
+                time.sleep(delay)
+                delay *= 2
+        else:
+            pytest.fail("Cannot start MySQL server")
+
+        container['host'] = host
+        container['port'] = host_port
+        container['conn_params'] = server_params
+
+        yield container
+    finally:
+        docker.kill(container=container['Id'])
+        docker.remove_container(container['Id'])

tests/sa/test_sa_connection.py

@@ -22,7 +22,7 @@ def setUp(self):
         self.loop = asyncio.new_event_loop()
         asyncio.set_event_loop(None)
         self.host = os.environ.get('MYSQL_HOST', 'localhost')
-        self.port = os.environ.get('MYSQL_PORT', 3306)
+        self.port = int(os.environ.get('MYSQL_PORT', 3306))
         self.user = os.environ.get('MYSQL_USER', 'root')
         self.db = os.environ.get('MYSQL_DB', 'test_pymysql')
         self.password = os.environ.get('MYSQL_PASSWORD', '')
@@ -37,6 +37,7 @@ def connect(self, **kwargs):
                                   password=self.password,
                                   host=self.host,
                                   loop=self.loop,
+                                  port=self.port,
                                   **kwargs)
         yield from conn.autocommit(True)
         cur = yield from conn.cursor()

tests/sa/test_sa_engine.py

@@ -19,7 +19,7 @@ def setUp(self):
         self.loop = asyncio.new_event_loop()
         asyncio.set_event_loop(None)
         self.host = os.environ.get('MYSQL_HOST', 'localhost')
-        self.port = os.environ.get('MYSQL_PORT', 3306)
+        self.port = int(os.environ.get('MYSQL_PORT', 3306))
         self.user = os.environ.get('MYSQL_USER', 'root')
         self.db = os.environ.get('MYSQL_DB', 'test_pymysql')
         self.password = os.environ.get('MYSQL_PASSWORD', '')
@@ -38,6 +38,7 @@ def make_engine(self, use_loop=True, **kwargs):
                                                 user=self.user,
                                                 password=self.password,
                                                 host=self.host,
+                                                port=self.port,
                                                 loop=self.loop,
                                                 minsize=10,
                                                 **kwargs))
@@ -46,6 +47,7 @@ def make_engine(self, use_loop=True, **kwargs):
                                                 user=self.user,
                                                 password=self.password,
                                                 host=self.host,
+                                                port=self.port,
                                                 minsize=10,
                                                 **kwargs))
 

tests/sa/test_sa_transaction.py

@@ -33,7 +33,7 @@ def setUp(self):
         self.loop = asyncio.new_event_loop()
         asyncio.set_event_loop(None)
         self.host = os.environ.get('MYSQL_HOST', 'localhost')
-        self.port = os.environ.get('MYSQL_PORT', 3306)
+        self.port = int(os.environ.get('MYSQL_PORT', 3306))
         self.user = os.environ.get('MYSQL_USER', 'root')
         self.db = os.environ.get('MYSQL_DB', 'test_pymysql')
         self.password = os.environ.get('MYSQL_PASSWORD', '')
@@ -58,6 +58,7 @@ def connect(self, **kwargs):
                                   user=self.user,
                                   password=self.password,
                                   host=self.host,
+                                  port=self.port,
                                   loop=self.loop,
                                   **kwargs)
         # TODO: fix this, should autocommit be enabled by default?

tests/ssl_resources/README.md

@@ -0,0 +1,20 @@
+# MySQL TLS Stuff
+
+This folder contains some resources to be mounted into a mysql container to support TLS
+
+Most of the instructions were taken from here https://dev.mysql.com/doc/refman/5.7/en/creating-ssl-files-using-openssl.html
+
+# Generating certificates
+```bash
+openssl genrsa 2048 > ca-key.pem
+openssl req -new -x509 -nodes -days 3600 -key ca-key.pem -out ca.pem
+
+openssl req -newkey rsa:2048 -days 3600 -nodes -keyout server-key.pem -out server-req.pem
+openssl rsa -in server-key.pem -out server-key.pem
+openssl x509 -req -in server-req.pem -days 3600 -CA ca.pem -CAkey ca-key.pem -set_serial 01 -out server-cert.pem
+```
+The current files under `ssl/` have the default values provided by openssl.
+
+# MySQL Config
+MySQL imports all `.cnf` files under `/etc/mysql/conf.d` so a `tls.cnf` is placed in there referencing the SSL CA 
+cert and server cert and key. The entire `ssl/` directory should be mounted on the container to `/etc/mysql/ssl/`

tests/ssl_resources/ssl/ca-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA4QGo8oVPe8HCIqfJXYVxZbHAZDVlMK/e6ypQM463B8lNg6Ce
+dd0QfAVIs3wuXtRTVD6ELVFDrmcg/eBjM00Idh2GcI0ojDAPcECBvvk0SWYrUBFn
+bou5R1tPwCb8yY601dNExteT61egVhTQd8Vz6NjN9tkMqrpnZ3pn86QqnFWuDOvJ
+5YkOAHOjXipkb8ba+Y4Jl50N6KZH2vrHwdWiO/DYQJLuvoOtYI6a+AXNH+ZSNvid
+Vz9nb3taUQcNxUlYtTByPfnU9wo/dzDFpNKvC5yScPvh2AmJUrEKNEYkFY/YbqLc
+iCXOoBfh1EHqmR2EASjanr+LgwTJfXoX3bQeQQIDAQABAoIBAQDLQ3igPhXjstHy
+BKlANwCN4dnvrNzQ8s/qmbsCGHb4Lb48nqkHyMDPiOZ4XkJ1oFH21NMLLVJ7Buci
+8cYr3fc63MlKe/qZSgFoYp3TK8U0WXvfRRmvH8Is2CxfZdkPLD/ouoZzKuSRwgMy
+QHNi/5kKTHEkAkgTI3muXUHzM+baeknisEqXqCib1yY4FfX2Vnip1dTbbj1gEbGu
+vgFM67uXsoKeQ8ykTF5ZUDIphP+tFWgLMZA2L3iTgRSZJztWwxUMa9NZ8erxy25h
+HPntC5OELndKEbO+su7wCMkw8w2cNA1V6yyQlwSoIRD1fhit/v4hOmvofTpoikXH
+DaCJjgwBAoGBAP84Wq8nrp+79BWRX/qsivERtkGnfjFIDbWBhj70az/CgCV8OWWu
+td6VEEl/Gk7IHsrmOySF0EMYYeMoc5WQKMZbb+1x2m7ovxEwS/UGwNGdtZswyOJw
+y0LKI6dJPQUiZm/O2m0w1zs6/CvvIfu9QOUTxyvwub4lM8UW/gDy79URAoGBAOGx
+q83cTMHKY32dNU/IVaMpw9OxojAWpWXYOEqyV7hM2+gw/lrdBKRyWwB3l3smwet1
+FvKINCZ0bTIRbz/UsNtXp8lISTvw5bQhGsQEYx6ncBeBeHN50zSVoR2xBfqu3pQ4
+G5V/UI82hba7QUDXkuMJ2T7dcZixLk1vp3y+LvYxAoGBANkGYc7J7qs0F6Xzfeta
+p7fA+PuxYxSjEc1DfBWyoDSSv4egr+owe8Tveu8UnxlZAR5GUwqGo4c6h5qzvj3z
+XUj3XiFKjJV9Y2RJbn3IpVRaSKDUBi7P/XgpDdJl6/aevv7aplDtlEhwqxjs+zfn
+QfTKMbbCuB/h4Lj7CTljW+ARAoGAchB7hgVK/b4t3jRv1yymq1nWUM077RXk7b4D
+ZS0RTGH72jO4uW9ugzYQbAIFGwaRh1CcEmNoB+9bqKxLD3WNFK4ObJoN+S9cyFba
+0iptdfalnhufJq1xYugkj38CSJnMgBiDSGEZ8+dYWOv2pLDO2dQGadE9MjCJ+DTv
+7wmnbmECgYBnkVNH0wOvdMuT0vovm18zqtH04PmQGg5JXgQjtpn/6iC9BjRaF2Nz
+Jj9arX00KxIO5vkzbyt5ht5fzk4dpXoG5ozOqnqbj6WNEfgJDeyCkqo1gZOMZSQ/
+YPjKGL7rhXbE/FsNEH90nG0NpIAk1ibtD9sYn6LBBYJcuauegOgE7g==
+-----END RSA PRIVATE KEY-----

tests/ssl_resources/ssl/ca.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDXTCCAkWgAwIBAgIJAP2JtVGC0ZPKMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQwHhcNMTgwMzI5MjEzMzA1WhcNMjgwMjA1MjEzMzA1WjBF
+MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEA4QGo8oVPe8HCIqfJXYVxZbHAZDVlMK/e6ypQM463B8lNg6Cedd0QfAVI
+s3wuXtRTVD6ELVFDrmcg/eBjM00Idh2GcI0ojDAPcECBvvk0SWYrUBFnbou5R1tP
+wCb8yY601dNExteT61egVhTQd8Vz6NjN9tkMqrpnZ3pn86QqnFWuDOvJ5YkOAHOj
+Xipkb8ba+Y4Jl50N6KZH2vrHwdWiO/DYQJLuvoOtYI6a+AXNH+ZSNvidVz9nb3ta
+UQcNxUlYtTByPfnU9wo/dzDFpNKvC5yScPvh2AmJUrEKNEYkFY/YbqLciCXOoBfh
+1EHqmR2EASjanr+LgwTJfXoX3bQeQQIDAQABo1AwTjAdBgNVHQ4EFgQUyNN4RMiv
+zhCNNvMta2kOoiw/SXswHwYDVR0jBBgwFoAUyNN4RMivzhCNNvMta2kOoiw/SXsw
+DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAyAiIqfKoemjnJmoMqcrg
+N/7QhC9P79rzwKQOEFkCFWJFuGiOD6GhPPCu/9ssrl5vBEwDLdl0V4AeGq8DeKV+
+SNON1o6Y6uUAiX5uYK5Asv0avQUu7SS+uvE3YhTELjbvp4vqdLCUjBqq4KZoEA+F
+4hXCltPVdOItztzAd7hgktYrkJeDA1M7sZHTv26HaO6vJ0trUdb4tvqzShzMCvN/
+2s9ZJAVBZL77Px40yUPiK6cjpq5fGcUen0zBumymBRFOb8ykvq7azUdjk6sz65Vb
+Q3kgsKGBHjVOPfXf00YWvgG3NePX3FsBEHtYbBDSD6k+aXQ1WfOUB8mtdwjyEnzD
+7w==
+-----END CERTIFICATE-----