support mysql

client/controller/build.gradle

@@ -45,7 +45,7 @@ dependencies {
     compile group: 'commons-cli', name: 'commons-cli', version: '1.2'
 
     // https://mvnrepository.com/artifact/mysql/mysql-connector-java
-    compile group: 'mysql', name: 'mysql-connector-java', version: '5.1.6'
+    compile group: 'mysql', name: 'mysql-connector-java', version: '8.0.12'
 
     // https://mvnrepository.com/artifact/org.postgresql/postgresql
     compile group: 'org.postgresql', name: 'postgresql', version: '9.4-1201-jdbc41'

client/controller/src/main/java/com/controller/collectors/MySQLCollector.java

@@ -16,6 +16,7 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Map;
+import java.util.HashMap;
 import org.apache.log4j.Logger;
 
 /** */
@@ -28,6 +29,10 @@ public class MySQLCollector extends DBCollector {
 
   private static final String METRICS_SQL = "SHOW STATUS";
   
+  private static final String METRICS_SQL2 =
+      "SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS where subsystem = 'transaction';";
+  private HashMap<String, String> innodbMetrics = new HashMap<>(); 
+
   public MySQLCollector(String oriDBUrl, String username, String password) {
     try {
       Connection conn = DriverManager.getConnection(oriDBUrl, username, password);
@@ -50,6 +55,12 @@ public class MySQLCollector extends DBCollector {
       while (out.next()) {
         dbMetrics.put(out.getString(1).toLowerCase(), out.getString(2));
       }
+
+      out = s.executeQuery(METRICS_SQL2);
+      while (out.next()) {
+        innodbMetrics.put(out.getString(1).toLowerCase(), out.getString(2));
+      }
+
       conn.close();
     } catch (SQLException e) {
       LOG.error("Error while collecting DB parameters: " + e.getMessage());
@@ -93,6 +104,13 @@ public class MySQLCollector extends DBCollector {
       }
       // "global" is a a placeholder
       jobGlobal.put("global", job);
+
+      JSONObject job2 = new JSONObject();
+      for (Map.Entry<String, String> entry : innodbMetrics.entrySet()) {
+        job2.put(entry.getKey(), entry.getValue());
+      }
+      jobGlobal.put("innodb_metrics", job2);
+      
       stringer.value(jobGlobal);
       stringer.key(JSON_LOCAL_KEY);
       stringer.value(null);

client/driver/driver_config.py

@@ -5,7 +5,7 @@ import os
 #==========================================================
 
 # Location of the database host relative to this driver
-# Valid values: local, remote, or docker
+# Valid values: local, remote, docker or remote_docker
 HOST_CONN = 'local'
 
 # The name of the Docker container for the target database
@@ -15,6 +15,7 @@ CONTAINER_NAME = None  # e.g., 'postgres_container'
 # Host SSH login credentials (only required if HOST_CONN=remote)
 LOGIN_NAME = None
 LOGIN_HOST = None
+LOGIN_PASSWORD = None
 LOGIN_PORT = None  # Set when using a port other than the SSH default
 
 
@@ -22,7 +23,7 @@ LOGIN_PORT = None  # Set when using a port other than the SSH default
 #  DATABASE OPTIONS
 #==========================================================
 
-# Either Postgres or Oracle
+# Postgres, Oracle or Mysql
 DB_TYPE = 'postgres'
 
 # Name of the database
@@ -50,12 +51,19 @@ DB_CONF = '/etc/postgresql/9.6/main/postgresql.conf'
 DB_DUMP_DIR = '/var/lib/postgresql/9.6/main/dumpfiles'
 
 # Base config settings to always include when installing new configurations
+if DB_TYPE == 'mysql':
+    BASE_DB_CONF = {
+        'innodb_monitor_enable': 'all',
+    }
+elif DB_TYPE == 'postgres':
     BASE_DB_CONF = {
         'track_counts': 'on',
         'track_functions': 'all',
         'track_io_timing': 'on',
         'autovacuum': 'off',
     }
+else:
+    BASE_DB_CONF = None
 
 # Name of the device on the database server to monitor the disk usage, or None to disable
 DATABASE_DISK = None
@@ -88,7 +96,11 @@ RESULT_DIR = os.path.join(DRIVER_HOME, 'results')
 TEMP_DIR = '/tmp/driver'
 
 # Path to the directory for storing database dump files
+if DB_DUMP_DIR is None:
     DB_DUMP_DIR = os.path.join(DRIVER_HOME, 'dumpfiles')
+    if not os.path.exists(DB_DUMP_DIR):
+        os.mkdir(DB_DUMP_DIR)
+
 
 # Reload the database after running this many iterations
 RELOAD_INTERVAL = 10
@@ -102,7 +114,7 @@ MAX_DISK_USAGE = 90
 WARMUP_ITERATIONS = 0
 
 # Let the database initialize for this many seconds after it restarts
-RESTART_SLEEP_SEC = 300
+RESTART_SLEEP_SEC = 30
 
 #==========================================================
 #  OLTPBENCHMARK OPTIONS
@@ -123,7 +135,7 @@ OLTPBENCH_BENCH = 'tpcc'
 #==========================================================
 
 # Path to the controller directory
-CONTROLLER_HOME = os.path.expanduser('~/ottertune/client/controller')
+CONTROLLER_HOME = DRIVER_HOME + '/../controller'
 
 # Path to the controller configuration file
 CONTROLLER_CONFIG = os.path.join(CONTROLLER_HOME, 'config/postgres_config.json')

client/driver/fabfile.py

@@ -36,6 +36,7 @@ fabric_output.update({
 })
 env.abort_exception = FabricException
 env.hosts = [dconf.LOGIN]
+env.password = dconf.LOGIN_PASSWORD
 
 # Create local directories
 for _d in (dconf.RESULT_DIR, dconf.LOG_DIR, dconf.TEMP_DIR):
@@ -82,6 +83,8 @@ def create_controller_config():
         dburl_fmt = 'jdbc:postgresql://{host}:{port}/{db}'.format
     elif dconf.DB_TYPE == 'oracle':
         dburl_fmt = 'jdbc:oracle:thin:@{host}:{port}:{db}'.format
+    elif dconf.DB_TYPE == 'mysql':
+        dburl_fmt = 'jdbc:mysql://{host}:{port}/{db}?useSSL=false'.format
     else:
         raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
 
@@ -107,9 +110,18 @@ def restart_database():
             # becaues there's no init system running and the only process running
             # in the container is postgres itself
             local('docker restart {}'.format(dconf.CONTAINER_NAME))
+        elif dconf.HOST_CONN == 'remote_docker':
+            run('docker restart {}'.format(dconf.CONTAINER_NAME), remote_only=True)
         else:
             sudo('pg_ctl -D {} -w -t 600 restart -m fast'.format(
                 dconf.PG_DATADIR), user=dconf.ADMIN_USER, capture=False)
+    elif dconf.DB_TYPE == 'mysql':
+        if dconf.HOST_CONN == 'docker':
+            local('docker restart {}'.format(dconf.CONTAINER_NAME))
+        elif dconf.HOST_CONN == 'remote_docker':
+            run('docker restart {}'.format(dconf.CONTAINER_NAME), remote_only=True)
+        else:
+            sudo('service mysql restart')
     elif dconf.DB_TYPE == 'oracle':
         db_log_path = os.path.join(os.path.split(dconf.DB_CONF)[0], 'startup.log')
         local_log_path = os.path.join(dconf.LOG_DIR, 'startup.log')
@@ -135,6 +147,9 @@ def drop_database():
     if dconf.DB_TYPE == 'postgres':
         run("PGPASSWORD={} dropdb -e --if-exists {} -U {} -h {}".format(
             dconf.DB_PASSWORD, dconf.DB_NAME, dconf.DB_USER, dconf.DB_HOST))
+    elif dconf.DB_TYPE == 'mysql':
+        run("mysql --user={} --password={} -e 'drop database if exists {}'".format(
+            dconf.DB_USER, dconf.DB_PASSWORD, dconf.DB_NAME))
     else:
         raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
 
@@ -144,6 +159,9 @@ def create_database():
     if dconf.DB_TYPE == 'postgres':
         run("PGPASSWORD={} createdb -e {} -U {} -h {}".format(
             dconf.DB_PASSWORD, dconf.DB_NAME, dconf.DB_USER, dconf.DB_HOST))
+    elif dconf.DB_TYPE == 'mysql':
+        run("mysql --user={} --password={} -e 'create database {}'".format(
+            dconf.DB_USER, dconf.DB_PASSWORD, dconf.DB_NAME))
     else:
         raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
 
@@ -172,9 +190,20 @@ def drop_user():
         raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
 
 @task
-def reset_conf():
+def reset_conf(always=True):
+    if always:
         change_conf()
+        return
 
+    # reset the config only if it has not been changed by Ottertune,
+    # i.e. OtterTune signal line is not in the config file.
+    signal = "# configurations recommended by ottertune:\n"
+    tmp_conf_in = os.path.join(dconf.TEMP_DIR, os.path.basename(dconf.DB_CONF) + '.in')
+    get(dconf.DB_CONF, tmp_conf_in)
+    with open(tmp_conf_in, 'r') as f:
+        lines = f.readlines()
+    if signal not in lines:
+        change_conf()
 
 @task
 def change_conf(next_conf=None):
@@ -191,11 +220,15 @@ def change_conf(next_conf=None):
 
     signal_idx = lines.index(signal)
     lines = lines[0:signal_idx + 1]
+
+    if dconf.DB_TYPE == 'mysql':
+        lines.append('[mysqld]\n')
+
     if dconf.BASE_DB_CONF:
         assert isinstance(dconf.BASE_DB_CONF, dict), \
             (type(dconf.BASE_DB_CONF), dconf.BASE_DB_CONF)
-        base_conf = ['{} = {}\n'.format(*c) for c in sorted(dconf.BASE_DB_CONF.items())]
+        for name, value in sorted(dconf.BASE_DB_CONF.items()):
-        lines.extend(base_conf)
+            lines.append('{} = {}\n'.format(name, value))
 
     if isinstance(next_conf, str):
         with open(next_conf, 'r') as f:
@@ -209,6 +242,10 @@ def change_conf(next_conf=None):
     for name, value in recommendation.items():
         if dconf.DB_TYPE == 'oracle' and isinstance(value, str):
             value = value.strip('B')
+        # If innodb_flush_method is set to NULL on a Unix-like system,
+        # the fsync option is used by default.
+        if name == 'innodb_flush_method' and value == '':
+            value = "fsync"
         lines.append('{} = {}\n'.format(name, value))
     lines.append('\n')
 
@@ -223,6 +260,10 @@ def change_conf(next_conf=None):
 
 @task
 def load_oltpbench():
+    if os.path.exists(dconf.OLTPBENCH_CONFIG) is False:
+        msg = 'oltpbench config {} does not exist, '.format(dconf.OLTPBENCH_CONFIG)
+        msg += 'please double check the option in driver_config.py'
+        raise Exception(msg)
     cmd = "./oltpbenchmark -b {} -c {} --create=true --load=true".\
           format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG)
     with lcd(dconf.OLTPBENCH_HOME):  # pylint: disable=not-context-manager
@@ -231,6 +272,10 @@ def load_oltpbench():
 
 @task
 def run_oltpbench():
+    if os.path.exists(dconf.OLTPBENCH_CONFIG) is False:
+        msg = 'oltpbench config {} does not exist, '.format(dconf.OLTPBENCH_CONFIG)
+        msg += 'please double check the option in driver_config.py'
+        raise Exception(msg)
     cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile".\
           format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG)
     with lcd(dconf.OLTPBENCH_HOME):  # pylint: disable=not-context-manager
@@ -239,6 +284,10 @@ def run_oltpbench():
 
 @task
 def run_oltpbench_bg():
+    if os.path.exists(dconf.OLTPBENCH_CONFIG) is False:
+        msg = 'oltpbench config {} does not exist, '.format(dconf.OLTPBENCH_CONFIG)
+        msg += 'please double check the option in driver_config.py'
+        raise Exception(msg)
     cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile > {} 2>&1 &".\
           format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG, dconf.OLTPBENCH_LOG)
     with lcd(dconf.OLTPBENCH_HOME):  # pylint: disable=not-context-manager
@@ -247,7 +296,7 @@ def run_oltpbench_bg():
 
 @task
 def run_controller():
-    if not os.path.exists(dconf.CONTROLLER_CONFIG):
+    LOG.info('Controller config path: %s', dconf.CONTROLLER_CONFIG)
     create_controller_config()
     cmd = 'gradle run -PappArgs="-c {} -d output/" --no-daemon > {}'.\
           format(dconf.CONTROLLER_CONFIG, dconf.CONTROLLER_LOG)
@@ -287,9 +336,9 @@ def save_next_config(next_config, t=None):
 
 @task
 def free_cache():
-    if dconf.HOST_CONN != 'docker':
+    if dconf.HOST_CONN not in ['docker', 'remote_docker']:
         with show('everything'), settings(warn_only=True):  # pylint: disable=not-context-manager
-            res = sudo("sh -c \"echo 3 > /proc/sys/vm/drop_caches\"")
+            res = sudo("sh -c \"echo 3 > /proc/sys/vm/drop_caches\"", remote_only=True)
             if res.failed:
                 LOG.error('%s (return code %s)', res.stderr.strip(), res.return_code)
 
@@ -299,7 +348,6 @@ def upload_result(result_dir=None, prefix=None, upload_code=None):
     result_dir = result_dir or os.path.join(dconf.CONTROLLER_HOME, 'output')
     prefix = prefix or ''
     upload_code = upload_code or dconf.UPLOAD_CODE
-
     files = {}
     for base in ('summary', 'knobs', 'metrics_before', 'metrics_after'):
         fpath = os.path.join(result_dir, prefix + base + '.json')
@@ -451,7 +499,7 @@ def dump_database():
             LOG.info('%s already exists ! ', dumpfile)
             return False
 
-    if dconf.ORACLE_FLASH_BACK:
+    if dconf.DB_TYPE == 'oracle' and dconf.ORACLE_FLASH_BACK:
         LOG.info('create restore point %s for database %s in %s', dconf.RESTORE_POINT,
                  dconf.DB_NAME, dconf.RECOVERY_FILE_DEST)
     else:
@@ -469,6 +517,9 @@ def dump_database():
         run('PGPASSWORD={} pg_dump -U {} -h {} -F c -d {} > {}'.format(
             dconf.DB_PASSWORD, dconf.DB_USER, dconf.DB_HOST, dconf.DB_NAME,
             dumpfile))
+    elif dconf.DB_TYPE == 'mysql':
+        sudo('mysqldump --user={} --password={} --databases {} > {}'.format(
+            dconf.DB_USER, dconf.DB_PASSWORD, dconf.DB_NAME, dumpfile))
     else:
         raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
     return True
@@ -501,6 +552,8 @@ def restore_database():
         create_database()
         run('PGPASSWORD={} pg_restore -U {} -h {} -n public -j 8 -F c -d {} {}'.format(
             dconf.DB_PASSWORD, dconf.DB_USER, dconf.DB_HOST, dconf.DB_NAME, dumpfile))
+    elif dconf.DB_TYPE == 'mysql':
+        run('mysql --user={} --password={} < {}'.format(dconf.DB_USER, dconf.DB_PASSWORD, dumpfile))
     else:
         raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
     LOG.info('Finish restoring database')
@@ -603,7 +656,9 @@ def loop(i):
 def run_loops(max_iter=10):
     # dump database if it's not done before.
     dump = dump_database()
-
+    # put the BASE_DB_CONF in the config file
+    # e.g., mysql needs to set innodb_monitor_enable to track innodb metrics
+    reset_conf(False)
     for i in range(int(max_iter)):
         # restart database
         restart_succeeded = restart_database()
@@ -622,12 +677,14 @@ def run_loops(max_iter=10):
 
         # reload database periodically
         if dconf.RELOAD_INTERVAL > 0:
+            # wait 5 secs after restarting databases
+            time.sleep(5)
             if i % dconf.RELOAD_INTERVAL == 0:
                 if i == 0 and dump is False:
                     restore_database()
                 elif i > 0:
                     restore_database()
-
+        LOG.info('Wait %s seconds after restarting database', dconf.RESTART_SLEEP_SEC)
         time.sleep(dconf.RESTART_SLEEP_SEC)
         LOG.info('The %s-th Loop Starts / Total Loops %s', i + 1, max_iter)
         loop(i % dconf.RELOAD_INTERVAL if dconf.RELOAD_INTERVAL > 0 else i)

client/driver/utils.py

@@ -23,7 +23,7 @@ def load_driver_conf():
         if dconf.HOST_CONN == 'local':
             login_str = 'localhost'
 
-        elif dconf.HOST_CONN == 'remote':
+        elif dconf.HOST_CONN in ['remote', 'remote_docker']:
             if not dconf.LOGIN_HOST:
                 raise ValueError("LOGIN_HOST must be set if HOST_CONN=remote")
 
@@ -65,7 +65,7 @@ def get_content(response):
 
 
 @task
-def run(cmd, capture=True, **kwargs):
+def run(cmd, capture=True, remote_only=False, **kwargs):
     capture = parse_bool(capture)
 
     try:
@@ -73,14 +73,23 @@ def run(cmd, capture=True, **kwargs):
             res = _run(cmd, **kwargs)
         elif dconf.HOST_CONN == 'local':
             res = local(cmd, capture=capture, **kwargs)
-        else:  # docker
+        else:  # docker or remote_docker
             opts = ''
             cmdd = cmd
             if cmd.endswith('&'):
                 cmdd = cmd[:-1].strip()
                 opts = '-d '
-            res = local('docker exec {} -ti {} /bin/bash -c "{}"'.format(
+            if remote_only:
-                opts, dconf.CONTAINER_NAME, cmdd), capture=capture, **kwargs)
+                docker_cmd = cmdd
+            else:
+                docker_cmd = 'docker exec {} -ti {} /bin/bash -c "{}"'.format(
+                    opts, dconf.CONTAINER_NAME, cmdd)
+            if dconf.HOST_CONN == 'docker':
+                res = local(docker_cmd, capture=capture, **kwargs)
+            elif dconf.HOST_CONN == 'remote_docker':
+                res = _run(docker_cmd, **kwargs)
+            else:
+                raise Exception('wrong HOST_CONN type {}'.format(dconf.HOST_CONN))
     except TypeError as e:
         err = str(e).strip()
         if 'unexpected keyword argument' in err:
@@ -93,7 +102,7 @@ def run(cmd, capture=True, **kwargs):
 
 
 @task
-def sudo(cmd, user=None, capture=True, **kwargs):
+def sudo(cmd, user=None, capture=True, remote_only=False, **kwargs):
     capture = parse_bool(capture)
 
     if dconf.HOST_CONN == 'remote':
@@ -105,14 +114,22 @@ def sudo(cmd, user=None, capture=True, **kwargs):
             pre_cmd += '-u {} '.format(user)
         res = local(pre_cmd + cmd, capture=capture, **kwargs)
 
-    else:  # docker
+    else:  # docker or remote_docker
         user = user or 'root'
         opts = '-ti -u {}'.format(user or 'root')
         if user == 'root':
             opts += ' -w /'
-        res = local('docker exec {} {} /bin/bash -c "{}"'.format(
+        if remote_only:
-            opts, dconf.CONTAINER_NAME, cmd), capture=capture)
+            docker_cmd = cmd
-
+        else:
+            docker_cmd = 'docker exec {} {} /bin/bash -c "{}"'.format(
+                opts, dconf.CONTAINER_NAME, cmd)
+        if dconf.HOST_CONN == 'docker':
+            res = local(docker_cmd, capture=capture, **kwargs)
+        elif dconf.HOST_CONN == 'remote_docker':
+            res = _sudo(docker_cmd, **kwargs)
+        else:
+            raise Exception('wrong HOST_CONN type {}'.format(dconf.HOST_CONN))
     return res
 
 
@@ -126,8 +143,15 @@ def get(remote_path, local_path, use_sudo=False):
         pre_cmd = 'sudo ' if use_sudo else ''
         opts = '-r' if os.path.isdir(remote_path) else ''
         res = local('{}cp {} {} {}'.format(pre_cmd, opts, remote_path, local_path))
-    else:  # docker
+    else:  # docker or remote_docker
-        res = local('docker cp {}:{} {}'.format(dconf.CONTAINER_NAME, remote_path, local_path))
+        docker_cmd = 'docker cp -L {}:{} {}'.format(dconf.CONTAINER_NAME, remote_path, local_path)
+        if dconf.HOST_CONN == 'docker':
+            res = local(docker_cmd)
+        elif dconf.HOST_CONN == 'remote_docker':
+            res = sudo(docker_cmd, remote_only=True)
+            res = _get(local_path, local_path, use_sudo)
+        else:
+            raise Exception('wrong HOST_CONN type {}'.format(dconf.HOST_CONN))
     return res
 
 
@@ -141,8 +165,15 @@ def put(local_path, remote_path, use_sudo=False):
         pre_cmd = 'sudo ' if use_sudo else ''
         opts = '-r' if os.path.isdir(local_path) else ''
         res = local('{}cp {} {} {}'.format(pre_cmd, opts, local_path, remote_path))
-    else:  # docker
+    else:  # docker or remote_docker
-        res = local('docker cp {} {}:{}'.format(local_path, dconf.CONTAINER_NAME, remote_path))
+        docker_cmd = 'docker cp -L {} {}:{}'.format(local_path, dconf.CONTAINER_NAME, remote_path)
+        if dconf.HOST_CONN == 'docker':
+            res = local(docker_cmd)
+        elif dconf.HOST_CONN == 'remote_docker':
+            res = _put(local_path, local_path, use_sudo=True)
+            res = sudo(docker_cmd, remote_only=True)
+        else:
+            raise Exception('wrong HOST_CONN type {}'.format(dconf.HOST_CONN))
     return res
 
 

server/website/website/db/base/parser.py

@@ -89,8 +89,11 @@ class BaseParser:
         return converted
 
     def convert_real(self, real_value, metadata):
+        try:
             return float(real_value)
-
+        except ValueError:
+            raise Exception('Cannot convert knob {} from {} to float'.format(
+                metadata.name, real_value))
     def convert_string(self, string_value, metadata):
         return string_value
 
@@ -124,10 +127,10 @@ class BaseParser:
 
             if metadata.vartype == VarType.BOOL:
                 if not self._check_knob_bool_val(value):
-                    raise Exception('Knob boolean value not valid! '
+                    raise Exception('Knob {} boolean value not valid! '
                                     'Boolean values should be one of: {}, '
                                     'but the actual value is: {}'
-                                    .format(self.valid_boolean_val_to_string(),
+                                    .format(name, self.valid_boolean_val_to_string(),
                                             str(value)))
                 conv_value = self.convert_bool(value, metadata)
 
@@ -137,17 +140,17 @@ class BaseParser:
             elif metadata.vartype == VarType.INTEGER:
                 conv_value = self.convert_integer(value, metadata)
                 if not self._check_knob_num_in_range(conv_value, metadata):
-                    raise Exception('Knob integer num value not in range! '
+                    raise Exception('Knob {} integer num value not in range! '
                                     'min: {}, max: {}, actual: {}'
-                                    .format(metadata.minval,
+                                    .format(name, metadata.minval,
                                             metadata.maxval, str(conv_value)))
 
             elif metadata.vartype == VarType.REAL:
                 conv_value = self.convert_real(value, metadata)
                 if not self._check_knob_num_in_range(conv_value, metadata):
-                    raise Exception('Knob real num value not in range! '
+                    raise Exception('Knob {} real num value not in range! '
                                     'min: {}, max: {}, actual: {}'
-                                    .format(metadata.minval,
+                                    .format(name, metadata.minval,
                                             metadata.maxval, str(conv_value)))
 
             elif metadata.vartype == VarType.STRING:
@@ -328,7 +331,8 @@ class BaseParser:
                     'Invalid metric type: {}'.format(metric.metric_type))
         return valid_metrics, diffs
 
-    def calculate_change_in_metrics(self, metrics_start, metrics_end, fix_metric_type=True):
+    def calculate_change_in_metrics(self, metrics_start, metrics_end,
+                                    fix_metric_type=True, allow_negative=True):
         metric_catalog = {m.name: m for m in MetricCatalog.objects.filter(dbms__id=self.dbms_id)}
         adjusted_metrics = {}
 
@@ -350,9 +354,14 @@ class BaseParser:
                 if fix_metric_type:
                     if adj_val < 0:
                         adj_val = end_val
-                        LOG.debug("Changing metric %s from COUNTER to STATISTICS", met_name)
+                        LOG.warning("Changing metric %s from COUNTER to STATISTICS", met_name)
                         met_info.metric_type = MetricType.STATISTICS
                         met_info.save()
+                if allow_negative:
+                    LOG.warning('%s metric type %s value is negative (start=%s, end=%s, diff=%s)',
+                                met_name, MetricType.name(met_info.metric_type), start_val, end_val,
+                                end_val - start_val)
+                else:
                     assert adj_val >= 0, \
                         '{} wrong metric type: {} (start={}, end={}, diff={})'.format(
                             met_name, MetricType.name(met_info.metric_type), start_val,

server/website/website/db/base/target_objective.py

@@ -87,10 +87,11 @@ class TargetObjectives:
         from ..myrocks.target_objective import target_objective_list as _myrocks_list  # pylint: disable=import-outside-toplevel
         from ..oracle.target_objective import target_objective_list as _oracle_list  # pylint: disable=import-outside-toplevel
         from ..postgres.target_objective import target_objective_list as _postgres_list  # pylint: disable=import-outside-toplevel
+        from ..mysql.target_objective import target_objective_list as _mysql_list  # pylint: disable=import-outside-toplevel
 
         if not self.registered():
             LOG.info('Registering target objectives...')
-            full_list = _myrocks_list + _oracle_list + _postgres_list
+            full_list = _myrocks_list + _oracle_list + _postgres_list + _mysql_list
             for dbms_type, target_objective_instance in full_list:
                 dbmss = models.DBMSCatalog.objects.filter(type=dbms_type)
                 name = target_objective_instance.name

server/website/website/db/mysql/parser.py

@@ -0,0 +1,53 @@
+#
+# OtterTune - parser.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+
+from website.types import KnobUnitType
+from website.utils import ConversionUtil
+from ..base.parser import BaseParser  # pylint: disable=relative-beyond-top-level
+
+
+# pylint: disable=no-self-use
+class MysqlParser(BaseParser):
+
+    def __init__(self, dbms_obj):
+        super().__init__(dbms_obj)
+        self.bytes_system = (
+            (1024 ** 4, 'T'),
+            (1024 ** 3, 'G'),
+            (1024 ** 2, 'M'),
+            (1024 ** 1, 'k'),
+        )
+        self.time_system = None
+        self.min_bytes_unit = 'k'
+        self.valid_true_val = ("on", "true", "yes", '1', 'enabled')
+        self.valid_false_val = ("off", "false", "no", '0', 'disabled')
+
+    def convert_integer(self, int_value, metadata):
+        # Collected knobs/metrics do not show unit, convert to int directly
+        if len(str(int_value)) == 0:
+            # The value collected from the database is empty
+            return 0
+        try:
+            try:
+                converted = int(int_value)
+            except ValueError:
+                converted = int(float(int_value))
+
+        except ValueError:
+            raise Exception('Invalid integer format for {}: {}'.format(
+                metadata.name, int_value))
+        return converted
+
+    def format_integer(self, int_value, metadata):
+        int_value = int(round(int_value))
+        if int_value > 0 and metadata.unit == KnobUnitType.BYTES:
+            int_value = ConversionUtil.get_human_readable2(
+                int_value, self.bytes_system, self.min_bytes_unit)
+        return int_value
+
+    def parse_version_string(self, version_string):
+        s = version_string.split('.')[0] + '.' + version_string.split('.')[1]
+        return s

server/website/website/db/mysql/target_objective.py

@@ -0,0 +1,14 @@
+#
+# OtterTune - target_objective.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+
+from website.types import DBMSType
+from ..base.target_objective import BaseThroughput  # pylint: disable=relative-beyond-top-level
+
+target_objective_list = tuple((DBMSType.MYSQL, target_obj) for target_obj in [  # pylint: disable=invalid-name
+    BaseThroughput(transactions_counter=('innodb_metrics.trx_rw_commits',
+                                         'innodb_metrics.trx_ro_commits',
+                                         'innodb_metrics.trx_nl_ro_commits'))
+])

server/website/website/db/parser.py

@@ -10,6 +10,7 @@ from website.types import DBMSType
 from .myrocks.parser import MyRocksParser
 from .postgres.parser import PostgresParser
 from .oracle.parser import OracleParser
+from .mysql.parser import MysqlParser
 
 _DBMS_PARSERS = {}
 
@@ -25,6 +26,8 @@ def _get(dbms_id):
             clz = MyRocksParser
         elif obj.type == DBMSType.ORACLE:
             clz = OracleParser
+        elif obj.type == DBMSType.MYSQL:
+            clz = MysqlParser
         else:
             raise NotImplementedError('Implement me! {}'.format(obj))
 

server/website/website/fixtures/dbms_catalog.json

@@ -39,6 +39,30 @@
         "version":"5.6"
      }
   },
+  {
+    "model":"website.DBMSCatalog",
+    "pk":11,
+    "fields":{
+        "type":1,
+        "version":"5.6"
+     }
+  },
+  {
+    "model":"website.DBMSCatalog",
+    "pk":12,
+    "fields":{
+        "type":1,
+        "version":"5.7"
+     }
+  },
+    {
+    "model":"website.DBMSCatalog",
+    "pk":13,
+    "fields":{
+        "type":1,
+        "version":"8.0"
+     }
+  },
   {
     "model":"website.DBMSCatalog",
     "pk":121,

server/website/website/migrations/0001_initial.py

@@ -59,7 +59,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('name', models.CharField(max_length=128)),
                 ('vartype', models.IntegerField(choices=[(1, 'STRING'), (2, 'INTEGER'), (3, 'REAL'), (4, 'BOOL'), (5, 'ENUM'), (6, 'TIMESTAMP')], verbose_name='variable type')),
-                ('unit', models.IntegerField(choices=[(1, 'bytes'), (2, 'milliseconds'), (3, 'other')])),
+                ('unit', models.IntegerField(choices=[(1, 'bytes'), (2, 'milliseconds'), (3, 'other'), (4, 'microseconds'), (5, 'seconds')])),
                 ('category', models.TextField(null=True)),
                 ('summary', models.TextField(null=True, verbose_name='description')),
                 ('description', models.TextField(null=True)),

server/website/website/migrations/0003_load_initial_data.py

@@ -25,6 +25,8 @@ def load_initial_data(apps, schema_editor):
         "oracle-121_metrics.json",
         "oracle-19_knobs.json",
         "oracle-19_metrics.json",
+        "mysql-56_knobs.json",
+        "mysql-56_metrics.json",
     ]
     for fixture in initial_data_fixtures:
         call_command("loaddata", fixture, app_label="website")

server/website/website/set_default_knobs.py

@@ -35,6 +35,17 @@ DEFAULT_TUNABLE_KNOBS = {
         "global.shared_pool_size",
         "global.sort_area_size",
     },
+    DBMSType.MYSQL: {
+        "global.innodb_buffer_pool_size",
+        "global.innodb_thread_sleep_delay",
+        "global.innodb_flush_method",
+        "global.innodb_log_file_size",
+        "global.innodb_max_dirty_pages_pct_lwm",
+        "global.innodb_read_ahead_threshold",
+        "global.innodb_adaptive_max_sleep_delay",
+        "global.innodb_buffer_pool_instances",
+        "global.thread_cache_size",
+    },
 }
 
 # Bytes in a GB

server/website/website/types.py

@@ -130,11 +130,15 @@ class KnobUnitType(BaseType):
     BYTES = 1
     MILLISECONDS = 2
     OTHER = 3
+    MICROSECONDS = 4
+    SECONDS = 5
 
     TYPE_NAMES = {
         BYTES: 'bytes',
         MILLISECONDS: 'milliseconds',
         OTHER: 'other',
+        MICROSECONDS: 'microseconds',
+        SECONDS: 'seconds',
     }
 
 

server/website/website/views.py

@@ -606,6 +606,17 @@ def handle_result_files(session, files, execution_times=None):
                                 'invalid characters! It should only contain '
                                 'alpha-numeric, underscore(_) and hyphen(-)')
 
+        try:
+            # Check that we support this DBMS and version
+            dbms = DBMSCatalog.objects.get(
+                type=dbms_type, version=dbms_version)
+        except ObjectDoesNotExist:
+            try:
+                dbms_version = parser.parse_version_string(dbms_type, dbms_version)
+            except Exception:  # pylint: disable=broad-except
+                LOG.warning('Cannot parse dbms version %s', dbms_version)
+                return HttpResponse('{} v{} is not yet supported.'.format(
+                    dbms_type, dbms_version))
             try:
                 # Check that we support this DBMS and version
                 dbms = DBMSCatalog.objects.get(