public/ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

support mysql

Browse Source
This commit is contained in:
bohanjason
2020-04-23 03:46:58 -04:00
committed by Dana Van Aken
parent 5b697b9c56
commit 5c422dd010
18 changed files with 13326 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client/controller/build.gradle
View File

@@ -45,7 +45,7 @@ dependencies {
compile group: 'commons-cli', name: 'commons-cli', version: '1.2'
// https://mvnrepository.com/artifact/mysql/mysql-connector-java
compile group: 'mysql', name: 'mysql-connector-java', version: '5.1.6'
compile group: 'mysql', name: 'mysql-connector-java', version: '8.0.12'
// https://mvnrepository.com/artifact/org.postgresql/postgresql
compile group: 'org.postgresql', name: 'postgresql', version: '9.4-1201-jdbc41'

18
client/controller/src/main/java/com/controller/collectors/MySQLCollector.java
View File

@@ -16,6 +16,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import java.util.HashMap;
import org.apache.log4j.Logger;
/** */
@@ -27,6 +28,10 @@ public class MySQLCollector extends DBCollector {
private static final String PARAMETERS_SQL = "SHOW VARIABLES;";
private static final String METRICS_SQL = "SHOW STATUS";
private static final String METRICS_SQL2 =
"SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS where subsystem = 'transaction';";
private HashMap<String, String> innodbMetrics = new HashMap<>();
public MySQLCollector(String oriDBUrl, String username, String password) {
try {
@@ -50,6 +55,12 @@ public class MySQLCollector extends DBCollector {
while (out.next()) {
dbMetrics.put(out.getString(1).toLowerCase(), out.getString(2));
}
out = s.executeQuery(METRICS_SQL2);
while (out.next()) {
innodbMetrics.put(out.getString(1).toLowerCase(), out.getString(2));
}
conn.close();
} catch (SQLException e) {
LOG.error("Error while collecting DB parameters: " + e.getMessage());
@@ -93,6 +104,13 @@ public class MySQLCollector extends DBCollector {
}
// "global" is a a placeholder
jobGlobal.put("global", job);
JSONObject job2 = new JSONObject();
for (Map.Entry<String, String> entry : innodbMetrics.entrySet()) {
job2.put(entry.getKey(), entry.getValue());
}
jobGlobal.put("innodb_metrics", job2);
stringer.value(jobGlobal);
stringer.key(JSON_LOCAL_KEY);
stringer.value(null);

34
client/driver/driver_config.py
View File

@@ -5,7 +5,7 @@ import os
#==========================================================
# Location of the database host relative to this driver
# Valid values: local, remote, or docker
# Valid values: local, remote, docker or remote_docker
HOST_CONN = 'local'
# The name of the Docker container for the target database
@@ -15,6 +15,7 @@ CONTAINER_NAME = None # e.g., 'postgres_container'
# Host SSH login credentials (only required if HOST_CONN=remote)
LOGIN_NAME = None
LOGIN_HOST = None
LOGIN_PASSWORD = None
LOGIN_PORT = None # Set when using a port other than the SSH default
@@ -22,7 +23,7 @@ LOGIN_PORT = None # Set when using a port other than the SSH default
# DATABASE OPTIONS
#==========================================================
# Either Postgres or Oracle
# Postgres, Oracle or Mysql
DB_TYPE = 'postgres'
# Name of the database
@@ -50,12 +51,19 @@ DB_CONF = '/etc/postgresql/9.6/main/postgresql.conf'
DB_DUMP_DIR = '/var/lib/postgresql/9.6/main/dumpfiles'
# Base config settings to always include when installing new configurations
BASE_DB_CONF = {
'track_counts': 'on',
'track_functions': 'all',
'track_io_timing': 'on',
'autovacuum': 'off',
}
if DB_TYPE == 'mysql':
BASE_DB_CONF = {
'innodb_monitor_enable': 'all',
}
elif DB_TYPE == 'postgres':
BASE_DB_CONF = {
'track_counts': 'on',
'track_functions': 'all',
'track_io_timing': 'on',
'autovacuum': 'off',
}
else:
BASE_DB_CONF = None
# Name of the device on the database server to monitor the disk usage, or None to disable
DATABASE_DISK = None
@@ -88,7 +96,11 @@ RESULT_DIR = os.path.join(DRIVER_HOME, 'results')
TEMP_DIR = '/tmp/driver'
# Path to the directory for storing database dump files
DB_DUMP_DIR = os.path.join(DRIVER_HOME, 'dumpfiles')
if DB_DUMP_DIR is None:
DB_DUMP_DIR = os.path.join(DRIVER_HOME, 'dumpfiles')
if not os.path.exists(DB_DUMP_DIR):
os.mkdir(DB_DUMP_DIR)
# Reload the database after running this many iterations
RELOAD_INTERVAL = 10
@@ -102,7 +114,7 @@ MAX_DISK_USAGE = 90
WARMUP_ITERATIONS = 0
# Let the database initialize for this many seconds after it restarts
RESTART_SLEEP_SEC = 300
RESTART_SLEEP_SEC = 30
#==========================================================
# OLTPBENCHMARK OPTIONS
@@ -123,7 +135,7 @@ OLTPBENCH_BENCH = 'tpcc'
#==========================================================
# Path to the controller directory
CONTROLLER_HOME = os.path.expanduser('~/ottertune/client/controller')
CONTROLLER_HOME = DRIVER_HOME + '/../controller'
# Path to the controller configuration file
CONTROLLER_CONFIG = os.path.join(CONTROLLER_HOME, 'config/postgres_config.json')

81
client/driver/fabfile.py vendored
View File

@@ -36,6 +36,7 @@ fabric_output.update({
})
env.abort_exception = FabricException
env.hosts = [dconf.LOGIN]
env.password = dconf.LOGIN_PASSWORD
# Create local directories
for _d in (dconf.RESULT_DIR, dconf.LOG_DIR, dconf.TEMP_DIR):
@@ -82,6 +83,8 @@ def create_controller_config():
dburl_fmt = 'jdbc:postgresql://{host}:{port}/{db}'.format
elif dconf.DB_TYPE == 'oracle':
dburl_fmt = 'jdbc:oracle:thin:@{host}:{port}:{db}'.format
elif dconf.DB_TYPE == 'mysql':
dburl_fmt = 'jdbc:mysql://{host}:{port}/{db}?useSSL=false'.format
else:
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
@@ -107,9 +110,18 @@ def restart_database():
# becaues there's no init system running and the only process running
# in the container is postgres itself
local('docker restart {}'.format(dconf.CONTAINER_NAME))
elif dconf.HOST_CONN == 'remote_docker':
run('docker restart {}'.format(dconf.CONTAINER_NAME), remote_only=True)
else:
sudo('pg_ctl -D {} -w -t 600 restart -m fast'.format(
dconf.PG_DATADIR), user=dconf.ADMIN_USER, capture=False)
elif dconf.DB_TYPE == 'mysql':
if dconf.HOST_CONN == 'docker':
local('docker restart {}'.format(dconf.CONTAINER_NAME))
elif dconf.HOST_CONN == 'remote_docker':
run('docker restart {}'.format(dconf.CONTAINER_NAME), remote_only=True)
else:
sudo('service mysql restart')
elif dconf.DB_TYPE == 'oracle':
db_log_path = os.path.join(os.path.split(dconf.DB_CONF)[0], 'startup.log')
local_log_path = os.path.join(dconf.LOG_DIR, 'startup.log')
@@ -135,6 +147,9 @@ def drop_database():
if dconf.DB_TYPE == 'postgres':
run("PGPASSWORD={} dropdb -e --if-exists {} -U {} -h {}".format(
dconf.DB_PASSWORD, dconf.DB_NAME, dconf.DB_USER, dconf.DB_HOST))
elif dconf.DB_TYPE == 'mysql':
run("mysql --user={} --password={} -e 'drop database if exists {}'".format(
dconf.DB_USER, dconf.DB_PASSWORD, dconf.DB_NAME))
else:
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
@@ -144,6 +159,9 @@ def create_database():
if dconf.DB_TYPE == 'postgres':
run("PGPASSWORD={} createdb -e {} -U {} -h {}".format(
dconf.DB_PASSWORD, dconf.DB_NAME, dconf.DB_USER, dconf.DB_HOST))
elif dconf.DB_TYPE == 'mysql':
run("mysql --user={} --password={} -e 'create database {}'".format(
dconf.DB_USER, dconf.DB_PASSWORD, dconf.DB_NAME))
else:
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
@@ -172,9 +190,20 @@ def drop_user():
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
@task
def reset_conf():
change_conf()
def reset_conf(always=True):
if always:
change_conf()
return
# reset the config only if it has not been changed by Ottertune,
# i.e. OtterTune signal line is not in the config file.
signal = "# configurations recommended by ottertune:\n"
tmp_conf_in = os.path.join(dconf.TEMP_DIR, os.path.basename(dconf.DB_CONF) + '.in')
get(dconf.DB_CONF, tmp_conf_in)
with open(tmp_conf_in, 'r') as f:
lines = f.readlines()
if signal not in lines:
change_conf()
@task
def change_conf(next_conf=None):
@@ -191,11 +220,15 @@ def change_conf(next_conf=None):
signal_idx = lines.index(signal)
lines = lines[0:signal_idx + 1]
if dconf.DB_TYPE == 'mysql':
lines.append('[mysqld]\n')
if dconf.BASE_DB_CONF:
assert isinstance(dconf.BASE_DB_CONF, dict), \
(type(dconf.BASE_DB_CONF), dconf.BASE_DB_CONF)
base_conf = ['{} = {}\n'.format(*c) for c in sorted(dconf.BASE_DB_CONF.items())]
lines.extend(base_conf)
for name, value in sorted(dconf.BASE_DB_CONF.items()):
lines.append('{} = {}\n'.format(name, value))
if isinstance(next_conf, str):
with open(next_conf, 'r') as f:
@@ -209,6 +242,10 @@ def change_conf(next_conf=None):
for name, value in recommendation.items():
if dconf.DB_TYPE == 'oracle' and isinstance(value, str):
value = value.strip('B')
# If innodb_flush_method is set to NULL on a Unix-like system,
# the fsync option is used by default.
if name == 'innodb_flush_method' and value == '':
value = "fsync"
lines.append('{} = {}\n'.format(name, value))
lines.append('\n')
@@ -223,6 +260,10 @@ def change_conf(next_conf=None):
@task
def load_oltpbench():
if os.path.exists(dconf.OLTPBENCH_CONFIG) is False:
msg = 'oltpbench config {} does not exist, '.format(dconf.OLTPBENCH_CONFIG)
msg += 'please double check the option in driver_config.py'
raise Exception(msg)
cmd = "./oltpbenchmark -b {} -c {} --create=true --load=true".\
format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG)
with lcd(dconf.OLTPBENCH_HOME): # pylint: disable=not-context-manager
@@ -231,6 +272,10 @@ def load_oltpbench():
@task
def run_oltpbench():
if os.path.exists(dconf.OLTPBENCH_CONFIG) is False:
msg = 'oltpbench config {} does not exist, '.format(dconf.OLTPBENCH_CONFIG)
msg += 'please double check the option in driver_config.py'
raise Exception(msg)
cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile".\
format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG)
with lcd(dconf.OLTPBENCH_HOME): # pylint: disable=not-context-manager
@@ -239,6 +284,10 @@ def run_oltpbench():
@task
def run_oltpbench_bg():
if os.path.exists(dconf.OLTPBENCH_CONFIG) is False:
msg = 'oltpbench config {} does not exist, '.format(dconf.OLTPBENCH_CONFIG)
msg += 'please double check the option in driver_config.py'
raise Exception(msg)
cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile > {} 2>&1 &".\
format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG, dconf.OLTPBENCH_LOG)
with lcd(dconf.OLTPBENCH_HOME): # pylint: disable=not-context-manager
@@ -247,8 +296,8 @@ def run_oltpbench_bg():
@task
def run_controller():
if not os.path.exists(dconf.CONTROLLER_CONFIG):
create_controller_config()
LOG.info('Controller config path: %s', dconf.CONTROLLER_CONFIG)
create_controller_config()
cmd = 'gradle run -PappArgs="-c {} -d output/" --no-daemon > {}'.\
format(dconf.CONTROLLER_CONFIG, dconf.CONTROLLER_LOG)
with lcd(dconf.CONTROLLER_HOME): # pylint: disable=not-context-manager
@@ -287,9 +336,9 @@ def save_next_config(next_config, t=None):
@task
def free_cache():
if dconf.HOST_CONN != 'docker':
if dconf.HOST_CONN not in ['docker', 'remote_docker']:
with show('everything'), settings(warn_only=True): # pylint: disable=not-context-manager
res = sudo("sh -c \"echo 3 > /proc/sys/vm/drop_caches\"")
res = sudo("sh -c \"echo 3 > /proc/sys/vm/drop_caches\"", remote_only=True)
if res.failed:
LOG.error('%s (return code %s)', res.stderr.strip(), res.return_code)
@@ -299,7 +348,6 @@ def upload_result(result_dir=None, prefix=None, upload_code=None):
result_dir = result_dir or os.path.join(dconf.CONTROLLER_HOME, 'output')
prefix = prefix or ''
upload_code = upload_code or dconf.UPLOAD_CODE
files = {}
for base in ('summary', 'knobs', 'metrics_before', 'metrics_after'):
fpath = os.path.join(result_dir, prefix + base + '.json')
@@ -451,7 +499,7 @@ def dump_database():
LOG.info('%s already exists ! ', dumpfile)
return False
if dconf.ORACLE_FLASH_BACK:
if dconf.DB_TYPE == 'oracle' and dconf.ORACLE_FLASH_BACK:
LOG.info('create restore point %s for database %s in %s', dconf.RESTORE_POINT,
dconf.DB_NAME, dconf.RECOVERY_FILE_DEST)
else:
@@ -469,6 +517,9 @@ def dump_database():
run('PGPASSWORD={} pg_dump -U {} -h {} -F c -d {} > {}'.format(
dconf.DB_PASSWORD, dconf.DB_USER, dconf.DB_HOST, dconf.DB_NAME,
dumpfile))
elif dconf.DB_TYPE == 'mysql':
sudo('mysqldump --user={} --password={} --databases {} > {}'.format(
dconf.DB_USER, dconf.DB_PASSWORD, dconf.DB_NAME, dumpfile))
else:
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
return True
@@ -501,6 +552,8 @@ def restore_database():
create_database()
run('PGPASSWORD={} pg_restore -U {} -h {} -n public -j 8 -F c -d {} {}'.format(
dconf.DB_PASSWORD, dconf.DB_USER, dconf.DB_HOST, dconf.DB_NAME, dumpfile))
elif dconf.DB_TYPE == 'mysql':
run('mysql --user={} --password={} < {}'.format(dconf.DB_USER, dconf.DB_PASSWORD, dumpfile))
else:
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
LOG.info('Finish restoring database')
@@ -603,7 +656,9 @@ def loop(i):
def run_loops(max_iter=10):
# dump database if it's not done before.
dump = dump_database()
# put the BASE_DB_CONF in the config file
# e.g., mysql needs to set innodb_monitor_enable to track innodb metrics
reset_conf(False)
for i in range(int(max_iter)):
# restart database
restart_succeeded = restart_database()
@@ -622,12 +677,14 @@ def run_loops(max_iter=10):
# reload database periodically
if dconf.RELOAD_INTERVAL > 0:
# wait 5 secs after restarting databases
time.sleep(5)
if i % dconf.RELOAD_INTERVAL == 0:
if i == 0 and dump is False:
restore_database()
elif i > 0:
restore_database()
LOG.info('Wait %s seconds after restarting database', dconf.RESTART_SLEEP_SEC)
time.sleep(dconf.RESTART_SLEEP_SEC)
LOG.info('The %s-th Loop Starts / Total Loops %s', i + 1, max_iter)
loop(i % dconf.RELOAD_INTERVAL if dconf.RELOAD_INTERVAL > 0 else i)

59
client/driver/utils.py
View File

@@ -23,7 +23,7 @@ def load_driver_conf():
if dconf.HOST_CONN == 'local':
login_str = 'localhost'
elif dconf.HOST_CONN == 'remote':
elif dconf.HOST_CONN in ['remote', 'remote_docker']:
if not dconf.LOGIN_HOST:
raise ValueError("LOGIN_HOST must be set if HOST_CONN=remote")
@@ -65,7 +65,7 @@ def get_content(response):
@task
def run(cmd, capture=True, **kwargs):
def run(cmd, capture=True, remote_only=False, **kwargs):
capture = parse_bool(capture)
try:
@@ -73,14 +73,23 @@ def run(cmd, capture=True, **kwargs):
res = _run(cmd, **kwargs)
elif dconf.HOST_CONN == 'local':
res = local(cmd, capture=capture, **kwargs)
else: # docker
else: # docker or remote_docker
opts = ''
cmdd = cmd
if cmd.endswith('&'):
cmdd = cmd[:-1].strip()
opts = '-d '
res = local('docker exec {} -ti {} /bin/bash -c "{}"'.format(
opts, dconf.CONTAINER_NAME, cmdd), capture=capture, **kwargs)
if remote_only:
docker_cmd = cmdd
else:
docker_cmd = 'docker exec {} -ti {} /bin/bash -c "{}"'.format(
opts, dconf.CONTAINER_NAME, cmdd)
if dconf.HOST_CONN == 'docker':
res = local(docker_cmd, capture=capture, **kwargs)
elif dconf.HOST_CONN == 'remote_docker':
res = _run(docker_cmd, **kwargs)
else:
raise Exception('wrong HOST_CONN type {}'.format(dconf.HOST_CONN))
except TypeError as e:
err = str(e).strip()
if 'unexpected keyword argument' in err:
@@ -93,7 +102,7 @@ def run(cmd, capture=True, **kwargs):
@task
def sudo(cmd, user=None, capture=True, **kwargs):
def sudo(cmd, user=None, capture=True, remote_only=False, **kwargs):
capture = parse_bool(capture)
if dconf.HOST_CONN == 'remote':
@@ -105,14 +114,22 @@ def sudo(cmd, user=None, capture=True, **kwargs):
pre_cmd += '-u {} '.format(user)
res = local(pre_cmd + cmd, capture=capture, **kwargs)
else: # docker
else: # docker or remote_docker
user = user or 'root'
opts = '-ti -u {}'.format(user or 'root')
if user == 'root':
opts += ' -w /'
res = local('docker exec {} {} /bin/bash -c "{}"'.format(
opts, dconf.CONTAINER_NAME, cmd), capture=capture)
if remote_only:
docker_cmd = cmd
else:
docker_cmd = 'docker exec {} {} /bin/bash -c "{}"'.format(
opts, dconf.CONTAINER_NAME, cmd)
if dconf.HOST_CONN == 'docker':
res = local(docker_cmd, capture=capture, **kwargs)
elif dconf.HOST_CONN == 'remote_docker':
res = _sudo(docker_cmd, **kwargs)
else:
raise Exception('wrong HOST_CONN type {}'.format(dconf.HOST_CONN))
return res
@@ -126,8 +143,15 @@ def get(remote_path, local_path, use_sudo=False):
pre_cmd = 'sudo ' if use_sudo else ''
opts = '-r' if os.path.isdir(remote_path) else ''
res = local('{}cp {} {} {}'.format(pre_cmd, opts, remote_path, local_path))
else: # docker
res = local('docker cp {}:{} {}'.format(dconf.CONTAINER_NAME, remote_path, local_path))
else: # docker or remote_docker
docker_cmd = 'docker cp -L {}:{} {}'.format(dconf.CONTAINER_NAME, remote_path, local_path)
if dconf.HOST_CONN == 'docker':
res = local(docker_cmd)
elif dconf.HOST_CONN == 'remote_docker':
res = sudo(docker_cmd, remote_only=True)
res = _get(local_path, local_path, use_sudo)
else:
raise Exception('wrong HOST_CONN type {}'.format(dconf.HOST_CONN))
return res
@@ -141,8 +165,15 @@ def put(local_path, remote_path, use_sudo=False):
pre_cmd = 'sudo ' if use_sudo else ''
opts = '-r' if os.path.isdir(local_path) else ''
res = local('{}cp {} {} {}'.format(pre_cmd, opts, local_path, remote_path))
else: # docker
res = local('docker cp {} {}:{}'.format(local_path, dconf.CONTAINER_NAME, remote_path))
else: # docker or remote_docker
docker_cmd = 'docker cp -L {} {}:{}'.format(local_path, dconf.CONTAINER_NAME, remote_path)
if dconf.HOST_CONN == 'docker':
res = local(docker_cmd)
elif dconf.HOST_CONN == 'remote_docker':
res = _put(local_path, local_path, use_sudo=True)
res = sudo(docker_cmd, remote_only=True)
else:
raise Exception('wrong HOST_CONN type {}'.format(dconf.HOST_CONN))
return res