diff --git a/client/driver/configs/postgres_base.conf b/client/driver/configs/postgres_base.conf deleted file mode 100644 index c57b0c9..0000000 --- a/client/driver/configs/postgres_base.conf +++ /dev/null @@ -1,4 +0,0 @@ -track_counts = on -track_functions = all -track_io_timing = on -autovacuum = off diff --git a/client/driver/driver_config.json b/client/driver/driver_config.json deleted file mode 100644 index ee4d568..0000000 --- a/client/driver/driver_config.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "database_type" : "postgres", - "database_name" : "tpcc", - "database_disk": "", - "database_conf": "/etc/postgresql/9.6/main/postgresql.conf", - "base_database_conf": "configs/postgres_base.conf", - "pg_datadir": "/var/lib/postgresql/9.6/main", - "database_save_path": "~/ottertune/client/driver/dumpfiles", - "username" : "dbuser", - "password" : "dbpassword", - "oltpbench_home": "~/oltpbench", - "oltpbench_config": "~/oltpbench/config/tpcc_config_postgres.xml", - "oltpbench_workload": "tpcc", - "controller_home": "~/ottertune/client/controller", - "log_path": "~/ottertune/client/driver/log", - "save_path": "~/ottertune/client/driver/results", - "upload_url" : "http://127.0.0.1:8000", - "upload_code" : "I5I10PXK3PK27FM86YYS", - "lhs_knob_path" : "~/ottertune/client/driver/knobs/postgres-96.json", - "lhs_save_path" : "~/ottertune/client/driver/configs", - "oracle_awr_enabled": false, - "warmup_iterations": 0, - "sleep_time": 300 -} diff --git a/client/driver/driver_config.py b/client/driver/driver_config.py new file mode 100644 index 0000000..64cf3e0 --- /dev/null +++ b/client/driver/driver_config.py @@ -0,0 +1,143 @@ +import os + +#========================================================== +# HOST LOGIN +#========================================================== + +# Location of the database host relative to this driver +# Valid values: local, remote, or docker +HOST_CONN = 'local' + +# The name of the Docker container for the target database +# (only required if HOST_CONN = docker) +CONTAINER_NAME = None # e.g., 'postgres_container' + +# Host SSH login credentials (only required if HOST_CONN=remote) +LOGIN_NAME = None +LOGIN_HOST = None +LOGIN_PORT = None # Set when using a port other than the SSH default + + +#========================================================== +# DATABASE OPTIONS +#========================================================== + +# Either Postgres or Oracle +DB_TYPE = 'postgres' + +# Name of the database +DB_NAME = 'tpcc' + +# Database username +DB_USER = 'dbuser' + +# Password for DB_USER +DB_PASSWORD = 'dbpassword' + +# Database host address +DB_HOST = 'localhost' + +# Database port +DB_PORT = '5432' + +# Path to the configuration file on the database server +DB_CONF = '/etc/postgresql/9.6/main/postgresql.conf' + +# Base config settings to always include when installing new configurations +BASE_DB_CONF = { + 'track_counts': 'on', + 'track_functions': 'all', + 'track_io_timing': 'on', + 'autovacuum': 'off', +} + +# Name of the device on the database server to monitor the disk usage, or None to disable +DATABASE_DISK = None + +# Set this to a different database version to override the current version +OVERRIDE_DB_VERSION = None + +# POSTGRES-SPECIFIC OPTIONS >>> +PG_DATADIR = '/var/lib/postgresql/9.6/main' + +# ORACLE-SPECIFIC OPTIONS >>> +ORACLE_AWR_ENABLED = False + + +#========================================================== +# DRIVER OPTIONS +#========================================================== + +# Path to this driver +DRIVER_HOME = os.path.realpath(__file__) +print('DRIVER HOME: {}'.format(DRIVER_HOME)) + +# Path to the directory for storing results +RESULT_DIR = os.path.join(DRIVER_HOME, 'results') + +# Path to the directory for storing database dump files +DB_DUMP_DIR = os.path.join(DRIVER_HOME, 'dumpfiles') + +# Reload the database after running this many iterations +RELOAD_INTERVAL = 10 + +# The maximum allowable disk usage percentage. Reload the database +# whenever the current disk usage exceeds this value. +MAX_DISK_USAGE = 90 + +# Execute this many warmup iterations before uploading the next result +# to the website +WARMUP_ITERATIONS = 0 + +# Let the database initialize for this many seconds after it restarts +RESTART_SLEEP_SEC = 300 + +#========================================================== +# OLTPBENCHMARK OPTIONS +#========================================================== + +# Path to OLTPBench directory +OLTPBENCH_HOME = os.path.expanduser('~/oltpbench') + +# Path to the OLTPBench configuration file +OLTPBENCH_CONFIG = os.path.join(OLTPBENCH_HOME, 'config/tpcc_config_postgres.xml') + +# Name of the benchmark to run +OLTPBENCH_BENCH = 'tpcc' + + +#========================================================== +# CONTROLLER OPTIONS +#========================================================== + +# Path to the controller directory +CONTROLLER_HOME = os.path.expanduser('~/ottertune/client/controller') + +# Path to the controller configuration file +CONTROLLER_CONFIG = os.path.join(CONTROLLER_HOME, 'config/postgres_config.json') + + +#========================================================== +# LOGGING OPTIONS +#========================================================== + +LOG_LEVEL = 'DEBUG' + +# Path to log directory +LOG_DIR = os.path.join(DRIVER_HOME, 'log') + +# Log files +DRIVER_LOG = os.path.join(LOG_DIR, 'driver.log') +OLTPBENCH_LOG = os.path.join(LOG_DIR, 'oltpbench.log') +CONTROLLER_LOG = os.path.join(LOG_DIR, 'controller.log') + + +#========================================================== +# WEBSITE OPTIONS +#========================================================== + +# OtterTune website URL +WEBSITE_URL = 'http://127.0.0.1:8000' + +# Code for uploading new results to the website +UPLOAD_CODE = 'I5I10PXK3PK27FM86YYS' diff --git a/client/driver/fabfile.py b/client/driver/fabfile.py index b3ad036..fae1c5c 100644 --- a/client/driver/fabfile.py +++ b/client/driver/fabfile.py @@ -23,38 +23,55 @@ import requests from fabric.api import env, local, task, lcd from fabric.state import output as fabric_output +import driver_config as dconf + + +def _setup(): + global LOGIN + + # Determine correct login for the type of host connection + if dconf.HOST_CONN == 'local': + LOGIN = 'localhost' + + elif dconf.HOST_CONN == 'remote': + if not dconf.LOGIN_HOST: + raise ValueError("LOGIN_HOST must be set if HOST_CONN=remote") + + LOGIN = dconf.LOGIN_HOST + if dconf.LOGIN_NAME: + LOGIN = '{}@{}'.format(dconf.LOGIN_NAME, LOGIN) + + if dconf.LOGIN_PORT: + LOGIN += ':{}'.format(dconf.LOGIN_PORT) + + elif dconf.HOST_CONN == 'docker': + if not dconf.CONTAINER_NAME: + raise ValueError("CONTAINER_NAME must be set if HOST_CONN=docker") + LOGIN = 'localhost' + + else: + raise ValueError(("Invalid HOST_CONN: {}. Valid values are " + "'local', 'remote', or 'docker'.").format(dconf.HOST_CONN)) + + # Create all output directories + for d in (dconf.RESULT_DIR, dconf.LOG_DIR, dconf.DB_DUMP_DIR): + os.makedirs(d, exist_ok=True) + + +_setup() + + # Fabric environment settings -env.hosts = ['localhost'] +env.hosts = [LOGIN] fabric_output.update({ 'running': True, 'stdout': True, }) -# intervals of restoring the databse -RELOAD_INTERVAL = 10 -# maximum disk usage -MAX_DISK_USAGE = 90 - -# Load config -with open('driver_config.json', 'r') as _f: - CONF = {k: os.path.expanduser(v) if isinstance(v, str) and v.startswith('~') else v - for k, v in json.load(_f).items()} - -# Create output directories -for _dir in (CONF['database_save_path'], CONF['log_path'], CONF['save_path'], - CONF['lhs_save_path']): - os.makedirs(_dir, exist_ok=True) - -# Define paths -CONF['driver_log'] = os.path.join(CONF['log_path'], 'driver.log') -CONF['oltpbench_log'] = os.path.join(CONF['log_path'], 'oltpbench.log') -CONF['controller_log'] = os.path.join(CONF['log_path'], 'controller.log') -CONF['controller_config'] = os.path.join(CONF['controller_home'], 'config', - '{}_config.json'.format(CONF['database_type'])) # Configure logging LOG = logging.getLogger(__name__) -LOG.setLevel(logging.DEBUG) +LOG.setLevel(getattr(logging, dconf.LOG_LEVEL, logging.DEBUG)) Formatter = logging.Formatter( # pylint: disable=invalid-name fmt='%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s', datefmt='%m-%d-%Y %H:%M:%S') @@ -62,7 +79,7 @@ ConsoleHandler = logging.StreamHandler() # pylint: disable=invalid-name ConsoleHandler.setFormatter(Formatter) LOG.addHandler(ConsoleHandler) FileHandler = RotatingFileHandler( # pylint: disable=invalid-name - CONF['driver_log'], maxBytes=50000, backupCount=2) + dconf.DRIVER_LOG, maxBytes=50000, backupCount=2) FileHandler.setFormatter(Formatter) LOG.addHandler(FileHandler) @@ -75,7 +92,7 @@ def _parse_bool(value): @task def check_disk_usage(): - partition = CONF['database_disk'] + partition = dconf.DATABASE_DISK disk_use = 0 if partition: cmd = "df -h {}".format(partition) @@ -95,55 +112,55 @@ def check_memory_usage(): @task def create_controller_config(): - if CONF['database_type'] == 'postgres': + if dconf.DB_TYPE == 'postgres': dburl_fmt = 'jdbc:postgresql://localhost:5432/{db}'.format - elif CONF['database_type'] == 'oracle': + elif dconf.DB_TYPE == 'oracle': dburl_fmt = 'jdbc:oracle:thin:@localhost:1521:{db}'.format else: - raise Exception("Database Type {} Not Implemented !".format(CONF['database_type'])) + raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE)) config = dict( - database_type=CONF['database_type'], - database_url=dburl_fmt(db=CONF['database_name']), - username=CONF['username'], - password=CONF['password'], + database_type=dconf.DB_TYPE, + database_url=dburl_fmt(db=dconf.DB_NAME), + username=dconf.DB_USER, + password=dconf.DB_PASSWORD, upload_code='DEPRECATED', upload_url='DEPRECATED', - workload_name=CONF['oltpbench_workload'] + workload_name=dconf.OLTPBENCH_BENCH ) - with open(CONF['controller_config'], 'w') as f: + with open(dconf.CONTROLLER_CONFIG, 'w') as f: json.dump(config, f, indent=2) @task def restart_database(): - if CONF['database_type'] == 'postgres': - cmd = 'sudo -u postgres pg_ctl -D {} -w -t 600 restart -m fast'.format(CONF['pg_datadir']) - elif CONF['database_type'] == 'oracle': + if dconf.DB_TYPE == 'postgres': + cmd = 'sudo -u postgres pg_ctl -D {} -w -t 600 restart -m fast'.format(dconf.PG_DATADIR) + elif dconf.DB_TYPE == 'oracle': cmd = 'sh oracleScripts/shutdownOracle.sh && sh oracleScripts/startupOracle.sh' else: - raise Exception("Database Type {} Not Implemented !".format(CONF['database_type'])) + raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE)) local(cmd) @task def drop_database(): - if CONF['database_type'] == 'postgres': + if dconf.DB_TYPE == 'postgres': cmd = "PGPASSWORD={} dropdb -e --if-exists {} -U {}".\ - format(CONF['password'], CONF['database_name'], CONF['username']) + format(dconf.DB_PASSWORD, dconf.DB_NAME, dconf.DB_USER) else: - raise Exception("Database Type {} Not Implemented !".format(CONF['database_type'])) + raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE)) local(cmd) @task def create_database(): - if CONF['database_type'] == 'postgres': + if dconf.DB_TYPE == 'postgres': cmd = "PGPASSWORD={} createdb -e {} -U {}".\ - format(CONF['password'], CONF['database_name'], CONF['username']) + format(dconf.DB_PASSWORD, dconf.DB_NAME, dconf.DB_USER) else: - raise Exception("Database Type {} Not Implemented !".format(CONF['database_type'])) + raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE)) local(cmd) @@ -157,7 +174,7 @@ def change_conf(next_conf=None): signal = "# configurations recommended by ottertune:\n" next_conf = next_conf or {} - with open(CONF['database_conf'], 'r') as f: + with open(dconf.DB_CONF, 'r') as f: lines = f.readlines() if signal not in lines: @@ -165,10 +182,10 @@ def change_conf(next_conf=None): signal_idx = lines.index(signal) lines = lines[0:signal_idx + 1] - if CONF.__contains__('base_database_conf'): - with open(CONF['base_database_conf'], 'r') as f: - base_confs = f.readlines() - lines.extend(base_confs) + if dconf.BASE_DB_CONF: + assert isinstance(dconf.BASE_DB_CONF, dict) + base_conf = ['{} = {}\n'.format(*c) for c in sorted(dconf.BASE_DB_CONF.items())] + lines.extend(base_conf) if isinstance(next_conf, str): with open(next_conf, 'r') as f: @@ -180,60 +197,60 @@ def change_conf(next_conf=None): assert isinstance(recommendation, dict) for name, value in recommendation.items(): - if CONF['database_type'] == 'oracle' and isinstance(value, str): + if dconf.DB_TYPE == 'oracle' and isinstance(value, str): value = value.strip('B') lines.append('{} = {}\n'.format(name, value)) lines.append('\n') - tmpconf = 'tmp_' + os.path.basename(CONF['database_conf']) + tmpconf = 'tmp_' + os.path.basename(dconf.DB_CONF) with open(tmpconf, 'w') as f: f.write(''.join(lines)) - local('sudo cp {0} {0}.ottertune.bak'.format(CONF['database_conf'])) - local('sudo mv {} {}'.format(tmpconf, CONF['database_conf'])) + local('sudo cp {0} {0}.ottertune.bak'.format(dconf.DB_CONF)) + local('sudo mv {} {}'.format(tmpconf, dconf.DB_CONF)) @task def load_oltpbench(): cmd = "./oltpbenchmark -b {} -c {} --create=true --load=true".\ - format(CONF['oltpbench_workload'], CONF['oltpbench_config']) - with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager + format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG) + with lcd(dconf.OLTPBENCH_HOME): # pylint: disable=not-context-manager local(cmd) @task def run_oltpbench(): cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile".\ - format(CONF['oltpbench_workload'], CONF['oltpbench_config']) - with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager + format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG) + with lcd(dconf.OLTPBENCH_HOME): # pylint: disable=not-context-manager local(cmd) @task def run_oltpbench_bg(): cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile > {} 2>&1 &".\ - format(CONF['oltpbench_workload'], CONF['oltpbench_config'], CONF['oltpbench_log']) - with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager + format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG, dconf.OLTPBENCH_LOG) + with lcd(dconf.OLTPBENCH_HOME): # pylint: disable=not-context-manager local(cmd) @task def run_controller(): - if not os.path.exists(CONF['controller_config']): + if not os.path.exists(dconf.CONTROLLER_CONFIG): create_controller_config() cmd = 'gradle run -PappArgs="-c {} -d output/" --no-daemon > {}'.\ - format(CONF['controller_config'], CONF['controller_log']) - with lcd(CONF['controller_home']): # pylint: disable=not-context-manager + format(dconf.CONTROLLER_CONFIG, dconf.CONTROLLER_LOG) + with lcd(dconf.CONTROLLER_HOME): # pylint: disable=not-context-manager local(cmd) @task def signal_controller(): - pidfile = os.path.join(CONF['controller_home'], 'pid.txt') + pidfile = os.path.join(dconf.CONTROLLER_HOME, 'pid.txt') with open(pidfile, 'r') as f: pid = int(f.read()) cmd = 'sudo kill -2 {}'.format(pid) - with lcd(CONF['controller_home']): # pylint: disable=not-context-manager + with lcd(dconf.CONTROLLER_HOME): # pylint: disable=not-context-manager local(cmd) @@ -242,8 +259,8 @@ def save_dbms_result(): t = int(time.time()) files = ['knobs.json', 'metrics_after.json', 'metrics_before.json', 'summary.json'] for f_ in files: - srcfile = os.path.join(CONF['controller_home'], 'output', f_) - dstfile = os.path.join(CONF['save_path'], '{}__{}'.format(t, f_)) + srcfile = os.path.join(dconf.CONTROLLER_HOME, 'output', f_) + dstfile = os.path.join(dconf.RESULT_DIR, '{}__{}'.format(t, f_)) local('cp {} {}'.format(srcfile, dstfile)) return t @@ -252,7 +269,7 @@ def save_dbms_result(): def save_next_config(next_config, t=None): if not t: t = int(time.time()) - with open(os.path.join(CONF['save_path'], '{}__next_config.json'.format(t)), 'w') as f: + with open(os.path.join(dconf.RESULT_DIR, '{}__next_config.json'.format(t)), 'w') as f: json.dump(next_config, f, indent=2) return t @@ -265,9 +282,9 @@ def free_cache(): @task def upload_result(result_dir=None, prefix=None, upload_code=None): - result_dir = result_dir or os.path.join(CONF['controller_home'], 'output') + result_dir = result_dir or os.path.join(dconf.CONTROLLER_HOME, 'output') prefix = prefix or '' - upload_code = upload_code or CONF['upload_code'] + upload_code = upload_code or dconf.UPLOAD_CODE files = {} for base in ('summary', 'knobs', 'metrics_before', 'metrics_after'): @@ -275,18 +292,17 @@ def upload_result(result_dir=None, prefix=None, upload_code=None): # Replaces the true db version with the specified version to allow for # testing versions not officially supported by OtterTune - if base == 'summary' and 'override_database_version' in CONF and \ - CONF['override_database_version']: + if base == 'summary' and dconf.OVERRIDE_DB_VERSION: with open(fpath, 'r') as f: summary = json.load(f) summary['real_database_version'] = summary['database_version'] - summary['database_version'] = CONF['override_database_version'] + summary['database_version'] = dconf.OVERRIDE_DB_VERSION with open(fpath, 'w') as f: json.dump(summary, f, indent=1) files[base] = open(fpath, 'rb') - response = requests.post(CONF['upload_url'] + '/new_result/', files=files, + response = requests.post(dconf.WEBSITE_URL + '/new_result/', files=files, data={'upload_code': upload_code}) if response.status_code != 200: raise Exception('Error uploading result.\nStatus: {}\nMessage: {}\n'.format( @@ -304,8 +320,8 @@ def upload_result(result_dir=None, prefix=None, upload_code=None): def get_result(max_time_sec=180, interval_sec=5, upload_code=None): max_time_sec = int(max_time_sec) interval_sec = int(interval_sec) - upload_code = upload_code or CONF['upload_code'] - url = CONF['upload_url'] + '/query_and_get/' + upload_code + upload_code = upload_code or dconf.UPLOAD_CODE + url = dconf.WEBSITE_URL + '/query_and_get/' + upload_code elapsed = 0 response_dict = None response = '' @@ -354,7 +370,7 @@ def get_result(max_time_sec=180, interval_sec=5, upload_code=None): @task def download_debug_info(pprint=False): pprint = _parse_bool(pprint) - url = '{}/dump/{}'.format(CONF['upload_url'], CONF['upload_code']) + url = '{}/dump/{}'.format(dconf.WEBSITE_URL, dconf.UPLOAD_CODE) params = {'pp': int(True)} if pprint else {} rsp = requests.get(url, params=params) @@ -381,7 +397,7 @@ def add_udf(): @task def upload_batch(result_dir=None, sort=True, upload_code=None): - result_dir = result_dir or CONF['save_path'] + result_dir = result_dir or dconf.RESULT_DIR sort = _parse_bool(sort) results = glob.glob(os.path.join(result_dir, '*__summary.json')) if sort: @@ -399,42 +415,42 @@ def upload_batch(result_dir=None, sort=True, upload_code=None): @task def dump_database(): - db_file_path = os.path.join(CONF['database_save_path'], CONF['database_name'] + '.dump') + db_file_path = os.path.join(dconf.DB_DUMP_DIR, dconf.DB_NAME + '.dump') if os.path.exists(db_file_path): LOG.info('%s already exists ! ', db_file_path) return False else: - LOG.info('Dump database %s to %s', CONF['database_name'], db_file_path) + LOG.info('Dump database %s to %s', dconf.DB_NAME, db_file_path) # You must create a directory named dpdata through sqlplus in your Oracle database - if CONF['database_type'] == 'oracle': + if dconf.DB_TYPE == 'oracle': cmd = 'expdp {}/{}@{} schemas={} dumpfile={}.dump DIRECTORY=dpdata'.format( 'c##tpcc', 'oracle', 'orcldb', 'c##tpcc', 'orcldb') - elif CONF['database_type'] == 'postgres': - cmd = 'PGPASSWORD={} pg_dump -U {} -F c -d {} > {}'.format(CONF['password'], - CONF['username'], - CONF['database_name'], + elif dconf.DB_TYPE == 'postgres': + cmd = 'PGPASSWORD={} pg_dump -U {} -F c -d {} > {}'.format(dconf.DB_PASSWORD, + dconf.DB_USER, + dconf.DB_NAME, db_file_path) else: - raise Exception("Database Type {} Not Implemented !".format(CONF['database_type'])) + raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE)) local(cmd) return True @task def restore_database(): - if CONF['database_type'] == 'oracle': + if dconf.DB_TYPE == 'oracle': # You must create a directory named dpdata through sqlplus in your Oracle database # The following script assumes such directory exists. # You may want to modify the username, password, and dump file name in the script cmd = 'sh oracleScripts/restoreOracle.sh' - elif CONF['database_type'] == 'postgres': - db_file_path = '{}/{}.dump'.format(CONF['database_save_path'], CONF['database_name']) + elif dconf.DB_TYPE == 'postgres': + db_file_path = '{}/{}.dump'.format(dconf.DB_DUMP_DIR, dconf.DB_NAME) drop_database() create_database() cmd = 'PGPASSWORD={} pg_restore -U {} -n public -j 8 -F c -d {} {}'.\ - format(CONF['password'], CONF['username'], CONF['database_name'], db_file_path) + format(dconf.DB_PASSWORD, dconf.DB_USER, dconf.DB_NAME, db_file_path) else: - raise Exception("Database Type {} Not Implemented !".format(CONF['database_type'])) + raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE)) LOG.info('Start restoring database') local(cmd) LOG.info('Finish restoring database') @@ -442,8 +458,8 @@ def restore_database(): def _ready_to_start_oltpbench(): ready = False - if os.path.exists(CONF['controller_log']): - with open(CONF['controller_log'], 'r') as f: + if os.path.exists(dconf.CONTROLLER_LOG): + with open(dconf.CONTROLLER_LOG, 'r') as f: content = f.read() ready = 'Output the process pid to' in content return ready @@ -451,18 +467,18 @@ def _ready_to_start_oltpbench(): def _ready_to_start_controller(): ready = False - if os.path.exists(CONF['oltpbench_log']): - with open(CONF['oltpbench_log'], 'r') as f: + if os.path.exists(dconf.OLTPBENCH_LOG): + with open(dconf.OLTPBENCH_LOG, 'r') as f: content = f.read() ready = 'Warmup complete, starting measurements' in content return ready def _ready_to_shut_down_controller(): - pidfile = os.path.join(CONF['controller_home'], 'pid.txt') + pidfile = os.path.join(dconf.CONTROLLER_HOME, 'pid.txt') ready = False - if os.path.exists(pidfile) and os.path.exists(CONF['oltpbench_log']): - with open(CONF['oltpbench_log'], 'r') as f: + if os.path.exists(pidfile) and os.path.exists(dconf.OLTPBENCH_LOG): + with open(dconf.OLTPBENCH_LOG, 'r') as f: content = f.read() ready = 'Output throughput samples into file' in content return ready @@ -470,17 +486,11 @@ def _ready_to_shut_down_controller(): def clean_logs(): # remove oltpbench log - cmd = 'rm -f {}'.format(CONF['oltpbench_log']) + cmd = 'rm -f {}'.format(dconf.OLTPBENCH_LOG) local(cmd) # remove controller log - cmd = 'rm -f {}'.format(CONF['controller_log']) - local(cmd) - - -@task -def lhs_samples(count=10): - cmd = 'python3 lhs.py {} {} {}'.format(count, CONF['lhs_knob_path'], CONF['lhs_save_path']) + cmd = 'rm -f {}'.format(dconf.CONTROLLER_LOG) local(cmd) @@ -494,11 +504,11 @@ def loop(i): # restart database restart_database() - time.sleep(CONF['sleep_time']) + time.sleep(dconf.RESTART_SLEEP_SEC) # check disk usage - if check_disk_usage() > MAX_DISK_USAGE: - LOG.warning('Exceeds max disk usage %s', MAX_DISK_USAGE) + if check_disk_usage() > dconf.MAX_DISK_USAGE: + LOG.warning('Exceeds max disk usage %s', dconf.MAX_DISK_USAGE) # run controller from another process p = Process(target=run_controller, args=()) @@ -532,7 +542,7 @@ def loop(i): # save result result_timestamp = save_dbms_result() - if i >= CONF['warmup_iterations']: + if i >= dconf.WARMUP_ITERATIONS: # upload result upload_result() @@ -546,96 +556,14 @@ def loop(i): change_conf(response['recommendation']) -@task -def run_lhs(): - datadir = CONF['lhs_save_path'] - samples = glob.glob(os.path.join(datadir, 'config_*')) - - # dump database if it's not done before. - dump = dump_database() - - result_timestamp = None - for i, sample in enumerate(samples): - # reload database periodically - if RELOAD_INTERVAL > 0: - if i % RELOAD_INTERVAL == 0: - if i == 0 and dump is False: - restore_database() - elif i > 0: - restore_database() - # free cache - free_cache() - - LOG.info('\n\n Start %s-th sample %s \n\n', i, sample) - # check memory usage - # check_memory_usage() - - # check disk usage - if check_disk_usage() > MAX_DISK_USAGE: - LOG.warning('Exceeds max disk usage %s', MAX_DISK_USAGE) - - # load the next lhs-sampled config - with open(sample, 'r') as f: - next_config = json.load(f, object_pairs_hook=OrderedDict) - save_next_config(next_config, t=result_timestamp) - - # remove oltpbench log and controller log - clean_logs() - - # change config - change_conf(next_config) - - # restart database - restart_database() - - if CONF.get('oracle_awr_enabled', False): - # create snapshot for oracle AWR report - if CONF['database_type'] == 'oracle': - local('sh snapshotOracle.sh') - - # run controller from another process - p = Process(target=run_controller, args=()) - p.start() - - # run oltpbench as a background job - while not _ready_to_start_oltpbench(): - pass - run_oltpbench_bg() - LOG.info('Run OLTP-Bench') - - while not _ready_to_start_controller(): - pass - signal_controller() - LOG.info('Start the first collection') - - while not _ready_to_shut_down_controller(): - pass - # stop the experiment - signal_controller() - LOG.info('Start the second collection, shut down the controller') - - p.join() - - # save result - result_timestamp = save_dbms_result() - - # upload result - upload_result() - - if CONF.get('oracle_awr_enabled', False): - # create oracle AWR report for performance analysis - if CONF['database_type'] == 'oracle': - local('sh oracleScripts/snapshotOracle.sh && sh oracleScripts/awrOracle.sh') - - @task def run_loops(max_iter=1): # dump database if it's not done before. dump = dump_database() for i in range(int(max_iter)): - if RELOAD_INTERVAL > 0: - if i % RELOAD_INTERVAL == 0: + if dconf.RELOAD_INTERVAL > 0: + if i % dconf.RELOAD_INTERVAL == 0: if i == 0 and dump is False: restart_database() restore_database() @@ -643,13 +571,13 @@ def run_loops(max_iter=1): restore_database() LOG.info('The %s-th Loop Starts / Total Loops %s', i + 1, max_iter) - loop(i % RELOAD_INTERVAL) + loop(i % dconf.RELOAD_INTERVAL) LOG.info('The %s-th Loop Ends / Total Loops %s', i + 1, max_iter) @task def rename_batch(result_dir=None): - result_dir = result_dir or CONF['save_path'] + result_dir = result_dir or dconf.RESULT_DIR results = glob.glob(os.path.join(result_dir, '*__summary.json')) results = sorted(results) for i, result in enumerate(results): @@ -752,7 +680,7 @@ def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10): elapsed = 0 while elapsed <= max_time_sec: - response = requests.get(CONF['upload_url'] + '/test/pipeline/') + response = requests.get(dconf.WEBSITE_URL + '/test/pipeline/') response = response.content LOG.info(response) if 'False' in str(response): @@ -766,7 +694,7 @@ def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10): def integration_tests(): # Create test website - response = requests.get(CONF['upload_url'] + '/test/create/') + response = requests.get(dconf.WEBSITE_URL + '/test/create/') LOG.info(response.content) # Upload training data