diff --git a/client/driver/configs/postgres_base.conf b/client/driver/configs/postgres_base.conf new file mode 100644 index 0000000..c57b0c9 --- /dev/null +++ b/client/driver/configs/postgres_base.conf @@ -0,0 +1,4 @@ +track_counts = on +track_functions = all +track_io_timing = on +autovacuum = off diff --git a/client/driver/driver_config.json b/client/driver/driver_config.json index efd6bb7..ee4d568 100644 --- a/client/driver/driver_config.json +++ b/client/driver/driver_config.json @@ -1,20 +1,24 @@ -{ - "database_type" : "postgres", - "database_name" : "tpcc", - "database_disk": "", - "database_conf": "/etc/postgresql/9.6/main/postgresql.conf", - "database_save_path": "~/ottertune/client/driver/dumpfiles", - "username" : "dbuser", - "password" : "dbpassword", - "oltpbench_home": "~/oltpbench", - "oltpbench_config": "~/oltpbench/config/tpcc_config_postgres.xml", - "oltpbench_workload": "tpcc", - "controller_home": "~/ottertune/client/controller", - "log_path": "~/ottertune/client/driver/log", - "save_path": "~/ottertune/client/driver/results", - "upload_url" : "http://127.0.0.1:8000", - "upload_code" : "I5I10PXK3PK27FM86YYS", - "lhs_knob_path" : "~/ottertune/client/driver/knobs/postgres-96.json", - "lhs_save_path" : "~/ottertune/client/driver/configs", - "oracle_awr_enabled": false -} +{ + "database_type" : "postgres", + "database_name" : "tpcc", + "database_disk": "", + "database_conf": "/etc/postgresql/9.6/main/postgresql.conf", + "base_database_conf": "configs/postgres_base.conf", + "pg_datadir": "/var/lib/postgresql/9.6/main", + "database_save_path": "~/ottertune/client/driver/dumpfiles", + "username" : "dbuser", + "password" : "dbpassword", + "oltpbench_home": "~/oltpbench", + "oltpbench_config": "~/oltpbench/config/tpcc_config_postgres.xml", + "oltpbench_workload": "tpcc", + "controller_home": "~/ottertune/client/controller", + "log_path": "~/ottertune/client/driver/log", + "save_path": "~/ottertune/client/driver/results", + "upload_url" : "http://127.0.0.1:8000", + "upload_code" : "I5I10PXK3PK27FM86YYS", + "lhs_knob_path" : "~/ottertune/client/driver/knobs/postgres-96.json", + "lhs_save_path" : "~/ottertune/client/driver/configs", + "oracle_awr_enabled": false, + "warmup_iterations": 0, + "sleep_time": 300 +} diff --git a/client/driver/fabfile.py b/client/driver/fabfile.py index b537d77..98ca2a1 100644 --- a/client/driver/fabfile.py +++ b/client/driver/fabfile.py @@ -34,8 +34,6 @@ fabric_output.update({ RELOAD_INTERVAL = 10 # maximum disk usage MAX_DISK_USAGE = 90 -# Postgres datadir -PG_DATADIR = '/var/lib/postgresql/9.6/main' # Load config with open('driver_config.json', 'r') as _f: @@ -121,7 +119,7 @@ def create_controller_config(): @task def restart_database(): if CONF['database_type'] == 'postgres': - cmd = 'sudo -u postgres pg_ctl -D {} -w restart'.format(PG_DATADIR) + cmd = 'sudo -u postgres pg_ctl -D {} -w -t 600 restart -m fast'.format(CONF['pg_datadir']) elif CONF['database_type'] == 'oracle': cmd = 'sh oracleScripts/shutdownOracle.sh && sh oracleScripts/startupOracle.sh' else: @@ -167,6 +165,10 @@ def change_conf(next_conf=None): signal_idx = lines.index(signal) lines = lines[0:signal_idx + 1] + if CONF.__contains__('base_database_conf'): + with open(CONF['base_database_conf'], 'r') as f: + base_confs = f.readlines() + lines.extend(base_confs) if isinstance(next_conf, str): with open(next_conf, 'r') as f: @@ -481,8 +483,7 @@ def lhs_samples(count=10): @task -def loop(): - +def loop(i): # free cache free_cache() @@ -491,6 +492,7 @@ def loop(): # restart database restart_database() + time.sleep(CONF['sleep_time']) # check disk usage if check_disk_usage() > MAX_DISK_USAGE: @@ -516,6 +518,7 @@ def loop(): # stop the experiment while not _ready_to_shut_down_controller(): time.sleep(1) + signal_controller() LOG.info('Start the second collection, shut down the controller') @@ -527,17 +530,18 @@ def loop(): # save result result_timestamp = save_dbms_result() - # upload result - upload_result() + if i >= CONF['warmup_iterations']: + # upload result + upload_result() - # get result - response = get_result() - - # save next config - save_next_config(response, t=result_timestamp) - - # change config - change_conf(response['recommendation']) + # get result + response = get_result() + + # save next config + save_next_config(response, t=result_timestamp) + + # change config + change_conf(response['recommendation']) @task @@ -631,10 +635,11 @@ def run_loops(max_iter=1): if RELOAD_INTERVAL > 0: if i % RELOAD_INTERVAL == 0: if i == 0 and dump is False: + restart_database() restore_database() elif i > 0: restore_database() LOG.info('The %s-th Loop Starts / Total Loops %s', i + 1, max_iter) - loop() + loop(i % RELOAD_INTERVAL) LOG.info('The %s-th Loop Ends / Total Loops %s', i + 1, max_iter)