public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

achieve stable performance

This commit is contained in:
yangdsh 2019-10-29 00:07:34 +00:00 committed by Dana Van Aken
parent 9f71d1c8de
commit cf65db2231
3 changed files with 49 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
client/driver/configs/postgres_base.conf Normal file
View File

@ -0,0 +1,4 @@
track_counts = on
track_functions = all
track_io_timing = on
autovacuum = off

44
client/driver/driver_config.json
View File

@ -1,20 +1,24 @@
{ {
"database_type" : "postgres", "database_type" : "postgres",
"database_name" : "tpcc", "database_name" : "tpcc",
"database_disk": "", "database_disk": "",
"database_conf": "/etc/postgresql/9.6/main/postgresql.conf", "database_conf": "/etc/postgresql/9.6/main/postgresql.conf",
"database_save_path": "~/ottertune/client/driver/dumpfiles", "base_database_conf": "configs/postgres_base.conf",
"username" : "dbuser", "pg_datadir": "/var/lib/postgresql/9.6/main",
"password" : "dbpassword", "database_save_path": "~/ottertune/client/driver/dumpfiles",
"oltpbench_home": "~/oltpbench", "username" : "dbuser",
"oltpbench_config": "~/oltpbench/config/tpcc_config_postgres.xml", "password" : "dbpassword",
"oltpbench_workload": "tpcc", "oltpbench_home": "~/oltpbench",
"controller_home": "~/ottertune/client/controller", "oltpbench_config": "~/oltpbench/config/tpcc_config_postgres.xml",
"log_path": "~/ottertune/client/driver/log", "oltpbench_workload": "tpcc",
"save_path": "~/ottertune/client/driver/results", "controller_home": "~/ottertune/client/controller",
"upload_url" : "http://127.0.0.1:8000", "log_path": "~/ottertune/client/driver/log",
"upload_code" : "I5I10PXK3PK27FM86YYS", "save_path": "~/ottertune/client/driver/results",
"lhs_knob_path" : "~/ottertune/client/driver/knobs/postgres-96.json", "upload_url" : "http://127.0.0.1:8000",
"lhs_save_path" : "~/ottertune/client/driver/configs", "upload_code" : "I5I10PXK3PK27FM86YYS",
"oracle_awr_enabled": false "lhs_knob_path" : "~/ottertune/client/driver/knobs/postgres-96.json",
} "lhs_save_path" : "~/ottertune/client/driver/configs",
"oracle_awr_enabled": false,
"warmup_iterations": 0,
"sleep_time": 300
}

37
client/driver/fabfile.py vendored
View File

@ -34,8 +34,6 @@ fabric_output.update({
RELOAD_INTERVAL = 10 RELOAD_INTERVAL = 10
# maximum disk usage # maximum disk usage
MAX_DISK_USAGE = 90 MAX_DISK_USAGE = 90
# Postgres datadir
PG_DATADIR = '/var/lib/postgresql/9.6/main'
# Load config # Load config
with open('driver_config.json', 'r') as _f: with open('driver_config.json', 'r') as _f:
@ -121,7 +119,7 @@ def create_controller_config():
@task @task
def restart_database(): def restart_database():
if CONF['database_type'] == 'postgres': if CONF['database_type'] == 'postgres':
cmd = 'sudo -u postgres pg_ctl -D {} -w restart'.format(PG_DATADIR) cmd = 'sudo -u postgres pg_ctl -D {} -w -t 600 restart -m fast'.format(CONF['pg_datadir'])
elif CONF['database_type'] == 'oracle': elif CONF['database_type'] == 'oracle':
cmd = 'sh oracleScripts/shutdownOracle.sh && sh oracleScripts/startupOracle.sh' cmd = 'sh oracleScripts/shutdownOracle.sh && sh oracleScripts/startupOracle.sh'
else: else:
@ -167,6 +165,10 @@ def change_conf(next_conf=None):
signal_idx = lines.index(signal) signal_idx = lines.index(signal)
lines = lines[0:signal_idx + 1] lines = lines[0:signal_idx + 1]
if CONF.__contains__('base_database_conf'):
with open(CONF['base_database_conf'], 'r') as f:
base_confs = f.readlines()
lines.extend(base_confs)
if isinstance(next_conf, str): if isinstance(next_conf, str):
with open(next_conf, 'r') as f: with open(next_conf, 'r') as f:
@ -481,8 +483,7 @@ def lhs_samples(count=10):
@task @task
def loop(): def loop(i):
# free cache # free cache
free_cache() free_cache()
@ -491,6 +492,7 @@ def loop():
# restart database # restart database
restart_database() restart_database()
time.sleep(CONF['sleep_time'])
# check disk usage # check disk usage
if check_disk_usage() > MAX_DISK_USAGE: if check_disk_usage() > MAX_DISK_USAGE:
@ -516,6 +518,7 @@ def loop():
# stop the experiment # stop the experiment
while not _ready_to_shut_down_controller(): while not _ready_to_shut_down_controller():
time.sleep(1) time.sleep(1)
signal_controller() signal_controller()
LOG.info('Start the second collection, shut down the controller') LOG.info('Start the second collection, shut down the controller')
@ -527,17 +530,18 @@ def loop():
# save result # save result
result_timestamp = save_dbms_result() result_timestamp = save_dbms_result()
# upload result if i >= CONF['warmup_iterations']:
upload_result() # upload result
upload_result()
# get result # get result
response = get_result() response = get_result()
# save next config # save next config
save_next_config(response, t=result_timestamp) save_next_config(response, t=result_timestamp)
# change config # change config
change_conf(response['recommendation']) change_conf(response['recommendation'])
@task @task
@ -631,10 +635,11 @@ def run_loops(max_iter=1):
if RELOAD_INTERVAL > 0: if RELOAD_INTERVAL > 0:
if i % RELOAD_INTERVAL == 0: if i % RELOAD_INTERVAL == 0:
if i == 0 and dump is False: if i == 0 and dump is False:
restart_database()
restore_database() restore_database()
elif i > 0: elif i > 0:
restore_database() restore_database()
LOG.info('The %s-th Loop Starts / Total Loops %s', i + 1, max_iter) LOG.info('The %s-th Loop Starts / Total Loops %s', i + 1, max_iter)
loop() loop(i % RELOAD_INTERVAL)
LOG.info('The %s-th Loop Ends / Total Loops %s', i + 1, max_iter) LOG.info('The %s-th Loop Ends / Total Loops %s', i + 1, max_iter)