Changed the format of the driver config from JSON to a python settings file to make it easier to update / read.

client/driver/configs/postgres_base.conf

@@ -1,4 +0,0 @@
-track_counts = on
-track_functions = all
-track_io_timing = on
-autovacuum = off

client/driver/driver_config.json

@@ -1,24 +0,0 @@
-{
-  "database_type" : "postgres",
-  "database_name" : "tpcc",
-  "database_disk": "",
-  "database_conf": "/etc/postgresql/9.6/main/postgresql.conf",
-  "base_database_conf": "configs/postgres_base.conf",
-  "pg_datadir": "/var/lib/postgresql/9.6/main",
-  "database_save_path": "~/ottertune/client/driver/dumpfiles",
-  "username" : "dbuser",
-  "password" : "dbpassword",
-  "oltpbench_home": "~/oltpbench",
-  "oltpbench_config": "~/oltpbench/config/tpcc_config_postgres.xml",
-  "oltpbench_workload": "tpcc",
-  "controller_home": "~/ottertune/client/controller",
-  "log_path": "~/ottertune/client/driver/log",
-  "save_path": "~/ottertune/client/driver/results",
-  "upload_url" : "http://127.0.0.1:8000",
-  "upload_code" : "I5I10PXK3PK27FM86YYS",
-  "lhs_knob_path" : "~/ottertune/client/driver/knobs/postgres-96.json",
-  "lhs_save_path" : "~/ottertune/client/driver/configs",
-  "oracle_awr_enabled": false,
-  "warmup_iterations": 0,
-  "sleep_time": 300
-}

client/driver/driver_config.py

@@ -0,0 +1,143 @@
+import os
+
+#==========================================================
+#  HOST LOGIN
+#==========================================================
+
+# Location of the database host relative to this driver
+# Valid values: local, remote, or docker
+HOST_CONN = 'local'
+
+# The name of the Docker container for the target database
+# (only required if HOST_CONN = docker)
+CONTAINER_NAME = None  # e.g., 'postgres_container'
+
+# Host SSH login credentials (only required if HOST_CONN=remote)
+LOGIN_NAME = None
+LOGIN_HOST = None
+LOGIN_PORT = None  # Set when using a port other than the SSH default
+
+
+#==========================================================
+#  DATABASE OPTIONS
+#==========================================================
+
+# Either Postgres or Oracle
+DB_TYPE = 'postgres'
+
+# Name of the database
+DB_NAME = 'tpcc'
+
+# Database username
+DB_USER = 'dbuser'
+
+# Password for DB_USER
+DB_PASSWORD = 'dbpassword'
+
+# Database host address
+DB_HOST = 'localhost'
+
+# Database port
+DB_PORT = '5432'
+
+# Path to the configuration file on the database server
+DB_CONF = '/etc/postgresql/9.6/main/postgresql.conf'
+
+# Base config settings to always include when installing new configurations
+BASE_DB_CONF = {
+    'track_counts': 'on',
+    'track_functions': 'all',
+    'track_io_timing': 'on',
+    'autovacuum': 'off',
+}
+
+# Name of the device on the database server to monitor the disk usage, or None to disable
+DATABASE_DISK = None
+
+# Set this to a different database version to override the current version
+OVERRIDE_DB_VERSION = None
+
+# POSTGRES-SPECIFIC OPTIONS >>>
+PG_DATADIR = '/var/lib/postgresql/9.6/main'
+
+# ORACLE-SPECIFIC OPTIONS >>>
+ORACLE_AWR_ENABLED = False
+
+
+#==========================================================
+#  DRIVER OPTIONS
+#==========================================================
+
+# Path to this driver
+DRIVER_HOME = os.path.realpath(__file__)
+print('DRIVER HOME: {}'.format(DRIVER_HOME))
+
+# Path to the directory for storing results
+RESULT_DIR = os.path.join(DRIVER_HOME, 'results')
+
+# Path to the directory for storing database dump files
+DB_DUMP_DIR = os.path.join(DRIVER_HOME, 'dumpfiles')
+
+# Reload the database after running this many iterations
+RELOAD_INTERVAL = 10
+
+# The maximum allowable disk usage percentage. Reload the database
+# whenever the current disk usage exceeds this value.
+MAX_DISK_USAGE = 90
+
+# Execute this many warmup iterations before uploading the next result
+# to the website
+WARMUP_ITERATIONS = 0
+
+# Let the database initialize for this many seconds after it restarts
+RESTART_SLEEP_SEC = 300
+
+#==========================================================
+#  OLTPBENCHMARK OPTIONS
+#==========================================================
+
+# Path to OLTPBench directory
+OLTPBENCH_HOME = os.path.expanduser('~/oltpbench')
+
+# Path to the OLTPBench configuration file
+OLTPBENCH_CONFIG = os.path.join(OLTPBENCH_HOME, 'config/tpcc_config_postgres.xml')
+
+# Name of the benchmark to run
+OLTPBENCH_BENCH = 'tpcc'
+
+
+#==========================================================
+#  CONTROLLER OPTIONS
+#==========================================================
+
+# Path to the controller directory
+CONTROLLER_HOME = os.path.expanduser('~/ottertune/client/controller')
+
+# Path to the controller configuration file
+CONTROLLER_CONFIG = os.path.join(CONTROLLER_HOME, 'config/postgres_config.json')
+
+
+#==========================================================
+#  LOGGING OPTIONS
+#==========================================================
+
+LOG_LEVEL = 'DEBUG'
+
+# Path to log directory
+LOG_DIR = os.path.join(DRIVER_HOME, 'log')
+
+# Log files
+DRIVER_LOG = os.path.join(LOG_DIR, 'driver.log')
+OLTPBENCH_LOG = os.path.join(LOG_DIR, 'oltpbench.log')
+CONTROLLER_LOG = os.path.join(LOG_DIR, 'controller.log')
+
+
+#==========================================================
+#  WEBSITE OPTIONS
+#==========================================================
+
+# OtterTune website URL
+WEBSITE_URL = 'http://127.0.0.1:8000'
+
+# Code for uploading new results to the website
+UPLOAD_CODE = 'I5I10PXK3PK27FM86YYS'

client/driver/fabfile.py

@@ -23,38 +23,55 @@ import requests
 from fabric.api import env, local, task, lcd
 from fabric.state import output as fabric_output
 
+import driver_config as dconf
+
+
+def _setup():
+    global LOGIN
+
+    # Determine correct login for the type of host connection
+    if dconf.HOST_CONN == 'local':
+        LOGIN = 'localhost'
+
+    elif dconf.HOST_CONN == 'remote':
+        if not dconf.LOGIN_HOST:
+            raise ValueError("LOGIN_HOST must be set if HOST_CONN=remote")
+
+        LOGIN = dconf.LOGIN_HOST
+        if dconf.LOGIN_NAME:
+            LOGIN = '{}@{}'.format(dconf.LOGIN_NAME, LOGIN)
+
+        if dconf.LOGIN_PORT:
+            LOGIN += ':{}'.format(dconf.LOGIN_PORT)
+
+    elif dconf.HOST_CONN == 'docker':
+        if not dconf.CONTAINER_NAME:
+            raise ValueError("CONTAINER_NAME must be set if HOST_CONN=docker")
+        LOGIN = 'localhost'
+
+    else:
+        raise ValueError(("Invalid HOST_CONN: {}. Valid values are "
+                          "'local', 'remote', or 'docker'.").format(dconf.HOST_CONN))
+
+    # Create all output directories
+    for d in (dconf.RESULT_DIR, dconf.LOG_DIR, dconf.DB_DUMP_DIR):
+        os.makedirs(d, exist_ok=True)
+
+
+_setup()
+
+
 # Fabric environment settings
-env.hosts = ['localhost']
+env.hosts = [LOGIN]
 fabric_output.update({
     'running': True,
     'stdout': True,
 })
 
-# intervals of restoring the databse
-RELOAD_INTERVAL = 10
-# maximum disk usage
-MAX_DISK_USAGE = 90
-
-# Load config
-with open('driver_config.json', 'r') as _f:
-    CONF = {k: os.path.expanduser(v) if isinstance(v, str) and v.startswith('~') else v
-            for k, v in json.load(_f).items()}
-
-# Create output directories
-for _dir in (CONF['database_save_path'], CONF['log_path'], CONF['save_path'],
-             CONF['lhs_save_path']):
-    os.makedirs(_dir, exist_ok=True)
-
-# Define paths
-CONF['driver_log'] = os.path.join(CONF['log_path'], 'driver.log')
-CONF['oltpbench_log'] = os.path.join(CONF['log_path'], 'oltpbench.log')
-CONF['controller_log'] = os.path.join(CONF['log_path'], 'controller.log')
-CONF['controller_config'] = os.path.join(CONF['controller_home'], 'config',
-                                         '{}_config.json'.format(CONF['database_type']))
 
 # Configure logging
 LOG = logging.getLogger(__name__)
-LOG.setLevel(logging.DEBUG)
+LOG.setLevel(getattr(logging, dconf.LOG_LEVEL, logging.DEBUG))
 Formatter = logging.Formatter(  # pylint: disable=invalid-name
     fmt='%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
     datefmt='%m-%d-%Y %H:%M:%S')
@@ -62,7 +79,7 @@ ConsoleHandler = logging.StreamHandler()  # pylint: disable=invalid-name
 ConsoleHandler.setFormatter(Formatter)
 LOG.addHandler(ConsoleHandler)
 FileHandler = RotatingFileHandler(  # pylint: disable=invalid-name
-    CONF['driver_log'], maxBytes=50000, backupCount=2)
+    dconf.DRIVER_LOG, maxBytes=50000, backupCount=2)
 FileHandler.setFormatter(Formatter)
 LOG.addHandler(FileHandler)
 
@@ -75,7 +92,7 @@ def _parse_bool(value):
 
 @task
 def check_disk_usage():
-    partition = CONF['database_disk']
+    partition = dconf.DATABASE_DISK
     disk_use = 0
     if partition:
         cmd = "df -h {}".format(partition)
@@ -95,55 +112,55 @@ def check_memory_usage():
 
 @task
 def create_controller_config():
-    if CONF['database_type'] == 'postgres':
+    if dconf.DB_TYPE == 'postgres':
         dburl_fmt = 'jdbc:postgresql://localhost:5432/{db}'.format
-    elif CONF['database_type'] == 'oracle':
+    elif dconf.DB_TYPE == 'oracle':
         dburl_fmt = 'jdbc:oracle:thin:@localhost:1521:{db}'.format
     else:
-        raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
+        raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
 
     config = dict(
-        database_type=CONF['database_type'],
+        database_type=dconf.DB_TYPE,
-        database_url=dburl_fmt(db=CONF['database_name']),
+        database_url=dburl_fmt(db=dconf.DB_NAME),
-        username=CONF['username'],
+        username=dconf.DB_USER,
-        password=CONF['password'],
+        password=dconf.DB_PASSWORD,
         upload_code='DEPRECATED',
         upload_url='DEPRECATED',
-        workload_name=CONF['oltpbench_workload']
+        workload_name=dconf.OLTPBENCH_BENCH
     )
 
-    with open(CONF['controller_config'], 'w') as f:
+    with open(dconf.CONTROLLER_CONFIG, 'w') as f:
         json.dump(config, f, indent=2)
 
 
 @task
 def restart_database():
-    if CONF['database_type'] == 'postgres':
+    if dconf.DB_TYPE == 'postgres':
-        cmd = 'sudo -u postgres pg_ctl -D {} -w -t 600 restart -m fast'.format(CONF['pg_datadir'])
+        cmd = 'sudo -u postgres pg_ctl -D {} -w -t 600 restart -m fast'.format(dconf.PG_DATADIR)
-    elif CONF['database_type'] == 'oracle':
+    elif dconf.DB_TYPE == 'oracle':
         cmd = 'sh oracleScripts/shutdownOracle.sh && sh oracleScripts/startupOracle.sh'
     else:
-        raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
+        raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
     local(cmd)
 
 
 @task
 def drop_database():
-    if CONF['database_type'] == 'postgres':
+    if dconf.DB_TYPE == 'postgres':
         cmd = "PGPASSWORD={} dropdb -e --if-exists {} -U {}".\
-              format(CONF['password'], CONF['database_name'], CONF['username'])
+              format(dconf.DB_PASSWORD, dconf.DB_NAME, dconf.DB_USER)
     else:
-        raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
+        raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
     local(cmd)
 
 
 @task
 def create_database():
-    if CONF['database_type'] == 'postgres':
+    if dconf.DB_TYPE == 'postgres':
         cmd = "PGPASSWORD={} createdb -e {} -U {}".\
-              format(CONF['password'], CONF['database_name'], CONF['username'])
+              format(dconf.DB_PASSWORD, dconf.DB_NAME, dconf.DB_USER)
     else:
-        raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
+        raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
     local(cmd)
 
 
@@ -157,7 +174,7 @@ def change_conf(next_conf=None):
     signal = "# configurations recommended by ottertune:\n"
     next_conf = next_conf or {}
 
-    with open(CONF['database_conf'], 'r') as f:
+    with open(dconf.DB_CONF, 'r') as f:
         lines = f.readlines()
 
     if signal not in lines:
@@ -165,10 +182,10 @@ def change_conf(next_conf=None):
 
     signal_idx = lines.index(signal)
     lines = lines[0:signal_idx + 1]
-    if CONF.__contains__('base_database_conf'):
+    if dconf.BASE_DB_CONF:
-        with open(CONF['base_database_conf'], 'r') as f:
+        assert isinstance(dconf.BASE_DB_CONF, dict)
-            base_confs = f.readlines()
+        base_conf = ['{} = {}\n'.format(*c) for c in sorted(dconf.BASE_DB_CONF.items())]
-        lines.extend(base_confs)
+        lines.extend(base_conf)
 
     if isinstance(next_conf, str):
         with open(next_conf, 'r') as f:
@@ -180,60 +197,60 @@ def change_conf(next_conf=None):
     assert isinstance(recommendation, dict)
 
     for name, value in recommendation.items():
-        if CONF['database_type'] == 'oracle' and isinstance(value, str):
+        if dconf.DB_TYPE == 'oracle' and isinstance(value, str):
             value = value.strip('B')
         lines.append('{} = {}\n'.format(name, value))
     lines.append('\n')
 
-    tmpconf = 'tmp_' + os.path.basename(CONF['database_conf'])
+    tmpconf = 'tmp_' + os.path.basename(dconf.DB_CONF)
     with open(tmpconf, 'w') as f:
         f.write(''.join(lines))
 
-    local('sudo cp {0} {0}.ottertune.bak'.format(CONF['database_conf']))
+    local('sudo cp {0} {0}.ottertune.bak'.format(dconf.DB_CONF))
-    local('sudo mv {} {}'.format(tmpconf, CONF['database_conf']))
+    local('sudo mv {} {}'.format(tmpconf, dconf.DB_CONF))
 
 
 @task
 def load_oltpbench():
     cmd = "./oltpbenchmark -b {} -c {} --create=true --load=true".\
-          format(CONF['oltpbench_workload'], CONF['oltpbench_config'])
+          format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG)
-    with lcd(CONF['oltpbench_home']):  # pylint: disable=not-context-manager
+    with lcd(dconf.OLTPBENCH_HOME):  # pylint: disable=not-context-manager
         local(cmd)
 
 
 @task
 def run_oltpbench():
     cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile".\
-          format(CONF['oltpbench_workload'], CONF['oltpbench_config'])
+          format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG)
-    with lcd(CONF['oltpbench_home']):  # pylint: disable=not-context-manager
+    with lcd(dconf.OLTPBENCH_HOME):  # pylint: disable=not-context-manager
         local(cmd)
 
 
 @task
 def run_oltpbench_bg():
     cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile > {} 2>&1 &".\
-          format(CONF['oltpbench_workload'], CONF['oltpbench_config'], CONF['oltpbench_log'])
+          format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG, dconf.OLTPBENCH_LOG)
-    with lcd(CONF['oltpbench_home']):  # pylint: disable=not-context-manager
+    with lcd(dconf.OLTPBENCH_HOME):  # pylint: disable=not-context-manager
         local(cmd)
 
 
 @task
 def run_controller():
-    if not os.path.exists(CONF['controller_config']):
+    if not os.path.exists(dconf.CONTROLLER_CONFIG):
         create_controller_config()
     cmd = 'gradle run -PappArgs="-c {} -d output/" --no-daemon > {}'.\
-          format(CONF['controller_config'], CONF['controller_log'])
+          format(dconf.CONTROLLER_CONFIG, dconf.CONTROLLER_LOG)
-    with lcd(CONF['controller_home']):  # pylint: disable=not-context-manager
+    with lcd(dconf.CONTROLLER_HOME):  # pylint: disable=not-context-manager
         local(cmd)
 
 
 @task
 def signal_controller():
-    pidfile = os.path.join(CONF['controller_home'], 'pid.txt')
+    pidfile = os.path.join(dconf.CONTROLLER_HOME, 'pid.txt')
     with open(pidfile, 'r') as f:
         pid = int(f.read())
     cmd = 'sudo kill -2 {}'.format(pid)
-    with lcd(CONF['controller_home']):  # pylint: disable=not-context-manager
+    with lcd(dconf.CONTROLLER_HOME):  # pylint: disable=not-context-manager
         local(cmd)
 
 
@@ -242,8 +259,8 @@ def save_dbms_result():
     t = int(time.time())
     files = ['knobs.json', 'metrics_after.json', 'metrics_before.json', 'summary.json']
     for f_ in files:
-        srcfile = os.path.join(CONF['controller_home'], 'output', f_)
+        srcfile = os.path.join(dconf.CONTROLLER_HOME, 'output', f_)
-        dstfile = os.path.join(CONF['save_path'], '{}__{}'.format(t, f_))
+        dstfile = os.path.join(dconf.RESULT_DIR, '{}__{}'.format(t, f_))
         local('cp {} {}'.format(srcfile, dstfile))
     return t
 
@@ -252,7 +269,7 @@ def save_dbms_result():
 def save_next_config(next_config, t=None):
     if not t:
         t = int(time.time())
-    with open(os.path.join(CONF['save_path'], '{}__next_config.json'.format(t)), 'w') as f:
+    with open(os.path.join(dconf.RESULT_DIR, '{}__next_config.json'.format(t)), 'w') as f:
         json.dump(next_config, f, indent=2)
     return t
 
@@ -265,9 +282,9 @@ def free_cache():
 
 @task
 def upload_result(result_dir=None, prefix=None, upload_code=None):
-    result_dir = result_dir or os.path.join(CONF['controller_home'], 'output')
+    result_dir = result_dir or os.path.join(dconf.CONTROLLER_HOME, 'output')
     prefix = prefix or ''
-    upload_code = upload_code or CONF['upload_code']
+    upload_code = upload_code or dconf.UPLOAD_CODE
 
     files = {}
     for base in ('summary', 'knobs', 'metrics_before', 'metrics_after'):
@@ -275,18 +292,17 @@ def upload_result(result_dir=None, prefix=None, upload_code=None):
 
         # Replaces the true db version with the specified version to allow for
         # testing versions not officially supported by OtterTune
-        if base == 'summary' and 'override_database_version' in CONF and \
+        if base == 'summary' and dconf.OVERRIDE_DB_VERSION:
-                CONF['override_database_version']:
             with open(fpath, 'r') as f:
                 summary = json.load(f)
             summary['real_database_version'] = summary['database_version']
-            summary['database_version'] = CONF['override_database_version']
+            summary['database_version'] = dconf.OVERRIDE_DB_VERSION
             with open(fpath, 'w') as f:
                 json.dump(summary, f, indent=1)
 
         files[base] = open(fpath, 'rb')
 
-    response = requests.post(CONF['upload_url'] + '/new_result/', files=files,
+    response = requests.post(dconf.WEBSITE_URL + '/new_result/', files=files,
                              data={'upload_code': upload_code})
     if response.status_code != 200:
         raise Exception('Error uploading result.\nStatus: {}\nMessage: {}\n'.format(
@@ -304,8 +320,8 @@ def upload_result(result_dir=None, prefix=None, upload_code=None):
 def get_result(max_time_sec=180, interval_sec=5, upload_code=None):
     max_time_sec = int(max_time_sec)
     interval_sec = int(interval_sec)
-    upload_code = upload_code or CONF['upload_code']
+    upload_code = upload_code or dconf.UPLOAD_CODE
-    url = CONF['upload_url'] + '/query_and_get/' + upload_code
+    url = dconf.WEBSITE_URL + '/query_and_get/' + upload_code
     elapsed = 0
     response_dict = None
     response = ''
@@ -354,7 +370,7 @@ def get_result(max_time_sec=180, interval_sec=5, upload_code=None):
 @task
 def download_debug_info(pprint=False):
     pprint = _parse_bool(pprint)
-    url = '{}/dump/{}'.format(CONF['upload_url'], CONF['upload_code'])
+    url = '{}/dump/{}'.format(dconf.WEBSITE_URL, dconf.UPLOAD_CODE)
     params = {'pp': int(True)} if pprint else {}
     rsp = requests.get(url, params=params)
 
@@ -381,7 +397,7 @@ def add_udf():
 
 @task
 def upload_batch(result_dir=None, sort=True, upload_code=None):
-    result_dir = result_dir or CONF['save_path']
+    result_dir = result_dir or dconf.RESULT_DIR
     sort = _parse_bool(sort)
     results = glob.glob(os.path.join(result_dir, '*__summary.json'))
     if sort:
@@ -399,42 +415,42 @@ def upload_batch(result_dir=None, sort=True, upload_code=None):
 
 @task
 def dump_database():
-    db_file_path = os.path.join(CONF['database_save_path'], CONF['database_name'] + '.dump')
+    db_file_path = os.path.join(dconf.DB_DUMP_DIR, dconf.DB_NAME + '.dump')
     if os.path.exists(db_file_path):
         LOG.info('%s already exists ! ', db_file_path)
         return False
     else:
-        LOG.info('Dump database %s to %s', CONF['database_name'], db_file_path)
+        LOG.info('Dump database %s to %s', dconf.DB_NAME, db_file_path)
         # You must create a directory named dpdata through sqlplus in your Oracle database
-        if CONF['database_type'] == 'oracle':
+        if dconf.DB_TYPE == 'oracle':
             cmd = 'expdp {}/{}@{} schemas={} dumpfile={}.dump DIRECTORY=dpdata'.format(
                 'c##tpcc', 'oracle', 'orcldb', 'c##tpcc', 'orcldb')
-        elif CONF['database_type'] == 'postgres':
+        elif dconf.DB_TYPE == 'postgres':
-            cmd = 'PGPASSWORD={} pg_dump -U {} -F c -d {} > {}'.format(CONF['password'],
+            cmd = 'PGPASSWORD={} pg_dump -U {} -F c -d {} > {}'.format(dconf.DB_PASSWORD,
-                                                                       CONF['username'],
+                                                                       dconf.DB_USER,
-                                                                       CONF['database_name'],
+                                                                       dconf.DB_NAME,
                                                                        db_file_path)
         else:
-            raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
+            raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
         local(cmd)
         return True
 
 
 @task
 def restore_database():
-    if CONF['database_type'] == 'oracle':
+    if dconf.DB_TYPE == 'oracle':
         # You must create a directory named dpdata through sqlplus in your Oracle database
         # The following script assumes such directory exists.
         # You may want to modify the username, password, and dump file name in the script
         cmd = 'sh oracleScripts/restoreOracle.sh'
-    elif CONF['database_type'] == 'postgres':
+    elif dconf.DB_TYPE == 'postgres':
-        db_file_path = '{}/{}.dump'.format(CONF['database_save_path'], CONF['database_name'])
+        db_file_path = '{}/{}.dump'.format(dconf.DB_DUMP_DIR, dconf.DB_NAME)
         drop_database()
         create_database()
         cmd = 'PGPASSWORD={} pg_restore -U {} -n public -j 8 -F c -d {} {}'.\
-              format(CONF['password'], CONF['username'], CONF['database_name'], db_file_path)
+              format(dconf.DB_PASSWORD, dconf.DB_USER, dconf.DB_NAME, db_file_path)
     else:
-        raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
+        raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
     LOG.info('Start restoring database')
     local(cmd)
     LOG.info('Finish restoring database')
@@ -442,8 +458,8 @@ def restore_database():
 
 def _ready_to_start_oltpbench():
     ready = False
-    if os.path.exists(CONF['controller_log']):
+    if os.path.exists(dconf.CONTROLLER_LOG):
-        with open(CONF['controller_log'], 'r') as f:
+        with open(dconf.CONTROLLER_LOG, 'r') as f:
             content = f.read()
         ready = 'Output the process pid to' in content
     return ready
@@ -451,18 +467,18 @@ def _ready_to_start_oltpbench():
 
 def _ready_to_start_controller():
     ready = False
-    if os.path.exists(CONF['oltpbench_log']):
+    if os.path.exists(dconf.OLTPBENCH_LOG):
-        with open(CONF['oltpbench_log'], 'r') as f:
+        with open(dconf.OLTPBENCH_LOG, 'r') as f:
             content = f.read()
         ready = 'Warmup complete, starting measurements' in content
     return ready
 
 
 def _ready_to_shut_down_controller():
-    pidfile = os.path.join(CONF['controller_home'], 'pid.txt')
+    pidfile = os.path.join(dconf.CONTROLLER_HOME, 'pid.txt')
     ready = False
-    if os.path.exists(pidfile) and os.path.exists(CONF['oltpbench_log']):
+    if os.path.exists(pidfile) and os.path.exists(dconf.OLTPBENCH_LOG):
-        with open(CONF['oltpbench_log'], 'r') as f:
+        with open(dconf.OLTPBENCH_LOG, 'r') as f:
             content = f.read()
         ready = 'Output throughput samples into file' in content
     return ready
@@ -470,17 +486,11 @@ def _ready_to_shut_down_controller():
 
 def clean_logs():
     # remove oltpbench log
-    cmd = 'rm -f {}'.format(CONF['oltpbench_log'])
+    cmd = 'rm -f {}'.format(dconf.OLTPBENCH_LOG)
     local(cmd)
 
     # remove controller log
-    cmd = 'rm -f {}'.format(CONF['controller_log'])
+    cmd = 'rm -f {}'.format(dconf.CONTROLLER_LOG)
-    local(cmd)
-
-
-@task
-def lhs_samples(count=10):
-    cmd = 'python3 lhs.py {} {} {}'.format(count, CONF['lhs_knob_path'], CONF['lhs_save_path'])
     local(cmd)
 
 
@@ -494,11 +504,11 @@ def loop(i):
 
     # restart database
     restart_database()
-    time.sleep(CONF['sleep_time'])
+    time.sleep(dconf.RESTART_SLEEP_SEC)
 
     # check disk usage
-    if check_disk_usage() > MAX_DISK_USAGE:
+    if check_disk_usage() > dconf.MAX_DISK_USAGE:
-        LOG.warning('Exceeds max disk usage %s', MAX_DISK_USAGE)
+        LOG.warning('Exceeds max disk usage %s', dconf.MAX_DISK_USAGE)
 
     # run controller from another process
     p = Process(target=run_controller, args=())
@@ -532,7 +542,7 @@ def loop(i):
     # save result
     result_timestamp = save_dbms_result()
 
-    if i >= CONF['warmup_iterations']:
+    if i >= dconf.WARMUP_ITERATIONS:
         # upload result
         upload_result()
 
@@ -546,96 +556,14 @@ def loop(i):
         change_conf(response['recommendation'])
 
 
-@task
-def run_lhs():
-    datadir = CONF['lhs_save_path']
-    samples = glob.glob(os.path.join(datadir, 'config_*'))
-
-    # dump database if it's not done before.
-    dump = dump_database()
-
-    result_timestamp = None
-    for i, sample in enumerate(samples):
-        # reload database periodically
-        if RELOAD_INTERVAL > 0:
-            if i % RELOAD_INTERVAL == 0:
-                if i == 0 and dump is False:
-                    restore_database()
-                elif i > 0:
-                    restore_database()
-        # free cache
-        free_cache()
-
-        LOG.info('\n\n Start %s-th sample %s \n\n', i, sample)
-        # check memory usage
-        # check_memory_usage()
-
-        # check disk usage
-        if check_disk_usage() > MAX_DISK_USAGE:
-            LOG.warning('Exceeds max disk usage %s', MAX_DISK_USAGE)
-
-        # load the next lhs-sampled config
-        with open(sample, 'r') as f:
-            next_config = json.load(f, object_pairs_hook=OrderedDict)
-        save_next_config(next_config, t=result_timestamp)
-
-        # remove oltpbench log and controller log
-        clean_logs()
-
-        # change config
-        change_conf(next_config)
-
-        # restart database
-        restart_database()
-
-        if CONF.get('oracle_awr_enabled', False):
-            # create snapshot for oracle AWR report
-            if CONF['database_type'] == 'oracle':
-                local('sh snapshotOracle.sh')
-
-        # run controller from another process
-        p = Process(target=run_controller, args=())
-        p.start()
-
-        # run oltpbench as a background job
-        while not _ready_to_start_oltpbench():
-            pass
-        run_oltpbench_bg()
-        LOG.info('Run OLTP-Bench')
-
-        while not _ready_to_start_controller():
-            pass
-        signal_controller()
-        LOG.info('Start the first collection')
-
-        while not _ready_to_shut_down_controller():
-            pass
-        # stop the experiment
-        signal_controller()
-        LOG.info('Start the second collection, shut down the controller')
-
-        p.join()
-
-        # save result
-        result_timestamp = save_dbms_result()
-
-        # upload result
-        upload_result()
-
-        if CONF.get('oracle_awr_enabled', False):
-            # create oracle AWR report for performance analysis
-            if CONF['database_type'] == 'oracle':
-                local('sh oracleScripts/snapshotOracle.sh && sh oracleScripts/awrOracle.sh')
-
-
 @task
 def run_loops(max_iter=1):
     # dump database if it's not done before.
     dump = dump_database()
 
     for i in range(int(max_iter)):
-        if RELOAD_INTERVAL > 0:
+        if dconf.RELOAD_INTERVAL > 0:
-            if i % RELOAD_INTERVAL == 0:
+            if i % dconf.RELOAD_INTERVAL == 0:
                 if i == 0 and dump is False:
                     restart_database()
                     restore_database()
@@ -643,13 +571,13 @@ def run_loops(max_iter=1):
                     restore_database()
 
         LOG.info('The %s-th Loop Starts / Total Loops %s', i + 1, max_iter)
-        loop(i % RELOAD_INTERVAL)
+        loop(i % dconf.RELOAD_INTERVAL)
         LOG.info('The %s-th Loop Ends / Total Loops %s', i + 1, max_iter)
 
 
 @task
 def rename_batch(result_dir=None):
-    result_dir = result_dir or CONF['save_path']
+    result_dir = result_dir or dconf.RESULT_DIR
     results = glob.glob(os.path.join(result_dir, '*__summary.json'))
     results = sorted(results)
     for i, result in enumerate(results):
@@ -752,7 +680,7 @@ def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10):
     elapsed = 0
 
     while elapsed <= max_time_sec:
-        response = requests.get(CONF['upload_url'] + '/test/pipeline/')
+        response = requests.get(dconf.WEBSITE_URL + '/test/pipeline/')
         response = response.content
         LOG.info(response)
         if 'False' in str(response):
@@ -766,7 +694,7 @@ def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10):
 def integration_tests():
 
     # Create test website
-    response = requests.get(CONF['upload_url'] + '/test/create/')
+    response = requests.get(dconf.WEBSITE_URL + '/test/create/')
     LOG.info(response.content)
 
     # Upload training data