Changed the format of the driver config from JSON to a python settings file to make it easier to update / read.
This commit is contained in:
dvanaken
Dana Van Aken
parent
159e26e1c7
commit
8ee52a64bf
|
|
@ -1,4 +0,0 @@
|
|||
track_counts = on
|
||||
track_functions = all
|
||||
track_io_timing = on
|
||||
autovacuum = off
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
{
|
||||
"database_type" : "postgres",
|
||||
"database_name" : "tpcc",
|
||||
"database_disk": "",
|
||||
"database_conf": "/etc/postgresql/9.6/main/postgresql.conf",
|
||||
"base_database_conf": "configs/postgres_base.conf",
|
||||
"pg_datadir": "/var/lib/postgresql/9.6/main",
|
||||
"database_save_path": "~/ottertune/client/driver/dumpfiles",
|
||||
"username" : "dbuser",
|
||||
"password" : "dbpassword",
|
||||
"oltpbench_home": "~/oltpbench",
|
||||
"oltpbench_config": "~/oltpbench/config/tpcc_config_postgres.xml",
|
||||
"oltpbench_workload": "tpcc",
|
||||
"controller_home": "~/ottertune/client/controller",
|
||||
"log_path": "~/ottertune/client/driver/log",
|
||||
"save_path": "~/ottertune/client/driver/results",
|
||||
"upload_url" : "http://127.0.0.1:8000",
|
||||
"upload_code" : "I5I10PXK3PK27FM86YYS",
|
||||
"lhs_knob_path" : "~/ottertune/client/driver/knobs/postgres-96.json",
|
||||
"lhs_save_path" : "~/ottertune/client/driver/configs",
|
||||
"oracle_awr_enabled": false,
|
||||
"warmup_iterations": 0,
|
||||
"sleep_time": 300
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
import os
|
||||
|
||||
#==========================================================
|
||||
# HOST LOGIN
|
||||
#==========================================================
|
||||
|
||||
# Location of the database host relative to this driver
|
||||
# Valid values: local, remote, or docker
|
||||
HOST_CONN = 'local'
|
||||
|
||||
# The name of the Docker container for the target database
|
||||
# (only required if HOST_CONN = docker)
|
||||
CONTAINER_NAME = None # e.g., 'postgres_container'
|
||||
|
||||
# Host SSH login credentials (only required if HOST_CONN=remote)
|
||||
LOGIN_NAME = None
|
||||
LOGIN_HOST = None
|
||||
LOGIN_PORT = None # Set when using a port other than the SSH default
|
||||
|
||||
|
||||
#==========================================================
|
||||
# DATABASE OPTIONS
|
||||
#==========================================================
|
||||
|
||||
# Either Postgres or Oracle
|
||||
DB_TYPE = 'postgres'
|
||||
|
||||
# Name of the database
|
||||
DB_NAME = 'tpcc'
|
||||
|
||||
# Database username
|
||||
DB_USER = 'dbuser'
|
||||
|
||||
# Password for DB_USER
|
||||
DB_PASSWORD = 'dbpassword'
|
||||
|
||||
# Database host address
|
||||
DB_HOST = 'localhost'
|
||||
|
||||
# Database port
|
||||
DB_PORT = '5432'
|
||||
|
||||
# Path to the configuration file on the database server
|
||||
DB_CONF = '/etc/postgresql/9.6/main/postgresql.conf'
|
||||
|
||||
# Base config settings to always include when installing new configurations
|
||||
BASE_DB_CONF = {
|
||||
'track_counts': 'on',
|
||||
'track_functions': 'all',
|
||||
'track_io_timing': 'on',
|
||||
'autovacuum': 'off',
|
||||
}
|
||||
|
||||
# Name of the device on the database server to monitor the disk usage, or None to disable
|
||||
DATABASE_DISK = None
|
||||
|
||||
# Set this to a different database version to override the current version
|
||||
OVERRIDE_DB_VERSION = None
|
||||
|
||||
# POSTGRES-SPECIFIC OPTIONS >>>
|
||||
PG_DATADIR = '/var/lib/postgresql/9.6/main'
|
||||
|
||||
# ORACLE-SPECIFIC OPTIONS >>>
|
||||
ORACLE_AWR_ENABLED = False
|
||||
|
||||
|
||||
#==========================================================
|
||||
# DRIVER OPTIONS
|
||||
#==========================================================
|
||||
|
||||
# Path to this driver
|
||||
DRIVER_HOME = os.path.realpath(__file__)
|
||||
print('DRIVER HOME: {}'.format(DRIVER_HOME))
|
||||
|
||||
# Path to the directory for storing results
|
||||
RESULT_DIR = os.path.join(DRIVER_HOME, 'results')
|
||||
|
||||
# Path to the directory for storing database dump files
|
||||
DB_DUMP_DIR = os.path.join(DRIVER_HOME, 'dumpfiles')
|
||||
|
||||
# Reload the database after running this many iterations
|
||||
RELOAD_INTERVAL = 10
|
||||
|
||||
# The maximum allowable disk usage percentage. Reload the database
|
||||
# whenever the current disk usage exceeds this value.
|
||||
MAX_DISK_USAGE = 90
|
||||
|
||||
# Execute this many warmup iterations before uploading the next result
|
||||
# to the website
|
||||
WARMUP_ITERATIONS = 0
|
||||
|
||||
# Let the database initialize for this many seconds after it restarts
|
||||
RESTART_SLEEP_SEC = 300
|
||||
|
||||
#==========================================================
|
||||
# OLTPBENCHMARK OPTIONS
|
||||
#==========================================================
|
||||
|
||||
# Path to OLTPBench directory
|
||||
OLTPBENCH_HOME = os.path.expanduser('~/oltpbench')
|
||||
|
||||
# Path to the OLTPBench configuration file
|
||||
OLTPBENCH_CONFIG = os.path.join(OLTPBENCH_HOME, 'config/tpcc_config_postgres.xml')
|
||||
|
||||
# Name of the benchmark to run
|
||||
OLTPBENCH_BENCH = 'tpcc'
|
||||
|
||||
|
||||
#==========================================================
|
||||
# CONTROLLER OPTIONS
|
||||
#==========================================================
|
||||
|
||||
# Path to the controller directory
|
||||
CONTROLLER_HOME = os.path.expanduser('~/ottertune/client/controller')
|
||||
|
||||
# Path to the controller configuration file
|
||||
CONTROLLER_CONFIG = os.path.join(CONTROLLER_HOME, 'config/postgres_config.json')
|
||||
|
||||
|
||||
#==========================================================
|
||||
# LOGGING OPTIONS
|
||||
#==========================================================
|
||||
|
||||
LOG_LEVEL = 'DEBUG'
|
||||
|
||||
# Path to log directory
|
||||
LOG_DIR = os.path.join(DRIVER_HOME, 'log')
|
||||
|
||||
# Log files
|
||||
DRIVER_LOG = os.path.join(LOG_DIR, 'driver.log')
|
||||
OLTPBENCH_LOG = os.path.join(LOG_DIR, 'oltpbench.log')
|
||||
CONTROLLER_LOG = os.path.join(LOG_DIR, 'controller.log')
|
||||
|
||||
|
||||
#==========================================================
|
||||
# WEBSITE OPTIONS
|
||||
#==========================================================
|
||||
|
||||
# OtterTune website URL
|
||||
WEBSITE_URL = 'http://127.0.0.1:8000'
|
||||
|
||||
# Code for uploading new results to the website
|
||||
UPLOAD_CODE = 'I5I10PXK3PK27FM86YYS'
|
||||
|
|
@ -23,38 +23,55 @@ import requests
|
|||
from fabric.api import env, local, task, lcd
|
||||
from fabric.state import output as fabric_output
|
||||
|
||||
import driver_config as dconf
|
||||
|
||||
|
||||
def _setup():
|
||||
global LOGIN
|
||||
|
||||
# Determine correct login for the type of host connection
|
||||
if dconf.HOST_CONN == 'local':
|
||||
LOGIN = 'localhost'
|
||||
|
||||
elif dconf.HOST_CONN == 'remote':
|
||||
if not dconf.LOGIN_HOST:
|
||||
raise ValueError("LOGIN_HOST must be set if HOST_CONN=remote")
|
||||
|
||||
LOGIN = dconf.LOGIN_HOST
|
||||
if dconf.LOGIN_NAME:
|
||||
LOGIN = '{}@{}'.format(dconf.LOGIN_NAME, LOGIN)
|
||||
|
||||
if dconf.LOGIN_PORT:
|
||||
LOGIN += ':{}'.format(dconf.LOGIN_PORT)
|
||||
|
||||
elif dconf.HOST_CONN == 'docker':
|
||||
if not dconf.CONTAINER_NAME:
|
||||
raise ValueError("CONTAINER_NAME must be set if HOST_CONN=docker")
|
||||
LOGIN = 'localhost'
|
||||
|
||||
else:
|
||||
raise ValueError(("Invalid HOST_CONN: {}. Valid values are "
|
||||
"'local', 'remote', or 'docker'.").format(dconf.HOST_CONN))
|
||||
|
||||
# Create all output directories
|
||||
for d in (dconf.RESULT_DIR, dconf.LOG_DIR, dconf.DB_DUMP_DIR):
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
|
||||
_setup()
|
||||
|
||||
|
||||
# Fabric environment settings
|
||||
env.hosts = ['localhost']
|
||||
env.hosts = [LOGIN]
|
||||
fabric_output.update({
|
||||
'running': True,
|
||||
'stdout': True,
|
||||
})
|
||||
|
||||
# intervals of restoring the databse
|
||||
RELOAD_INTERVAL = 10
|
||||
# maximum disk usage
|
||||
MAX_DISK_USAGE = 90
|
||||
|
||||
# Load config
|
||||
with open('driver_config.json', 'r') as _f:
|
||||
CONF = {k: os.path.expanduser(v) if isinstance(v, str) and v.startswith('~') else v
|
||||
for k, v in json.load(_f).items()}
|
||||
|
||||
# Create output directories
|
||||
for _dir in (CONF['database_save_path'], CONF['log_path'], CONF['save_path'],
|
||||
CONF['lhs_save_path']):
|
||||
os.makedirs(_dir, exist_ok=True)
|
||||
|
||||
# Define paths
|
||||
CONF['driver_log'] = os.path.join(CONF['log_path'], 'driver.log')
|
||||
CONF['oltpbench_log'] = os.path.join(CONF['log_path'], 'oltpbench.log')
|
||||
CONF['controller_log'] = os.path.join(CONF['log_path'], 'controller.log')
|
||||
CONF['controller_config'] = os.path.join(CONF['controller_home'], 'config',
|
||||
'{}_config.json'.format(CONF['database_type']))
|
||||
|
||||
# Configure logging
|
||||
LOG = logging.getLogger(__name__)
|
||||
LOG.setLevel(logging.DEBUG)
|
||||
LOG.setLevel(getattr(logging, dconf.LOG_LEVEL, logging.DEBUG))
|
||||
Formatter = logging.Formatter( # pylint: disable=invalid-name
|
||||
fmt='%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
|
||||
datefmt='%m-%d-%Y %H:%M:%S')
|
||||
|
|
@ -62,7 +79,7 @@ ConsoleHandler = logging.StreamHandler() # pylint: disable=invalid-name
|
|||
ConsoleHandler.setFormatter(Formatter)
|
||||
LOG.addHandler(ConsoleHandler)
|
||||
FileHandler = RotatingFileHandler( # pylint: disable=invalid-name
|
||||
CONF['driver_log'], maxBytes=50000, backupCount=2)
|
||||
dconf.DRIVER_LOG, maxBytes=50000, backupCount=2)
|
||||
FileHandler.setFormatter(Formatter)
|
||||
LOG.addHandler(FileHandler)
|
||||
|
||||
|
|
@ -75,7 +92,7 @@ def _parse_bool(value):
|
|||
|
||||
@task
|
||||
def check_disk_usage():
|
||||
partition = CONF['database_disk']
|
||||
partition = dconf.DATABASE_DISK
|
||||
disk_use = 0
|
||||
if partition:
|
||||
cmd = "df -h {}".format(partition)
|
||||
|
|
@ -95,55 +112,55 @@ def check_memory_usage():
|
|||
|
||||
@task
|
||||
def create_controller_config():
|
||||
if CONF['database_type'] == 'postgres':
|
||||
if dconf.DB_TYPE == 'postgres':
|
||||
dburl_fmt = 'jdbc:postgresql://localhost:5432/{db}'.format
|
||||
elif CONF['database_type'] == 'oracle':
|
||||
elif dconf.DB_TYPE == 'oracle':
|
||||
dburl_fmt = 'jdbc:oracle:thin:@localhost:1521:{db}'.format
|
||||
else:
|
||||
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
|
||||
|
||||
config = dict(
|
||||
database_type=CONF['database_type'],
|
||||
database_url=dburl_fmt(db=CONF['database_name']),
|
||||
username=CONF['username'],
|
||||
password=CONF['password'],
|
||||
database_type=dconf.DB_TYPE,
|
||||
database_url=dburl_fmt(db=dconf.DB_NAME),
|
||||
username=dconf.DB_USER,
|
||||
password=dconf.DB_PASSWORD,
|
||||
upload_code='DEPRECATED',
|
||||
upload_url='DEPRECATED',
|
||||
workload_name=CONF['oltpbench_workload']
|
||||
workload_name=dconf.OLTPBENCH_BENCH
|
||||
)
|
||||
|
||||
with open(CONF['controller_config'], 'w') as f:
|
||||
with open(dconf.CONTROLLER_CONFIG, 'w') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
|
||||
|
||||
@task
|
||||
def restart_database():
|
||||
if CONF['database_type'] == 'postgres':
|
||||
cmd = 'sudo -u postgres pg_ctl -D {} -w -t 600 restart -m fast'.format(CONF['pg_datadir'])
|
||||
elif CONF['database_type'] == 'oracle':
|
||||
if dconf.DB_TYPE == 'postgres':
|
||||
cmd = 'sudo -u postgres pg_ctl -D {} -w -t 600 restart -m fast'.format(dconf.PG_DATADIR)
|
||||
elif dconf.DB_TYPE == 'oracle':
|
||||
cmd = 'sh oracleScripts/shutdownOracle.sh && sh oracleScripts/startupOracle.sh'
|
||||
else:
|
||||
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
|
||||
local(cmd)
|
||||
|
||||
|
||||
@task
|
||||
def drop_database():
|
||||
if CONF['database_type'] == 'postgres':
|
||||
if dconf.DB_TYPE == 'postgres':
|
||||
cmd = "PGPASSWORD={} dropdb -e --if-exists {} -U {}".\
|
||||
format(CONF['password'], CONF['database_name'], CONF['username'])
|
||||
format(dconf.DB_PASSWORD, dconf.DB_NAME, dconf.DB_USER)
|
||||
else:
|
||||
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
|
||||
local(cmd)
|
||||
|
||||
|
||||
@task
|
||||
def create_database():
|
||||
if CONF['database_type'] == 'postgres':
|
||||
if dconf.DB_TYPE == 'postgres':
|
||||
cmd = "PGPASSWORD={} createdb -e {} -U {}".\
|
||||
format(CONF['password'], CONF['database_name'], CONF['username'])
|
||||
format(dconf.DB_PASSWORD, dconf.DB_NAME, dconf.DB_USER)
|
||||
else:
|
||||
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
|
||||
local(cmd)
|
||||
|
||||
|
||||
|
|
@ -157,7 +174,7 @@ def change_conf(next_conf=None):
|
|||
signal = "# configurations recommended by ottertune:\n"
|
||||
next_conf = next_conf or {}
|
||||
|
||||
with open(CONF['database_conf'], 'r') as f:
|
||||
with open(dconf.DB_CONF, 'r') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
if signal not in lines:
|
||||
|
|
@ -165,10 +182,10 @@ def change_conf(next_conf=None):
|
|||
|
||||
signal_idx = lines.index(signal)
|
||||
lines = lines[0:signal_idx + 1]
|
||||
if CONF.__contains__('base_database_conf'):
|
||||
with open(CONF['base_database_conf'], 'r') as f:
|
||||
base_confs = f.readlines()
|
||||
lines.extend(base_confs)
|
||||
if dconf.BASE_DB_CONF:
|
||||
assert isinstance(dconf.BASE_DB_CONF, dict)
|
||||
base_conf = ['{} = {}\n'.format(*c) for c in sorted(dconf.BASE_DB_CONF.items())]
|
||||
lines.extend(base_conf)
|
||||
|
||||
if isinstance(next_conf, str):
|
||||
with open(next_conf, 'r') as f:
|
||||
|
|
@ -180,60 +197,60 @@ def change_conf(next_conf=None):
|
|||
assert isinstance(recommendation, dict)
|
||||
|
||||
for name, value in recommendation.items():
|
||||
if CONF['database_type'] == 'oracle' and isinstance(value, str):
|
||||
if dconf.DB_TYPE == 'oracle' and isinstance(value, str):
|
||||
value = value.strip('B')
|
||||
lines.append('{} = {}\n'.format(name, value))
|
||||
lines.append('\n')
|
||||
|
||||
tmpconf = 'tmp_' + os.path.basename(CONF['database_conf'])
|
||||
tmpconf = 'tmp_' + os.path.basename(dconf.DB_CONF)
|
||||
with open(tmpconf, 'w') as f:
|
||||
f.write(''.join(lines))
|
||||
|
||||
local('sudo cp {0} {0}.ottertune.bak'.format(CONF['database_conf']))
|
||||
local('sudo mv {} {}'.format(tmpconf, CONF['database_conf']))
|
||||
local('sudo cp {0} {0}.ottertune.bak'.format(dconf.DB_CONF))
|
||||
local('sudo mv {} {}'.format(tmpconf, dconf.DB_CONF))
|
||||
|
||||
|
||||
@task
|
||||
def load_oltpbench():
|
||||
cmd = "./oltpbenchmark -b {} -c {} --create=true --load=true".\
|
||||
format(CONF['oltpbench_workload'], CONF['oltpbench_config'])
|
||||
with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager
|
||||
format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG)
|
||||
with lcd(dconf.OLTPBENCH_HOME): # pylint: disable=not-context-manager
|
||||
local(cmd)
|
||||
|
||||
|
||||
@task
|
||||
def run_oltpbench():
|
||||
cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile".\
|
||||
format(CONF['oltpbench_workload'], CONF['oltpbench_config'])
|
||||
with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager
|
||||
format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG)
|
||||
with lcd(dconf.OLTPBENCH_HOME): # pylint: disable=not-context-manager
|
||||
local(cmd)
|
||||
|
||||
|
||||
@task
|
||||
def run_oltpbench_bg():
|
||||
cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile > {} 2>&1 &".\
|
||||
format(CONF['oltpbench_workload'], CONF['oltpbench_config'], CONF['oltpbench_log'])
|
||||
with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager
|
||||
format(dconf.OLTPBENCH_BENCH, dconf.OLTPBENCH_CONFIG, dconf.OLTPBENCH_LOG)
|
||||
with lcd(dconf.OLTPBENCH_HOME): # pylint: disable=not-context-manager
|
||||
local(cmd)
|
||||
|
||||
|
||||
@task
|
||||
def run_controller():
|
||||
if not os.path.exists(CONF['controller_config']):
|
||||
if not os.path.exists(dconf.CONTROLLER_CONFIG):
|
||||
create_controller_config()
|
||||
cmd = 'gradle run -PappArgs="-c {} -d output/" --no-daemon > {}'.\
|
||||
format(CONF['controller_config'], CONF['controller_log'])
|
||||
with lcd(CONF['controller_home']): # pylint: disable=not-context-manager
|
||||
format(dconf.CONTROLLER_CONFIG, dconf.CONTROLLER_LOG)
|
||||
with lcd(dconf.CONTROLLER_HOME): # pylint: disable=not-context-manager
|
||||
local(cmd)
|
||||
|
||||
|
||||
@task
|
||||
def signal_controller():
|
||||
pidfile = os.path.join(CONF['controller_home'], 'pid.txt')
|
||||
pidfile = os.path.join(dconf.CONTROLLER_HOME, 'pid.txt')
|
||||
with open(pidfile, 'r') as f:
|
||||
pid = int(f.read())
|
||||
cmd = 'sudo kill -2 {}'.format(pid)
|
||||
with lcd(CONF['controller_home']): # pylint: disable=not-context-manager
|
||||
with lcd(dconf.CONTROLLER_HOME): # pylint: disable=not-context-manager
|
||||
local(cmd)
|
||||
|
||||
|
||||
|
|
@ -242,8 +259,8 @@ def save_dbms_result():
|
|||
t = int(time.time())
|
||||
files = ['knobs.json', 'metrics_after.json', 'metrics_before.json', 'summary.json']
|
||||
for f_ in files:
|
||||
srcfile = os.path.join(CONF['controller_home'], 'output', f_)
|
||||
dstfile = os.path.join(CONF['save_path'], '{}__{}'.format(t, f_))
|
||||
srcfile = os.path.join(dconf.CONTROLLER_HOME, 'output', f_)
|
||||
dstfile = os.path.join(dconf.RESULT_DIR, '{}__{}'.format(t, f_))
|
||||
local('cp {} {}'.format(srcfile, dstfile))
|
||||
return t
|
||||
|
||||
|
|
@ -252,7 +269,7 @@ def save_dbms_result():
|
|||
def save_next_config(next_config, t=None):
|
||||
if not t:
|
||||
t = int(time.time())
|
||||
with open(os.path.join(CONF['save_path'], '{}__next_config.json'.format(t)), 'w') as f:
|
||||
with open(os.path.join(dconf.RESULT_DIR, '{}__next_config.json'.format(t)), 'w') as f:
|
||||
json.dump(next_config, f, indent=2)
|
||||
return t
|
||||
|
||||
|
|
@ -265,9 +282,9 @@ def free_cache():
|
|||
|
||||
@task
|
||||
def upload_result(result_dir=None, prefix=None, upload_code=None):
|
||||
result_dir = result_dir or os.path.join(CONF['controller_home'], 'output')
|
||||
result_dir = result_dir or os.path.join(dconf.CONTROLLER_HOME, 'output')
|
||||
prefix = prefix or ''
|
||||
upload_code = upload_code or CONF['upload_code']
|
||||
upload_code = upload_code or dconf.UPLOAD_CODE
|
||||
|
||||
files = {}
|
||||
for base in ('summary', 'knobs', 'metrics_before', 'metrics_after'):
|
||||
|
|
@ -275,18 +292,17 @@ def upload_result(result_dir=None, prefix=None, upload_code=None):
|
|||
|
||||
# Replaces the true db version with the specified version to allow for
|
||||
# testing versions not officially supported by OtterTune
|
||||
if base == 'summary' and 'override_database_version' in CONF and \
|
||||
CONF['override_database_version']:
|
||||
if base == 'summary' and dconf.OVERRIDE_DB_VERSION:
|
||||
with open(fpath, 'r') as f:
|
||||
summary = json.load(f)
|
||||
summary['real_database_version'] = summary['database_version']
|
||||
summary['database_version'] = CONF['override_database_version']
|
||||
summary['database_version'] = dconf.OVERRIDE_DB_VERSION
|
||||
with open(fpath, 'w') as f:
|
||||
json.dump(summary, f, indent=1)
|
||||
|
||||
files[base] = open(fpath, 'rb')
|
||||
|
||||
response = requests.post(CONF['upload_url'] + '/new_result/', files=files,
|
||||
response = requests.post(dconf.WEBSITE_URL + '/new_result/', files=files,
|
||||
data={'upload_code': upload_code})
|
||||
if response.status_code != 200:
|
||||
raise Exception('Error uploading result.\nStatus: {}\nMessage: {}\n'.format(
|
||||
|
|
@ -304,8 +320,8 @@ def upload_result(result_dir=None, prefix=None, upload_code=None):
|
|||
def get_result(max_time_sec=180, interval_sec=5, upload_code=None):
|
||||
max_time_sec = int(max_time_sec)
|
||||
interval_sec = int(interval_sec)
|
||||
upload_code = upload_code or CONF['upload_code']
|
||||
url = CONF['upload_url'] + '/query_and_get/' + upload_code
|
||||
upload_code = upload_code or dconf.UPLOAD_CODE
|
||||
url = dconf.WEBSITE_URL + '/query_and_get/' + upload_code
|
||||
elapsed = 0
|
||||
response_dict = None
|
||||
response = ''
|
||||
|
|
@ -354,7 +370,7 @@ def get_result(max_time_sec=180, interval_sec=5, upload_code=None):
|
|||
@task
|
||||
def download_debug_info(pprint=False):
|
||||
pprint = _parse_bool(pprint)
|
||||
url = '{}/dump/{}'.format(CONF['upload_url'], CONF['upload_code'])
|
||||
url = '{}/dump/{}'.format(dconf.WEBSITE_URL, dconf.UPLOAD_CODE)
|
||||
params = {'pp': int(True)} if pprint else {}
|
||||
rsp = requests.get(url, params=params)
|
||||
|
||||
|
|
@ -381,7 +397,7 @@ def add_udf():
|
|||
|
||||
@task
|
||||
def upload_batch(result_dir=None, sort=True, upload_code=None):
|
||||
result_dir = result_dir or CONF['save_path']
|
||||
result_dir = result_dir or dconf.RESULT_DIR
|
||||
sort = _parse_bool(sort)
|
||||
results = glob.glob(os.path.join(result_dir, '*__summary.json'))
|
||||
if sort:
|
||||
|
|
@ -399,42 +415,42 @@ def upload_batch(result_dir=None, sort=True, upload_code=None):
|
|||
|
||||
@task
|
||||
def dump_database():
|
||||
db_file_path = os.path.join(CONF['database_save_path'], CONF['database_name'] + '.dump')
|
||||
db_file_path = os.path.join(dconf.DB_DUMP_DIR, dconf.DB_NAME + '.dump')
|
||||
if os.path.exists(db_file_path):
|
||||
LOG.info('%s already exists ! ', db_file_path)
|
||||
return False
|
||||
else:
|
||||
LOG.info('Dump database %s to %s', CONF['database_name'], db_file_path)
|
||||
LOG.info('Dump database %s to %s', dconf.DB_NAME, db_file_path)
|
||||
# You must create a directory named dpdata through sqlplus in your Oracle database
|
||||
if CONF['database_type'] == 'oracle':
|
||||
if dconf.DB_TYPE == 'oracle':
|
||||
cmd = 'expdp {}/{}@{} schemas={} dumpfile={}.dump DIRECTORY=dpdata'.format(
|
||||
'c##tpcc', 'oracle', 'orcldb', 'c##tpcc', 'orcldb')
|
||||
elif CONF['database_type'] == 'postgres':
|
||||
cmd = 'PGPASSWORD={} pg_dump -U {} -F c -d {} > {}'.format(CONF['password'],
|
||||
CONF['username'],
|
||||
CONF['database_name'],
|
||||
elif dconf.DB_TYPE == 'postgres':
|
||||
cmd = 'PGPASSWORD={} pg_dump -U {} -F c -d {} > {}'.format(dconf.DB_PASSWORD,
|
||||
dconf.DB_USER,
|
||||
dconf.DB_NAME,
|
||||
db_file_path)
|
||||
else:
|
||||
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
|
||||
local(cmd)
|
||||
return True
|
||||
|
||||
|
||||
@task
|
||||
def restore_database():
|
||||
if CONF['database_type'] == 'oracle':
|
||||
if dconf.DB_TYPE == 'oracle':
|
||||
# You must create a directory named dpdata through sqlplus in your Oracle database
|
||||
# The following script assumes such directory exists.
|
||||
# You may want to modify the username, password, and dump file name in the script
|
||||
cmd = 'sh oracleScripts/restoreOracle.sh'
|
||||
elif CONF['database_type'] == 'postgres':
|
||||
db_file_path = '{}/{}.dump'.format(CONF['database_save_path'], CONF['database_name'])
|
||||
elif dconf.DB_TYPE == 'postgres':
|
||||
db_file_path = '{}/{}.dump'.format(dconf.DB_DUMP_DIR, dconf.DB_NAME)
|
||||
drop_database()
|
||||
create_database()
|
||||
cmd = 'PGPASSWORD={} pg_restore -U {} -n public -j 8 -F c -d {} {}'.\
|
||||
format(CONF['password'], CONF['username'], CONF['database_name'], db_file_path)
|
||||
format(dconf.DB_PASSWORD, dconf.DB_USER, dconf.DB_NAME, db_file_path)
|
||||
else:
|
||||
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
|
||||
LOG.info('Start restoring database')
|
||||
local(cmd)
|
||||
LOG.info('Finish restoring database')
|
||||
|
|
@ -442,8 +458,8 @@ def restore_database():
|
|||
|
||||
def _ready_to_start_oltpbench():
|
||||
ready = False
|
||||
if os.path.exists(CONF['controller_log']):
|
||||
with open(CONF['controller_log'], 'r') as f:
|
||||
if os.path.exists(dconf.CONTROLLER_LOG):
|
||||
with open(dconf.CONTROLLER_LOG, 'r') as f:
|
||||
content = f.read()
|
||||
ready = 'Output the process pid to' in content
|
||||
return ready
|
||||
|
|
@ -451,18 +467,18 @@ def _ready_to_start_oltpbench():
|
|||
|
||||
def _ready_to_start_controller():
|
||||
ready = False
|
||||
if os.path.exists(CONF['oltpbench_log']):
|
||||
with open(CONF['oltpbench_log'], 'r') as f:
|
||||
if os.path.exists(dconf.OLTPBENCH_LOG):
|
||||
with open(dconf.OLTPBENCH_LOG, 'r') as f:
|
||||
content = f.read()
|
||||
ready = 'Warmup complete, starting measurements' in content
|
||||
return ready
|
||||
|
||||
|
||||
def _ready_to_shut_down_controller():
|
||||
pidfile = os.path.join(CONF['controller_home'], 'pid.txt')
|
||||
pidfile = os.path.join(dconf.CONTROLLER_HOME, 'pid.txt')
|
||||
ready = False
|
||||
if os.path.exists(pidfile) and os.path.exists(CONF['oltpbench_log']):
|
||||
with open(CONF['oltpbench_log'], 'r') as f:
|
||||
if os.path.exists(pidfile) and os.path.exists(dconf.OLTPBENCH_LOG):
|
||||
with open(dconf.OLTPBENCH_LOG, 'r') as f:
|
||||
content = f.read()
|
||||
ready = 'Output throughput samples into file' in content
|
||||
return ready
|
||||
|
|
@ -470,17 +486,11 @@ def _ready_to_shut_down_controller():
|
|||
|
||||
def clean_logs():
|
||||
# remove oltpbench log
|
||||
cmd = 'rm -f {}'.format(CONF['oltpbench_log'])
|
||||
cmd = 'rm -f {}'.format(dconf.OLTPBENCH_LOG)
|
||||
local(cmd)
|
||||
|
||||
# remove controller log
|
||||
cmd = 'rm -f {}'.format(CONF['controller_log'])
|
||||
local(cmd)
|
||||
|
||||
|
||||
@task
|
||||
def lhs_samples(count=10):
|
||||
cmd = 'python3 lhs.py {} {} {}'.format(count, CONF['lhs_knob_path'], CONF['lhs_save_path'])
|
||||
cmd = 'rm -f {}'.format(dconf.CONTROLLER_LOG)
|
||||
local(cmd)
|
||||
|
||||
|
||||
|
|
@ -494,11 +504,11 @@ def loop(i):
|
|||
|
||||
# restart database
|
||||
restart_database()
|
||||
time.sleep(CONF['sleep_time'])
|
||||
time.sleep(dconf.RESTART_SLEEP_SEC)
|
||||
|
||||
# check disk usage
|
||||
if check_disk_usage() > MAX_DISK_USAGE:
|
||||
LOG.warning('Exceeds max disk usage %s', MAX_DISK_USAGE)
|
||||
if check_disk_usage() > dconf.MAX_DISK_USAGE:
|
||||
LOG.warning('Exceeds max disk usage %s', dconf.MAX_DISK_USAGE)
|
||||
|
||||
# run controller from another process
|
||||
p = Process(target=run_controller, args=())
|
||||
|
|
@ -532,7 +542,7 @@ def loop(i):
|
|||
# save result
|
||||
result_timestamp = save_dbms_result()
|
||||
|
||||
if i >= CONF['warmup_iterations']:
|
||||
if i >= dconf.WARMUP_ITERATIONS:
|
||||
# upload result
|
||||
upload_result()
|
||||
|
||||
|
|
@ -546,96 +556,14 @@ def loop(i):
|
|||
change_conf(response['recommendation'])
|
||||
|
||||
|
||||
@task
|
||||
def run_lhs():
|
||||
datadir = CONF['lhs_save_path']
|
||||
samples = glob.glob(os.path.join(datadir, 'config_*'))
|
||||
|
||||
# dump database if it's not done before.
|
||||
dump = dump_database()
|
||||
|
||||
result_timestamp = None
|
||||
for i, sample in enumerate(samples):
|
||||
# reload database periodically
|
||||
if RELOAD_INTERVAL > 0:
|
||||
if i % RELOAD_INTERVAL == 0:
|
||||
if i == 0 and dump is False:
|
||||
restore_database()
|
||||
elif i > 0:
|
||||
restore_database()
|
||||
# free cache
|
||||
free_cache()
|
||||
|
||||
LOG.info('\n\n Start %s-th sample %s \n\n', i, sample)
|
||||
# check memory usage
|
||||
# check_memory_usage()
|
||||
|
||||
# check disk usage
|
||||
if check_disk_usage() > MAX_DISK_USAGE:
|
||||
LOG.warning('Exceeds max disk usage %s', MAX_DISK_USAGE)
|
||||
|
||||
# load the next lhs-sampled config
|
||||
with open(sample, 'r') as f:
|
||||
next_config = json.load(f, object_pairs_hook=OrderedDict)
|
||||
save_next_config(next_config, t=result_timestamp)
|
||||
|
||||
# remove oltpbench log and controller log
|
||||
clean_logs()
|
||||
|
||||
# change config
|
||||
change_conf(next_config)
|
||||
|
||||
# restart database
|
||||
restart_database()
|
||||
|
||||
if CONF.get('oracle_awr_enabled', False):
|
||||
# create snapshot for oracle AWR report
|
||||
if CONF['database_type'] == 'oracle':
|
||||
local('sh snapshotOracle.sh')
|
||||
|
||||
# run controller from another process
|
||||
p = Process(target=run_controller, args=())
|
||||
p.start()
|
||||
|
||||
# run oltpbench as a background job
|
||||
while not _ready_to_start_oltpbench():
|
||||
pass
|
||||
run_oltpbench_bg()
|
||||
LOG.info('Run OLTP-Bench')
|
||||
|
||||
while not _ready_to_start_controller():
|
||||
pass
|
||||
signal_controller()
|
||||
LOG.info('Start the first collection')
|
||||
|
||||
while not _ready_to_shut_down_controller():
|
||||
pass
|
||||
# stop the experiment
|
||||
signal_controller()
|
||||
LOG.info('Start the second collection, shut down the controller')
|
||||
|
||||
p.join()
|
||||
|
||||
# save result
|
||||
result_timestamp = save_dbms_result()
|
||||
|
||||
# upload result
|
||||
upload_result()
|
||||
|
||||
if CONF.get('oracle_awr_enabled', False):
|
||||
# create oracle AWR report for performance analysis
|
||||
if CONF['database_type'] == 'oracle':
|
||||
local('sh oracleScripts/snapshotOracle.sh && sh oracleScripts/awrOracle.sh')
|
||||
|
||||
|
||||
@task
|
||||
def run_loops(max_iter=1):
|
||||
# dump database if it's not done before.
|
||||
dump = dump_database()
|
||||
|
||||
for i in range(int(max_iter)):
|
||||
if RELOAD_INTERVAL > 0:
|
||||
if i % RELOAD_INTERVAL == 0:
|
||||
if dconf.RELOAD_INTERVAL > 0:
|
||||
if i % dconf.RELOAD_INTERVAL == 0:
|
||||
if i == 0 and dump is False:
|
||||
restart_database()
|
||||
restore_database()
|
||||
|
|
@ -643,13 +571,13 @@ def run_loops(max_iter=1):
|
|||
restore_database()
|
||||
|
||||
LOG.info('The %s-th Loop Starts / Total Loops %s', i + 1, max_iter)
|
||||
loop(i % RELOAD_INTERVAL)
|
||||
loop(i % dconf.RELOAD_INTERVAL)
|
||||
LOG.info('The %s-th Loop Ends / Total Loops %s', i + 1, max_iter)
|
||||
|
||||
|
||||
@task
|
||||
def rename_batch(result_dir=None):
|
||||
result_dir = result_dir or CONF['save_path']
|
||||
result_dir = result_dir or dconf.RESULT_DIR
|
||||
results = glob.glob(os.path.join(result_dir, '*__summary.json'))
|
||||
results = sorted(results)
|
||||
for i, result in enumerate(results):
|
||||
|
|
@ -752,7 +680,7 @@ def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10):
|
|||
elapsed = 0
|
||||
|
||||
while elapsed <= max_time_sec:
|
||||
response = requests.get(CONF['upload_url'] + '/test/pipeline/')
|
||||
response = requests.get(dconf.WEBSITE_URL + '/test/pipeline/')
|
||||
response = response.content
|
||||
LOG.info(response)
|
||||
if 'False' in str(response):
|
||||
|
|
@ -766,7 +694,7 @@ def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10):
|
|||
def integration_tests():
|
||||
|
||||
# Create test website
|
||||
response = requests.get(CONF['upload_url'] + '/test/create/')
|
||||
response = requests.get(dconf.WEBSITE_URL + '/test/create/')
|
||||
LOG.info(response.content)
|
||||
|
||||
# Upload training data
|
||||
|
|
|
|||
Loading…
Reference in New Issue