ottertune/client/driver/lhs.py

127 lines
3.2 KiB
Python
Raw Normal View History

2019-08-23 08:47:19 -07:00
#
# OtterTune - lhs.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
import sys
import json
import os
import numpy as np
from pyDOE import lhs
from scipy.stats import uniform
from hurry.filesize import size
BYTES_SYSTEM = [
(1024 ** 5, 'PB'),
(1024 ** 4, 'TB'),
(1024 ** 3, 'GB'),
(1024 ** 2, 'MB'),
(1024 ** 1, 'kB'),
(1024 ** 0, 'B'),
]
TIME_SYSTEM = [
(1000 * 60 * 60 * 24, 'd'),
(1000 * 60 * 60, 'h'),
(1000 * 60, 'min'),
(1000, 's'),
(1, 'ms'),
]
def get_raw_size(value, system):
for factor, suffix in system:
if value.endswith(suffix):
if len(value) == len(suffix):
amount = 1
else:
try:
amount = int(value[:-len(suffix)])
except ValueError:
continue
return amount * factor
return None
def get_knob_raw(value, knob_type):
if knob_type == 'integer':
return int(value)
elif knob_type == 'float':
return float(value)
elif knob_type == 'bytes':
return get_raw_size(value, BYTES_SYSTEM)
elif knob_type == 'time':
return get_raw_size(value, TIME_SYSTEM)
else:
raise Exception('Knob Type does not support')
def get_knob_readable(value, knob_type):
if knob_type == 'integer':
return int(round(value))
elif knob_type == 'float':
return float(value)
elif knob_type == 'bytes':
value = int(round(value))
return size(value, system=BYTES_SYSTEM)
elif knob_type == 'time':
value = int(round(value))
return size(value, system=TIME_SYSTEM)
else:
raise Exception('Knob Type does not support')
def get_knobs_readable(values, types):
result = []
for i, value in enumerate(values):
result.append(get_knob_readable(value, types[i]))
return result
def main(args):
if (len(sys.argv) != 4):
raise Exception("Usage: python3 lhs.py [Samples Count] [Knob Path] [Save Path]")
knob_path = args[2]
save_path = args[3]
with open(knob_path, "r") as f:
tuning_knobs = json.load(f)
names = []
maxvals = []
minvals = []
types = []
for knob in tuning_knobs:
names.append(knob['name'])
maxvals.append(get_knob_raw(knob['tuning_range']['maxval'], knob['type']))
minvals.append(get_knob_raw(knob['tuning_range']['minval'], knob['type']))
types.append(knob['type'])
nsamples = int(args[1])
nfeats = len(tuning_knobs)
samples = lhs(nfeats, samples=nsamples, criterion='maximin')
maxvals = np.array(maxvals)
minvals = np.array(minvals)
scales = maxvals - minvals
for fidx in range(nfeats):
samples[:, fidx] = uniform(loc=minvals[fidx], scale=scales[fidx]).ppf(samples[:, fidx])
samples_readable = []
for sample in samples:
samples_readable.append(get_knobs_readable(sample, types))
config = {'recommendation': {}}
for sidx in range(nsamples):
for fidx in range(nfeats):
config["recommendation"][names[fidx]] = samples_readable[sidx][fidx]
with open(os.path.join(save_path, 'config_' + str(sidx)), 'w+') as f:
f.write(json.dumps(config))
if __name__ == '__main__':
main(sys.argv)