public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix enum bug

This commit is contained in:
yangdsh 2019-12-10 02:12:55 +00:00 committed by Dana Van Aken
parent a7724233d3
commit 642d7c3875
7 changed files with 42 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
client/driver/utils.py
View File

@ -75,7 +75,7 @@ def run(cmd, capture=True, **kwargs):
if cmd.endswith('&'):
cmdd = cmd[:-1].strip()
opts = '-d '
res = local('docker exec {} -ti {} /bin/bash -c "{}"'.format(
res = local('sudo docker exec {} -ti {} /bin/bash -c "{}"'.format(
opts, dconf.CONTAINER_NAME, cmdd), capture=capture, **kwargs)
except TypeError as e:
err = str(e).strip()
@ -106,7 +106,7 @@ def sudo(cmd, user=None, capture=True, **kwargs):
opts = '-ti -u {}'.format(user or 'root')
if user == 'root':
opts += ' -w /'
res = local('docker exec {} {} /bin/bash -c "{}"'.format(
res = local('sudo docker exec {} {} /bin/bash -c "{}"'.format(
opts, dconf.CONTAINER_NAME, cmd), capture=capture)
return res
@ -123,7 +123,7 @@ def get(remote_path, local_path, use_sudo=False):
opts = '-r' if os.path.isdir(remote_path) else ''
res = local('{}cp {} {} {}'.format(pre_cmd, opts, remote_path, local_path))
else: # docker
res = local('docker cp {}:{} {}'.format(dconf.CONTAINER_NAME, remote_path, local_path))
res = local('sudo docker cp {}:{} {}'.format(dconf.CONTAINER_NAME, remote_path, local_path))
return res
@ -138,7 +138,7 @@ def put(local_path, remote_path, use_sudo=False):
opts = '-r' if os.path.isdir(local_path) else ''
res = local('{}cp {} {} {}'.format(pre_cmd, opts, local_path, remote_path))
else: # docker
res = local('docker cp {} {}:{}'.format(local_path, dconf.CONTAINER_NAME, remote_path))
res = local('sudo docker cp {} {}:{}'.format(local_path, dconf.CONTAINER_NAME, remote_path))
return res

2
server/website/website/db/base/parser.py
View File

@ -361,7 +361,7 @@ class BaseParser:
return configuration
def format_bool(self, bool_value, metadata):
return self.true_value if bool_value == BooleanType.TRUE else self.false_value
return self.true_value if int(round(bool_value)) == BooleanType.TRUE else self.false_value
def format_enum(self, enum_value, metadata):
enumvals = metadata.enumvals.split(',')

2
server/website/website/management/commands/resetwebsite.py
View File

@ -22,7 +22,7 @@ class Command(BaseCommand):
port = DATABASES['default']['PORT']
if engine.endswith('mysql'):
db_cmd_fmt = 'mysql -u {user} -p{passwd} -h {host} -P {port} -N -B -e "{{cmd}}"'
db_cmd_fmt = 'mysql -u {user} -p -h {host} -P {port} -N -B -e "{{cmd}}"'
elif engine.endswith('postgresql'):
db_cmd_fmt = 'PGPASSWORD={passwd} psql -U {user} -h {host} -p {port} -c "{{cmd}}"'
else:

8
server/website/website/models.py
View File

@ -193,6 +193,14 @@ class SessionKnobManager(models.Manager):
knob_dict['minval'] = sess_knob.minval
knob_dict['maxval'] = sess_knob.maxval
knob_dict['tunable'] = sess_knob.tunable
if knob_dict['vartype'] is VarType.ENUM:
enumvals = knob_dict['enumvals'].split(',')
knob_dict["minval"] = 0
knob_dict["maxval"] = len(enumvals) - 1
if knob_dict['vartype'] is VarType.BOOL:
knob_dict["minval"] = 0
knob_dict["maxval"] = 1
return knob_dicts
@staticmethod

28
server/website/website/tasks/async_tasks.py
View File

@ -110,12 +110,6 @@ class ConfigurationRecommendation(UpdateTask): # pylint: disable=abstract-metho
result_id = retval['result_id']
result = Result.objects.get(pk=result_id)
# Create next configuration to try
config = db.parser.create_knob_configuration(result.dbms.pk, retval['recommendation'])
retval['recommendation'] = config
result.next_configuration = JSONUtil.dumps(retval)
result.save()
# Replace result with formatted result
formatted_params = db.parser.format_dbms_knobs(result.dbms.pk, retval['recommendation'])
task_meta = TaskMeta.objects.get(task_id=task_id)
@ -123,6 +117,12 @@ class ConfigurationRecommendation(UpdateTask): # pylint: disable=abstract-metho
task_meta.result = retval
task_meta.save()
# Create next configuration to try
config = db.parser.create_knob_configuration(result.dbms.pk, retval['recommendation'])
retval['recommendation'] = config
result.next_configuration = JSONUtil.dumps(retval)
result.save()
def clean_knob_data(knob_matrix, knob_labels, session):
# Makes sure that all knobs in the dbms are included in the knob_matrix and knob_labels
@ -138,7 +138,11 @@ def clean_knob_data(knob_matrix, knob_labels, session):
for knob in missing_columns:
knob_object = KnobCatalog.objects.get(dbms=session.dbms, name=knob, tunable=True)
index = knob_cat.index(knob)
matrix = np.insert(matrix, index, knob_object.default, axis=1)
try:
default_val = float(knob_object.default)
except ValueError:
default_val = 0
matrix = np.insert(matrix, index, default_val, axis=1)
knob_labels.insert(index, knob)
# If they are useless columns in the matrix
if unused_columns:
@ -272,6 +276,10 @@ def gen_lhs_samples(knobs, nsamples):
for fidx in range(nfeats):
if types[fidx] == VarType.INTEGER:
lhs_samples[-1][names[fidx]] = int(round(samples[sidx][fidx]))
elif types[fidx] == VarType.BOOL:
lhs_samples[-1][names[fidx]] = int(round(samples[sidx][fidx]))
elif types[fidx] == VarType.ENUM:
lhs_samples[-1][names[fidx]] = int(round(samples[sidx][fidx]))
elif types[fidx] == VarType.REAL:
lhs_samples[-1][names[fidx]] = float(samples[sidx][fidx])
else:
@ -461,10 +469,12 @@ def configuration_recommendation(recommendation_input):
if not np.array_equal(X_columnlabels, target_data['X_columnlabels']):
raise Exception(('The workload and target data should have '
'identical X columnlabels (sorted knob names)'))
'identical X columnlabels (sorted knob names)'),
X_columnlabels, target_data['X_columnlabels'])
if not np.array_equal(y_columnlabels, target_data['y_columnlabels']):
raise Exception(('The workload and target data should have '
'identical y columnlabels (sorted metric names)'))
'identical y columnlabels (sorted metric names)'),
y_columnlabels, target_data['y_columnlabels'])
# Filter Xs by top 10 ranked knobs
ranked_knobs = PipelineData.objects.get(

9
server/website/website/utils.py
View File

@ -113,7 +113,14 @@ class DataUtil(object):
knob_object = KnobCatalog.objects.get(dbms=session.dbms, name=knob, tunable=True)
knob_session_object = SessionKnob.objects.filter(knob=knob_object, session=session,
tunable=True)
if knob_session_object.exists():
if knob_object.vartype is VarType.ENUM:
enumvals = knob_object.enumvals.split(',')
minval = 0
maxval = len(enumvals) - 1
elif knob_object.vartype is VarType.BOOL:
minval = 0
maxval = 1
elif knob_session_object.exists():
minval = float(knob_session_object[0].minval)
maxval = float(knob_session_object[0].maxval)
else:

1
server/website/website/views.py
View File

@ -487,6 +487,7 @@ def handle_result_files(session, files):
knob_data.knobs = JSONUtil.dumps(all_knobs)
data_knobs = JSONUtil.loads(knob_data.data)
last_conf = parser.convert_dbms_knobs(result.dbms.pk, last_conf)
for knob in data_knobs.keys():
for tunable_knob in last_conf.keys():
if tunable_knob in knob: