diff --git a/client/driver/utils.py b/client/driver/utils.py index af66ca1..76522b3 100644 --- a/client/driver/utils.py +++ b/client/driver/utils.py @@ -75,7 +75,7 @@ def run(cmd, capture=True, **kwargs): if cmd.endswith('&'): cmdd = cmd[:-1].strip() opts = '-d ' - res = local('docker exec {} -ti {} /bin/bash -c "{}"'.format( + res = local('sudo docker exec {} -ti {} /bin/bash -c "{}"'.format( opts, dconf.CONTAINER_NAME, cmdd), capture=capture, **kwargs) except TypeError as e: err = str(e).strip() @@ -106,7 +106,7 @@ def sudo(cmd, user=None, capture=True, **kwargs): opts = '-ti -u {}'.format(user or 'root') if user == 'root': opts += ' -w /' - res = local('docker exec {} {} /bin/bash -c "{}"'.format( + res = local('sudo docker exec {} {} /bin/bash -c "{}"'.format( opts, dconf.CONTAINER_NAME, cmd), capture=capture) return res @@ -123,7 +123,7 @@ def get(remote_path, local_path, use_sudo=False): opts = '-r' if os.path.isdir(remote_path) else '' res = local('{}cp {} {} {}'.format(pre_cmd, opts, remote_path, local_path)) else: # docker - res = local('docker cp {}:{} {}'.format(dconf.CONTAINER_NAME, remote_path, local_path)) + res = local('sudo docker cp {}:{} {}'.format(dconf.CONTAINER_NAME, remote_path, local_path)) return res @@ -138,7 +138,7 @@ def put(local_path, remote_path, use_sudo=False): opts = '-r' if os.path.isdir(local_path) else '' res = local('{}cp {} {} {}'.format(pre_cmd, opts, local_path, remote_path)) else: # docker - res = local('docker cp {} {}:{}'.format(local_path, dconf.CONTAINER_NAME, remote_path)) + res = local('sudo docker cp {} {}:{}'.format(local_path, dconf.CONTAINER_NAME, remote_path)) return res diff --git a/server/website/website/db/base/parser.py b/server/website/website/db/base/parser.py index efd7058..3a82855 100644 --- a/server/website/website/db/base/parser.py +++ b/server/website/website/db/base/parser.py @@ -361,7 +361,7 @@ class BaseParser: return configuration def format_bool(self, bool_value, metadata): - return self.true_value if bool_value == BooleanType.TRUE else self.false_value + return self.true_value if int(round(bool_value)) == BooleanType.TRUE else self.false_value def format_enum(self, enum_value, metadata): enumvals = metadata.enumvals.split(',') diff --git a/server/website/website/management/commands/resetwebsite.py b/server/website/website/management/commands/resetwebsite.py index a743dc5..69ec07a 100644 --- a/server/website/website/management/commands/resetwebsite.py +++ b/server/website/website/management/commands/resetwebsite.py @@ -22,7 +22,7 @@ class Command(BaseCommand): port = DATABASES['default']['PORT'] if engine.endswith('mysql'): - db_cmd_fmt = 'mysql -u {user} -p{passwd} -h {host} -P {port} -N -B -e "{{cmd}}"' + db_cmd_fmt = 'mysql -u {user} -p -h {host} -P {port} -N -B -e "{{cmd}}"' elif engine.endswith('postgresql'): db_cmd_fmt = 'PGPASSWORD={passwd} psql -U {user} -h {host} -p {port} -c "{{cmd}}"' else: diff --git a/server/website/website/models.py b/server/website/website/models.py index 32e783b..94368d6 100644 --- a/server/website/website/models.py +++ b/server/website/website/models.py @@ -193,6 +193,14 @@ class SessionKnobManager(models.Manager): knob_dict['minval'] = sess_knob.minval knob_dict['maxval'] = sess_knob.maxval knob_dict['tunable'] = sess_knob.tunable + if knob_dict['vartype'] is VarType.ENUM: + enumvals = knob_dict['enumvals'].split(',') + knob_dict["minval"] = 0 + knob_dict["maxval"] = len(enumvals) - 1 + if knob_dict['vartype'] is VarType.BOOL: + knob_dict["minval"] = 0 + knob_dict["maxval"] = 1 + return knob_dicts @staticmethod diff --git a/server/website/website/tasks/async_tasks.py b/server/website/website/tasks/async_tasks.py index 50d7216..287d1b5 100644 --- a/server/website/website/tasks/async_tasks.py +++ b/server/website/website/tasks/async_tasks.py @@ -110,12 +110,6 @@ class ConfigurationRecommendation(UpdateTask): # pylint: disable=abstract-metho result_id = retval['result_id'] result = Result.objects.get(pk=result_id) - # Create next configuration to try - config = db.parser.create_knob_configuration(result.dbms.pk, retval['recommendation']) - retval['recommendation'] = config - result.next_configuration = JSONUtil.dumps(retval) - result.save() - # Replace result with formatted result formatted_params = db.parser.format_dbms_knobs(result.dbms.pk, retval['recommendation']) task_meta = TaskMeta.objects.get(task_id=task_id) @@ -123,6 +117,12 @@ class ConfigurationRecommendation(UpdateTask): # pylint: disable=abstract-metho task_meta.result = retval task_meta.save() + # Create next configuration to try + config = db.parser.create_knob_configuration(result.dbms.pk, retval['recommendation']) + retval['recommendation'] = config + result.next_configuration = JSONUtil.dumps(retval) + result.save() + def clean_knob_data(knob_matrix, knob_labels, session): # Makes sure that all knobs in the dbms are included in the knob_matrix and knob_labels @@ -138,7 +138,11 @@ def clean_knob_data(knob_matrix, knob_labels, session): for knob in missing_columns: knob_object = KnobCatalog.objects.get(dbms=session.dbms, name=knob, tunable=True) index = knob_cat.index(knob) - matrix = np.insert(matrix, index, knob_object.default, axis=1) + try: + default_val = float(knob_object.default) + except ValueError: + default_val = 0 + matrix = np.insert(matrix, index, default_val, axis=1) knob_labels.insert(index, knob) # If they are useless columns in the matrix if unused_columns: @@ -272,6 +276,10 @@ def gen_lhs_samples(knobs, nsamples): for fidx in range(nfeats): if types[fidx] == VarType.INTEGER: lhs_samples[-1][names[fidx]] = int(round(samples[sidx][fidx])) + elif types[fidx] == VarType.BOOL: + lhs_samples[-1][names[fidx]] = int(round(samples[sidx][fidx])) + elif types[fidx] == VarType.ENUM: + lhs_samples[-1][names[fidx]] = int(round(samples[sidx][fidx])) elif types[fidx] == VarType.REAL: lhs_samples[-1][names[fidx]] = float(samples[sidx][fidx]) else: @@ -461,10 +469,12 @@ def configuration_recommendation(recommendation_input): if not np.array_equal(X_columnlabels, target_data['X_columnlabels']): raise Exception(('The workload and target data should have ' - 'identical X columnlabels (sorted knob names)')) + 'identical X columnlabels (sorted knob names)'), + X_columnlabels, target_data['X_columnlabels']) if not np.array_equal(y_columnlabels, target_data['y_columnlabels']): raise Exception(('The workload and target data should have ' - 'identical y columnlabels (sorted metric names)')) + 'identical y columnlabels (sorted metric names)'), + y_columnlabels, target_data['y_columnlabels']) # Filter Xs by top 10 ranked knobs ranked_knobs = PipelineData.objects.get( diff --git a/server/website/website/utils.py b/server/website/website/utils.py index 4afde37..7544ecd 100644 --- a/server/website/website/utils.py +++ b/server/website/website/utils.py @@ -113,7 +113,14 @@ class DataUtil(object): knob_object = KnobCatalog.objects.get(dbms=session.dbms, name=knob, tunable=True) knob_session_object = SessionKnob.objects.filter(knob=knob_object, session=session, tunable=True) - if knob_session_object.exists(): + if knob_object.vartype is VarType.ENUM: + enumvals = knob_object.enumvals.split(',') + minval = 0 + maxval = len(enumvals) - 1 + elif knob_object.vartype is VarType.BOOL: + minval = 0 + maxval = 1 + elif knob_session_object.exists(): minval = float(knob_session_object[0].minval) maxval = float(knob_session_object[0].maxval) else: diff --git a/server/website/website/views.py b/server/website/website/views.py index 6c6cb1b..ce12dfb 100644 --- a/server/website/website/views.py +++ b/server/website/website/views.py @@ -487,6 +487,7 @@ def handle_result_files(session, files): knob_data.knobs = JSONUtil.dumps(all_knobs) data_knobs = JSONUtil.loads(knob_data.data) + last_conf = parser.convert_dbms_knobs(result.dbms.pk, last_conf) for knob in data_knobs.keys(): for tunable_knob in last_conf.keys(): if tunable_knob in knob: