From cebc9586665818d89296e072d42101ad77e14645 Mon Sep 17 00:00:00 2001 From: yangdsh Date: Wed, 4 Mar 2020 01:25:28 +0000 Subject: [PATCH] search for knob ranges --- script/formatting/config/pylintrc | 2 +- .../migrations/0011_knob_bound_fields.py | 25 +++ server/website/website/models.py | 32 ++-- server/website/website/tasks/async_tasks.py | 178 +++++++++++++++--- 4 files changed, 195 insertions(+), 42 deletions(-) create mode 100644 server/website/website/migrations/0011_knob_bound_fields.py diff --git a/script/formatting/config/pylintrc b/script/formatting/config/pylintrc index d0d4b14..75a3ddc 100644 --- a/script/formatting/config/pylintrc +++ b/script/formatting/config/pylintrc @@ -13,7 +13,7 @@ profile=no # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=CVS,.git,manage.py,0001_initial.py,0002_enable_compression.py,0003_load_initial_data.py,0004_add_lhs.py,0005_add_workload_field.py,0006_session_hyperparameters.py,0007_executiontime.py,0008_change_result_taskids_field.py,0009_change_executiontime_function_field.py,0010_add_pipeline_data_field.py,credentials.py,create_knob_settings.py +ignore=CVS,.git,manage.py,0001_initial.py,0002_enable_compression.py,0003_load_initial_data.py,0004_add_lhs.py,0005_add_workload_field.py,0006_session_hyperparameters.py,0007_executiontime.py,0008_change_result_taskids_field.py,0009_change_executiontime_function_field.py,0010_add_pipeline_data_field.py,0011_knob_bound_fields.py,credentials.py,create_knob_settings.py # ignore-patterns=**/migrations/*.py diff --git a/server/website/website/migrations/0011_knob_bound_fields.py b/server/website/website/migrations/0011_knob_bound_fields.py new file mode 100644 index 0000000..be910aa --- /dev/null +++ b/server/website/website/migrations/0011_knob_bound_fields.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-03-03 21:07 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('website', '0009_change_executiontime_function_field'), + ] + + operations = [ + migrations.AddField( + model_name='sessionknob', + name='lowerbound', + field=models.CharField(max_length=32, null=True, verbose_name='lowerbound'), + ), + migrations.AddField( + model_name='sessionknob', + name='upperbound', + field=models.CharField(max_length=32, null=True, verbose_name='upperbound'), + ), + ] diff --git a/server/website/website/models.py b/server/website/website/models.py index 3226815..b896a91 100644 --- a/server/website/website/models.py +++ b/server/website/website/models.py @@ -233,18 +233,20 @@ class SessionKnobManager(models.Manager): session=session, tunable=True).prefetch_related('knob') session_knobs = {s.knob.pk: s for s in session_knobs} knob_dicts = list(KnobCatalog.objects.filter(id__in=session_knobs.keys()).values()) - for knob_dict in knob_dicts: - sess_knob = session_knobs[knob_dict['id']] - knob_dict['minval'] = sess_knob.minval - knob_dict['maxval'] = sess_knob.maxval - knob_dict['tunable'] = sess_knob.tunable - if knob_dict['vartype'] is VarType.ENUM: - enumvals = knob_dict['enumvals'].split(',') - knob_dict["minval"] = 0 - knob_dict["maxval"] = len(enumvals) - 1 - if knob_dict['vartype'] is VarType.BOOL: - knob_dict["minval"] = 0 - knob_dict["maxval"] = 1 + for knob_info in knob_dicts: + sess_knob = session_knobs[knob_info['id']] + knob_info['minval'] = sess_knob.minval + knob_info['maxval'] = sess_knob.maxval + knob_info['upperbound'] = sess_knob.upperbound + knob_info['lowerbound'] = sess_knob.lowerbound + knob_info['tunable'] = sess_knob.tunable + if knob_info['vartype'] is VarType.ENUM: + enumvals = knob_info['enumvals'].split(',') + knob_info["minval"] = 0 + knob_info["maxval"] = len(enumvals) - 1 + if knob_info['vartype'] is VarType.BOOL: + knob_info["minval"] = 0 + knob_info["maxval"] = 1 return knob_dicts @@ -275,6 +277,10 @@ class SessionKnobManager(models.Manager): session_knob.minval = settings["minval"] session_knob.maxval = settings["maxval"] session_knob.tunable = settings["tunable"] + if "upperbound" in settings: + session_knob.upperbound = settings["upperbound"] + if "lowerbound" in settings: + session_knob.lowerbound = settings["lowerbound"] session_knob.save() if cascade: knob = KnobCatalog.objects.get(name=session_knob.name, dbms=session.dbms) @@ -302,6 +308,8 @@ class SessionKnob(BaseModel): knob = models.ForeignKey(KnobCatalog) minval = models.CharField(max_length=32, null=True, verbose_name="minimum value") maxval = models.CharField(max_length=32, null=True, verbose_name="maximum value") + upperbound = models.CharField(max_length=32, null=True, verbose_name="upperbound") + lowerbound = models.CharField(max_length=32, null=True, verbose_name="lowerbound") tunable = models.BooleanField(verbose_name="tunable") diff --git a/server/website/website/tasks/async_tasks.py b/server/website/website/tasks/async_tasks.py index 9d75fe3..35989b1 100644 --- a/server/website/website/tasks/async_tasks.py +++ b/server/website/website/tasks/async_tasks.py @@ -30,7 +30,7 @@ from analysis.gpr.predict import gpflow_predict from analysis.preprocessing import Bin, DummyEncoder from analysis.constraints import ParamConstraintHelper from website.models import (PipelineData, PipelineRun, Result, Workload, SessionKnob, - MetricCatalog, ExecutionTime) + MetricCatalog, ExecutionTime, KnobCatalog) from website import db from website.types import PipelineTaskType, AlgorithmType, VarType from website.utils import DataUtil, JSONUtil @@ -69,7 +69,7 @@ class MapWorkloadTask(BaseTask): # pylint: disable=abstract-method new_res = None # Replace result with formatted result - if not args[0][0]['bad'] and args[0][0]['mapped_workload'] is not None: + if args[0][0]['status'] == 'good' and args[0][0]['mapped_workload'] is not None: new_res = { 'scores': sorted(args[0][0]['scores'].items()), 'mapped_workload_id': args[0][0]['mapped_workload'], @@ -178,47 +178,137 @@ def save_execution_time(start_ts, fn, result): start_time=start_time, execution_time=exec_time, result=result) +def choose_value_in_range(num1, num2): + if num1 < 1: + num1 = num1 + 1 + if num2 < 1: + num2 = num2 + 1 + log_num1 = np.log(num1) + log_num2 = np.log(num2) + return np.exp((log_num1 + log_num2) / 2) + + +def test_knob_range(knob_info, newest_result, good_val, bad_val, mode): + session = newest_result.session + knob = KnobCatalog.objects.get(name=knob_info['name'], dbms=session.dbms) + knob_file = newest_result.knob_data + knob_values = JSONUtil.loads(knob_file.data) + last_value = float(knob_values[knob.name]) + session_knob = SessionKnob.objects.get(session=session, knob=knob) + # The collected knob value may be different from the expected value + # We use the expected value to set the knob range + expected_value = choose_value_in_range(good_val, bad_val) + + session_results = Result.objects.filter(session=session).order_by("-id") + last_conf_value = '' + if len(session_results) > 1: + last_conf = session_results[1].next_configuration + if last_conf is not None: + last_conf = JSONUtil.loads(last_conf)["recommendation"] + # The names cannot be matched directly because of the 'global.' prefix + for name in last_conf.keys(): + if name in knob.name: + last_conf_value = last_conf[name] + fomatted_expect_value = db.parser.format_dbms_knobs( + session.dbms.pk, {knob.name: expected_value})[knob.name] + + # The last result was testing the max_range of this knob + if last_conf_value == fomatted_expect_value: + # Fixme: '*' is a special symbol indicating that the knob setting is invalid + # In the future we can add a field to indicate if the knob setting is invalid + if '*' in knob_file.name: + if mode == 'lowerbound': + session_knob.lowerbound = str(int(expected_value)) + else: + session_knob.upperbound = str(int(expected_value)) + next_value = choose_value_in_range(expected_value, good_val) + else: + if mode == 'lowerbound': + session_knob.minval = str(int(expected_value)) + if expected_value < last_value / 10: + session_knob.minval = str(int(last_value)) + session_knob.lowerbound = str(int(last_value)) + else: + session_knob.maxval = str(int(expected_value)) + next_value = choose_value_in_range(expected_value, bad_val) + session_knob.save() + else: + next_value = expected_value + + if mode == 'lowerbound': + next_config = {knob.name: next_value} + else: + knobs = SessionKnob.objects.get_knobs_for_session(session) + next_config = gen_test_maxval_data(knobs, knob.name, next_value) + + agg_data = DataUtil.aggregate_data(Result.objects.filter(pk=newest_result.pk)) + agg_data['newest_result_id'] = newest_result.pk + agg_data['status'] = 'range_test' + agg_data['config_recommend'] = next_config + LOG.debug('Testing %s of %s.\n\ndata=%s\n', mode, knob.name, + JSONUtil.dumps(agg_data, pprint=True)) + save_execution_time(time.time(), "aggregate_target_results", newest_result) + return agg_data + + @shared_task(base=IgnoreResultTask, name='aggregate_target_results') def aggregate_target_results(result_id, algorithm): start_ts = time.time() + agg_data = DataUtil.aggregate_data(Result.objects.filter(pk=result_id)) + newest_result = Result.objects.get(pk=result_id) + session = newest_result.session + knobs = SessionKnob.objects.get_knobs_for_session(session) + + # Check that the minvals of tunable knobs are all decided + for knob_info in knobs: + if 'lowerbound' in knob_info and knob_info['lowerbound'] is not None: + lowerbound = float(knob_info['lowerbound']) + minval = float(knob_info['minval']) + if lowerbound < minval * 0.7: + # We need to do binary search to determine the minval of this knob + return test_knob_range(knob_info, + newest_result, minval, lowerbound, 'lowerbound'), algorithm + + # Check that the maxvals of tunable knobs are all decided + for knob_info in knobs: + if 'upperbound' in knob_info and knob_info['upperbound'] is not None: + upperbound = float(knob_info['upperbound']) + maxval = float(knob_info['maxval']) + if upperbound > maxval * 1.5: + # We need to do binary search to determine the maxval of this knob + return test_knob_range(knob_info, + newest_result, maxval, upperbound, 'upperbound'), algorithm + # Check that we've completed the background tasks at least once. We need # this data in order to make a configuration recommendation (until we # implement a sampling technique to generate new training data). - newest_result = Result.objects.get(pk=result_id) has_pipeline_data = PipelineData.objects.filter(workload=newest_result.workload).exists() - if not has_pipeline_data or newest_result.session.tuning_session == 'lhs': - if not has_pipeline_data and newest_result.session.tuning_session == 'tuning_session': + if not has_pipeline_data or session.tuning_session == 'lhs': + if not has_pipeline_data and session.tuning_session == 'tuning_session': LOG.debug("Background tasks haven't ran for this workload yet, picking data with lhs.") - all_samples = JSONUtil.loads(newest_result.session.lhs_samples) + all_samples = JSONUtil.loads(session.lhs_samples) if len(all_samples) == 0: - knobs = SessionKnob.objects.get_knobs_for_session(newest_result.session) - if newest_result.session.tuning_session == 'lhs': + if session.tuning_session == 'lhs': all_samples = gen_lhs_samples(knobs, 100) else: all_samples = gen_lhs_samples(knobs, 10) LOG.debug('%s: Generated LHS.\n\ndata=%s\n', AlgorithmType.name(algorithm), JSONUtil.dumps(all_samples[:5], pprint=True)) samples = all_samples.pop() - result = Result.objects.filter(pk=result_id) - agg_data = DataUtil.aggregate_data(result) agg_data['newest_result_id'] = result_id - agg_data['bad'] = True + agg_data['status'] = 'lhs' agg_data['config_recommend'] = samples - newest_result.session.lhs_samples = JSONUtil.dumps(all_samples) - newest_result.session.save() + session.lhs_samples = JSONUtil.dumps(all_samples) + session.save() LOG.debug('%s: Got LHS config.\n\ndata=%s\n', AlgorithmType.name(algorithm), JSONUtil.dumps(agg_data, pprint=True)) - elif newest_result.session.tuning_session == 'randomly_generate': - result = Result.objects.filter(pk=result_id) - knobs = SessionKnob.objects.get_knobs_for_session(newest_result.session) - + elif session.tuning_session == 'randomly_generate': # generate a config randomly random_knob_result = gen_random_data(knobs) - agg_data = DataUtil.aggregate_data(result) agg_data['newest_result_id'] = result_id - agg_data['bad'] = True + agg_data['status'] = 'random' agg_data['config_recommend'] = random_knob_result LOG.debug('%s: Finished generating a random config.\n\ndata=%s\n', AlgorithmType.name(algorithm), JSONUtil.dumps(agg_data, pprint=True)) @@ -226,19 +316,19 @@ def aggregate_target_results(result_id, algorithm): else: # Aggregate all knob config results tried by the target so far in this # tuning session and this tuning workload. - target_results = Result.objects.filter(session=newest_result.session, + target_results = Result.objects.filter(session=session, dbms=newest_result.dbms, workload=newest_result.workload) if len(target_results) == 0: raise Exception('Cannot find any results for session_id={}, dbms_id={}' - .format(newest_result.session, newest_result.dbms)) + .format(session, newest_result.dbms)) agg_data = DataUtil.aggregate_data(target_results) agg_data['newest_result_id'] = result_id - agg_data['bad'] = False + agg_data['status'] = 'good' # Clean knob data cleaned_agg_data = clean_knob_data(agg_data['X_matrix'], agg_data['X_columnlabels'], - newest_result.session) + session) agg_data['X_matrix'] = np.array(cleaned_agg_data[0]) agg_data['X_columnlabels'] = np.array(cleaned_agg_data[1]) @@ -248,6 +338,31 @@ def aggregate_target_results(result_id, algorithm): return agg_data, algorithm +def gen_test_maxval_data(knobs, test_knob, next_value): + next_config = {} + for knob in knobs: + name = knob["name"] + if name == test_knob: + next_config[name] = next_value + elif knob["vartype"] == VarType.BOOL: + next_config[name] = False + elif knob["vartype"] == VarType.ENUM: + next_config[name] = 0 + elif knob["vartype"] == VarType.INTEGER: + next_config[name] = int(knob["minval"]) + elif knob["vartype"] == VarType.REAL: + next_config[name] = float(knob["minval"]) + elif knob["vartype"] == VarType.STRING: + next_config[name] = "None" + elif knob["vartype"] == VarType.TIMESTAMP: + next_config[name] = "None" + else: + raise Exception( + 'Unknown variable type: {}'.format(knob["vartype"])) + + return next_config + + def gen_random_data(knobs): random_knob_result = {} for knob in knobs: @@ -685,14 +800,19 @@ def configuration_recommendation(recommendation_input): session = newest_result.session params = JSONUtil.loads(session.hyperparameters) - if target_data['bad'] is True: - if session.tuning_session == 'randomly_generate': - info = 'Randomly generated' + if target_data['status'] != 'good': + LOG.info(target_data['status']) + if target_data['status'] == 'random': + info = 'The config is generated by Random' + elif target_data['status'] == 'lhs': + info = 'The config is generated by LHS' + elif target_data['status'] == 'range_test': + info = 'Searching for the valid ranges of knobs' else: - info = 'WARNING: no training data, the config is generated by LHS' + info = 'Unknown' target_data_res = create_and_save_recommendation( recommended_knobs=target_data['config_recommend'], result=newest_result, - status='bad', info=info, + status=target_data['status'], info=info, pipeline_run=target_data['pipeline_run']) LOG.debug('%s: Skipping configuration recommendation.\nData:\n%s\n\n', AlgorithmType.name(algorithm), target_data) @@ -830,7 +950,7 @@ def map_workload(map_workload_input): start_ts = time.time() target_data, algorithm = map_workload_input - if target_data['bad']: + if target_data['status'] != 'good': assert target_data is not None target_data['pipeline_run'] = None LOG.debug('%s: Skipping workload mapping.\n\ndata=%s\n',