From 0baaefeb92eb0617a3f9547e9c0bc88baa34fe08 Mon Sep 17 00:00:00 2001 From: yangdsh Date: Wed, 11 Mar 2020 03:46:55 +0000 Subject: [PATCH] fix bugs when number of tunable knobs changes or are inconsistent --- server/website/website/db/base/parser.py | 20 +++++++++++++++----- server/website/website/db/myrocks/parser.py | 7 ++++--- server/website/website/db/parser.py | 4 ++-- server/website/website/tasks/async_tasks.py | 2 +- server/website/website/views.py | 8 +++++--- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/server/website/website/db/base/parser.py b/server/website/website/db/base/parser.py index f61d911..853c4c0 100644 --- a/server/website/website/db/base/parser.py +++ b/server/website/website/db/base/parser.py @@ -60,6 +60,9 @@ class BaseParser: return res def convert_integer(self, int_value, metadata): + if len(str(int_value)) == 0: + # The value collected from the database is empty + return 0 try: try: converted = int(int_value) @@ -74,8 +77,12 @@ class BaseParser: converted = ConversionUtil.get_raw_size( int_value, system=self.time_system) else: - raise Exception( - 'Unknown unit type: {}'.format(metadata.unit)) + # If vartype is OTHER, try to decode it with bytes and time systems + converted = ConversionUtil.get_raw_size( + int_value, system=self.bytes_system) + if converted is None: + converted = ConversionUtil.get_raw_size( + int_value, system=self.time_system) if converted is None: raise Exception('Invalid integer format for {}: {}'.format( metadata.name, int_value)) @@ -99,10 +106,11 @@ class BaseParser: str_false += str(bval) + ' ' return str_true + '; ' + str_false - def convert_dbms_knobs(self, knobs): + def convert_dbms_knobs(self, knobs, knob_catalog=None): knob_data = {} - tunable_knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True) - for metadata in tunable_knob_catalog: + if knob_catalog is None: + knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True) + for metadata in knob_catalog: name = metadata.name if name not in knobs: if name.startswith('global.'): @@ -159,6 +167,8 @@ class BaseParser: return knob_data def _check_knob_num_in_range(self, value, mdata, fix_knob_range=True): + if mdata.minval is None or mdata.maxval is None: + return True minval = float(mdata.minval) maxval = float(mdata.maxval) if fix_knob_range: diff --git a/server/website/website/db/myrocks/parser.py b/server/website/website/db/myrocks/parser.py index 6aee107..01b6319 100644 --- a/server/website/website/db/myrocks/parser.py +++ b/server/website/website/db/myrocks/parser.py @@ -180,12 +180,13 @@ class MyRocksParser(BaseParser): return metric_data - def convert_dbms_knobs(self, knobs): + def convert_dbms_knobs(self, knobs, knob_catalog=None): knob_data = {} - tunable_knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True) + if knob_catalog is None: + knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True) for name, value in list(knobs.items()): prt_name = self.partial_name(name) - metadata = tunable_knob_catalog.filter(name=prt_name).first() + metadata = knob_catalog.filter(name=prt_name).first() if metadata: conv_value = None if metadata.vartype == VarType.BOOL: diff --git a/server/website/website/db/parser.py b/server/website/website/db/parser.py index 97fdf7d..9e1cb1d 100644 --- a/server/website/website/db/parser.py +++ b/server/website/website/db/parser.py @@ -48,8 +48,8 @@ def parse_version_string(dbms_type, version_string): return parsed_version -def convert_dbms_knobs(dbms_id, knobs): - return _get(dbms_id).convert_dbms_knobs(knobs) +def convert_dbms_knobs(dbms_id, knobs, knob_catalog=None): + return _get(dbms_id).convert_dbms_knobs(knobs, knob_catalog) def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective): diff --git a/server/website/website/tasks/async_tasks.py b/server/website/website/tasks/async_tasks.py index b5d4157..148c10e 100644 --- a/server/website/website/tasks/async_tasks.py +++ b/server/website/website/tasks/async_tasks.py @@ -90,7 +90,7 @@ class ConfigurationRecommendation(BaseTask): # pylint: disable=abstract-method def clean_knob_data(knob_matrix, knob_labels, session): - # Makes sure that all knobs in the dbms are included in the knob_matrix and knob_labels + # Filter and amend knob_matrix and knob_labels according to the tunable knobs in the session knob_matrix = np.array(knob_matrix) session_knobs = SessionKnob.objects.get_knobs_for_session(session) knob_cat = [k['name'] for k in session_knobs] diff --git a/server/website/website/views.py b/server/website/website/views.py index b28c2ab..45b04c8 100644 --- a/server/website/website/views.py +++ b/server/website/website/views.py @@ -617,11 +617,13 @@ def handle_result_files(session, files, execution_times=None): # Load, process, and store the knobs in the DBMS's configuration knob_dict, knob_diffs = parser.parse_dbms_knobs( dbms.pk, JSONUtil.loads(files['knobs'])) - tunable_knob_dict = parser.convert_dbms_knobs( - dbms.pk, knob_dict) + knob_to_convert = KnobCatalog.objects.filter(dbms=dbms).exclude( + vartype=VarType.STRING).exclude(vartype=VarType.TIMESTAMP) + converted_knob_dict = parser.convert_dbms_knobs( + dbms.pk, knob_dict, knob_to_convert) knob_data = KnobData.objects.create_knob_data( session, JSONUtil.dumps(knob_dict, pprint=True, sort=True), - JSONUtil.dumps(tunable_knob_dict, pprint=True, sort=True), dbms) + JSONUtil.dumps(converted_knob_dict, pprint=True, sort=True), dbms) # Load, process, and store the runtime metrics exposed by the DBMS initial_metric_dict, initial_metric_diffs = parser.parse_dbms_metrics(