fix bugs when number of tunable knobs changes or are inconsistent

server/website/website/db/base/parser.py

@@ -60,6 +60,9 @@ class BaseParser:
         return res
 
     def convert_integer(self, int_value, metadata):
+        if len(str(int_value)) == 0:
+            # The value collected from the database is empty
+            return 0
         try:
             try:
                 converted = int(int_value)
@@ -74,8 +77,12 @@ class BaseParser:
                 converted = ConversionUtil.get_raw_size(
                     int_value, system=self.time_system)
             else:
-                raise Exception(
-                    'Unknown unit type: {}'.format(metadata.unit))
+                # If vartype is OTHER, try to decode it with bytes and time systems
+                converted = ConversionUtil.get_raw_size(
+                    int_value, system=self.bytes_system)
+                if converted is None:
+                    converted = ConversionUtil.get_raw_size(
+                        int_value, system=self.time_system)
         if converted is None:
             raise Exception('Invalid integer format for {}: {}'.format(
                 metadata.name, int_value))
@@ -99,10 +106,11 @@ class BaseParser:
             str_false += str(bval) + ' '
         return str_true + '; ' + str_false
 
-    def convert_dbms_knobs(self, knobs):
+    def convert_dbms_knobs(self, knobs, knob_catalog=None):
         knob_data = {}
-        tunable_knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True)
-        for metadata in tunable_knob_catalog:
+        if knob_catalog is None:
+            knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True)
+        for metadata in knob_catalog:
             name = metadata.name
             if name not in knobs:
                 if name.startswith('global.'):
@@ -159,6 +167,8 @@ class BaseParser:
         return knob_data
 
     def _check_knob_num_in_range(self, value, mdata, fix_knob_range=True):
+        if mdata.minval is None or mdata.maxval is None:
+            return True
         minval = float(mdata.minval)
         maxval = float(mdata.maxval)
         if fix_knob_range:

server/website/website/db/myrocks/parser.py

@@ -180,12 +180,13 @@ class MyRocksParser(BaseParser):
 
         return metric_data
 
-    def convert_dbms_knobs(self, knobs):
+    def convert_dbms_knobs(self, knobs, knob_catalog=None):
         knob_data = {}
-        tunable_knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True)
+        if knob_catalog is None:
+            knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True)
         for name, value in list(knobs.items()):
             prt_name = self.partial_name(name)
-            metadata = tunable_knob_catalog.filter(name=prt_name).first()
+            metadata = knob_catalog.filter(name=prt_name).first()
             if metadata:
                 conv_value = None
                 if metadata.vartype == VarType.BOOL:

server/website/website/db/parser.py

@@ -48,8 +48,8 @@ def parse_version_string(dbms_type, version_string):
     return parsed_version
 
 
-def convert_dbms_knobs(dbms_id, knobs):
-    return _get(dbms_id).convert_dbms_knobs(knobs)
+def convert_dbms_knobs(dbms_id, knobs, knob_catalog=None):
+    return _get(dbms_id).convert_dbms_knobs(knobs, knob_catalog)
 
 
 def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective):

server/website/website/tasks/async_tasks.py

@@ -90,7 +90,7 @@ class ConfigurationRecommendation(BaseTask):  # pylint: disable=abstract-method
 
 
 def clean_knob_data(knob_matrix, knob_labels, session):
-    # Makes sure that all knobs in the dbms are included in the knob_matrix and knob_labels
+    # Filter and amend knob_matrix and knob_labels according to the tunable knobs in the session
     knob_matrix = np.array(knob_matrix)
     session_knobs = SessionKnob.objects.get_knobs_for_session(session)
     knob_cat = [k['name'] for k in session_knobs]

server/website/website/views.py

@@ -617,11 +617,13 @@ def handle_result_files(session, files, execution_times=None):
         # Load, process, and store the knobs in the DBMS's configuration
         knob_dict, knob_diffs = parser.parse_dbms_knobs(
             dbms.pk, JSONUtil.loads(files['knobs']))
-        tunable_knob_dict = parser.convert_dbms_knobs(
-            dbms.pk, knob_dict)
+        knob_to_convert = KnobCatalog.objects.filter(dbms=dbms).exclude(
+            vartype=VarType.STRING).exclude(vartype=VarType.TIMESTAMP)
+        converted_knob_dict = parser.convert_dbms_knobs(
+            dbms.pk, knob_dict, knob_to_convert)
         knob_data = KnobData.objects.create_knob_data(
             session, JSONUtil.dumps(knob_dict, pprint=True, sort=True),
-            JSONUtil.dumps(tunable_knob_dict, pprint=True, sort=True), dbms)
+            JSONUtil.dumps(converted_knob_dict, pprint=True, sort=True), dbms)
 
         # Load, process, and store the runtime metrics exposed by the DBMS
         initial_metric_dict, initial_metric_diffs = parser.parse_dbms_metrics(