Added outer db module and moved parser code into it

server/website/website/db/__init__.py

@@ -0,0 +1,5 @@
+#
+# OtterTune - __init__.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#

server/website/website/db/base/__init__.py

@@ -0,0 +1,5 @@
+#
+# OtterTune - __init__.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#

server/website/website/db/base/parser.py

@@ -0,0 +1,422 @@
+#
+# OtterTune - parser.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+from collections import OrderedDict
+
+from website.models import KnobCatalog, KnobUnitType, MetricCatalog
+from website.types import BooleanType, MetricType, VarType
+from website.utils import ConversionUtil
+
+
+# pylint: disable=no-self-use
+class BaseParser:
+
+    def __init__(self, dbms_obj):
+        knobs = KnobCatalog.objects.filter(dbms=dbms_obj)
+        self.knob_catalog_ = {k.name: k for k in knobs}
+        self.tunable_knob_catalog_ = {
+            k: v for k, v in self.knob_catalog_.items() if
+            v.tunable is True}
+
+        metrics = MetricCatalog.objects.filter(dbms=dbms_obj)
+        self.metric_catalog_ = {m.name: m for m in metrics}
+        numeric_mtypes = (MetricType.COUNTER, MetricType.STATISTICS)
+        self.numeric_metric_catalog_ = {
+            m: v for m, v in self.metric_catalog_.items() if
+            v.metric_type in numeric_mtypes}
+
+        self.valid_true_val = ("on", "true", "yes")
+        self.valid_false_val = ("off", "false", "no")
+        self.true_value = 'on'
+        self.false_value = 'off'
+
+        self.bytes_system = ConversionUtil.DEFAULT_BYTES_SYSTEM
+        self.time_system = ConversionUtil.DEFAULT_TIME_SYSTEM
+        self.min_bytes_unit = 'kB'
+        self.min_time_unit = 'ms'
+
+    @property
+    def transactions_counter(self):
+        raise NotImplementedError()
+
+    @property
+    def latency_timer(self):
+        raise NotImplementedError()
+
+    def target_metric(self, target_objective=None):
+        if target_objective == 'throughput_txn_per_sec' or target_objective is None:
+            # throughput
+            res = self.transactions_counter
+        elif target_objective == '99th_lat_ms':
+            # 99 percentile latency
+            res = self.latency_timer
+        else:
+            raise Exception("Target Objective {} Not Supported".format(target_objective))
+
+        return res
+
+    def parse_version_string(self, version_string):
+        return version_string
+
+    def convert_bool(self, bool_value, metadata):
+        if isinstance(bool_value, str):
+            bool_value = bool_value.lower()
+
+        if bool_value in self.valid_true_val:
+            res = BooleanType.TRUE
+        elif bool_value in self.valid_false_val:
+            res = BooleanType.FALSE
+        else:
+            raise Exception("Invalid Boolean {}".format(bool_value))
+
+        return res
+
+    def convert_enum(self, enum_value, metadata):
+        enumvals = metadata.enumvals.split(',')
+        try:
+            res = enumvals.index(enum_value)
+        except ValueError:
+            raise Exception('Invalid enum value for variable {} ({})'.format(
+                metadata.name, enum_value))
+
+        return res
+
+    def convert_integer(self, int_value, metadata):
+        try:
+            try:
+                converted = int(int_value)
+            except ValueError:
+                converted = int(float(int_value))
+
+        except ValueError:
+            if metadata.unit == KnobUnitType.BYTES:
+                converted = ConversionUtil.get_raw_size(
+                    int_value, system=self.bytes_system)
+            elif metadata.unit == KnobUnitType.MILLISECONDS:
+                converted = ConversionUtil.get_raw_size(
+                    int_value, system=self.time_system)
+            else:
+                raise Exception(
+                    'Unknown unit type: {}'.format(metadata.unit))
+        if converted is None:
+            raise Exception('Invalid integer format for {}: {}'.format(
+                metadata.name, int_value))
+        return converted
+
+    def convert_real(self, real_value, metadata):
+        return float(real_value)
+
+    def convert_string(self, string_value, metadata):
+        return string_value
+
+    def convert_timestamp(self, timestamp_value, metadata):
+        return timestamp_value
+
+    def valid_boolean_val_to_string(self):
+        str_true = 'valid true values: '
+        for bval in self.valid_true_val:
+            str_true += str(bval) + ' '
+        str_false = 'valid false values: '
+        for bval in self.valid_false_val:
+            str_false += str(bval) + ' '
+        return str_true + '; ' + str_false
+
+    def convert_dbms_knobs(self, knobs):
+        knob_data = {}
+        for name, metadata in list(self.tunable_knob_catalog_.items()):
+            if metadata.tunable is False:
+                continue
+            if name not in knobs:
+                continue
+            value = knobs[name]
+            conv_value = None
+
+            if metadata.vartype == VarType.BOOL:
+                if not self._check_knob_bool_val(value):
+                    raise Exception('Knob boolean value not valid! '
+                                    'Boolean values should be one of: {}, '
+                                    'but the actual value is: {}'
+                                    .format(self.valid_boolean_val_to_string(),
+                                            str(value)))
+                conv_value = self.convert_bool(value, metadata)
+
+            elif metadata.vartype == VarType.ENUM:
+                conv_value = self.convert_enum(value, metadata)
+
+            elif metadata.vartype == VarType.INTEGER:
+                conv_value = self.convert_integer(value, metadata)
+                if not self._check_knob_num_in_range(conv_value, metadata):
+                    raise Exception('Knob integer num value not in range! '
+                                    'min: {}, max: {}, actual: {}'
+                                    .format(metadata.minval,
+                                            metadata.maxval, str(conv_value)))
+
+            elif metadata.vartype == VarType.REAL:
+                conv_value = self.convert_real(value, metadata)
+                if not self._check_knob_num_in_range(conv_value, metadata):
+                    raise Exception('Knob real num value not in range! '
+                                    'min: {}, max: {}, actual: {}'
+                                    .format(metadata.minval,
+                                            metadata.maxval, str(conv_value)))
+
+            elif metadata.vartype == VarType.STRING:
+                conv_value = self.convert_string(value, metadata)
+
+            elif metadata.vartype == VarType.TIMESTAMP:
+                conv_value = self.convert_timestamp(value, metadata)
+
+            else:
+                raise Exception(
+                    'Unknown variable type: {}'.format(metadata.vartype))
+
+            if conv_value is None:
+                raise Exception('Param value for {} cannot be null'.format(name))
+            knob_data[name] = conv_value
+
+        return knob_data
+
+    def _check_knob_num_in_range(self, value, mdata):
+        return value >= float(mdata.minval) and value <= float(mdata.maxval)
+
+    def _check_knob_bool_val(self, value):
+        if isinstance(str, value):
+            value = value.lower()
+        return value in self.valid_true_val or value in self.valid_false_val
+
+    def convert_dbms_metrics(self, metrics, observation_time, target_objective=None):
+        #         if len(metrics) != len(self.numeric_metric_catalog_):
+        #             raise Exception('The number of metrics should be equal!')
+        metric_data = {}
+        for name, metadata in list(self.numeric_metric_catalog_.items()):
+            value = metrics[name]
+            if metadata.metric_type == MetricType.COUNTER:
+                converted = self.convert_integer(value, metadata)
+                metric_data[name] = float(converted) / observation_time
+            elif metadata.metric_type == MetricType.STATISTICS:
+                converted = self.convert_integer(value, metadata)
+                metric_data[name] = float(converted)
+            else:
+                raise Exception(
+                    'Unknown metric type for {}: {}'.format(name, metadata.metric_type))
+
+        if target_objective is not None and self.target_metric(target_objective) not in metric_data:
+            raise Exception("Cannot find objective function")
+
+        if target_objective is not None:
+            metric_data[target_objective] = metric_data[self.target_metric(target_objective)]
+        else:
+            # default
+            metric_data['throughput_txn_per_sec'] = \
+                metric_data[self.target_metric(target_objective)]
+
+        return metric_data
+
+    def extract_valid_variables(self, variables, catalog, default_value=None):
+        valid_variables = {}
+        diff_log = []
+        valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())}
+
+        # First check that the names of all variables are valid (i.e., listed
+        # in the official catalog). Invalid variables are logged as 'extras'.
+        # Variable names that are valid but differ in capitalization are still
+        # added to valid_variables but with the proper capitalization. They
+        # are also logged as 'miscapitalized'.
+        for var_name, var_value in list(variables.items()):
+            lc_var_name = var_name.lower()
+            if lc_var_name in valid_lc_variables:
+                valid_name = valid_lc_variables[lc_var_name].name
+                if var_name != valid_name:
+                    diff_log.append(('miscapitalized', valid_name, var_name, var_value))
+                valid_variables[valid_name] = var_value
+            else:
+                diff_log.append(('extra', None, var_name, var_value))
+
+        # Next find all item names that are listed in the catalog but missing from
+        # variables. Missing variables are added to valid_variables with the given
+        # default_value if provided (or the item's actual default value if not) and
+        # logged as 'missing'.
+        lc_variables = {k.lower(): v for k, v in list(variables.items())}
+        for valid_lc_name, metadata in list(valid_lc_variables.items()):
+            if valid_lc_name not in lc_variables:
+                diff_log.append(('missing', metadata.name, None, None))
+                valid_variables[metadata.name] = default_value if \
+                    default_value is not None else metadata.default
+        assert len(valid_variables) == len(catalog)
+        return valid_variables, diff_log
+
+    def parse_helper(self, scope, valid_variables, view_variables):
+        for view_name, variables in list(view_variables.items()):
+            for var_name, var_value in list(variables.items()):
+                full_name = '{}.{}'.format(view_name, var_name)
+                if full_name not in valid_variables:
+                    valid_variables[full_name] = []
+                valid_variables[full_name].append(var_value)
+        return valid_variables
+
+    def parse_dbms_variables(self, variables):
+        valid_variables = {}
+        for scope, sub_vars in list(variables.items()):
+            if sub_vars is None:
+                continue
+            if scope == 'global':
+                valid_variables.update(self.parse_helper(scope, valid_variables, sub_vars))
+            elif scope == 'local':
+                for _, viewnames in list(sub_vars.items()):
+                    for viewname, objnames in list(viewnames.items()):
+                        for _, view_vars in list(objnames.items()):
+                            valid_variables.update(self.parse_helper(
+                                scope, valid_variables, {viewname: view_vars}))
+            else:
+                raise Exception('Unsupported variable scope: {}'.format(scope))
+        return valid_variables
+
+    def parse_dbms_knobs(self, knobs):
+        valid_knobs = self.parse_dbms_variables(knobs)
+
+        for k in list(valid_knobs.keys()):
+            assert len(valid_knobs[k]) == 1
+            valid_knobs[k] = valid_knobs[k][0]
+        # Extract all valid knobs
+        return self.extract_valid_variables(valid_knobs, self.knob_catalog_)
+
+    def parse_dbms_metrics(self, metrics):
+        # Some DBMSs measure different types of stats (e.g., global, local)
+        # at different scopes (e.g. indexes, # tables, database) so for now
+        # we just combine them
+        valid_metrics = self.parse_dbms_variables(metrics)
+
+        # Extract all valid metrics
+        valid_metrics, diffs = self.extract_valid_variables(
+            valid_metrics, self.metric_catalog_, default_value='0')
+
+        # Combine values
+        for name, values in list(valid_metrics.items()):
+            metric = self.metric_catalog_[name]
+            if metric.metric_type == MetricType.INFO or len(values) == 1:
+                valid_metrics[name] = values[0]
+            elif metric.metric_type == MetricType.COUNTER or \
+                    metric.metric_type == MetricType.STATISTICS:
+                conv_fn = int if metric.vartype == VarType.INTEGER else float
+                values = [conv_fn(v) for v in values if v is not None]
+                if len(values) == 0:
+                    valid_metrics[name] = 0
+                else:
+                    valid_metrics[name] = str(sum(values))
+            else:
+                raise Exception(
+                    'Invalid metric type: {}'.format(metric.metric_type))
+        return valid_metrics, diffs
+
+    def calculate_change_in_metrics(self, metrics_start, metrics_end):
+        adjusted_metrics = {}
+        for met_name, start_val in list(metrics_start.items()):
+            end_val = metrics_end[met_name]
+            met_info = self.metric_catalog_[met_name]
+            if met_info.vartype == VarType.INTEGER or \
+                    met_info.vartype == VarType.REAL:
+                conversion_fn = self.convert_integer if \
+                    met_info.vartype == VarType.INTEGER else \
+                    self.convert_real
+                start_val = conversion_fn(start_val, met_info)
+                end_val = conversion_fn(end_val, met_info)
+                if met_info.metric_type == MetricType.COUNTER:
+                    adj_val = end_val - start_val
+                else:  # MetricType.STATISTICS or MetricType.INFO
+                    adj_val = end_val
+                assert adj_val >= 0, \
+                    '{} wrong metric type: {} (start={}, end={}, diff={})'.format(
+                        met_name, MetricType.name(met_info.metric_type), start_val,
+                        end_val, end_val - start_val)
+
+                adjusted_metrics[met_name] = adj_val
+            else:
+                # This metric is either a bool, enum, string, or timestamp
+                # so take last recorded value from metrics_end
+                adjusted_metrics[met_name] = end_val
+        return adjusted_metrics
+
+    def create_knob_configuration(self, tuning_knobs):
+        configuration = {}
+        for knob_name, knob_value in sorted(tuning_knobs.items()):
+            # FIX ME: for now it only shows the global knobs, works for Postgres
+            if knob_name.startswith('global.'):
+                knob_name_global = knob_name[knob_name.find('.') + 1:]
+                configuration[knob_name_global] = knob_value
+
+        configuration = OrderedDict(sorted(configuration.items()))
+        return configuration
+
+    def format_bool(self, bool_value, metadata):
+        return self.true_value if bool_value == BooleanType.TRUE else self.false_value
+
+    def format_enum(self, enum_value, metadata):
+        enumvals = metadata.enumvals.split(',')
+        return enumvals[int(round(enum_value))]
+
+    # def format_integer(self, int_value, metadata):
+    #     return int(round(int_value))
+
+    def format_integer(self, int_value, metadata):
+        int_value = int(round(int_value))
+        if metadata.unit != KnobUnitType.OTHER and int_value > 0:
+            if metadata.unit == KnobUnitType.BYTES:
+                int_value = ConversionUtil.get_human_readable2(
+                    int_value, self.bytes_system, 'kB')
+            elif metadata.unit == KnobUnitType.MILLISECONDS:
+                int_value = ConversionUtil.get_human_readable2(
+                    int_value, self.time_system, 'ms')
+            else:
+                raise Exception(
+                    'Invalid unit type for {}: {}'.format(
+                        metadata.name, metadata.unit))
+
+        return int_value
+
+    def format_real(self, real_value, metadata):
+        return round(float(real_value), 3)
+
+    def format_string(self, string_value, metadata):
+        return string_value
+
+    def format_timestamp(self, timestamp_value, metadata):
+        return timestamp_value
+
+    def format_dbms_knobs(self, knobs):
+        formatted_knobs = {}
+        for knob_name, knob_value in list(knobs.items()):
+            metadata = self.knob_catalog_.get(knob_name, None)
+            if (metadata is None):
+                raise Exception('Unknown knob {}'.format(knob_name))
+            fvalue = None
+            if metadata.vartype == VarType.BOOL:
+                fvalue = self.format_bool(knob_value, metadata)
+            elif metadata.vartype == VarType.ENUM:
+                fvalue = self.format_enum(knob_value, metadata)
+            elif metadata.vartype == VarType.INTEGER:
+                fvalue = self.format_integer(knob_value, metadata)
+            elif metadata.vartype == VarType.REAL:
+                fvalue = self.format_real(knob_value, metadata)
+            elif metadata.vartype == VarType.STRING:
+                fvalue = self.format_string(knob_value, metadata)
+            elif metadata.vartype == VarType.TIMESTAMP:
+                fvalue = self.format_timestamp(knob_value, metadata)
+            else:
+                raise Exception('Unknown variable type for {}: {}'.format(
+                    knob_name, metadata.vartype))
+            if fvalue is None:
+                raise Exception('Cannot format value for {}: {}'.format(
+                    knob_name, knob_value))
+            formatted_knobs[knob_name] = fvalue
+        return formatted_knobs
+
+    def filter_numeric_metrics(self, metrics):
+        return OrderedDict(((k, v) for k, v in list(metrics.items()) if
+                            k in self.numeric_metric_catalog_))
+
+    def filter_tunable_knobs(self, knobs):
+        return OrderedDict(((k, v) for k, v in list(knobs.items()) if
+                            k in self.tunable_knob_catalog_))
+
+# pylint: enable=no-self-use

server/website/website/db/myrocks/__init__.py

@@ -0,0 +1,5 @@
+#
+# OtterTune - __init__.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#

server/website/website/db/myrocks/parser.py

@@ -0,0 +1,208 @@
+#
+# OtterTune - parser.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+
+import re
+from collections import OrderedDict
+
+from ..base.parser import BaseParser
+from website.types import MetricType, VarType
+
+
+class MyRocksParser(BaseParser):
+
+    @property
+    def transactions_counter(self):
+        return 'session_status.questions'
+
+    @property
+    def latency_timer(self):
+        raise NotImplementedError()
+
+    def parse_version_string(self, version_string):
+        dbms_version = version_string.split(',')[0]
+        return re.search(r'\d+\.\d+(?=\.\d+)', dbms_version).group(0)
+
+    def parse_helper(self, scope, valid_variables, view_variables):
+        for view_name, variables in list(view_variables.items()):
+            if scope == 'local':
+                for obj_name, sub_vars in list(variables.items()):
+                    for var_name, var_value in list(sub_vars.items()):  # local
+                        full_name = '{}.{}.{}'.format(view_name, obj_name, var_name)
+                        valid_variables[full_name] = var_value
+            elif scope == 'global':
+                for var_name, var_value in list(variables.items()):  # global
+                    full_name = '{}.{}'.format(view_name, var_name)
+                    valid_variables[full_name] = var_value
+            else:
+                raise Exception('Unsupported variable scope: {}'.format(scope))
+        return valid_variables
+
+    # global variable fullname: viewname.varname
+    # local variable fullname: viewname.objname.varname
+    # return format: valid_variables = {var_fullname:var_val}
+    def parse_dbms_variables(self, variables):
+        valid_variables = {}
+        for scope, sub_vars in list(variables.items()):
+            if sub_vars is None:
+                continue
+            if scope == 'global':
+                valid_variables.update(self.parse_helper('global', valid_variables, sub_vars))
+            elif scope == 'local':
+                for _, viewnames in list(sub_vars.items()):
+                    for viewname, objnames in list(viewnames.items()):
+                        for obj_name, view_vars in list(objnames.items()):
+                            valid_variables.update(self.parse_helper(
+                                'local', valid_variables, {viewname: {obj_name: view_vars}}))
+            else:
+                raise Exception('Unsupported variable scope: {}'.format(scope))
+        return valid_variables
+
+    # local variable: viewname.objname.varname
+    # global variable: viewname.varname
+    # This function is to change local variable fullname to viewname.varname, global
+    # variable remains same. This is because local varialbe in knob_catalog is in
+    # parial format (i,e. viewname.varname)
+    @staticmethod
+    def partial_name(full_name):
+        var_name = full_name.split('.')
+        if len(var_name) == 2:  # global variable
+            return full_name
+        elif len(var_name) == 3:  # local variable
+            return var_name[0] + '.' + var_name[2]
+        else:
+            raise Exception('Invalid variable full name: {}'.format(full_name))
+
+    def extract_valid_variables(self, variables, catalog, default_value=None):
+        valid_variables = {}
+        diff_log = []
+        valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())}
+
+        # First check that the names of all variables are valid (i.e., listed
+        # in the official catalog). Invalid variables are logged as 'extras'.
+        # Variable names that are valid but differ in capitalization are still
+        # added to valid_variables but with the proper capitalization. They
+        # are also logged as 'miscapitalized'.
+        for var_name, var_value in list(variables.items()):
+            lc_var_name = var_name.lower()
+            prt_name = self.partial_name(lc_var_name)
+            if prt_name in valid_lc_variables:
+                valid_name = valid_lc_variables[prt_name].name
+                if prt_name != valid_name:
+                    diff_log.append(('miscapitalized', valid_name, var_name, var_value))
+                valid_variables[var_name] = var_value
+            else:
+                diff_log.append(('extra', None, var_name, var_value))
+
+        # Next find all item names that are listed in the catalog but missing from
+        # variables. Missing global variables are added to valid_variables with
+        # the given default_value if provided (or the item's actual default value
+        # if not) and logged as 'missing'. For now missing local variables are
+        # not added to valid_variables
+        lc_variables = {self.partial_name(k.lower()): v
+                        for k, v in list(variables.items())}
+        for valid_lc_name, metadata in list(valid_lc_variables.items()):
+            if valid_lc_name not in lc_variables:
+                diff_log.append(('missing', metadata.name, None, None))
+                if metadata.scope == 'global':
+                    valid_variables[metadata.name] = default_value if \
+                        default_value is not None else metadata.default
+        return valid_variables, diff_log
+
+    def calculate_change_in_metrics(self, metrics_start, metrics_end):
+        adjusted_metrics = {}
+        for met_name, start_val in list(metrics_start.items()):
+            end_val = metrics_end[met_name]
+            met_info = self.metric_catalog_[self.partial_name(met_name)]
+            if met_info.vartype == VarType.INTEGER or \
+                    met_info.vartype == VarType.REAL:
+                conversion_fn = self.convert_integer if \
+                    met_info.vartype == VarType.INTEGER else \
+                    self.convert_real
+                start_val = conversion_fn(start_val, met_info)
+                end_val = conversion_fn(end_val, met_info)
+                adj_val = end_val - start_val
+                assert adj_val >= 0
+                adjusted_metrics[met_name] = adj_val
+            else:
+                # This metric is either a bool, enum, string, or timestamp
+                # so take last recorded value from metrics_end
+                adjusted_metrics[met_name] = end_val
+        return adjusted_metrics
+
+    def parse_dbms_knobs(self, knobs):
+        valid_knobs = self.parse_dbms_variables(knobs)
+        # Extract all valid knobs
+        return self.extract_valid_variables(
+            valid_knobs, self.knob_catalog_)
+
+    def parse_dbms_metrics(self, metrics):
+        valid_metrics = self.parse_dbms_variables(metrics)
+        # Extract all valid metrics
+        valid_metrics, diffs = self.extract_valid_variables(
+            valid_metrics, self.metric_catalog_, default_value='0')
+        return valid_metrics, diffs
+
+    def convert_dbms_metrics(self, metrics, observation_time, target_objective=None):
+        metric_data = {}
+        for name, value in list(metrics.items()):
+            prt_name = self.partial_name(name)
+            if prt_name in self.numeric_metric_catalog_:
+                metadata = self.numeric_metric_catalog_[prt_name]
+                if metadata.metric_type == MetricType.COUNTER:
+                    converted = self.convert_integer(value, metadata)
+                    metric_data[name] = float(converted) / observation_time
+                else:
+                    raise Exception('Unknown metric type for {}: {}'.format(
+                        name, metadata.metric_type))
+
+        if target_objective is not None and self.target_metric(target_objective) not in metric_data:
+            raise Exception("Cannot find objective function")
+
+        if target_objective is not None:
+            metric_data[target_objective] = metric_data[self.target_metric(target_objective)]
+        else:
+            # default
+            metric_data['throughput_txn_per_sec'] = \
+                metric_data[self.target_metric(target_objective)]
+        return metric_data
+
+    def convert_dbms_knobs(self, knobs):
+        knob_data = {}
+        for name, value in list(knobs.items()):
+            prt_name = self.partial_name(name)
+            if prt_name in self.tunable_knob_catalog_:
+                metadata = self.tunable_knob_catalog_[prt_name]
+                assert(metadata.tunable)
+                value = knobs[name]
+                conv_value = None
+                if metadata.vartype == VarType.BOOL:
+                    conv_value = self.convert_bool(value, metadata)
+                elif metadata.vartype == VarType.ENUM:
+                    conv_value = self.convert_enum(value, metadata)
+                elif metadata.vartype == VarType.INTEGER:
+                    conv_value = self.convert_integer(value, metadata)
+                elif metadata.vartype == VarType.REAL:
+                    conv_value = self.convert_real(value, metadata)
+                elif metadata.vartype == VarType.STRING:
+                    conv_value = self.convert_string(value, metadata)
+                elif metadata.vartype == VarType.TIMESTAMP:
+                    conv_value = self.convert_timestamp(value, metadata)
+                else:
+                    raise Exception(
+                        'Unknown variable type: {}'.format(metadata.vartype))
+                if conv_value is None:
+                    raise Exception(
+                        'Param value for {} cannot be null'.format(name))
+                knob_data[name] = conv_value
+        return knob_data
+
+    def filter_numeric_metrics(self, metrics):
+        return OrderedDict([(k, v) for k, v in list(metrics.items()) if
+                            self.partial_name(k) in self.numeric_metric_catalog_])
+
+    def filter_tunable_knobs(self, knobs):
+        return OrderedDict([(k, v) for k, v in list(knobs.items()) if
+                            self.partial_name(k) in self.tunable_knob_catalog_])

server/website/website/db/oracle/__init__.py

@@ -0,0 +1,5 @@
+#
+# OtterTune - __init__.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#

server/website/website/db/oracle/parser.py

@@ -0,0 +1,30 @@
+#
+# OtterTune - parser.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+
+from ..base.parser import BaseParser
+
+
+class OracleParser(BaseParser):
+
+    def __init__(self, dbms_obj):
+        super().__init__(dbms_obj)
+        self.true_value = 'TRUE'
+        self.false_value = 'FALSE'
+        self.bytes_system = (
+            (1024 ** 4, 'T'),
+            (1024 ** 3, 'G'),
+            (1024 ** 2, 'M'),
+            (1024 ** 1, 'k'),
+        )
+        self.min_bytes_unit = 'k'
+
+    @property
+    def transactions_counter(self):
+        return 'global.user commits'
+
+    @property
+    def latency_timer(self):
+        raise NotImplementedError()

server/website/website/db/parser.py

@@ -0,0 +1,86 @@
+#
+# OtterTune - parser.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+
+from website.models import DBMSCatalog
+from website.types import DBMSType
+
+from .myrocks.parser import MyRocksParser
+from .postgres.parser import PostgresParser
+from .oracle.parser import OracleParser
+
+_DBMS_PARSERS = {}
+
+
+def _get(dbms_id):
+    dbms_id = int(dbms_id)
+    db_parser = _DBMS_PARSERS.get(dbms_id, None)
+    if db_parser is None:
+        obj = DBMSCatalog.objects.get(id=dbms_id)
+        if obj.type == DBMSType.POSTGRES:
+            clz = PostgresParser
+        elif obj.type == DBMSType.MYROCKS:
+            clz = MyRocksParser
+        elif obj.type == DBMSType.ORACLE:
+            clz = OracleParser
+        else:
+            raise NotImplementedError('Implement me! {}'.format(obj))
+
+        db_parser = clz(obj)
+        _DBMS_PARSERS[dbms_id] = db_parser
+
+    return db_parser
+
+
+def parse_version_string(dbms_type, version_string):
+    dbmss = DBMSCatalog.objects.filter(type=dbms_type)
+    parsed_version = None
+    for instance in dbmss:
+        db_parser = _get(instance.pk)
+        try:
+            parsed_version = db_parser.parse_version_string(version_string)
+        except AttributeError:
+            pass
+        if parsed_version is not None:
+            break
+    return parsed_version
+
+
+def convert_dbms_knobs(dbms_id, knobs):
+    return _get(dbms_id).convert_dbms_knobs(knobs)
+
+
+def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective=None):
+    return _get(dbms_id).convert_dbms_metrics(
+        numeric_metrics, observation_time, target_objective)
+
+
+def parse_dbms_knobs(dbms_id, knobs):
+    return _get(dbms_id).parse_dbms_knobs(knobs)
+
+
+def parse_dbms_metrics(dbms_id, metrics):
+    return _get(dbms_id).parse_dbms_metrics(metrics)
+
+
+def create_knob_configuration(dbms_id, tuning_knobs):
+    return _get(dbms_id).create_knob_configuration(tuning_knobs)
+
+
+def format_dbms_knobs(dbms_id, knobs):
+    return _get(dbms_id).format_dbms_knobs(knobs)
+
+
+def filter_numeric_metrics(dbms_id, metrics):
+    return _get(dbms_id).filter_numeric_metrics(metrics)
+
+
+def filter_tunable_knobs(dbms_id, knobs):
+    return _get(dbms_id).filter_tunable_knobs(knobs)
+
+
+def calculate_change_in_metrics(dbms_id, metrics_start, metrics_end):
+    return _get(dbms_id).calculate_change_in_metrics(
+        metrics_start, metrics_end)

server/website/website/db/postgres/__init__.py

@@ -0,0 +1,5 @@
+#
+# OtterTune - __init__.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#

server/website/website/db/postgres/parser.py

@@ -0,0 +1,32 @@
+#
+# OtterTune - parser.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+
+import re
+
+from ..base.parser import BaseParser
+from website.utils import ConversionUtil
+
+
+class PostgresParser(BaseParser):
+
+    def __init__(self, dbms_obj):
+        super().__init__(dbms_obj)
+        self.valid_true_val = ("on", "true", "yes", 1)
+        self.valid_false_val = ("off", "false", "no", 0)
+        self.bytes_system = [(f, s) for f, s in ConversionUtil.DEFAULT_BYTES_SYSTEM
+                             if s in ('TB', 'GB', 'MB', 'kB')]
+
+    @property
+    def transactions_counter(self):
+        return 'pg_stat_database.xact_commit'
+
+    @property
+    def latency_timer(self):
+        raise NotImplementedError()
+
+    def parse_version_string(self, version_string):
+        dbms_version = version_string.split(',')[0]
+        return re.search(r'\d+\.\d+(?=\.\d+)', dbms_version).group(0)