Added outer db module and moved parser code into it

server/website/tests/test_parser.py

@@ -7,7 +7,8 @@
 from abc import ABCMeta, abstractmethod
 import mock
 from django.test import TestCase
-from website.parser.postgres import PostgresParser
+from website.db import parser
+# from website.db.parser.postgres import PostgresParser
 from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType
 from website.models import DBMSCatalog, KnobCatalog
 
@@ -221,7 +222,7 @@ class PostgresParserTests(BaseParserTests, TestCase):
     def setUp(self):
         dbms_obj = DBMSCatalog.objects.filter(
             type=DBMSType.POSTGRES, version="9.6").first()
-        self.test_dbms = PostgresParser(dbms_obj)
+        self.test_dbms = parser._get(dbms_obj.pk)  # pylint: disable=protected-access
 
     def test_convert_dbms_knobs(self):
         super().test_convert_dbms_knobs()

server/website/website/db/__init__.py

@@ -3,4 +3,3 @@
 #
 # Copyright (c) 2017-18, Carnegie Mellon University Database Group
 #
-from .parser import Parser

server/website/website/db/base/__init__.py

@@ -0,0 +1,5 @@
+#
+# OtterTune - __init__.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#

server/website/website/db/base/parser.py

@@ -1,5 +1,5 @@
 #
-# OtterTune - base.py
+# OtterTune - parser.py
 #
 # Copyright (c) 2017-18, Carnegie Mellon University Database Group
 #
@@ -213,8 +213,7 @@ class BaseParser:
 
         return metric_data
 
-    @staticmethod
+    def extract_valid_variables(self, variables, catalog, default_value=None):
-    def extract_valid_variables(variables, catalog, default_value=None):
         valid_variables = {}
         diff_log = []
         valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())}
@@ -280,8 +279,7 @@ class BaseParser:
             assert len(valid_knobs[k]) == 1
             valid_knobs[k] = valid_knobs[k][0]
         # Extract all valid knobs
-        return BaseParser.extract_valid_variables(
+        return self.extract_valid_variables(valid_knobs, self.knob_catalog_)
-            valid_knobs, self.knob_catalog_)
 
     def parse_dbms_metrics(self, metrics):
         # Some DBMSs measure different types of stats (e.g., global, local)
@@ -290,7 +288,7 @@ class BaseParser:
         valid_metrics = self.parse_dbms_variables(metrics)
 
         # Extract all valid metrics
-        valid_metrics, diffs = BaseParser.extract_valid_variables(
+        valid_metrics, diffs = self.extract_valid_variables(
             valid_metrics, self.metric_catalog_, default_value='0')
 
         # Combine values

server/website/website/db/myrocks/__init__.py

@@ -0,0 +1,5 @@
+#
+# OtterTune - __init__.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#

server/website/website/db/myrocks/parser.py

@@ -1,5 +1,5 @@
 #
-# OtterTune - myrocks.py
+# OtterTune - parser.py
 #
 # Copyright (c) 2017-18, Carnegie Mellon University Database Group
 #
@@ -7,7 +7,7 @@
 import re
 from collections import OrderedDict
 
-from .base import BaseParser
+from ..base.parser import BaseParser
 from website.types import MetricType, VarType
 
 
@@ -75,8 +75,7 @@ class MyRocksParser(BaseParser):
         else:
             raise Exception('Invalid variable full name: {}'.format(full_name))
 
-    @staticmethod
+    def extract_valid_variables(self, variables, catalog, default_value=None):
-    def extract_valid_variables(variables, catalog, default_value=None):
         valid_variables = {}
         diff_log = []
         valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())}
@@ -88,7 +87,7 @@ class MyRocksParser(BaseParser):
         # are also logged as 'miscapitalized'.
         for var_name, var_value in list(variables.items()):
             lc_var_name = var_name.lower()
-            prt_name = MyRocksParser.partial_name(lc_var_name)
+            prt_name = self.partial_name(lc_var_name)
             if prt_name in valid_lc_variables:
                 valid_name = valid_lc_variables[prt_name].name
                 if prt_name != valid_name:
@@ -102,7 +101,7 @@ class MyRocksParser(BaseParser):
         # the given default_value if provided (or the item's actual default value
         # if not) and logged as 'missing'. For now missing local variables are
         # not added to valid_variables
-        lc_variables = {MyRocksParser.partial_name(k.lower()): v
+        lc_variables = {self.partial_name(k.lower()): v
                         for k, v in list(variables.items())}
         for valid_lc_name, metadata in list(valid_lc_variables.items()):
             if valid_lc_name not in lc_variables:
@@ -116,7 +115,7 @@ class MyRocksParser(BaseParser):
         adjusted_metrics = {}
         for met_name, start_val in list(metrics_start.items()):
             end_val = metrics_end[met_name]
-            met_info = self.metric_catalog_[MyRocksParser.partial_name(met_name)]
+            met_info = self.metric_catalog_[self.partial_name(met_name)]
             if met_info.vartype == VarType.INTEGER or \
                     met_info.vartype == VarType.REAL:
                 conversion_fn = self.convert_integer if \
@@ -136,20 +135,20 @@ class MyRocksParser(BaseParser):
     def parse_dbms_knobs(self, knobs):
         valid_knobs = self.parse_dbms_variables(knobs)
         # Extract all valid knobs
-        return MyRocksParser.extract_valid_variables(
+        return self.extract_valid_variables(
             valid_knobs, self.knob_catalog_)
 
     def parse_dbms_metrics(self, metrics):
         valid_metrics = self.parse_dbms_variables(metrics)
         # Extract all valid metrics
-        valid_metrics, diffs = MyRocksParser.extract_valid_variables(
+        valid_metrics, diffs = self.extract_valid_variables(
             valid_metrics, self.metric_catalog_, default_value='0')
         return valid_metrics, diffs
 
     def convert_dbms_metrics(self, metrics, observation_time, target_objective=None):
         metric_data = {}
         for name, value in list(metrics.items()):
-            prt_name = MyRocksParser.partial_name(name)
+            prt_name = self.partial_name(name)
             if prt_name in self.numeric_metric_catalog_:
                 metadata = self.numeric_metric_catalog_[prt_name]
                 if metadata.metric_type == MetricType.COUNTER:
@@ -173,7 +172,7 @@ class MyRocksParser(BaseParser):
     def convert_dbms_knobs(self, knobs):
         knob_data = {}
         for name, value in list(knobs.items()):
-            prt_name = MyRocksParser.partial_name(name)
+            prt_name = self.partial_name(name)
             if prt_name in self.tunable_knob_catalog_:
                 metadata = self.tunable_knob_catalog_[prt_name]
                 assert(metadata.tunable)
@@ -202,8 +201,8 @@ class MyRocksParser(BaseParser):
 
     def filter_numeric_metrics(self, metrics):
         return OrderedDict([(k, v) for k, v in list(metrics.items()) if
-                            MyRocksParser.partial_name(k) in self.numeric_metric_catalog_])
+                            self.partial_name(k) in self.numeric_metric_catalog_])
 
     def filter_tunable_knobs(self, knobs):
         return OrderedDict([(k, v) for k, v in list(knobs.items()) if
-                            MyRocksParser.partial_name(k) in self.tunable_knob_catalog_])
+                            self.partial_name(k) in self.tunable_knob_catalog_])

server/website/website/db/oracle/__init__.py

@@ -0,0 +1,5 @@
+#
+# OtterTune - __init__.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#

server/website/website/db/oracle/parser.py

@@ -1,10 +1,10 @@
 #
-# OtterTune - oracle.py
+# OtterTune - parser.py
 #
 # Copyright (c) 2017-18, Carnegie Mellon University Database Group
 #
 
-from .base import BaseParser
+from ..base.parser import BaseParser
 
 
 class OracleParser(BaseParser):

server/website/website/db/parser.py

@@ -0,0 +1,86 @@
+#
+# OtterTune - parser.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+
+from website.models import DBMSCatalog
+from website.types import DBMSType
+
+from .myrocks.parser import MyRocksParser
+from .postgres.parser import PostgresParser
+from .oracle.parser import OracleParser
+
+_DBMS_PARSERS = {}
+
+
+def _get(dbms_id):
+    dbms_id = int(dbms_id)
+    db_parser = _DBMS_PARSERS.get(dbms_id, None)
+    if db_parser is None:
+        obj = DBMSCatalog.objects.get(id=dbms_id)
+        if obj.type == DBMSType.POSTGRES:
+            clz = PostgresParser
+        elif obj.type == DBMSType.MYROCKS:
+            clz = MyRocksParser
+        elif obj.type == DBMSType.ORACLE:
+            clz = OracleParser
+        else:
+            raise NotImplementedError('Implement me! {}'.format(obj))
+
+        db_parser = clz(obj)
+        _DBMS_PARSERS[dbms_id] = db_parser
+
+    return db_parser
+
+
+def parse_version_string(dbms_type, version_string):
+    dbmss = DBMSCatalog.objects.filter(type=dbms_type)
+    parsed_version = None
+    for instance in dbmss:
+        db_parser = _get(instance.pk)
+        try:
+            parsed_version = db_parser.parse_version_string(version_string)
+        except AttributeError:
+            pass
+        if parsed_version is not None:
+            break
+    return parsed_version
+
+
+def convert_dbms_knobs(dbms_id, knobs):
+    return _get(dbms_id).convert_dbms_knobs(knobs)
+
+
+def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective=None):
+    return _get(dbms_id).convert_dbms_metrics(
+        numeric_metrics, observation_time, target_objective)
+
+
+def parse_dbms_knobs(dbms_id, knobs):
+    return _get(dbms_id).parse_dbms_knobs(knobs)
+
+
+def parse_dbms_metrics(dbms_id, metrics):
+    return _get(dbms_id).parse_dbms_metrics(metrics)
+
+
+def create_knob_configuration(dbms_id, tuning_knobs):
+    return _get(dbms_id).create_knob_configuration(tuning_knobs)
+
+
+def format_dbms_knobs(dbms_id, knobs):
+    return _get(dbms_id).format_dbms_knobs(knobs)
+
+
+def filter_numeric_metrics(dbms_id, metrics):
+    return _get(dbms_id).filter_numeric_metrics(metrics)
+
+
+def filter_tunable_knobs(dbms_id, knobs):
+    return _get(dbms_id).filter_tunable_knobs(knobs)
+
+
+def calculate_change_in_metrics(dbms_id, metrics_start, metrics_end):
+    return _get(dbms_id).calculate_change_in_metrics(
+        metrics_start, metrics_end)

server/website/website/db/postgres/__init__.py

@@ -0,0 +1,5 @@
+#
+# OtterTune - __init__.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#

server/website/website/db/postgres/parser.py

@@ -1,12 +1,12 @@
 #
-# OtterTune - postgres.py
+# OtterTune - parser.py
 #
 # Copyright (c) 2017-18, Carnegie Mellon University Database Group
 #
 
 import re
 
-from .base import BaseParser
+from ..base.parser import BaseParser
 from website.utils import ConversionUtil
 
 

server/website/website/parser/parser.py

@@ -1,97 +0,0 @@
-#
-# OtterTune - parser.py
-#
-# Copyright (c) 2017-18, Carnegie Mellon University Database Group
-#
-
-from website.models import DBMSCatalog
-from website.types import DBMSType
-
-from .myrocks import MyRocksParser
-from .postgres import PostgresParser
-from .oracle import OracleParser
-
-
-class Parser():
-
-    __DBMS_UTILS_IMPLS = None
-
-    @staticmethod
-    def __utils(dbms_id=None):
-        if Parser.__DBMS_UTILS_IMPLS is None:
-
-            parsers = {}
-            for obj in DBMSCatalog.objects.all():
-                if obj.type == DBMSType.POSTGRES:
-                    clz = PostgresParser
-                elif obj.type == DBMSType.MYROCKS:
-                    clz = MyRocksParser
-                elif obj.type == DBMSType.ORACLE:
-                    clz = OracleParser
-                else:
-                    raise NotImplementedError('Implement me! {}'.format(obj))
-
-                parsers[obj.pk] = clz(obj)
-
-            Parser.__DBMS_UTILS_IMPLS = parsers
-
-        try:
-            if dbms_id is None:
-                return Parser.__DBMS_UTILS_IMPLS
-            return Parser.__DBMS_UTILS_IMPLS[dbms_id]
-        except KeyError:
-            raise NotImplementedError(
-                'Implement me! ({})'.format(dbms_id))
-
-    @staticmethod
-    def parse_version_string(dbms_type, version_string):
-        for k, v in list(Parser.__utils(dbms_type).items()):
-            dbms = DBMSCatalog.objects.get(pk=k)
-            if dbms.type == dbms_type:
-                try:
-                    return v.parse_version_string(version_string)
-                except AttributeError:
-                    pass
-        return None
-
-    @staticmethod
-    def convert_dbms_knobs(dbms_id, knobs):
-        return Parser.__utils(dbms_id).convert_dbms_knobs(knobs)
-
-    @staticmethod
-    def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective=None):
-        return Parser.__utils(dbms_id).convert_dbms_metrics(
-            numeric_metrics, observation_time, target_objective)
-
-    @staticmethod
-    def parse_dbms_knobs(dbms_id, knobs):
-        return Parser.__utils(dbms_id).parse_dbms_knobs(knobs)
-
-    @staticmethod
-    def parse_dbms_metrics(dbms_id, metrics):
-        return Parser.__utils(dbms_id).parse_dbms_metrics(metrics)
-
-    @staticmethod
-    def get_nondefault_knob_settings(dbms_id, knobs):
-        return Parser.__utils(dbms_id).get_nondefault_knob_settings(knobs)
-
-    @staticmethod
-    def create_knob_configuration(dbms_id, tuning_knobs):
-        return Parser.__utils(dbms_id).create_knob_configuration(tuning_knobs)
-
-    @staticmethod
-    def format_dbms_knobs(dbms_id, knobs):
-        return Parser.__utils(dbms_id).format_dbms_knobs(knobs)
-
-    @staticmethod
-    def filter_numeric_metrics(dbms_id, metrics):
-        return Parser.__utils(dbms_id).filter_numeric_metrics(metrics)
-
-    @staticmethod
-    def filter_tunable_knobs(dbms_id, knobs):
-        return Parser.__utils(dbms_id).filter_tunable_knobs(knobs)
-
-    @staticmethod
-    def calculate_change_in_metrics(dbms_id, metrics_start, metrics_end):
-        return Parser.__utils(dbms_id).calculate_change_in_metrics(
-            metrics_start, metrics_end)

server/website/website/tasks/async_tasks.py

@@ -20,7 +20,7 @@ from analysis.preprocessing import Bin, DummyEncoder
 from analysis.constraints import ParamConstraintHelper
 from website.models import (PipelineData, PipelineRun, Result, Workload, KnobCatalog,
                             MetricCatalog, SessionKnob)
-from website.parser import Parser
+from website.db import parser
 from website.types import PipelineTaskType, AlgorithmType
 from website.utils import DataUtil, JSONUtil
 from website.settings import IMPORTANT_KNOB_NUMBER, NUM_SAMPLES, TOP_NUM_CONFIG  # pylint: disable=no-name-in-module
@@ -102,14 +102,14 @@ class ConfigurationRecommendation(UpdateTask):  # pylint: disable=abstract-metho
         result = Result.objects.get(pk=result_id)
 
         # Replace result with formatted result
-        formatted_params = Parser.format_dbms_knobs(result.dbms.pk, retval['recommendation'])
+        formatted_params = parser.format_dbms_knobs(result.dbms.pk, retval['recommendation'])
         task_meta = TaskMeta.objects.get(task_id=task_id)
         retval['recommendation'] = formatted_params
         task_meta.result = retval
         task_meta.save()
 
         # Create next configuration to try
-        config = Parser.create_knob_configuration(result.dbms.pk, retval['recommendation'])
+        config = parser.create_knob_configuration(result.dbms.pk, retval['recommendation'])
         retval['recommendation'] = config
         result.next_configuration = JSONUtil.dumps(retval)
         result.save()

server/website/website/views.py

@@ -27,11 +27,11 @@ from django.views.decorators.csrf import csrf_exempt
 from django.forms.models import model_to_dict
 from pytz import timezone
 
+from .db import parser
 from .forms import NewResultForm, ProjectForm, SessionForm, SessionKnobForm
 from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog,
                      MetricData, MetricManager, Project, Result, Session, Workload,
                      SessionKnob)
-from .parser import Parser
 from .tasks import (aggregate_target_results, map_workload, train_ddpg,
                     configuration_recommendation, configuration_recommendation_ddpg)
 from .types import (DBMSType, KnobUnitType, MetricType,
@@ -487,23 +487,23 @@ def handle_result_files(session, files):
                             '(actual=' + dbms.full_name + ')')
 
     # Load, process, and store the knobs in the DBMS's configuration
-    knob_dict, knob_diffs = Parser.parse_dbms_knobs(
+    knob_dict, knob_diffs = parser.parse_dbms_knobs(
         dbms.pk, JSONUtil.loads(files['knobs']))
-    tunable_knob_dict = Parser.convert_dbms_knobs(
+    tunable_knob_dict = parser.convert_dbms_knobs(
         dbms.pk, knob_dict)
     knob_data = KnobData.objects.create_knob_data(
         session, JSONUtil.dumps(knob_dict, pprint=True, sort=True),
         JSONUtil.dumps(tunable_knob_dict, pprint=True, sort=True), dbms)
 
     # Load, process, and store the runtime metrics exposed by the DBMS
-    initial_metric_dict, initial_metric_diffs = Parser.parse_dbms_metrics(
+    initial_metric_dict, initial_metric_diffs = parser.parse_dbms_metrics(
         dbms.pk, JSONUtil.loads(files['metrics_before']))
-    final_metric_dict, final_metric_diffs = Parser.parse_dbms_metrics(
+    final_metric_dict, final_metric_diffs = parser.parse_dbms_metrics(
         dbms.pk, JSONUtil.loads(files['metrics_after']))
-    metric_dict = Parser.calculate_change_in_metrics(
+    metric_dict = parser.calculate_change_in_metrics(
         dbms.pk, initial_metric_dict, final_metric_dict)
     initial_metric_diffs.extend(final_metric_diffs)
-    numeric_metric_dict = Parser.convert_dbms_metrics(
+    numeric_metric_dict = parser.convert_dbms_metrics(
         dbms.pk, metric_dict, observation_time, session.target_objective)
     metric_data = MetricData.objects.create_metric_data(
         session, JSONUtil.dumps(metric_dict, pprint=True, sort=True),
@@ -658,11 +658,11 @@ def metric_data_view(request, project_id, session_id, data_id):  # pylint: disab
 def dbms_data_view(request, context, dbms_data):
     if context['data_type'] == 'knobs':
         model_class = KnobData
-        filter_fn = Parser.filter_tunable_knobs
+        filter_fn = parser.filter_tunable_knobs
         obj_data = dbms_data.knobs
     else:
         model_class = MetricData
-        filter_fn = Parser.filter_numeric_metrics
+        filter_fn = parser.filter_numeric_metrics
         obj_data = dbms_data.metrics
 
     dbms_id = dbms_data.dbms.pk