From f68c23e975d04152f0890045ddd27ab701b5b4e9 Mon Sep 17 00:00:00 2001 From: Dana Van Aken Date: Tue, 8 Oct 2019 11:24:04 -0400 Subject: [PATCH] Added outer db module and moved parser code into it --- server/website/tests/test_parser.py | 5 +- .../website/{parser => db}/__init__.py | 1 - server/website/website/db/base/__init__.py | 5 + .../{parser/base.py => db/base/parser.py} | 10 +- server/website/website/db/myrocks/__init__.py | 5 + .../myrocks.py => db/myrocks/parser.py} | 25 +++-- server/website/website/db/oracle/__init__.py | 5 + .../{parser/oracle.py => db/oracle/parser.py} | 4 +- server/website/website/db/parser.py | 86 ++++++++++++++++ .../website/website/db/postgres/__init__.py | 5 + .../postgres.py => db/postgres/parser.py} | 4 +- server/website/website/parser/parser.py | 97 ------------------- server/website/website/tasks/async_tasks.py | 6 +- server/website/website/views.py | 18 ++-- 14 files changed, 141 insertions(+), 135 deletions(-) rename server/website/website/{parser => db}/__init__.py (78%) create mode 100644 server/website/website/db/base/__init__.py rename server/website/website/{parser/base.py => db/base/parser.py} (98%) create mode 100644 server/website/website/db/myrocks/__init__.py rename server/website/website/{parser/myrocks.py => db/myrocks/parser.py} (92%) create mode 100644 server/website/website/db/oracle/__init__.py rename server/website/website/{parser/oracle.py => db/oracle/parser.py} (90%) create mode 100644 server/website/website/db/parser.py create mode 100644 server/website/website/db/postgres/__init__.py rename server/website/website/{parser/postgres.py => db/postgres/parser.py} (93%) delete mode 100644 server/website/website/parser/parser.py diff --git a/server/website/tests/test_parser.py b/server/website/tests/test_parser.py index 4aa2fb1..5caaf6f 100644 --- a/server/website/tests/test_parser.py +++ b/server/website/tests/test_parser.py @@ -7,7 +7,8 @@ from abc import ABCMeta, abstractmethod import mock from django.test import TestCase -from website.parser.postgres import PostgresParser +from website.db import parser +# from website.db.parser.postgres import PostgresParser from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType from website.models import DBMSCatalog, KnobCatalog @@ -221,7 +222,7 @@ class PostgresParserTests(BaseParserTests, TestCase): def setUp(self): dbms_obj = DBMSCatalog.objects.filter( type=DBMSType.POSTGRES, version="9.6").first() - self.test_dbms = PostgresParser(dbms_obj) + self.test_dbms = parser._get(dbms_obj.pk) # pylint: disable=protected-access def test_convert_dbms_knobs(self): super().test_convert_dbms_knobs() diff --git a/server/website/website/parser/__init__.py b/server/website/website/db/__init__.py similarity index 78% rename from server/website/website/parser/__init__.py rename to server/website/website/db/__init__.py index b87acd5..4e851cb 100644 --- a/server/website/website/parser/__init__.py +++ b/server/website/website/db/__init__.py @@ -3,4 +3,3 @@ # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # -from .parser import Parser diff --git a/server/website/website/db/base/__init__.py b/server/website/website/db/base/__init__.py new file mode 100644 index 0000000..4e851cb --- /dev/null +++ b/server/website/website/db/base/__init__.py @@ -0,0 +1,5 @@ +# +# OtterTune - __init__.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# diff --git a/server/website/website/parser/base.py b/server/website/website/db/base/parser.py similarity index 98% rename from server/website/website/parser/base.py rename to server/website/website/db/base/parser.py index 08ad7c9..cf990be 100644 --- a/server/website/website/parser/base.py +++ b/server/website/website/db/base/parser.py @@ -1,5 +1,5 @@ # -# OtterTune - base.py +# OtterTune - parser.py # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # @@ -213,8 +213,7 @@ class BaseParser: return metric_data - @staticmethod - def extract_valid_variables(variables, catalog, default_value=None): + def extract_valid_variables(self, variables, catalog, default_value=None): valid_variables = {} diff_log = [] valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())} @@ -280,8 +279,7 @@ class BaseParser: assert len(valid_knobs[k]) == 1 valid_knobs[k] = valid_knobs[k][0] # Extract all valid knobs - return BaseParser.extract_valid_variables( - valid_knobs, self.knob_catalog_) + return self.extract_valid_variables(valid_knobs, self.knob_catalog_) def parse_dbms_metrics(self, metrics): # Some DBMSs measure different types of stats (e.g., global, local) @@ -290,7 +288,7 @@ class BaseParser: valid_metrics = self.parse_dbms_variables(metrics) # Extract all valid metrics - valid_metrics, diffs = BaseParser.extract_valid_variables( + valid_metrics, diffs = self.extract_valid_variables( valid_metrics, self.metric_catalog_, default_value='0') # Combine values diff --git a/server/website/website/db/myrocks/__init__.py b/server/website/website/db/myrocks/__init__.py new file mode 100644 index 0000000..4e851cb --- /dev/null +++ b/server/website/website/db/myrocks/__init__.py @@ -0,0 +1,5 @@ +# +# OtterTune - __init__.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# diff --git a/server/website/website/parser/myrocks.py b/server/website/website/db/myrocks/parser.py similarity index 92% rename from server/website/website/parser/myrocks.py rename to server/website/website/db/myrocks/parser.py index 9e8877f..8eea22b 100644 --- a/server/website/website/parser/myrocks.py +++ b/server/website/website/db/myrocks/parser.py @@ -1,5 +1,5 @@ # -# OtterTune - myrocks.py +# OtterTune - parser.py # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # @@ -7,7 +7,7 @@ import re from collections import OrderedDict -from .base import BaseParser +from ..base.parser import BaseParser from website.types import MetricType, VarType @@ -75,8 +75,7 @@ class MyRocksParser(BaseParser): else: raise Exception('Invalid variable full name: {}'.format(full_name)) - @staticmethod - def extract_valid_variables(variables, catalog, default_value=None): + def extract_valid_variables(self, variables, catalog, default_value=None): valid_variables = {} diff_log = [] valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())} @@ -88,7 +87,7 @@ class MyRocksParser(BaseParser): # are also logged as 'miscapitalized'. for var_name, var_value in list(variables.items()): lc_var_name = var_name.lower() - prt_name = MyRocksParser.partial_name(lc_var_name) + prt_name = self.partial_name(lc_var_name) if prt_name in valid_lc_variables: valid_name = valid_lc_variables[prt_name].name if prt_name != valid_name: @@ -102,7 +101,7 @@ class MyRocksParser(BaseParser): # the given default_value if provided (or the item's actual default value # if not) and logged as 'missing'. For now missing local variables are # not added to valid_variables - lc_variables = {MyRocksParser.partial_name(k.lower()): v + lc_variables = {self.partial_name(k.lower()): v for k, v in list(variables.items())} for valid_lc_name, metadata in list(valid_lc_variables.items()): if valid_lc_name not in lc_variables: @@ -116,7 +115,7 @@ class MyRocksParser(BaseParser): adjusted_metrics = {} for met_name, start_val in list(metrics_start.items()): end_val = metrics_end[met_name] - met_info = self.metric_catalog_[MyRocksParser.partial_name(met_name)] + met_info = self.metric_catalog_[self.partial_name(met_name)] if met_info.vartype == VarType.INTEGER or \ met_info.vartype == VarType.REAL: conversion_fn = self.convert_integer if \ @@ -136,20 +135,20 @@ class MyRocksParser(BaseParser): def parse_dbms_knobs(self, knobs): valid_knobs = self.parse_dbms_variables(knobs) # Extract all valid knobs - return MyRocksParser.extract_valid_variables( + return self.extract_valid_variables( valid_knobs, self.knob_catalog_) def parse_dbms_metrics(self, metrics): valid_metrics = self.parse_dbms_variables(metrics) # Extract all valid metrics - valid_metrics, diffs = MyRocksParser.extract_valid_variables( + valid_metrics, diffs = self.extract_valid_variables( valid_metrics, self.metric_catalog_, default_value='0') return valid_metrics, diffs def convert_dbms_metrics(self, metrics, observation_time, target_objective=None): metric_data = {} for name, value in list(metrics.items()): - prt_name = MyRocksParser.partial_name(name) + prt_name = self.partial_name(name) if prt_name in self.numeric_metric_catalog_: metadata = self.numeric_metric_catalog_[prt_name] if metadata.metric_type == MetricType.COUNTER: @@ -173,7 +172,7 @@ class MyRocksParser(BaseParser): def convert_dbms_knobs(self, knobs): knob_data = {} for name, value in list(knobs.items()): - prt_name = MyRocksParser.partial_name(name) + prt_name = self.partial_name(name) if prt_name in self.tunable_knob_catalog_: metadata = self.tunable_knob_catalog_[prt_name] assert(metadata.tunable) @@ -202,8 +201,8 @@ class MyRocksParser(BaseParser): def filter_numeric_metrics(self, metrics): return OrderedDict([(k, v) for k, v in list(metrics.items()) if - MyRocksParser.partial_name(k) in self.numeric_metric_catalog_]) + self.partial_name(k) in self.numeric_metric_catalog_]) def filter_tunable_knobs(self, knobs): return OrderedDict([(k, v) for k, v in list(knobs.items()) if - MyRocksParser.partial_name(k) in self.tunable_knob_catalog_]) + self.partial_name(k) in self.tunable_knob_catalog_]) diff --git a/server/website/website/db/oracle/__init__.py b/server/website/website/db/oracle/__init__.py new file mode 100644 index 0000000..4e851cb --- /dev/null +++ b/server/website/website/db/oracle/__init__.py @@ -0,0 +1,5 @@ +# +# OtterTune - __init__.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# diff --git a/server/website/website/parser/oracle.py b/server/website/website/db/oracle/parser.py similarity index 90% rename from server/website/website/parser/oracle.py rename to server/website/website/db/oracle/parser.py index 0957060..04024a0 100644 --- a/server/website/website/parser/oracle.py +++ b/server/website/website/db/oracle/parser.py @@ -1,10 +1,10 @@ # -# OtterTune - oracle.py +# OtterTune - parser.py # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # -from .base import BaseParser +from ..base.parser import BaseParser class OracleParser(BaseParser): diff --git a/server/website/website/db/parser.py b/server/website/website/db/parser.py new file mode 100644 index 0000000..e6475f1 --- /dev/null +++ b/server/website/website/db/parser.py @@ -0,0 +1,86 @@ +# +# OtterTune - parser.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# + +from website.models import DBMSCatalog +from website.types import DBMSType + +from .myrocks.parser import MyRocksParser +from .postgres.parser import PostgresParser +from .oracle.parser import OracleParser + +_DBMS_PARSERS = {} + + +def _get(dbms_id): + dbms_id = int(dbms_id) + db_parser = _DBMS_PARSERS.get(dbms_id, None) + if db_parser is None: + obj = DBMSCatalog.objects.get(id=dbms_id) + if obj.type == DBMSType.POSTGRES: + clz = PostgresParser + elif obj.type == DBMSType.MYROCKS: + clz = MyRocksParser + elif obj.type == DBMSType.ORACLE: + clz = OracleParser + else: + raise NotImplementedError('Implement me! {}'.format(obj)) + + db_parser = clz(obj) + _DBMS_PARSERS[dbms_id] = db_parser + + return db_parser + + +def parse_version_string(dbms_type, version_string): + dbmss = DBMSCatalog.objects.filter(type=dbms_type) + parsed_version = None + for instance in dbmss: + db_parser = _get(instance.pk) + try: + parsed_version = db_parser.parse_version_string(version_string) + except AttributeError: + pass + if parsed_version is not None: + break + return parsed_version + + +def convert_dbms_knobs(dbms_id, knobs): + return _get(dbms_id).convert_dbms_knobs(knobs) + + +def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective=None): + return _get(dbms_id).convert_dbms_metrics( + numeric_metrics, observation_time, target_objective) + + +def parse_dbms_knobs(dbms_id, knobs): + return _get(dbms_id).parse_dbms_knobs(knobs) + + +def parse_dbms_metrics(dbms_id, metrics): + return _get(dbms_id).parse_dbms_metrics(metrics) + + +def create_knob_configuration(dbms_id, tuning_knobs): + return _get(dbms_id).create_knob_configuration(tuning_knobs) + + +def format_dbms_knobs(dbms_id, knobs): + return _get(dbms_id).format_dbms_knobs(knobs) + + +def filter_numeric_metrics(dbms_id, metrics): + return _get(dbms_id).filter_numeric_metrics(metrics) + + +def filter_tunable_knobs(dbms_id, knobs): + return _get(dbms_id).filter_tunable_knobs(knobs) + + +def calculate_change_in_metrics(dbms_id, metrics_start, metrics_end): + return _get(dbms_id).calculate_change_in_metrics( + metrics_start, metrics_end) diff --git a/server/website/website/db/postgres/__init__.py b/server/website/website/db/postgres/__init__.py new file mode 100644 index 0000000..4e851cb --- /dev/null +++ b/server/website/website/db/postgres/__init__.py @@ -0,0 +1,5 @@ +# +# OtterTune - __init__.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# diff --git a/server/website/website/parser/postgres.py b/server/website/website/db/postgres/parser.py similarity index 93% rename from server/website/website/parser/postgres.py rename to server/website/website/db/postgres/parser.py index 2748b13..e0f378f 100644 --- a/server/website/website/parser/postgres.py +++ b/server/website/website/db/postgres/parser.py @@ -1,12 +1,12 @@ # -# OtterTune - postgres.py +# OtterTune - parser.py # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # import re -from .base import BaseParser +from ..base.parser import BaseParser from website.utils import ConversionUtil diff --git a/server/website/website/parser/parser.py b/server/website/website/parser/parser.py deleted file mode 100644 index 200a6de..0000000 --- a/server/website/website/parser/parser.py +++ /dev/null @@ -1,97 +0,0 @@ -# -# OtterTune - parser.py -# -# Copyright (c) 2017-18, Carnegie Mellon University Database Group -# - -from website.models import DBMSCatalog -from website.types import DBMSType - -from .myrocks import MyRocksParser -from .postgres import PostgresParser -from .oracle import OracleParser - - -class Parser(): - - __DBMS_UTILS_IMPLS = None - - @staticmethod - def __utils(dbms_id=None): - if Parser.__DBMS_UTILS_IMPLS is None: - - parsers = {} - for obj in DBMSCatalog.objects.all(): - if obj.type == DBMSType.POSTGRES: - clz = PostgresParser - elif obj.type == DBMSType.MYROCKS: - clz = MyRocksParser - elif obj.type == DBMSType.ORACLE: - clz = OracleParser - else: - raise NotImplementedError('Implement me! {}'.format(obj)) - - parsers[obj.pk] = clz(obj) - - Parser.__DBMS_UTILS_IMPLS = parsers - - try: - if dbms_id is None: - return Parser.__DBMS_UTILS_IMPLS - return Parser.__DBMS_UTILS_IMPLS[dbms_id] - except KeyError: - raise NotImplementedError( - 'Implement me! ({})'.format(dbms_id)) - - @staticmethod - def parse_version_string(dbms_type, version_string): - for k, v in list(Parser.__utils(dbms_type).items()): - dbms = DBMSCatalog.objects.get(pk=k) - if dbms.type == dbms_type: - try: - return v.parse_version_string(version_string) - except AttributeError: - pass - return None - - @staticmethod - def convert_dbms_knobs(dbms_id, knobs): - return Parser.__utils(dbms_id).convert_dbms_knobs(knobs) - - @staticmethod - def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective=None): - return Parser.__utils(dbms_id).convert_dbms_metrics( - numeric_metrics, observation_time, target_objective) - - @staticmethod - def parse_dbms_knobs(dbms_id, knobs): - return Parser.__utils(dbms_id).parse_dbms_knobs(knobs) - - @staticmethod - def parse_dbms_metrics(dbms_id, metrics): - return Parser.__utils(dbms_id).parse_dbms_metrics(metrics) - - @staticmethod - def get_nondefault_knob_settings(dbms_id, knobs): - return Parser.__utils(dbms_id).get_nondefault_knob_settings(knobs) - - @staticmethod - def create_knob_configuration(dbms_id, tuning_knobs): - return Parser.__utils(dbms_id).create_knob_configuration(tuning_knobs) - - @staticmethod - def format_dbms_knobs(dbms_id, knobs): - return Parser.__utils(dbms_id).format_dbms_knobs(knobs) - - @staticmethod - def filter_numeric_metrics(dbms_id, metrics): - return Parser.__utils(dbms_id).filter_numeric_metrics(metrics) - - @staticmethod - def filter_tunable_knobs(dbms_id, knobs): - return Parser.__utils(dbms_id).filter_tunable_knobs(knobs) - - @staticmethod - def calculate_change_in_metrics(dbms_id, metrics_start, metrics_end): - return Parser.__utils(dbms_id).calculate_change_in_metrics( - metrics_start, metrics_end) diff --git a/server/website/website/tasks/async_tasks.py b/server/website/website/tasks/async_tasks.py index b3d8c3d..12ba472 100644 --- a/server/website/website/tasks/async_tasks.py +++ b/server/website/website/tasks/async_tasks.py @@ -20,7 +20,7 @@ from analysis.preprocessing import Bin, DummyEncoder from analysis.constraints import ParamConstraintHelper from website.models import (PipelineData, PipelineRun, Result, Workload, KnobCatalog, MetricCatalog, SessionKnob) -from website.parser import Parser +from website.db import parser from website.types import PipelineTaskType, AlgorithmType from website.utils import DataUtil, JSONUtil from website.settings import IMPORTANT_KNOB_NUMBER, NUM_SAMPLES, TOP_NUM_CONFIG # pylint: disable=no-name-in-module @@ -102,14 +102,14 @@ class ConfigurationRecommendation(UpdateTask): # pylint: disable=abstract-metho result = Result.objects.get(pk=result_id) # Replace result with formatted result - formatted_params = Parser.format_dbms_knobs(result.dbms.pk, retval['recommendation']) + formatted_params = parser.format_dbms_knobs(result.dbms.pk, retval['recommendation']) task_meta = TaskMeta.objects.get(task_id=task_id) retval['recommendation'] = formatted_params task_meta.result = retval task_meta.save() # Create next configuration to try - config = Parser.create_knob_configuration(result.dbms.pk, retval['recommendation']) + config = parser.create_knob_configuration(result.dbms.pk, retval['recommendation']) retval['recommendation'] = config result.next_configuration = JSONUtil.dumps(retval) result.save() diff --git a/server/website/website/views.py b/server/website/website/views.py index 7a62f8e..1827fac 100644 --- a/server/website/website/views.py +++ b/server/website/website/views.py @@ -27,11 +27,11 @@ from django.views.decorators.csrf import csrf_exempt from django.forms.models import model_to_dict from pytz import timezone +from .db import parser from .forms import NewResultForm, ProjectForm, SessionForm, SessionKnobForm from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog, MetricData, MetricManager, Project, Result, Session, Workload, SessionKnob) -from .parser import Parser from .tasks import (aggregate_target_results, map_workload, train_ddpg, configuration_recommendation, configuration_recommendation_ddpg) from .types import (DBMSType, KnobUnitType, MetricType, @@ -487,23 +487,23 @@ def handle_result_files(session, files): '(actual=' + dbms.full_name + ')') # Load, process, and store the knobs in the DBMS's configuration - knob_dict, knob_diffs = Parser.parse_dbms_knobs( + knob_dict, knob_diffs = parser.parse_dbms_knobs( dbms.pk, JSONUtil.loads(files['knobs'])) - tunable_knob_dict = Parser.convert_dbms_knobs( + tunable_knob_dict = parser.convert_dbms_knobs( dbms.pk, knob_dict) knob_data = KnobData.objects.create_knob_data( session, JSONUtil.dumps(knob_dict, pprint=True, sort=True), JSONUtil.dumps(tunable_knob_dict, pprint=True, sort=True), dbms) # Load, process, and store the runtime metrics exposed by the DBMS - initial_metric_dict, initial_metric_diffs = Parser.parse_dbms_metrics( + initial_metric_dict, initial_metric_diffs = parser.parse_dbms_metrics( dbms.pk, JSONUtil.loads(files['metrics_before'])) - final_metric_dict, final_metric_diffs = Parser.parse_dbms_metrics( + final_metric_dict, final_metric_diffs = parser.parse_dbms_metrics( dbms.pk, JSONUtil.loads(files['metrics_after'])) - metric_dict = Parser.calculate_change_in_metrics( + metric_dict = parser.calculate_change_in_metrics( dbms.pk, initial_metric_dict, final_metric_dict) initial_metric_diffs.extend(final_metric_diffs) - numeric_metric_dict = Parser.convert_dbms_metrics( + numeric_metric_dict = parser.convert_dbms_metrics( dbms.pk, metric_dict, observation_time, session.target_objective) metric_data = MetricData.objects.create_metric_data( session, JSONUtil.dumps(metric_dict, pprint=True, sort=True), @@ -658,11 +658,11 @@ def metric_data_view(request, project_id, session_id, data_id): # pylint: disab def dbms_data_view(request, context, dbms_data): if context['data_type'] == 'knobs': model_class = KnobData - filter_fn = Parser.filter_tunable_knobs + filter_fn = parser.filter_tunable_knobs obj_data = dbms_data.knobs else: model_class = MetricData - filter_fn = Parser.filter_numeric_metrics + filter_fn = parser.filter_numeric_metrics obj_data = dbms_data.metrics dbms_id = dbms_data.dbms.pk