Added outer db module and moved parser code into it

This commit is contained in:
Dana Van Aken 2019-10-08 11:24:04 -04:00
parent a3833d83b1
commit f68c23e975
14 changed files with 141 additions and 135 deletions

View File

@ -7,7 +7,8 @@
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
import mock import mock
from django.test import TestCase from django.test import TestCase
from website.parser.postgres import PostgresParser from website.db import parser
# from website.db.parser.postgres import PostgresParser
from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType
from website.models import DBMSCatalog, KnobCatalog from website.models import DBMSCatalog, KnobCatalog
@ -221,7 +222,7 @@ class PostgresParserTests(BaseParserTests, TestCase):
def setUp(self): def setUp(self):
dbms_obj = DBMSCatalog.objects.filter( dbms_obj = DBMSCatalog.objects.filter(
type=DBMSType.POSTGRES, version="9.6").first() type=DBMSType.POSTGRES, version="9.6").first()
self.test_dbms = PostgresParser(dbms_obj) self.test_dbms = parser._get(dbms_obj.pk) # pylint: disable=protected-access
def test_convert_dbms_knobs(self): def test_convert_dbms_knobs(self):
super().test_convert_dbms_knobs() super().test_convert_dbms_knobs()

View File

@ -3,4 +3,3 @@
# #
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
from .parser import Parser

View File

@ -0,0 +1,5 @@
#
# OtterTune - __init__.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#

View File

@ -1,5 +1,5 @@
# #
# OtterTune - base.py # OtterTune - parser.py
# #
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
@ -213,8 +213,7 @@ class BaseParser:
return metric_data return metric_data
@staticmethod def extract_valid_variables(self, variables, catalog, default_value=None):
def extract_valid_variables(variables, catalog, default_value=None):
valid_variables = {} valid_variables = {}
diff_log = [] diff_log = []
valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())} valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())}
@ -280,8 +279,7 @@ class BaseParser:
assert len(valid_knobs[k]) == 1 assert len(valid_knobs[k]) == 1
valid_knobs[k] = valid_knobs[k][0] valid_knobs[k] = valid_knobs[k][0]
# Extract all valid knobs # Extract all valid knobs
return BaseParser.extract_valid_variables( return self.extract_valid_variables(valid_knobs, self.knob_catalog_)
valid_knobs, self.knob_catalog_)
def parse_dbms_metrics(self, metrics): def parse_dbms_metrics(self, metrics):
# Some DBMSs measure different types of stats (e.g., global, local) # Some DBMSs measure different types of stats (e.g., global, local)
@ -290,7 +288,7 @@ class BaseParser:
valid_metrics = self.parse_dbms_variables(metrics) valid_metrics = self.parse_dbms_variables(metrics)
# Extract all valid metrics # Extract all valid metrics
valid_metrics, diffs = BaseParser.extract_valid_variables( valid_metrics, diffs = self.extract_valid_variables(
valid_metrics, self.metric_catalog_, default_value='0') valid_metrics, self.metric_catalog_, default_value='0')
# Combine values # Combine values

View File

@ -0,0 +1,5 @@
#
# OtterTune - __init__.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#

View File

@ -1,5 +1,5 @@
# #
# OtterTune - myrocks.py # OtterTune - parser.py
# #
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
@ -7,7 +7,7 @@
import re import re
from collections import OrderedDict from collections import OrderedDict
from .base import BaseParser from ..base.parser import BaseParser
from website.types import MetricType, VarType from website.types import MetricType, VarType
@ -75,8 +75,7 @@ class MyRocksParser(BaseParser):
else: else:
raise Exception('Invalid variable full name: {}'.format(full_name)) raise Exception('Invalid variable full name: {}'.format(full_name))
@staticmethod def extract_valid_variables(self, variables, catalog, default_value=None):
def extract_valid_variables(variables, catalog, default_value=None):
valid_variables = {} valid_variables = {}
diff_log = [] diff_log = []
valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())} valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())}
@ -88,7 +87,7 @@ class MyRocksParser(BaseParser):
# are also logged as 'miscapitalized'. # are also logged as 'miscapitalized'.
for var_name, var_value in list(variables.items()): for var_name, var_value in list(variables.items()):
lc_var_name = var_name.lower() lc_var_name = var_name.lower()
prt_name = MyRocksParser.partial_name(lc_var_name) prt_name = self.partial_name(lc_var_name)
if prt_name in valid_lc_variables: if prt_name in valid_lc_variables:
valid_name = valid_lc_variables[prt_name].name valid_name = valid_lc_variables[prt_name].name
if prt_name != valid_name: if prt_name != valid_name:
@ -102,7 +101,7 @@ class MyRocksParser(BaseParser):
# the given default_value if provided (or the item's actual default value # the given default_value if provided (or the item's actual default value
# if not) and logged as 'missing'. For now missing local variables are # if not) and logged as 'missing'. For now missing local variables are
# not added to valid_variables # not added to valid_variables
lc_variables = {MyRocksParser.partial_name(k.lower()): v lc_variables = {self.partial_name(k.lower()): v
for k, v in list(variables.items())} for k, v in list(variables.items())}
for valid_lc_name, metadata in list(valid_lc_variables.items()): for valid_lc_name, metadata in list(valid_lc_variables.items()):
if valid_lc_name not in lc_variables: if valid_lc_name not in lc_variables:
@ -116,7 +115,7 @@ class MyRocksParser(BaseParser):
adjusted_metrics = {} adjusted_metrics = {}
for met_name, start_val in list(metrics_start.items()): for met_name, start_val in list(metrics_start.items()):
end_val = metrics_end[met_name] end_val = metrics_end[met_name]
met_info = self.metric_catalog_[MyRocksParser.partial_name(met_name)] met_info = self.metric_catalog_[self.partial_name(met_name)]
if met_info.vartype == VarType.INTEGER or \ if met_info.vartype == VarType.INTEGER or \
met_info.vartype == VarType.REAL: met_info.vartype == VarType.REAL:
conversion_fn = self.convert_integer if \ conversion_fn = self.convert_integer if \
@ -136,20 +135,20 @@ class MyRocksParser(BaseParser):
def parse_dbms_knobs(self, knobs): def parse_dbms_knobs(self, knobs):
valid_knobs = self.parse_dbms_variables(knobs) valid_knobs = self.parse_dbms_variables(knobs)
# Extract all valid knobs # Extract all valid knobs
return MyRocksParser.extract_valid_variables( return self.extract_valid_variables(
valid_knobs, self.knob_catalog_) valid_knobs, self.knob_catalog_)
def parse_dbms_metrics(self, metrics): def parse_dbms_metrics(self, metrics):
valid_metrics = self.parse_dbms_variables(metrics) valid_metrics = self.parse_dbms_variables(metrics)
# Extract all valid metrics # Extract all valid metrics
valid_metrics, diffs = MyRocksParser.extract_valid_variables( valid_metrics, diffs = self.extract_valid_variables(
valid_metrics, self.metric_catalog_, default_value='0') valid_metrics, self.metric_catalog_, default_value='0')
return valid_metrics, diffs return valid_metrics, diffs
def convert_dbms_metrics(self, metrics, observation_time, target_objective=None): def convert_dbms_metrics(self, metrics, observation_time, target_objective=None):
metric_data = {} metric_data = {}
for name, value in list(metrics.items()): for name, value in list(metrics.items()):
prt_name = MyRocksParser.partial_name(name) prt_name = self.partial_name(name)
if prt_name in self.numeric_metric_catalog_: if prt_name in self.numeric_metric_catalog_:
metadata = self.numeric_metric_catalog_[prt_name] metadata = self.numeric_metric_catalog_[prt_name]
if metadata.metric_type == MetricType.COUNTER: if metadata.metric_type == MetricType.COUNTER:
@ -173,7 +172,7 @@ class MyRocksParser(BaseParser):
def convert_dbms_knobs(self, knobs): def convert_dbms_knobs(self, knobs):
knob_data = {} knob_data = {}
for name, value in list(knobs.items()): for name, value in list(knobs.items()):
prt_name = MyRocksParser.partial_name(name) prt_name = self.partial_name(name)
if prt_name in self.tunable_knob_catalog_: if prt_name in self.tunable_knob_catalog_:
metadata = self.tunable_knob_catalog_[prt_name] metadata = self.tunable_knob_catalog_[prt_name]
assert(metadata.tunable) assert(metadata.tunable)
@ -202,8 +201,8 @@ class MyRocksParser(BaseParser):
def filter_numeric_metrics(self, metrics): def filter_numeric_metrics(self, metrics):
return OrderedDict([(k, v) for k, v in list(metrics.items()) if return OrderedDict([(k, v) for k, v in list(metrics.items()) if
MyRocksParser.partial_name(k) in self.numeric_metric_catalog_]) self.partial_name(k) in self.numeric_metric_catalog_])
def filter_tunable_knobs(self, knobs): def filter_tunable_knobs(self, knobs):
return OrderedDict([(k, v) for k, v in list(knobs.items()) if return OrderedDict([(k, v) for k, v in list(knobs.items()) if
MyRocksParser.partial_name(k) in self.tunable_knob_catalog_]) self.partial_name(k) in self.tunable_knob_catalog_])

View File

@ -0,0 +1,5 @@
#
# OtterTune - __init__.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#

View File

@ -1,10 +1,10 @@
# #
# OtterTune - oracle.py # OtterTune - parser.py
# #
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
from .base import BaseParser from ..base.parser import BaseParser
class OracleParser(BaseParser): class OracleParser(BaseParser):

View File

@ -0,0 +1,86 @@
#
# OtterTune - parser.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from website.models import DBMSCatalog
from website.types import DBMSType
from .myrocks.parser import MyRocksParser
from .postgres.parser import PostgresParser
from .oracle.parser import OracleParser
_DBMS_PARSERS = {}
def _get(dbms_id):
dbms_id = int(dbms_id)
db_parser = _DBMS_PARSERS.get(dbms_id, None)
if db_parser is None:
obj = DBMSCatalog.objects.get(id=dbms_id)
if obj.type == DBMSType.POSTGRES:
clz = PostgresParser
elif obj.type == DBMSType.MYROCKS:
clz = MyRocksParser
elif obj.type == DBMSType.ORACLE:
clz = OracleParser
else:
raise NotImplementedError('Implement me! {}'.format(obj))
db_parser = clz(obj)
_DBMS_PARSERS[dbms_id] = db_parser
return db_parser
def parse_version_string(dbms_type, version_string):
dbmss = DBMSCatalog.objects.filter(type=dbms_type)
parsed_version = None
for instance in dbmss:
db_parser = _get(instance.pk)
try:
parsed_version = db_parser.parse_version_string(version_string)
except AttributeError:
pass
if parsed_version is not None:
break
return parsed_version
def convert_dbms_knobs(dbms_id, knobs):
return _get(dbms_id).convert_dbms_knobs(knobs)
def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective=None):
return _get(dbms_id).convert_dbms_metrics(
numeric_metrics, observation_time, target_objective)
def parse_dbms_knobs(dbms_id, knobs):
return _get(dbms_id).parse_dbms_knobs(knobs)
def parse_dbms_metrics(dbms_id, metrics):
return _get(dbms_id).parse_dbms_metrics(metrics)
def create_knob_configuration(dbms_id, tuning_knobs):
return _get(dbms_id).create_knob_configuration(tuning_knobs)
def format_dbms_knobs(dbms_id, knobs):
return _get(dbms_id).format_dbms_knobs(knobs)
def filter_numeric_metrics(dbms_id, metrics):
return _get(dbms_id).filter_numeric_metrics(metrics)
def filter_tunable_knobs(dbms_id, knobs):
return _get(dbms_id).filter_tunable_knobs(knobs)
def calculate_change_in_metrics(dbms_id, metrics_start, metrics_end):
return _get(dbms_id).calculate_change_in_metrics(
metrics_start, metrics_end)

View File

@ -0,0 +1,5 @@
#
# OtterTune - __init__.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#

View File

@ -1,12 +1,12 @@
# #
# OtterTune - postgres.py # OtterTune - parser.py
# #
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
import re import re
from .base import BaseParser from ..base.parser import BaseParser
from website.utils import ConversionUtil from website.utils import ConversionUtil

View File

@ -1,97 +0,0 @@
#
# OtterTune - parser.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from website.models import DBMSCatalog
from website.types import DBMSType
from .myrocks import MyRocksParser
from .postgres import PostgresParser
from .oracle import OracleParser
class Parser():
__DBMS_UTILS_IMPLS = None
@staticmethod
def __utils(dbms_id=None):
if Parser.__DBMS_UTILS_IMPLS is None:
parsers = {}
for obj in DBMSCatalog.objects.all():
if obj.type == DBMSType.POSTGRES:
clz = PostgresParser
elif obj.type == DBMSType.MYROCKS:
clz = MyRocksParser
elif obj.type == DBMSType.ORACLE:
clz = OracleParser
else:
raise NotImplementedError('Implement me! {}'.format(obj))
parsers[obj.pk] = clz(obj)
Parser.__DBMS_UTILS_IMPLS = parsers
try:
if dbms_id is None:
return Parser.__DBMS_UTILS_IMPLS
return Parser.__DBMS_UTILS_IMPLS[dbms_id]
except KeyError:
raise NotImplementedError(
'Implement me! ({})'.format(dbms_id))
@staticmethod
def parse_version_string(dbms_type, version_string):
for k, v in list(Parser.__utils(dbms_type).items()):
dbms = DBMSCatalog.objects.get(pk=k)
if dbms.type == dbms_type:
try:
return v.parse_version_string(version_string)
except AttributeError:
pass
return None
@staticmethod
def convert_dbms_knobs(dbms_id, knobs):
return Parser.__utils(dbms_id).convert_dbms_knobs(knobs)
@staticmethod
def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective=None):
return Parser.__utils(dbms_id).convert_dbms_metrics(
numeric_metrics, observation_time, target_objective)
@staticmethod
def parse_dbms_knobs(dbms_id, knobs):
return Parser.__utils(dbms_id).parse_dbms_knobs(knobs)
@staticmethod
def parse_dbms_metrics(dbms_id, metrics):
return Parser.__utils(dbms_id).parse_dbms_metrics(metrics)
@staticmethod
def get_nondefault_knob_settings(dbms_id, knobs):
return Parser.__utils(dbms_id).get_nondefault_knob_settings(knobs)
@staticmethod
def create_knob_configuration(dbms_id, tuning_knobs):
return Parser.__utils(dbms_id).create_knob_configuration(tuning_knobs)
@staticmethod
def format_dbms_knobs(dbms_id, knobs):
return Parser.__utils(dbms_id).format_dbms_knobs(knobs)
@staticmethod
def filter_numeric_metrics(dbms_id, metrics):
return Parser.__utils(dbms_id).filter_numeric_metrics(metrics)
@staticmethod
def filter_tunable_knobs(dbms_id, knobs):
return Parser.__utils(dbms_id).filter_tunable_knobs(knobs)
@staticmethod
def calculate_change_in_metrics(dbms_id, metrics_start, metrics_end):
return Parser.__utils(dbms_id).calculate_change_in_metrics(
metrics_start, metrics_end)

View File

@ -20,7 +20,7 @@ from analysis.preprocessing import Bin, DummyEncoder
from analysis.constraints import ParamConstraintHelper from analysis.constraints import ParamConstraintHelper
from website.models import (PipelineData, PipelineRun, Result, Workload, KnobCatalog, from website.models import (PipelineData, PipelineRun, Result, Workload, KnobCatalog,
MetricCatalog, SessionKnob) MetricCatalog, SessionKnob)
from website.parser import Parser from website.db import parser
from website.types import PipelineTaskType, AlgorithmType from website.types import PipelineTaskType, AlgorithmType
from website.utils import DataUtil, JSONUtil from website.utils import DataUtil, JSONUtil
from website.settings import IMPORTANT_KNOB_NUMBER, NUM_SAMPLES, TOP_NUM_CONFIG # pylint: disable=no-name-in-module from website.settings import IMPORTANT_KNOB_NUMBER, NUM_SAMPLES, TOP_NUM_CONFIG # pylint: disable=no-name-in-module
@ -102,14 +102,14 @@ class ConfigurationRecommendation(UpdateTask): # pylint: disable=abstract-metho
result = Result.objects.get(pk=result_id) result = Result.objects.get(pk=result_id)
# Replace result with formatted result # Replace result with formatted result
formatted_params = Parser.format_dbms_knobs(result.dbms.pk, retval['recommendation']) formatted_params = parser.format_dbms_knobs(result.dbms.pk, retval['recommendation'])
task_meta = TaskMeta.objects.get(task_id=task_id) task_meta = TaskMeta.objects.get(task_id=task_id)
retval['recommendation'] = formatted_params retval['recommendation'] = formatted_params
task_meta.result = retval task_meta.result = retval
task_meta.save() task_meta.save()
# Create next configuration to try # Create next configuration to try
config = Parser.create_knob_configuration(result.dbms.pk, retval['recommendation']) config = parser.create_knob_configuration(result.dbms.pk, retval['recommendation'])
retval['recommendation'] = config retval['recommendation'] = config
result.next_configuration = JSONUtil.dumps(retval) result.next_configuration = JSONUtil.dumps(retval)
result.save() result.save()

View File

@ -27,11 +27,11 @@ from django.views.decorators.csrf import csrf_exempt
from django.forms.models import model_to_dict from django.forms.models import model_to_dict
from pytz import timezone from pytz import timezone
from .db import parser
from .forms import NewResultForm, ProjectForm, SessionForm, SessionKnobForm from .forms import NewResultForm, ProjectForm, SessionForm, SessionKnobForm
from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog, from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog,
MetricData, MetricManager, Project, Result, Session, Workload, MetricData, MetricManager, Project, Result, Session, Workload,
SessionKnob) SessionKnob)
from .parser import Parser
from .tasks import (aggregate_target_results, map_workload, train_ddpg, from .tasks import (aggregate_target_results, map_workload, train_ddpg,
configuration_recommendation, configuration_recommendation_ddpg) configuration_recommendation, configuration_recommendation_ddpg)
from .types import (DBMSType, KnobUnitType, MetricType, from .types import (DBMSType, KnobUnitType, MetricType,
@ -487,23 +487,23 @@ def handle_result_files(session, files):
'(actual=' + dbms.full_name + ')') '(actual=' + dbms.full_name + ')')
# Load, process, and store the knobs in the DBMS's configuration # Load, process, and store the knobs in the DBMS's configuration
knob_dict, knob_diffs = Parser.parse_dbms_knobs( knob_dict, knob_diffs = parser.parse_dbms_knobs(
dbms.pk, JSONUtil.loads(files['knobs'])) dbms.pk, JSONUtil.loads(files['knobs']))
tunable_knob_dict = Parser.convert_dbms_knobs( tunable_knob_dict = parser.convert_dbms_knobs(
dbms.pk, knob_dict) dbms.pk, knob_dict)
knob_data = KnobData.objects.create_knob_data( knob_data = KnobData.objects.create_knob_data(
session, JSONUtil.dumps(knob_dict, pprint=True, sort=True), session, JSONUtil.dumps(knob_dict, pprint=True, sort=True),
JSONUtil.dumps(tunable_knob_dict, pprint=True, sort=True), dbms) JSONUtil.dumps(tunable_knob_dict, pprint=True, sort=True), dbms)
# Load, process, and store the runtime metrics exposed by the DBMS # Load, process, and store the runtime metrics exposed by the DBMS
initial_metric_dict, initial_metric_diffs = Parser.parse_dbms_metrics( initial_metric_dict, initial_metric_diffs = parser.parse_dbms_metrics(
dbms.pk, JSONUtil.loads(files['metrics_before'])) dbms.pk, JSONUtil.loads(files['metrics_before']))
final_metric_dict, final_metric_diffs = Parser.parse_dbms_metrics( final_metric_dict, final_metric_diffs = parser.parse_dbms_metrics(
dbms.pk, JSONUtil.loads(files['metrics_after'])) dbms.pk, JSONUtil.loads(files['metrics_after']))
metric_dict = Parser.calculate_change_in_metrics( metric_dict = parser.calculate_change_in_metrics(
dbms.pk, initial_metric_dict, final_metric_dict) dbms.pk, initial_metric_dict, final_metric_dict)
initial_metric_diffs.extend(final_metric_diffs) initial_metric_diffs.extend(final_metric_diffs)
numeric_metric_dict = Parser.convert_dbms_metrics( numeric_metric_dict = parser.convert_dbms_metrics(
dbms.pk, metric_dict, observation_time, session.target_objective) dbms.pk, metric_dict, observation_time, session.target_objective)
metric_data = MetricData.objects.create_metric_data( metric_data = MetricData.objects.create_metric_data(
session, JSONUtil.dumps(metric_dict, pprint=True, sort=True), session, JSONUtil.dumps(metric_dict, pprint=True, sort=True),
@ -658,11 +658,11 @@ def metric_data_view(request, project_id, session_id, data_id): # pylint: disab
def dbms_data_view(request, context, dbms_data): def dbms_data_view(request, context, dbms_data):
if context['data_type'] == 'knobs': if context['data_type'] == 'knobs':
model_class = KnobData model_class = KnobData
filter_fn = Parser.filter_tunable_knobs filter_fn = parser.filter_tunable_knobs
obj_data = dbms_data.knobs obj_data = dbms_data.knobs
else: else:
model_class = MetricData model_class = MetricData
filter_fn = Parser.filter_numeric_metrics filter_fn = parser.filter_numeric_metrics
obj_data = dbms_data.metrics obj_data = dbms_data.metrics
dbms_id = dbms_data.dbms.pk dbms_id = dbms_data.dbms.pk