public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added outer db module and moved parser code into it

This commit is contained in:
Dana Van Aken 2019-10-08 11:24:04 -04:00
parent a3833d83b1
commit f68c23e975
14 changed files with 141 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
server/website/tests/test_parser.py
View File

@ -7,7 +7,8 @@
from abc import ABCMeta, abstractmethod
import mock
from django.test import TestCase
from website.parser.postgres import PostgresParser
from website.db import parser
# from website.db.parser.postgres import PostgresParser
from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType
from website.models import DBMSCatalog, KnobCatalog
@ -221,7 +222,7 @@ class PostgresParserTests(BaseParserTests, TestCase):
def setUp(self):
dbms_obj = DBMSCatalog.objects.filter(
type=DBMSType.POSTGRES, version="9.6").first()
self.test_dbms = PostgresParser(dbms_obj)
self.test_dbms = parser._get(dbms_obj.pk) # pylint: disable=protected-access
def test_convert_dbms_knobs(self):
super().test_convert_dbms_knobs()

1
server/website/website/parser/__init__.py → server/website/website/db/__init__.py
View File

@ -3,4 +3,3 @@
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from .parser import Parser

5
server/website/website/db/base/__init__.py Normal file
View File

@ -0,0 +1,5 @@
#
# OtterTune - __init__.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#

10
server/website/website/parser/base.py → server/website/website/db/base/parser.py
View File

@ -1,5 +1,5 @@
#
# OtterTune - base.py
# OtterTune - parser.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
@ -213,8 +213,7 @@ class BaseParser:
return metric_data
@staticmethod
def extract_valid_variables(variables, catalog, default_value=None):
def extract_valid_variables(self, variables, catalog, default_value=None):
valid_variables = {}
diff_log = []
valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())}
@ -280,8 +279,7 @@ class BaseParser:
assert len(valid_knobs[k]) == 1
valid_knobs[k] = valid_knobs[k][0]
# Extract all valid knobs
return BaseParser.extract_valid_variables(
valid_knobs, self.knob_catalog_)
return self.extract_valid_variables(valid_knobs, self.knob_catalog_)
def parse_dbms_metrics(self, metrics):
# Some DBMSs measure different types of stats (e.g., global, local)
@ -290,7 +288,7 @@ class BaseParser:
valid_metrics = self.parse_dbms_variables(metrics)
# Extract all valid metrics
valid_metrics, diffs = BaseParser.extract_valid_variables(
valid_metrics, diffs = self.extract_valid_variables(
valid_metrics, self.metric_catalog_, default_value='0')
# Combine values

5
server/website/website/db/myrocks/__init__.py Normal file
View File

@ -0,0 +1,5 @@
#
# OtterTune - __init__.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#

25
server/website/website/parser/myrocks.py → server/website/website/db/myrocks/parser.py
View File

@ -1,5 +1,5 @@
#
# OtterTune - myrocks.py
# OtterTune - parser.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
@ -7,7 +7,7 @@
import re
from collections import OrderedDict
from .base import BaseParser
from ..base.parser import BaseParser
from website.types import MetricType, VarType
@ -75,8 +75,7 @@ class MyRocksParser(BaseParser):
else:
raise Exception('Invalid variable full name: {}'.format(full_name))
@staticmethod
def extract_valid_variables(variables, catalog, default_value=None):
def extract_valid_variables(self, variables, catalog, default_value=None):
valid_variables = {}
diff_log = []
valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())}
@ -88,7 +87,7 @@ class MyRocksParser(BaseParser):
# are also logged as 'miscapitalized'.
for var_name, var_value in list(variables.items()):
lc_var_name = var_name.lower()
prt_name = MyRocksParser.partial_name(lc_var_name)
prt_name = self.partial_name(lc_var_name)
if prt_name in valid_lc_variables:
valid_name = valid_lc_variables[prt_name].name
if prt_name != valid_name:
@ -102,7 +101,7 @@ class MyRocksParser(BaseParser):
# the given default_value if provided (or the item's actual default value
# if not) and logged as 'missing'. For now missing local variables are
# not added to valid_variables
lc_variables = {MyRocksParser.partial_name(k.lower()): v
lc_variables = {self.partial_name(k.lower()): v
for k, v in list(variables.items())}
for valid_lc_name, metadata in list(valid_lc_variables.items()):
if valid_lc_name not in lc_variables:
@ -116,7 +115,7 @@ class MyRocksParser(BaseParser):
adjusted_metrics = {}
for met_name, start_val in list(metrics_start.items()):
end_val = metrics_end[met_name]
met_info = self.metric_catalog_[MyRocksParser.partial_name(met_name)]
met_info = self.metric_catalog_[self.partial_name(met_name)]
if met_info.vartype == VarType.INTEGER or \
met_info.vartype == VarType.REAL:
conversion_fn = self.convert_integer if \
@ -136,20 +135,20 @@ class MyRocksParser(BaseParser):
def parse_dbms_knobs(self, knobs):
valid_knobs = self.parse_dbms_variables(knobs)
# Extract all valid knobs
return MyRocksParser.extract_valid_variables(
return self.extract_valid_variables(
valid_knobs, self.knob_catalog_)
def parse_dbms_metrics(self, metrics):
valid_metrics = self.parse_dbms_variables(metrics)
# Extract all valid metrics
valid_metrics, diffs = MyRocksParser.extract_valid_variables(
valid_metrics, diffs = self.extract_valid_variables(
valid_metrics, self.metric_catalog_, default_value='0')
return valid_metrics, diffs
def convert_dbms_metrics(self, metrics, observation_time, target_objective=None):
metric_data = {}
for name, value in list(metrics.items()):
prt_name = MyRocksParser.partial_name(name)
prt_name = self.partial_name(name)
if prt_name in self.numeric_metric_catalog_:
metadata = self.numeric_metric_catalog_[prt_name]
if metadata.metric_type == MetricType.COUNTER:
@ -173,7 +172,7 @@ class MyRocksParser(BaseParser):
def convert_dbms_knobs(self, knobs):
knob_data = {}
for name, value in list(knobs.items()):
prt_name = MyRocksParser.partial_name(name)
prt_name = self.partial_name(name)
if prt_name in self.tunable_knob_catalog_:
metadata = self.tunable_knob_catalog_[prt_name]
assert(metadata.tunable)
@ -202,8 +201,8 @@ class MyRocksParser(BaseParser):
def filter_numeric_metrics(self, metrics):
return OrderedDict([(k, v) for k, v in list(metrics.items()) if
MyRocksParser.partial_name(k) in self.numeric_metric_catalog_])
self.partial_name(k) in self.numeric_metric_catalog_])
def filter_tunable_knobs(self, knobs):
return OrderedDict([(k, v) for k, v in list(knobs.items()) if
MyRocksParser.partial_name(k) in self.tunable_knob_catalog_])
self.partial_name(k) in self.tunable_knob_catalog_])

5
server/website/website/db/oracle/__init__.py Normal file
View File

@ -0,0 +1,5 @@
#
# OtterTune - __init__.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#

4
server/website/website/parser/oracle.py → server/website/website/db/oracle/parser.py
View File

@ -1,10 +1,10 @@
#
# OtterTune - oracle.py
# OtterTune - parser.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from .base import BaseParser
from ..base.parser import BaseParser
class OracleParser(BaseParser):

86
server/website/website/db/parser.py Normal file
View File

@ -0,0 +1,86 @@
#
# OtterTune - parser.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from website.models import DBMSCatalog
from website.types import DBMSType
from .myrocks.parser import MyRocksParser
from .postgres.parser import PostgresParser
from .oracle.parser import OracleParser
_DBMS_PARSERS = {}
def _get(dbms_id):
dbms_id = int(dbms_id)
db_parser = _DBMS_PARSERS.get(dbms_id, None)
if db_parser is None:
obj = DBMSCatalog.objects.get(id=dbms_id)
if obj.type == DBMSType.POSTGRES:
clz = PostgresParser
elif obj.type == DBMSType.MYROCKS:
clz = MyRocksParser
elif obj.type == DBMSType.ORACLE:
clz = OracleParser
else:
raise NotImplementedError('Implement me! {}'.format(obj))
db_parser = clz(obj)
_DBMS_PARSERS[dbms_id] = db_parser
return db_parser
def parse_version_string(dbms_type, version_string):
dbmss = DBMSCatalog.objects.filter(type=dbms_type)
parsed_version = None
for instance in dbmss:
db_parser = _get(instance.pk)
try:
parsed_version = db_parser.parse_version_string(version_string)
except AttributeError:
pass
if parsed_version is not None:
break
return parsed_version
def convert_dbms_knobs(dbms_id, knobs):
return _get(dbms_id).convert_dbms_knobs(knobs)
def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective=None):
return _get(dbms_id).convert_dbms_metrics(
numeric_metrics, observation_time, target_objective)
def parse_dbms_knobs(dbms_id, knobs):
return _get(dbms_id).parse_dbms_knobs(knobs)
def parse_dbms_metrics(dbms_id, metrics):
return _get(dbms_id).parse_dbms_metrics(metrics)
def create_knob_configuration(dbms_id, tuning_knobs):
return _get(dbms_id).create_knob_configuration(tuning_knobs)
def format_dbms_knobs(dbms_id, knobs):
return _get(dbms_id).format_dbms_knobs(knobs)
def filter_numeric_metrics(dbms_id, metrics):
return _get(dbms_id).filter_numeric_metrics(metrics)
def filter_tunable_knobs(dbms_id, knobs):
return _get(dbms_id).filter_tunable_knobs(knobs)
def calculate_change_in_metrics(dbms_id, metrics_start, metrics_end):
return _get(dbms_id).calculate_change_in_metrics(
metrics_start, metrics_end)

5
server/website/website/db/postgres/__init__.py Normal file
View File

@ -0,0 +1,5 @@
#
# OtterTune - __init__.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#

4
server/website/website/parser/postgres.py → server/website/website/db/postgres/parser.py
View File

@ -1,12 +1,12 @@
#
# OtterTune - postgres.py
# OtterTune - parser.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
import re
from .base import BaseParser
from ..base.parser import BaseParser
from website.utils import ConversionUtil

97
server/website/website/parser/parser.py
View File

@ -1,97 +0,0 @@
#
# OtterTune - parser.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from website.models import DBMSCatalog
from website.types import DBMSType
from .myrocks import MyRocksParser
from .postgres import PostgresParser
from .oracle import OracleParser
class Parser():
__DBMS_UTILS_IMPLS = None
@staticmethod
def __utils(dbms_id=None):
if Parser.__DBMS_UTILS_IMPLS is None:
parsers = {}
for obj in DBMSCatalog.objects.all():
if obj.type == DBMSType.POSTGRES:
clz = PostgresParser
elif obj.type == DBMSType.MYROCKS:
clz = MyRocksParser
elif obj.type == DBMSType.ORACLE:
clz = OracleParser
else:
raise NotImplementedError('Implement me! {}'.format(obj))
parsers[obj.pk] = clz(obj)
Parser.__DBMS_UTILS_IMPLS = parsers
try:
if dbms_id is None:
return Parser.__DBMS_UTILS_IMPLS
return Parser.__DBMS_UTILS_IMPLS[dbms_id]
except KeyError:
raise NotImplementedError(
'Implement me! ({})'.format(dbms_id))
@staticmethod
def parse_version_string(dbms_type, version_string):
for k, v in list(Parser.__utils(dbms_type).items()):
dbms = DBMSCatalog.objects.get(pk=k)
if dbms.type == dbms_type:
try:
return v.parse_version_string(version_string)
except AttributeError:
pass
return None
@staticmethod
def convert_dbms_knobs(dbms_id, knobs):
return Parser.__utils(dbms_id).convert_dbms_knobs(knobs)
@staticmethod
def convert_dbms_metrics(dbms_id, numeric_metrics, observation_time, target_objective=None):
return Parser.__utils(dbms_id).convert_dbms_metrics(
numeric_metrics, observation_time, target_objective)
@staticmethod
def parse_dbms_knobs(dbms_id, knobs):
return Parser.__utils(dbms_id).parse_dbms_knobs(knobs)
@staticmethod
def parse_dbms_metrics(dbms_id, metrics):
return Parser.__utils(dbms_id).parse_dbms_metrics(metrics)
@staticmethod
def get_nondefault_knob_settings(dbms_id, knobs):
return Parser.__utils(dbms_id).get_nondefault_knob_settings(knobs)
@staticmethod
def create_knob_configuration(dbms_id, tuning_knobs):
return Parser.__utils(dbms_id).create_knob_configuration(tuning_knobs)
@staticmethod
def format_dbms_knobs(dbms_id, knobs):
return Parser.__utils(dbms_id).format_dbms_knobs(knobs)
@staticmethod
def filter_numeric_metrics(dbms_id, metrics):
return Parser.__utils(dbms_id).filter_numeric_metrics(metrics)
@staticmethod
def filter_tunable_knobs(dbms_id, knobs):
return Parser.__utils(dbms_id).filter_tunable_knobs(knobs)
@staticmethod
def calculate_change_in_metrics(dbms_id, metrics_start, metrics_end):
return Parser.__utils(dbms_id).calculate_change_in_metrics(
metrics_start, metrics_end)

6
server/website/website/tasks/async_tasks.py
View File

@ -20,7 +20,7 @@ from analysis.preprocessing import Bin, DummyEncoder
from analysis.constraints import ParamConstraintHelper
from website.models import (PipelineData, PipelineRun, Result, Workload, KnobCatalog,
MetricCatalog, SessionKnob)
from website.parser import Parser
from website.db import parser
from website.types import PipelineTaskType, AlgorithmType
from website.utils import DataUtil, JSONUtil
from website.settings import IMPORTANT_KNOB_NUMBER, NUM_SAMPLES, TOP_NUM_CONFIG # pylint: disable=no-name-in-module
@ -102,14 +102,14 @@ class ConfigurationRecommendation(UpdateTask): # pylint: disable=abstract-metho
result = Result.objects.get(pk=result_id)
# Replace result with formatted result
formatted_params = Parser.format_dbms_knobs(result.dbms.pk, retval['recommendation'])
formatted_params = parser.format_dbms_knobs(result.dbms.pk, retval['recommendation'])
task_meta = TaskMeta.objects.get(task_id=task_id)
retval['recommendation'] = formatted_params
task_meta.result = retval
task_meta.save()
# Create next configuration to try
config = Parser.create_knob_configuration(result.dbms.pk, retval['recommendation'])
config = parser.create_knob_configuration(result.dbms.pk, retval['recommendation'])
retval['recommendation'] = config
result.next_configuration = JSONUtil.dumps(retval)
result.save()

18
server/website/website/views.py
View File

@ -27,11 +27,11 @@ from django.views.decorators.csrf import csrf_exempt
from django.forms.models import model_to_dict
from pytz import timezone
from .db import parser
from .forms import NewResultForm, ProjectForm, SessionForm, SessionKnobForm
from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog,
MetricData, MetricManager, Project, Result, Session, Workload,
SessionKnob)
from .parser import Parser
from .tasks import (aggregate_target_results, map_workload, train_ddpg,
configuration_recommendation, configuration_recommendation_ddpg)
from .types import (DBMSType, KnobUnitType, MetricType,
@ -487,23 +487,23 @@ def handle_result_files(session, files):
'(actual=' + dbms.full_name + ')')
# Load, process, and store the knobs in the DBMS's configuration
knob_dict, knob_diffs = Parser.parse_dbms_knobs(
knob_dict, knob_diffs = parser.parse_dbms_knobs(
dbms.pk, JSONUtil.loads(files['knobs']))
tunable_knob_dict = Parser.convert_dbms_knobs(
tunable_knob_dict = parser.convert_dbms_knobs(
dbms.pk, knob_dict)
knob_data = KnobData.objects.create_knob_data(
session, JSONUtil.dumps(knob_dict, pprint=True, sort=True),
JSONUtil.dumps(tunable_knob_dict, pprint=True, sort=True), dbms)
# Load, process, and store the runtime metrics exposed by the DBMS
initial_metric_dict, initial_metric_diffs = Parser.parse_dbms_metrics(
initial_metric_dict, initial_metric_diffs = parser.parse_dbms_metrics(
dbms.pk, JSONUtil.loads(files['metrics_before']))
final_metric_dict, final_metric_diffs = Parser.parse_dbms_metrics(
final_metric_dict, final_metric_diffs = parser.parse_dbms_metrics(
dbms.pk, JSONUtil.loads(files['metrics_after']))
metric_dict = Parser.calculate_change_in_metrics(
metric_dict = parser.calculate_change_in_metrics(
dbms.pk, initial_metric_dict, final_metric_dict)
initial_metric_diffs.extend(final_metric_diffs)
numeric_metric_dict = Parser.convert_dbms_metrics(
numeric_metric_dict = parser.convert_dbms_metrics(
dbms.pk, metric_dict, observation_time, session.target_objective)
metric_data = MetricData.objects.create_metric_data(
session, JSONUtil.dumps(metric_dict, pprint=True, sort=True),
@ -658,11 +658,11 @@ def metric_data_view(request, project_id, session_id, data_id): # pylint: disab
def dbms_data_view(request, context, dbms_data):
if context['data_type'] == 'knobs':
model_class = KnobData
filter_fn = Parser.filter_tunable_knobs
filter_fn = parser.filter_tunable_knobs
obj_data = dbms_data.knobs
else:
model_class = MetricData
filter_fn = Parser.filter_numeric_metrics
filter_fn = parser.filter_numeric_metrics
obj_data = dbms_data.metrics
dbms_id = dbms_data.dbms.pk