public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated parser classes to not cache knob/metric catalogs and replaced all queries that filter for numeric metrics with the new MetricType.numeric() method.

This commit is contained in:
dvanaken 2020-01-13 15:42:13 -05:00 committed by Dana Van Aken
parent e1a41928f6
commit 7b962c4065
5 changed files with 65 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
server/website/tests/test_parser.py
View File

@ -9,7 +9,7 @@ import mock
from django.test import TestCase
from website.db import parser, target_objectives
from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType
from website.models import DBMSCatalog, KnobCatalog
from website.models import DBMSCatalog, KnobCatalog, MetricCatalog
class BaseParserTests(object, metaclass=ABCMeta):
@ -214,6 +214,11 @@ class PostgresParserTests(BaseParserTests, TestCase):
dbms_obj = DBMSCatalog.objects.filter(
type=DBMSType.POSTGRES, version="9.6").first()
self.test_dbms = parser._get(dbms_obj.pk) # pylint: disable=protected-access
self.knob_catalog = KnobCatalog.objects.filter(dbms=dbms_obj)
self.tunable_knob_catalog = self.knob_catalog.filter(tunable=True)
self.metric_catalog = MetricCatalog.objects.filter(dbms=dbms_obj)
self.numeric_metric_catalog = self.metric_catalog.filter(
metric_type__in=MetricType.numeric())
def test_convert_dbms_knobs(self):
super().test_convert_dbms_knobs()
@ -251,16 +256,17 @@ class PostgresParserTests(BaseParserTests, TestCase):
txns_counter = target_obj_instance.transactions_counter
test_metrics = {}
for met in self.numeric_metric_catalog:
test_metrics[met.name] = 2
for key in list(self.test_dbms.numeric_metric_catalog_.keys()):
test_metrics[key] = 2
test_metrics[txns_counter] = 10
test_metrics['pg_FAKE_METRIC'] = 0
self.assertEqual(test_metrics.get(target_obj), None)
test_convert_metrics = self.test_dbms.convert_dbms_metrics(test_metrics, 0.1, target_obj)
for key, metadata in list(self.test_dbms.numeric_metric_catalog_.items()):
for metadata in self.numeric_metric_catalog:
key = metadata.name
if key == txns_counter:
self.assertEqual(test_convert_metrics[key], 10 / 0.1)
continue
@ -285,11 +291,11 @@ class PostgresParserTests(BaseParserTests, TestCase):
self.test_dbms.parse_version_string("1.0")
def test_extract_valid_variables(self):
num_tunable_knobs = len(list(self.test_dbms.tunable_knob_catalog_.keys()))
num_tunable_knobs = len(self.tunable_knob_catalog)
test_empty, test_empty_diff = self.test_dbms.extract_valid_variables(
{}, self.test_dbms.tunable_knob_catalog_)
self.assertEqual(len(list(test_empty.keys())), num_tunable_knobs)
{}, self.tunable_knob_catalog)
self.assertEqual(len(test_empty), num_tunable_knobs)
self.assertEqual(len(test_empty_diff), num_tunable_knobs)
test_vars = {'global.wal_sync_method': 'fsync',
@ -303,7 +309,7 @@ class PostgresParserTests(BaseParserTests, TestCase):
'global.FAKE_KNOB': 'fake'}
tune_extract, tune_diff = self.test_dbms.extract_valid_variables(
test_vars, self.test_dbms.tunable_knob_catalog_)
test_vars, self.tunable_knob_catalog)
self.assertTrue(('miscapitalized', 'global.wal_buffers',
'global.Wal_buffers', 1024) in tune_diff)
@ -317,10 +323,10 @@ class PostgresParserTests(BaseParserTests, TestCase):
self.assertEqual(tune_extract.get('global.wal_buffers'), 1024)
self.assertEqual(tune_extract.get('global.Wal_buffers'), None)
self.assertEqual(len(tune_extract), len(self.test_dbms.tunable_knob_catalog_))
self.assertEqual(len(tune_extract), num_tunable_knobs)
nontune_extract, nontune_diff = self.test_dbms.extract_valid_variables(
test_vars, self.test_dbms.knob_catalog_)
test_vars, self.knob_catalog)
self.assertTrue(('miscapitalized', 'global.wal_buffers',
'global.Wal_buffers', 1024) in nontune_diff)
@ -519,14 +525,14 @@ class PostgresParserTests(BaseParserTests, TestCase):
'cpu_tuple_cost': 0.55,
'force_parallel_mode': 'regress',
'FAKE_KNOB': 'fake'}}}
num_knobs = len(self.knob_catalog)
(test_parse_dict, test_parse_log) = self.test_dbms.parse_dbms_knobs(test_knobs)
test_parse_dict, test_parse_log = self.test_dbms.parse_dbms_knobs(test_knobs)
self.assertEqual(len(test_parse_log), len(list(self.test_dbms.knob_catalog_.keys())) - 7)
self.assertEqual(len(test_parse_log), num_knobs - 7)
self.assertTrue(('extra', None, 'global.FAKE_KNOB', 'fake') in test_parse_log)
self.assertEqual(len(list(test_parse_dict.keys())),
len(list(self.test_dbms.knob_catalog_.keys())))
self.assertEqual(len(test_parse_dict), num_knobs)
self.assertEqual(test_parse_dict['global.wal_sync_method'], 'fsync')
self.assertEqual(test_parse_dict['global.random_page_cost'], 0.22)
@ -552,8 +558,7 @@ class PostgresParserTests(BaseParserTests, TestCase):
# Doesn't support table or index scope
with self.assertRaises(Exception):
num_metrics = len(self.metric_catalog)
test_parse_dict, test_parse_log = self.test_dbms.parse_dbms_metrics(test_metrics)
self.assertEqual(len(list(test_parse_dict.keys())),
len(list(self.test_dbms.metric_catalog_.keys())))
self.assertEqual(len(test_parse_log),
len(list(self.test_dbms.metric_catalog_.keys())) - 14)
self.assertEqual(len(test_parse_dict), num_metrics)
self.assertEqual(len(test_parse_log), num_metrics - 14)

55
server/website/website/db/base/parser.py
View File

@ -7,8 +7,8 @@ import logging
from collections import OrderedDict
from website.models import KnobCatalog, KnobUnitType, MetricCatalog
from website.types import BooleanType, MetricType, VarType
from website.models import KnobCatalog, MetricCatalog
from website.types import BooleanType, KnobUnitType, MetricType, VarType
from website.utils import ConversionUtil
from .. import target_objectives
@ -20,18 +20,6 @@ class BaseParser:
def __init__(self, dbms_obj):
self.dbms_id = int(dbms_obj.pk)
knobs = KnobCatalog.objects.filter(dbms=dbms_obj)
self.knob_catalog_ = {k.name: k for k in knobs}
self.tunable_knob_catalog_ = {
k: v for k, v in self.knob_catalog_.items() if
v.tunable is True}
metrics = MetricCatalog.objects.filter(dbms=dbms_obj)
self.metric_catalog_ = {m.name: m for m in metrics}
numeric_mtypes = (MetricType.COUNTER, MetricType.STATISTICS)
self.numeric_metric_catalog_ = {
m: v for m, v in self.metric_catalog_.items() if
v.metric_type in numeric_mtypes}
self.valid_true_val = ("on", "true", "yes")
self.valid_false_val = ("off", "false", "no")
@ -113,9 +101,9 @@ class BaseParser:
def convert_dbms_knobs(self, knobs):
knob_data = {}
for name, metadata in list(self.tunable_knob_catalog_.items()):
if metadata.tunable is False:
continue
tunable_knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True)
for metadata in tunable_knob_catalog:
name = metadata.name
if name not in knobs:
continue
value = knobs[name]
@ -187,8 +175,11 @@ class BaseParser:
metric_data = {}
# Same as metric_data except COUNTER metrics are not divided by the time
base_metric_data = {}
numeric_metric_catalog = MetricCatalog.objects.filter(
dbms__id=self.dbms_id, metric_type__in=MetricType.numeric())
for name, metadata in self.numeric_metric_catalog_.items():
for metadata in numeric_metric_catalog:
name = metadata.name
value = metrics[name]
if metadata.vartype == VarType.INTEGER:
@ -223,7 +214,7 @@ class BaseParser:
def extract_valid_variables(self, variables, catalog, default_value=None):
valid_variables = {}
diff_log = []
valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())}
valid_lc_variables = {k.name.lower(): k for k in catalog}
# First check that the names of all variables are valid (i.e., listed
# in the official catalog). Invalid variables are logged as 'extras'.
@ -286,7 +277,8 @@ class BaseParser:
assert len(valid_knobs[k]) == 1
valid_knobs[k] = valid_knobs[k][0]
# Extract all valid knobs
return self.extract_valid_variables(valid_knobs, self.knob_catalog_)
knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id)
return self.extract_valid_variables(valid_knobs, knob_catalog)
def parse_dbms_metrics(self, metrics):
# Some DBMSs measure different types of stats (e.g., global, local)
@ -295,16 +287,16 @@ class BaseParser:
valid_metrics = self.parse_dbms_variables(metrics)
# Extract all valid metrics
metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id)
valid_metrics, diffs = self.extract_valid_variables(
valid_metrics, self.metric_catalog_, default_value='0')
valid_metrics, metric_catalog, default_value='0')
# Combine values
for name, values in list(valid_metrics.items()):
metric = self.metric_catalog_[name]
if metric.metric_type == MetricType.INFO or len(values) == 1:
metric = metric_catalog.get(name=name)
if metric.metric_type in MetricType.nonnumeric() or len(values) == 1:
valid_metrics[name] = values[0]
elif metric.metric_type == MetricType.COUNTER or \
metric.metric_type == MetricType.STATISTICS:
elif metric.metric_type in MetricType.numeric():
conv_fn = int if metric.vartype == VarType.INTEGER else float
values = [conv_fn(v) for v in values if v is not None]
if len(values) == 0:
@ -317,10 +309,11 @@ class BaseParser:
return valid_metrics, diffs
def calculate_change_in_metrics(self, metrics_start, metrics_end, fix_metric_type=True):
metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id)
adjusted_metrics = {}
for met_name, start_val in list(metrics_start.items()):
end_val = metrics_end[met_name]
met_info = self.metric_catalog_[met_name]
met_info = metric_catalog.get(name=met_name)
if met_info.vartype == VarType.INTEGER or \
met_info.vartype == VarType.REAL:
conversion_fn = self.convert_integer if \
@ -332,13 +325,13 @@ class BaseParser:
adj_val = end_val - start_val
else: # MetricType.STATISTICS or MetricType.INFO
adj_val = end_val
if fix_metric_type:
if adj_val < 0:
adj_val = end_val
LOG.debug("Changing metric %s from COUNTER to STATISTICS", met_name)
metric_fixed = self.metric_catalog_[met_name]
metric_fixed.metric_type = MetricType.STATISTICS
metric_fixed.save()
met_info.metric_type = MetricType.STATISTICS
met_info.save()
assert adj_val >= 0, \
'{} wrong metric type: {} (start={}, end={}, diff={})'.format(
met_name, MetricType.name(met_info.metric_type), start_val,
@ -397,9 +390,7 @@ class BaseParser:
def format_dbms_knobs(self, knobs):
formatted_knobs = {}
for knob_name, knob_value in list(knobs.items()):
metadata = self.knob_catalog_.get(knob_name, None)
if (metadata is None):
raise Exception('Unknown knob {}'.format(knob_name))
metadata = KnobCatalog.objects.get(dbms__id=self.dbms_id, name=knob_name)
fvalue = None
if metadata.vartype == VarType.BOOL:
fvalue = self.format_bool(knob_value, metadata)

5
server/website/website/db/base/target_objective.py
View File

@ -94,8 +94,9 @@ class TargetObjectives:
self._registry[dbms_id][name] = target_objective_instance
if dbms_id not in self._metric_metadatas:
numeric_metrics = models.MetricCatalog.objects.filter(dbms=dbms).exclude(
metric_type=types.MetricType.INFO).values_list('name', flat=True)
numeric_metrics = models.MetricCatalog.objects.filter(
dbms=dbms, metric_type__in=types.MetricType.numeric()).values_list(
'name', flat=True)
self._metric_metadatas[dbms_id] = [(mname, BaseMetric(mname)) for mname
in sorted(numeric_metrics)]

27
server/website/website/db/myrocks/parser.py
View File

@ -9,6 +9,7 @@ from collections import OrderedDict
from ..base.parser import BaseParser
from .. import target_objectives
from website.models import KnobCatalog, MetricCatalog
from website.types import MetricType, VarType
@ -71,7 +72,7 @@ class MyRocksParser(BaseParser):
def extract_valid_variables(self, variables, catalog, default_value=None):
valid_variables = {}
diff_log = []
valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())}
valid_lc_variables = {k.name.lower(): k for k in catalog}
# First check that the names of all variables are valid (i.e., listed
# in the official catalog). Invalid variables are logged as 'extras'.
@ -106,9 +107,10 @@ class MyRocksParser(BaseParser):
def calculate_change_in_metrics(self, metrics_start, metrics_end):
adjusted_metrics = {}
metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id)
for met_name, start_val in list(metrics_start.items()):
end_val = metrics_end[met_name]
met_info = self.metric_catalog_[self.partial_name(met_name)]
met_info = metric_catalog.get(name=self.partial_name(met_name))
if met_info.vartype == VarType.INTEGER or \
met_info.vartype == VarType.REAL:
conversion_fn = self.convert_integer if \
@ -127,26 +129,28 @@ class MyRocksParser(BaseParser):
def parse_dbms_knobs(self, knobs):
valid_knobs = self.parse_dbms_variables(knobs)
knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id)
# Extract all valid knobs
return self.extract_valid_variables(
valid_knobs, self.knob_catalog_)
return self.extract_valid_variables(valid_knobs, knob_catalog)
def parse_dbms_metrics(self, metrics):
valid_metrics = self.parse_dbms_variables(metrics)
metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id)
# Extract all valid metrics
valid_metrics, diffs = self.extract_valid_variables(
valid_metrics, self.metric_catalog_, default_value='0')
valid_metrics, metric_catalog, default_value='0')
return valid_metrics, diffs
def convert_dbms_metrics(self, metrics, observation_time, target_objective):
base_metric_data = {}
metric_data = {}
numeric_metric_catalog = MetricCatalog.objects.filter(
dbms__id=self.dbms_id, metric_type__in=MetricType.numeric())
for name, value in list(metrics.items()):
prt_name = self.partial_name(name)
metadata = numeric_metric_catalog.filter(name=prt_name).first()
if prt_name in self.numeric_metric_catalog_:
metadata = self.numeric_metric_catalog_[prt_name]
if metadata:
if metadata.vartype == VarType.INTEGER:
converted = float(self.convert_integer(value, metadata))
elif metadata.vartype == VarType.REAL:
@ -178,12 +182,11 @@ class MyRocksParser(BaseParser):
def convert_dbms_knobs(self, knobs):
knob_data = {}
tunable_knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True)
for name, value in list(knobs.items()):
prt_name = self.partial_name(name)
if prt_name in self.tunable_knob_catalog_:
metadata = self.tunable_knob_catalog_[prt_name]
assert(metadata.tunable)
value = knobs[name]
metadata = tunable_knob_catalog.filter(name=prt_name).first()
if metadata:
conv_value = None
if metadata.vartype == VarType.BOOL:
conv_value = self.convert_bool(value, metadata)

3
server/website/website/views.py
View File

@ -774,9 +774,8 @@ def dbms_data_view(request, context, dbms_data, session, target_obj):
'knob__name', flat=True))
else:
model_class = MetricData
num_types = (MetricType.COUNTER, MetricType.STATISTICS)
featured_names = set(MetricCatalog.objects.filter(
dbms=session.dbms, metric_type__in=num_types).values_list(
dbms=session.dbms, metric_type__in=MetricCtype.numeric()).values_list(
'name', flat=True))
obj_data = getattr(dbms_data, data_type)