public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated parser classes to not cache knob/metric catalogs and replaced all queries that filter for numeric metrics with the new MetricType.numeric() method.

This commit is contained in:
dvanaken 2020-01-13 15:42:13 -05:00 committed by Dana Van Aken
parent e1a41928f6
commit 7b962c4065
5 changed files with 65 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
server/website/tests/test_parser.py
View File

@ -9,7 +9,7 @@ import mock
from django.test import TestCase from django.test import TestCase
from website.db import parser, target_objectives from website.db import parser, target_objectives
from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType
from website.models import DBMSCatalog, KnobCatalog from website.models import DBMSCatalog, KnobCatalog, MetricCatalog
class BaseParserTests(object, metaclass=ABCMeta): class BaseParserTests(object, metaclass=ABCMeta):
@ -214,6 +214,11 @@ class PostgresParserTests(BaseParserTests, TestCase):
dbms_obj = DBMSCatalog.objects.filter( dbms_obj = DBMSCatalog.objects.filter(
type=DBMSType.POSTGRES, version="9.6").first() type=DBMSType.POSTGRES, version="9.6").first()
self.test_dbms = parser._get(dbms_obj.pk) # pylint: disable=protected-access self.test_dbms = parser._get(dbms_obj.pk) # pylint: disable=protected-access
self.knob_catalog = KnobCatalog.objects.filter(dbms=dbms_obj)
self.tunable_knob_catalog = self.knob_catalog.filter(tunable=True)
self.metric_catalog = MetricCatalog.objects.filter(dbms=dbms_obj)
self.numeric_metric_catalog = self.metric_catalog.filter(
metric_type__in=MetricType.numeric())
def test_convert_dbms_knobs(self): def test_convert_dbms_knobs(self):
super().test_convert_dbms_knobs() super().test_convert_dbms_knobs()
@ -251,16 +256,17 @@ class PostgresParserTests(BaseParserTests, TestCase):
txns_counter = target_obj_instance.transactions_counter txns_counter = target_obj_instance.transactions_counter
test_metrics = {} test_metrics = {}
for met in self.numeric_metric_catalog:
test_metrics[met.name] = 2
for key in list(self.test_dbms.numeric_metric_catalog_.keys()):
test_metrics[key] = 2
test_metrics[txns_counter] = 10 test_metrics[txns_counter] = 10
test_metrics['pg_FAKE_METRIC'] = 0 test_metrics['pg_FAKE_METRIC'] = 0
self.assertEqual(test_metrics.get(target_obj), None) self.assertEqual(test_metrics.get(target_obj), None)
test_convert_metrics = self.test_dbms.convert_dbms_metrics(test_metrics, 0.1, target_obj) test_convert_metrics = self.test_dbms.convert_dbms_metrics(test_metrics, 0.1, target_obj)
for key, metadata in list(self.test_dbms.numeric_metric_catalog_.items()): for metadata in self.numeric_metric_catalog:
key = metadata.name
if key == txns_counter: if key == txns_counter:
self.assertEqual(test_convert_metrics[key], 10 / 0.1) self.assertEqual(test_convert_metrics[key], 10 / 0.1)
continue continue
@ -285,11 +291,11 @@ class PostgresParserTests(BaseParserTests, TestCase):
self.test_dbms.parse_version_string("1.0") self.test_dbms.parse_version_string("1.0")
def test_extract_valid_variables(self): def test_extract_valid_variables(self):
num_tunable_knobs = len(list(self.test_dbms.tunable_knob_catalog_.keys())) num_tunable_knobs = len(self.tunable_knob_catalog)
test_empty, test_empty_diff = self.test_dbms.extract_valid_variables( test_empty, test_empty_diff = self.test_dbms.extract_valid_variables(
{}, self.test_dbms.tunable_knob_catalog_) {}, self.tunable_knob_catalog)
self.assertEqual(len(list(test_empty.keys())), num_tunable_knobs) self.assertEqual(len(test_empty), num_tunable_knobs)
self.assertEqual(len(test_empty_diff), num_tunable_knobs) self.assertEqual(len(test_empty_diff), num_tunable_knobs)
test_vars = {'global.wal_sync_method': 'fsync', test_vars = {'global.wal_sync_method': 'fsync',
@ -303,7 +309,7 @@ class PostgresParserTests(BaseParserTests, TestCase):
'global.FAKE_KNOB': 'fake'} 'global.FAKE_KNOB': 'fake'}
tune_extract, tune_diff = self.test_dbms.extract_valid_variables( tune_extract, tune_diff = self.test_dbms.extract_valid_variables(
test_vars, self.test_dbms.tunable_knob_catalog_) test_vars, self.tunable_knob_catalog)
self.assertTrue(('miscapitalized', 'global.wal_buffers', self.assertTrue(('miscapitalized', 'global.wal_buffers',
'global.Wal_buffers', 1024) in tune_diff) 'global.Wal_buffers', 1024) in tune_diff)
@ -317,10 +323,10 @@ class PostgresParserTests(BaseParserTests, TestCase):
self.assertEqual(tune_extract.get('global.wal_buffers'), 1024) self.assertEqual(tune_extract.get('global.wal_buffers'), 1024)
self.assertEqual(tune_extract.get('global.Wal_buffers'), None) self.assertEqual(tune_extract.get('global.Wal_buffers'), None)
self.assertEqual(len(tune_extract), len(self.test_dbms.tunable_knob_catalog_)) self.assertEqual(len(tune_extract), num_tunable_knobs)
nontune_extract, nontune_diff = self.test_dbms.extract_valid_variables( nontune_extract, nontune_diff = self.test_dbms.extract_valid_variables(
test_vars, self.test_dbms.knob_catalog_) test_vars, self.knob_catalog)
self.assertTrue(('miscapitalized', 'global.wal_buffers', self.assertTrue(('miscapitalized', 'global.wal_buffers',
'global.Wal_buffers', 1024) in nontune_diff) 'global.Wal_buffers', 1024) in nontune_diff)
@ -519,14 +525,14 @@ class PostgresParserTests(BaseParserTests, TestCase):
'cpu_tuple_cost': 0.55, 'cpu_tuple_cost': 0.55,
'force_parallel_mode': 'regress', 'force_parallel_mode': 'regress',
'FAKE_KNOB': 'fake'}}} 'FAKE_KNOB': 'fake'}}}
num_knobs = len(self.knob_catalog)
(test_parse_dict, test_parse_log) = self.test_dbms.parse_dbms_knobs(test_knobs) test_parse_dict, test_parse_log = self.test_dbms.parse_dbms_knobs(test_knobs)
self.assertEqual(len(test_parse_log), len(list(self.test_dbms.knob_catalog_.keys())) - 7) self.assertEqual(len(test_parse_log), num_knobs - 7)
self.assertTrue(('extra', None, 'global.FAKE_KNOB', 'fake') in test_parse_log) self.assertTrue(('extra', None, 'global.FAKE_KNOB', 'fake') in test_parse_log)
self.assertEqual(len(list(test_parse_dict.keys())), self.assertEqual(len(test_parse_dict), num_knobs)
len(list(self.test_dbms.knob_catalog_.keys())))
self.assertEqual(test_parse_dict['global.wal_sync_method'], 'fsync') self.assertEqual(test_parse_dict['global.wal_sync_method'], 'fsync')
self.assertEqual(test_parse_dict['global.random_page_cost'], 0.22) self.assertEqual(test_parse_dict['global.random_page_cost'], 0.22)
@ -552,8 +558,7 @@ class PostgresParserTests(BaseParserTests, TestCase):
# Doesn't support table or index scope # Doesn't support table or index scope
with self.assertRaises(Exception): with self.assertRaises(Exception):
num_metrics = len(self.metric_catalog)
test_parse_dict, test_parse_log = self.test_dbms.parse_dbms_metrics(test_metrics) test_parse_dict, test_parse_log = self.test_dbms.parse_dbms_metrics(test_metrics)
self.assertEqual(len(list(test_parse_dict.keys())), self.assertEqual(len(test_parse_dict), num_metrics)
len(list(self.test_dbms.metric_catalog_.keys()))) self.assertEqual(len(test_parse_log), num_metrics - 14)
self.assertEqual(len(test_parse_log),
len(list(self.test_dbms.metric_catalog_.keys())) - 14)

55
server/website/website/db/base/parser.py
View File

@ -7,8 +7,8 @@ import logging
from collections import OrderedDict from collections import OrderedDict
from website.models import KnobCatalog, KnobUnitType, MetricCatalog from website.models import KnobCatalog, MetricCatalog
from website.types import BooleanType, MetricType, VarType from website.types import BooleanType, KnobUnitType, MetricType, VarType
from website.utils import ConversionUtil from website.utils import ConversionUtil
from .. import target_objectives from .. import target_objectives
@ -20,18 +20,6 @@ class BaseParser:
def __init__(self, dbms_obj): def __init__(self, dbms_obj):
self.dbms_id = int(dbms_obj.pk) self.dbms_id = int(dbms_obj.pk)
knobs = KnobCatalog.objects.filter(dbms=dbms_obj)
self.knob_catalog_ = {k.name: k for k in knobs}
self.tunable_knob_catalog_ = {
k: v for k, v in self.knob_catalog_.items() if
v.tunable is True}
metrics = MetricCatalog.objects.filter(dbms=dbms_obj)
self.metric_catalog_ = {m.name: m for m in metrics}
numeric_mtypes = (MetricType.COUNTER, MetricType.STATISTICS)
self.numeric_metric_catalog_ = {
m: v for m, v in self.metric_catalog_.items() if
v.metric_type in numeric_mtypes}
self.valid_true_val = ("on", "true", "yes") self.valid_true_val = ("on", "true", "yes")
self.valid_false_val = ("off", "false", "no") self.valid_false_val = ("off", "false", "no")
@ -113,9 +101,9 @@ class BaseParser:
def convert_dbms_knobs(self, knobs): def convert_dbms_knobs(self, knobs):
knob_data = {} knob_data = {}
for name, metadata in list(self.tunable_knob_catalog_.items()): tunable_knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True)
if metadata.tunable is False: for metadata in tunable_knob_catalog:
continue name = metadata.name
if name not in knobs: if name not in knobs:
continue continue
value = knobs[name] value = knobs[name]
@ -187,8 +175,11 @@ class BaseParser:
metric_data = {} metric_data = {}
# Same as metric_data except COUNTER metrics are not divided by the time # Same as metric_data except COUNTER metrics are not divided by the time
base_metric_data = {} base_metric_data = {}
numeric_metric_catalog = MetricCatalog.objects.filter(
dbms__id=self.dbms_id, metric_type__in=MetricType.numeric())
for name, metadata in self.numeric_metric_catalog_.items(): for metadata in numeric_metric_catalog:
name = metadata.name
value = metrics[name] value = metrics[name]
if metadata.vartype == VarType.INTEGER: if metadata.vartype == VarType.INTEGER:
@ -223,7 +214,7 @@ class BaseParser:
def extract_valid_variables(self, variables, catalog, default_value=None): def extract_valid_variables(self, variables, catalog, default_value=None):
valid_variables = {} valid_variables = {}
diff_log = [] diff_log = []
valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())} valid_lc_variables = {k.name.lower(): k for k in catalog}
# First check that the names of all variables are valid (i.e., listed # First check that the names of all variables are valid (i.e., listed
# in the official catalog). Invalid variables are logged as 'extras'. # in the official catalog). Invalid variables are logged as 'extras'.
@ -286,7 +277,8 @@ class BaseParser:
assert len(valid_knobs[k]) == 1 assert len(valid_knobs[k]) == 1
valid_knobs[k] = valid_knobs[k][0] valid_knobs[k] = valid_knobs[k][0]
# Extract all valid knobs # Extract all valid knobs
return self.extract_valid_variables(valid_knobs, self.knob_catalog_) knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id)
return self.extract_valid_variables(valid_knobs, knob_catalog)
def parse_dbms_metrics(self, metrics): def parse_dbms_metrics(self, metrics):
# Some DBMSs measure different types of stats (e.g., global, local) # Some DBMSs measure different types of stats (e.g., global, local)
@ -295,16 +287,16 @@ class BaseParser:
valid_metrics = self.parse_dbms_variables(metrics) valid_metrics = self.parse_dbms_variables(metrics)
# Extract all valid metrics # Extract all valid metrics
metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id)
valid_metrics, diffs = self.extract_valid_variables( valid_metrics, diffs = self.extract_valid_variables(
valid_metrics, self.metric_catalog_, default_value='0') valid_metrics, metric_catalog, default_value='0')
# Combine values # Combine values
for name, values in list(valid_metrics.items()): for name, values in list(valid_metrics.items()):
metric = self.metric_catalog_[name] metric = metric_catalog.get(name=name)
if metric.metric_type == MetricType.INFO or len(values) == 1: if metric.metric_type in MetricType.nonnumeric() or len(values) == 1:
valid_metrics[name] = values[0] valid_metrics[name] = values[0]
elif metric.metric_type == MetricType.COUNTER or \ elif metric.metric_type in MetricType.numeric():
metric.metric_type == MetricType.STATISTICS:
conv_fn = int if metric.vartype == VarType.INTEGER else float conv_fn = int if metric.vartype == VarType.INTEGER else float
values = [conv_fn(v) for v in values if v is not None] values = [conv_fn(v) for v in values if v is not None]
if len(values) == 0: if len(values) == 0:
@ -317,10 +309,11 @@ class BaseParser:
return valid_metrics, diffs return valid_metrics, diffs
def calculate_change_in_metrics(self, metrics_start, metrics_end, fix_metric_type=True): def calculate_change_in_metrics(self, metrics_start, metrics_end, fix_metric_type=True):
metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id)
adjusted_metrics = {} adjusted_metrics = {}
for met_name, start_val in list(metrics_start.items()): for met_name, start_val in list(metrics_start.items()):
end_val = metrics_end[met_name] end_val = metrics_end[met_name]
met_info = self.metric_catalog_[met_name] met_info = metric_catalog.get(name=met_name)
if met_info.vartype == VarType.INTEGER or \ if met_info.vartype == VarType.INTEGER or \
met_info.vartype == VarType.REAL: met_info.vartype == VarType.REAL:
conversion_fn = self.convert_integer if \ conversion_fn = self.convert_integer if \
@ -332,13 +325,13 @@ class BaseParser:
adj_val = end_val - start_val adj_val = end_val - start_val
else: # MetricType.STATISTICS or MetricType.INFO else: # MetricType.STATISTICS or MetricType.INFO
adj_val = end_val adj_val = end_val
if fix_metric_type: if fix_metric_type:
if adj_val < 0: if adj_val < 0:
adj_val = end_val adj_val = end_val
LOG.debug("Changing metric %s from COUNTER to STATISTICS", met_name) LOG.debug("Changing metric %s from COUNTER to STATISTICS", met_name)
metric_fixed = self.metric_catalog_[met_name] met_info.metric_type = MetricType.STATISTICS
metric_fixed.metric_type = MetricType.STATISTICS met_info.save()
metric_fixed.save()
assert adj_val >= 0, \ assert adj_val >= 0, \
'{} wrong metric type: {} (start={}, end={}, diff={})'.format( '{} wrong metric type: {} (start={}, end={}, diff={})'.format(
met_name, MetricType.name(met_info.metric_type), start_val, met_name, MetricType.name(met_info.metric_type), start_val,
@ -397,9 +390,7 @@ class BaseParser:
def format_dbms_knobs(self, knobs): def format_dbms_knobs(self, knobs):
formatted_knobs = {} formatted_knobs = {}
for knob_name, knob_value in list(knobs.items()): for knob_name, knob_value in list(knobs.items()):
metadata = self.knob_catalog_.get(knob_name, None) metadata = KnobCatalog.objects.get(dbms__id=self.dbms_id, name=knob_name)
if (metadata is None):
raise Exception('Unknown knob {}'.format(knob_name))
fvalue = None fvalue = None
if metadata.vartype == VarType.BOOL: if metadata.vartype == VarType.BOOL:
fvalue = self.format_bool(knob_value, metadata) fvalue = self.format_bool(knob_value, metadata)

5
server/website/website/db/base/target_objective.py
View File

@ -94,8 +94,9 @@ class TargetObjectives:
self._registry[dbms_id][name] = target_objective_instance self._registry[dbms_id][name] = target_objective_instance
if dbms_id not in self._metric_metadatas: if dbms_id not in self._metric_metadatas:
numeric_metrics = models.MetricCatalog.objects.filter(dbms=dbms).exclude( numeric_metrics = models.MetricCatalog.objects.filter(
metric_type=types.MetricType.INFO).values_list('name', flat=True) dbms=dbms, metric_type__in=types.MetricType.numeric()).values_list(
'name', flat=True)
self._metric_metadatas[dbms_id] = [(mname, BaseMetric(mname)) for mname self._metric_metadatas[dbms_id] = [(mname, BaseMetric(mname)) for mname
in sorted(numeric_metrics)] in sorted(numeric_metrics)]

27
server/website/website/db/myrocks/parser.py
View File

@ -9,6 +9,7 @@ from collections import OrderedDict
from ..base.parser import BaseParser from ..base.parser import BaseParser
from .. import target_objectives from .. import target_objectives
from website.models import KnobCatalog, MetricCatalog
from website.types import MetricType, VarType from website.types import MetricType, VarType
@ -71,7 +72,7 @@ class MyRocksParser(BaseParser):
def extract_valid_variables(self, variables, catalog, default_value=None): def extract_valid_variables(self, variables, catalog, default_value=None):
valid_variables = {} valid_variables = {}
diff_log = [] diff_log = []
valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())} valid_lc_variables = {k.name.lower(): k for k in catalog}
# First check that the names of all variables are valid (i.e., listed # First check that the names of all variables are valid (i.e., listed
# in the official catalog). Invalid variables are logged as 'extras'. # in the official catalog). Invalid variables are logged as 'extras'.
@ -106,9 +107,10 @@ class MyRocksParser(BaseParser):
def calculate_change_in_metrics(self, metrics_start, metrics_end): def calculate_change_in_metrics(self, metrics_start, metrics_end):
adjusted_metrics = {} adjusted_metrics = {}
metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id)
for met_name, start_val in list(metrics_start.items()): for met_name, start_val in list(metrics_start.items()):
end_val = metrics_end[met_name] end_val = metrics_end[met_name]
met_info = self.metric_catalog_[self.partial_name(met_name)] met_info = metric_catalog.get(name=self.partial_name(met_name))
if met_info.vartype == VarType.INTEGER or \ if met_info.vartype == VarType.INTEGER or \
met_info.vartype == VarType.REAL: met_info.vartype == VarType.REAL:
conversion_fn = self.convert_integer if \ conversion_fn = self.convert_integer if \
@ -127,26 +129,28 @@ class MyRocksParser(BaseParser):
def parse_dbms_knobs(self, knobs): def parse_dbms_knobs(self, knobs):
valid_knobs = self.parse_dbms_variables(knobs) valid_knobs = self.parse_dbms_variables(knobs)
knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id)
# Extract all valid knobs # Extract all valid knobs
return self.extract_valid_variables( return self.extract_valid_variables(valid_knobs, knob_catalog)
valid_knobs, self.knob_catalog_)
def parse_dbms_metrics(self, metrics): def parse_dbms_metrics(self, metrics):
valid_metrics = self.parse_dbms_variables(metrics) valid_metrics = self.parse_dbms_variables(metrics)
metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id)
# Extract all valid metrics # Extract all valid metrics
valid_metrics, diffs = self.extract_valid_variables( valid_metrics, diffs = self.extract_valid_variables(
valid_metrics, self.metric_catalog_, default_value='0') valid_metrics, metric_catalog, default_value='0')
return valid_metrics, diffs return valid_metrics, diffs
def convert_dbms_metrics(self, metrics, observation_time, target_objective): def convert_dbms_metrics(self, metrics, observation_time, target_objective):
base_metric_data = {} base_metric_data = {}
metric_data = {} metric_data = {}
numeric_metric_catalog = MetricCatalog.objects.filter(
dbms__id=self.dbms_id, metric_type__in=MetricType.numeric())
for name, value in list(metrics.items()): for name, value in list(metrics.items()):
prt_name = self.partial_name(name) prt_name = self.partial_name(name)
metadata = numeric_metric_catalog.filter(name=prt_name).first()
if prt_name in self.numeric_metric_catalog_: if metadata:
metadata = self.numeric_metric_catalog_[prt_name]
if metadata.vartype == VarType.INTEGER: if metadata.vartype == VarType.INTEGER:
converted = float(self.convert_integer(value, metadata)) converted = float(self.convert_integer(value, metadata))
elif metadata.vartype == VarType.REAL: elif metadata.vartype == VarType.REAL:
@ -178,12 +182,11 @@ class MyRocksParser(BaseParser):
def convert_dbms_knobs(self, knobs): def convert_dbms_knobs(self, knobs):
knob_data = {} knob_data = {}
tunable_knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True)
for name, value in list(knobs.items()): for name, value in list(knobs.items()):
prt_name = self.partial_name(name) prt_name = self.partial_name(name)
if prt_name in self.tunable_knob_catalog_: metadata = tunable_knob_catalog.filter(name=prt_name).first()
metadata = self.tunable_knob_catalog_[prt_name] if metadata:
assert(metadata.tunable)
value = knobs[name]
conv_value = None conv_value = None
if metadata.vartype == VarType.BOOL: if metadata.vartype == VarType.BOOL:
conv_value = self.convert_bool(value, metadata) conv_value = self.convert_bool(value, metadata)

3
server/website/website/views.py
View File

@ -774,9 +774,8 @@ def dbms_data_view(request, context, dbms_data, session, target_obj):
'knob__name', flat=True)) 'knob__name', flat=True))
else: else:
model_class = MetricData model_class = MetricData
num_types = (MetricType.COUNTER, MetricType.STATISTICS)
featured_names = set(MetricCatalog.objects.filter( featured_names = set(MetricCatalog.objects.filter(
dbms=session.dbms, metric_type__in=num_types).values_list( dbms=session.dbms, metric_type__in=MetricCtype.numeric()).values_list(
'name', flat=True)) 'name', flat=True))
obj_data = getattr(dbms_data, data_type) obj_data = getattr(dbms_data, data_type)