diff --git a/server/website/tests/test_parser.py b/server/website/tests/test_parser.py index 8ea4555..83cafc0 100644 --- a/server/website/tests/test_parser.py +++ b/server/website/tests/test_parser.py @@ -9,7 +9,7 @@ import mock from django.test import TestCase from website.db import parser, target_objectives from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType -from website.models import DBMSCatalog, KnobCatalog +from website.models import DBMSCatalog, KnobCatalog, MetricCatalog class BaseParserTests(object, metaclass=ABCMeta): @@ -214,6 +214,11 @@ class PostgresParserTests(BaseParserTests, TestCase): dbms_obj = DBMSCatalog.objects.filter( type=DBMSType.POSTGRES, version="9.6").first() self.test_dbms = parser._get(dbms_obj.pk) # pylint: disable=protected-access + self.knob_catalog = KnobCatalog.objects.filter(dbms=dbms_obj) + self.tunable_knob_catalog = self.knob_catalog.filter(tunable=True) + self.metric_catalog = MetricCatalog.objects.filter(dbms=dbms_obj) + self.numeric_metric_catalog = self.metric_catalog.filter( + metric_type__in=MetricType.numeric()) def test_convert_dbms_knobs(self): super().test_convert_dbms_knobs() @@ -251,16 +256,17 @@ class PostgresParserTests(BaseParserTests, TestCase): txns_counter = target_obj_instance.transactions_counter test_metrics = {} + for met in self.numeric_metric_catalog: + test_metrics[met.name] = 2 - for key in list(self.test_dbms.numeric_metric_catalog_.keys()): - test_metrics[key] = 2 test_metrics[txns_counter] = 10 test_metrics['pg_FAKE_METRIC'] = 0 self.assertEqual(test_metrics.get(target_obj), None) test_convert_metrics = self.test_dbms.convert_dbms_metrics(test_metrics, 0.1, target_obj) - for key, metadata in list(self.test_dbms.numeric_metric_catalog_.items()): + for metadata in self.numeric_metric_catalog: + key = metadata.name if key == txns_counter: self.assertEqual(test_convert_metrics[key], 10 / 0.1) continue @@ -285,11 +291,11 @@ class PostgresParserTests(BaseParserTests, TestCase): self.test_dbms.parse_version_string("1.0") def test_extract_valid_variables(self): - num_tunable_knobs = len(list(self.test_dbms.tunable_knob_catalog_.keys())) + num_tunable_knobs = len(self.tunable_knob_catalog) test_empty, test_empty_diff = self.test_dbms.extract_valid_variables( - {}, self.test_dbms.tunable_knob_catalog_) - self.assertEqual(len(list(test_empty.keys())), num_tunable_knobs) + {}, self.tunable_knob_catalog) + self.assertEqual(len(test_empty), num_tunable_knobs) self.assertEqual(len(test_empty_diff), num_tunable_knobs) test_vars = {'global.wal_sync_method': 'fsync', @@ -303,7 +309,7 @@ class PostgresParserTests(BaseParserTests, TestCase): 'global.FAKE_KNOB': 'fake'} tune_extract, tune_diff = self.test_dbms.extract_valid_variables( - test_vars, self.test_dbms.tunable_knob_catalog_) + test_vars, self.tunable_knob_catalog) self.assertTrue(('miscapitalized', 'global.wal_buffers', 'global.Wal_buffers', 1024) in tune_diff) @@ -317,10 +323,10 @@ class PostgresParserTests(BaseParserTests, TestCase): self.assertEqual(tune_extract.get('global.wal_buffers'), 1024) self.assertEqual(tune_extract.get('global.Wal_buffers'), None) - self.assertEqual(len(tune_extract), len(self.test_dbms.tunable_knob_catalog_)) + self.assertEqual(len(tune_extract), num_tunable_knobs) nontune_extract, nontune_diff = self.test_dbms.extract_valid_variables( - test_vars, self.test_dbms.knob_catalog_) + test_vars, self.knob_catalog) self.assertTrue(('miscapitalized', 'global.wal_buffers', 'global.Wal_buffers', 1024) in nontune_diff) @@ -519,14 +525,14 @@ class PostgresParserTests(BaseParserTests, TestCase): 'cpu_tuple_cost': 0.55, 'force_parallel_mode': 'regress', 'FAKE_KNOB': 'fake'}}} + num_knobs = len(self.knob_catalog) - (test_parse_dict, test_parse_log) = self.test_dbms.parse_dbms_knobs(test_knobs) + test_parse_dict, test_parse_log = self.test_dbms.parse_dbms_knobs(test_knobs) - self.assertEqual(len(test_parse_log), len(list(self.test_dbms.knob_catalog_.keys())) - 7) + self.assertEqual(len(test_parse_log), num_knobs - 7) self.assertTrue(('extra', None, 'global.FAKE_KNOB', 'fake') in test_parse_log) - self.assertEqual(len(list(test_parse_dict.keys())), - len(list(self.test_dbms.knob_catalog_.keys()))) + self.assertEqual(len(test_parse_dict), num_knobs) self.assertEqual(test_parse_dict['global.wal_sync_method'], 'fsync') self.assertEqual(test_parse_dict['global.random_page_cost'], 0.22) @@ -552,8 +558,7 @@ class PostgresParserTests(BaseParserTests, TestCase): # Doesn't support table or index scope with self.assertRaises(Exception): + num_metrics = len(self.metric_catalog) test_parse_dict, test_parse_log = self.test_dbms.parse_dbms_metrics(test_metrics) - self.assertEqual(len(list(test_parse_dict.keys())), - len(list(self.test_dbms.metric_catalog_.keys()))) - self.assertEqual(len(test_parse_log), - len(list(self.test_dbms.metric_catalog_.keys())) - 14) + self.assertEqual(len(test_parse_dict), num_metrics) + self.assertEqual(len(test_parse_log), num_metrics - 14) diff --git a/server/website/website/db/base/parser.py b/server/website/website/db/base/parser.py index 8c9c667..99e433f 100644 --- a/server/website/website/db/base/parser.py +++ b/server/website/website/db/base/parser.py @@ -7,8 +7,8 @@ import logging from collections import OrderedDict -from website.models import KnobCatalog, KnobUnitType, MetricCatalog -from website.types import BooleanType, MetricType, VarType +from website.models import KnobCatalog, MetricCatalog +from website.types import BooleanType, KnobUnitType, MetricType, VarType from website.utils import ConversionUtil from .. import target_objectives @@ -20,18 +20,6 @@ class BaseParser: def __init__(self, dbms_obj): self.dbms_id = int(dbms_obj.pk) - knobs = KnobCatalog.objects.filter(dbms=dbms_obj) - self.knob_catalog_ = {k.name: k for k in knobs} - self.tunable_knob_catalog_ = { - k: v for k, v in self.knob_catalog_.items() if - v.tunable is True} - - metrics = MetricCatalog.objects.filter(dbms=dbms_obj) - self.metric_catalog_ = {m.name: m for m in metrics} - numeric_mtypes = (MetricType.COUNTER, MetricType.STATISTICS) - self.numeric_metric_catalog_ = { - m: v for m, v in self.metric_catalog_.items() if - v.metric_type in numeric_mtypes} self.valid_true_val = ("on", "true", "yes") self.valid_false_val = ("off", "false", "no") @@ -113,9 +101,9 @@ class BaseParser: def convert_dbms_knobs(self, knobs): knob_data = {} - for name, metadata in list(self.tunable_knob_catalog_.items()): - if metadata.tunable is False: - continue + tunable_knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True) + for metadata in tunable_knob_catalog: + name = metadata.name if name not in knobs: continue value = knobs[name] @@ -187,8 +175,11 @@ class BaseParser: metric_data = {} # Same as metric_data except COUNTER metrics are not divided by the time base_metric_data = {} + numeric_metric_catalog = MetricCatalog.objects.filter( + dbms__id=self.dbms_id, metric_type__in=MetricType.numeric()) - for name, metadata in self.numeric_metric_catalog_.items(): + for metadata in numeric_metric_catalog: + name = metadata.name value = metrics[name] if metadata.vartype == VarType.INTEGER: @@ -223,7 +214,7 @@ class BaseParser: def extract_valid_variables(self, variables, catalog, default_value=None): valid_variables = {} diff_log = [] - valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())} + valid_lc_variables = {k.name.lower(): k for k in catalog} # First check that the names of all variables are valid (i.e., listed # in the official catalog). Invalid variables are logged as 'extras'. @@ -286,7 +277,8 @@ class BaseParser: assert len(valid_knobs[k]) == 1 valid_knobs[k] = valid_knobs[k][0] # Extract all valid knobs - return self.extract_valid_variables(valid_knobs, self.knob_catalog_) + knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id) + return self.extract_valid_variables(valid_knobs, knob_catalog) def parse_dbms_metrics(self, metrics): # Some DBMSs measure different types of stats (e.g., global, local) @@ -295,16 +287,16 @@ class BaseParser: valid_metrics = self.parse_dbms_variables(metrics) # Extract all valid metrics + metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id) valid_metrics, diffs = self.extract_valid_variables( - valid_metrics, self.metric_catalog_, default_value='0') + valid_metrics, metric_catalog, default_value='0') # Combine values for name, values in list(valid_metrics.items()): - metric = self.metric_catalog_[name] - if metric.metric_type == MetricType.INFO or len(values) == 1: + metric = metric_catalog.get(name=name) + if metric.metric_type in MetricType.nonnumeric() or len(values) == 1: valid_metrics[name] = values[0] - elif metric.metric_type == MetricType.COUNTER or \ - metric.metric_type == MetricType.STATISTICS: + elif metric.metric_type in MetricType.numeric(): conv_fn = int if metric.vartype == VarType.INTEGER else float values = [conv_fn(v) for v in values if v is not None] if len(values) == 0: @@ -317,10 +309,11 @@ class BaseParser: return valid_metrics, diffs def calculate_change_in_metrics(self, metrics_start, metrics_end, fix_metric_type=True): + metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id) adjusted_metrics = {} for met_name, start_val in list(metrics_start.items()): end_val = metrics_end[met_name] - met_info = self.metric_catalog_[met_name] + met_info = metric_catalog.get(name=met_name) if met_info.vartype == VarType.INTEGER or \ met_info.vartype == VarType.REAL: conversion_fn = self.convert_integer if \ @@ -332,13 +325,13 @@ class BaseParser: adj_val = end_val - start_val else: # MetricType.STATISTICS or MetricType.INFO adj_val = end_val + if fix_metric_type: if adj_val < 0: adj_val = end_val LOG.debug("Changing metric %s from COUNTER to STATISTICS", met_name) - metric_fixed = self.metric_catalog_[met_name] - metric_fixed.metric_type = MetricType.STATISTICS - metric_fixed.save() + met_info.metric_type = MetricType.STATISTICS + met_info.save() assert adj_val >= 0, \ '{} wrong metric type: {} (start={}, end={}, diff={})'.format( met_name, MetricType.name(met_info.metric_type), start_val, @@ -397,9 +390,7 @@ class BaseParser: def format_dbms_knobs(self, knobs): formatted_knobs = {} for knob_name, knob_value in list(knobs.items()): - metadata = self.knob_catalog_.get(knob_name, None) - if (metadata is None): - raise Exception('Unknown knob {}'.format(knob_name)) + metadata = KnobCatalog.objects.get(dbms__id=self.dbms_id, name=knob_name) fvalue = None if metadata.vartype == VarType.BOOL: fvalue = self.format_bool(knob_value, metadata) diff --git a/server/website/website/db/base/target_objective.py b/server/website/website/db/base/target_objective.py index 21c1e3d..84e9030 100644 --- a/server/website/website/db/base/target_objective.py +++ b/server/website/website/db/base/target_objective.py @@ -94,8 +94,9 @@ class TargetObjectives: self._registry[dbms_id][name] = target_objective_instance if dbms_id not in self._metric_metadatas: - numeric_metrics = models.MetricCatalog.objects.filter(dbms=dbms).exclude( - metric_type=types.MetricType.INFO).values_list('name', flat=True) + numeric_metrics = models.MetricCatalog.objects.filter( + dbms=dbms, metric_type__in=types.MetricType.numeric()).values_list( + 'name', flat=True) self._metric_metadatas[dbms_id] = [(mname, BaseMetric(mname)) for mname in sorted(numeric_metrics)] diff --git a/server/website/website/db/myrocks/parser.py b/server/website/website/db/myrocks/parser.py index 435a390..08fe432 100644 --- a/server/website/website/db/myrocks/parser.py +++ b/server/website/website/db/myrocks/parser.py @@ -9,6 +9,7 @@ from collections import OrderedDict from ..base.parser import BaseParser from .. import target_objectives +from website.models import KnobCatalog, MetricCatalog from website.types import MetricType, VarType @@ -71,7 +72,7 @@ class MyRocksParser(BaseParser): def extract_valid_variables(self, variables, catalog, default_value=None): valid_variables = {} diff_log = [] - valid_lc_variables = {k.lower(): v for k, v in list(catalog.items())} + valid_lc_variables = {k.name.lower(): k for k in catalog} # First check that the names of all variables are valid (i.e., listed # in the official catalog). Invalid variables are logged as 'extras'. @@ -106,9 +107,10 @@ class MyRocksParser(BaseParser): def calculate_change_in_metrics(self, metrics_start, metrics_end): adjusted_metrics = {} + metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id) for met_name, start_val in list(metrics_start.items()): end_val = metrics_end[met_name] - met_info = self.metric_catalog_[self.partial_name(met_name)] + met_info = metric_catalog.get(name=self.partial_name(met_name)) if met_info.vartype == VarType.INTEGER or \ met_info.vartype == VarType.REAL: conversion_fn = self.convert_integer if \ @@ -127,26 +129,28 @@ class MyRocksParser(BaseParser): def parse_dbms_knobs(self, knobs): valid_knobs = self.parse_dbms_variables(knobs) + knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id) # Extract all valid knobs - return self.extract_valid_variables( - valid_knobs, self.knob_catalog_) + return self.extract_valid_variables(valid_knobs, knob_catalog) def parse_dbms_metrics(self, metrics): valid_metrics = self.parse_dbms_variables(metrics) + metric_catalog = MetricCatalog.objects.filter(dbms__id=self.dbms_id) # Extract all valid metrics valid_metrics, diffs = self.extract_valid_variables( - valid_metrics, self.metric_catalog_, default_value='0') + valid_metrics, metric_catalog, default_value='0') return valid_metrics, diffs def convert_dbms_metrics(self, metrics, observation_time, target_objective): base_metric_data = {} metric_data = {} + numeric_metric_catalog = MetricCatalog.objects.filter( + dbms__id=self.dbms_id, metric_type__in=MetricType.numeric()) for name, value in list(metrics.items()): prt_name = self.partial_name(name) + metadata = numeric_metric_catalog.filter(name=prt_name).first() - if prt_name in self.numeric_metric_catalog_: - metadata = self.numeric_metric_catalog_[prt_name] - + if metadata: if metadata.vartype == VarType.INTEGER: converted = float(self.convert_integer(value, metadata)) elif metadata.vartype == VarType.REAL: @@ -178,12 +182,11 @@ class MyRocksParser(BaseParser): def convert_dbms_knobs(self, knobs): knob_data = {} + tunable_knob_catalog = KnobCatalog.objects.filter(dbms__id=self.dbms_id, tunable=True) for name, value in list(knobs.items()): prt_name = self.partial_name(name) - if prt_name in self.tunable_knob_catalog_: - metadata = self.tunable_knob_catalog_[prt_name] - assert(metadata.tunable) - value = knobs[name] + metadata = tunable_knob_catalog.filter(name=prt_name).first() + if metadata: conv_value = None if metadata.vartype == VarType.BOOL: conv_value = self.convert_bool(value, metadata) diff --git a/server/website/website/views.py b/server/website/website/views.py index d9a3763..02160bf 100644 --- a/server/website/website/views.py +++ b/server/website/website/views.py @@ -774,9 +774,8 @@ def dbms_data_view(request, context, dbms_data, session, target_obj): 'knob__name', flat=True)) else: model_class = MetricData - num_types = (MetricType.COUNTER, MetricType.STATISTICS) featured_names = set(MetricCatalog.objects.filter( - dbms=session.dbms, metric_type__in=num_types).values_list( + dbms=session.dbms, metric_type__in=MetricCtype.numeric()).values_list( 'name', flat=True)) obj_data = getattr(dbms_data, data_type)