public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Parser cleanup

This commit is contained in:
Dana Van Aken 2019-08-27 19:21:10 -04:00
parent aa01f17eae
commit 7aea07f4c1
8 changed files with 149 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
server/website/tests/test_parser.py
View File

@ -7,9 +7,9 @@
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
import mock import mock
from django.test import TestCase from django.test import TestCase
from website.parser.postgres import PostgresParser, Postgres96Parser from website.parser.postgres import PostgresParser
from website.types import BooleanType, VarType, KnobUnitType, MetricType from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType
from website.models import KnobCatalog from website.models import DBMSCatalog, KnobCatalog
class BaseParserTests(object, metaclass=ABCMeta): class BaseParserTests(object, metaclass=ABCMeta):
@ -161,8 +161,10 @@ class BaseParserTests(object, metaclass=ABCMeta):
mock_other_knob = mock.Mock(spec=KnobCatalog) mock_other_knob = mock.Mock(spec=KnobCatalog)
mock_other_knob.unit = KnobUnitType.OTHER mock_other_knob.unit = KnobUnitType.OTHER
self.assertEqual(self.test_dbms.format_bool(BooleanType.TRUE, mock_other_knob), 'on') self.assertEqual(self.test_dbms.format_bool(BooleanType.TRUE, mock_other_knob),
self.assertEqual(self.test_dbms.format_bool(BooleanType.FALSE, mock_other_knob), 'off') self.test_dbms.true_value)
self.assertEqual(self.test_dbms.format_bool(BooleanType.FALSE, mock_other_knob),
self.test_dbms.false_value)
def test_format_enum(self): def test_format_enum(self):
mock_enum_knob = mock.Mock(spec=KnobCatalog) mock_enum_knob = mock.Mock(spec=KnobCatalog)
@ -217,13 +219,15 @@ class BaseParserTests(object, metaclass=ABCMeta):
pass pass
class Postgres96ParserTests(BaseParserTests, TestCase): class PostgresParserTests(BaseParserTests, TestCase):
def setUp(self): def setUp(self):
self.test_dbms = Postgres96Parser(9.6) dbms_obj = DBMSCatalog.objects.filter(
type=DBMSType.POSTGRES, version="9.6").first()
self.test_dbms = PostgresParser(dbms_obj)
def test_convert_dbms_knobs(self): def test_convert_dbms_knobs(self):
super(Postgres96ParserTests, self).test_convert_dbms_knobs() super().test_convert_dbms_knobs()
test_knobs = {'global.wal_sync_method': 'open_sync', # Enum test_knobs = {'global.wal_sync_method': 'open_sync', # Enum
'global.random_page_cost': 0.22, # Real 'global.random_page_cost': 0.22, # Real
@ -250,7 +254,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
self.assertEqual(self.test_dbms.convert_dbms_knobs(test_nontune_knobs), {}) self.assertEqual(self.test_dbms.convert_dbms_knobs(test_nontune_knobs), {})
def test_convert_dbms_metrics(self): def test_convert_dbms_metrics(self):
super(Postgres96ParserTests, self).test_convert_dbms_metrics() super().test_convert_dbms_metrics()
test_metrics = {} test_metrics = {}
@ -361,7 +365,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
self.assertEqual(nontune_extract.get('global.GEQO_EFFORT'), None) self.assertEqual(nontune_extract.get('global.GEQO_EFFORT'), None)
def test_convert_integer(self): def test_convert_integer(self):
super(Postgres96ParserTests, self).test_convert_integer() super().test_convert_integer()
# Convert Integer # Convert Integer
knob_unit_bytes = KnobUnitType() knob_unit_bytes = KnobUnitType()
@ -387,6 +391,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
self.assertEqual(self.test_dbms self.assertEqual(self.test_dbms
.convert_integer('10min', knob_unit_time), 600000) .convert_integer('10min', knob_unit_time), 600000)
self.assertEqual(self.test_dbms.convert_integer('1s', knob_unit_time), 1000) self.assertEqual(self.test_dbms.convert_integer('1s', knob_unit_time), 1000)
self.assertEqual(self.test_dbms.convert_integer('5000ms', knob_unit_time), 5000)
test_exceptions = [('A', knob_unit_other), test_exceptions = [('A', knob_unit_other),
('', knob_unit_other), ('', knob_unit_other),
@ -400,7 +405,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
self.test_dbms.convert_integer(failure_case, knob_unit) self.test_dbms.convert_integer(failure_case, knob_unit)
def test_calculate_change_in_metrics(self): def test_calculate_change_in_metrics(self):
super(Postgres96ParserTests, self).test_calculate_change_in_metrics() super().test_calculate_change_in_metrics()
test_metric_start = {'pg_stat_bgwriter.buffers_alloc': 256, test_metric_start = {'pg_stat_bgwriter.buffers_alloc': 256,
'pg_stat_archiver.last_failed_wal': "today", 'pg_stat_archiver.last_failed_wal': "today",
@ -473,8 +478,6 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
self.assertEqual(test_config.get(k), v) self.assertEqual(test_config.get(k), v)
def test_format_integer(self): def test_format_integer(self):
test_dbms = PostgresParser(2)
knob_unit_bytes = KnobUnitType() knob_unit_bytes = KnobUnitType()
knob_unit_bytes.unit = 1 knob_unit_bytes.unit = 1
knob_unit_time = KnobUnitType() knob_unit_time = KnobUnitType()
@ -482,22 +485,22 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
knob_unit_other = KnobUnitType() knob_unit_other = KnobUnitType()
knob_unit_other.unit = 3 knob_unit_other.unit = 3
self.assertEqual(test_dbms.format_integer(5, knob_unit_other), 5) self.assertEqual(self.test_dbms.format_integer(5, knob_unit_other), 5)
self.assertEqual(test_dbms.format_integer(0, knob_unit_other), 0) self.assertEqual(self.test_dbms.format_integer(0, knob_unit_other), 0)
self.assertEqual(test_dbms.format_integer(-1, knob_unit_other), -1) self.assertEqual(self.test_dbms.format_integer(-1, knob_unit_other), -1)
self.assertEqual(test_dbms.format_integer(5120, knob_unit_bytes), '5kB') self.assertEqual(self.test_dbms.format_integer(5120, knob_unit_bytes), '5kB')
self.assertEqual(test_dbms.format_integer(4194304, knob_unit_bytes), '4MB') self.assertEqual(self.test_dbms.format_integer(4194304, knob_unit_bytes), '4MB')
self.assertEqual(test_dbms.format_integer(4194500, knob_unit_bytes), '4MB') self.assertEqual(self.test_dbms.format_integer(4194500, knob_unit_bytes), '4MB')
self.assertEqual(test_dbms.format_integer(86400000, knob_unit_time), '1d') self.assertEqual(self.test_dbms.format_integer(86400000, knob_unit_time), '1d')
self.assertEqual(test_dbms.format_integer(72000000, knob_unit_time), '20h') self.assertEqual(self.test_dbms.format_integer(72000000, knob_unit_time), '20h')
self.assertEqual(test_dbms.format_integer(600000, knob_unit_time), '10min') self.assertEqual(self.test_dbms.format_integer(600000, knob_unit_time), '10min')
self.assertEqual(test_dbms.format_integer(1000, knob_unit_time), '1s') self.assertEqual(self.test_dbms.format_integer(1000, knob_unit_time), '1s')
self.assertEqual(test_dbms.format_integer(500, knob_unit_time), '500ms') self.assertEqual(self.test_dbms.format_integer(500, knob_unit_time), '500ms')
def test_format_dbms_knobs(self): def test_format_dbms_knobs(self):
super(Postgres96ParserTests, self).test_format_dbms_knobs() super().test_format_dbms_knobs()
test_knobs = {'global.wal_sync_method': 2, # Enum test_knobs = {'global.wal_sync_method': 2, # Enum
'global.random_page_cost': 0.22, # Real 'global.random_page_cost': 0.22, # Real
@ -520,7 +523,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
self.assertEqual(test_formatted_knobs.get('global.wal_buffers'), '1kB') self.assertEqual(test_formatted_knobs.get('global.wal_buffers'), '1kB')
def test_filter_numeric_metrics(self): def test_filter_numeric_metrics(self):
super(Postgres96ParserTests, self).test_filter_numeric_metrics() super().test_filter_numeric_metrics()
test_metrics = {'pg_stat_bgwriter.checkpoints_req': (2, 'global'), test_metrics = {'pg_stat_bgwriter.checkpoints_req': (2, 'global'),
'pg_stat_archiver.last_failed_wal': (1, 'global'), 'pg_stat_archiver.last_failed_wal': (1, 'global'),
@ -549,7 +552,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
self.assertEqual(filtered_metrics.get('pg_FAKE_KNOB'), None) self.assertEqual(filtered_metrics.get('pg_FAKE_KNOB'), None)
def test_filter_tunable_knobs(self): def test_filter_tunable_knobs(self):
super(Postgres96ParserTests, self).test_filter_tunable_knobs() super().test_filter_tunable_knobs()
test_knobs = {'global.wal_sync_method': 5, test_knobs = {'global.wal_sync_method': 5,
'global.random_page_cost': 3, 'global.random_page_cost': 3,
@ -571,7 +574,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
self.assertEqual(filtered_knobs.get('global.FAKE_KNOB'), None) self.assertEqual(filtered_knobs.get('global.FAKE_KNOB'), None)
def test_parse_helper(self): def test_parse_helper(self):
super(Postgres96ParserTests, self).test_parse_helper() super().test_parse_helper()
test_view_vars = {'global': {'wal_sync_method': 'open_sync', test_view_vars = {'global': {'wal_sync_method': 'open_sync',
'random_page_cost': 0.22}, 'random_page_cost': 0.22},

17
server/website/tests/test_utils.py
View File

@ -8,7 +8,6 @@ import string
import numpy as np import numpy as np
from django.test import TestCase from django.test import TestCase
from website.utils import JSONUtil, MediaUtil, DataUtil, ConversionUtil, LabelUtil, TaskUtil from website.utils import JSONUtil, MediaUtil, DataUtil, ConversionUtil, LabelUtil, TaskUtil
from website.parser.postgres import PostgresParser
from website.types import LabelStyleType, VarType from website.types import LabelStyleType, VarType
from website.models import Result, DBMSCatalog from website.models import Result, DBMSCatalog
@ -225,9 +224,9 @@ class DataUtilTest(TestCase):
'global.bgwriter_delay', 'global.bgwriter_delay',
'global.wal_writer_delay', 'global.wal_writer_delay',
'global.work_mem'] 'global.work_mem']
postgres96 = DBMSCatalog.objects.get(pk=1) postgresdb = DBMSCatalog.objects.get(pk=1)
categorical_info = DataUtil.dummy_encoder_helper(featured_knobs, categorical_info = DataUtil.dummy_encoder_helper(featured_knobs,
dbms=postgres96) dbms=postgresdb)
self.assertEqual(len(categorical_info['n_values']), 0) self.assertEqual(len(categorical_info['n_values']), 0)
self.assertEqual(len(categorical_info['categorical_features']), 0) self.assertEqual(len(categorical_info['categorical_features']), 0)
self.assertEqual(categorical_info['cat_columnlabels'], []) self.assertEqual(categorical_info['cat_columnlabels'], [])
@ -239,9 +238,9 @@ class DataUtilTest(TestCase):
'global.wal_writer_delay', 'global.wal_writer_delay',
'global.work_mem', 'global.work_mem',
'global.wal_sync_method'] # last knob categorical 'global.wal_sync_method'] # last knob categorical
postgres96 = DBMSCatalog.objects.get(pk=1) postgresdb = DBMSCatalog.objects.get(pk=1)
categorical_info = DataUtil.dummy_encoder_helper(featured_knobs, categorical_info = DataUtil.dummy_encoder_helper(featured_knobs,
dbms=postgres96) dbms=postgresdb)
self.assertEqual(len(categorical_info['n_values']), 1) self.assertEqual(len(categorical_info['n_values']), 1)
self.assertEqual(categorical_info['n_values'][0], 4) self.assertEqual(categorical_info['n_values'][0], 4)
self.assertEqual(len(categorical_info['categorical_features']), 1) self.assertEqual(len(categorical_info['categorical_features']), 1)
@ -257,7 +256,7 @@ class ConversionUtilTest(TestCase):
byte_ans = [1024**5, 2 * 1024**4, 3 * 1024**3, 4 * 1024**2, 5 * 1024**1, 6] byte_ans = [1024**5, 2 * 1024**4, 3 * 1024**3, 4 * 1024**2, 5 * 1024**1, 6]
for i, byte_test in enumerate(byte_test_convert): for i, byte_test in enumerate(byte_test_convert):
byte_conversion = ConversionUtil.get_raw_size( byte_conversion = ConversionUtil.get_raw_size(
byte_test, system=PostgresParser.POSTGRES_BYTES_SYSTEM) byte_test, system=ConversionUtil.DEFAULT_BYTES_SYSTEM)
self.assertEqual(byte_conversion, byte_ans[i]) self.assertEqual(byte_conversion, byte_ans[i])
# Time - In Milliseconds # Time - In Milliseconds
@ -265,7 +264,7 @@ class ConversionUtilTest(TestCase):
day_ans = [1000, 1000, 600000, 72000000, 86400000] day_ans = [1000, 1000, 600000, 72000000, 86400000]
for i, day_test in enumerate(day_test_convert): for i, day_test in enumerate(day_test_convert):
day_conversion = ConversionUtil.get_raw_size( day_conversion = ConversionUtil.get_raw_size(
day_test, system=PostgresParser.POSTGRES_TIME_SYSTEM) day_test, system=ConversionUtil.DEFAULT_TIME_SYSTEM)
self.assertEqual(day_conversion, day_ans[i]) self.assertEqual(day_conversion, day_ans[i])
def test_get_human_readable(self): def test_get_human_readable(self):
@ -275,7 +274,7 @@ class ConversionUtilTest(TestCase):
byte_ans = ['1PB', '2TB', '3GB', '4MB', '5kB', '6B'] byte_ans = ['1PB', '2TB', '3GB', '4MB', '5kB', '6B']
for i, byte_test in enumerate(byte_test_convert): for i, byte_test in enumerate(byte_test_convert):
byte_readable = ConversionUtil.get_human_readable( byte_readable = ConversionUtil.get_human_readable(
byte_test, system=PostgresParser.POSTGRES_BYTES_SYSTEM) byte_test, system=ConversionUtil.DEFAULT_BYTES_SYSTEM)
self.assertEqual(byte_readable, byte_ans[i]) self.assertEqual(byte_readable, byte_ans[i])
# Time # Time
@ -283,7 +282,7 @@ class ConversionUtilTest(TestCase):
day_ans = ['500ms', '1s', '55s', '10min', '20h', '1d'] day_ans = ['500ms', '1s', '55s', '10min', '20h', '1d']
for i, day_test in enumerate(day_test_convert): for i, day_test in enumerate(day_test_convert):
day_readable = ConversionUtil.get_human_readable( day_readable = ConversionUtil.get_human_readable(
day_test, system=PostgresParser.POSTGRES_TIME_SYSTEM) day_test, system=ConversionUtil.DEFAULT_TIME_SYSTEM)
self.assertEqual(day_readable, day_ans[i]) self.assertEqual(day_readable, day_ans[i])

84
server/website/website/parser/base.py
View File

@ -3,13 +3,6 @@
# #
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
'''
Created on Dec 12, 2017
@author: dvanaken
Parser interface.
'''
from abc import ABCMeta, abstractmethod, abstractproperty from abc import ABCMeta, abstractmethod, abstractproperty
from collections import OrderedDict from collections import OrderedDict
@ -21,20 +14,24 @@ from website.types import BooleanType, MetricType, VarType
# pylint: disable=no-self-use # pylint: disable=no-self-use
class BaseParser(object, metaclass=ABCMeta): class BaseParser(object, metaclass=ABCMeta):
def __init__(self, dbms_id): def __init__(self, dbms_obj):
self.dbms_id_ = dbms_id knobs = KnobCatalog.objects.filter(dbms=dbms_obj)
knobs = KnobCatalog.objects.filter(dbms__pk=self.dbms_id_)
self.knob_catalog_ = {k.name: k for k in knobs} self.knob_catalog_ = {k.name: k for k in knobs}
self.tunable_knob_catalog_ = {k: v for k, v in self.tunable_knob_catalog_ = {
list(self.knob_catalog_.items()) if v.tunable is True} k: v for k, v in self.knob_catalog_.items() if
metrics = MetricCatalog.objects.filter(dbms__pk=self.dbms_id_) v.tunable is True}
metrics = MetricCatalog.objects.filter(dbms=dbms_obj)
self.metric_catalog_ = {m.name: m for m in metrics} self.metric_catalog_ = {m.name: m for m in metrics}
self.numeric_metric_catalog_ = {m: v for m, v in numeric_mtypes = (MetricType.COUNTER, MetricType.STATISTICS)
list(self.metric_catalog_.items()) if self.numeric_metric_catalog_ = {
v.metric_type == MetricType.COUNTER or m: v for m, v in self.metric_catalog_.items() if
v.metric_type == MetricType.STATISTICS} v.metric_type in numeric_mtypes}
self.valid_true_val = list()
self.valid_false_val = list() self.valid_true_val = ("on", "true", "yes")
self.valid_false_val = ("off", "false", "no")
self.true_value = 'on'
self.false_value = 'off'
@abstractproperty @abstractproperty
def base_configuration_settings(self): def base_configuration_settings(self):
@ -55,38 +52,49 @@ class BaseParser(object, metaclass=ABCMeta):
def target_metric(self, target_objective=None): def target_metric(self, target_objective=None):
if target_objective == 'throughput_txn_per_sec' or target_objective is None: if target_objective == 'throughput_txn_per_sec' or target_objective is None:
# throughput # throughput
return self.transactions_counter res = self.transactions_counter
elif target_objective == '99th_lat_ms': elif target_objective == '99th_lat_ms':
# 99 percentile latency # 99 percentile latency
return self.latency_timer res = self.latency_timer
else: else:
raise Exception("Target Objective {} Not Supported".format(target_objective)) raise Exception("Target Objective {} Not Supported".format(target_objective))
return res
@abstractmethod @abstractmethod
def parse_version_string(self, version_string): def parse_version_string(self, version_string):
pass pass
def convert_bool(self, bool_value, metadata): def convert_bool(self, bool_value, metadata):
if isinstance(bool_value, str):
bool_value = bool_value.lower()
if bool_value in self.valid_true_val: if bool_value in self.valid_true_val:
return BooleanType.TRUE res = BooleanType.TRUE
elif bool_value in self.valid_false_val: elif bool_value in self.valid_false_val:
return BooleanType.FALSE res = BooleanType.FALSE
else: else:
raise Exception("Invalid Boolean {}".format(bool_value)) raise Exception("Invalid Boolean {}".format(bool_value))
return res
def convert_enum(self, enum_value, metadata): def convert_enum(self, enum_value, metadata):
enumvals = metadata.enumvals.split(',') enumvals = metadata.enumvals.split(',')
try: try:
return enumvals.index(enum_value) res = enumvals.index(enum_value)
except ValueError: except ValueError:
raise Exception('Invalid enum value for variable {} ({})'.format( raise Exception('Invalid enum value for variable {} ({})'.format(
metadata.name, enum_value)) metadata.name, enum_value))
return res
def convert_integer(self, int_value, metadata): def convert_integer(self, int_value, metadata):
try: try:
return int(int_value) res = int(int_value)
except ValueError: except ValueError:
return int(float(int_value)) res = int(float(int_value))
return res
def convert_real(self, real_value, metadata): def convert_real(self, real_value, metadata):
return float(real_value) return float(real_value)
@ -115,6 +123,7 @@ class BaseParser(object, metaclass=ABCMeta):
continue continue
value = knobs[name] value = knobs[name]
conv_value = None conv_value = None
if metadata.vartype == VarType.BOOL: if metadata.vartype == VarType.BOOL:
if not self._check_knob_bool_val(value): if not self._check_knob_bool_val(value):
raise Exception('Knob boolean value not valid! ' raise Exception('Knob boolean value not valid! '
@ -123,8 +132,10 @@ class BaseParser(object, metaclass=ABCMeta):
.format(self.valid_boolean_val_to_string(), .format(self.valid_boolean_val_to_string(),
str(value))) str(value)))
conv_value = self.convert_bool(value, metadata) conv_value = self.convert_bool(value, metadata)
elif metadata.vartype == VarType.ENUM: elif metadata.vartype == VarType.ENUM:
conv_value = self.convert_enum(value, metadata) conv_value = self.convert_enum(value, metadata)
elif metadata.vartype == VarType.INTEGER: elif metadata.vartype == VarType.INTEGER:
conv_value = self.convert_integer(value, metadata) conv_value = self.convert_integer(value, metadata)
if not self._check_knob_num_in_range(conv_value, metadata): if not self._check_knob_num_in_range(conv_value, metadata):
@ -132,6 +143,7 @@ class BaseParser(object, metaclass=ABCMeta):
'min: {}, max: {}, actual: {}' 'min: {}, max: {}, actual: {}'
.format(metadata.minval, .format(metadata.minval,
metadata.maxval, str(conv_value))) metadata.maxval, str(conv_value)))
elif metadata.vartype == VarType.REAL: elif metadata.vartype == VarType.REAL:
conv_value = self.convert_real(value, metadata) conv_value = self.convert_real(value, metadata)
if not self._check_knob_num_in_range(conv_value, metadata): if not self._check_knob_num_in_range(conv_value, metadata):
@ -139,23 +151,29 @@ class BaseParser(object, metaclass=ABCMeta):
'min: {}, max: {}, actual: {}' 'min: {}, max: {}, actual: {}'
.format(metadata.minval, .format(metadata.minval,
metadata.maxval, str(conv_value))) metadata.maxval, str(conv_value)))
elif metadata.vartype == VarType.STRING: elif metadata.vartype == VarType.STRING:
conv_value = self.convert_string(value, metadata) conv_value = self.convert_string(value, metadata)
elif metadata.vartype == VarType.TIMESTAMP: elif metadata.vartype == VarType.TIMESTAMP:
conv_value = self.convert_timestamp(value, metadata) conv_value = self.convert_timestamp(value, metadata)
else: else:
raise Exception( raise Exception(
'Unknown variable type: {}'.format(metadata.vartype)) 'Unknown variable type: {}'.format(metadata.vartype))
if conv_value is None: if conv_value is None:
raise Exception( raise Exception('Param value for {} cannot be null'.format(name))
'Param value for {} cannot be null'.format(name))
knob_data[name] = conv_value knob_data[name] = conv_value
return knob_data return knob_data
def _check_knob_num_in_range(self, value, mdata): def _check_knob_num_in_range(self, value, mdata):
return value >= float(mdata.minval) and value <= float(mdata.maxval) return value >= float(mdata.minval) and value <= float(mdata.maxval)
def _check_knob_bool_val(self, value): def _check_knob_bool_val(self, value):
if isinstance(str, value):
value = value.lower()
return value in self.valid_true_val or value in self.valid_false_val return value in self.valid_true_val or value in self.valid_false_val
def convert_dbms_metrics(self, metrics, observation_time, target_objective=None): def convert_dbms_metrics(self, metrics, observation_time, target_objective=None):
@ -331,7 +349,7 @@ class BaseParser(object, metaclass=ABCMeta):
return nondefault_settings return nondefault_settings
def format_bool(self, bool_value, metadata): def format_bool(self, bool_value, metadata):
return 'on' if bool_value == BooleanType.TRUE else 'off' return self.true_value if bool_value == BooleanType.TRUE else self.false_value
def format_enum(self, enum_value, metadata): def format_enum(self, enum_value, metadata):
enumvals = metadata.enumvals.split(',') enumvals = metadata.enumvals.split(',')
@ -378,11 +396,11 @@ class BaseParser(object, metaclass=ABCMeta):
return formatted_knobs return formatted_knobs
def filter_numeric_metrics(self, metrics): def filter_numeric_metrics(self, metrics):
return OrderedDict([(k, v) for k, v in list(metrics.items()) if return OrderedDict(((k, v) for k, v in list(metrics.items()) if
k in self.numeric_metric_catalog_]) k in self.numeric_metric_catalog_))
def filter_tunable_knobs(self, knobs): def filter_tunable_knobs(self, knobs):
return OrderedDict([(k, v) for k, v in list(knobs.items()) if return OrderedDict(((k, v) for k, v in list(knobs.items()) if
k in self.tunable_knob_catalog_]) k in self.tunable_knob_catalog_))
# pylint: enable=no-self-use # pylint: enable=no-self-use

46
server/website/website/parser/myrocks.py
View File

@ -3,40 +3,17 @@
# #
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
'''
Created on Jan 16, 2018
@author: bohan
'''
import re import re
from collections import OrderedDict from collections import OrderedDict
from .base import BaseParser from .base import BaseParser
from website.models import DBMSCatalog from website.types import KnobUnitType, MetricType, VarType
from website.types import DBMSType, KnobUnitType, MetricType, VarType
from website.utils import ConversionUtil from website.utils import ConversionUtil
class MyRocksParser(BaseParser): class MyRocksParser(BaseParser):
MYROCKS_BYTES_SYSTEM = [
(1024 ** 5, 'PB'),
(1024 ** 4, 'TB'),
(1024 ** 3, 'GB'),
(1024 ** 2, 'MB'),
(1024 ** 1, 'kB'),
(1024 ** 0, 'B'),
]
MYROCKS_TIME_SYSTEM = [
(1000 * 60 * 60 * 24, 'd'),
(1000 * 60 * 60, 'h'),
(1000 * 60, 'min'),
(1, 'ms'),
(1000, 's'),
]
MYROCKS_BASE_KNOBS = { MYROCKS_BASE_KNOBS = {
'session_variables.rocksdb_max_open_files': '-1' 'session_variables.rocksdb_max_open_files': '-1'
} }
@ -59,15 +36,15 @@ class MyRocksParser(BaseParser):
def convert_integer(self, int_value, metadata): def convert_integer(self, int_value, metadata):
converted = None converted = None
try: try:
converted = super(MyRocksParser, self).convert_integer( converted = super().convert_integer(
int_value, metadata) int_value, metadata)
except ValueError: except ValueError:
if metadata.unit == KnobUnitType.BYTES: if metadata.unit == KnobUnitType.BYTES:
converted = ConversionUtil.get_raw_size( converted = ConversionUtil.get_raw_size(
int_value, system=self.MYROCKS_BYTES_SYSTEM) int_value, system=ConversionUtil.DEFAULT_BYTES_SYSTEM)
elif metadata.unit == KnobUnitType.MILLISECONDS: elif metadata.unit == KnobUnitType.MILLISECONDS:
converted = ConversionUtil.get_raw_size( converted = ConversionUtil.get_raw_size(
int_value, system=self.MYROCKS_TIME_SYSTEM) int_value, system=ConversionUtil.DEFAULT_TIME_SYSTEM)
else: else:
raise Exception('Unknown unit type: {}'.format(metadata.unit)) raise Exception('Unknown unit type: {}'.format(metadata.unit))
if converted is None: if converted is None:
@ -79,16 +56,15 @@ class MyRocksParser(BaseParser):
if metadata.unit != KnobUnitType.OTHER and int_value > 0: if metadata.unit != KnobUnitType.OTHER and int_value > 0:
if metadata.unit == KnobUnitType.BYTES: if metadata.unit == KnobUnitType.BYTES:
int_value = ConversionUtil.get_human_readable( int_value = ConversionUtil.get_human_readable(
int_value, MyRocksParser.MYROCKS_BYTES_SYSTEM) int_value, ConversionUtil.DEFAULT_BYTES_SYSTEM)
elif metadata.unit == KnobUnitType.MILLISECONDS: elif metadata.unit == KnobUnitType.MILLISECONDS:
int_value = ConversionUtil.get_human_readable( int_value = ConversionUtil.get_human_readable(
int_value, MyRocksParser.MYROCKS_TIME_SYSTEM) int_value, ConversionUtil.DEFAULT_TIME_SYSTEM)
else: else:
raise Exception('Invalid unit type for {}: {}'.format( raise Exception('Invalid unit type for {}: {}'.format(
metadata.name, metadata.unit)) metadata.name, metadata.unit))
else: else:
int_value = super(MyRocksParser, self).format_integer( int_value = super().format_integer(int_value, metadata)
int_value, metadata)
return int_value return int_value
def parse_version_string(self, version_string): def parse_version_string(self, version_string):
@ -277,11 +253,3 @@ class MyRocksParser(BaseParser):
def filter_tunable_knobs(self, knobs): def filter_tunable_knobs(self, knobs):
return OrderedDict([(k, v) for k, v in list(knobs.items()) if return OrderedDict([(k, v) for k, v in list(knobs.items()) if
MyRocksParser.partial_name(k) in self.tunable_knob_catalog_]) MyRocksParser.partial_name(k) in self.tunable_knob_catalog_])
class MyRocks56Parser(MyRocksParser):
def __init__(self):
dbms = DBMSCatalog.objects.get(
type=DBMSType.MYROCKS, version='5.6')
super(MyRocks56Parser, self).__init__(dbms.pk)

18
server/website/website/parser/oracle.py
View File

@ -5,16 +5,14 @@
# #
from .base import BaseParser from .base import BaseParser
from website.models import DBMSCatalog
from website.types import DBMSType
class OracleParser(BaseParser): class OracleParser(BaseParser):
def __init__(self, dbms_id): def __init__(self, dbms_obj):
super(OracleParser, self).__init__(dbms_id) super().__init__(dbms_obj)
self.valid_true_val = ["TRUE", "true", "yes", 1] self.true_value = 'TRUE'
self.valid_false_val = ["FALSE", "false", "no", 0] self.false_value = 'FALSE'
ORACLE_BASE_KNOBS = { ORACLE_BASE_KNOBS = {
} }
@ -37,11 +35,3 @@ class OracleParser(BaseParser):
def parse_version_string(self, version_string): def parse_version_string(self, version_string):
return version_string return version_string
class Oracle19Parser(OracleParser):
def __init__(self):
dbms = DBMSCatalog.objects.get(
type=DBMSType.ORACLE, version='19.0.0.0.0')
super(Oracle19Parser, self).__init__(dbms.pk)

44
server/website/website/parser/parser.py
View File

@ -3,42 +3,38 @@
# #
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
'''
Created on Dec 12, 2017
@author: dvanaken
'''
from website.models import DBMSCatalog from website.models import DBMSCatalog
from website.types import DBMSType from website.types import DBMSType
from .myrocks import MyRocks56Parser from .myrocks import MyRocksParser
from .postgres import Postgres96Parser, PostgresOldParser from .postgres import PostgresParser
from .oracle import Oracle19Parser from .oracle import OracleParser
class Parser(object): class Parser():
__DBMS_UTILS_IMPLS = None __DBMS_UTILS_IMPLS = None
@staticmethod @staticmethod
def __utils(dbms_id=None): def __utils(dbms_id=None):
if Parser.__DBMS_UTILS_IMPLS is None: if Parser.__DBMS_UTILS_IMPLS is None:
Parser.__DBMS_UTILS_IMPLS = {
DBMSCatalog.objects.get( parsers = {}
type=DBMSType.POSTGRES, version='9.3').pk: PostgresOldParser('9.3'), for obj in DBMSCatalog.objects.all():
DBMSCatalog.objects.get( if obj.type == DBMSType.POSTGRES:
type=DBMSType.POSTGRES, version='9.2').pk: PostgresOldParser('9.2'), clz = PostgresParser
DBMSCatalog.objects.get( elif obj.type == DBMSType.MYROCKS:
type=DBMSType.POSTGRES, version='9.6').pk: Postgres96Parser('9.6'), clz = MyRocksParser
DBMSCatalog.objects.get( elif obj.type == DBMSType.ORACLE:
type=DBMSType.POSTGRES, version='9.4').pk: Postgres96Parser('9.4'), clz = OracleParser
DBMSCatalog.objects.get( else:
type=DBMSType.MYROCKS, version='5.6').pk: MyRocks56Parser(), raise NotImplementedError('Implement me! {}'.format(obj))
DBMSCatalog.objects.get(
type=DBMSType.ORACLE, version='19.0.0.0.0').pk: Oracle19Parser() parsers[obj.pk] = clz(obj)
}
Parser.__DBMS_UTILS_IMPLS = parsers
try: try:
if dbms_id is None: if dbms_id is None:
return Parser.__DBMS_UTILS_IMPLS return Parser.__DBMS_UTILS_IMPLS

63
server/website/website/parser/postgres.py
View File

@ -3,43 +3,20 @@
# #
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
'''
Created on Dec 12, 2017
@author: dvanaken
'''
import re import re
from .base import BaseParser from .base import BaseParser
from website.models import DBMSCatalog from website.types import KnobUnitType
from website.types import DBMSType, KnobUnitType
from website.utils import ConversionUtil from website.utils import ConversionUtil
class PostgresParser(BaseParser): class PostgresParser(BaseParser):
def __init__(self, dbms_id): def __init__(self, dbms_obj):
super(PostgresParser, self).__init__(dbms_id) super().__init__(dbms_obj)
self.valid_true_val = ["on", "true", "yes", 1] self.valid_true_val = ("on", "true", "yes", 1)
self.valid_false_val = ["off", "false", "no", 0] self.valid_false_val = ("off", "false", "no", 0)
POSTGRES_BYTES_SYSTEM = [
(1024 ** 5, 'PB'),
(1024 ** 4, 'TB'),
(1024 ** 3, 'GB'),
(1024 ** 2, 'MB'),
(1024 ** 1, 'kB'),
(1024 ** 0, 'B'),
]
POSTGRES_TIME_SYSTEM = [
(1000 * 60 * 60 * 24, 'd'),
(1000 * 60 * 60, 'h'),
(1000 * 60, 'min'),
(1000, 's'),
(1, 'ms'),
]
POSTGRES_BASE_KNOBS = { POSTGRES_BASE_KNOBS = {
'global.data_directory': None, 'global.data_directory': None,
@ -76,15 +53,14 @@ class PostgresParser(BaseParser):
def convert_integer(self, int_value, metadata): def convert_integer(self, int_value, metadata):
converted = None converted = None
try: try:
converted = super(PostgresParser, self).convert_integer( converted = super().convert_integer(int_value, metadata)
int_value, metadata)
except ValueError: except ValueError:
if metadata.unit == KnobUnitType.BYTES: if metadata.unit == KnobUnitType.BYTES:
converted = ConversionUtil.get_raw_size( converted = ConversionUtil.get_raw_size(
int_value, system=self.POSTGRES_BYTES_SYSTEM) int_value, system=ConversionUtil.DEFAULT_BYTES_SYSTEM)
elif metadata.unit == KnobUnitType.MILLISECONDS: elif metadata.unit == KnobUnitType.MILLISECONDS:
converted = ConversionUtil.get_raw_size( converted = ConversionUtil.get_raw_size(
int_value, system=self.POSTGRES_TIME_SYSTEM) int_value, system=ConversionUtil.DEFAULT_TIME_SYSTEM)
else: else:
raise Exception( raise Exception(
'Unknown unit type: {}'.format(metadata.unit)) 'Unknown unit type: {}'.format(metadata.unit))
@ -98,35 +74,18 @@ class PostgresParser(BaseParser):
if metadata.unit != KnobUnitType.OTHER and int_value > 0: if metadata.unit != KnobUnitType.OTHER and int_value > 0:
if metadata.unit == KnobUnitType.BYTES: if metadata.unit == KnobUnitType.BYTES:
int_value = ConversionUtil.get_human_readable( int_value = ConversionUtil.get_human_readable(
int_value, PostgresParser.POSTGRES_BYTES_SYSTEM) int_value, ConversionUtil.DEFAULT_BYTES_SYSTEM)
elif metadata.unit == KnobUnitType.MILLISECONDS: elif metadata.unit == KnobUnitType.MILLISECONDS:
int_value = ConversionUtil.get_human_readable( int_value = ConversionUtil.get_human_readable(
int_value, PostgresParser.POSTGRES_TIME_SYSTEM) int_value, ConversionUtil.DEFAULT_TIME_SYSTEM)
else: else:
raise Exception( raise Exception(
'Invalid unit type for {}: {}'.format( 'Invalid unit type for {}: {}'.format(
metadata.name, metadata.unit)) metadata.name, metadata.unit))
else: else:
int_value = super(PostgresParser, self).format_integer( int_value = super().format_integer(int_value, metadata)
int_value, metadata)
return int_value return int_value
def parse_version_string(self, version_string): def parse_version_string(self, version_string):
dbms_version = version_string.split(',')[0] dbms_version = version_string.split(',')[0]
return re.search(r'\d+\.\d+(?=\.\d+)', dbms_version).group(0) return re.search(r'\d+\.\d+(?=\.\d+)', dbms_version).group(0)
class Postgres96Parser(PostgresParser):
def __init__(self, version):
dbms = DBMSCatalog.objects.get(
type=DBMSType.POSTGRES, version=version)
super(Postgres96Parser, self).__init__(dbms.pk)
class PostgresOldParser(PostgresParser):
def __init__(self, version):
dbms = DBMSCatalog.objects.get(
type=DBMSType.POSTGRES, version=version)
super(PostgresOldParser, self).__init__(dbms.pk)

17
server/website/website/utils.py
View File

@ -205,6 +205,23 @@ class DataUtil(object):
class ConversionUtil(object): class ConversionUtil(object):
DEFAULT_BYTES_SYSTEM = (
(1024 ** 5, 'PB'),
(1024 ** 4, 'TB'),
(1024 ** 3, 'GB'),
(1024 ** 2, 'MB'),
(1024 ** 1, 'kB'),
(1024 ** 0, 'B'),
)
DEFAULT_TIME_SYSTEM = (
(1000 * 60 * 60 * 24, 'd'),
(1000 * 60 * 60, 'h'),
(1000 * 60, 'min'),
(1000, 's'),
(1, 'ms'),
)
@staticmethod @staticmethod
def get_raw_size(value, system): def get_raw_size(value, system):
for factor, suffix in system: for factor, suffix in system: