Parser cleanup
This commit is contained in:
Dana Van Aken
parent
aa01f17eae
commit
7aea07f4c1
|
|
@ -7,9 +7,9 @@
|
|||
from abc import ABCMeta, abstractmethod
|
||||
import mock
|
||||
from django.test import TestCase
|
||||
from website.parser.postgres import PostgresParser, Postgres96Parser
|
||||
from website.types import BooleanType, VarType, KnobUnitType, MetricType
|
||||
from website.models import KnobCatalog
|
||||
from website.parser.postgres import PostgresParser
|
||||
from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType
|
||||
from website.models import DBMSCatalog, KnobCatalog
|
||||
|
||||
|
||||
class BaseParserTests(object, metaclass=ABCMeta):
|
||||
|
|
@ -161,8 +161,10 @@ class BaseParserTests(object, metaclass=ABCMeta):
|
|||
mock_other_knob = mock.Mock(spec=KnobCatalog)
|
||||
mock_other_knob.unit = KnobUnitType.OTHER
|
||||
|
||||
self.assertEqual(self.test_dbms.format_bool(BooleanType.TRUE, mock_other_knob), 'on')
|
||||
self.assertEqual(self.test_dbms.format_bool(BooleanType.FALSE, mock_other_knob), 'off')
|
||||
self.assertEqual(self.test_dbms.format_bool(BooleanType.TRUE, mock_other_knob),
|
||||
self.test_dbms.true_value)
|
||||
self.assertEqual(self.test_dbms.format_bool(BooleanType.FALSE, mock_other_knob),
|
||||
self.test_dbms.false_value)
|
||||
|
||||
def test_format_enum(self):
|
||||
mock_enum_knob = mock.Mock(spec=KnobCatalog)
|
||||
|
|
@ -217,13 +219,15 @@ class BaseParserTests(object, metaclass=ABCMeta):
|
|||
pass
|
||||
|
||||
|
||||
class Postgres96ParserTests(BaseParserTests, TestCase):
|
||||
class PostgresParserTests(BaseParserTests, TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.test_dbms = Postgres96Parser(9.6)
|
||||
dbms_obj = DBMSCatalog.objects.filter(
|
||||
type=DBMSType.POSTGRES, version="9.6").first()
|
||||
self.test_dbms = PostgresParser(dbms_obj)
|
||||
|
||||
def test_convert_dbms_knobs(self):
|
||||
super(Postgres96ParserTests, self).test_convert_dbms_knobs()
|
||||
super().test_convert_dbms_knobs()
|
||||
|
||||
test_knobs = {'global.wal_sync_method': 'open_sync', # Enum
|
||||
'global.random_page_cost': 0.22, # Real
|
||||
|
|
@ -250,7 +254,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
|
|||
self.assertEqual(self.test_dbms.convert_dbms_knobs(test_nontune_knobs), {})
|
||||
|
||||
def test_convert_dbms_metrics(self):
|
||||
super(Postgres96ParserTests, self).test_convert_dbms_metrics()
|
||||
super().test_convert_dbms_metrics()
|
||||
|
||||
test_metrics = {}
|
||||
|
||||
|
|
@ -361,7 +365,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
|
|||
self.assertEqual(nontune_extract.get('global.GEQO_EFFORT'), None)
|
||||
|
||||
def test_convert_integer(self):
|
||||
super(Postgres96ParserTests, self).test_convert_integer()
|
||||
super().test_convert_integer()
|
||||
|
||||
# Convert Integer
|
||||
knob_unit_bytes = KnobUnitType()
|
||||
|
|
@ -387,6 +391,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
|
|||
self.assertEqual(self.test_dbms
|
||||
.convert_integer('10min', knob_unit_time), 600000)
|
||||
self.assertEqual(self.test_dbms.convert_integer('1s', knob_unit_time), 1000)
|
||||
self.assertEqual(self.test_dbms.convert_integer('5000ms', knob_unit_time), 5000)
|
||||
|
||||
test_exceptions = [('A', knob_unit_other),
|
||||
('', knob_unit_other),
|
||||
|
|
@ -400,7 +405,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
|
|||
self.test_dbms.convert_integer(failure_case, knob_unit)
|
||||
|
||||
def test_calculate_change_in_metrics(self):
|
||||
super(Postgres96ParserTests, self).test_calculate_change_in_metrics()
|
||||
super().test_calculate_change_in_metrics()
|
||||
|
||||
test_metric_start = {'pg_stat_bgwriter.buffers_alloc': 256,
|
||||
'pg_stat_archiver.last_failed_wal': "today",
|
||||
|
|
@ -473,8 +478,6 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
|
|||
self.assertEqual(test_config.get(k), v)
|
||||
|
||||
def test_format_integer(self):
|
||||
test_dbms = PostgresParser(2)
|
||||
|
||||
knob_unit_bytes = KnobUnitType()
|
||||
knob_unit_bytes.unit = 1
|
||||
knob_unit_time = KnobUnitType()
|
||||
|
|
@ -482,22 +485,22 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
|
|||
knob_unit_other = KnobUnitType()
|
||||
knob_unit_other.unit = 3
|
||||
|
||||
self.assertEqual(test_dbms.format_integer(5, knob_unit_other), 5)
|
||||
self.assertEqual(test_dbms.format_integer(0, knob_unit_other), 0)
|
||||
self.assertEqual(test_dbms.format_integer(-1, knob_unit_other), -1)
|
||||
self.assertEqual(self.test_dbms.format_integer(5, knob_unit_other), 5)
|
||||
self.assertEqual(self.test_dbms.format_integer(0, knob_unit_other), 0)
|
||||
self.assertEqual(self.test_dbms.format_integer(-1, knob_unit_other), -1)
|
||||
|
||||
self.assertEqual(test_dbms.format_integer(5120, knob_unit_bytes), '5kB')
|
||||
self.assertEqual(test_dbms.format_integer(4194304, knob_unit_bytes), '4MB')
|
||||
self.assertEqual(test_dbms.format_integer(4194500, knob_unit_bytes), '4MB')
|
||||
self.assertEqual(self.test_dbms.format_integer(5120, knob_unit_bytes), '5kB')
|
||||
self.assertEqual(self.test_dbms.format_integer(4194304, knob_unit_bytes), '4MB')
|
||||
self.assertEqual(self.test_dbms.format_integer(4194500, knob_unit_bytes), '4MB')
|
||||
|
||||
self.assertEqual(test_dbms.format_integer(86400000, knob_unit_time), '1d')
|
||||
self.assertEqual(test_dbms.format_integer(72000000, knob_unit_time), '20h')
|
||||
self.assertEqual(test_dbms.format_integer(600000, knob_unit_time), '10min')
|
||||
self.assertEqual(test_dbms.format_integer(1000, knob_unit_time), '1s')
|
||||
self.assertEqual(test_dbms.format_integer(500, knob_unit_time), '500ms')
|
||||
self.assertEqual(self.test_dbms.format_integer(86400000, knob_unit_time), '1d')
|
||||
self.assertEqual(self.test_dbms.format_integer(72000000, knob_unit_time), '20h')
|
||||
self.assertEqual(self.test_dbms.format_integer(600000, knob_unit_time), '10min')
|
||||
self.assertEqual(self.test_dbms.format_integer(1000, knob_unit_time), '1s')
|
||||
self.assertEqual(self.test_dbms.format_integer(500, knob_unit_time), '500ms')
|
||||
|
||||
def test_format_dbms_knobs(self):
|
||||
super(Postgres96ParserTests, self).test_format_dbms_knobs()
|
||||
super().test_format_dbms_knobs()
|
||||
|
||||
test_knobs = {'global.wal_sync_method': 2, # Enum
|
||||
'global.random_page_cost': 0.22, # Real
|
||||
|
|
@ -520,7 +523,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
|
|||
self.assertEqual(test_formatted_knobs.get('global.wal_buffers'), '1kB')
|
||||
|
||||
def test_filter_numeric_metrics(self):
|
||||
super(Postgres96ParserTests, self).test_filter_numeric_metrics()
|
||||
super().test_filter_numeric_metrics()
|
||||
|
||||
test_metrics = {'pg_stat_bgwriter.checkpoints_req': (2, 'global'),
|
||||
'pg_stat_archiver.last_failed_wal': (1, 'global'),
|
||||
|
|
@ -549,7 +552,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
|
|||
self.assertEqual(filtered_metrics.get('pg_FAKE_KNOB'), None)
|
||||
|
||||
def test_filter_tunable_knobs(self):
|
||||
super(Postgres96ParserTests, self).test_filter_tunable_knobs()
|
||||
super().test_filter_tunable_knobs()
|
||||
|
||||
test_knobs = {'global.wal_sync_method': 5,
|
||||
'global.random_page_cost': 3,
|
||||
|
|
@ -571,7 +574,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase):
|
|||
self.assertEqual(filtered_knobs.get('global.FAKE_KNOB'), None)
|
||||
|
||||
def test_parse_helper(self):
|
||||
super(Postgres96ParserTests, self).test_parse_helper()
|
||||
super().test_parse_helper()
|
||||
|
||||
test_view_vars = {'global': {'wal_sync_method': 'open_sync',
|
||||
'random_page_cost': 0.22},
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import string
|
|||
import numpy as np
|
||||
from django.test import TestCase
|
||||
from website.utils import JSONUtil, MediaUtil, DataUtil, ConversionUtil, LabelUtil, TaskUtil
|
||||
from website.parser.postgres import PostgresParser
|
||||
from website.types import LabelStyleType, VarType
|
||||
from website.models import Result, DBMSCatalog
|
||||
|
||||
|
|
@ -225,9 +224,9 @@ class DataUtilTest(TestCase):
|
|||
'global.bgwriter_delay',
|
||||
'global.wal_writer_delay',
|
||||
'global.work_mem']
|
||||
postgres96 = DBMSCatalog.objects.get(pk=1)
|
||||
postgresdb = DBMSCatalog.objects.get(pk=1)
|
||||
categorical_info = DataUtil.dummy_encoder_helper(featured_knobs,
|
||||
dbms=postgres96)
|
||||
dbms=postgresdb)
|
||||
self.assertEqual(len(categorical_info['n_values']), 0)
|
||||
self.assertEqual(len(categorical_info['categorical_features']), 0)
|
||||
self.assertEqual(categorical_info['cat_columnlabels'], [])
|
||||
|
|
@ -239,9 +238,9 @@ class DataUtilTest(TestCase):
|
|||
'global.wal_writer_delay',
|
||||
'global.work_mem',
|
||||
'global.wal_sync_method'] # last knob categorical
|
||||
postgres96 = DBMSCatalog.objects.get(pk=1)
|
||||
postgresdb = DBMSCatalog.objects.get(pk=1)
|
||||
categorical_info = DataUtil.dummy_encoder_helper(featured_knobs,
|
||||
dbms=postgres96)
|
||||
dbms=postgresdb)
|
||||
self.assertEqual(len(categorical_info['n_values']), 1)
|
||||
self.assertEqual(categorical_info['n_values'][0], 4)
|
||||
self.assertEqual(len(categorical_info['categorical_features']), 1)
|
||||
|
|
@ -257,7 +256,7 @@ class ConversionUtilTest(TestCase):
|
|||
byte_ans = [1024**5, 2 * 1024**4, 3 * 1024**3, 4 * 1024**2, 5 * 1024**1, 6]
|
||||
for i, byte_test in enumerate(byte_test_convert):
|
||||
byte_conversion = ConversionUtil.get_raw_size(
|
||||
byte_test, system=PostgresParser.POSTGRES_BYTES_SYSTEM)
|
||||
byte_test, system=ConversionUtil.DEFAULT_BYTES_SYSTEM)
|
||||
self.assertEqual(byte_conversion, byte_ans[i])
|
||||
|
||||
# Time - In Milliseconds
|
||||
|
|
@ -265,7 +264,7 @@ class ConversionUtilTest(TestCase):
|
|||
day_ans = [1000, 1000, 600000, 72000000, 86400000]
|
||||
for i, day_test in enumerate(day_test_convert):
|
||||
day_conversion = ConversionUtil.get_raw_size(
|
||||
day_test, system=PostgresParser.POSTGRES_TIME_SYSTEM)
|
||||
day_test, system=ConversionUtil.DEFAULT_TIME_SYSTEM)
|
||||
self.assertEqual(day_conversion, day_ans[i])
|
||||
|
||||
def test_get_human_readable(self):
|
||||
|
|
@ -275,7 +274,7 @@ class ConversionUtilTest(TestCase):
|
|||
byte_ans = ['1PB', '2TB', '3GB', '4MB', '5kB', '6B']
|
||||
for i, byte_test in enumerate(byte_test_convert):
|
||||
byte_readable = ConversionUtil.get_human_readable(
|
||||
byte_test, system=PostgresParser.POSTGRES_BYTES_SYSTEM)
|
||||
byte_test, system=ConversionUtil.DEFAULT_BYTES_SYSTEM)
|
||||
self.assertEqual(byte_readable, byte_ans[i])
|
||||
|
||||
# Time
|
||||
|
|
@ -283,7 +282,7 @@ class ConversionUtilTest(TestCase):
|
|||
day_ans = ['500ms', '1s', '55s', '10min', '20h', '1d']
|
||||
for i, day_test in enumerate(day_test_convert):
|
||||
day_readable = ConversionUtil.get_human_readable(
|
||||
day_test, system=PostgresParser.POSTGRES_TIME_SYSTEM)
|
||||
day_test, system=ConversionUtil.DEFAULT_TIME_SYSTEM)
|
||||
self.assertEqual(day_readable, day_ans[i])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,13 +3,6 @@
|
|||
#
|
||||
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||
#
|
||||
'''
|
||||
Created on Dec 12, 2017
|
||||
|
||||
@author: dvanaken
|
||||
|
||||
Parser interface.
|
||||
'''
|
||||
|
||||
from abc import ABCMeta, abstractmethod, abstractproperty
|
||||
from collections import OrderedDict
|
||||
|
|
@ -21,20 +14,24 @@ from website.types import BooleanType, MetricType, VarType
|
|||
# pylint: disable=no-self-use
|
||||
class BaseParser(object, metaclass=ABCMeta):
|
||||
|
||||
def __init__(self, dbms_id):
|
||||
self.dbms_id_ = dbms_id
|
||||
knobs = KnobCatalog.objects.filter(dbms__pk=self.dbms_id_)
|
||||
def __init__(self, dbms_obj):
|
||||
knobs = KnobCatalog.objects.filter(dbms=dbms_obj)
|
||||
self.knob_catalog_ = {k.name: k for k in knobs}
|
||||
self.tunable_knob_catalog_ = {k: v for k, v in
|
||||
list(self.knob_catalog_.items()) if v.tunable is True}
|
||||
metrics = MetricCatalog.objects.filter(dbms__pk=self.dbms_id_)
|
||||
self.tunable_knob_catalog_ = {
|
||||
k: v for k, v in self.knob_catalog_.items() if
|
||||
v.tunable is True}
|
||||
|
||||
metrics = MetricCatalog.objects.filter(dbms=dbms_obj)
|
||||
self.metric_catalog_ = {m.name: m for m in metrics}
|
||||
self.numeric_metric_catalog_ = {m: v for m, v in
|
||||
list(self.metric_catalog_.items()) if
|
||||
v.metric_type == MetricType.COUNTER or
|
||||
v.metric_type == MetricType.STATISTICS}
|
||||
self.valid_true_val = list()
|
||||
self.valid_false_val = list()
|
||||
numeric_mtypes = (MetricType.COUNTER, MetricType.STATISTICS)
|
||||
self.numeric_metric_catalog_ = {
|
||||
m: v for m, v in self.metric_catalog_.items() if
|
||||
v.metric_type in numeric_mtypes}
|
||||
|
||||
self.valid_true_val = ("on", "true", "yes")
|
||||
self.valid_false_val = ("off", "false", "no")
|
||||
self.true_value = 'on'
|
||||
self.false_value = 'off'
|
||||
|
||||
@abstractproperty
|
||||
def base_configuration_settings(self):
|
||||
|
|
@ -55,38 +52,49 @@ class BaseParser(object, metaclass=ABCMeta):
|
|||
def target_metric(self, target_objective=None):
|
||||
if target_objective == 'throughput_txn_per_sec' or target_objective is None:
|
||||
# throughput
|
||||
return self.transactions_counter
|
||||
res = self.transactions_counter
|
||||
elif target_objective == '99th_lat_ms':
|
||||
# 99 percentile latency
|
||||
return self.latency_timer
|
||||
res = self.latency_timer
|
||||
else:
|
||||
raise Exception("Target Objective {} Not Supported".format(target_objective))
|
||||
|
||||
return res
|
||||
|
||||
@abstractmethod
|
||||
def parse_version_string(self, version_string):
|
||||
pass
|
||||
|
||||
def convert_bool(self, bool_value, metadata):
|
||||
if isinstance(bool_value, str):
|
||||
bool_value = bool_value.lower()
|
||||
|
||||
if bool_value in self.valid_true_val:
|
||||
return BooleanType.TRUE
|
||||
res = BooleanType.TRUE
|
||||
elif bool_value in self.valid_false_val:
|
||||
return BooleanType.FALSE
|
||||
res = BooleanType.FALSE
|
||||
else:
|
||||
raise Exception("Invalid Boolean {}".format(bool_value))
|
||||
|
||||
return res
|
||||
|
||||
def convert_enum(self, enum_value, metadata):
|
||||
enumvals = metadata.enumvals.split(',')
|
||||
try:
|
||||
return enumvals.index(enum_value)
|
||||
res = enumvals.index(enum_value)
|
||||
except ValueError:
|
||||
raise Exception('Invalid enum value for variable {} ({})'.format(
|
||||
metadata.name, enum_value))
|
||||
|
||||
return res
|
||||
|
||||
def convert_integer(self, int_value, metadata):
|
||||
try:
|
||||
return int(int_value)
|
||||
res = int(int_value)
|
||||
except ValueError:
|
||||
return int(float(int_value))
|
||||
res = int(float(int_value))
|
||||
|
||||
return res
|
||||
|
||||
def convert_real(self, real_value, metadata):
|
||||
return float(real_value)
|
||||
|
|
@ -115,6 +123,7 @@ class BaseParser(object, metaclass=ABCMeta):
|
|||
continue
|
||||
value = knobs[name]
|
||||
conv_value = None
|
||||
|
||||
if metadata.vartype == VarType.BOOL:
|
||||
if not self._check_knob_bool_val(value):
|
||||
raise Exception('Knob boolean value not valid! '
|
||||
|
|
@ -123,8 +132,10 @@ class BaseParser(object, metaclass=ABCMeta):
|
|||
.format(self.valid_boolean_val_to_string(),
|
||||
str(value)))
|
||||
conv_value = self.convert_bool(value, metadata)
|
||||
|
||||
elif metadata.vartype == VarType.ENUM:
|
||||
conv_value = self.convert_enum(value, metadata)
|
||||
|
||||
elif metadata.vartype == VarType.INTEGER:
|
||||
conv_value = self.convert_integer(value, metadata)
|
||||
if not self._check_knob_num_in_range(conv_value, metadata):
|
||||
|
|
@ -132,6 +143,7 @@ class BaseParser(object, metaclass=ABCMeta):
|
|||
'min: {}, max: {}, actual: {}'
|
||||
.format(metadata.minval,
|
||||
metadata.maxval, str(conv_value)))
|
||||
|
||||
elif metadata.vartype == VarType.REAL:
|
||||
conv_value = self.convert_real(value, metadata)
|
||||
if not self._check_knob_num_in_range(conv_value, metadata):
|
||||
|
|
@ -139,23 +151,29 @@ class BaseParser(object, metaclass=ABCMeta):
|
|||
'min: {}, max: {}, actual: {}'
|
||||
.format(metadata.minval,
|
||||
metadata.maxval, str(conv_value)))
|
||||
|
||||
elif metadata.vartype == VarType.STRING:
|
||||
conv_value = self.convert_string(value, metadata)
|
||||
|
||||
elif metadata.vartype == VarType.TIMESTAMP:
|
||||
conv_value = self.convert_timestamp(value, metadata)
|
||||
|
||||
else:
|
||||
raise Exception(
|
||||
'Unknown variable type: {}'.format(metadata.vartype))
|
||||
|
||||
if conv_value is None:
|
||||
raise Exception(
|
||||
'Param value for {} cannot be null'.format(name))
|
||||
raise Exception('Param value for {} cannot be null'.format(name))
|
||||
knob_data[name] = conv_value
|
||||
|
||||
return knob_data
|
||||
|
||||
def _check_knob_num_in_range(self, value, mdata):
|
||||
return value >= float(mdata.minval) and value <= float(mdata.maxval)
|
||||
|
||||
def _check_knob_bool_val(self, value):
|
||||
if isinstance(str, value):
|
||||
value = value.lower()
|
||||
return value in self.valid_true_val or value in self.valid_false_val
|
||||
|
||||
def convert_dbms_metrics(self, metrics, observation_time, target_objective=None):
|
||||
|
|
@ -331,7 +349,7 @@ class BaseParser(object, metaclass=ABCMeta):
|
|||
return nondefault_settings
|
||||
|
||||
def format_bool(self, bool_value, metadata):
|
||||
return 'on' if bool_value == BooleanType.TRUE else 'off'
|
||||
return self.true_value if bool_value == BooleanType.TRUE else self.false_value
|
||||
|
||||
def format_enum(self, enum_value, metadata):
|
||||
enumvals = metadata.enumvals.split(',')
|
||||
|
|
@ -378,11 +396,11 @@ class BaseParser(object, metaclass=ABCMeta):
|
|||
return formatted_knobs
|
||||
|
||||
def filter_numeric_metrics(self, metrics):
|
||||
return OrderedDict([(k, v) for k, v in list(metrics.items()) if
|
||||
k in self.numeric_metric_catalog_])
|
||||
return OrderedDict(((k, v) for k, v in list(metrics.items()) if
|
||||
k in self.numeric_metric_catalog_))
|
||||
|
||||
def filter_tunable_knobs(self, knobs):
|
||||
return OrderedDict([(k, v) for k, v in list(knobs.items()) if
|
||||
k in self.tunable_knob_catalog_])
|
||||
return OrderedDict(((k, v) for k, v in list(knobs.items()) if
|
||||
k in self.tunable_knob_catalog_))
|
||||
|
||||
# pylint: enable=no-self-use
|
||||
|
|
|
|||
|
|
@ -3,40 +3,17 @@
|
|||
#
|
||||
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||
#
|
||||
'''
|
||||
Created on Jan 16, 2018
|
||||
|
||||
@author: bohan
|
||||
'''
|
||||
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
|
||||
from .base import BaseParser
|
||||
from website.models import DBMSCatalog
|
||||
from website.types import DBMSType, KnobUnitType, MetricType, VarType
|
||||
from website.types import KnobUnitType, MetricType, VarType
|
||||
from website.utils import ConversionUtil
|
||||
|
||||
|
||||
class MyRocksParser(BaseParser):
|
||||
|
||||
MYROCKS_BYTES_SYSTEM = [
|
||||
(1024 ** 5, 'PB'),
|
||||
(1024 ** 4, 'TB'),
|
||||
(1024 ** 3, 'GB'),
|
||||
(1024 ** 2, 'MB'),
|
||||
(1024 ** 1, 'kB'),
|
||||
(1024 ** 0, 'B'),
|
||||
]
|
||||
|
||||
MYROCKS_TIME_SYSTEM = [
|
||||
(1000 * 60 * 60 * 24, 'd'),
|
||||
(1000 * 60 * 60, 'h'),
|
||||
(1000 * 60, 'min'),
|
||||
(1, 'ms'),
|
||||
(1000, 's'),
|
||||
]
|
||||
|
||||
MYROCKS_BASE_KNOBS = {
|
||||
'session_variables.rocksdb_max_open_files': '-1'
|
||||
}
|
||||
|
|
@ -59,15 +36,15 @@ class MyRocksParser(BaseParser):
|
|||
def convert_integer(self, int_value, metadata):
|
||||
converted = None
|
||||
try:
|
||||
converted = super(MyRocksParser, self).convert_integer(
|
||||
converted = super().convert_integer(
|
||||
int_value, metadata)
|
||||
except ValueError:
|
||||
if metadata.unit == KnobUnitType.BYTES:
|
||||
converted = ConversionUtil.get_raw_size(
|
||||
int_value, system=self.MYROCKS_BYTES_SYSTEM)
|
||||
int_value, system=ConversionUtil.DEFAULT_BYTES_SYSTEM)
|
||||
elif metadata.unit == KnobUnitType.MILLISECONDS:
|
||||
converted = ConversionUtil.get_raw_size(
|
||||
int_value, system=self.MYROCKS_TIME_SYSTEM)
|
||||
int_value, system=ConversionUtil.DEFAULT_TIME_SYSTEM)
|
||||
else:
|
||||
raise Exception('Unknown unit type: {}'.format(metadata.unit))
|
||||
if converted is None:
|
||||
|
|
@ -79,16 +56,15 @@ class MyRocksParser(BaseParser):
|
|||
if metadata.unit != KnobUnitType.OTHER and int_value > 0:
|
||||
if metadata.unit == KnobUnitType.BYTES:
|
||||
int_value = ConversionUtil.get_human_readable(
|
||||
int_value, MyRocksParser.MYROCKS_BYTES_SYSTEM)
|
||||
int_value, ConversionUtil.DEFAULT_BYTES_SYSTEM)
|
||||
elif metadata.unit == KnobUnitType.MILLISECONDS:
|
||||
int_value = ConversionUtil.get_human_readable(
|
||||
int_value, MyRocksParser.MYROCKS_TIME_SYSTEM)
|
||||
int_value, ConversionUtil.DEFAULT_TIME_SYSTEM)
|
||||
else:
|
||||
raise Exception('Invalid unit type for {}: {}'.format(
|
||||
metadata.name, metadata.unit))
|
||||
else:
|
||||
int_value = super(MyRocksParser, self).format_integer(
|
||||
int_value, metadata)
|
||||
int_value = super().format_integer(int_value, metadata)
|
||||
return int_value
|
||||
|
||||
def parse_version_string(self, version_string):
|
||||
|
|
@ -277,11 +253,3 @@ class MyRocksParser(BaseParser):
|
|||
def filter_tunable_knobs(self, knobs):
|
||||
return OrderedDict([(k, v) for k, v in list(knobs.items()) if
|
||||
MyRocksParser.partial_name(k) in self.tunable_knob_catalog_])
|
||||
|
||||
|
||||
class MyRocks56Parser(MyRocksParser):
|
||||
|
||||
def __init__(self):
|
||||
dbms = DBMSCatalog.objects.get(
|
||||
type=DBMSType.MYROCKS, version='5.6')
|
||||
super(MyRocks56Parser, self).__init__(dbms.pk)
|
||||
|
|
|
|||
|
|
@ -5,16 +5,14 @@
|
|||
#
|
||||
|
||||
from .base import BaseParser
|
||||
from website.models import DBMSCatalog
|
||||
from website.types import DBMSType
|
||||
|
||||
|
||||
class OracleParser(BaseParser):
|
||||
|
||||
def __init__(self, dbms_id):
|
||||
super(OracleParser, self).__init__(dbms_id)
|
||||
self.valid_true_val = ["TRUE", "true", "yes", 1]
|
||||
self.valid_false_val = ["FALSE", "false", "no", 0]
|
||||
def __init__(self, dbms_obj):
|
||||
super().__init__(dbms_obj)
|
||||
self.true_value = 'TRUE'
|
||||
self.false_value = 'FALSE'
|
||||
|
||||
ORACLE_BASE_KNOBS = {
|
||||
}
|
||||
|
|
@ -37,11 +35,3 @@ class OracleParser(BaseParser):
|
|||
|
||||
def parse_version_string(self, version_string):
|
||||
return version_string
|
||||
|
||||
|
||||
class Oracle19Parser(OracleParser):
|
||||
|
||||
def __init__(self):
|
||||
dbms = DBMSCatalog.objects.get(
|
||||
type=DBMSType.ORACLE, version='19.0.0.0.0')
|
||||
super(Oracle19Parser, self).__init__(dbms.pk)
|
||||
|
|
|
|||
|
|
@ -3,42 +3,38 @@
|
|||
#
|
||||
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||
#
|
||||
'''
|
||||
Created on Dec 12, 2017
|
||||
|
||||
@author: dvanaken
|
||||
'''
|
||||
|
||||
|
||||
from website.models import DBMSCatalog
|
||||
from website.types import DBMSType
|
||||
|
||||
from .myrocks import MyRocks56Parser
|
||||
from .postgres import Postgres96Parser, PostgresOldParser
|
||||
from .oracle import Oracle19Parser
|
||||
from .myrocks import MyRocksParser
|
||||
from .postgres import PostgresParser
|
||||
from .oracle import OracleParser
|
||||
|
||||
|
||||
class Parser(object):
|
||||
class Parser():
|
||||
|
||||
__DBMS_UTILS_IMPLS = None
|
||||
|
||||
@staticmethod
|
||||
def __utils(dbms_id=None):
|
||||
if Parser.__DBMS_UTILS_IMPLS is None:
|
||||
Parser.__DBMS_UTILS_IMPLS = {
|
||||
DBMSCatalog.objects.get(
|
||||
type=DBMSType.POSTGRES, version='9.3').pk: PostgresOldParser('9.3'),
|
||||
DBMSCatalog.objects.get(
|
||||
type=DBMSType.POSTGRES, version='9.2').pk: PostgresOldParser('9.2'),
|
||||
DBMSCatalog.objects.get(
|
||||
type=DBMSType.POSTGRES, version='9.6').pk: Postgres96Parser('9.6'),
|
||||
DBMSCatalog.objects.get(
|
||||
type=DBMSType.POSTGRES, version='9.4').pk: Postgres96Parser('9.4'),
|
||||
DBMSCatalog.objects.get(
|
||||
type=DBMSType.MYROCKS, version='5.6').pk: MyRocks56Parser(),
|
||||
DBMSCatalog.objects.get(
|
||||
type=DBMSType.ORACLE, version='19.0.0.0.0').pk: Oracle19Parser()
|
||||
}
|
||||
|
||||
parsers = {}
|
||||
for obj in DBMSCatalog.objects.all():
|
||||
if obj.type == DBMSType.POSTGRES:
|
||||
clz = PostgresParser
|
||||
elif obj.type == DBMSType.MYROCKS:
|
||||
clz = MyRocksParser
|
||||
elif obj.type == DBMSType.ORACLE:
|
||||
clz = OracleParser
|
||||
else:
|
||||
raise NotImplementedError('Implement me! {}'.format(obj))
|
||||
|
||||
parsers[obj.pk] = clz(obj)
|
||||
|
||||
Parser.__DBMS_UTILS_IMPLS = parsers
|
||||
|
||||
try:
|
||||
if dbms_id is None:
|
||||
return Parser.__DBMS_UTILS_IMPLS
|
||||
|
|
|
|||
|
|
@ -3,43 +3,20 @@
|
|||
#
|
||||
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||
#
|
||||
'''
|
||||
Created on Dec 12, 2017
|
||||
|
||||
@author: dvanaken
|
||||
'''
|
||||
|
||||
import re
|
||||
|
||||
from .base import BaseParser
|
||||
from website.models import DBMSCatalog
|
||||
from website.types import DBMSType, KnobUnitType
|
||||
from website.types import KnobUnitType
|
||||
from website.utils import ConversionUtil
|
||||
|
||||
|
||||
class PostgresParser(BaseParser):
|
||||
|
||||
def __init__(self, dbms_id):
|
||||
super(PostgresParser, self).__init__(dbms_id)
|
||||
self.valid_true_val = ["on", "true", "yes", 1]
|
||||
self.valid_false_val = ["off", "false", "no", 0]
|
||||
|
||||
POSTGRES_BYTES_SYSTEM = [
|
||||
(1024 ** 5, 'PB'),
|
||||
(1024 ** 4, 'TB'),
|
||||
(1024 ** 3, 'GB'),
|
||||
(1024 ** 2, 'MB'),
|
||||
(1024 ** 1, 'kB'),
|
||||
(1024 ** 0, 'B'),
|
||||
]
|
||||
|
||||
POSTGRES_TIME_SYSTEM = [
|
||||
(1000 * 60 * 60 * 24, 'd'),
|
||||
(1000 * 60 * 60, 'h'),
|
||||
(1000 * 60, 'min'),
|
||||
(1000, 's'),
|
||||
(1, 'ms'),
|
||||
]
|
||||
def __init__(self, dbms_obj):
|
||||
super().__init__(dbms_obj)
|
||||
self.valid_true_val = ("on", "true", "yes", 1)
|
||||
self.valid_false_val = ("off", "false", "no", 0)
|
||||
|
||||
POSTGRES_BASE_KNOBS = {
|
||||
'global.data_directory': None,
|
||||
|
|
@ -76,15 +53,14 @@ class PostgresParser(BaseParser):
|
|||
def convert_integer(self, int_value, metadata):
|
||||
converted = None
|
||||
try:
|
||||
converted = super(PostgresParser, self).convert_integer(
|
||||
int_value, metadata)
|
||||
converted = super().convert_integer(int_value, metadata)
|
||||
except ValueError:
|
||||
if metadata.unit == KnobUnitType.BYTES:
|
||||
converted = ConversionUtil.get_raw_size(
|
||||
int_value, system=self.POSTGRES_BYTES_SYSTEM)
|
||||
int_value, system=ConversionUtil.DEFAULT_BYTES_SYSTEM)
|
||||
elif metadata.unit == KnobUnitType.MILLISECONDS:
|
||||
converted = ConversionUtil.get_raw_size(
|
||||
int_value, system=self.POSTGRES_TIME_SYSTEM)
|
||||
int_value, system=ConversionUtil.DEFAULT_TIME_SYSTEM)
|
||||
else:
|
||||
raise Exception(
|
||||
'Unknown unit type: {}'.format(metadata.unit))
|
||||
|
|
@ -98,35 +74,18 @@ class PostgresParser(BaseParser):
|
|||
if metadata.unit != KnobUnitType.OTHER and int_value > 0:
|
||||
if metadata.unit == KnobUnitType.BYTES:
|
||||
int_value = ConversionUtil.get_human_readable(
|
||||
int_value, PostgresParser.POSTGRES_BYTES_SYSTEM)
|
||||
int_value, ConversionUtil.DEFAULT_BYTES_SYSTEM)
|
||||
elif metadata.unit == KnobUnitType.MILLISECONDS:
|
||||
int_value = ConversionUtil.get_human_readable(
|
||||
int_value, PostgresParser.POSTGRES_TIME_SYSTEM)
|
||||
int_value, ConversionUtil.DEFAULT_TIME_SYSTEM)
|
||||
else:
|
||||
raise Exception(
|
||||
'Invalid unit type for {}: {}'.format(
|
||||
metadata.name, metadata.unit))
|
||||
else:
|
||||
int_value = super(PostgresParser, self).format_integer(
|
||||
int_value, metadata)
|
||||
int_value = super().format_integer(int_value, metadata)
|
||||
return int_value
|
||||
|
||||
def parse_version_string(self, version_string):
|
||||
dbms_version = version_string.split(',')[0]
|
||||
return re.search(r'\d+\.\d+(?=\.\d+)', dbms_version).group(0)
|
||||
|
||||
|
||||
class Postgres96Parser(PostgresParser):
|
||||
|
||||
def __init__(self, version):
|
||||
dbms = DBMSCatalog.objects.get(
|
||||
type=DBMSType.POSTGRES, version=version)
|
||||
super(Postgres96Parser, self).__init__(dbms.pk)
|
||||
|
||||
|
||||
class PostgresOldParser(PostgresParser):
|
||||
|
||||
def __init__(self, version):
|
||||
dbms = DBMSCatalog.objects.get(
|
||||
type=DBMSType.POSTGRES, version=version)
|
||||
super(PostgresOldParser, self).__init__(dbms.pk)
|
||||
|
|
|
|||
|
|
@ -205,6 +205,23 @@ class DataUtil(object):
|
|||
|
||||
class ConversionUtil(object):
|
||||
|
||||
DEFAULT_BYTES_SYSTEM = (
|
||||
(1024 ** 5, 'PB'),
|
||||
(1024 ** 4, 'TB'),
|
||||
(1024 ** 3, 'GB'),
|
||||
(1024 ** 2, 'MB'),
|
||||
(1024 ** 1, 'kB'),
|
||||
(1024 ** 0, 'B'),
|
||||
)
|
||||
|
||||
DEFAULT_TIME_SYSTEM = (
|
||||
(1000 * 60 * 60 * 24, 'd'),
|
||||
(1000 * 60 * 60, 'h'),
|
||||
(1000 * 60, 'min'),
|
||||
(1000, 's'),
|
||||
(1, 'ms'),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_raw_size(value, system):
|
||||
for factor, suffix in system:
|
||||
|
|
|
|||
Loading…
Reference in New Issue