public/ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Parser cleanup

Browse Source
This commit is contained in:
Dana Van Aken
2019-08-27 19:21:10 -04:00
parent aa01f17eae
commit 7aea07f4c1
8 changed files with 149 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
server/website/website/parser/base.py
View File

@@ -3,13 +3,6 @@
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
'''
Created on Dec 12, 2017
@author: dvanaken
Parser interface.
'''
from abc import ABCMeta, abstractmethod, abstractproperty
from collections import OrderedDict
@@ -21,20 +14,24 @@ from website.types import BooleanType, MetricType, VarType
# pylint: disable=no-self-use
class BaseParser(object, metaclass=ABCMeta):
def __init__(self, dbms_id):
self.dbms_id_ = dbms_id
knobs = KnobCatalog.objects.filter(dbms__pk=self.dbms_id_)
def __init__(self, dbms_obj):
knobs = KnobCatalog.objects.filter(dbms=dbms_obj)
self.knob_catalog_ = {k.name: k for k in knobs}
self.tunable_knob_catalog_ = {k: v for k, v in
list(self.knob_catalog_.items()) if v.tunable is True}
metrics = MetricCatalog.objects.filter(dbms__pk=self.dbms_id_)
self.tunable_knob_catalog_ = {
k: v for k, v in self.knob_catalog_.items() if
v.tunable is True}
metrics = MetricCatalog.objects.filter(dbms=dbms_obj)
self.metric_catalog_ = {m.name: m for m in metrics}
self.numeric_metric_catalog_ = {m: v for m, v in
list(self.metric_catalog_.items()) if
v.metric_type == MetricType.COUNTER or
v.metric_type == MetricType.STATISTICS}
self.valid_true_val = list()
self.valid_false_val = list()
numeric_mtypes = (MetricType.COUNTER, MetricType.STATISTICS)
self.numeric_metric_catalog_ = {
m: v for m, v in self.metric_catalog_.items() if
v.metric_type in numeric_mtypes}
self.valid_true_val = ("on", "true", "yes")
self.valid_false_val = ("off", "false", "no")
self.true_value = 'on'
self.false_value = 'off'
@abstractproperty
def base_configuration_settings(self):
@@ -55,38 +52,49 @@ class BaseParser(object, metaclass=ABCMeta):
def target_metric(self, target_objective=None):
if target_objective == 'throughput_txn_per_sec' or target_objective is None:
# throughput
return self.transactions_counter
res = self.transactions_counter
elif target_objective == '99th_lat_ms':
# 99 percentile latency
return self.latency_timer
res = self.latency_timer
else:
raise Exception("Target Objective {} Not Supported".format(target_objective))
return res
@abstractmethod
def parse_version_string(self, version_string):
pass
def convert_bool(self, bool_value, metadata):
if isinstance(bool_value, str):
bool_value = bool_value.lower()
if bool_value in self.valid_true_val:
return BooleanType.TRUE
res = BooleanType.TRUE
elif bool_value in self.valid_false_val:
return BooleanType.FALSE
res = BooleanType.FALSE
else:
raise Exception("Invalid Boolean {}".format(bool_value))
return res
def convert_enum(self, enum_value, metadata):
enumvals = metadata.enumvals.split(',')
try:
return enumvals.index(enum_value)
res = enumvals.index(enum_value)
except ValueError:
raise Exception('Invalid enum value for variable {} ({})'.format(
metadata.name, enum_value))
return res
def convert_integer(self, int_value, metadata):
try:
return int(int_value)
res = int(int_value)
except ValueError:
return int(float(int_value))
res = int(float(int_value))
return res
def convert_real(self, real_value, metadata):
return float(real_value)
@@ -115,6 +123,7 @@ class BaseParser(object, metaclass=ABCMeta):
continue
value = knobs[name]
conv_value = None
if metadata.vartype == VarType.BOOL:
if not self._check_knob_bool_val(value):
raise Exception('Knob boolean value not valid! '
@@ -123,8 +132,10 @@ class BaseParser(object, metaclass=ABCMeta):
.format(self.valid_boolean_val_to_string(),
str(value)))
conv_value = self.convert_bool(value, metadata)
elif metadata.vartype == VarType.ENUM:
conv_value = self.convert_enum(value, metadata)
elif metadata.vartype == VarType.INTEGER:
conv_value = self.convert_integer(value, metadata)
if not self._check_knob_num_in_range(conv_value, metadata):
@@ -132,6 +143,7 @@ class BaseParser(object, metaclass=ABCMeta):
'min: {}, max: {}, actual: {}'
.format(metadata.minval,
metadata.maxval, str(conv_value)))
elif metadata.vartype == VarType.REAL:
conv_value = self.convert_real(value, metadata)
if not self._check_knob_num_in_range(conv_value, metadata):
@@ -139,23 +151,29 @@ class BaseParser(object, metaclass=ABCMeta):
'min: {}, max: {}, actual: {}'
.format(metadata.minval,
metadata.maxval, str(conv_value)))
elif metadata.vartype == VarType.STRING:
conv_value = self.convert_string(value, metadata)
elif metadata.vartype == VarType.TIMESTAMP:
conv_value = self.convert_timestamp(value, metadata)
else:
raise Exception(
'Unknown variable type: {}'.format(metadata.vartype))
if conv_value is None:
raise Exception(
'Param value for {} cannot be null'.format(name))
raise Exception('Param value for {} cannot be null'.format(name))
knob_data[name] = conv_value
return knob_data
def _check_knob_num_in_range(self, value, mdata):
return value >= float(mdata.minval) and value <= float(mdata.maxval)
def _check_knob_bool_val(self, value):
if isinstance(str, value):
value = value.lower()
return value in self.valid_true_val or value in self.valid_false_val
def convert_dbms_metrics(self, metrics, observation_time, target_objective=None):
@@ -331,7 +349,7 @@ class BaseParser(object, metaclass=ABCMeta):
return nondefault_settings
def format_bool(self, bool_value, metadata):
return 'on' if bool_value == BooleanType.TRUE else 'off'
return self.true_value if bool_value == BooleanType.TRUE else self.false_value
def format_enum(self, enum_value, metadata):
enumvals = metadata.enumvals.split(',')
@@ -378,11 +396,11 @@ class BaseParser(object, metaclass=ABCMeta):
return formatted_knobs
def filter_numeric_metrics(self, metrics):
return OrderedDict([(k, v) for k, v in list(metrics.items()) if
k in self.numeric_metric_catalog_])
return OrderedDict(((k, v) for k, v in list(metrics.items()) if
k in self.numeric_metric_catalog_))
def filter_tunable_knobs(self, knobs):
return OrderedDict([(k, v) for k, v in list(knobs.items()) if
k in self.tunable_knob_catalog_])
return OrderedDict(((k, v) for k, v in list(knobs.items()) if
k in self.tunable_knob_catalog_))
# pylint: enable=no-self-use

46
server/website/website/parser/myrocks.py
View File

@@ -3,40 +3,17 @@
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
'''
Created on Jan 16, 2018
@author: bohan
'''
import re
from collections import OrderedDict
from .base import BaseParser
from website.models import DBMSCatalog
from website.types import DBMSType, KnobUnitType, MetricType, VarType
from website.types import KnobUnitType, MetricType, VarType
from website.utils import ConversionUtil
class MyRocksParser(BaseParser):
MYROCKS_BYTES_SYSTEM = [
(1024 ** 5, 'PB'),
(1024 ** 4, 'TB'),
(1024 ** 3, 'GB'),
(1024 ** 2, 'MB'),
(1024 ** 1, 'kB'),
(1024 ** 0, 'B'),
]
MYROCKS_TIME_SYSTEM = [
(1000 * 60 * 60 * 24, 'd'),
(1000 * 60 * 60, 'h'),
(1000 * 60, 'min'),
(1, 'ms'),
(1000, 's'),
]
MYROCKS_BASE_KNOBS = {
'session_variables.rocksdb_max_open_files': '-1'
}
@@ -59,15 +36,15 @@ class MyRocksParser(BaseParser):
def convert_integer(self, int_value, metadata):
converted = None
try:
converted = super(MyRocksParser, self).convert_integer(
converted = super().convert_integer(
int_value, metadata)
except ValueError:
if metadata.unit == KnobUnitType.BYTES:
converted = ConversionUtil.get_raw_size(
int_value, system=self.MYROCKS_BYTES_SYSTEM)
int_value, system=ConversionUtil.DEFAULT_BYTES_SYSTEM)
elif metadata.unit == KnobUnitType.MILLISECONDS:
converted = ConversionUtil.get_raw_size(
int_value, system=self.MYROCKS_TIME_SYSTEM)
int_value, system=ConversionUtil.DEFAULT_TIME_SYSTEM)
else:
raise Exception('Unknown unit type: {}'.format(metadata.unit))
if converted is None:
@@ -79,16 +56,15 @@ class MyRocksParser(BaseParser):
if metadata.unit != KnobUnitType.OTHER and int_value > 0:
if metadata.unit == KnobUnitType.BYTES:
int_value = ConversionUtil.get_human_readable(
int_value, MyRocksParser.MYROCKS_BYTES_SYSTEM)
int_value, ConversionUtil.DEFAULT_BYTES_SYSTEM)
elif metadata.unit == KnobUnitType.MILLISECONDS:
int_value = ConversionUtil.get_human_readable(
int_value, MyRocksParser.MYROCKS_TIME_SYSTEM)
int_value, ConversionUtil.DEFAULT_TIME_SYSTEM)
else:
raise Exception('Invalid unit type for {}: {}'.format(
metadata.name, metadata.unit))
else:
int_value = super(MyRocksParser, self).format_integer(
int_value, metadata)
int_value = super().format_integer(int_value, metadata)
return int_value
def parse_version_string(self, version_string):
@@ -277,11 +253,3 @@ class MyRocksParser(BaseParser):
def filter_tunable_knobs(self, knobs):
return OrderedDict([(k, v) for k, v in list(knobs.items()) if
MyRocksParser.partial_name(k) in self.tunable_knob_catalog_])
class MyRocks56Parser(MyRocksParser):
def __init__(self):
dbms = DBMSCatalog.objects.get(
type=DBMSType.MYROCKS, version='5.6')
super(MyRocks56Parser, self).__init__(dbms.pk)

18
server/website/website/parser/oracle.py
View File

@@ -5,16 +5,14 @@
#
from .base import BaseParser
from website.models import DBMSCatalog
from website.types import DBMSType
class OracleParser(BaseParser):
def __init__(self, dbms_id):
super(OracleParser, self).__init__(dbms_id)
self.valid_true_val = ["TRUE", "true", "yes", 1]
self.valid_false_val = ["FALSE", "false", "no", 0]
def __init__(self, dbms_obj):
super().__init__(dbms_obj)
self.true_value = 'TRUE'
self.false_value = 'FALSE'
ORACLE_BASE_KNOBS = {
}
@@ -37,11 +35,3 @@ class OracleParser(BaseParser):
def parse_version_string(self, version_string):
return version_string
class Oracle19Parser(OracleParser):
def __init__(self):
dbms = DBMSCatalog.objects.get(
type=DBMSType.ORACLE, version='19.0.0.0.0')
super(Oracle19Parser, self).__init__(dbms.pk)

44
server/website/website/parser/parser.py
View File

@@ -3,42 +3,38 @@
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
'''
Created on Dec 12, 2017
@author: dvanaken
'''
from website.models import DBMSCatalog
from website.types import DBMSType
from .myrocks import MyRocks56Parser
from .postgres import Postgres96Parser, PostgresOldParser
from .oracle import Oracle19Parser
from .myrocks import MyRocksParser
from .postgres import PostgresParser
from .oracle import OracleParser
class Parser(object):
class Parser():
__DBMS_UTILS_IMPLS = None
@staticmethod
def __utils(dbms_id=None):
if Parser.__DBMS_UTILS_IMPLS is None:
Parser.__DBMS_UTILS_IMPLS = {
DBMSCatalog.objects.get(
type=DBMSType.POSTGRES, version='9.3').pk: PostgresOldParser('9.3'),
DBMSCatalog.objects.get(
type=DBMSType.POSTGRES, version='9.2').pk: PostgresOldParser('9.2'),
DBMSCatalog.objects.get(
type=DBMSType.POSTGRES, version='9.6').pk: Postgres96Parser('9.6'),
DBMSCatalog.objects.get(
type=DBMSType.POSTGRES, version='9.4').pk: Postgres96Parser('9.4'),
DBMSCatalog.objects.get(
type=DBMSType.MYROCKS, version='5.6').pk: MyRocks56Parser(),
DBMSCatalog.objects.get(
type=DBMSType.ORACLE, version='19.0.0.0.0').pk: Oracle19Parser()
}
parsers = {}
for obj in DBMSCatalog.objects.all():
if obj.type == DBMSType.POSTGRES:
clz = PostgresParser
elif obj.type == DBMSType.MYROCKS:
clz = MyRocksParser
elif obj.type == DBMSType.ORACLE:
clz = OracleParser
else:
raise NotImplementedError('Implement me! {}'.format(obj))
parsers[obj.pk] = clz(obj)
Parser.__DBMS_UTILS_IMPLS = parsers
try:
if dbms_id is None:
return Parser.__DBMS_UTILS_IMPLS

63
server/website/website/parser/postgres.py
View File

@@ -3,43 +3,20 @@
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
'''
Created on Dec 12, 2017
@author: dvanaken
'''
import re
from .base import BaseParser
from website.models import DBMSCatalog
from website.types import DBMSType, KnobUnitType
from website.types import KnobUnitType
from website.utils import ConversionUtil
class PostgresParser(BaseParser):
def __init__(self, dbms_id):
super(PostgresParser, self).__init__(dbms_id)
self.valid_true_val = ["on", "true", "yes", 1]
self.valid_false_val = ["off", "false", "no", 0]
POSTGRES_BYTES_SYSTEM = [
(1024 ** 5, 'PB'),
(1024 ** 4, 'TB'),
(1024 ** 3, 'GB'),
(1024 ** 2, 'MB'),
(1024 ** 1, 'kB'),
(1024 ** 0, 'B'),
]
POSTGRES_TIME_SYSTEM = [
(1000 * 60 * 60 * 24, 'd'),
(1000 * 60 * 60, 'h'),
(1000 * 60, 'min'),
(1000, 's'),
(1, 'ms'),
]
def __init__(self, dbms_obj):
super().__init__(dbms_obj)
self.valid_true_val = ("on", "true", "yes", 1)
self.valid_false_val = ("off", "false", "no", 0)
POSTGRES_BASE_KNOBS = {
'global.data_directory': None,
@@ -76,15 +53,14 @@ class PostgresParser(BaseParser):
def convert_integer(self, int_value, metadata):
converted = None
try:
converted = super(PostgresParser, self).convert_integer(
int_value, metadata)
converted = super().convert_integer(int_value, metadata)
except ValueError:
if metadata.unit == KnobUnitType.BYTES:
converted = ConversionUtil.get_raw_size(
int_value, system=self.POSTGRES_BYTES_SYSTEM)
int_value, system=ConversionUtil.DEFAULT_BYTES_SYSTEM)
elif metadata.unit == KnobUnitType.MILLISECONDS:
converted = ConversionUtil.get_raw_size(
int_value, system=self.POSTGRES_TIME_SYSTEM)
int_value, system=ConversionUtil.DEFAULT_TIME_SYSTEM)
else:
raise Exception(
'Unknown unit type: {}'.format(metadata.unit))
@@ -98,35 +74,18 @@ class PostgresParser(BaseParser):
if metadata.unit != KnobUnitType.OTHER and int_value > 0:
if metadata.unit == KnobUnitType.BYTES:
int_value = ConversionUtil.get_human_readable(
int_value, PostgresParser.POSTGRES_BYTES_SYSTEM)
int_value, ConversionUtil.DEFAULT_BYTES_SYSTEM)
elif metadata.unit == KnobUnitType.MILLISECONDS:
int_value = ConversionUtil.get_human_readable(
int_value, PostgresParser.POSTGRES_TIME_SYSTEM)
int_value, ConversionUtil.DEFAULT_TIME_SYSTEM)
else:
raise Exception(
'Invalid unit type for {}: {}'.format(
metadata.name, metadata.unit))
else:
int_value = super(PostgresParser, self).format_integer(
int_value, metadata)
int_value = super().format_integer(int_value, metadata)
return int_value
def parse_version_string(self, version_string):
dbms_version = version_string.split(',')[0]
return re.search(r'\d+\.\d+(?=\.\d+)', dbms_version).group(0)
class Postgres96Parser(PostgresParser):
def __init__(self, version):
dbms = DBMSCatalog.objects.get(
type=DBMSType.POSTGRES, version=version)
super(Postgres96Parser, self).__init__(dbms.pk)
class PostgresOldParser(PostgresParser):
def __init__(self, version):
dbms = DBMSCatalog.objects.get(
type=DBMSType.POSTGRES, version=version)
super(PostgresOldParser, self).__init__(dbms.pk)

17
server/website/website/utils.py
View File

@@ -205,6 +205,23 @@ class DataUtil(object):
class ConversionUtil(object):
DEFAULT_BYTES_SYSTEM = (
(1024 ** 5, 'PB'),
(1024 ** 4, 'TB'),
(1024 ** 3, 'GB'),
(1024 ** 2, 'MB'),
(1024 ** 1, 'kB'),
(1024 ** 0, 'B'),
)
DEFAULT_TIME_SYSTEM = (
(1000 * 60 * 60 * 24, 'd'),
(1000 * 60 * 60, 'h'),
(1000 * 60, 'min'),
(1000, 's'),
(1, 'ms'),
)
@staticmethod
def get_raw_size(value, system):
for factor, suffix in system: