From 7aea07f4c122ec6ddcb6d6900bc6fb41e4f81d43 Mon Sep 17 00:00:00 2001 From: Dana Van Aken Date: Tue, 27 Aug 2019 19:21:10 -0400 Subject: [PATCH] Parser cleanup --- server/website/tests/test_parser.py | 59 ++++++++-------- server/website/tests/test_utils.py | 17 +++-- server/website/website/parser/base.py | 84 ++++++++++++++--------- server/website/website/parser/myrocks.py | 46 ++----------- server/website/website/parser/oracle.py | 18 ++--- server/website/website/parser/parser.py | 44 ++++++------ server/website/website/parser/postgres.py | 63 +++-------------- server/website/website/utils.py | 17 +++++ 8 files changed, 149 insertions(+), 199 deletions(-) diff --git a/server/website/tests/test_parser.py b/server/website/tests/test_parser.py index ae81244..d7f45ca 100644 --- a/server/website/tests/test_parser.py +++ b/server/website/tests/test_parser.py @@ -7,9 +7,9 @@ from abc import ABCMeta, abstractmethod import mock from django.test import TestCase -from website.parser.postgres import PostgresParser, Postgres96Parser -from website.types import BooleanType, VarType, KnobUnitType, MetricType -from website.models import KnobCatalog +from website.parser.postgres import PostgresParser +from website.types import BooleanType, DBMSType, VarType, KnobUnitType, MetricType +from website.models import DBMSCatalog, KnobCatalog class BaseParserTests(object, metaclass=ABCMeta): @@ -161,8 +161,10 @@ class BaseParserTests(object, metaclass=ABCMeta): mock_other_knob = mock.Mock(spec=KnobCatalog) mock_other_knob.unit = KnobUnitType.OTHER - self.assertEqual(self.test_dbms.format_bool(BooleanType.TRUE, mock_other_knob), 'on') - self.assertEqual(self.test_dbms.format_bool(BooleanType.FALSE, mock_other_knob), 'off') + self.assertEqual(self.test_dbms.format_bool(BooleanType.TRUE, mock_other_knob), + self.test_dbms.true_value) + self.assertEqual(self.test_dbms.format_bool(BooleanType.FALSE, mock_other_knob), + self.test_dbms.false_value) def test_format_enum(self): mock_enum_knob = mock.Mock(spec=KnobCatalog) @@ -217,13 +219,15 @@ class BaseParserTests(object, metaclass=ABCMeta): pass -class Postgres96ParserTests(BaseParserTests, TestCase): +class PostgresParserTests(BaseParserTests, TestCase): def setUp(self): - self.test_dbms = Postgres96Parser(9.6) + dbms_obj = DBMSCatalog.objects.filter( + type=DBMSType.POSTGRES, version="9.6").first() + self.test_dbms = PostgresParser(dbms_obj) def test_convert_dbms_knobs(self): - super(Postgres96ParserTests, self).test_convert_dbms_knobs() + super().test_convert_dbms_knobs() test_knobs = {'global.wal_sync_method': 'open_sync', # Enum 'global.random_page_cost': 0.22, # Real @@ -250,7 +254,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase): self.assertEqual(self.test_dbms.convert_dbms_knobs(test_nontune_knobs), {}) def test_convert_dbms_metrics(self): - super(Postgres96ParserTests, self).test_convert_dbms_metrics() + super().test_convert_dbms_metrics() test_metrics = {} @@ -361,7 +365,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase): self.assertEqual(nontune_extract.get('global.GEQO_EFFORT'), None) def test_convert_integer(self): - super(Postgres96ParserTests, self).test_convert_integer() + super().test_convert_integer() # Convert Integer knob_unit_bytes = KnobUnitType() @@ -387,6 +391,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase): self.assertEqual(self.test_dbms .convert_integer('10min', knob_unit_time), 600000) self.assertEqual(self.test_dbms.convert_integer('1s', knob_unit_time), 1000) + self.assertEqual(self.test_dbms.convert_integer('5000ms', knob_unit_time), 5000) test_exceptions = [('A', knob_unit_other), ('', knob_unit_other), @@ -400,7 +405,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase): self.test_dbms.convert_integer(failure_case, knob_unit) def test_calculate_change_in_metrics(self): - super(Postgres96ParserTests, self).test_calculate_change_in_metrics() + super().test_calculate_change_in_metrics() test_metric_start = {'pg_stat_bgwriter.buffers_alloc': 256, 'pg_stat_archiver.last_failed_wal': "today", @@ -473,8 +478,6 @@ class Postgres96ParserTests(BaseParserTests, TestCase): self.assertEqual(test_config.get(k), v) def test_format_integer(self): - test_dbms = PostgresParser(2) - knob_unit_bytes = KnobUnitType() knob_unit_bytes.unit = 1 knob_unit_time = KnobUnitType() @@ -482,22 +485,22 @@ class Postgres96ParserTests(BaseParserTests, TestCase): knob_unit_other = KnobUnitType() knob_unit_other.unit = 3 - self.assertEqual(test_dbms.format_integer(5, knob_unit_other), 5) - self.assertEqual(test_dbms.format_integer(0, knob_unit_other), 0) - self.assertEqual(test_dbms.format_integer(-1, knob_unit_other), -1) + self.assertEqual(self.test_dbms.format_integer(5, knob_unit_other), 5) + self.assertEqual(self.test_dbms.format_integer(0, knob_unit_other), 0) + self.assertEqual(self.test_dbms.format_integer(-1, knob_unit_other), -1) - self.assertEqual(test_dbms.format_integer(5120, knob_unit_bytes), '5kB') - self.assertEqual(test_dbms.format_integer(4194304, knob_unit_bytes), '4MB') - self.assertEqual(test_dbms.format_integer(4194500, knob_unit_bytes), '4MB') + self.assertEqual(self.test_dbms.format_integer(5120, knob_unit_bytes), '5kB') + self.assertEqual(self.test_dbms.format_integer(4194304, knob_unit_bytes), '4MB') + self.assertEqual(self.test_dbms.format_integer(4194500, knob_unit_bytes), '4MB') - self.assertEqual(test_dbms.format_integer(86400000, knob_unit_time), '1d') - self.assertEqual(test_dbms.format_integer(72000000, knob_unit_time), '20h') - self.assertEqual(test_dbms.format_integer(600000, knob_unit_time), '10min') - self.assertEqual(test_dbms.format_integer(1000, knob_unit_time), '1s') - self.assertEqual(test_dbms.format_integer(500, knob_unit_time), '500ms') + self.assertEqual(self.test_dbms.format_integer(86400000, knob_unit_time), '1d') + self.assertEqual(self.test_dbms.format_integer(72000000, knob_unit_time), '20h') + self.assertEqual(self.test_dbms.format_integer(600000, knob_unit_time), '10min') + self.assertEqual(self.test_dbms.format_integer(1000, knob_unit_time), '1s') + self.assertEqual(self.test_dbms.format_integer(500, knob_unit_time), '500ms') def test_format_dbms_knobs(self): - super(Postgres96ParserTests, self).test_format_dbms_knobs() + super().test_format_dbms_knobs() test_knobs = {'global.wal_sync_method': 2, # Enum 'global.random_page_cost': 0.22, # Real @@ -520,7 +523,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase): self.assertEqual(test_formatted_knobs.get('global.wal_buffers'), '1kB') def test_filter_numeric_metrics(self): - super(Postgres96ParserTests, self).test_filter_numeric_metrics() + super().test_filter_numeric_metrics() test_metrics = {'pg_stat_bgwriter.checkpoints_req': (2, 'global'), 'pg_stat_archiver.last_failed_wal': (1, 'global'), @@ -549,7 +552,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase): self.assertEqual(filtered_metrics.get('pg_FAKE_KNOB'), None) def test_filter_tunable_knobs(self): - super(Postgres96ParserTests, self).test_filter_tunable_knobs() + super().test_filter_tunable_knobs() test_knobs = {'global.wal_sync_method': 5, 'global.random_page_cost': 3, @@ -571,7 +574,7 @@ class Postgres96ParserTests(BaseParserTests, TestCase): self.assertEqual(filtered_knobs.get('global.FAKE_KNOB'), None) def test_parse_helper(self): - super(Postgres96ParserTests, self).test_parse_helper() + super().test_parse_helper() test_view_vars = {'global': {'wal_sync_method': 'open_sync', 'random_page_cost': 0.22}, diff --git a/server/website/tests/test_utils.py b/server/website/tests/test_utils.py index 5d7a9f3..e0103f4 100644 --- a/server/website/tests/test_utils.py +++ b/server/website/tests/test_utils.py @@ -8,7 +8,6 @@ import string import numpy as np from django.test import TestCase from website.utils import JSONUtil, MediaUtil, DataUtil, ConversionUtil, LabelUtil, TaskUtil -from website.parser.postgres import PostgresParser from website.types import LabelStyleType, VarType from website.models import Result, DBMSCatalog @@ -225,9 +224,9 @@ class DataUtilTest(TestCase): 'global.bgwriter_delay', 'global.wal_writer_delay', 'global.work_mem'] - postgres96 = DBMSCatalog.objects.get(pk=1) + postgresdb = DBMSCatalog.objects.get(pk=1) categorical_info = DataUtil.dummy_encoder_helper(featured_knobs, - dbms=postgres96) + dbms=postgresdb) self.assertEqual(len(categorical_info['n_values']), 0) self.assertEqual(len(categorical_info['categorical_features']), 0) self.assertEqual(categorical_info['cat_columnlabels'], []) @@ -239,9 +238,9 @@ class DataUtilTest(TestCase): 'global.wal_writer_delay', 'global.work_mem', 'global.wal_sync_method'] # last knob categorical - postgres96 = DBMSCatalog.objects.get(pk=1) + postgresdb = DBMSCatalog.objects.get(pk=1) categorical_info = DataUtil.dummy_encoder_helper(featured_knobs, - dbms=postgres96) + dbms=postgresdb) self.assertEqual(len(categorical_info['n_values']), 1) self.assertEqual(categorical_info['n_values'][0], 4) self.assertEqual(len(categorical_info['categorical_features']), 1) @@ -257,7 +256,7 @@ class ConversionUtilTest(TestCase): byte_ans = [1024**5, 2 * 1024**4, 3 * 1024**3, 4 * 1024**2, 5 * 1024**1, 6] for i, byte_test in enumerate(byte_test_convert): byte_conversion = ConversionUtil.get_raw_size( - byte_test, system=PostgresParser.POSTGRES_BYTES_SYSTEM) + byte_test, system=ConversionUtil.DEFAULT_BYTES_SYSTEM) self.assertEqual(byte_conversion, byte_ans[i]) # Time - In Milliseconds @@ -265,7 +264,7 @@ class ConversionUtilTest(TestCase): day_ans = [1000, 1000, 600000, 72000000, 86400000] for i, day_test in enumerate(day_test_convert): day_conversion = ConversionUtil.get_raw_size( - day_test, system=PostgresParser.POSTGRES_TIME_SYSTEM) + day_test, system=ConversionUtil.DEFAULT_TIME_SYSTEM) self.assertEqual(day_conversion, day_ans[i]) def test_get_human_readable(self): @@ -275,7 +274,7 @@ class ConversionUtilTest(TestCase): byte_ans = ['1PB', '2TB', '3GB', '4MB', '5kB', '6B'] for i, byte_test in enumerate(byte_test_convert): byte_readable = ConversionUtil.get_human_readable( - byte_test, system=PostgresParser.POSTGRES_BYTES_SYSTEM) + byte_test, system=ConversionUtil.DEFAULT_BYTES_SYSTEM) self.assertEqual(byte_readable, byte_ans[i]) # Time @@ -283,7 +282,7 @@ class ConversionUtilTest(TestCase): day_ans = ['500ms', '1s', '55s', '10min', '20h', '1d'] for i, day_test in enumerate(day_test_convert): day_readable = ConversionUtil.get_human_readable( - day_test, system=PostgresParser.POSTGRES_TIME_SYSTEM) + day_test, system=ConversionUtil.DEFAULT_TIME_SYSTEM) self.assertEqual(day_readable, day_ans[i]) diff --git a/server/website/website/parser/base.py b/server/website/website/parser/base.py index 494d283..86c2967 100644 --- a/server/website/website/parser/base.py +++ b/server/website/website/parser/base.py @@ -3,13 +3,6 @@ # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # -''' -Created on Dec 12, 2017 - -@author: dvanaken - -Parser interface. -''' from abc import ABCMeta, abstractmethod, abstractproperty from collections import OrderedDict @@ -21,20 +14,24 @@ from website.types import BooleanType, MetricType, VarType # pylint: disable=no-self-use class BaseParser(object, metaclass=ABCMeta): - def __init__(self, dbms_id): - self.dbms_id_ = dbms_id - knobs = KnobCatalog.objects.filter(dbms__pk=self.dbms_id_) + def __init__(self, dbms_obj): + knobs = KnobCatalog.objects.filter(dbms=dbms_obj) self.knob_catalog_ = {k.name: k for k in knobs} - self.tunable_knob_catalog_ = {k: v for k, v in - list(self.knob_catalog_.items()) if v.tunable is True} - metrics = MetricCatalog.objects.filter(dbms__pk=self.dbms_id_) + self.tunable_knob_catalog_ = { + k: v for k, v in self.knob_catalog_.items() if + v.tunable is True} + + metrics = MetricCatalog.objects.filter(dbms=dbms_obj) self.metric_catalog_ = {m.name: m for m in metrics} - self.numeric_metric_catalog_ = {m: v for m, v in - list(self.metric_catalog_.items()) if - v.metric_type == MetricType.COUNTER or - v.metric_type == MetricType.STATISTICS} - self.valid_true_val = list() - self.valid_false_val = list() + numeric_mtypes = (MetricType.COUNTER, MetricType.STATISTICS) + self.numeric_metric_catalog_ = { + m: v for m, v in self.metric_catalog_.items() if + v.metric_type in numeric_mtypes} + + self.valid_true_val = ("on", "true", "yes") + self.valid_false_val = ("off", "false", "no") + self.true_value = 'on' + self.false_value = 'off' @abstractproperty def base_configuration_settings(self): @@ -55,38 +52,49 @@ class BaseParser(object, metaclass=ABCMeta): def target_metric(self, target_objective=None): if target_objective == 'throughput_txn_per_sec' or target_objective is None: # throughput - return self.transactions_counter + res = self.transactions_counter elif target_objective == '99th_lat_ms': # 99 percentile latency - return self.latency_timer + res = self.latency_timer else: raise Exception("Target Objective {} Not Supported".format(target_objective)) + return res + @abstractmethod def parse_version_string(self, version_string): pass def convert_bool(self, bool_value, metadata): + if isinstance(bool_value, str): + bool_value = bool_value.lower() + if bool_value in self.valid_true_val: - return BooleanType.TRUE + res = BooleanType.TRUE elif bool_value in self.valid_false_val: - return BooleanType.FALSE + res = BooleanType.FALSE else: raise Exception("Invalid Boolean {}".format(bool_value)) + return res + def convert_enum(self, enum_value, metadata): enumvals = metadata.enumvals.split(',') try: - return enumvals.index(enum_value) + res = enumvals.index(enum_value) except ValueError: raise Exception('Invalid enum value for variable {} ({})'.format( metadata.name, enum_value)) + return res + def convert_integer(self, int_value, metadata): try: - return int(int_value) + res = int(int_value) except ValueError: - return int(float(int_value)) + res = int(float(int_value)) + + return res def convert_real(self, real_value, metadata): return float(real_value) @@ -115,6 +123,7 @@ class BaseParser(object, metaclass=ABCMeta): continue value = knobs[name] conv_value = None + if metadata.vartype == VarType.BOOL: if not self._check_knob_bool_val(value): raise Exception('Knob boolean value not valid! ' @@ -123,8 +132,10 @@ class BaseParser(object, metaclass=ABCMeta): .format(self.valid_boolean_val_to_string(), str(value))) conv_value = self.convert_bool(value, metadata) + elif metadata.vartype == VarType.ENUM: conv_value = self.convert_enum(value, metadata) + elif metadata.vartype == VarType.INTEGER: conv_value = self.convert_integer(value, metadata) if not self._check_knob_num_in_range(conv_value, metadata): @@ -132,6 +143,7 @@ class BaseParser(object, metaclass=ABCMeta): 'min: {}, max: {}, actual: {}' .format(metadata.minval, metadata.maxval, str(conv_value))) + elif metadata.vartype == VarType.REAL: conv_value = self.convert_real(value, metadata) if not self._check_knob_num_in_range(conv_value, metadata): @@ -139,23 +151,29 @@ class BaseParser(object, metaclass=ABCMeta): 'min: {}, max: {}, actual: {}' .format(metadata.minval, metadata.maxval, str(conv_value))) + elif metadata.vartype == VarType.STRING: conv_value = self.convert_string(value, metadata) + elif metadata.vartype == VarType.TIMESTAMP: conv_value = self.convert_timestamp(value, metadata) + else: raise Exception( 'Unknown variable type: {}'.format(metadata.vartype)) + if conv_value is None: - raise Exception( - 'Param value for {} cannot be null'.format(name)) + raise Exception('Param value for {} cannot be null'.format(name)) knob_data[name] = conv_value + return knob_data def _check_knob_num_in_range(self, value, mdata): return value >= float(mdata.minval) and value <= float(mdata.maxval) def _check_knob_bool_val(self, value): + if isinstance(str, value): + value = value.lower() return value in self.valid_true_val or value in self.valid_false_val def convert_dbms_metrics(self, metrics, observation_time, target_objective=None): @@ -331,7 +349,7 @@ class BaseParser(object, metaclass=ABCMeta): return nondefault_settings def format_bool(self, bool_value, metadata): - return 'on' if bool_value == BooleanType.TRUE else 'off' + return self.true_value if bool_value == BooleanType.TRUE else self.false_value def format_enum(self, enum_value, metadata): enumvals = metadata.enumvals.split(',') @@ -378,11 +396,11 @@ class BaseParser(object, metaclass=ABCMeta): return formatted_knobs def filter_numeric_metrics(self, metrics): - return OrderedDict([(k, v) for k, v in list(metrics.items()) if - k in self.numeric_metric_catalog_]) + return OrderedDict(((k, v) for k, v in list(metrics.items()) if + k in self.numeric_metric_catalog_)) def filter_tunable_knobs(self, knobs): - return OrderedDict([(k, v) for k, v in list(knobs.items()) if - k in self.tunable_knob_catalog_]) + return OrderedDict(((k, v) for k, v in list(knobs.items()) if + k in self.tunable_knob_catalog_)) # pylint: enable=no-self-use diff --git a/server/website/website/parser/myrocks.py b/server/website/website/parser/myrocks.py index bbb4ce6..fbfc5ae 100644 --- a/server/website/website/parser/myrocks.py +++ b/server/website/website/parser/myrocks.py @@ -3,40 +3,17 @@ # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # -''' -Created on Jan 16, 2018 - -@author: bohan -''' import re from collections import OrderedDict from .base import BaseParser -from website.models import DBMSCatalog -from website.types import DBMSType, KnobUnitType, MetricType, VarType +from website.types import KnobUnitType, MetricType, VarType from website.utils import ConversionUtil class MyRocksParser(BaseParser): - MYROCKS_BYTES_SYSTEM = [ - (1024 ** 5, 'PB'), - (1024 ** 4, 'TB'), - (1024 ** 3, 'GB'), - (1024 ** 2, 'MB'), - (1024 ** 1, 'kB'), - (1024 ** 0, 'B'), - ] - - MYROCKS_TIME_SYSTEM = [ - (1000 * 60 * 60 * 24, 'd'), - (1000 * 60 * 60, 'h'), - (1000 * 60, 'min'), - (1, 'ms'), - (1000, 's'), - ] - MYROCKS_BASE_KNOBS = { 'session_variables.rocksdb_max_open_files': '-1' } @@ -59,15 +36,15 @@ class MyRocksParser(BaseParser): def convert_integer(self, int_value, metadata): converted = None try: - converted = super(MyRocksParser, self).convert_integer( + converted = super().convert_integer( int_value, metadata) except ValueError: if metadata.unit == KnobUnitType.BYTES: converted = ConversionUtil.get_raw_size( - int_value, system=self.MYROCKS_BYTES_SYSTEM) + int_value, system=ConversionUtil.DEFAULT_BYTES_SYSTEM) elif metadata.unit == KnobUnitType.MILLISECONDS: converted = ConversionUtil.get_raw_size( - int_value, system=self.MYROCKS_TIME_SYSTEM) + int_value, system=ConversionUtil.DEFAULT_TIME_SYSTEM) else: raise Exception('Unknown unit type: {}'.format(metadata.unit)) if converted is None: @@ -79,16 +56,15 @@ class MyRocksParser(BaseParser): if metadata.unit != KnobUnitType.OTHER and int_value > 0: if metadata.unit == KnobUnitType.BYTES: int_value = ConversionUtil.get_human_readable( - int_value, MyRocksParser.MYROCKS_BYTES_SYSTEM) + int_value, ConversionUtil.DEFAULT_BYTES_SYSTEM) elif metadata.unit == KnobUnitType.MILLISECONDS: int_value = ConversionUtil.get_human_readable( - int_value, MyRocksParser.MYROCKS_TIME_SYSTEM) + int_value, ConversionUtil.DEFAULT_TIME_SYSTEM) else: raise Exception('Invalid unit type for {}: {}'.format( metadata.name, metadata.unit)) else: - int_value = super(MyRocksParser, self).format_integer( - int_value, metadata) + int_value = super().format_integer(int_value, metadata) return int_value def parse_version_string(self, version_string): @@ -277,11 +253,3 @@ class MyRocksParser(BaseParser): def filter_tunable_knobs(self, knobs): return OrderedDict([(k, v) for k, v in list(knobs.items()) if MyRocksParser.partial_name(k) in self.tunable_knob_catalog_]) - - -class MyRocks56Parser(MyRocksParser): - - def __init__(self): - dbms = DBMSCatalog.objects.get( - type=DBMSType.MYROCKS, version='5.6') - super(MyRocks56Parser, self).__init__(dbms.pk) diff --git a/server/website/website/parser/oracle.py b/server/website/website/parser/oracle.py index e0c8985..5520c2f 100644 --- a/server/website/website/parser/oracle.py +++ b/server/website/website/parser/oracle.py @@ -5,16 +5,14 @@ # from .base import BaseParser -from website.models import DBMSCatalog -from website.types import DBMSType class OracleParser(BaseParser): - def __init__(self, dbms_id): - super(OracleParser, self).__init__(dbms_id) - self.valid_true_val = ["TRUE", "true", "yes", 1] - self.valid_false_val = ["FALSE", "false", "no", 0] + def __init__(self, dbms_obj): + super().__init__(dbms_obj) + self.true_value = 'TRUE' + self.false_value = 'FALSE' ORACLE_BASE_KNOBS = { } @@ -37,11 +35,3 @@ class OracleParser(BaseParser): def parse_version_string(self, version_string): return version_string - - -class Oracle19Parser(OracleParser): - - def __init__(self): - dbms = DBMSCatalog.objects.get( - type=DBMSType.ORACLE, version='19.0.0.0.0') - super(Oracle19Parser, self).__init__(dbms.pk) diff --git a/server/website/website/parser/parser.py b/server/website/website/parser/parser.py index 2209a41..13bcac9 100644 --- a/server/website/website/parser/parser.py +++ b/server/website/website/parser/parser.py @@ -3,42 +3,38 @@ # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # -''' -Created on Dec 12, 2017 - -@author: dvanaken -''' - from website.models import DBMSCatalog from website.types import DBMSType -from .myrocks import MyRocks56Parser -from .postgres import Postgres96Parser, PostgresOldParser -from .oracle import Oracle19Parser +from .myrocks import MyRocksParser +from .postgres import PostgresParser +from .oracle import OracleParser -class Parser(object): +class Parser(): __DBMS_UTILS_IMPLS = None @staticmethod def __utils(dbms_id=None): if Parser.__DBMS_UTILS_IMPLS is None: - Parser.__DBMS_UTILS_IMPLS = { - DBMSCatalog.objects.get( - type=DBMSType.POSTGRES, version='9.3').pk: PostgresOldParser('9.3'), - DBMSCatalog.objects.get( - type=DBMSType.POSTGRES, version='9.2').pk: PostgresOldParser('9.2'), - DBMSCatalog.objects.get( - type=DBMSType.POSTGRES, version='9.6').pk: Postgres96Parser('9.6'), - DBMSCatalog.objects.get( - type=DBMSType.POSTGRES, version='9.4').pk: Postgres96Parser('9.4'), - DBMSCatalog.objects.get( - type=DBMSType.MYROCKS, version='5.6').pk: MyRocks56Parser(), - DBMSCatalog.objects.get( - type=DBMSType.ORACLE, version='19.0.0.0.0').pk: Oracle19Parser() - } + + parsers = {} + for obj in DBMSCatalog.objects.all(): + if obj.type == DBMSType.POSTGRES: + clz = PostgresParser + elif obj.type == DBMSType.MYROCKS: + clz = MyRocksParser + elif obj.type == DBMSType.ORACLE: + clz = OracleParser + else: + raise NotImplementedError('Implement me! {}'.format(obj)) + + parsers[obj.pk] = clz(obj) + + Parser.__DBMS_UTILS_IMPLS = parsers + try: if dbms_id is None: return Parser.__DBMS_UTILS_IMPLS diff --git a/server/website/website/parser/postgres.py b/server/website/website/parser/postgres.py index f8acf8b..00000e0 100644 --- a/server/website/website/parser/postgres.py +++ b/server/website/website/parser/postgres.py @@ -3,43 +3,20 @@ # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # -''' -Created on Dec 12, 2017 - -@author: dvanaken -''' import re from .base import BaseParser -from website.models import DBMSCatalog -from website.types import DBMSType, KnobUnitType +from website.types import KnobUnitType from website.utils import ConversionUtil class PostgresParser(BaseParser): - def __init__(self, dbms_id): - super(PostgresParser, self).__init__(dbms_id) - self.valid_true_val = ["on", "true", "yes", 1] - self.valid_false_val = ["off", "false", "no", 0] - - POSTGRES_BYTES_SYSTEM = [ - (1024 ** 5, 'PB'), - (1024 ** 4, 'TB'), - (1024 ** 3, 'GB'), - (1024 ** 2, 'MB'), - (1024 ** 1, 'kB'), - (1024 ** 0, 'B'), - ] - - POSTGRES_TIME_SYSTEM = [ - (1000 * 60 * 60 * 24, 'd'), - (1000 * 60 * 60, 'h'), - (1000 * 60, 'min'), - (1000, 's'), - (1, 'ms'), - ] + def __init__(self, dbms_obj): + super().__init__(dbms_obj) + self.valid_true_val = ("on", "true", "yes", 1) + self.valid_false_val = ("off", "false", "no", 0) POSTGRES_BASE_KNOBS = { 'global.data_directory': None, @@ -76,15 +53,14 @@ class PostgresParser(BaseParser): def convert_integer(self, int_value, metadata): converted = None try: - converted = super(PostgresParser, self).convert_integer( - int_value, metadata) + converted = super().convert_integer(int_value, metadata) except ValueError: if metadata.unit == KnobUnitType.BYTES: converted = ConversionUtil.get_raw_size( - int_value, system=self.POSTGRES_BYTES_SYSTEM) + int_value, system=ConversionUtil.DEFAULT_BYTES_SYSTEM) elif metadata.unit == KnobUnitType.MILLISECONDS: converted = ConversionUtil.get_raw_size( - int_value, system=self.POSTGRES_TIME_SYSTEM) + int_value, system=ConversionUtil.DEFAULT_TIME_SYSTEM) else: raise Exception( 'Unknown unit type: {}'.format(metadata.unit)) @@ -98,35 +74,18 @@ class PostgresParser(BaseParser): if metadata.unit != KnobUnitType.OTHER and int_value > 0: if metadata.unit == KnobUnitType.BYTES: int_value = ConversionUtil.get_human_readable( - int_value, PostgresParser.POSTGRES_BYTES_SYSTEM) + int_value, ConversionUtil.DEFAULT_BYTES_SYSTEM) elif metadata.unit == KnobUnitType.MILLISECONDS: int_value = ConversionUtil.get_human_readable( - int_value, PostgresParser.POSTGRES_TIME_SYSTEM) + int_value, ConversionUtil.DEFAULT_TIME_SYSTEM) else: raise Exception( 'Invalid unit type for {}: {}'.format( metadata.name, metadata.unit)) else: - int_value = super(PostgresParser, self).format_integer( - int_value, metadata) + int_value = super().format_integer(int_value, metadata) return int_value def parse_version_string(self, version_string): dbms_version = version_string.split(',')[0] return re.search(r'\d+\.\d+(?=\.\d+)', dbms_version).group(0) - - -class Postgres96Parser(PostgresParser): - - def __init__(self, version): - dbms = DBMSCatalog.objects.get( - type=DBMSType.POSTGRES, version=version) - super(Postgres96Parser, self).__init__(dbms.pk) - - -class PostgresOldParser(PostgresParser): - - def __init__(self, version): - dbms = DBMSCatalog.objects.get( - type=DBMSType.POSTGRES, version=version) - super(PostgresOldParser, self).__init__(dbms.pk) diff --git a/server/website/website/utils.py b/server/website/website/utils.py index 59784ee..40b2e2e 100644 --- a/server/website/website/utils.py +++ b/server/website/website/utils.py @@ -205,6 +205,23 @@ class DataUtil(object): class ConversionUtil(object): + DEFAULT_BYTES_SYSTEM = ( + (1024 ** 5, 'PB'), + (1024 ** 4, 'TB'), + (1024 ** 3, 'GB'), + (1024 ** 2, 'MB'), + (1024 ** 1, 'kB'), + (1024 ** 0, 'B'), + ) + + DEFAULT_TIME_SYSTEM = ( + (1000 * 60 * 60 * 24, 'd'), + (1000 * 60 * 60, 'h'), + (1000 * 60, 'min'), + (1000, 's'), + (1, 'ms'), + ) + @staticmethod def get_raw_size(value, system): for factor, suffix in system: