public/ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

support mysql

Browse Source
This commit is contained in:
bohanjason
2020-04-23 03:46:58 -04:00
committed by Dana Van Aken
parent 5b697b9c56
commit 5c422dd010
18 changed files with 13326 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
server/website/website/db/base/parser.py
View File

@@ -89,8 +89,11 @@ class BaseParser:
return converted
def convert_real(self, real_value, metadata):
return float(real_value)
try:
return float(real_value)
except ValueError:
raise Exception('Cannot convert knob {} from {} to float'.format(
metadata.name, real_value))
def convert_string(self, string_value, metadata):
return string_value
@@ -124,10 +127,10 @@ class BaseParser:
if metadata.vartype == VarType.BOOL:
if not self._check_knob_bool_val(value):
raise Exception('Knob boolean value not valid! '
raise Exception('Knob {} boolean value not valid! '
'Boolean values should be one of: {}, '
'but the actual value is: {}'
.format(self.valid_boolean_val_to_string(),
.format(name, self.valid_boolean_val_to_string(),
str(value)))
conv_value = self.convert_bool(value, metadata)
@@ -137,17 +140,17 @@ class BaseParser:
elif metadata.vartype == VarType.INTEGER:
conv_value = self.convert_integer(value, metadata)
if not self._check_knob_num_in_range(conv_value, metadata):
raise Exception('Knob integer num value not in range! '
raise Exception('Knob {} integer num value not in range! '
'min: {}, max: {}, actual: {}'
.format(metadata.minval,
.format(name, metadata.minval,
metadata.maxval, str(conv_value)))
elif metadata.vartype == VarType.REAL:
conv_value = self.convert_real(value, metadata)
if not self._check_knob_num_in_range(conv_value, metadata):
raise Exception('Knob real num value not in range! '
raise Exception('Knob {} real num value not in range! '
'min: {}, max: {}, actual: {}'
.format(metadata.minval,
.format(name, metadata.minval,
metadata.maxval, str(conv_value)))
elif metadata.vartype == VarType.STRING:
@@ -328,7 +331,8 @@ class BaseParser:
'Invalid metric type: {}'.format(metric.metric_type))
return valid_metrics, diffs
def calculate_change_in_metrics(self, metrics_start, metrics_end, fix_metric_type=True):
def calculate_change_in_metrics(self, metrics_start, metrics_end,
fix_metric_type=True, allow_negative=True):
metric_catalog = {m.name: m for m in MetricCatalog.objects.filter(dbms__id=self.dbms_id)}
adjusted_metrics = {}
@@ -350,13 +354,18 @@ class BaseParser:
if fix_metric_type:
if adj_val < 0:
adj_val = end_val
LOG.debug("Changing metric %s from COUNTER to STATISTICS", met_name)
LOG.warning("Changing metric %s from COUNTER to STATISTICS", met_name)
met_info.metric_type = MetricType.STATISTICS
met_info.save()
assert adj_val >= 0, \
'{} wrong metric type: {} (start={}, end={}, diff={})'.format(
met_name, MetricType.name(met_info.metric_type), start_val,
end_val, end_val - start_val)
if allow_negative:
LOG.warning('%s metric type %s value is negative (start=%s, end=%s, diff=%s)',
met_name, MetricType.name(met_info.metric_type), start_val, end_val,
end_val - start_val)
else:
assert adj_val >= 0, \
'{} wrong metric type: {} (start={}, end={}, diff={})'.format(
met_name, MetricType.name(met_info.metric_type), start_val,
end_val, end_val - start_val)
adjusted_metrics[met_name] = adj_val
else:

3
server/website/website/db/base/target_objective.py
View File

@@ -87,10 +87,11 @@ class TargetObjectives:
from ..myrocks.target_objective import target_objective_list as _myrocks_list # pylint: disable=import-outside-toplevel
from ..oracle.target_objective import target_objective_list as _oracle_list # pylint: disable=import-outside-toplevel
from ..postgres.target_objective import target_objective_list as _postgres_list # pylint: disable=import-outside-toplevel
from ..mysql.target_objective import target_objective_list as _mysql_list # pylint: disable=import-outside-toplevel
if not self.registered():
LOG.info('Registering target objectives...')
full_list = _myrocks_list + _oracle_list + _postgres_list
full_list = _myrocks_list + _oracle_list + _postgres_list + _mysql_list
for dbms_type, target_objective_instance in full_list:
dbmss = models.DBMSCatalog.objects.filter(type=dbms_type)
name = target_objective_instance.name

53
server/website/website/db/mysql/parser.py Normal file
View File

@@ -0,0 +1,53 @@
#
# OtterTune - parser.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from website.types import KnobUnitType
from website.utils import ConversionUtil
from ..base.parser import BaseParser # pylint: disable=relative-beyond-top-level
# pylint: disable=no-self-use
class MysqlParser(BaseParser):
def __init__(self, dbms_obj):
super().__init__(dbms_obj)
self.bytes_system = (
(1024 ** 4, 'T'),
(1024 ** 3, 'G'),
(1024 ** 2, 'M'),
(1024 ** 1, 'k'),
)
self.time_system = None
self.min_bytes_unit = 'k'
self.valid_true_val = ("on", "true", "yes", '1', 'enabled')
self.valid_false_val = ("off", "false", "no", '0', 'disabled')
def convert_integer(self, int_value, metadata):
# Collected knobs/metrics do not show unit, convert to int directly
if len(str(int_value)) == 0:
# The value collected from the database is empty
return 0
try:
try:
converted = int(int_value)
except ValueError:
converted = int(float(int_value))
except ValueError:
raise Exception('Invalid integer format for {}: {}'.format(
metadata.name, int_value))
return converted
def format_integer(self, int_value, metadata):
int_value = int(round(int_value))
if int_value > 0 and metadata.unit == KnobUnitType.BYTES:
int_value = ConversionUtil.get_human_readable2(
int_value, self.bytes_system, self.min_bytes_unit)
return int_value
def parse_version_string(self, version_string):
s = version_string.split('.')[0] + '.' + version_string.split('.')[1]
return s

14
server/website/website/db/mysql/target_objective.py Normal file
View File

@@ -0,0 +1,14 @@
#
# OtterTune - target_objective.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from website.types import DBMSType
from ..base.target_objective import BaseThroughput # pylint: disable=relative-beyond-top-level
target_objective_list = tuple((DBMSType.MYSQL, target_obj) for target_obj in [ # pylint: disable=invalid-name
BaseThroughput(transactions_counter=('innodb_metrics.trx_rw_commits',
'innodb_metrics.trx_ro_commits',
'innodb_metrics.trx_nl_ro_commits'))
])

3
server/website/website/db/parser.py
View File

@@ -10,6 +10,7 @@ from website.types import DBMSType
from .myrocks.parser import MyRocksParser
from .postgres.parser import PostgresParser
from .oracle.parser import OracleParser
from .mysql.parser import MysqlParser
_DBMS_PARSERS = {}
@@ -25,6 +26,8 @@ def _get(dbms_id):
clz = MyRocksParser
elif obj.type == DBMSType.ORACLE:
clz = OracleParser
elif obj.type == DBMSType.MYSQL:
clz = MysqlParser
else:
raise NotImplementedError('Implement me! {}'.format(obj))

24
server/website/website/fixtures/dbms_catalog.json
View File

@@ -39,6 +39,30 @@
"version":"5.6"
}
},
{
"model":"website.DBMSCatalog",
"pk":11,
"fields":{
"type":1,
"version":"5.6"
}
},
{
"model":"website.DBMSCatalog",
"pk":12,
"fields":{
"type":1,
"version":"5.7"
}
},
{
"model":"website.DBMSCatalog",
"pk":13,
"fields":{
"type":1,
"version":"8.0"
}
},
{
"model":"website.DBMSCatalog",
"pk":121,

8842
server/website/website/fixtures/mysql-56_knobs.json Normal file
View File

File diff suppressed because it is too large Load Diff

4178
server/website/website/fixtures/mysql-56_metrics.json Normal file
View File

File diff suppressed because it is too large Load Diff

2
server/website/website/migrations/0001_initial.py
View File

@@ -59,7 +59,7 @@ class Migration(migrations.Migration):
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128)),
('vartype', models.IntegerField(choices=[(1, 'STRING'), (2, 'INTEGER'), (3, 'REAL'), (4, 'BOOL'), (5, 'ENUM'), (6, 'TIMESTAMP')], verbose_name='variable type')),
('unit', models.IntegerField(choices=[(1, 'bytes'), (2, 'milliseconds'), (3, 'other')])),
('unit', models.IntegerField(choices=[(1, 'bytes'), (2, 'milliseconds'), (3, 'other'), (4, 'microseconds'), (5, 'seconds')])),
('category', models.TextField(null=True)),
('summary', models.TextField(null=True, verbose_name='description')),
('description', models.TextField(null=True)),

2
server/website/website/migrations/0003_load_initial_data.py
View File

@@ -25,6 +25,8 @@ def load_initial_data(apps, schema_editor):
"oracle-121_metrics.json",
"oracle-19_knobs.json",
"oracle-19_metrics.json",
"mysql-56_knobs.json",
"mysql-56_metrics.json",
]
for fixture in initial_data_fixtures:
call_command("loaddata", fixture, app_label="website")

11
server/website/website/set_default_knobs.py
View File

@@ -35,6 +35,17 @@ DEFAULT_TUNABLE_KNOBS = {
"global.shared_pool_size",
"global.sort_area_size",
},
DBMSType.MYSQL: {
"global.innodb_buffer_pool_size",
"global.innodb_thread_sleep_delay",
"global.innodb_flush_method",
"global.innodb_log_file_size",
"global.innodb_max_dirty_pages_pct_lwm",
"global.innodb_read_ahead_threshold",
"global.innodb_adaptive_max_sleep_delay",
"global.innodb_buffer_pool_instances",
"global.thread_cache_size",
},
}
# Bytes in a GB

4
server/website/website/types.py
View File

@@ -130,11 +130,15 @@ class KnobUnitType(BaseType):
BYTES = 1
MILLISECONDS = 2
OTHER = 3
MICROSECONDS = 4
SECONDS = 5
TYPE_NAMES = {
BYTES: 'bytes',
MILLISECONDS: 'milliseconds',
OTHER: 'other',
MICROSECONDS: 'microseconds',
SECONDS: 'seconds',
}

15
server/website/website/views.py
View File

@@ -611,8 +611,19 @@ def handle_result_files(session, files, execution_times=None):
dbms = DBMSCatalog.objects.get(
type=dbms_type, version=dbms_version)
except ObjectDoesNotExist:
return HttpResponse('{} v{} is not yet supported.'.format(
dbms_type, dbms_version))
try:
dbms_version = parser.parse_version_string(dbms_type, dbms_version)
except Exception: # pylint: disable=broad-except
LOG.warning('Cannot parse dbms version %s', dbms_version)
return HttpResponse('{} v{} is not yet supported.'.format(
dbms_type, dbms_version))
try:
# Check that we support this DBMS and version
dbms = DBMSCatalog.objects.get(
type=dbms_type, version=dbms_version)
except ObjectDoesNotExist:
return HttpResponse('{} v{} is not yet supported.'.format(
dbms_type, dbms_version))
if dbms != session.dbms:
return HttpResponse('The DBMS must match the type and version '