public/ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit with BSL

Browse Source
This commit is contained in:
Andy Pavlo
2019-08-23 11:47:19 -04:00
commit 3e564ce922
286 changed files with 177642 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

255
server/website/website/migrations/0001_initial.py Normal file
View File

@@ -0,0 +1,255 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.10.1 on 2018-03-26 02:21
from django.conf import settings
import django.core.validators
from django.db import migrations, models
import django.db.models.deletion
import re
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='BackupData',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('raw_knobs', models.TextField()),
('raw_initial_metrics', models.TextField()),
('raw_final_metrics', models.TextField()),
('raw_summary', models.TextField()),
('knob_log', models.TextField()),
('metric_log', models.TextField()),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='DBMSCatalog',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('type', models.IntegerField(choices=[(1, b'MySQL'), (2, b'Postgres'), (3, b'Db2'), (4, b'Oracle'), (5, b'SQL Server'), (6, b'SQLite'), (7, b'HStore'), (8, b'Vector'), (9, b'MyRocks')])),
('version', models.CharField(max_length=16)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Hardware',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('type', models.IntegerField()),
('name', models.CharField(max_length=32)),
('cpu', models.IntegerField()),
('memory', models.FloatField()),
('storage', models.CharField(max_length=64, validators=[django.core.validators.RegexValidator(re.compile('^\\d+(?:\\,\\d+)*\\Z'), code='invalid', message='Enter only digits separated by commas.')])),
('storage_type', models.CharField(max_length=16)),
('additional_specs', models.TextField(null=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='KnobCatalog',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128)),
('vartype', models.IntegerField(choices=[(1, b'STRING'), (2, b'INTEGER'), (3, b'REAL'), (4, b'BOOL'), (5, b'ENUM'), (6, b'TIMESTAMP')], verbose_name=b'variable type')),
('unit', models.IntegerField(choices=[(1, b'bytes'), (2, b'milliseconds'), (3, b'other')])),
('category', models.TextField(null=True)),
('summary', models.TextField(null=True, verbose_name=b'description')),
('description', models.TextField(null=True)),
('scope', models.CharField(max_length=16)),
('minval', models.CharField(max_length=32, null=True, verbose_name=b'minimum value')),
('maxval', models.CharField(max_length=32, null=True, verbose_name=b'maximum value')),
('default', models.TextField(verbose_name=b'default value')),
('enumvals', models.TextField(null=True, verbose_name=b'valid values')),
('context', models.CharField(max_length=32)),
('tunable', models.BooleanField(verbose_name=b'tunable')),
('resource', models.IntegerField(choices=[(1, b'Memory'), (2, b'CPU'), (3, b'Storage'), (4, b'Other')], default=4)),
('dbms', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.DBMSCatalog')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='KnobData',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=50)),
('creation_time', models.DateTimeField()),
('data', models.TextField()),
('knobs', models.TextField()),
('dbms', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.DBMSCatalog')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='MetricCatalog',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128)),
('vartype', models.IntegerField(choices=[(1, b'STRING'), (2, b'INTEGER'), (3, b'REAL'), (4, b'BOOL'), (5, b'ENUM'), (6, b'TIMESTAMP')])),
('summary', models.TextField(null=True, verbose_name=b'description')),
('scope', models.CharField(max_length=16)),
('metric_type', models.IntegerField(choices=[(1, b'COUNTER'), (2, b'INFO'), (3,b'STATISTICS')])),
('dbms', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.DBMSCatalog')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='MetricData',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=50)),
('creation_time', models.DateTimeField()),
('data', models.TextField()),
('metrics', models.TextField()),
('dbms', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.DBMSCatalog')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='PipelineData',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('task_type', models.IntegerField(choices=[(1, b'Pruned Metrics'), (2, b'Ranked Knobs'), (3, b'Knob Data'), (4, b'Metric Data')])),
('data', models.TextField()),
('creation_time', models.DateTimeField()),
],
),
migrations.CreateModel(
name='PipelineRun',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('start_time', models.DateTimeField()),
('end_time', models.DateTimeField(null=True)),
],
options={
'ordering': ['-id'],
},
),
migrations.CreateModel(
name='Project',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=64, verbose_name=b'project name')),
('description', models.TextField(blank=True, null=True)),
('creation_time', models.DateTimeField()),
('last_update', models.DateTimeField()),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Result',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('creation_time', models.DateTimeField()),
('observation_start_time', models.DateTimeField()),
('observation_end_time', models.DateTimeField()),
('observation_time', models.FloatField()),
('task_ids', models.CharField(max_length=180, null=True)),
('next_configuration', models.TextField(null=True)),
('dbms', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.DBMSCatalog')),
('knob_data', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.KnobData')),
('metric_data', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.MetricData')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Session',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=64, verbose_name=b'session name')),
('description', models.TextField(blank=True, null=True)),
('creation_time', models.DateTimeField()),
('last_update', models.DateTimeField()),
('upload_code', models.CharField(max_length=30, unique=True)),
('tuning_session', models.CharField(choices=[('tuning_sesion', 'Tuning Session'), ('no_tuning_session', 'No Tuning'), ('randomly_generate', 'Randomly Generate')], max_length=64)),
('target_objective', models.CharField(choices=[(b'throughput_txn_per_sec', b'Throughput'), (b'99th_lat_ms', b'99 Percentile Latency')], max_length=64, null=True)),
('nondefault_settings', models.TextField(null=True)),
('dbms', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.DBMSCatalog')),
('hardware', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Hardware')),
('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Project')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Workload',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128, verbose_name=b'workload name')),
('dbms', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.DBMSCatalog')),
('hardware', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Hardware')),
],
),
migrations.AddField(
model_name='result',
name='session',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Session', verbose_name=b'session name'),
),
migrations.AddField(
model_name='result',
name='workload',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Workload'),
),
migrations.AddField(
model_name='pipelinedata',
name='pipeline_run',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.PipelineRun'),
),
migrations.AddField(
model_name='pipelinedata',
name='workload',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Workload'),
),
migrations.AddField(
model_name='metricdata',
name='session',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Session'),
),
migrations.AddField(
model_name='knobdata',
name='session',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Session'),
),
migrations.AddField(
model_name='backupdata',
name='result',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Result'),
),
migrations.AlterUniqueTogether(
name='workload',
unique_together=set([('dbms', 'hardware', 'name')]),
),
migrations.AlterUniqueTogether(
name='pipelinedata',
unique_together=set([('pipeline_run', 'task_type', 'workload')]),
),
]

59
server/website/website/migrations/0002_enable_compression.py Normal file
View File

@@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
import logging
from django.conf import settings
#from django.core.exceptions import ProgrammingError
from django.db import connection, migrations, ProgrammingError
LOG = logging.getLogger(__name__)
TABLES_TO_COMPRESS = [
"website_backupdata",
"website_knobdata",
"website_metricdata",
"website_pipelinedata",
]
class Migration(migrations.Migration):
dependencies = [
('website', '0001_initial'),
]
try:
LOG.info('***** VENDOR: %s', connection.vendor)
if connection.vendor == 'mysql':
version = (0, 0, 0)
with connection.cursor() as cursor:
cursor.execute('SELECT VERSION()')
version = cursor.fetchone()[0]
version = version.split('-')[0]
version = version.split('.')
version = tuple(int(v) for v in version)
LOG.info('***** DB VERSION: %s', version)
if version >= (5, 7, 0):
operations = [
migrations.RunSQL(["ALTER TABLE " + table_name + " COMPRESSION='zlib';",
"OPTIMIZE TABLE " + table_name + ";"],
["ALTER TABLE " + table_name + " COMPRESSION='none';",
"OPTIMIZE TABLE " + table_name + ";"])
for table_name in TABLES_TO_COMPRESS
]
LOG.info('***** DONE ENABLING COMPRESSION')
else:
operations = []
LOG.info('***** COMPRESSION NOT SUPPORTED: %s < (5, 7, 0)', version)
else:
LOG.info('***** DB COMPRESSION NOT SUPPORTED: %s', connection.vendor)
except ProgrammingError as err:
LOG.warning("Error applying migration '0002_enable_compression'... Skipping")
operations = []

21
server/website/website/migrations/0003_background_task_optimization.py Normal file
View File

@@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.10.1 on 2018-08-02 07:58
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('website', '0002_enable_compression'),
]
operations = [
migrations.AddField(
model_name='workload',
name='status',
field=models.IntegerField(choices=[(1, 'MODIFIED'), (2, 'PROCESSING'), (3, 'PROCESSED')], default=1, editable=False),
)
]

49
server/website/website/migrations/0004_load_initial_data.py Normal file
View File

@@ -0,0 +1,49 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.10.1 on 2017-12-07 19:51
from django.core.management import call_command
from django.db import migrations
def load_initial_data(apps, schema_editor):
initial_data_fixtures = [
"dbms_catalog.json",
"postgres-96_knobs.json",
"postgres-96_metrics.json",
"postgres-92_knobs.json",
"postgres-92_metrics.json",
"postgres-93_knobs.json",
"postgres-93_metrics.json",
"postgres-94_knobs.json",
"postgres-94_metrics.json",
"myrocks-5.6_knobs.json",
"myrocks-5.6_metrics.json",
"oracle_knobs.json",
"oracle_metrics.json"
]
for fixture in initial_data_fixtures:
call_command("loaddata", fixture, app_label="website")
def unload_initial_data(apps, schema_editor):
model_names = [
"DBMSCatalog",
"KnobCatalog",
"MetricCatalog",
"Hardware"
]
for model_name in model_names:
model = apps.get_model("website", model_name)
model.objects.all().delete()
class Migration(migrations.Migration):
dependencies = [
('website', '0003_background_task_optimization'),
]
operations = [
migrations.RunPython(load_initial_data, unload_initial_data)
]

138
server/website/website/migrations/0005_adding_session_knob.py Normal file
View File

@@ -0,0 +1,138 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.10.1 on 2019-08-07 18:18
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('website', '0004_load_initial_data'),
]
operations = [
migrations.CreateModel(
name='SessionKnob',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('minval', models.CharField(max_length=32, null=True, verbose_name='minimum value')),
('maxval', models.CharField(max_length=32, null=True, verbose_name='maximum value')),
('tunable', models.BooleanField(verbose_name='tunable')),
],
options={
'abstract': False,
},
),
migrations.AlterField(
model_name='dbmscatalog',
name='type',
field=models.IntegerField(choices=[(1, 'MySQL'), (2, 'Postgres'), (3, 'Db2'), (4, 'Oracle'), (6, 'SQLite'), (7, 'HStore'), (8, 'Vector'), (5, 'SQL Server'), (9, 'MyRocks')]),
),
migrations.AlterField(
model_name='knobcatalog',
name='default',
field=models.TextField(verbose_name='default value'),
),
migrations.AlterField(
model_name='knobcatalog',
name='enumvals',
field=models.TextField(null=True, verbose_name='valid values'),
),
migrations.AlterField(
model_name='knobcatalog',
name='maxval',
field=models.CharField(max_length=32, null=True, verbose_name='maximum value'),
),
migrations.AlterField(
model_name='knobcatalog',
name='minval',
field=models.CharField(max_length=32, null=True, verbose_name='minimum value'),
),
migrations.AlterField(
model_name='knobcatalog',
name='resource',
field=models.IntegerField(choices=[(1, 'Memory'), (2, 'CPU'), (3, 'Storage'), (4, 'Other')], default=4),
),
migrations.AlterField(
model_name='knobcatalog',
name='summary',
field=models.TextField(null=True, verbose_name='description'),
),
migrations.AlterField(
model_name='knobcatalog',
name='tunable',
field=models.BooleanField(verbose_name='tunable'),
),
migrations.AlterField(
model_name='knobcatalog',
name='unit',
field=models.IntegerField(choices=[(1, 'bytes'), (2, 'milliseconds'), (3, 'other')]),
),
migrations.AlterField(
model_name='knobcatalog',
name='vartype',
field=models.IntegerField(choices=[(1, 'STRING'), (2, 'INTEGER'), (3, 'REAL'), (4, 'BOOL'), (5, 'ENUM'), (6, 'TIMESTAMP')], verbose_name='variable type'),
),
migrations.AlterField(
model_name='metriccatalog',
name='metric_type',
field=models.IntegerField(choices=[(1, 'COUNTER'), (2, 'INFO'), (3, 'STATISTICS')]),
),
migrations.AlterField(
model_name='metriccatalog',
name='summary',
field=models.TextField(null=True, verbose_name='description'),
),
migrations.AlterField(
model_name='metriccatalog',
name='vartype',
field=models.IntegerField(choices=[(1, 'STRING'), (2, 'INTEGER'), (3, 'REAL'), (4, 'BOOL'), (5, 'ENUM'), (6, 'TIMESTAMP')]),
),
migrations.AlterField(
model_name='pipelinedata',
name='task_type',
field=models.IntegerField(choices=[(1, 'Pruned Metrics'), (2, 'Ranked Knobs'), (3, 'Knob Data'), (4, 'Metric Data')]),
),
migrations.AlterField(
model_name='project',
name='name',
field=models.CharField(max_length=64, verbose_name='project name'),
),
migrations.AlterField(
model_name='result',
name='session',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Session', verbose_name='session name'),
),
migrations.AlterField(
model_name='session',
name='name',
field=models.CharField(max_length=64, verbose_name='session name'),
),
migrations.AlterField(
model_name='session',
name='target_objective',
field=models.CharField(choices=[('throughput_txn_per_sec', 'Throughput'), ('99th_lat_ms', '99 Percentile Latency')], max_length=64, null=True),
),
migrations.AlterField(
model_name='session',
name='tuning_session',
field=models.CharField(choices=[('tuning_session', 'Tuning Session'), ('no_tuning_session', 'No Tuning'), ('randomly_generate', 'Randomly Generate')], default='tuning_sesion', max_length=64),
),
migrations.AlterField(
model_name='workload',
name='name',
field=models.CharField(max_length=128, verbose_name='workload name'),
),
migrations.AddField(
model_name='sessionknob',
name='knob',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.KnobCatalog'),
),
migrations.AddField(
model_name='sessionknob',
name='session',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='website.Session'),
),
]

0
server/website/website/migrations/__init__.py Normal file
View File