Initial commit with BSL

server/website/script/fixture_generators/knob_identification/.gitignore

@@ -0,0 +1 @@
+*.txt

server/website/script/fixture_generators/knob_identification/create_ranked_knobs.py

@@ -0,0 +1,80 @@
+#
+# OtterTune - create_ranked_knobs.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+import logging
+import os
+import shutil
+import json
+import itertools
+
+LOG = logging.getLogger(__name__)
+
+DATADIR = '/dataset/oltpbench/first_paper_experiments/analysis/knob_identification'
+
+DBMSS = {'postgres-9.6': 1}
+HARDWARES = {'m3.xlarge': 16}
+TIMESTAMP = '2016-12-04 11:00'
+TASK_TYPE = 2
+
+PREFIX = 'global'
+MODEL = 'website.PipelineResult'
+VALIDATE = True
+EXTRA_EXCEPTIONS = {
+    PREFIX + '.' + 'checkpoint_segments',
+}
+
+
+def validate_postgres(knobs, dbms):
+    with open('../knob_settings/{}/{}_knobs.json'.format(dbms.replace('-', '_'),
+                                                         dbms.replace('.', '')), 'r') as f:
+        knob_info = json.load(f)
+        knob_info = {k['fields']['name']: k['fields'] for k in knob_info}
+    for kname, kinfo in list(knob_info.items()):
+        if kname not in knobs and kinfo['tunable'] is True:
+            knobs.append(kname)
+            LOG.warning("Adding missing knob to end of list (%s)", kname)
+    knob_names = list(knob_info.keys())
+    for kname in knobs:
+        if kname not in knob_names:
+            if kname not in EXTRA_EXCEPTIONS:
+                raise Exception('Extra knob: {}'.format(kname))
+            knobs.remove(kname)
+            LOG.warning("Removing extra knob (%s)", kname)
+
+
+def main():
+    for dbms, hw in itertools.product(list(DBMSS.keys()), HARDWARES):
+        datapath = os.path.join(DATADIR, '{}_{}'.format(dbms, hw))
+        if not os.path.exists(datapath):
+            raise IOError('Path does not exist: {}'.format(datapath))
+        with open(os.path.join(datapath, 'featured_knobs.txt'), 'r') as f:
+            knobs = [k.strip() for k in f.read().split('\n')]
+        knobs = [PREFIX + '.' + k for k in knobs]
+        if VALIDATE and dbms.startswith('postgres'):
+            validate_postgres(knobs, dbms)
+
+        basename = '{}_{}_ranked_knobs'.format(dbms, hw).replace('.', '')
+        with open(basename + '.txt', 'w') as f:
+            f.write('\n'.join(knobs))
+
+        django_entry = [{
+            'model': MODEL,
+            'fields': {
+                'dbms': DBMSS[dbms],
+                'hardware': HARDWARES[hw],
+                'creation_timestamp': TIMESTAMP,
+                'task_type': TASK_TYPE,
+                'value': json.dumps(knobs, indent=4)
+            }
+        }]
+        savepath = basename + '.json'
+        with open(savepath, 'w') as f:
+            json.dump(django_entry, f, indent=4)
+
+        shutil.copy(savepath, '../../../preload/{}'.format(savepath))
+
+
+if __name__ == '__main__':
+    main()

server/website/script/fixture_generators/knob_identification/postgres-96_m3xlarge_ranked_knobs.json

@@ -0,0 +1,12 @@
+[
+    {
+        "fields": {
+            "hardware": 16, 
+            "dbms": 1, 
+            "task_type": 2, 
+            "creation_timestamp": "2016-12-04 11:00", 
+            "value": "[\n    \"global.shared_buffers\", \n    \"global.effective_cache_size\", \n    \"global.bgwriter_lru_maxpages\", \n    \"global.bgwriter_delay\", \n    \"global.checkpoint_completion_target\", \n    \"global.deadlock_timeout\", \n    \"global.default_statistics_target\", \n    \"global.effective_io_concurrency\", \n    \"global.checkpoint_timeout\", \n    \"global.commit_delay\", \n    \"global.commit_siblings\", \n    \"global.wal_buffers\", \n    \"global.temp_buffers\", \n    \"global.from_collapse_limit\", \n    \"global.join_collapse_limit\", \n    \"global.bgwriter_lru_multiplier\", \n    \"global.random_page_cost\", \n    \"global.work_mem\", \n    \"global.maintenance_work_mem\", \n    \"global.min_wal_size\", \n    \"global.max_parallel_workers_per_gather\", \n    \"global.seq_page_cost\", \n    \"global.max_worker_processes\", \n    \"global.wal_sync_method\", \n    \"global.checkpoint_flush_after\", \n    \"global.wal_writer_delay\", \n    \"global.backend_flush_after\", \n    \"global.bgwriter_flush_after\", \n    \"global.min_parallel_relation_size\", \n    \"global.wal_writer_flush_after\", \n    \"global.max_wal_size\"\n]"
+        }, 
+        "model": "website.PipelineResult"
+    }
+]