public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed setting of session knobs to be case-insensitive and added git commit hash to server info

This commit is contained in:
Dana Van Aken 2020-03-05 14:37:11 -05:00
parent f737a0493e
commit f61021c32f
3 changed files with 62 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
server/website/website/models.py
View File

@ -267,17 +267,23 @@ class SessionKnobManager(models.Manager):
@staticmethod
def set_knob_min_max_tunability(session, knob_dicts, cascade=True, disable_others=False):
# Returns a dict of the knob
session_knobs = SessionKnob.objects.filter(session=session)
for session_knob in session_knobs:
if session_knob.name in knob_dicts:
session_knob.minval = knob_dicts[session_knob.name]["minval"]
session_knob.maxval = knob_dicts[session_knob.name]["maxval"]
session_knob.tunable = knob_dicts[session_knob.name]["tunable"]
knob_dicts = {k.lower(): v for k, v in knob_dicts.items()}
session_knobs = {k.name.lower(): k for k in SessionKnob.objects.filter(session=session)}
for lower_name, session_knob in session_knobs.items():
if lower_name in knob_dicts:
settings = knob_dicts[lower_name]
session_knob.minval = settings["minval"]
session_knob.maxval = settings["maxval"]
session_knob.tunable = settings["tunable"]
session_knob.save()
if cascade:
knob = KnobCatalog.objects.get(name=session_knob.name,
dbms=session_knob.session.dbms)
knob = KnobCatalog.objects.get(name=session_knob.name, dbms=session.dbms)
knob.tunable = session_knob.tunable
if knob.vartype in (VarType.INTEGER, VarType.REAL):
if knob.minval is None or session_knob.minval < float(knob.minval):
knob.minval = session_knob.minval
if knob.maxval is None or session_knob.maxval > float(knob.maxval):
knob.maxval = session_knob.maxval
knob.save()
elif disable_others:
# Set all knobs not in knob_dicts to not tunable

22
server/website/website/utils.py
View File

@ -3,7 +3,6 @@
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
import celery
import datetime
import json
import logging
@ -15,7 +14,9 @@ import time
from collections import OrderedDict
from io import BytesIO
from random import choice
from subprocess import Popen, PIPE
import celery
import numpy as np
from django.contrib.auth.models import User
from django.core.management import call_command
@ -517,3 +518,22 @@ def check_and_run_celery():
return 'celery stopped but is restarted successfully'
LOG.warning('Cannot restart celery.')
return 'celery stopped and cannot be restarted'
def git_hash():
sha = ''
if os.path.exists('/app/.git_commit'):
with open('/app/.git_commit', 'r') as f:
lines = f.read().strip().split('\n')
for line in lines:
if line.startswith('base='):
sha = line.strip().split('=', 1)[1]
break
else:
try:
p = Popen("git log -1 --format=format:%H", shell=True, stdout=PIPE, stderr=PIPE)
sha = p.communicate()[0].decode('utf-8')
except OSError as e:
LOG.warning("Failed to get git commit hash.\n\n%s\n\n", e, exc_info=True)
return sha

51
server/website/website/views.py
View File

@ -10,6 +10,7 @@ import os
import re
import shutil
import socket
import time
from collections import OrderedDict
from io import StringIO
@ -1258,29 +1259,31 @@ def train_ddpg_loops(request, session_id): # pylint: disable=unused-argument
def alt_get_info(request, name): # pylint: disable=unused-argument
# Backdoor method for getting basic info
if name in ('website', 'logs'):
tmpdir = os.path.realpath('.info')
shutil.rmtree(tmpdir, ignore_errors=True)
os.makedirs(tmpdir, exist_ok=True)
tmpdir = os.path.realpath('.info_{}'.format(int(time.time())))
os.makedirs(tmpdir, exist_ok=False)
if name == 'website':
filepath = os.path.join(tmpdir, 'website_dump.json.gz')
call_command('dumpwebsite', dumpfile=filepath, compress=True)
else: # name == 'logs'
base_name = os.path.join(tmpdir, 'website_logs')
root_dir, base_dir = os.path.split(LOG_DIR)
filepath = shutil.make_archive(
base_name, format='gztar', base_dir=base_dir, root_dir=root_dir)
f = open(filepath, 'rb')
try:
cfile = File(f)
response = HttpResponse(cfile, content_type='application/x-gzip')
response['Content-Length'] = cfile.size
response['Content-Disposition'] = 'attachment; filename={}'.format(
os.path.basename(filepath))
if name == 'website':
filepath = os.path.join(tmpdir, 'website_dump.json.gz')
call_command('dumpwebsite', dumpfile=filepath, compress=True)
else: # name == 'logs'
base_dir = 'website_log'
base_name = os.path.join(tmpdir, base_dir)
shutil.copytree(LOG_DIR, base_name)
filepath = shutil.make_archive(
base_name, 'gztar', tmpdir, base_dir)
f = open(filepath, 'rb')
try:
cfile = File(f)
response = HttpResponse(cfile, content_type='application/x-gzip')
response['Content-Length'] = cfile.size
response['Content-Disposition'] = 'attachment; filename={}'.format(
os.path.basename(filepath))
finally:
f.close()
finally:
f.close()
shutil.rmtree(tmpdir, ignore_errors=True)
shutil.rmtree(tmpdir, ignore_errors=True)
else:
info = {}
msg = ''
@ -1293,6 +1296,7 @@ def alt_get_info(request, name): # pylint: disable=unused-argument
v = 'localhost'
info['db_' + k] = v
info['hostname'] = socket.gethostname()
info['git_commit_hash'] = utils.git_hash()
msg = "Successfully retrieved info for '{}'.".format(name)
elif name in app_models.__dict__ and hasattr(app_models.__dict__[name], 'objects'):
data = {k: v[0] for k, v in request.POST.lists()}
@ -1504,8 +1508,7 @@ def alt_create_or_edit_session(request):
session = get_object_or_404(Session, upload_code=data['upload_code'])
else:
project = get_object_or_404(Project, name=project_name, user=user)
session = get_object_or_404(Session, name=session_name, project=project,
user=user)
session = get_object_or_404(Session, name=session_name, project=project, user=user)
for k, v in data.items():
setattr(session, k, v)
@ -1550,9 +1553,9 @@ def alt_create_or_edit_session(request):
res['hardware'] = model_to_dict(session.hardware)
res['algorithm'] = AlgorithmType.name(res['algorithm'])
sk = SessionKnob.objects.get_knobs_for_session(session)
sess_knobs = []
sess_knobs = {}
for knob in sk:
sess_knobs.append({x: knob[x] for x in ('minval', 'maxval', 'tunable')})
sess_knobs[knob['name']] = {x: knob[x] for x in ('minval', 'maxval', 'tunable')}
res['session_knobs'] = sess_knobs
if warnings: