diff --git a/server/website/website/models.py b/server/website/website/models.py index c7005e0..8a30d42 100644 --- a/server/website/website/models.py +++ b/server/website/website/models.py @@ -267,17 +267,23 @@ class SessionKnobManager(models.Manager): @staticmethod def set_knob_min_max_tunability(session, knob_dicts, cascade=True, disable_others=False): # Returns a dict of the knob - session_knobs = SessionKnob.objects.filter(session=session) - for session_knob in session_knobs: - if session_knob.name in knob_dicts: - session_knob.minval = knob_dicts[session_knob.name]["minval"] - session_knob.maxval = knob_dicts[session_knob.name]["maxval"] - session_knob.tunable = knob_dicts[session_knob.name]["tunable"] + knob_dicts = {k.lower(): v for k, v in knob_dicts.items()} + session_knobs = {k.name.lower(): k for k in SessionKnob.objects.filter(session=session)} + for lower_name, session_knob in session_knobs.items(): + if lower_name in knob_dicts: + settings = knob_dicts[lower_name] + session_knob.minval = settings["minval"] + session_knob.maxval = settings["maxval"] + session_knob.tunable = settings["tunable"] session_knob.save() if cascade: - knob = KnobCatalog.objects.get(name=session_knob.name, - dbms=session_knob.session.dbms) + knob = KnobCatalog.objects.get(name=session_knob.name, dbms=session.dbms) knob.tunable = session_knob.tunable + if knob.vartype in (VarType.INTEGER, VarType.REAL): + if knob.minval is None or session_knob.minval < float(knob.minval): + knob.minval = session_knob.minval + if knob.maxval is None or session_knob.maxval > float(knob.maxval): + knob.maxval = session_knob.maxval knob.save() elif disable_others: # Set all knobs not in knob_dicts to not tunable diff --git a/server/website/website/utils.py b/server/website/website/utils.py index 40b0fb6..07b75f4 100644 --- a/server/website/website/utils.py +++ b/server/website/website/utils.py @@ -3,7 +3,6 @@ # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # -import celery import datetime import json import logging @@ -15,7 +14,9 @@ import time from collections import OrderedDict from io import BytesIO from random import choice +from subprocess import Popen, PIPE +import celery import numpy as np from django.contrib.auth.models import User from django.core.management import call_command @@ -517,3 +518,22 @@ def check_and_run_celery(): return 'celery stopped but is restarted successfully' LOG.warning('Cannot restart celery.') return 'celery stopped and cannot be restarted' + + +def git_hash(): + sha = '' + if os.path.exists('/app/.git_commit'): + with open('/app/.git_commit', 'r') as f: + lines = f.read().strip().split('\n') + for line in lines: + if line.startswith('base='): + sha = line.strip().split('=', 1)[1] + break + else: + try: + p = Popen("git log -1 --format=format:%H", shell=True, stdout=PIPE, stderr=PIPE) + sha = p.communicate()[0].decode('utf-8') + except OSError as e: + LOG.warning("Failed to get git commit hash.\n\n%s\n\n", e, exc_info=True) + + return sha diff --git a/server/website/website/views.py b/server/website/website/views.py index 2bbf8c9..f0b8614 100644 --- a/server/website/website/views.py +++ b/server/website/website/views.py @@ -10,6 +10,7 @@ import os import re import shutil import socket +import time from collections import OrderedDict from io import StringIO @@ -1258,29 +1259,31 @@ def train_ddpg_loops(request, session_id): # pylint: disable=unused-argument def alt_get_info(request, name): # pylint: disable=unused-argument # Backdoor method for getting basic info if name in ('website', 'logs'): - tmpdir = os.path.realpath('.info') - shutil.rmtree(tmpdir, ignore_errors=True) - os.makedirs(tmpdir, exist_ok=True) + tmpdir = os.path.realpath('.info_{}'.format(int(time.time()))) + os.makedirs(tmpdir, exist_ok=False) - if name == 'website': - filepath = os.path.join(tmpdir, 'website_dump.json.gz') - call_command('dumpwebsite', dumpfile=filepath, compress=True) - else: # name == 'logs' - base_name = os.path.join(tmpdir, 'website_logs') - root_dir, base_dir = os.path.split(LOG_DIR) - filepath = shutil.make_archive( - base_name, format='gztar', base_dir=base_dir, root_dir=root_dir) - - f = open(filepath, 'rb') try: - cfile = File(f) - response = HttpResponse(cfile, content_type='application/x-gzip') - response['Content-Length'] = cfile.size - response['Content-Disposition'] = 'attachment; filename={}'.format( - os.path.basename(filepath)) + if name == 'website': + filepath = os.path.join(tmpdir, 'website_dump.json.gz') + call_command('dumpwebsite', dumpfile=filepath, compress=True) + else: # name == 'logs' + base_dir = 'website_log' + base_name = os.path.join(tmpdir, base_dir) + shutil.copytree(LOG_DIR, base_name) + filepath = shutil.make_archive( + base_name, 'gztar', tmpdir, base_dir) + + f = open(filepath, 'rb') + try: + cfile = File(f) + response = HttpResponse(cfile, content_type='application/x-gzip') + response['Content-Length'] = cfile.size + response['Content-Disposition'] = 'attachment; filename={}'.format( + os.path.basename(filepath)) + finally: + f.close() finally: - f.close() - shutil.rmtree(tmpdir, ignore_errors=True) + shutil.rmtree(tmpdir, ignore_errors=True) else: info = {} msg = '' @@ -1293,6 +1296,7 @@ def alt_get_info(request, name): # pylint: disable=unused-argument v = 'localhost' info['db_' + k] = v info['hostname'] = socket.gethostname() + info['git_commit_hash'] = utils.git_hash() msg = "Successfully retrieved info for '{}'.".format(name) elif name in app_models.__dict__ and hasattr(app_models.__dict__[name], 'objects'): data = {k: v[0] for k, v in request.POST.lists()} @@ -1504,8 +1508,7 @@ def alt_create_or_edit_session(request): session = get_object_or_404(Session, upload_code=data['upload_code']) else: project = get_object_or_404(Project, name=project_name, user=user) - session = get_object_or_404(Session, name=session_name, project=project, - user=user) + session = get_object_or_404(Session, name=session_name, project=project, user=user) for k, v in data.items(): setattr(session, k, v) @@ -1550,9 +1553,9 @@ def alt_create_or_edit_session(request): res['hardware'] = model_to_dict(session.hardware) res['algorithm'] = AlgorithmType.name(res['algorithm']) sk = SessionKnob.objects.get_knobs_for_session(session) - sess_knobs = [] + sess_knobs = {} for knob in sk: - sess_knobs.append({x: knob[x] for x in ('minval', 'maxval', 'tunable')}) + sess_knobs[knob['name']] = {x: knob[x] for x in ('minval', 'maxval', 'tunable')} res['session_knobs'] = sess_knobs if warnings: