Cleanup backdoor methods

docker/install.sh

@@ -61,15 +61,11 @@ then
     rm_pkgs="unzip wget"
     install_gradle=true
 
-elif [ "$service" = "driver-internal" ]
-then
-    apt_pkgs="openssh-server vim"
-
 else
     echo ""
     echo "ERROR: Invalid value for service: '$service'"
     echo ""
-    echo "Usage: $0 [base|web|driver|driver-internal]"
+    echo "Usage: $0 [base|web|driver]"
     exit 1
 fi
 

server/website/website/urls.py

@@ -74,7 +74,6 @@ urlpatterns = [
     url(r'^create/user/', website_views.alt_create_user, name='backdoor_create_user'),
     url(r'^delete/user/', website_views.alt_delete_user, name='backdoor_delete_user'),
     url(r'^info/(?P<name>[0-9a-zA-Z]+)', website_views.alt_get_info, name="backdoor_info"),
-    url(r'^set_constants/', website_views.alt_set_constants, name="backdoor_set_constants"),
 
     # train ddpg with results in the given session
     url(r'^train_ddpg/sessions/(?P<session_id>[0-9]+)$', website_views.train_ddpg_loops, name='train_ddpg_loops'),

server/website/website/utils.py

@@ -355,19 +355,6 @@ class LabelUtil(object):
         return style_labels
 
 
-def set_constant(name, value):
-    getattr(constants, name)  # Throw exception if not a valid option
-    setattr(constants, name, value)
-
-
-def get_constants():
-    constants_dict = OrderedDict()
-    for name, value in sorted(constants.__dict__.items()):
-        if not name.startswith('_') and name == name.upper():
-            constants_dict[name] = value
-    return constants_dict
-
-
 def dump_debug_info(session, pretty_print=False):
     files = {}
 
@@ -443,9 +430,6 @@ def dump_debug_info(session, pretty_print=False):
             lv['level'] = logging.getLevelName(lv['level'])
         files['logs/{}.log'.format(logger_name)] = log_values
 
-    # Save settings
-    files['constants.json'] = get_constants()
-
     timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
     root = 'debug_{}'.format(timestamp)
 

server/website/website/views.py

@@ -22,7 +22,6 @@ from django.contrib.auth.forms import PasswordChangeForm
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.files.base import ContentFile, File
 from django.core.management import call_command
-from django.db.utils import IntegrityError
 from django.forms.models import model_to_dict
 from django.http import HttpResponse, QueryDict
 from django.shortcuts import redirect, render, get_object_or_404
@@ -40,8 +39,7 @@ from .forms import NewResultForm, ProjectForm, SessionForm, SessionKnobForm
 from .models import (BackupData, DBMSCatalog, ExecutionTime, Hardware, KnobCatalog, KnobData,
                      MetricCatalog, MetricData, PipelineRun, Project, Result, Session,
                      SessionKnob, User, Workload)
-from .tasks import (aggregate_target_results, map_workload, train_ddpg,
+from .tasks import train_ddpg
-                    configuration_recommendation, configuration_recommendation_ddpg)
 from .types import (DBMSType, KnobUnitType, MetricType,
                     TaskType, VarType, WorkloadStatusType, AlgorithmType)
 from .utils import (JSONUtil, LabelUtil, MediaUtil, TaskUtil)
@@ -1256,11 +1254,7 @@ def train_ddpg_loops(request, session_id):  # pylint: disable=unused-argument
 @csrf_exempt
 def alt_get_info(request, name):  # pylint: disable=unused-argument
     # Backdoor method for getting basic info
-    if name == 'constants':
+    if name in ('website', 'logs'):
-        info = utils.get_constants()
-        response = HttpResponse(JSONUtil.dumps(info))
-
-    elif name in ('website', 'logs'):
         tmpdir = os.path.realpath('.info')
         shutil.rmtree(tmpdir, ignore_errors=True)
         os.makedirs(tmpdir, exist_ok=True)
@@ -1285,200 +1279,194 @@ def alt_get_info(request, name):  # pylint: disable=unused-argument
             f.close()
         shutil.rmtree(tmpdir, ignore_errors=True)
     else:
-        LOG.warning("Invalid name for info request: %s", name)
+        message = "Invalid name for info request: '{}'".format(name)
-        response = HttpResponse("Invalid name for info request: {}".format(name), status=400)
+        LOG.warning(message)
+        content = dict(message='ERROR: ' + message, info=None, name=name)
+        response = HttpResponse(JSONUtil.dumps(content), content_type='application/json',
+                                status=400)
     return response
 
 
-@csrf_exempt
+def _alt_checker(request, response, required_data=None, authenticate_user=False):
-def alt_set_constants(request):
+    required_data = required_data or ()
-    constants = JSONUtil.loads(request.POST.get('constants', '{}'))
+    data = {k: v[0] for k, v in request.POST.lists()}
-    for name, value in constants.items():
+
-        try:
+    missing = [k for k in required_data if k not in data]
-            utils.set_constant(name, value)
+    if missing:
-        except AttributeError as e:
+        err_msg = "Request is missing required data: {}".format(', '.join(missing))
-            LOG.warning(e)
+        response['message'] = 'ERROR: ' + err_msg
-            return HttpResponse(e, status=400)
+        LOG.warning(err_msg)
-    return HttpResponse("Successfully updated constants: {}".format(
+        return HttpResponse(JSONUtil.dumps(response), content_type='application/json', status=400)
-        ', '.join('{}={}'.format(k, v) for k, v in constants.items())))
+
+    if authenticate_user:
+        user = authenticate(User, username=data['username'], password=data['password'])
+        if not user:
+            err_msg = "Unable to authenticate user '{}'.".format(data['username'])
+            LOG.warning(err_msg)
+            response.update(message='ERROR: ' + err_msg)
+            return HttpResponse(JSONUtil.dumps(response), content_type='application/json',
+                                status=400)
+        data['user'] = user
+
+    return data
 
 
 @csrf_exempt
 def alt_create_user(request):
-    response = dict(created=False, error=None, user=None)
+    response = dict(created=False, message=None, user=None)
-    if request.method != 'POST':
+    res = _alt_checker(request, response, required_data=('username', 'password'))
-        err_msg = "Request was not a post!"
+    if isinstance(res, HttpResponse):
-        response.update(error=err_msg)
+        return res
-        LOG.warning(err_msg)
-        return HttpResponse(JSONUtil.dumps(response), status=400)
-
-    data = {k: v[0] for k, v in request.POST.lists()}
-    missing = [k for k in ('username', 'password') if k not in data]
-    if missing:
-        err_msg = "Request is missing required data: {}".format(', '.join(missing))
-        LOG.warning(err_msg)
-        response.update(error=err_msg)
-        return HttpResponse(JSONUtil.dumps(response), status=400)
 
+    data = res
     user, created = utils.create_user(**data)
-    response.update(user=user, created=created)
     if created:
-        LOG.info("Successfully created user '%s': %s", data['username'], user)
+        msg = "Successfully created user '{}'.".format(data['username'])
-        status = 200
+        LOG.info(msg)
     else:
-        err_msg = "ERROR: User '{}' already exists: {}".format(data['username'], user)
+        msg = "User '{}' already exists.".format(data['username'])
-        response.update(error=err_msg)
+        LOG.warning(msg)
-        LOG.warning(err_msg)
+        msg = 'WARNING: ' + msg
-        status = 400
 
-    response['user'] = model_to_dict(response['user'])
+    response.update(user=model_to_dict(user), created=created, message=msg)
-    return HttpResponse(JSONUtil.dumps(response), status=status)
+    return HttpResponse(JSONUtil.dumps(response), content_type='application/json', status=200)
 
 
 @csrf_exempt
 def alt_delete_user(request):
-    response = dict(deleted=False, error=None, delete_info=None)
+    response = dict(deleted=False, message=None, delete_info=None)
-    if request.method != 'POST':
+    res = _alt_checker(request, response, required_data=('username',))
-        err_msg = "Request was not a post!"
+    if isinstance(res, HttpResponse):
-        response.update(error=err_msg)
+        return res
-        LOG.warning(err_msg)
-        return HttpResponse(JSONUtil.dumps(response), status=400)
-
-    data = {k: v[0] for k, v in request.POST.lists()}
-    missing = [k for k in ('username',) if k not in data]
-    if missing:
-        err_msg = "Request is missing required data: {}".format(', '.join(missing))
-        response.update(error=err_msg)
-        LOG.warning(err_msg)
-        return HttpResponse(JSONUtil.dumps(response), status=400)
 
+    data = res
     delete_info, deleted = utils.delete_user(**data)
-    response.update(deleted=deleted, delete_info=delete_info)
     if deleted:
-        LOG.info("Successfully deleted user '%s': %s", data['username'], delete_info)
+        msg = "Successfully deleted user '{}'.".format(data['username'])
-        status = 200
+        LOG.info(msg)
     else:
-        err_msg = "User '{}' does not exist".format(data['username'])
+        msg = "User '{}' does not exist.".format(data['username'])
-        LOG.warning(err_msg)
+        LOG.warning(msg)
-        response.update(error=err_msg)
+        msg = 'WARNING: ' + msg
-        status = 400
 
-    return HttpResponse(JSONUtil.dumps(response), status=status)
+    response.update(message=msg, deleted=deleted, delete_info=delete_info)
+    return HttpResponse(JSONUtil.dumps(response), content_type='application/json', status=200)
 
 
 @csrf_exempt
 def alt_create_or_edit_project(request):
-    response = dict(created=False, error=None, project=None)
+    response = dict(created=False, updated=False, message=None, project=None)
-    if request.method != 'POST':
+    res = _alt_checker(request, response, required_data=('username', 'password', 'name'),
-        err_msg = "Request was not a post!"
+                       authenticate_user=True)
-        response.update(error=err_msg)
+    if isinstance(res, HttpResponse):
-        LOG.warning(err_msg)
+        return res
-        return HttpResponse(JSONUtil.dumps(response), status=400)
-
-    data = {k: v[0] for k, v in request.POST.lists()}
-    missing = [k for k in ('username', 'password', 'name') if k not in data]
-    if missing:
-        err_msg = "Request is missing required data: {}".format(', '.join(missing))
-        response.update(error=err_msg)
-        LOG.warning(err_msg)
-        return HttpResponse(JSONUtil.dumps(response), status=400)
-
-    username = data.pop('username')
-    password = data.pop('password')
-    user = authenticate(User, username=username, password=password)
-    if not user:
-        err_msg = "ERROR: Unable to authenticate user '{}'.".format(username)
-        response.update(error=err_msg)
-        LOG.warning(err_msg)
-        return HttpResponse(JSONUtil.dumps(response), status=400)
 
+    data = res
+    user = data.pop('user')
+    data.pop('username')
+    data.pop('password')
     project_name = data.pop('name')
+
     ts = now()
+    created = False
+    updated = False
 
     if request.path == reverse('backdoor_create_project'):
-        try:
+        defaults = dict(creation_time=ts, last_update=ts, **data)
-            project = Project.objects.create(user=user, name=project_name, last_update=ts,
+        project, created = Project.objects.get_or_create(
-                                             creation_time=ts, **data)
+            user=user, name=project_name, defaults=defaults)
-        except IntegrityError:
+
-            err_msg = "ERROR: Project '{}' already exists.".format(project_name)
+        if created:
-            project = Project.objects.get(user=user, name=project_name)
+            msg = "Successfully created project '{}'.".format(project_name)
-            response.update(error=err_msg, project=model_to_dict(project))
+        else:
-            LOG.warning(err_msg)
+            msg = "Project '{}' already exists.".format(project_name)
-            return HttpResponse(JSONUtil.dumps(response), status=400)
+            LOG.warning(msg)
+            msg = 'WARNING: ' + msg
     else:
         project = get_object_or_404(Project, name=project_name, user=user)
         for k, v in data.items():
             setattr(project, k, v)
         project.last_update = ts
         project.save()
+        msg = "Successfully updated project '{}'".format(project_name)
+        updated = True
 
-    response.update(created=True, project=model_to_dict(project))
+    response.update(message=msg, project=model_to_dict(project), created=created, updated=updated)
-    return HttpResponse(JSONUtil.dumps(response))
+    return HttpResponse(JSONUtil.dumps(response), content_type='application/json', status=200)
 
 
 @csrf_exempt
 def alt_create_or_edit_session(request):
-    response = dict(created=False, error=None, session=None)
+    response = dict(created=False, updated=False, message=None, session=None)
-    if request.method != 'POST':
+    authenticate_user = True
-        err_msg = "Request was not a post!"
+
-        response.update(error=err_msg)
+    if request.path == reverse('backdoor_create_session'):
-        LOG.warning(err_msg)
+        required_data = (
-        return HttpResponse(JSONUtil.dumps(response), status=400)
+            'username', 'password', 'project_name', 'name', 'dbms_type', 'dbms_version')
+    else:
+        if 'upload_code' in request.POST:
+            required_data = ()
+            authenticate_user = False
+        else:
+            required_data = ('username', 'password', 'project_name', 'name')
+
+    res = _alt_checker(request, response, required_data=required_data,
+                       authenticate_user=authenticate_user)
+    if isinstance(res, HttpResponse):
+        return res
+
+    data = res
+    warnings = []
 
-    data = {k: v[0] for k, v in request.POST.lists()}
     if 'hardware' in data:
-        err_msg = "Custom hardware objects are not supported."
+        data.pop('hardware')
-        response.update(error=err_msg)
+        warn_msg = "Custom hardware objects are not supported."
-        LOG.warning(err_msg)
+        LOG.warning(warn_msg)
-        return HttpResponse(JSONUtil.dumps(response), status=400)
+        warnings.append('WARNING: ' + warn_msg)
 
-    required_fields = ('username', 'password', 'project_name', 'name', 'dbms_type',
+    created = False
-                       'dbms_version')
+    updated = False
-    missing = [k for k in required_fields if k not in data]
+    data.pop('username', None)
-    if missing:
+    data.pop('password', None)
-        err_msg = "Request is missing required data: {}".format(', '.join(missing))
+    user = data.pop('user', None)
-        response.update(error=err_msg)
+    project_name = data.pop('project_name', None)
-        LOG.warning(err_msg)
+    session_name = data.pop('name', None)
-        return HttpResponse(JSONUtil.dumps(response), status=400)
-
-    username = data.pop('username')
-    password = data.pop('password')
-    user = authenticate(User, username=username, password=password)
-    if not user:
-        err_msg = "ERROR: Unable to authenticate user '{}'.".format(username)
-        response.update(error=err_msg)
-        LOG.warning(err_msg)
-        return HttpResponse(JSONUtil.dumps(response), status=400)
-
-    project = get_object_or_404(Project, name=data.pop('project_name'), user=user)
-    dbms_type = DBMSType.type(data.pop('dbms_type'))
-    dbms = get_object_or_404(DBMSCatalog, type=dbms_type, version=data.pop('dbms_version'))
-
-    session_name = data.pop('name')
     if 'algorithm' in data:
         data['algorithm'] = AlgorithmType.type(data['algorithm'])
     session_knobs = data.pop('session_knobs', None)
     disable_others = data.pop('disable_others', False)
+    hyperparams = data.pop('hyperparameters', None)
     ts = now()
 
     if request.path == reverse('backdoor_create_session'):
+        defaults = {}
+        project = get_object_or_404(Project, name=project_name, user=user)
+        dbms_type = DBMSType.type(data.pop('dbms_type'))
+        dbms_version = data.pop('dbms_version')
+        defaults['dbms'] = get_object_or_404(DBMSCatalog, type=dbms_type, version=dbms_version)
         hardware, _ = Hardware.objects.get_or_create(pk=1)
-        upload_code = data.pop('upload_code', None) or MediaUtil.upload_code_generator()
+        defaults['hardware'] = hardware
-        try:
+        defaults['upload_code'] = data.pop('upload_code', None) or MediaUtil.upload_code_generator()
-            session = Session.objects.create(user=user, project=project, dbms=dbms,
+        defaults.update(creation_time=ts, last_update=ts, **data)
-                                             name=session_name, hardware=hardware,
-                                             upload_code=upload_code, creation_time=ts,
-                                             last_update=ts, **data)
-        except IntegrityError:
-            err_msg = "ERROR: Session '{}' already exists.".format(session_name)
-            session = Session.objects.get(user=user, project=project, name=session_name)
-            response.update(error=err_msg, project=model_to_dict(session))
-            LOG.warning(err_msg)
-            return HttpResponse(JSONUtil.dumps(response), status=400)
 
+        session, created = Session.objects.get_or_create(user=user, project=project,
+                                                         name=session_name, defaults=defaults)
+
+        if created:
+            msg = "Successfully created session '{}'.".format(session_name)
             set_default_knobs(session)
         else:
-        session = get_object_or_404(Session, name=session_name, project=project, user=user)
+            msg = "Session '{}' already exists.".format(session_name)
+            LOG.warning(msg)
+            msg = 'WARNING: ' + msg
+    else:
+        if 'upload_code' in data:
+            session = get_object_or_404(Session, upload_code=data['upload_code'])
+        else:
+            project = get_object_or_404(Project, name=project_name, user=user)
+            session = get_object_or_404(Session, name=session_name, project=project,
+                                        user=user)
+
         for k, v in data.items():
             setattr(session, k, v)
 
@@ -1489,11 +1477,32 @@ def alt_create_or_edit_session(request):
 
         session.last_update = ts
         session.save()
+        msg = "Successfully updated session '{}'.".format(session_name)
+        updated = True
 
+    if created or updated:
         if session_knobs:
             session_knobs = JSONUtil.loads(session_knobs)
-        SessionKnob.objects.set_knob_min_max_tunability(session, session_knobs,
+            SessionKnob.objects.set_knob_min_max_tunability(
-                                                        disable_others=disable_others)
+                session, session_knobs, disable_others=disable_others)
+
+        if hyperparams:
+            hyperparams = JSONUtil.loads(hyperparams)
+            sess_hyperparams = JSONUtil.loads(session.hyperparameters)
+            invalid = []
+
+            for k, v in hyperparams.items():
+                if k in sess_hyperparams:
+                    sess_hyperparams[k] = v
+                else:
+                    invalid.append('{}={}'.format(k, v))
+            session.save()
+            if invalid:
+                warn_msg = "Ignored invalid hyperparameters: {}".format(', '.join(invalid))
+                LOG.warning(warn_msg)
+                warnings.append("WARNING: " + warn_msg)
+
+    session.refresh_from_db()
     res = model_to_dict(session)
     res['dbms_id'] = res['dbms']
     res['dbms'] = session.dbms.full_name
@@ -1503,13 +1512,14 @@ def alt_create_or_edit_session(request):
     sk = SessionKnob.objects.get_knobs_for_session(session)
     sess_knobs = []
     for knob in sk:
-        sess_knobs.append(dict(
+        sess_knobs.append({x: knob[x] for x in ('minval', 'maxval', 'tunable')})
-            minval=knob['minval'],
-            maxval=knob['maxval'],
-            tunable=knob['tunable']))
     res['session_knobs'] = sess_knobs
-    response.update(created=True, session=res)
+
-    return HttpResponse(JSONUtil.dumps(response))
+    if warnings:
+        msg = '\n\n'.join(warnings + [msg])
+
+    response.update(message=msg, session=res, created=created, updated=updated)
+    return HttpResponse(JSONUtil.dumps(response), content_type='application/json', status=200)
 
 
 # integration test