Added backdoor functions for creating/editing/deleting users/projects/sessions in the website and driver.

client/driver/fabfile.py

@@ -663,6 +663,89 @@ def rename_batch(result_dir=None):
             os.rename(fpath, rename_path)
 
 
+def _http_content_to_json(content):
+    if isinstance(content, bytes):
+        content = content.decode('utf-8')
+    try:
+        json_content = json.loads(content)
+        decoded = True
+    except (TypeError, json.decoder.JSONDecodeError):
+        json_content = None
+        decoded = False
+
+    return json_content, decoded
+
+
+def _modify_website_object(obj_name, action, verbose=False, **kwargs):
+    verbose = _parse_bool(verbose)
+    if obj_name == 'project':
+        valid_actions = ('create', 'edit')
+    elif obj_name == 'session':
+        valid_actions = ('create', 'edit')
+    elif obj_name == 'user':
+        valid_actions = ('create', 'delete')
+    else:
+        raise ValueError('Invalid object: {}. Valid objects: project, session'.format(obj_name))
+
+    if action not in valid_actions:
+        raise ValueError('Invalid action: {}. Valid actions: {}'.format(
+            action, ', '.join(valid_actions)))
+
+    data = {}
+    for k, v in kwargs.items():
+        if isinstance(v, (dict, list, tuple)):
+            v = json.dumps(v)
+        data[k] = v
+
+    url_path = '/{}/{}/'.format(action, obj_name)
+    response = requests.post(CONF['upload_url'] + url_path, data=data)
+
+    content = response.content.decode('utf-8')
+    if response.status_code != 200:
+        raise Exception("Failed to {} new {}.\nStatus: {}\nMessage: {}\n".format(
+            action, obj_name, response.status_code, content))
+
+    json_content, decoded = _http_content_to_json(content)
+    if verbose:
+        if decoded:
+            LOG.info('\n%s_%s = %s', action.upper(), obj_name.upper(),
+                     json.dumps(json_content, indent=4))
+        else:
+            LOG.warning("Content could not be decoded.\n\n%s\n", content)
+
+    return response, json_content, decoded
+
+
+@task
+def create_website_user(**kwargs):
+    return _modify_website_object('user', 'create', **kwargs)
+
+
+@task
+def delete_website_user(**kwargs):
+    return _modify_website_object('user', 'delete', **kwargs)
+
+
+@task
+def create_website_project(**kwargs):
+    return _modify_website_object('project', 'create', **kwargs)
+
+
+@task
+def edit_website_project(**kwargs):
+    return _modify_website_object('project', 'edit', **kwargs)
+
+
+@task
+def create_website_session(**kwargs):
+    return _modify_website_object('session', 'create', **kwargs)
+
+
+@task
+def edit_website_session(**kwargs):
+    return _modify_website_object('session', 'edit', **kwargs)
+
+
 def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10):
     max_time_sec = int(max_time_sec)
     interval_sec = int(interval_sec)

script/formatting/config/pylintrc

@@ -13,9 +13,9 @@ profile=no
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=CVS,.git,manage.py
+ignore=CVS,.git,manage.py,0001_initial.py,0002_enable_compression.py,0003_load_initial_data.py,0004_add_lhs.py 
 
-ignore-patterns=**/migrations/*.py
+# ignore-patterns=**/migrations/*.py
 
 # Pickle collected data for later comparisons.
 persistent=no

server/website/website/admin.py

@@ -50,7 +50,7 @@ class SessionKnobAdmin(admin.ModelAdmin):
     list_display = ('knob', 'dbms', 'session', 'minval', 'maxval', 'tunable')
     list_filter = (('session__dbms', admin.RelatedOnlyFieldListFilter),
                    ('session', admin.RelatedOnlyFieldListFilter),
-                   ('tunable', admin.FieldListFilter))
+                   'tunable')
     ordering = ('session__dbms', 'session__name', '-tunable', 'knob__name')
 
     def dbms(self, instance):  # pylint: disable=no-self-use

server/website/website/models.py

@@ -211,15 +211,19 @@ class SessionKnobManager(models.Manager):
         return session_knob_dicts
 
     @staticmethod
-    def set_knob_min_max_tunability(session, knob_dicts):
+    def set_knob_min_max_tunability(session, knob_dicts, disable_others=False):
         # Returns a dict of the knob
         session_knobs = SessionKnob.objects.filter(session=session)
         for session_knob in session_knobs:
-            if knob_dicts.__contains__(session_knob.name):
+            if session_knob.name in knob_dicts:
                 session_knob.minval = knob_dicts[session_knob.name]["minval"]
                 session_knob.maxval = knob_dicts[session_knob.name]["maxval"]
                 session_knob.tunable = knob_dicts[session_knob.name]["tunable"]
                 session_knob.save()
+            elif disable_others:
+                # Set all knobs not in knob_dicts to not tunable
+                session_knob.tunable = False
+                session_knob.save()
 
 
 class SessionKnob(BaseModel):

server/website/website/urls.py

@@ -67,6 +67,12 @@ urlpatterns = [
     # Back door
     url(r'^query_and_get/(?P<upload_code>[0-9a-zA-Z]+)$', website_views.give_result, name="backdoor"),
     url(r'^dump/(?P<upload_code>[0-9a-zA-Z]+)', website_views.get_debug_info, name="backdoor_debug"),
+    url(r'^create/project/', website_views.alt_create_or_edit_project, name='backdoor_create_project'),
+    url(r'^edit/project/', website_views.alt_create_or_edit_project, name='backdoor_edit_project'),
+    url(r'^create/session/', website_views.alt_create_or_edit_session, name='backdoor_create_session'),
+    url(r'^edit/session/', website_views.alt_create_or_edit_session, name='backdoor_edit_session'),
+    url(r'^create/user/', website_views.alt_create_user, name='backdoor_create_user'),
+    url(r'^delete/user/', website_views.alt_delete_user, name='backdoor_delete_user'),
 
     # train ddpg with results in the given session
     url(r'^train_ddpg/sessions/(?P<session_id>[0-9]+)$', website_views.train_ddpg_loops, name='train_ddpg_loops'),

server/website/website/utils.py

@@ -16,6 +16,7 @@ from io import BytesIO
 from random import choice
 
 import numpy as np
+from django.contrib.auth.models import User
 from django.utils.text import capfirst
 from django_db_logger.models import StatusLog
 from djcelery.models import TaskMeta
@@ -437,3 +438,32 @@ def dump_debug_info(session, pretty_print=False):
 
     tarstream.seek(0)
     return tarstream, root
+
+
+def create_user(username, password, email=None, superuser=False):
+    user = User.objects.filter(username=username).first()
+    if user:
+        created = False
+    else:
+        if superuser:
+            email = email or '{}@noemail.com'.format(username)
+            _create_user = User.objects.create_superuser
+        else:
+            _create_user = User.objects.create_user
+
+        user = _create_user(username=username, password=password, email=email)
+        created = True
+
+    return user, created
+
+
+def delete_user(username):
+    user = User.objects.filter(username=username).first()
+    if user:
+        delete_info = user.delete()
+        deleted = True
+    else:
+        delete_info = None
+        deleted = False
+
+    return delete_info, deleted

server/website/website/views.py

@@ -9,13 +9,15 @@ import datetime
 import re
 from collections import OrderedDict
 
-from django.contrib.auth import login, logout
+from django.contrib.auth import authenticate, login, logout
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth import update_session_auth_hash
 from django.contrib.auth.forms import AuthenticationForm, UserCreationForm
 from django.contrib.auth.forms import PasswordChangeForm
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.files.base import ContentFile
+from django.db.utils import IntegrityError
+from django.forms.models import model_to_dict
 from django.http import HttpResponse, QueryDict
 from django.shortcuts import redirect, render, get_object_or_404
 from django.template.context_processors import csrf
@@ -24,9 +26,9 @@ from django.urls import reverse, reverse_lazy
 from django.utils.datetime_safe import datetime
 from django.utils.timezone import now
 from django.views.decorators.csrf import csrf_exempt
-from django.forms.models import model_to_dict
 from pytz import timezone
 
+from . import utils
 from .db import parser, target_objectives
 from .forms import NewResultForm, ProjectForm, SessionForm, SessionKnobForm
 from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog, User, Hardware,
@@ -35,7 +37,7 @@ from .tasks import (aggregate_target_results, map_workload, train_ddpg,
                     configuration_recommendation, configuration_recommendation_ddpg)
 from .types import (DBMSType, KnobUnitType, MetricType,
                     TaskType, VarType, WorkloadStatusType, AlgorithmType)
-from .utils import dump_debug_info, JSONUtil, LabelUtil, MediaUtil, TaskUtil
+from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil
 from .settings import TIME_ZONE
 
 from .set_default_knobs import set_default_knobs
@@ -744,7 +746,7 @@ def download_next_config(request):
 @login_required(login_url=reverse_lazy('login'))
 def download_debug_info(request, project_id, session_id):  # pylint: disable=unused-argument
     session = Session.objects.get(pk=session_id)
-    content, filename = dump_debug_info(session, pretty_print=False)
+    content, filename = utils.dump_debug_info(session, pretty_print=False)
     file = ContentFile(content.getvalue())
     response = HttpResponse(file, content_type='application/x-gzip')
     response['Content-Length'] = file.size
@@ -1027,7 +1029,7 @@ def get_debug_info(request, upload_code):  # pylint: disable=unused-argument
         LOG.warning("Invalid upload code: %s", upload_code)
         return HttpResponse("Invalid upload code: " + upload_code, status=400)
 
-    content, filename = dump_debug_info(session, pretty_print=pprint)
+    content, filename = utils.dump_debug_info(session, pretty_print=pprint)
     file = ContentFile(content.getvalue())
     response = HttpResponse(file, content_type='application/x-gzip')
     response['Content-Length'] = file.size
@@ -1043,6 +1045,205 @@ def train_ddpg_loops(request, session_id):  # pylint: disable=unused-argument
     return HttpResponse()
 
 
+@csrf_exempt
+def alt_create_user(request):
+    response = dict(created=False, error=None, user=None)
+    if request.method != 'POST':
+        err_msg = "Request was not a post!"
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    data = {k: v[0] for k, v in request.POST.lists()}
+    missing = [k for k in ('username', 'password') if k not in data]
+    if missing:
+        err_msg = "Request is missing required data: {}".format(', '.join(missing))
+        LOG.warning(err_msg)
+        response.update(error=err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    user, created = utils.create_user(**data)
+    response.update(user=user, created=created)
+    if created:
+        LOG.info("Successfully created user '%s': %s", data['username'], user)
+        status = 200
+    else:
+        err_msg = "ERROR: User '{}' already exists: {}".format(data['username'], user)
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        status = 400
+
+    response['user'] = model_to_dict(response['user'])
+    return HttpResponse(JSONUtil.dumps(response), status=status)
+
+
+@csrf_exempt
+def alt_delete_user(request):
+    response = dict(deleted=False, error=None, delete_info=None)
+    if request.method != 'POST':
+        err_msg = "Request was not a post!"
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    data = {k: v[0] for k, v in request.POST.lists()}
+    missing = [k for k in ('username',) if k not in data]
+    if missing:
+        err_msg = "Request is missing required data: {}".format(', '.join(missing))
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    delete_info, deleted = utils.delete_user(**data)
+    response.update(deleted=deleted, delete_info=delete_info)
+    if deleted:
+        LOG.info("Successfully deleted user '%s': %s", data['username'], delete_info)
+        status = 200
+    else:
+        err_msg = "User '{}' does not exist".format(data['username'])
+        LOG.warning(err_msg)
+        response.update(error=err_msg)
+        status = 400
+
+    return HttpResponse(JSONUtil.dumps(response), status=status)
+
+
+@csrf_exempt
+def alt_create_or_edit_project(request):
+    response = dict(created=False, error=None, project=None)
+    if request.method != 'POST':
+        err_msg = "Request was not a post!"
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    data = {k: v[0] for k, v in request.POST.lists()}
+    missing = [k for k in ('username', 'password', 'name') if k not in data]
+    if missing:
+        err_msg = "Request is missing required data: {}".format(', '.join(missing))
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    username = data.pop('username')
+    password = data.pop('password')
+    user = authenticate(User, username=username, password=password)
+    if not user:
+        err_msg = "ERROR: Unable to authenticate user '{}'.".format(username)
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    project_name = data.pop('name')
+    ts = now()
+
+    if request.path == reverse('backdoor_create_project'):
+        try:
+            project = Project.objects.create(user=user, name=project_name, last_update=ts,
+                                             creation_time=ts, **data)
+        except IntegrityError:
+            err_msg = "ERROR: Project '{}' already exists.".format(project_name)
+            project = Project.objects.get(user=user, name=project_name)
+            response.update(error=err_msg, project=model_to_dict(project))
+            LOG.warning(err_msg)
+            return HttpResponse(JSONUtil.dumps(response), status=400)
+    else:
+        project = get_object_or_404(Project, name=project_name, user=user)
+        for k, v in data.items():
+            setattr(project, k, v)
+        project.last_update = ts
+        project.save()
+
+    response.update(created=True, project=model_to_dict(project))
+    return HttpResponse(JSONUtil.dumps(response))
+
+
+@csrf_exempt
+def alt_create_or_edit_session(request):
+    response = dict(created=False, error=None, session=None)
+    if request.method != 'POST':
+        err_msg = "Request was not a post!"
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    data = {k: v[0] for k, v in request.POST.lists()}
+    if 'hardware' in data:
+        err_msg = "Custom hardware objects are not supported."
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    required_fields = ('username', 'password', 'project_name', 'name', 'dbms_type',
+                       'dbms_version')
+    missing = [k for k in required_fields if k not in data]
+    if missing:
+        err_msg = "Request is missing required data: {}".format(', '.join(missing))
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    username = data.pop('username')
+    password = data.pop('password')
+    user = authenticate(User, username=username, password=password)
+    if not user:
+        err_msg = "ERROR: Unable to authenticate user '{}'.".format(username)
+        response.update(error=err_msg)
+        LOG.warning(err_msg)
+        return HttpResponse(JSONUtil.dumps(response), status=400)
+
+    project = get_object_or_404(Project, name=data.pop('project_name'), user=user)
+    dbms_type = DBMSType.type(data.pop('dbms_type'))
+    dbms = get_object_or_404(DBMSCatalog, type=dbms_type, version=data.pop('dbms_version'))
+
+    session_name = data.pop('name')
+    if 'algorithm' in data:
+        data['algorithm'] = AlgorithmType.type(data['algorithm'])
+    session_knobs = data.pop('session_knobs', None)
+    ts = now()
+
+    if request.path == reverse('backdoor_create_session'):
+        hardware, _ = Hardware.objects.get_or_create(pk=1)
+        upload_code = data.pop('upload_code', None) or MediaUtil.upload_code_generator()
+        try:
+            session = Session.objects.create(user=user, project=project, dbms=dbms,
+                                             name=session_name, hardware=hardware,
+                                             upload_code=upload_code, creation_time=ts,
+                                             last_update=ts, **data)
+        except IntegrityError:
+            err_msg = "ERROR: Project '{}' already exists.".format(session_name)
+            session = Session.objects.get(user=user, project=project, name=session_name)
+            response.update(error=err_msg, project=model_to_dict(session))
+            LOG.warning(err_msg)
+            return HttpResponse(JSONUtil.dumps(response), status=400)
+
+        set_default_knobs(session)
+    else:
+        session = get_object_or_404(Session, name=session_name, project=project, user=user)
+        for k, v in data.items():
+            setattr(session, k, v)
+        session.last_update = ts
+        session.save()
+
+    if session_knobs:
+        session_knobs = JSONUtil.loads(session_knobs)
+        disable_others = session_knobs.pop('disable_others', False)
+        SessionKnob.objects.set_knob_min_max_tunability(session, session_knobs,
+                                                        disable_others=disable_others)
+
+    res = model_to_dict(session)
+    res['dbms_id'] = res['dbms']
+    res['dbms'] = session.dbms.full_name
+    res['hardware_id'] = res['hardware']
+    res['hardware'] = model_to_dict(session.hardware)
+    res['algorithm'] = AlgorithmType.name(res['algorithm'])
+    sess_knobs = SessionKnob.objects.get_knobs_for_session(
+        session, fields=('name', 'minval', 'maxval'))
+    res['session_knobs'] = sess_knobs
+    response.update(created=True, session=res)
+    return HttpResponse(JSONUtil.dumps(response))
+
+
 # integration test
 @csrf_exempt
 def pipeline_data_ready(request):  # pylint: disable=unused-argument