diff --git a/client/driver/fabfile.py b/client/driver/fabfile.py index c94d1ee..b3ad036 100644 --- a/client/driver/fabfile.py +++ b/client/driver/fabfile.py @@ -663,6 +663,89 @@ def rename_batch(result_dir=None): os.rename(fpath, rename_path) +def _http_content_to_json(content): + if isinstance(content, bytes): + content = content.decode('utf-8') + try: + json_content = json.loads(content) + decoded = True + except (TypeError, json.decoder.JSONDecodeError): + json_content = None + decoded = False + + return json_content, decoded + + +def _modify_website_object(obj_name, action, verbose=False, **kwargs): + verbose = _parse_bool(verbose) + if obj_name == 'project': + valid_actions = ('create', 'edit') + elif obj_name == 'session': + valid_actions = ('create', 'edit') + elif obj_name == 'user': + valid_actions = ('create', 'delete') + else: + raise ValueError('Invalid object: {}. Valid objects: project, session'.format(obj_name)) + + if action not in valid_actions: + raise ValueError('Invalid action: {}. Valid actions: {}'.format( + action, ', '.join(valid_actions))) + + data = {} + for k, v in kwargs.items(): + if isinstance(v, (dict, list, tuple)): + v = json.dumps(v) + data[k] = v + + url_path = '/{}/{}/'.format(action, obj_name) + response = requests.post(CONF['upload_url'] + url_path, data=data) + + content = response.content.decode('utf-8') + if response.status_code != 200: + raise Exception("Failed to {} new {}.\nStatus: {}\nMessage: {}\n".format( + action, obj_name, response.status_code, content)) + + json_content, decoded = _http_content_to_json(content) + if verbose: + if decoded: + LOG.info('\n%s_%s = %s', action.upper(), obj_name.upper(), + json.dumps(json_content, indent=4)) + else: + LOG.warning("Content could not be decoded.\n\n%s\n", content) + + return response, json_content, decoded + + +@task +def create_website_user(**kwargs): + return _modify_website_object('user', 'create', **kwargs) + + +@task +def delete_website_user(**kwargs): + return _modify_website_object('user', 'delete', **kwargs) + + +@task +def create_website_project(**kwargs): + return _modify_website_object('project', 'create', **kwargs) + + +@task +def edit_website_project(**kwargs): + return _modify_website_object('project', 'edit', **kwargs) + + +@task +def create_website_session(**kwargs): + return _modify_website_object('session', 'create', **kwargs) + + +@task +def edit_website_session(**kwargs): + return _modify_website_object('session', 'edit', **kwargs) + + def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10): max_time_sec = int(max_time_sec) interval_sec = int(interval_sec) diff --git a/script/formatting/config/pylintrc b/script/formatting/config/pylintrc index a539a61..4d77f42 100644 --- a/script/formatting/config/pylintrc +++ b/script/formatting/config/pylintrc @@ -13,9 +13,9 @@ profile=no # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=CVS,.git,manage.py +ignore=CVS,.git,manage.py,0001_initial.py,0002_enable_compression.py,0003_load_initial_data.py,0004_add_lhs.py -ignore-patterns=**/migrations/*.py +# ignore-patterns=**/migrations/*.py # Pickle collected data for later comparisons. persistent=no diff --git a/server/website/website/admin.py b/server/website/website/admin.py index f4f014d..b825b46 100644 --- a/server/website/website/admin.py +++ b/server/website/website/admin.py @@ -50,7 +50,7 @@ class SessionKnobAdmin(admin.ModelAdmin): list_display = ('knob', 'dbms', 'session', 'minval', 'maxval', 'tunable') list_filter = (('session__dbms', admin.RelatedOnlyFieldListFilter), ('session', admin.RelatedOnlyFieldListFilter), - ('tunable', admin.FieldListFilter)) + 'tunable') ordering = ('session__dbms', 'session__name', '-tunable', 'knob__name') def dbms(self, instance): # pylint: disable=no-self-use diff --git a/server/website/website/models.py b/server/website/website/models.py index 01db1a7..eb8542a 100644 --- a/server/website/website/models.py +++ b/server/website/website/models.py @@ -211,15 +211,19 @@ class SessionKnobManager(models.Manager): return session_knob_dicts @staticmethod - def set_knob_min_max_tunability(session, knob_dicts): + def set_knob_min_max_tunability(session, knob_dicts, disable_others=False): # Returns a dict of the knob session_knobs = SessionKnob.objects.filter(session=session) for session_knob in session_knobs: - if knob_dicts.__contains__(session_knob.name): + if session_knob.name in knob_dicts: session_knob.minval = knob_dicts[session_knob.name]["minval"] session_knob.maxval = knob_dicts[session_knob.name]["maxval"] session_knob.tunable = knob_dicts[session_knob.name]["tunable"] session_knob.save() + elif disable_others: + # Set all knobs not in knob_dicts to not tunable + session_knob.tunable = False + session_knob.save() class SessionKnob(BaseModel): diff --git a/server/website/website/urls.py b/server/website/website/urls.py index 30c5025..a5de75b 100644 --- a/server/website/website/urls.py +++ b/server/website/website/urls.py @@ -67,6 +67,12 @@ urlpatterns = [ # Back door url(r'^query_and_get/(?P[0-9a-zA-Z]+)$', website_views.give_result, name="backdoor"), url(r'^dump/(?P[0-9a-zA-Z]+)', website_views.get_debug_info, name="backdoor_debug"), + url(r'^create/project/', website_views.alt_create_or_edit_project, name='backdoor_create_project'), + url(r'^edit/project/', website_views.alt_create_or_edit_project, name='backdoor_edit_project'), + url(r'^create/session/', website_views.alt_create_or_edit_session, name='backdoor_create_session'), + url(r'^edit/session/', website_views.alt_create_or_edit_session, name='backdoor_edit_session'), + url(r'^create/user/', website_views.alt_create_user, name='backdoor_create_user'), + url(r'^delete/user/', website_views.alt_delete_user, name='backdoor_delete_user'), # train ddpg with results in the given session url(r'^train_ddpg/sessions/(?P[0-9]+)$', website_views.train_ddpg_loops, name='train_ddpg_loops'), diff --git a/server/website/website/utils.py b/server/website/website/utils.py index 844d546..4afde37 100644 --- a/server/website/website/utils.py +++ b/server/website/website/utils.py @@ -16,6 +16,7 @@ from io import BytesIO from random import choice import numpy as np +from django.contrib.auth.models import User from django.utils.text import capfirst from django_db_logger.models import StatusLog from djcelery.models import TaskMeta @@ -437,3 +438,32 @@ def dump_debug_info(session, pretty_print=False): tarstream.seek(0) return tarstream, root + + +def create_user(username, password, email=None, superuser=False): + user = User.objects.filter(username=username).first() + if user: + created = False + else: + if superuser: + email = email or '{}@noemail.com'.format(username) + _create_user = User.objects.create_superuser + else: + _create_user = User.objects.create_user + + user = _create_user(username=username, password=password, email=email) + created = True + + return user, created + + +def delete_user(username): + user = User.objects.filter(username=username).first() + if user: + delete_info = user.delete() + deleted = True + else: + delete_info = None + deleted = False + + return delete_info, deleted diff --git a/server/website/website/views.py b/server/website/website/views.py index 29b7286..5443a48 100644 --- a/server/website/website/views.py +++ b/server/website/website/views.py @@ -9,13 +9,15 @@ import datetime import re from collections import OrderedDict -from django.contrib.auth import login, logout +from django.contrib.auth import authenticate, login, logout from django.contrib.auth.decorators import login_required from django.contrib.auth import update_session_auth_hash from django.contrib.auth.forms import AuthenticationForm, UserCreationForm from django.contrib.auth.forms import PasswordChangeForm from django.core.exceptions import ObjectDoesNotExist from django.core.files.base import ContentFile +from django.db.utils import IntegrityError +from django.forms.models import model_to_dict from django.http import HttpResponse, QueryDict from django.shortcuts import redirect, render, get_object_or_404 from django.template.context_processors import csrf @@ -24,9 +26,9 @@ from django.urls import reverse, reverse_lazy from django.utils.datetime_safe import datetime from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt -from django.forms.models import model_to_dict from pytz import timezone +from . import utils from .db import parser, target_objectives from .forms import NewResultForm, ProjectForm, SessionForm, SessionKnobForm from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog, User, Hardware, @@ -35,7 +37,7 @@ from .tasks import (aggregate_target_results, map_workload, train_ddpg, configuration_recommendation, configuration_recommendation_ddpg) from .types import (DBMSType, KnobUnitType, MetricType, TaskType, VarType, WorkloadStatusType, AlgorithmType) -from .utils import dump_debug_info, JSONUtil, LabelUtil, MediaUtil, TaskUtil +from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil from .settings import TIME_ZONE from .set_default_knobs import set_default_knobs @@ -744,7 +746,7 @@ def download_next_config(request): @login_required(login_url=reverse_lazy('login')) def download_debug_info(request, project_id, session_id): # pylint: disable=unused-argument session = Session.objects.get(pk=session_id) - content, filename = dump_debug_info(session, pretty_print=False) + content, filename = utils.dump_debug_info(session, pretty_print=False) file = ContentFile(content.getvalue()) response = HttpResponse(file, content_type='application/x-gzip') response['Content-Length'] = file.size @@ -1027,7 +1029,7 @@ def get_debug_info(request, upload_code): # pylint: disable=unused-argument LOG.warning("Invalid upload code: %s", upload_code) return HttpResponse("Invalid upload code: " + upload_code, status=400) - content, filename = dump_debug_info(session, pretty_print=pprint) + content, filename = utils.dump_debug_info(session, pretty_print=pprint) file = ContentFile(content.getvalue()) response = HttpResponse(file, content_type='application/x-gzip') response['Content-Length'] = file.size @@ -1043,6 +1045,205 @@ def train_ddpg_loops(request, session_id): # pylint: disable=unused-argument return HttpResponse() +@csrf_exempt +def alt_create_user(request): + response = dict(created=False, error=None, user=None) + if request.method != 'POST': + err_msg = "Request was not a post!" + response.update(error=err_msg) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + data = {k: v[0] for k, v in request.POST.lists()} + missing = [k for k in ('username', 'password') if k not in data] + if missing: + err_msg = "Request is missing required data: {}".format(', '.join(missing)) + LOG.warning(err_msg) + response.update(error=err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + user, created = utils.create_user(**data) + response.update(user=user, created=created) + if created: + LOG.info("Successfully created user '%s': %s", data['username'], user) + status = 200 + else: + err_msg = "ERROR: User '{}' already exists: {}".format(data['username'], user) + response.update(error=err_msg) + LOG.warning(err_msg) + status = 400 + + response['user'] = model_to_dict(response['user']) + return HttpResponse(JSONUtil.dumps(response), status=status) + + +@csrf_exempt +def alt_delete_user(request): + response = dict(deleted=False, error=None, delete_info=None) + if request.method != 'POST': + err_msg = "Request was not a post!" + response.update(error=err_msg) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + data = {k: v[0] for k, v in request.POST.lists()} + missing = [k for k in ('username',) if k not in data] + if missing: + err_msg = "Request is missing required data: {}".format(', '.join(missing)) + response.update(error=err_msg) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + delete_info, deleted = utils.delete_user(**data) + response.update(deleted=deleted, delete_info=delete_info) + if deleted: + LOG.info("Successfully deleted user '%s': %s", data['username'], delete_info) + status = 200 + else: + err_msg = "User '{}' does not exist".format(data['username']) + LOG.warning(err_msg) + response.update(error=err_msg) + status = 400 + + return HttpResponse(JSONUtil.dumps(response), status=status) + + +@csrf_exempt +def alt_create_or_edit_project(request): + response = dict(created=False, error=None, project=None) + if request.method != 'POST': + err_msg = "Request was not a post!" + response.update(error=err_msg) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + data = {k: v[0] for k, v in request.POST.lists()} + missing = [k for k in ('username', 'password', 'name') if k not in data] + if missing: + err_msg = "Request is missing required data: {}".format(', '.join(missing)) + response.update(error=err_msg) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + username = data.pop('username') + password = data.pop('password') + user = authenticate(User, username=username, password=password) + if not user: + err_msg = "ERROR: Unable to authenticate user '{}'.".format(username) + response.update(error=err_msg) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + project_name = data.pop('name') + ts = now() + + if request.path == reverse('backdoor_create_project'): + try: + project = Project.objects.create(user=user, name=project_name, last_update=ts, + creation_time=ts, **data) + except IntegrityError: + err_msg = "ERROR: Project '{}' already exists.".format(project_name) + project = Project.objects.get(user=user, name=project_name) + response.update(error=err_msg, project=model_to_dict(project)) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + else: + project = get_object_or_404(Project, name=project_name, user=user) + for k, v in data.items(): + setattr(project, k, v) + project.last_update = ts + project.save() + + response.update(created=True, project=model_to_dict(project)) + return HttpResponse(JSONUtil.dumps(response)) + + +@csrf_exempt +def alt_create_or_edit_session(request): + response = dict(created=False, error=None, session=None) + if request.method != 'POST': + err_msg = "Request was not a post!" + response.update(error=err_msg) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + data = {k: v[0] for k, v in request.POST.lists()} + if 'hardware' in data: + err_msg = "Custom hardware objects are not supported." + response.update(error=err_msg) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + required_fields = ('username', 'password', 'project_name', 'name', 'dbms_type', + 'dbms_version') + missing = [k for k in required_fields if k not in data] + if missing: + err_msg = "Request is missing required data: {}".format(', '.join(missing)) + response.update(error=err_msg) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + username = data.pop('username') + password = data.pop('password') + user = authenticate(User, username=username, password=password) + if not user: + err_msg = "ERROR: Unable to authenticate user '{}'.".format(username) + response.update(error=err_msg) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + project = get_object_or_404(Project, name=data.pop('project_name'), user=user) + dbms_type = DBMSType.type(data.pop('dbms_type')) + dbms = get_object_or_404(DBMSCatalog, type=dbms_type, version=data.pop('dbms_version')) + + session_name = data.pop('name') + if 'algorithm' in data: + data['algorithm'] = AlgorithmType.type(data['algorithm']) + session_knobs = data.pop('session_knobs', None) + ts = now() + + if request.path == reverse('backdoor_create_session'): + hardware, _ = Hardware.objects.get_or_create(pk=1) + upload_code = data.pop('upload_code', None) or MediaUtil.upload_code_generator() + try: + session = Session.objects.create(user=user, project=project, dbms=dbms, + name=session_name, hardware=hardware, + upload_code=upload_code, creation_time=ts, + last_update=ts, **data) + except IntegrityError: + err_msg = "ERROR: Project '{}' already exists.".format(session_name) + session = Session.objects.get(user=user, project=project, name=session_name) + response.update(error=err_msg, project=model_to_dict(session)) + LOG.warning(err_msg) + return HttpResponse(JSONUtil.dumps(response), status=400) + + set_default_knobs(session) + else: + session = get_object_or_404(Session, name=session_name, project=project, user=user) + for k, v in data.items(): + setattr(session, k, v) + session.last_update = ts + session.save() + + if session_knobs: + session_knobs = JSONUtil.loads(session_knobs) + disable_others = session_knobs.pop('disable_others', False) + SessionKnob.objects.set_knob_min_max_tunability(session, session_knobs, + disable_others=disable_others) + + res = model_to_dict(session) + res['dbms_id'] = res['dbms'] + res['dbms'] = session.dbms.full_name + res['hardware_id'] = res['hardware'] + res['hardware'] = model_to_dict(session.hardware) + res['algorithm'] = AlgorithmType.name(res['algorithm']) + sess_knobs = SessionKnob.objects.get_knobs_for_session( + session, fields=('name', 'minval', 'maxval')) + res['session_knobs'] = sess_knobs + response.update(created=True, session=res) + return HttpResponse(JSONUtil.dumps(response)) + + # integration test @csrf_exempt def pipeline_data_ready(request): # pylint: disable=unused-argument