Added backdoor functions for creating/editing/deleting users/projects/sessions in the website and driver.
This commit is contained in:
dvanaken
Dana Van Aken
parent
b261cf2139
commit
4171a662f5
|
|
@ -663,6 +663,89 @@ def rename_batch(result_dir=None):
|
|||
os.rename(fpath, rename_path)
|
||||
|
||||
|
||||
def _http_content_to_json(content):
|
||||
if isinstance(content, bytes):
|
||||
content = content.decode('utf-8')
|
||||
try:
|
||||
json_content = json.loads(content)
|
||||
decoded = True
|
||||
except (TypeError, json.decoder.JSONDecodeError):
|
||||
json_content = None
|
||||
decoded = False
|
||||
|
||||
return json_content, decoded
|
||||
|
||||
|
||||
def _modify_website_object(obj_name, action, verbose=False, **kwargs):
|
||||
verbose = _parse_bool(verbose)
|
||||
if obj_name == 'project':
|
||||
valid_actions = ('create', 'edit')
|
||||
elif obj_name == 'session':
|
||||
valid_actions = ('create', 'edit')
|
||||
elif obj_name == 'user':
|
||||
valid_actions = ('create', 'delete')
|
||||
else:
|
||||
raise ValueError('Invalid object: {}. Valid objects: project, session'.format(obj_name))
|
||||
|
||||
if action not in valid_actions:
|
||||
raise ValueError('Invalid action: {}. Valid actions: {}'.format(
|
||||
action, ', '.join(valid_actions)))
|
||||
|
||||
data = {}
|
||||
for k, v in kwargs.items():
|
||||
if isinstance(v, (dict, list, tuple)):
|
||||
v = json.dumps(v)
|
||||
data[k] = v
|
||||
|
||||
url_path = '/{}/{}/'.format(action, obj_name)
|
||||
response = requests.post(CONF['upload_url'] + url_path, data=data)
|
||||
|
||||
content = response.content.decode('utf-8')
|
||||
if response.status_code != 200:
|
||||
raise Exception("Failed to {} new {}.\nStatus: {}\nMessage: {}\n".format(
|
||||
action, obj_name, response.status_code, content))
|
||||
|
||||
json_content, decoded = _http_content_to_json(content)
|
||||
if verbose:
|
||||
if decoded:
|
||||
LOG.info('\n%s_%s = %s', action.upper(), obj_name.upper(),
|
||||
json.dumps(json_content, indent=4))
|
||||
else:
|
||||
LOG.warning("Content could not be decoded.\n\n%s\n", content)
|
||||
|
||||
return response, json_content, decoded
|
||||
|
||||
|
||||
@task
|
||||
def create_website_user(**kwargs):
|
||||
return _modify_website_object('user', 'create', **kwargs)
|
||||
|
||||
|
||||
@task
|
||||
def delete_website_user(**kwargs):
|
||||
return _modify_website_object('user', 'delete', **kwargs)
|
||||
|
||||
|
||||
@task
|
||||
def create_website_project(**kwargs):
|
||||
return _modify_website_object('project', 'create', **kwargs)
|
||||
|
||||
|
||||
@task
|
||||
def edit_website_project(**kwargs):
|
||||
return _modify_website_object('project', 'edit', **kwargs)
|
||||
|
||||
|
||||
@task
|
||||
def create_website_session(**kwargs):
|
||||
return _modify_website_object('session', 'create', **kwargs)
|
||||
|
||||
|
||||
@task
|
||||
def edit_website_session(**kwargs):
|
||||
return _modify_website_object('session', 'edit', **kwargs)
|
||||
|
||||
|
||||
def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10):
|
||||
max_time_sec = int(max_time_sec)
|
||||
interval_sec = int(interval_sec)
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ profile=no
|
|||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS,.git,manage.py
|
||||
ignore=CVS,.git,manage.py,0001_initial.py,0002_enable_compression.py,0003_load_initial_data.py,0004_add_lhs.py
|
||||
|
||||
ignore-patterns=**/migrations/*.py
|
||||
# ignore-patterns=**/migrations/*.py
|
||||
|
||||
# Pickle collected data for later comparisons.
|
||||
persistent=no
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ class SessionKnobAdmin(admin.ModelAdmin):
|
|||
list_display = ('knob', 'dbms', 'session', 'minval', 'maxval', 'tunable')
|
||||
list_filter = (('session__dbms', admin.RelatedOnlyFieldListFilter),
|
||||
('session', admin.RelatedOnlyFieldListFilter),
|
||||
('tunable', admin.FieldListFilter))
|
||||
'tunable')
|
||||
ordering = ('session__dbms', 'session__name', '-tunable', 'knob__name')
|
||||
|
||||
def dbms(self, instance): # pylint: disable=no-self-use
|
||||
|
|
|
|||
|
|
@ -211,15 +211,19 @@ class SessionKnobManager(models.Manager):
|
|||
return session_knob_dicts
|
||||
|
||||
@staticmethod
|
||||
def set_knob_min_max_tunability(session, knob_dicts):
|
||||
def set_knob_min_max_tunability(session, knob_dicts, disable_others=False):
|
||||
# Returns a dict of the knob
|
||||
session_knobs = SessionKnob.objects.filter(session=session)
|
||||
for session_knob in session_knobs:
|
||||
if knob_dicts.__contains__(session_knob.name):
|
||||
if session_knob.name in knob_dicts:
|
||||
session_knob.minval = knob_dicts[session_knob.name]["minval"]
|
||||
session_knob.maxval = knob_dicts[session_knob.name]["maxval"]
|
||||
session_knob.tunable = knob_dicts[session_knob.name]["tunable"]
|
||||
session_knob.save()
|
||||
elif disable_others:
|
||||
# Set all knobs not in knob_dicts to not tunable
|
||||
session_knob.tunable = False
|
||||
session_knob.save()
|
||||
|
||||
|
||||
class SessionKnob(BaseModel):
|
||||
|
|
|
|||
|
|
@ -67,6 +67,12 @@ urlpatterns = [
|
|||
# Back door
|
||||
url(r'^query_and_get/(?P<upload_code>[0-9a-zA-Z]+)$', website_views.give_result, name="backdoor"),
|
||||
url(r'^dump/(?P<upload_code>[0-9a-zA-Z]+)', website_views.get_debug_info, name="backdoor_debug"),
|
||||
url(r'^create/project/', website_views.alt_create_or_edit_project, name='backdoor_create_project'),
|
||||
url(r'^edit/project/', website_views.alt_create_or_edit_project, name='backdoor_edit_project'),
|
||||
url(r'^create/session/', website_views.alt_create_or_edit_session, name='backdoor_create_session'),
|
||||
url(r'^edit/session/', website_views.alt_create_or_edit_session, name='backdoor_edit_session'),
|
||||
url(r'^create/user/', website_views.alt_create_user, name='backdoor_create_user'),
|
||||
url(r'^delete/user/', website_views.alt_delete_user, name='backdoor_delete_user'),
|
||||
|
||||
# train ddpg with results in the given session
|
||||
url(r'^train_ddpg/sessions/(?P<session_id>[0-9]+)$', website_views.train_ddpg_loops, name='train_ddpg_loops'),
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from io import BytesIO
|
|||
from random import choice
|
||||
|
||||
import numpy as np
|
||||
from django.contrib.auth.models import User
|
||||
from django.utils.text import capfirst
|
||||
from django_db_logger.models import StatusLog
|
||||
from djcelery.models import TaskMeta
|
||||
|
|
@ -437,3 +438,32 @@ def dump_debug_info(session, pretty_print=False):
|
|||
|
||||
tarstream.seek(0)
|
||||
return tarstream, root
|
||||
|
||||
|
||||
def create_user(username, password, email=None, superuser=False):
|
||||
user = User.objects.filter(username=username).first()
|
||||
if user:
|
||||
created = False
|
||||
else:
|
||||
if superuser:
|
||||
email = email or '{}@noemail.com'.format(username)
|
||||
_create_user = User.objects.create_superuser
|
||||
else:
|
||||
_create_user = User.objects.create_user
|
||||
|
||||
user = _create_user(username=username, password=password, email=email)
|
||||
created = True
|
||||
|
||||
return user, created
|
||||
|
||||
|
||||
def delete_user(username):
|
||||
user = User.objects.filter(username=username).first()
|
||||
if user:
|
||||
delete_info = user.delete()
|
||||
deleted = True
|
||||
else:
|
||||
delete_info = None
|
||||
deleted = False
|
||||
|
||||
return delete_info, deleted
|
||||
|
|
|
|||
|
|
@ -9,13 +9,15 @@ import datetime
|
|||
import re
|
||||
from collections import OrderedDict
|
||||
|
||||
from django.contrib.auth import login, logout
|
||||
from django.contrib.auth import authenticate, login, logout
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.contrib.auth import update_session_auth_hash
|
||||
from django.contrib.auth.forms import AuthenticationForm, UserCreationForm
|
||||
from django.contrib.auth.forms import PasswordChangeForm
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.core.files.base import ContentFile
|
||||
from django.db.utils import IntegrityError
|
||||
from django.forms.models import model_to_dict
|
||||
from django.http import HttpResponse, QueryDict
|
||||
from django.shortcuts import redirect, render, get_object_or_404
|
||||
from django.template.context_processors import csrf
|
||||
|
|
@ -24,9 +26,9 @@ from django.urls import reverse, reverse_lazy
|
|||
from django.utils.datetime_safe import datetime
|
||||
from django.utils.timezone import now
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
from django.forms.models import model_to_dict
|
||||
from pytz import timezone
|
||||
|
||||
from . import utils
|
||||
from .db import parser, target_objectives
|
||||
from .forms import NewResultForm, ProjectForm, SessionForm, SessionKnobForm
|
||||
from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog, User, Hardware,
|
||||
|
|
@ -35,7 +37,7 @@ from .tasks import (aggregate_target_results, map_workload, train_ddpg,
|
|||
configuration_recommendation, configuration_recommendation_ddpg)
|
||||
from .types import (DBMSType, KnobUnitType, MetricType,
|
||||
TaskType, VarType, WorkloadStatusType, AlgorithmType)
|
||||
from .utils import dump_debug_info, JSONUtil, LabelUtil, MediaUtil, TaskUtil
|
||||
from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil
|
||||
from .settings import TIME_ZONE
|
||||
|
||||
from .set_default_knobs import set_default_knobs
|
||||
|
|
@ -744,7 +746,7 @@ def download_next_config(request):
|
|||
@login_required(login_url=reverse_lazy('login'))
|
||||
def download_debug_info(request, project_id, session_id): # pylint: disable=unused-argument
|
||||
session = Session.objects.get(pk=session_id)
|
||||
content, filename = dump_debug_info(session, pretty_print=False)
|
||||
content, filename = utils.dump_debug_info(session, pretty_print=False)
|
||||
file = ContentFile(content.getvalue())
|
||||
response = HttpResponse(file, content_type='application/x-gzip')
|
||||
response['Content-Length'] = file.size
|
||||
|
|
@ -1027,7 +1029,7 @@ def get_debug_info(request, upload_code): # pylint: disable=unused-argument
|
|||
LOG.warning("Invalid upload code: %s", upload_code)
|
||||
return HttpResponse("Invalid upload code: " + upload_code, status=400)
|
||||
|
||||
content, filename = dump_debug_info(session, pretty_print=pprint)
|
||||
content, filename = utils.dump_debug_info(session, pretty_print=pprint)
|
||||
file = ContentFile(content.getvalue())
|
||||
response = HttpResponse(file, content_type='application/x-gzip')
|
||||
response['Content-Length'] = file.size
|
||||
|
|
@ -1043,6 +1045,205 @@ def train_ddpg_loops(request, session_id): # pylint: disable=unused-argument
|
|||
return HttpResponse()
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def alt_create_user(request):
|
||||
response = dict(created=False, error=None, user=None)
|
||||
if request.method != 'POST':
|
||||
err_msg = "Request was not a post!"
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
data = {k: v[0] for k, v in request.POST.lists()}
|
||||
missing = [k for k in ('username', 'password') if k not in data]
|
||||
if missing:
|
||||
err_msg = "Request is missing required data: {}".format(', '.join(missing))
|
||||
LOG.warning(err_msg)
|
||||
response.update(error=err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
user, created = utils.create_user(**data)
|
||||
response.update(user=user, created=created)
|
||||
if created:
|
||||
LOG.info("Successfully created user '%s': %s", data['username'], user)
|
||||
status = 200
|
||||
else:
|
||||
err_msg = "ERROR: User '{}' already exists: {}".format(data['username'], user)
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
status = 400
|
||||
|
||||
response['user'] = model_to_dict(response['user'])
|
||||
return HttpResponse(JSONUtil.dumps(response), status=status)
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def alt_delete_user(request):
|
||||
response = dict(deleted=False, error=None, delete_info=None)
|
||||
if request.method != 'POST':
|
||||
err_msg = "Request was not a post!"
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
data = {k: v[0] for k, v in request.POST.lists()}
|
||||
missing = [k for k in ('username',) if k not in data]
|
||||
if missing:
|
||||
err_msg = "Request is missing required data: {}".format(', '.join(missing))
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
delete_info, deleted = utils.delete_user(**data)
|
||||
response.update(deleted=deleted, delete_info=delete_info)
|
||||
if deleted:
|
||||
LOG.info("Successfully deleted user '%s': %s", data['username'], delete_info)
|
||||
status = 200
|
||||
else:
|
||||
err_msg = "User '{}' does not exist".format(data['username'])
|
||||
LOG.warning(err_msg)
|
||||
response.update(error=err_msg)
|
||||
status = 400
|
||||
|
||||
return HttpResponse(JSONUtil.dumps(response), status=status)
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def alt_create_or_edit_project(request):
|
||||
response = dict(created=False, error=None, project=None)
|
||||
if request.method != 'POST':
|
||||
err_msg = "Request was not a post!"
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
data = {k: v[0] for k, v in request.POST.lists()}
|
||||
missing = [k for k in ('username', 'password', 'name') if k not in data]
|
||||
if missing:
|
||||
err_msg = "Request is missing required data: {}".format(', '.join(missing))
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
username = data.pop('username')
|
||||
password = data.pop('password')
|
||||
user = authenticate(User, username=username, password=password)
|
||||
if not user:
|
||||
err_msg = "ERROR: Unable to authenticate user '{}'.".format(username)
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
project_name = data.pop('name')
|
||||
ts = now()
|
||||
|
||||
if request.path == reverse('backdoor_create_project'):
|
||||
try:
|
||||
project = Project.objects.create(user=user, name=project_name, last_update=ts,
|
||||
creation_time=ts, **data)
|
||||
except IntegrityError:
|
||||
err_msg = "ERROR: Project '{}' already exists.".format(project_name)
|
||||
project = Project.objects.get(user=user, name=project_name)
|
||||
response.update(error=err_msg, project=model_to_dict(project))
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
else:
|
||||
project = get_object_or_404(Project, name=project_name, user=user)
|
||||
for k, v in data.items():
|
||||
setattr(project, k, v)
|
||||
project.last_update = ts
|
||||
project.save()
|
||||
|
||||
response.update(created=True, project=model_to_dict(project))
|
||||
return HttpResponse(JSONUtil.dumps(response))
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def alt_create_or_edit_session(request):
|
||||
response = dict(created=False, error=None, session=None)
|
||||
if request.method != 'POST':
|
||||
err_msg = "Request was not a post!"
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
data = {k: v[0] for k, v in request.POST.lists()}
|
||||
if 'hardware' in data:
|
||||
err_msg = "Custom hardware objects are not supported."
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
required_fields = ('username', 'password', 'project_name', 'name', 'dbms_type',
|
||||
'dbms_version')
|
||||
missing = [k for k in required_fields if k not in data]
|
||||
if missing:
|
||||
err_msg = "Request is missing required data: {}".format(', '.join(missing))
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
username = data.pop('username')
|
||||
password = data.pop('password')
|
||||
user = authenticate(User, username=username, password=password)
|
||||
if not user:
|
||||
err_msg = "ERROR: Unable to authenticate user '{}'.".format(username)
|
||||
response.update(error=err_msg)
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
project = get_object_or_404(Project, name=data.pop('project_name'), user=user)
|
||||
dbms_type = DBMSType.type(data.pop('dbms_type'))
|
||||
dbms = get_object_or_404(DBMSCatalog, type=dbms_type, version=data.pop('dbms_version'))
|
||||
|
||||
session_name = data.pop('name')
|
||||
if 'algorithm' in data:
|
||||
data['algorithm'] = AlgorithmType.type(data['algorithm'])
|
||||
session_knobs = data.pop('session_knobs', None)
|
||||
ts = now()
|
||||
|
||||
if request.path == reverse('backdoor_create_session'):
|
||||
hardware, _ = Hardware.objects.get_or_create(pk=1)
|
||||
upload_code = data.pop('upload_code', None) or MediaUtil.upload_code_generator()
|
||||
try:
|
||||
session = Session.objects.create(user=user, project=project, dbms=dbms,
|
||||
name=session_name, hardware=hardware,
|
||||
upload_code=upload_code, creation_time=ts,
|
||||
last_update=ts, **data)
|
||||
except IntegrityError:
|
||||
err_msg = "ERROR: Project '{}' already exists.".format(session_name)
|
||||
session = Session.objects.get(user=user, project=project, name=session_name)
|
||||
response.update(error=err_msg, project=model_to_dict(session))
|
||||
LOG.warning(err_msg)
|
||||
return HttpResponse(JSONUtil.dumps(response), status=400)
|
||||
|
||||
set_default_knobs(session)
|
||||
else:
|
||||
session = get_object_or_404(Session, name=session_name, project=project, user=user)
|
||||
for k, v in data.items():
|
||||
setattr(session, k, v)
|
||||
session.last_update = ts
|
||||
session.save()
|
||||
|
||||
if session_knobs:
|
||||
session_knobs = JSONUtil.loads(session_knobs)
|
||||
disable_others = session_knobs.pop('disable_others', False)
|
||||
SessionKnob.objects.set_knob_min_max_tunability(session, session_knobs,
|
||||
disable_others=disable_others)
|
||||
|
||||
res = model_to_dict(session)
|
||||
res['dbms_id'] = res['dbms']
|
||||
res['dbms'] = session.dbms.full_name
|
||||
res['hardware_id'] = res['hardware']
|
||||
res['hardware'] = model_to_dict(session.hardware)
|
||||
res['algorithm'] = AlgorithmType.name(res['algorithm'])
|
||||
sess_knobs = SessionKnob.objects.get_knobs_for_session(
|
||||
session, fields=('name', 'minval', 'maxval'))
|
||||
res['session_knobs'] = sess_knobs
|
||||
response.update(created=True, session=res)
|
||||
return HttpResponse(JSONUtil.dumps(response))
|
||||
|
||||
|
||||
# integration test
|
||||
@csrf_exempt
|
||||
def pipeline_data_ready(request): # pylint: disable=unused-argument
|
||||
|
|
|
|||
Loading…
Reference in New Issue