public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated management commands

This commit is contained in:
dvanaken 2019-11-25 22:50:16 -05:00 committed by Dana Van Aken
parent 4ab2fdfd52
commit 9994a22f16
7 changed files with 126 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
server/website/.gitignore vendored
View File

@ -9,6 +9,8 @@ local_settings.py
celerybeat-schedule*
*.pid
# Raw data files #
##################
data/media/*
# Management commands #
#######################
debug_*.tar.gz
session_knobs.json
dump_website.json

24
server/website/website/management/commands/createuser.py
View File

@ -6,6 +6,8 @@
from django.contrib.auth.models import User
from django.core.management.base import BaseCommand
from website.utils import create_user # pylint: disable=no-name-in-module,import-error
class Command(BaseCommand):
help = 'Create a new user.'
@ -31,22 +33,16 @@ class Command(BaseCommand):
def handle(self, *args, **options):
username = options['username']
if User.objects.filter(username=username).exists():
self.stdout.write(self.style.NOTICE(
"ERROR: User '{}' already exists.".format(username)))
else:
password = options['password']
email = options['email']
superuser = options['superuser']
password = options['password']
email = options['email']
superuser = options['superuser']
if superuser:
email = email or '{}@noemail.com'.format(username)
create_user = User.objects.create_superuser
else:
create_user = User.objects.create_user
create_user(username=username, password=password, email=email)
_, created = create_user(username, password, email, superuser)
if created:
self.stdout.write(self.style.SUCCESS("Successfully created {} '{}'{}.".format(
'superuser' if superuser else 'user', username,
" ('{}')".format(email) if email else '')))
else:
self.stdout.write(self.style.NOTICE(
"ERROR: User '{}' already exists.".format(username)))

9
server/website/website/management/commands/deleteuser.py
View File

@ -6,6 +6,8 @@
from django.contrib.auth.models import User
from django.core.management.base import BaseCommand
from website.utils import delete_user # pylint: disable=no-name-in-module,import-error
class Command(BaseCommand):
help = 'Delete an existing user.'
@ -19,11 +21,10 @@ class Command(BaseCommand):
def handle(self, *args, **options):
username = options['username']
try:
user = User.objects.get(username=username)
user.delete()
_, deleted = delete_user(username)
if deleted:
self.stdout.write(self.style.SUCCESS(
"Successfully deleted user '{}'.".format(username)))
except User.DoesNotExist:
else:
self.stdout.write(self.style.NOTICE(
"ERROR: User '{}' does not exist.".format(username)))

16
server/website/website/management/commands/dumpknob.py
View File

@ -5,6 +5,7 @@
#
import json
import os
from collections import OrderedDict
from django.core.management.base import BaseCommand, CommandError
@ -22,13 +23,18 @@ class Command(BaseCommand):
parser.add_argument(
'-f', '--filename',
metavar='FILE',
default='session_knobs.json',
help='Name of the file to write the session knob tunability to. '
'Default: knob.json')
'Default: session_knobs.json')
parser.add_argument(
'-d', '--directory',
metavar='DIR',
help='Path of the directory to write the session knob tunability to. '
'Default: current directory')
parser.add_argument(
'--tunable-only',
action='store_true',
help='Dump tunable knobs only. Default: False')
def handle(self, *args, **options):
directory = options['directory'] or ''
@ -40,13 +46,13 @@ class Command(BaseCommand):
raise CommandError(
"ERROR: Session with upload code '{}' not exist.".format(options['uploadcode']))
session_knobs = SessionKnobManager.get_knob_min_max_tunability(session)
session_knobs = SessionKnobManager.get_knob_min_max_tunability(
session, tunable_only=options['tunable_only'])
filename = options['filename'] or 'knobs.json'
path = os.path.join(directory, filename)
path = os.path.join(directory, options['filename'])
with open(path, 'w') as f:
json.dump(session_knobs, f, indent=4)
json.dump(OrderedDict(sorted(session_knobs.items())), f, indent=4)
self.stdout.write(self.style.SUCCESS(
"Successfully dumped knob information to '{}'.".format(path)))

55
server/website/website/management/commands/loadknob.py
View File

@ -5,14 +5,38 @@
#
import json
import os
from argparse import RawTextHelpFormatter
from django.core.management.base import BaseCommand, CommandError
from website.models import Session, SessionKnob, SessionKnobManager
HELP = """
Load knobs for the session with the given upload code.
example of JSON file format:
{
"global.knob1": {
"minval": 0,
"maxval": 100,
"tunable": true
},
"global.knob2": {
"minval": 1000000,
"maxval": 2000000,
"tunable": false
}
}
"""
class Command(BaseCommand):
help = 'load knobs for the session with the given upload code.'
help = HELP
def create_parser(self, prog_name, subcommand):
parser = super(Command, self).create_parser(prog_name, subcommand)
parser.formatter_class = RawTextHelpFormatter
return parser
def add_arguments(self, parser):
parser.add_argument(
@ -22,31 +46,40 @@ class Command(BaseCommand):
parser.add_argument(
'-f', '--filename',
metavar='FILE',
default='session_knobs.json',
help='Name of the file to read the session knob tunability from. '
'Default: knob.json')
'Default: session_knobs.json')
parser.add_argument(
'-d', '--directory',
metavar='DIR',
help='Path of the directory to read the session knob tunability from. '
'Default: current directory')
parser.add_argument(
'--disable-others',
action='store_true',
help='Disable the knob tunability of all session knobs NOT included '
'in the JSON file. Default: False')
def handle(self, *args, **options):
directory = options['directory'] or ''
if directory and not os.path.exists(directory):
os.makedirs(directory)
path = os.path.join(directory, filename)
try:
with open(path, 'r') as f:
knobs = json.load(f)
except FileNotFoundError:
raise CommandError("ERROR: File '{}' does not exist.".format(path))
except json.decoder.JSONDecodeError:
raise CommandError("ERROR: Unable to decode JSON file '{}'.".format(path))
try:
session = Session.objects.get(upload_code=options['uploadcode'])
except Session.DoesNotExist:
raise CommandError(
"ERROR: Session with upload code '{}' not exist.".format(options['uploadcode']))
filename = options['filename'] or 'knobs.json'
path = os.path.join(directory, filename)
with open(path, 'r') as f:
knobs = json.load(f)
SessionKnobManager.set_knob_min_max_tunability(session, knobs)
SessionKnobManager.set_knob_min_max_tunability(
session, knobs, disable_others=options['disable_others'])
self.stdout.write(self.style.SUCCESS(
"Successfully load knob information from '{}'.".format(path)))

68
server/website/website/management/commands/resetwebsite.py
View File

@ -3,6 +3,8 @@
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from django import db
from django.core.management import call_command
from django.core.management.base import BaseCommand
from fabric.api import local
from website.settings import DATABASES
@ -11,40 +13,66 @@ from website.settings import DATABASES
class Command(BaseCommand):
help = 'dump the website; reset the website; load data from file if specified.'
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
engine = DATABASES['default']['ENGINE']
user = DATABASES['default']['USER']
passwd = DATABASES['default']['PASSWORD']
host = DATABASES['default']['HOST']
port = DATABASES['default']['PORT']
if engine.endswith('mysql'):
db_cmd_fmt = 'mysql -u {user} -p{passwd} -h {host} -P {port} -N -B -e "{{cmd}}"'
elif engine.endswith('postgresql'):
db_cmd_fmt = 'PGPASSWORD={passwd} psql -U {user} -h {host} -p {port} -c "{{cmd}}"'
else:
raise NotImplementedError("Database engine '{}' is not implemented.".format(engine))
self._db_cmd_fmt = db_cmd_fmt.format(user=user, passwd=passwd, host=host, port=port).format
def call_db_command(self, cmd):
local(self._db_cmd_fmt(cmd=cmd))
def add_arguments(self, parser):
parser.add_argument(
'-d', '--dumpfile',
metavar='FILE',
help='Name of the file to dump data to. '
'Default: dump_website.json')
default='dump_website.json',
help="Name of the file to dump data to. "
"Default: 'dump_website.json'")
parser.add_argument(
'-l', '--loadfile',
metavar='FILE',
help='Name of the file to load data from. ')
help="Name of the file to load data from. "
"Default: '' (no data loaded)")
def reset_website(self):
# WARNING: destroys the existing website and creates with all
# of the required inital data loaded (e.g., the KnobCatalog)
# Recreate the ottertune database
user = DATABASES['default']['USER']
passwd = DATABASES['default']['PASSWORD']
name = DATABASES['default']['NAME']
local("mysql -u {} -p{} -N -B -e \"DROP DATABASE IF EXISTS {}\"".format(
user, passwd, name))
local("mysql -u {} -p{} -N -B -e \"CREATE DATABASE {}\"".format(
user, passwd, name))
db.connections.close_all()
dbname = DATABASES['default']['NAME']
self.call_db_command("DROP DATABASE IF EXISTS {}".format(dbname))
self.call_db_command("CREATE DATABASE {}".format(dbname))
# Reinitialize the website
local('python manage.py migrate')
call_command('makemigrations', 'website')
call_command('migrate')
call_command('startcelery')
def handle(self, *args, **options):
dumpfile = options['dumpfile'] if options['dumpfile'] else 'dump_website.json'
local("python manage.py dumpdata admin auth django_db_logger djcelery sessions\
sites website > {}".format(dumpfile))
self.reset_website()
if options['loadfile']:
local("python manage.py loaddata '{}'".format(options['loadfile']))
dumpfile = options['dumpfile']
self.stdout.write("Dumping database to file '{}'...".format(dumpfile))
call_command('dumpdata', 'admin', 'auth', 'django_db_logger', 'djcelery', 'sessions',
'sites', 'website', output=dumpfile)
call_command('stopcelery')
self.stdout.write(self.style.SUCCESS(
"Successfully reset website."))
self.reset_website()
loadfile = options['loadfile']
if loadfile:
self.stdout.write("Loading database from file '{}'...".format(loadfile))
call_command('loaddata', loadfile)
self.stdout.write(self.style.SUCCESS("Successfully reset website."))

3
server/website/website/views.py
View File

@ -1005,9 +1005,12 @@ def give_result(request, upload_code): # pylint: disable=unused-argument
# If the task status was incomplete when we first queried latest_result
# but succeeded before the call to TaskUtil.get_task_status() finished
# then latest_result is stale and must be updated.
LOG.debug("Updating stale result (pk=%s)", latest_result.pk)
latest_result = Result.objects.get(id=latest_result.pk)
if not latest_result.next_configuration:
LOG.warning("Failed to get the next configuration from the latest result: %s",
model_to_dict(latest_result))
overall_status = 'FAILURE'
response = _failed_response(latest_result, tasks, num_completed, overall_status,
'Failed to get the next configuration.')