Updated management commands
This commit is contained in:
parent
4ab2fdfd52
commit
9994a22f16
|
@ -9,6 +9,8 @@ local_settings.py
|
|||
celerybeat-schedule*
|
||||
*.pid
|
||||
|
||||
# Raw data files #
|
||||
##################
|
||||
data/media/*
|
||||
# Management commands #
|
||||
#######################
|
||||
debug_*.tar.gz
|
||||
session_knobs.json
|
||||
dump_website.json
|
||||
|
|
|
@ -6,6 +6,8 @@
|
|||
from django.contrib.auth.models import User
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from website.utils import create_user # pylint: disable=no-name-in-module,import-error
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Create a new user.'
|
||||
|
@ -31,22 +33,16 @@ class Command(BaseCommand):
|
|||
|
||||
def handle(self, *args, **options):
|
||||
username = options['username']
|
||||
if User.objects.filter(username=username).exists():
|
||||
self.stdout.write(self.style.NOTICE(
|
||||
"ERROR: User '{}' already exists.".format(username)))
|
||||
else:
|
||||
password = options['password']
|
||||
email = options['email']
|
||||
superuser = options['superuser']
|
||||
password = options['password']
|
||||
email = options['email']
|
||||
superuser = options['superuser']
|
||||
|
||||
if superuser:
|
||||
email = email or '{}@noemail.com'.format(username)
|
||||
create_user = User.objects.create_superuser
|
||||
else:
|
||||
create_user = User.objects.create_user
|
||||
|
||||
create_user(username=username, password=password, email=email)
|
||||
_, created = create_user(username, password, email, superuser)
|
||||
|
||||
if created:
|
||||
self.stdout.write(self.style.SUCCESS("Successfully created {} '{}'{}.".format(
|
||||
'superuser' if superuser else 'user', username,
|
||||
" ('{}')".format(email) if email else '')))
|
||||
else:
|
||||
self.stdout.write(self.style.NOTICE(
|
||||
"ERROR: User '{}' already exists.".format(username)))
|
||||
|
|
|
@ -6,6 +6,8 @@
|
|||
from django.contrib.auth.models import User
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from website.utils import delete_user # pylint: disable=no-name-in-module,import-error
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Delete an existing user.'
|
||||
|
@ -19,11 +21,10 @@ class Command(BaseCommand):
|
|||
|
||||
def handle(self, *args, **options):
|
||||
username = options['username']
|
||||
try:
|
||||
user = User.objects.get(username=username)
|
||||
user.delete()
|
||||
_, deleted = delete_user(username)
|
||||
if deleted:
|
||||
self.stdout.write(self.style.SUCCESS(
|
||||
"Successfully deleted user '{}'.".format(username)))
|
||||
except User.DoesNotExist:
|
||||
else:
|
||||
self.stdout.write(self.style.NOTICE(
|
||||
"ERROR: User '{}' does not exist.".format(username)))
|
||||
|
|
|
@ -5,6 +5,7 @@
|
|||
#
|
||||
import json
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
|
@ -22,13 +23,18 @@ class Command(BaseCommand):
|
|||
parser.add_argument(
|
||||
'-f', '--filename',
|
||||
metavar='FILE',
|
||||
default='session_knobs.json',
|
||||
help='Name of the file to write the session knob tunability to. '
|
||||
'Default: knob.json')
|
||||
'Default: session_knobs.json')
|
||||
parser.add_argument(
|
||||
'-d', '--directory',
|
||||
metavar='DIR',
|
||||
help='Path of the directory to write the session knob tunability to. '
|
||||
'Default: current directory')
|
||||
parser.add_argument(
|
||||
'--tunable-only',
|
||||
action='store_true',
|
||||
help='Dump tunable knobs only. Default: False')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
directory = options['directory'] or ''
|
||||
|
@ -40,13 +46,13 @@ class Command(BaseCommand):
|
|||
raise CommandError(
|
||||
"ERROR: Session with upload code '{}' not exist.".format(options['uploadcode']))
|
||||
|
||||
session_knobs = SessionKnobManager.get_knob_min_max_tunability(session)
|
||||
session_knobs = SessionKnobManager.get_knob_min_max_tunability(
|
||||
session, tunable_only=options['tunable_only'])
|
||||
|
||||
filename = options['filename'] or 'knobs.json'
|
||||
path = os.path.join(directory, filename)
|
||||
path = os.path.join(directory, options['filename'])
|
||||
|
||||
with open(path, 'w') as f:
|
||||
json.dump(session_knobs, f, indent=4)
|
||||
json.dump(OrderedDict(sorted(session_knobs.items())), f, indent=4)
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(
|
||||
"Successfully dumped knob information to '{}'.".format(path)))
|
||||
|
|
|
@ -5,14 +5,38 @@
|
|||
#
|
||||
import json
|
||||
import os
|
||||
from argparse import RawTextHelpFormatter
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from website.models import Session, SessionKnob, SessionKnobManager
|
||||
|
||||
HELP = """
|
||||
Load knobs for the session with the given upload code.
|
||||
|
||||
example of JSON file format:
|
||||
{
|
||||
"global.knob1": {
|
||||
"minval": 0,
|
||||
"maxval": 100,
|
||||
"tunable": true
|
||||
},
|
||||
"global.knob2": {
|
||||
"minval": 1000000,
|
||||
"maxval": 2000000,
|
||||
"tunable": false
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'load knobs for the session with the given upload code.'
|
||||
help = HELP
|
||||
|
||||
def create_parser(self, prog_name, subcommand):
|
||||
parser = super(Command, self).create_parser(prog_name, subcommand)
|
||||
parser.formatter_class = RawTextHelpFormatter
|
||||
return parser
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
|
@ -22,31 +46,40 @@ class Command(BaseCommand):
|
|||
parser.add_argument(
|
||||
'-f', '--filename',
|
||||
metavar='FILE',
|
||||
default='session_knobs.json',
|
||||
help='Name of the file to read the session knob tunability from. '
|
||||
'Default: knob.json')
|
||||
'Default: session_knobs.json')
|
||||
parser.add_argument(
|
||||
'-d', '--directory',
|
||||
metavar='DIR',
|
||||
help='Path of the directory to read the session knob tunability from. '
|
||||
'Default: current directory')
|
||||
parser.add_argument(
|
||||
'--disable-others',
|
||||
action='store_true',
|
||||
help='Disable the knob tunability of all session knobs NOT included '
|
||||
'in the JSON file. Default: False')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
directory = options['directory'] or ''
|
||||
if directory and not os.path.exists(directory):
|
||||
os.makedirs(directory)
|
||||
path = os.path.join(directory, filename)
|
||||
|
||||
try:
|
||||
with open(path, 'r') as f:
|
||||
knobs = json.load(f)
|
||||
except FileNotFoundError:
|
||||
raise CommandError("ERROR: File '{}' does not exist.".format(path))
|
||||
except json.decoder.JSONDecodeError:
|
||||
raise CommandError("ERROR: Unable to decode JSON file '{}'.".format(path))
|
||||
|
||||
try:
|
||||
session = Session.objects.get(upload_code=options['uploadcode'])
|
||||
except Session.DoesNotExist:
|
||||
raise CommandError(
|
||||
"ERROR: Session with upload code '{}' not exist.".format(options['uploadcode']))
|
||||
|
||||
filename = options['filename'] or 'knobs.json'
|
||||
path = os.path.join(directory, filename)
|
||||
|
||||
with open(path, 'r') as f:
|
||||
knobs = json.load(f)
|
||||
|
||||
SessionKnobManager.set_knob_min_max_tunability(session, knobs)
|
||||
SessionKnobManager.set_knob_min_max_tunability(
|
||||
session, knobs, disable_others=options['disable_others'])
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(
|
||||
"Successfully load knob information from '{}'.".format(path)))
|
||||
|
|
|
@ -3,6 +3,8 @@
|
|||
#
|
||||
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||
#
|
||||
from django import db
|
||||
from django.core.management import call_command
|
||||
from django.core.management.base import BaseCommand
|
||||
from fabric.api import local
|
||||
from website.settings import DATABASES
|
||||
|
@ -11,40 +13,66 @@ from website.settings import DATABASES
|
|||
class Command(BaseCommand):
|
||||
help = 'dump the website; reset the website; load data from file if specified.'
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
engine = DATABASES['default']['ENGINE']
|
||||
user = DATABASES['default']['USER']
|
||||
passwd = DATABASES['default']['PASSWORD']
|
||||
host = DATABASES['default']['HOST']
|
||||
port = DATABASES['default']['PORT']
|
||||
|
||||
if engine.endswith('mysql'):
|
||||
db_cmd_fmt = 'mysql -u {user} -p{passwd} -h {host} -P {port} -N -B -e "{{cmd}}"'
|
||||
elif engine.endswith('postgresql'):
|
||||
db_cmd_fmt = 'PGPASSWORD={passwd} psql -U {user} -h {host} -p {port} -c "{{cmd}}"'
|
||||
else:
|
||||
raise NotImplementedError("Database engine '{}' is not implemented.".format(engine))
|
||||
|
||||
self._db_cmd_fmt = db_cmd_fmt.format(user=user, passwd=passwd, host=host, port=port).format
|
||||
|
||||
def call_db_command(self, cmd):
|
||||
local(self._db_cmd_fmt(cmd=cmd))
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
'-d', '--dumpfile',
|
||||
metavar='FILE',
|
||||
help='Name of the file to dump data to. '
|
||||
'Default: dump_website.json')
|
||||
default='dump_website.json',
|
||||
help="Name of the file to dump data to. "
|
||||
"Default: 'dump_website.json'")
|
||||
parser.add_argument(
|
||||
'-l', '--loadfile',
|
||||
metavar='FILE',
|
||||
help='Name of the file to load data from. ')
|
||||
help="Name of the file to load data from. "
|
||||
"Default: '' (no data loaded)")
|
||||
|
||||
def reset_website(self):
|
||||
# WARNING: destroys the existing website and creates with all
|
||||
# of the required inital data loaded (e.g., the KnobCatalog)
|
||||
|
||||
# Recreate the ottertune database
|
||||
user = DATABASES['default']['USER']
|
||||
passwd = DATABASES['default']['PASSWORD']
|
||||
name = DATABASES['default']['NAME']
|
||||
local("mysql -u {} -p{} -N -B -e \"DROP DATABASE IF EXISTS {}\"".format(
|
||||
user, passwd, name))
|
||||
local("mysql -u {} -p{} -N -B -e \"CREATE DATABASE {}\"".format(
|
||||
user, passwd, name))
|
||||
|
||||
db.connections.close_all()
|
||||
dbname = DATABASES['default']['NAME']
|
||||
self.call_db_command("DROP DATABASE IF EXISTS {}".format(dbname))
|
||||
self.call_db_command("CREATE DATABASE {}".format(dbname))
|
||||
|
||||
# Reinitialize the website
|
||||
local('python manage.py migrate')
|
||||
call_command('makemigrations', 'website')
|
||||
call_command('migrate')
|
||||
call_command('startcelery')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
dumpfile = options['dumpfile'] if options['dumpfile'] else 'dump_website.json'
|
||||
local("python manage.py dumpdata admin auth django_db_logger djcelery sessions\
|
||||
sites website > {}".format(dumpfile))
|
||||
self.reset_website()
|
||||
if options['loadfile']:
|
||||
local("python manage.py loaddata '{}'".format(options['loadfile']))
|
||||
dumpfile = options['dumpfile']
|
||||
self.stdout.write("Dumping database to file '{}'...".format(dumpfile))
|
||||
call_command('dumpdata', 'admin', 'auth', 'django_db_logger', 'djcelery', 'sessions',
|
||||
'sites', 'website', output=dumpfile)
|
||||
call_command('stopcelery')
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(
|
||||
"Successfully reset website."))
|
||||
self.reset_website()
|
||||
|
||||
loadfile = options['loadfile']
|
||||
if loadfile:
|
||||
self.stdout.write("Loading database from file '{}'...".format(loadfile))
|
||||
call_command('loaddata', loadfile)
|
||||
|
||||
self.stdout.write(self.style.SUCCESS("Successfully reset website."))
|
||||
|
|
|
@ -1005,9 +1005,12 @@ def give_result(request, upload_code): # pylint: disable=unused-argument
|
|||
# If the task status was incomplete when we first queried latest_result
|
||||
# but succeeded before the call to TaskUtil.get_task_status() finished
|
||||
# then latest_result is stale and must be updated.
|
||||
LOG.debug("Updating stale result (pk=%s)", latest_result.pk)
|
||||
latest_result = Result.objects.get(id=latest_result.pk)
|
||||
|
||||
if not latest_result.next_configuration:
|
||||
LOG.warning("Failed to get the next configuration from the latest result: %s",
|
||||
model_to_dict(latest_result))
|
||||
overall_status = 'FAILURE'
|
||||
response = _failed_response(latest_result, tasks, num_completed, overall_status,
|
||||
'Failed to get the next configuration.')
|
||||
|
|
Loading…
Reference in New Issue