From 9994a22f16fe4db18d29eb1e998b678804676ef2 Mon Sep 17 00:00:00 2001 From: dvanaken Date: Mon, 25 Nov 2019 22:50:16 -0500 Subject: [PATCH] Updated management commands --- server/website/.gitignore | 8 ++- .../website/management/commands/createuser.py | 24 +++---- .../website/management/commands/deleteuser.py | 9 +-- .../website/management/commands/dumpknob.py | 16 +++-- .../website/management/commands/loadknob.py | 55 ++++++++++++--- .../management/commands/resetwebsite.py | 68 +++++++++++++------ server/website/website/views.py | 3 + 7 files changed, 126 insertions(+), 57 deletions(-) diff --git a/server/website/.gitignore b/server/website/.gitignore index 628c1d0..a7fa9a3 100644 --- a/server/website/.gitignore +++ b/server/website/.gitignore @@ -9,6 +9,8 @@ local_settings.py celerybeat-schedule* *.pid -# Raw data files # -################## -data/media/* +# Management commands # +####################### +debug_*.tar.gz +session_knobs.json +dump_website.json diff --git a/server/website/website/management/commands/createuser.py b/server/website/website/management/commands/createuser.py index 278fdf0..90b2bdd 100644 --- a/server/website/website/management/commands/createuser.py +++ b/server/website/website/management/commands/createuser.py @@ -6,6 +6,8 @@ from django.contrib.auth.models import User from django.core.management.base import BaseCommand +from website.utils import create_user # pylint: disable=no-name-in-module,import-error + class Command(BaseCommand): help = 'Create a new user.' @@ -31,22 +33,16 @@ class Command(BaseCommand): def handle(self, *args, **options): username = options['username'] - if User.objects.filter(username=username).exists(): - self.stdout.write(self.style.NOTICE( - "ERROR: User '{}' already exists.".format(username))) - else: - password = options['password'] - email = options['email'] - superuser = options['superuser'] + password = options['password'] + email = options['email'] + superuser = options['superuser'] - if superuser: - email = email or '{}@noemail.com'.format(username) - create_user = User.objects.create_superuser - else: - create_user = User.objects.create_user - - create_user(username=username, password=password, email=email) + _, created = create_user(username, password, email, superuser) + if created: self.stdout.write(self.style.SUCCESS("Successfully created {} '{}'{}.".format( 'superuser' if superuser else 'user', username, " ('{}')".format(email) if email else ''))) + else: + self.stdout.write(self.style.NOTICE( + "ERROR: User '{}' already exists.".format(username))) diff --git a/server/website/website/management/commands/deleteuser.py b/server/website/website/management/commands/deleteuser.py index f682e59..39e050e 100644 --- a/server/website/website/management/commands/deleteuser.py +++ b/server/website/website/management/commands/deleteuser.py @@ -6,6 +6,8 @@ from django.contrib.auth.models import User from django.core.management.base import BaseCommand +from website.utils import delete_user # pylint: disable=no-name-in-module,import-error + class Command(BaseCommand): help = 'Delete an existing user.' @@ -19,11 +21,10 @@ class Command(BaseCommand): def handle(self, *args, **options): username = options['username'] - try: - user = User.objects.get(username=username) - user.delete() + _, deleted = delete_user(username) + if deleted: self.stdout.write(self.style.SUCCESS( "Successfully deleted user '{}'.".format(username))) - except User.DoesNotExist: + else: self.stdout.write(self.style.NOTICE( "ERROR: User '{}' does not exist.".format(username))) diff --git a/server/website/website/management/commands/dumpknob.py b/server/website/website/management/commands/dumpknob.py index 9e34945..05a3b5b 100644 --- a/server/website/website/management/commands/dumpknob.py +++ b/server/website/website/management/commands/dumpknob.py @@ -5,6 +5,7 @@ # import json import os +from collections import OrderedDict from django.core.management.base import BaseCommand, CommandError @@ -22,13 +23,18 @@ class Command(BaseCommand): parser.add_argument( '-f', '--filename', metavar='FILE', + default='session_knobs.json', help='Name of the file to write the session knob tunability to. ' - 'Default: knob.json') + 'Default: session_knobs.json') parser.add_argument( '-d', '--directory', metavar='DIR', help='Path of the directory to write the session knob tunability to. ' 'Default: current directory') + parser.add_argument( + '--tunable-only', + action='store_true', + help='Dump tunable knobs only. Default: False') def handle(self, *args, **options): directory = options['directory'] or '' @@ -40,13 +46,13 @@ class Command(BaseCommand): raise CommandError( "ERROR: Session with upload code '{}' not exist.".format(options['uploadcode'])) - session_knobs = SessionKnobManager.get_knob_min_max_tunability(session) + session_knobs = SessionKnobManager.get_knob_min_max_tunability( + session, tunable_only=options['tunable_only']) - filename = options['filename'] or 'knobs.json' - path = os.path.join(directory, filename) + path = os.path.join(directory, options['filename']) with open(path, 'w') as f: - json.dump(session_knobs, f, indent=4) + json.dump(OrderedDict(sorted(session_knobs.items())), f, indent=4) self.stdout.write(self.style.SUCCESS( "Successfully dumped knob information to '{}'.".format(path))) diff --git a/server/website/website/management/commands/loadknob.py b/server/website/website/management/commands/loadknob.py index 446368b..5f94470 100644 --- a/server/website/website/management/commands/loadknob.py +++ b/server/website/website/management/commands/loadknob.py @@ -5,14 +5,38 @@ # import json import os +from argparse import RawTextHelpFormatter from django.core.management.base import BaseCommand, CommandError from website.models import Session, SessionKnob, SessionKnobManager +HELP = """ +Load knobs for the session with the given upload code. + +example of JSON file format: + { + "global.knob1": { + "minval": 0, + "maxval": 100, + "tunable": true + }, + "global.knob2": { + "minval": 1000000, + "maxval": 2000000, + "tunable": false + } + } +""" + class Command(BaseCommand): - help = 'load knobs for the session with the given upload code.' + help = HELP + + def create_parser(self, prog_name, subcommand): + parser = super(Command, self).create_parser(prog_name, subcommand) + parser.formatter_class = RawTextHelpFormatter + return parser def add_arguments(self, parser): parser.add_argument( @@ -22,31 +46,40 @@ class Command(BaseCommand): parser.add_argument( '-f', '--filename', metavar='FILE', + default='session_knobs.json', help='Name of the file to read the session knob tunability from. ' - 'Default: knob.json') + 'Default: session_knobs.json') parser.add_argument( '-d', '--directory', metavar='DIR', help='Path of the directory to read the session knob tunability from. ' 'Default: current directory') + parser.add_argument( + '--disable-others', + action='store_true', + help='Disable the knob tunability of all session knobs NOT included ' + 'in the JSON file. Default: False') def handle(self, *args, **options): directory = options['directory'] or '' - if directory and not os.path.exists(directory): - os.makedirs(directory) + path = os.path.join(directory, filename) + + try: + with open(path, 'r') as f: + knobs = json.load(f) + except FileNotFoundError: + raise CommandError("ERROR: File '{}' does not exist.".format(path)) + except json.decoder.JSONDecodeError: + raise CommandError("ERROR: Unable to decode JSON file '{}'.".format(path)) + try: session = Session.objects.get(upload_code=options['uploadcode']) except Session.DoesNotExist: raise CommandError( "ERROR: Session with upload code '{}' not exist.".format(options['uploadcode'])) - filename = options['filename'] or 'knobs.json' - path = os.path.join(directory, filename) - - with open(path, 'r') as f: - knobs = json.load(f) - - SessionKnobManager.set_knob_min_max_tunability(session, knobs) + SessionKnobManager.set_knob_min_max_tunability( + session, knobs, disable_others=options['disable_others']) self.stdout.write(self.style.SUCCESS( "Successfully load knob information from '{}'.".format(path))) diff --git a/server/website/website/management/commands/resetwebsite.py b/server/website/website/management/commands/resetwebsite.py index fecf1c3..a743dc5 100644 --- a/server/website/website/management/commands/resetwebsite.py +++ b/server/website/website/management/commands/resetwebsite.py @@ -3,6 +3,8 @@ # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # +from django import db +from django.core.management import call_command from django.core.management.base import BaseCommand from fabric.api import local from website.settings import DATABASES @@ -11,40 +13,66 @@ from website.settings import DATABASES class Command(BaseCommand): help = 'dump the website; reset the website; load data from file if specified.' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + engine = DATABASES['default']['ENGINE'] + user = DATABASES['default']['USER'] + passwd = DATABASES['default']['PASSWORD'] + host = DATABASES['default']['HOST'] + port = DATABASES['default']['PORT'] + + if engine.endswith('mysql'): + db_cmd_fmt = 'mysql -u {user} -p{passwd} -h {host} -P {port} -N -B -e "{{cmd}}"' + elif engine.endswith('postgresql'): + db_cmd_fmt = 'PGPASSWORD={passwd} psql -U {user} -h {host} -p {port} -c "{{cmd}}"' + else: + raise NotImplementedError("Database engine '{}' is not implemented.".format(engine)) + + self._db_cmd_fmt = db_cmd_fmt.format(user=user, passwd=passwd, host=host, port=port).format + + def call_db_command(self, cmd): + local(self._db_cmd_fmt(cmd=cmd)) + def add_arguments(self, parser): parser.add_argument( '-d', '--dumpfile', metavar='FILE', - help='Name of the file to dump data to. ' - 'Default: dump_website.json') + default='dump_website.json', + help="Name of the file to dump data to. " + "Default: 'dump_website.json'") parser.add_argument( '-l', '--loadfile', metavar='FILE', - help='Name of the file to load data from. ') + help="Name of the file to load data from. " + "Default: '' (no data loaded)") def reset_website(self): # WARNING: destroys the existing website and creates with all # of the required inital data loaded (e.g., the KnobCatalog) # Recreate the ottertune database - user = DATABASES['default']['USER'] - passwd = DATABASES['default']['PASSWORD'] - name = DATABASES['default']['NAME'] - local("mysql -u {} -p{} -N -B -e \"DROP DATABASE IF EXISTS {}\"".format( - user, passwd, name)) - local("mysql -u {} -p{} -N -B -e \"CREATE DATABASE {}\"".format( - user, passwd, name)) - + db.connections.close_all() + dbname = DATABASES['default']['NAME'] + self.call_db_command("DROP DATABASE IF EXISTS {}".format(dbname)) + self.call_db_command("CREATE DATABASE {}".format(dbname)) + # Reinitialize the website - local('python manage.py migrate') + call_command('makemigrations', 'website') + call_command('migrate') + call_command('startcelery') def handle(self, *args, **options): - dumpfile = options['dumpfile'] if options['dumpfile'] else 'dump_website.json' - local("python manage.py dumpdata admin auth django_db_logger djcelery sessions\ - sites website > {}".format(dumpfile)) - self.reset_website() - if options['loadfile']: - local("python manage.py loaddata '{}'".format(options['loadfile'])) + dumpfile = options['dumpfile'] + self.stdout.write("Dumping database to file '{}'...".format(dumpfile)) + call_command('dumpdata', 'admin', 'auth', 'django_db_logger', 'djcelery', 'sessions', + 'sites', 'website', output=dumpfile) + call_command('stopcelery') - self.stdout.write(self.style.SUCCESS( - "Successfully reset website.")) + self.reset_website() + + loadfile = options['loadfile'] + if loadfile: + self.stdout.write("Loading database from file '{}'...".format(loadfile)) + call_command('loaddata', loadfile) + + self.stdout.write(self.style.SUCCESS("Successfully reset website.")) diff --git a/server/website/website/views.py b/server/website/website/views.py index 500db8d..439a799 100644 --- a/server/website/website/views.py +++ b/server/website/website/views.py @@ -1005,9 +1005,12 @@ def give_result(request, upload_code): # pylint: disable=unused-argument # If the task status was incomplete when we first queried latest_result # but succeeded before the call to TaskUtil.get_task_status() finished # then latest_result is stale and must be updated. + LOG.debug("Updating stale result (pk=%s)", latest_result.pk) latest_result = Result.objects.get(id=latest_result.pk) if not latest_result.next_configuration: + LOG.warning("Failed to get the next configuration from the latest result: %s", + model_to_dict(latest_result)) overall_status = 'FAILURE' response = _failed_response(latest_result, tasks, num_completed, overall_status, 'Failed to get the next configuration.')