Updated management commands

server/website/.gitignore

@@ -9,6 +9,8 @@ local_settings.py
 celerybeat-schedule*
 *.pid
 
-# Raw data files #
+# Management commands #
-##################
+#######################
-data/media/*
+debug_*.tar.gz
+session_knobs.json
+dump_website.json

server/website/website/management/commands/createuser.py

@@ -6,6 +6,8 @@
 from django.contrib.auth.models import User
 from django.core.management.base import BaseCommand
 
+from website.utils import create_user  # pylint: disable=no-name-in-module,import-error
+
 
 class Command(BaseCommand):
     help = 'Create a new user.'
@@ -31,22 +33,16 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         username = options['username']
-        if User.objects.filter(username=username).exists():
+        password = options['password']
-            self.stdout.write(self.style.NOTICE(
+        email = options['email']
-                "ERROR: User '{}' already exists.".format(username)))
+        superuser = options['superuser']
-        else:
-            password = options['password']
-            email = options['email']
-            superuser = options['superuser']
 
-            if superuser:
+        _, created = create_user(username, password, email, superuser)
-                email = email or '{}@noemail.com'.format(username)
-                create_user = User.objects.create_superuser
-            else:
-                create_user = User.objects.create_user
-
-            create_user(username=username, password=password, email=email)
 
+        if created:
             self.stdout.write(self.style.SUCCESS("Successfully created {} '{}'{}.".format(
                 'superuser' if superuser else 'user', username,
                 " ('{}')".format(email) if email else '')))
+        else:
+            self.stdout.write(self.style.NOTICE(
+                "ERROR: User '{}' already exists.".format(username)))

server/website/website/management/commands/deleteuser.py

@@ -6,6 +6,8 @@
 from django.contrib.auth.models import User
 from django.core.management.base import BaseCommand
 
+from website.utils import delete_user  # pylint: disable=no-name-in-module,import-error
+
 
 class Command(BaseCommand):
     help = 'Delete an existing user.'
@@ -19,11 +21,10 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         username = options['username']
-        try:
+        _, deleted = delete_user(username)
-            user = User.objects.get(username=username)
+        if deleted:
-            user.delete()
             self.stdout.write(self.style.SUCCESS(
                 "Successfully deleted user '{}'.".format(username)))
-        except User.DoesNotExist:
+        else:
             self.stdout.write(self.style.NOTICE(
                 "ERROR: User '{}' does not exist.".format(username)))

server/website/website/management/commands/dumpknob.py

@@ -5,6 +5,7 @@
 #
 import json
 import os
+from collections import OrderedDict
 
 from django.core.management.base import BaseCommand, CommandError
 
@@ -22,13 +23,18 @@ class Command(BaseCommand):
         parser.add_argument(
             '-f', '--filename',
             metavar='FILE',
+            default='session_knobs.json',
             help='Name of the file to write the session knob tunability to. '
-                 'Default: knob.json')
+                 'Default: session_knobs.json')
         parser.add_argument(
             '-d', '--directory',
             metavar='DIR',
             help='Path of the directory to write the session knob tunability to. '
                  'Default: current directory')
+        parser.add_argument(
+            '--tunable-only',
+            action='store_true',
+            help='Dump tunable knobs only. Default: False')
 
     def handle(self, *args, **options):
         directory = options['directory'] or ''
@@ -40,13 +46,13 @@ class Command(BaseCommand):
             raise CommandError(
                 "ERROR: Session with upload code '{}' not exist.".format(options['uploadcode']))
 
-        session_knobs = SessionKnobManager.get_knob_min_max_tunability(session)
+        session_knobs = SessionKnobManager.get_knob_min_max_tunability(
+            session, tunable_only=options['tunable_only'])
 
-        filename = options['filename'] or 'knobs.json'
+        path = os.path.join(directory, options['filename'])
-        path = os.path.join(directory, filename)
 
         with open(path, 'w') as f:
-            json.dump(session_knobs, f, indent=4)
+            json.dump(OrderedDict(sorted(session_knobs.items())), f, indent=4)
 
         self.stdout.write(self.style.SUCCESS(
             "Successfully dumped knob information to '{}'.".format(path)))

server/website/website/management/commands/loadknob.py

@@ -5,14 +5,38 @@
 #
 import json
 import os
+from argparse import RawTextHelpFormatter
 
 from django.core.management.base import BaseCommand, CommandError
 
 from website.models import Session, SessionKnob, SessionKnobManager
 
+HELP = """
+Load knobs for the session with the given upload code.
+
+example of JSON file format:
+  {
+      "global.knob1": {
+          "minval": 0,
+          "maxval": 100,
+          "tunable": true
+      },
+      "global.knob2": {
+          "minval": 1000000,
+          "maxval": 2000000,
+          "tunable": false
+      }
+  }
+"""
+
 
 class Command(BaseCommand):
-    help = 'load knobs for the session with the given upload code.'
+    help = HELP
+
+    def create_parser(self, prog_name, subcommand):
+        parser = super(Command, self).create_parser(prog_name, subcommand)
+        parser.formatter_class = RawTextHelpFormatter
+        return parser
 
     def add_arguments(self, parser):
         parser.add_argument(
@@ -22,31 +46,40 @@ class Command(BaseCommand):
         parser.add_argument(
             '-f', '--filename',
             metavar='FILE',
+            default='session_knobs.json',
             help='Name of the file to read the session knob tunability from. '
-                 'Default: knob.json')
+                 'Default: session_knobs.json')
         parser.add_argument(
             '-d', '--directory',
             metavar='DIR',
             help='Path of the directory to read the session knob tunability from. '
                  'Default: current directory')
+        parser.add_argument(
+            '--disable-others',
+            action='store_true',
+            help='Disable the knob tunability of all session knobs NOT included '
+                 'in the JSON file. Default: False')
 
     def handle(self, *args, **options):
         directory = options['directory'] or ''
-        if directory and not os.path.exists(directory):
+        path = os.path.join(directory, filename)
-            os.makedirs(directory)
+
+        try:
+            with open(path, 'r') as f:
+                knobs = json.load(f)
+        except FileNotFoundError:
+            raise CommandError("ERROR: File '{}' does not exist.".format(path))
+        except json.decoder.JSONDecodeError:
+            raise CommandError("ERROR: Unable to decode JSON file '{}'.".format(path))
+
         try:
             session = Session.objects.get(upload_code=options['uploadcode'])
         except Session.DoesNotExist:
             raise CommandError(
                 "ERROR: Session with upload code '{}' not exist.".format(options['uploadcode']))
 
-        filename = options['filename'] or 'knobs.json'
+        SessionKnobManager.set_knob_min_max_tunability(
-        path = os.path.join(directory, filename)
+            session, knobs, disable_others=options['disable_others'])
-
-        with open(path, 'r') as f:
-            knobs = json.load(f)
-
-        SessionKnobManager.set_knob_min_max_tunability(session, knobs)
 
         self.stdout.write(self.style.SUCCESS(
             "Successfully load knob information from '{}'.".format(path)))

server/website/website/management/commands/resetwebsite.py

@@ -3,6 +3,8 @@
 #
 # Copyright (c) 2017-18, Carnegie Mellon University Database Group
 #
+from django import db
+from django.core.management import call_command
 from django.core.management.base import BaseCommand
 from fabric.api import local
 from website.settings import DATABASES
@@ -11,40 +13,66 @@ from website.settings import DATABASES
 class Command(BaseCommand):
     help = 'dump the website; reset the website; load data from file if specified.'
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        engine = DATABASES['default']['ENGINE']
+        user = DATABASES['default']['USER']
+        passwd = DATABASES['default']['PASSWORD']
+        host = DATABASES['default']['HOST']
+        port = DATABASES['default']['PORT']
+
+        if engine.endswith('mysql'):
+            db_cmd_fmt = 'mysql -u {user} -p{passwd} -h {host} -P {port} -N -B -e "{{cmd}}"'
+        elif engine.endswith('postgresql'):
+            db_cmd_fmt = 'PGPASSWORD={passwd} psql -U {user} -h {host} -p {port} -c "{{cmd}}"'
+        else:
+            raise NotImplementedError("Database engine '{}' is not implemented.".format(engine))
+
+        self._db_cmd_fmt = db_cmd_fmt.format(user=user, passwd=passwd, host=host, port=port).format
+
+    def call_db_command(self, cmd):
+        local(self._db_cmd_fmt(cmd=cmd))
+
     def add_arguments(self, parser):
         parser.add_argument(
             '-d', '--dumpfile',
             metavar='FILE',
-            help='Name of the file to dump data to. '
+            default='dump_website.json',
-                 'Default: dump_website.json')
+            help="Name of the file to dump data to. "
+                 "Default: 'dump_website.json'")
         parser.add_argument(
             '-l', '--loadfile',
             metavar='FILE',
-            help='Name of the file to load data from. ')
+            help="Name of the file to load data from. "
+                 "Default: '' (no data loaded)")
 
     def reset_website(self):
         # WARNING: destroys the existing website and creates with all
         # of the required inital data loaded (e.g., the KnobCatalog)
     
         # Recreate the ottertune database
-        user = DATABASES['default']['USER']
+        db.connections.close_all()
-        passwd = DATABASES['default']['PASSWORD']
+        dbname = DATABASES['default']['NAME']
-        name = DATABASES['default']['NAME']
+        self.call_db_command("DROP DATABASE IF EXISTS {}".format(dbname))
-        local("mysql -u {} -p{} -N -B -e \"DROP DATABASE IF EXISTS {}\"".format(
+        self.call_db_command("CREATE DATABASE {}".format(dbname))
-            user, passwd, name))
-        local("mysql -u {} -p{} -N -B -e \"CREATE DATABASE {}\"".format(
-            user, passwd, name))
 
         # Reinitialize the website
-        local('python manage.py migrate')
+        call_command('makemigrations', 'website')
+        call_command('migrate')
+        call_command('startcelery')
 
     def handle(self, *args, **options):
-        dumpfile = options['dumpfile'] if options['dumpfile'] else 'dump_website.json'
+        dumpfile = options['dumpfile']
-        local("python manage.py dumpdata admin auth django_db_logger djcelery sessions\
+        self.stdout.write("Dumping database to file '{}'...".format(dumpfile))
-               sites website > {}".format(dumpfile))
+        call_command('dumpdata', 'admin', 'auth', 'django_db_logger', 'djcelery', 'sessions',
-        self.reset_website()
+                     'sites', 'website', output=dumpfile)
-        if options['loadfile']:
+        call_command('stopcelery')
-            local("python manage.py loaddata '{}'".format(options['loadfile']))
 
-        self.stdout.write(self.style.SUCCESS(
+        self.reset_website()
-            "Successfully reset website."))
+
+        loadfile = options['loadfile']
+        if loadfile:
+            self.stdout.write("Loading database from file '{}'...".format(loadfile))
+            call_command('loaddata', loadfile)
+
+        self.stdout.write(self.style.SUCCESS("Successfully reset website."))

server/website/website/views.py

@@ -1005,9 +1005,12 @@ def give_result(request, upload_code):  # pylint: disable=unused-argument
             # If the task status was incomplete when we first queried latest_result
             # but succeeded before the call to TaskUtil.get_task_status() finished
             # then latest_result is stale and must be updated.
+            LOG.debug("Updating stale result (pk=%s)", latest_result.pk)
             latest_result = Result.objects.get(id=latest_result.pk)
 
         if not latest_result.next_configuration:
+            LOG.warning("Failed to get the next configuration from the latest result: %s",
+                        model_to_dict(latest_result))
             overall_status = 'FAILURE'
             response = _failed_response(latest_result, tasks, num_completed, overall_status,
                                         'Failed to get the next configuration.')