Moved dumpdata code from resetwebsite into its own mgmt command, ignore errors from stopcelery kill command, added backdoor method to dump website backend database and logfiles

server/website/website/management/commands/dumpwebsite.py

@@ -0,0 +1,47 @@
+#
+# OtterTune - dumpwebsite.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+from django.core.management import call_command
+from django.core.management.base import BaseCommand
+from fabric.api import hide, local
+
+
+class Command(BaseCommand):
+    help = 'dump the website.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-d', '--dumpfile',
+            metavar='FILE',
+            default='dump_website.json',
+            help="Name of the file to dump data to. "
+                 "Default: 'dump_website.json[.gz]'")
+        parser.add_argument(
+            '--compress',
+            action='store_true',
+            help='Compress dump data (gzip). Default: False')
+
+    def handle(self, *args, **options):
+        dumpfile = options['dumpfile']
+        compress = options['compress']
+        if compress:
+            if dumpfile.endswith('.gz'):
+                dstfile = dumpfile
+                dumpfile = dumpfile[:-len('.gz')]
+            else:
+                dstfile = dumpfile + '.gz'
+        else:
+            dstfile = dumpfile
+
+        self.stdout.write("Dumping database to file '{}'...".format(dstfile))
+        call_command('dumpdata', 'admin', 'auth', 'django_db_logger', 'djcelery', 'sessions',
+                     'sites', 'website', output=dumpfile)
+
+        if compress:
+            with hide("commands"):  # pylint: disable=not-context-manager
+                local("gzip {}".format(dumpfile))
+
+        self.stdout.write(self.style.SUCCESS(
+            "Successfully dumped website to '{}'.".format(dstfile)))

server/website/website/management/commands/resetwebsite.py

@@ -49,7 +49,7 @@ class Command(BaseCommand):
     def reset_website(self):
         # WARNING: destroys the existing website and creates with all
         # of the required inital data loaded (e.g., the KnobCatalog)
-    
+
         # Recreate the ottertune database
         db.connections.close_all()
         dbname = DATABASES['default']['NAME']
@@ -62,10 +62,7 @@ class Command(BaseCommand):
         call_command('startcelery')
 
     def handle(self, *args, **options):
-        dumpfile = options['dumpfile']
+        call_command('dumpwebsite', dumpfile=options['dumpfile'])
-        self.stdout.write("Dumping database to file '{}'...".format(dumpfile))
-        call_command('dumpdata', 'admin', 'auth', 'django_db_logger', 'djcelery', 'sessions',
-                     'sites', 'website', output=dumpfile)
         call_command('stopcelery')
 
         self.reset_website()

server/website/website/management/commands/stopcelery.py

@@ -7,7 +7,7 @@ import os
 import time
 
 from django.core.management.base import BaseCommand
-from fabric.api import local
+from fabric.api import local, quiet, settings
 
 
 class Command(BaseCommand):
@@ -34,9 +34,10 @@ class Command(BaseCommand):
                 pidfile = options[name + '_pidfile']
                 with open(pidfile, 'r') as f:
                     pid = f.read()
-                local('kill {}'.format(pid))
+                with settings(warn_only=True):
+                    local('kill {}'.format(pid))
                 check_pidfiles.append((name, pidfile))
-            except Exception as e:
+            except Exception as e:  # pylint: disable=broad-except
                 self.stdout.write(self.style.NOTICE(
                     "ERROR: an exception occurred while stopping '{}':\n{}\n".format(name, e)))
 
@@ -54,3 +55,6 @@ class Command(BaseCommand):
             else:
                 self.stdout.write(self.style.SUCCESS(
                     "Successfully stopped '{}'.".format(name)))
+
+        with quiet():
+            local('rm -f {} {}'.format(options['celery_pidfile'], options['celerybeat_pidfile']))

server/website/website/views.py

@@ -6,8 +6,9 @@
 # pylint: disable=too-many-lines
 import logging
 import datetime
+import os
 import re
-import time
+import shutil
 from collections import OrderedDict
 
 from django.contrib.auth import authenticate, login, logout
@@ -16,7 +17,8 @@ from django.contrib.auth import update_session_auth_hash
 from django.contrib.auth.forms import AuthenticationForm, UserCreationForm
 from django.contrib.auth.forms import PasswordChangeForm
 from django.core.exceptions import ObjectDoesNotExist
-from django.core.files.base import ContentFile
+from django.core.files.base import ContentFile, File
+from django.core.management import call_command
 from django.db.utils import IntegrityError
 from django.forms.models import model_to_dict
 from django.http import HttpResponse, QueryDict
@@ -38,8 +40,8 @@ from .tasks import (aggregate_target_results, map_workload, train_ddpg,
                     configuration_recommendation, configuration_recommendation_ddpg)
 from .types import (DBMSType, KnobUnitType, MetricType,
                     TaskType, VarType, WorkloadStatusType, AlgorithmType)
-from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil, ConversionUtil
+from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil
-from .settings import TIME_ZONE
+from .settings import LOG_DIR, TIME_ZONE
 
 from .set_default_knobs import set_default_knobs
 
@@ -729,6 +731,7 @@ def knob_data_view(request, project_id, session_id, data_id):  # pylint: disable
     target_obj = JSONUtil.loads(result.metric_data.data)[session.target_objective]
     return dbms_data_view(request, context, knob_data, session, target_obj)
 
+
 @login_required(login_url=reverse_lazy('login'))
 def metric_data_view(request, project_id, session_id, data_id):  # pylint: disable=unused-argument
     metric_data = get_object_or_404(MetricData, pk=data_id)
@@ -894,8 +897,7 @@ def tuner_status_view(request, project_id, session_id, result_id):  # pylint: di
         total_runtime = (completion_time - res.creation_time).total_seconds()
         total_runtime = '{0:.2f} seconds'.format(total_runtime)
 
-    task_info = [(tname, task) for tname, task in
+    task_info = list(zip(TaskType.TYPE_NAMES.values(), tasks))
-                 zip(list(TaskType.TYPE_NAMES.values()), tasks)]
 
     context = {"id": result_id,
                "result": res,
@@ -1175,6 +1177,31 @@ def alt_get_info(request, name):  # pylint: disable=unused-argument
     if name == 'constants':
         info = utils.get_constants()
         response = HttpResponse(JSONUtil.dumps(info))
+
+    elif name in ('website', 'logs'):
+        tmpdir = os.path.realpath('.info')
+        shutil.rmtree(tmpdir, ignore_errors=True)
+        os.makedirs(tmpdir, exist_ok=True)
+
+        if name == 'website':
+            filepath = os.path.join(tmpdir, 'website_dump.json.gz')
+            call_command('dumpwebsite', dumpfile=filepath, compress=True)
+        else:  # name == 'logs'
+            base_name = os.path.join(tmpdir, 'website_logs')
+            root_dir, base_dir = os.path.split(LOG_DIR)
+            filepath = shutil.make_archive(
+                base_name, format='gztar', base_dir=base_dir, root_dir=root_dir)
+
+        f = open(filepath, 'rb')
+        try:
+            cfile = File(f)
+            response = HttpResponse(cfile, content_type='application/x-gzip')
+            response['Content-Length'] = cfile.size
+            response['Content-Disposition'] = 'attachment; filename={}'.format(
+                os.path.basename(filepath))
+        finally:
+            f.close()
+        shutil.rmtree(tmpdir, ignore_errors=True)
     else:
         LOG.warning("Invalid name for info request: %s", name)
         response = HttpResponse("Invalid name for info request: {}".format(name), status=400)