From 6bf50b892d795bf88991020f554c1754e403f4df Mon Sep 17 00:00:00 2001 From: dvanaken Date: Tue, 14 Jan 2020 06:02:31 -0500 Subject: [PATCH] Moved dumpdata code from resetwebsite into its own mgmt command, ignore errors from stopcelery kill command, added backdoor method to dump website backend database and logfiles --- .../management/commands/dumpwebsite.py | 47 +++++++++++++++++++ .../management/commands/resetwebsite.py | 7 +-- .../website/management/commands/stopcelery.py | 10 ++-- server/website/website/views.py | 39 ++++++++++++--- 4 files changed, 89 insertions(+), 14 deletions(-) create mode 100644 server/website/website/management/commands/dumpwebsite.py diff --git a/server/website/website/management/commands/dumpwebsite.py b/server/website/website/management/commands/dumpwebsite.py new file mode 100644 index 0000000..e3eb2ae --- /dev/null +++ b/server/website/website/management/commands/dumpwebsite.py @@ -0,0 +1,47 @@ +# +# OtterTune - dumpwebsite.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# +from django.core.management import call_command +from django.core.management.base import BaseCommand +from fabric.api import hide, local + + +class Command(BaseCommand): + help = 'dump the website.' + + def add_arguments(self, parser): + parser.add_argument( + '-d', '--dumpfile', + metavar='FILE', + default='dump_website.json', + help="Name of the file to dump data to. " + "Default: 'dump_website.json[.gz]'") + parser.add_argument( + '--compress', + action='store_true', + help='Compress dump data (gzip). Default: False') + + def handle(self, *args, **options): + dumpfile = options['dumpfile'] + compress = options['compress'] + if compress: + if dumpfile.endswith('.gz'): + dstfile = dumpfile + dumpfile = dumpfile[:-len('.gz')] + else: + dstfile = dumpfile + '.gz' + else: + dstfile = dumpfile + + self.stdout.write("Dumping database to file '{}'...".format(dstfile)) + call_command('dumpdata', 'admin', 'auth', 'django_db_logger', 'djcelery', 'sessions', + 'sites', 'website', output=dumpfile) + + if compress: + with hide("commands"): # pylint: disable=not-context-manager + local("gzip {}".format(dumpfile)) + + self.stdout.write(self.style.SUCCESS( + "Successfully dumped website to '{}'.".format(dstfile))) diff --git a/server/website/website/management/commands/resetwebsite.py b/server/website/website/management/commands/resetwebsite.py index 69ec07a..4fa6954 100644 --- a/server/website/website/management/commands/resetwebsite.py +++ b/server/website/website/management/commands/resetwebsite.py @@ -49,7 +49,7 @@ class Command(BaseCommand): def reset_website(self): # WARNING: destroys the existing website and creates with all # of the required inital data loaded (e.g., the KnobCatalog) - + # Recreate the ottertune database db.connections.close_all() dbname = DATABASES['default']['NAME'] @@ -62,10 +62,7 @@ class Command(BaseCommand): call_command('startcelery') def handle(self, *args, **options): - dumpfile = options['dumpfile'] - self.stdout.write("Dumping database to file '{}'...".format(dumpfile)) - call_command('dumpdata', 'admin', 'auth', 'django_db_logger', 'djcelery', 'sessions', - 'sites', 'website', output=dumpfile) + call_command('dumpwebsite', dumpfile=options['dumpfile']) call_command('stopcelery') self.reset_website() diff --git a/server/website/website/management/commands/stopcelery.py b/server/website/website/management/commands/stopcelery.py index c1091fb..2385255 100644 --- a/server/website/website/management/commands/stopcelery.py +++ b/server/website/website/management/commands/stopcelery.py @@ -7,7 +7,7 @@ import os import time from django.core.management.base import BaseCommand -from fabric.api import local +from fabric.api import local, quiet, settings class Command(BaseCommand): @@ -34,9 +34,10 @@ class Command(BaseCommand): pidfile = options[name + '_pidfile'] with open(pidfile, 'r') as f: pid = f.read() - local('kill {}'.format(pid)) + with settings(warn_only=True): + local('kill {}'.format(pid)) check_pidfiles.append((name, pidfile)) - except Exception as e: + except Exception as e: # pylint: disable=broad-except self.stdout.write(self.style.NOTICE( "ERROR: an exception occurred while stopping '{}':\n{}\n".format(name, e))) @@ -54,3 +55,6 @@ class Command(BaseCommand): else: self.stdout.write(self.style.SUCCESS( "Successfully stopped '{}'.".format(name))) + + with quiet(): + local('rm -f {} {}'.format(options['celery_pidfile'], options['celerybeat_pidfile'])) diff --git a/server/website/website/views.py b/server/website/website/views.py index 562964e..f6ee7df 100644 --- a/server/website/website/views.py +++ b/server/website/website/views.py @@ -6,8 +6,9 @@ # pylint: disable=too-many-lines import logging import datetime +import os import re -import time +import shutil from collections import OrderedDict from django.contrib.auth import authenticate, login, logout @@ -16,7 +17,8 @@ from django.contrib.auth import update_session_auth_hash from django.contrib.auth.forms import AuthenticationForm, UserCreationForm from django.contrib.auth.forms import PasswordChangeForm from django.core.exceptions import ObjectDoesNotExist -from django.core.files.base import ContentFile +from django.core.files.base import ContentFile, File +from django.core.management import call_command from django.db.utils import IntegrityError from django.forms.models import model_to_dict from django.http import HttpResponse, QueryDict @@ -38,8 +40,8 @@ from .tasks import (aggregate_target_results, map_workload, train_ddpg, configuration_recommendation, configuration_recommendation_ddpg) from .types import (DBMSType, KnobUnitType, MetricType, TaskType, VarType, WorkloadStatusType, AlgorithmType) -from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil, ConversionUtil -from .settings import TIME_ZONE +from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil +from .settings import LOG_DIR, TIME_ZONE from .set_default_knobs import set_default_knobs @@ -729,6 +731,7 @@ def knob_data_view(request, project_id, session_id, data_id): # pylint: disable target_obj = JSONUtil.loads(result.metric_data.data)[session.target_objective] return dbms_data_view(request, context, knob_data, session, target_obj) + @login_required(login_url=reverse_lazy('login')) def metric_data_view(request, project_id, session_id, data_id): # pylint: disable=unused-argument metric_data = get_object_or_404(MetricData, pk=data_id) @@ -894,8 +897,7 @@ def tuner_status_view(request, project_id, session_id, result_id): # pylint: di total_runtime = (completion_time - res.creation_time).total_seconds() total_runtime = '{0:.2f} seconds'.format(total_runtime) - task_info = [(tname, task) for tname, task in - zip(list(TaskType.TYPE_NAMES.values()), tasks)] + task_info = list(zip(TaskType.TYPE_NAMES.values(), tasks)) context = {"id": result_id, "result": res, @@ -1175,6 +1177,31 @@ def alt_get_info(request, name): # pylint: disable=unused-argument if name == 'constants': info = utils.get_constants() response = HttpResponse(JSONUtil.dumps(info)) + + elif name in ('website', 'logs'): + tmpdir = os.path.realpath('.info') + shutil.rmtree(tmpdir, ignore_errors=True) + os.makedirs(tmpdir, exist_ok=True) + + if name == 'website': + filepath = os.path.join(tmpdir, 'website_dump.json.gz') + call_command('dumpwebsite', dumpfile=filepath, compress=True) + else: # name == 'logs' + base_name = os.path.join(tmpdir, 'website_logs') + root_dir, base_dir = os.path.split(LOG_DIR) + filepath = shutil.make_archive( + base_name, format='gztar', base_dir=base_dir, root_dir=root_dir) + + f = open(filepath, 'rb') + try: + cfile = File(f) + response = HttpResponse(cfile, content_type='application/x-gzip') + response['Content-Length'] = cfile.size + response['Content-Disposition'] = 'attachment; filename={}'.format( + os.path.basename(filepath)) + finally: + f.close() + shutil.rmtree(tmpdir, ignore_errors=True) else: LOG.warning("Invalid name for info request: %s", name) response = HttpResponse("Invalid name for info request: {}".format(name), status=400)