Moved dumpdata code from resetwebsite into its own mgmt command, ignore errors from stopcelery kill command, added backdoor method to dump website backend database and logfiles

This commit is contained in:
dvanaken 2020-01-14 06:02:31 -05:00 committed by Dana Van Aken
parent 9f9f2845d6
commit 6bf50b892d
4 changed files with 89 additions and 14 deletions

View File

@ -0,0 +1,47 @@
#
# OtterTune - dumpwebsite.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from django.core.management import call_command
from django.core.management.base import BaseCommand
from fabric.api import hide, local
class Command(BaseCommand):
help = 'dump the website.'
def add_arguments(self, parser):
parser.add_argument(
'-d', '--dumpfile',
metavar='FILE',
default='dump_website.json',
help="Name of the file to dump data to. "
"Default: 'dump_website.json[.gz]'")
parser.add_argument(
'--compress',
action='store_true',
help='Compress dump data (gzip). Default: False')
def handle(self, *args, **options):
dumpfile = options['dumpfile']
compress = options['compress']
if compress:
if dumpfile.endswith('.gz'):
dstfile = dumpfile
dumpfile = dumpfile[:-len('.gz')]
else:
dstfile = dumpfile + '.gz'
else:
dstfile = dumpfile
self.stdout.write("Dumping database to file '{}'...".format(dstfile))
call_command('dumpdata', 'admin', 'auth', 'django_db_logger', 'djcelery', 'sessions',
'sites', 'website', output=dumpfile)
if compress:
with hide("commands"): # pylint: disable=not-context-manager
local("gzip {}".format(dumpfile))
self.stdout.write(self.style.SUCCESS(
"Successfully dumped website to '{}'.".format(dstfile)))

View File

@ -62,10 +62,7 @@ class Command(BaseCommand):
call_command('startcelery')
def handle(self, *args, **options):
dumpfile = options['dumpfile']
self.stdout.write("Dumping database to file '{}'...".format(dumpfile))
call_command('dumpdata', 'admin', 'auth', 'django_db_logger', 'djcelery', 'sessions',
'sites', 'website', output=dumpfile)
call_command('dumpwebsite', dumpfile=options['dumpfile'])
call_command('stopcelery')
self.reset_website()

View File

@ -7,7 +7,7 @@ import os
import time
from django.core.management.base import BaseCommand
from fabric.api import local
from fabric.api import local, quiet, settings
class Command(BaseCommand):
@ -34,9 +34,10 @@ class Command(BaseCommand):
pidfile = options[name + '_pidfile']
with open(pidfile, 'r') as f:
pid = f.read()
with settings(warn_only=True):
local('kill {}'.format(pid))
check_pidfiles.append((name, pidfile))
except Exception as e:
except Exception as e: # pylint: disable=broad-except
self.stdout.write(self.style.NOTICE(
"ERROR: an exception occurred while stopping '{}':\n{}\n".format(name, e)))
@ -54,3 +55,6 @@ class Command(BaseCommand):
else:
self.stdout.write(self.style.SUCCESS(
"Successfully stopped '{}'.".format(name)))
with quiet():
local('rm -f {} {}'.format(options['celery_pidfile'], options['celerybeat_pidfile']))

View File

@ -6,8 +6,9 @@
# pylint: disable=too-many-lines
import logging
import datetime
import os
import re
import time
import shutil
from collections import OrderedDict
from django.contrib.auth import authenticate, login, logout
@ -16,7 +17,8 @@ from django.contrib.auth import update_session_auth_hash
from django.contrib.auth.forms import AuthenticationForm, UserCreationForm
from django.contrib.auth.forms import PasswordChangeForm
from django.core.exceptions import ObjectDoesNotExist
from django.core.files.base import ContentFile
from django.core.files.base import ContentFile, File
from django.core.management import call_command
from django.db.utils import IntegrityError
from django.forms.models import model_to_dict
from django.http import HttpResponse, QueryDict
@ -38,8 +40,8 @@ from .tasks import (aggregate_target_results, map_workload, train_ddpg,
configuration_recommendation, configuration_recommendation_ddpg)
from .types import (DBMSType, KnobUnitType, MetricType,
TaskType, VarType, WorkloadStatusType, AlgorithmType)
from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil, ConversionUtil
from .settings import TIME_ZONE
from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil
from .settings import LOG_DIR, TIME_ZONE
from .set_default_knobs import set_default_knobs
@ -729,6 +731,7 @@ def knob_data_view(request, project_id, session_id, data_id): # pylint: disable
target_obj = JSONUtil.loads(result.metric_data.data)[session.target_objective]
return dbms_data_view(request, context, knob_data, session, target_obj)
@login_required(login_url=reverse_lazy('login'))
def metric_data_view(request, project_id, session_id, data_id): # pylint: disable=unused-argument
metric_data = get_object_or_404(MetricData, pk=data_id)
@ -894,8 +897,7 @@ def tuner_status_view(request, project_id, session_id, result_id): # pylint: di
total_runtime = (completion_time - res.creation_time).total_seconds()
total_runtime = '{0:.2f} seconds'.format(total_runtime)
task_info = [(tname, task) for tname, task in
zip(list(TaskType.TYPE_NAMES.values()), tasks)]
task_info = list(zip(TaskType.TYPE_NAMES.values(), tasks))
context = {"id": result_id,
"result": res,
@ -1175,6 +1177,31 @@ def alt_get_info(request, name): # pylint: disable=unused-argument
if name == 'constants':
info = utils.get_constants()
response = HttpResponse(JSONUtil.dumps(info))
elif name in ('website', 'logs'):
tmpdir = os.path.realpath('.info')
shutil.rmtree(tmpdir, ignore_errors=True)
os.makedirs(tmpdir, exist_ok=True)
if name == 'website':
filepath = os.path.join(tmpdir, 'website_dump.json.gz')
call_command('dumpwebsite', dumpfile=filepath, compress=True)
else: # name == 'logs'
base_name = os.path.join(tmpdir, 'website_logs')
root_dir, base_dir = os.path.split(LOG_DIR)
filepath = shutil.make_archive(
base_name, format='gztar', base_dir=base_dir, root_dir=root_dir)
f = open(filepath, 'rb')
try:
cfile = File(f)
response = HttpResponse(cfile, content_type='application/x-gzip')
response['Content-Length'] = cfile.size
response['Content-Disposition'] = 'attachment; filename={}'.format(
os.path.basename(filepath))
finally:
f.close()
shutil.rmtree(tmpdir, ignore_errors=True)
else:
LOG.warning("Invalid name for info request: %s", name)
response = HttpResponse("Invalid name for info request: {}".format(name), status=400)