From b3c42a81fb2cf05107305bd58ef6fb615e64e3db Mon Sep 17 00:00:00 2001 From: Dana Van Aken Date: Tue, 1 Oct 2019 20:17:23 -0400 Subject: [PATCH] Added django-db-logger for storing log messages in the database, removed random *.sh scripts from website code then extended manage.py with same commands --- client/controller/.gitignore | 5 + server/website/.gitignore | 5 +- server/website/config/.gitignore | 2 - server/website/config/postgresql.conf | 33 ----- .../website/{ => script/management}/beat.sh | 0 .../website/{ => script/management}/celery.sh | 0 .../website/{ => script/management}/django.sh | 0 .../{ => script/management}/fabfile.py | 0 server/website/website/admin.py | 97 +++++++++++---- .../website/management/commands/cleardblog.py | 16 +++ .../management/commands/startcelery.py | 116 ++++++++++++++++++ .../website/management/commands/stopcelery.py | 56 +++++++++ server/website/website/settings/common.py | 21 +++- 13 files changed, 286 insertions(+), 65 deletions(-) delete mode 100644 server/website/config/.gitignore delete mode 100644 server/website/config/postgresql.conf rename server/website/{ => script/management}/beat.sh (100%) rename server/website/{ => script/management}/celery.sh (100%) rename server/website/{ => script/management}/django.sh (100%) rename server/website/{ => script/management}/fabfile.py (100%) create mode 100644 server/website/website/management/commands/cleardblog.py create mode 100644 server/website/website/management/commands/startcelery.py create mode 100644 server/website/website/management/commands/stopcelery.py diff --git a/client/controller/.gitignore b/client/controller/.gitignore index 8f3a7e5..5b86e7d 100644 --- a/client/controller/.gitignore +++ b/client/controller/.gitignore @@ -26,3 +26,8 @@ lib/ # log file *.log + +# controller configuration files +config/* +!config/sample_*_config.json + diff --git a/server/website/.gitignore b/server/website/.gitignore index bcaf137..3399987 100644 --- a/server/website/.gitignore +++ b/server/website/.gitignore @@ -4,9 +4,10 @@ log/ *.log local_settings.py -# celery beat schedule file # -############################# +# celery/celerybeat # +##################### celerybeat-schedule +*.pid # Raw data files # ################## diff --git a/server/website/config/.gitignore b/server/website/config/.gitignore deleted file mode 100644 index 16443c5..0000000 --- a/server/website/config/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.bak -prod_supervisord.conf diff --git a/server/website/config/postgresql.conf b/server/website/config/postgresql.conf deleted file mode 100644 index 422b34f..0000000 --- a/server/website/config/postgresql.conf +++ /dev/null @@ -1,33 +0,0 @@ -# ----------------------------- -# PostgreSQL configuration file -# ----------------------------- -# -# This file consists of lines of the form: -# -# name = value -# -# (The "=" is optional.) Whitespace may be used. Comments are introduced with -# "#" anywhere on a line. The complete list of parameter names and allowed -# values can be found in the PostgreSQL documentation. -# -# The commented-out settings shown in this file represent the default values. -# Re-commenting a setting is NOT sufficient to revert it to the default value; -# you need to reload the server. -# -# This file is read on server startup and when the server receives a SIGHUP -# signal. If you edit the file on a running system, you have to SIGHUP the -# server for the changes to take effect, or use "pg_ctl reload". Some -# parameters, which are marked below, require a server shutdown and restart to -# take effect. -# -# Any parameter can also be given as a command-line option to the server, e.g., -# "postgres -c log_connections=on". Some parameters can be changed at run time -# with the "SET" SQL command. -# -# Memory units: kB = kilobytes Time units: ms = milliseconds -# MB = megabytes s = seconds -# GB = gigabytes min = minutes -# h = hours -# d = days - - diff --git a/server/website/beat.sh b/server/website/script/management/beat.sh similarity index 100% rename from server/website/beat.sh rename to server/website/script/management/beat.sh diff --git a/server/website/celery.sh b/server/website/script/management/celery.sh similarity index 100% rename from server/website/celery.sh rename to server/website/script/management/celery.sh diff --git a/server/website/django.sh b/server/website/script/management/django.sh similarity index 100% rename from server/website/django.sh rename to server/website/script/management/django.sh diff --git a/server/website/fabfile.py b/server/website/script/management/fabfile.py similarity index 100% rename from server/website/fabfile.py rename to server/website/script/management/fabfile.py diff --git a/server/website/website/admin.py b/server/website/website/admin.py index 90715ba..54cb093 100644 --- a/server/website/website/admin.py +++ b/server/website/website/admin.py @@ -4,7 +4,10 @@ # Copyright (c) 2017-18, Carnegie Mellon University Database Group # from django.contrib import admin -from djcelery.models import TaskMeta +from django.utils.html import format_html +from django_db_logger.admin import StatusLogAdmin +from django_db_logger.models import StatusLog +from djcelery import models as djcelery_models from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog, MetricData, @@ -45,12 +48,12 @@ class SessionAdmin(admin.ModelAdmin): class SessionKnobAdmin(admin.ModelAdmin): list_display = ('knob', 'dbms', 'session', 'minval', 'maxval', 'tunable') list_filter = (('session__dbms', admin.RelatedOnlyFieldListFilter), - ('session', admin.RelatedOnlyFieldListFilter), ('tunable')) + ('session', admin.RelatedOnlyFieldListFilter), + ('tunable', admin.FieldListFilter)) ordering = ('session__dbms', 'session__name', '-tunable', 'knob__name') - @staticmethod - def dbms(obj): - return obj.session.dbms + def dbms(self, instance): # pylint: disable=no-self-use + return instance.session.dbms class HardwareAdmin(admin.ModelAdmin): @@ -71,22 +74,6 @@ class MetricDataAdmin(admin.ModelAdmin): ordering = ('creation_time',) -class TaskMetaAdmin(admin.ModelAdmin): - list_display = ('id', 'status', 'task_result', 'date_done') - readonly_fields = ('id', 'task_id', 'status', 'result', 'date_done', - 'traceback', 'hidden', 'meta') - fields = readonly_fields - list_filter = ('status',) - ordering = ('date_done',) - - @staticmethod - def task_result(obj, maxlen=300): - res = obj.result - if res and len(res) > maxlen: - res = res[:maxlen] + '...' - return res - - class ResultAdmin(admin.ModelAdmin): readonly_fields = ('dbms', 'knob_data', 'metric_data', 'session', 'workload') list_display = ('id', 'dbms', 'session', 'workload', 'creation_time') @@ -120,6 +107,55 @@ class WorkloadAdmin(admin.ModelAdmin): ('hardware', admin.RelatedOnlyFieldListFilter)) +class TaskMetaAdmin(admin.ModelAdmin): + list_display = ('colored_status', 'task_result', 'date_done', 'task_traceback') + list_display_links = ('colored_status', 'task_result') + readonly_fields = ('id', 'task_id', 'status', 'result', 'date_done', + 'traceback', 'hidden', 'meta') + fields = readonly_fields + list_filter = ('status',) + list_per_page = 10 + ordering = ('date_done',) + max_field_length = 1000 + + @staticmethod + def color_field(text, status): + if status == 'SUCCESS': + color = 'green' + elif status in ('PENDING', 'RECEIVED', 'STARTED'): + color = 'orange' + else: + color = 'red' + return format_html('{}'.format(color, text)) + + def format_field(self, field): + text = str(field) if field else '' + if len(text) > self.max_field_length: + text = text[:self.max_field_length] + '...' + return text + + def colored_status(self, instance): + return self.color_field(instance.status, instance.status) + colored_status.short_description = 'Status' + + def task_traceback(self, instance): + text = self.format_field(instance.traceback) + return format_html('
{}
'.format(text)) + task_traceback.short_description = 'Traceback' + + def task_result(self, instance): + res = self.format_field(instance.result) + return self.color_field(res, instance.status) + task_result.short_description = 'Result' + + +class CustomStatusLogAdmin(StatusLogAdmin): + list_display = ('logger_name', 'colored_msg', 'traceback', 'create_datetime_format') + list_display_links = ('logger_name',) + list_filter = ('logger_name', 'level') + + +# Admin classes for website models admin.site.register(DBMSCatalog, DBMSCatalogAdmin) admin.site.register(KnobCatalog, KnobCatalogAdmin) admin.site.register(MetricCatalog, MetricCatalogAdmin) @@ -127,7 +163,6 @@ admin.site.register(Session, SessionAdmin) admin.site.register(Project, ProjectAdmin) admin.site.register(KnobData, KnobDataAdmin) admin.site.register(MetricData, MetricDataAdmin) -admin.site.register(TaskMeta, TaskMetaAdmin) admin.site.register(Result, ResultAdmin) admin.site.register(BackupData, BackupDataAdmin) admin.site.register(PipelineData, PipelineDataAdmin) @@ -135,3 +170,21 @@ admin.site.register(PipelineRun, PipelineRunAdmin) admin.site.register(Workload, WorkloadAdmin) admin.site.register(SessionKnob, SessionKnobAdmin) admin.site.register(Hardware, HardwareAdmin) + +# Admin classes for 3rd party models +admin.site.unregister(StatusLog) +admin.site.register(StatusLog, CustomStatusLogAdmin) +admin.site.register(djcelery_models.TaskMeta, TaskMetaAdmin) + +# Unregister empty djcelery models +UNUSED_DJCELERY_MODELS = ( + djcelery_models.CrontabSchedule, + djcelery_models.IntervalSchedule, + djcelery_models.PeriodicTask, + djcelery_models.TaskState, + djcelery_models.WorkerState, +) + +for model in UNUSED_DJCELERY_MODELS: + if model.objects.count() == 0: + admin.site.unregister(model) diff --git a/server/website/website/management/commands/cleardblog.py b/server/website/website/management/commands/cleardblog.py new file mode 100644 index 0000000..182d61d --- /dev/null +++ b/server/website/website/management/commands/cleardblog.py @@ -0,0 +1,16 @@ +# +# OtterTune - cleardblog.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# +from django.core.management.base import BaseCommand +from django_db_logger.models import StatusLog + + +class Command(BaseCommand): + help = 'Clear all log entries from the django_db_logger table.' + + def handle(self, *args, **options): + StatusLog.objects.all().delete() + self.stdout.write(self.style.SUCCESS( + "Successfully cleared the django_db_logger table.")) diff --git a/server/website/website/management/commands/startcelery.py b/server/website/website/management/commands/startcelery.py new file mode 100644 index 0000000..c2a2ced --- /dev/null +++ b/server/website/website/management/commands/startcelery.py @@ -0,0 +1,116 @@ +# +# OtterTune - startcelery.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# +import os +import time + +from django.conf import settings +from django.core.management.base import BaseCommand +from fabric.api import hide, lcd, local + + +class Command(BaseCommand): + help = 'Start celery and celerybeat in the background.' + celery_cmd = 'python3 manage.py {cmd} {opts} {pipe} &'.format + max_wait_sec = 15 + + def add_arguments(self, parser): + parser.add_argument( + '--celery-only', + action='store_true', + help='Start celery only (skip celerybeat).') + parser.add_argument( + '--celerybeat-only', + action='store_true', + help='Start celerybeat only (skip celery).') + parser.add_argument( + '--loglevel', + metavar='LOGLEVEL', + help='Logging level, choose between DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL. ' + 'Defaults to DEBUG if settings.DEBUG is true, otherwise INFO.') + parser.add_argument( + '--pool', + metavar='POOL_CLS', + default='threads', + help='Pool implementation: prefork (default), eventlet, gevent, solo or threads.' + 'Default: threads.') + parser.add_argument( + '--celery-pidfile', + metavar='PIDFILE', + default='celery.pid', + help='File used to store the process pid. The program will not start if this ' + 'file already exists and the pid is still alive. Default: celery.pid.') + parser.add_argument( + '--celerybeat-pidfile', + metavar='PIDFILE', + default='celerybeat.pid', + help='File used to store the process pid. The program will not start if this ' + 'file already exists and the pid is still alive. Default: celerybeat.pid.') + parser.add_argument( + '--celery-options', + metavar='OPTIONS', + help="A comma-separated list of additional options to pass to celery, " + "see 'python manage.py celery worker --help' for all available options. " + "IMPORTANT: the option's initial -/-- must be omitted. " + "Example: '--celery-options purge,include=foo.tasks,q'.") + parser.add_argument( + '--celerybeat-options', + metavar='OPTIONS', + help="A comma-separated list of additional options to pass to celerybeat, " + "see 'python manage.py celerybeat --help' for all available options. " + "IMPORTANT: the option's initial -/-- must be omitted. " + "Example: '--celerybeat-options uid=123,q'.") + + def _parse_suboptions(self, suboptions): + suboptions = suboptions or '' + parsed = [] + for opt in suboptions.split(','): + if opt: + opt = ('-{}' if len(opt) == 1 else '--{}').format(opt) + parsed.append(opt) + return parsed + + def handle(self, *args, **options): + loglevel = options['loglevel'] or ('DEBUG' if settings.DEBUG else 'INFO') + celery_options = [ + '--loglevel={}'.format(loglevel), + '--pool={}'.format(options['pool']), + '--pidfile={}'.format(options['celery_pidfile']), + ] + self._parse_suboptions(options['celery_options']) + celerybeat_options = [ + '--loglevel={}'.format(loglevel), + '--pidfile={}'.format(options['celerybeat_pidfile']), + ] + self._parse_suboptions(options['celerybeat_options']) + + pipe = '' if 'console' in settings.LOGGING['loggers']['celery']['handlers'] \ + else '> /dev/null 2>&1' + + with lcd(settings.PROJECT_ROOT), hide('command'): + if not options['celerybeat_only']: + local(self.celery_cmd( + cmd='celery worker', opts=' '.join(celery_options), pipe=pipe)) + + if not options['celery_only']: + local(self.celery_cmd( + cmd='celerybeat', opts=' '.join(celerybeat_options), pipe=pipe)) + + pidfiles = [options['celery_pidfile'], options['celerybeat_pidfile']] + wait_sec = 0 + + while wait_sec < self.max_wait_sec and len(pidfiles) > 0: + time.sleep(1) + wait_sec += 1 + + for i in range(len(pidfiles)): + if os.path.exists(pidfiles[i]): + pidfiles.pop(i) + + for name in ('celery', 'celerybeat'): + if os.path.exists(options[name + '_pidfile']): + self.stdout.write(self.style.SUCCESS( + "Successfully started '{}'.".format(name))) + else: + self.stdout.write(self.style.NOTICE( + "Failed to start '{}'.".format(name))) diff --git a/server/website/website/management/commands/stopcelery.py b/server/website/website/management/commands/stopcelery.py new file mode 100644 index 0000000..0eef50f --- /dev/null +++ b/server/website/website/management/commands/stopcelery.py @@ -0,0 +1,56 @@ +# +# OtterTune - stopcelery.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# +import os +import time + +from django.core.management.base import BaseCommand +from fabric.api import local + + +class Command(BaseCommand): + help = 'Start celery and celerybeat in the background.' + celery_cmd = 'python3 manage.py {cmd} {opts} &'.format + max_wait_sec = 15 + + def add_arguments(self, parser): + parser.add_argument( + '--celery-pidfile', + metavar='PIDFILE', + default='celery.pid', + help="Alternate path to the process' pid file if not located at ./celery.pid.") + parser.add_argument( + '--celerybeat-pidfile', + metavar='PIDFILE', + default='celerybeat.pid', + help="Alternate path to the process' pid file if not located at ./celerybeat.pid.") + + def handle(self, *args, **options): + check_pidfiles = [] + for name in ('celery', 'celerybeat'): + try: + pidfile = options[name + '_pidfile'] + with open(pidfile, 'r') as f: + pid = f.read() + local('kill {}'.format(pid)) + check_pidfiles.append((name, pidfile)) + except Exception as e: + self.stdout.write(self.style.NOTICE( + "ERROR: an exception occurred while stopping '{}':\n{}\n".format(name, e))) + + if check_pidfiles: + self.stdout.write("Waiting for processes to shutdown...\n") + for name, pidfile in check_pidfiles: + wait_sec = 0 + while os.path.exists(pidfile) and wait_sec < self.max_wait_sec: + time.sleep(1) + wait_sec += 1 + if os.path.exists(pidfile): + self.stdout.write(self.style.NOTICE( + "WARNING: file '{}' still exists after stopping {}.".format( + pidfile, name))) + else: + self.stdout.write(self.style.SUCCESS( + "Successfully stopped '{}'.".format(name))) diff --git a/server/website/website/settings/common.py b/server/website/website/settings/common.py index d24688b..cd0feb0 100644 --- a/server/website/website/settings/common.py +++ b/server/website/website/settings/common.py @@ -194,6 +194,7 @@ INSTALLED_APPS = ( 'djcelery', # 'django_celery_beat', 'website', + 'django_db_logger', ) # ============================================== @@ -253,6 +254,14 @@ LOGGING = { 'backupCount': 2, 'formatter': 'standard', }, + 'dblog': { + 'level': 'DEBUG', + 'class': 'django_db_logger.db_log_handler.DatabaseLogHandler', + }, + 'dblog_warn': { + 'level': 'WARN', + 'class': 'django_db_logger.db_log_handler.DatabaseLogHandler', + }, 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', @@ -271,26 +280,26 @@ LOGGING = { }, 'loggers': { 'django': { - 'handlers': ['console', 'logfile'], + 'handlers': ['console', 'logfile', 'dblog_warn'], 'propagate': True, 'level': 'WARN', }, 'django.db.backends': { - 'handlers': ['console', 'logfile'], + 'handlers': ['console', 'logfile', 'dblog_warn'], 'level': 'WARN', 'propagate': False, }, 'website': { - 'handlers': ['console', 'logfile'], + 'handlers': ['console', 'logfile', 'dblog'], 'level': 'DEBUG', }, 'django.request': { - 'handlers': ['console'], - 'level': 'DEBUG', + 'handlers': ['console', 'dblog_warn'], + 'level': 'INFO', 'propagate': False, }, 'celery': { - 'handlers': ['console', 'celery'], + 'handlers': ['celery', 'dblog'], 'level': 'DEBUG', 'propogate': True, },