Added django-db-logger for storing log messages in the database, removed random *.sh scripts from website code then extended manage.py with same commands

This commit is contained in:
Dana Van Aken 2019-10-01 20:17:23 -04:00
parent 6b8aeb6043
commit b3c42a81fb
13 changed files with 286 additions and 65 deletions

View File

@ -26,3 +26,8 @@ lib/
# log file # log file
*.log *.log
# controller configuration files
config/*
!config/sample_*_config.json

View File

@ -4,9 +4,10 @@ log/
*.log *.log
local_settings.py local_settings.py
# celery beat schedule file # # celery/celerybeat #
############################# #####################
celerybeat-schedule celerybeat-schedule
*.pid
# Raw data files # # Raw data files #
################## ##################

View File

@ -1,2 +0,0 @@
*.bak
prod_supervisord.conf

View File

@ -1,33 +0,0 @@
# -----------------------------
# PostgreSQL configuration file
# -----------------------------
#
# This file consists of lines of the form:
#
# name = value
#
# (The "=" is optional.) Whitespace may be used. Comments are introduced with
# "#" anywhere on a line. The complete list of parameter names and allowed
# values can be found in the PostgreSQL documentation.
#
# The commented-out settings shown in this file represent the default values.
# Re-commenting a setting is NOT sufficient to revert it to the default value;
# you need to reload the server.
#
# This file is read on server startup and when the server receives a SIGHUP
# signal. If you edit the file on a running system, you have to SIGHUP the
# server for the changes to take effect, or use "pg_ctl reload". Some
# parameters, which are marked below, require a server shutdown and restart to
# take effect.
#
# Any parameter can also be given as a command-line option to the server, e.g.,
# "postgres -c log_connections=on". Some parameters can be changed at run time
# with the "SET" SQL command.
#
# Memory units: kB = kilobytes Time units: ms = milliseconds
# MB = megabytes s = seconds
# GB = gigabytes min = minutes
# h = hours
# d = days

View File

@ -4,7 +4,10 @@
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
from django.contrib import admin from django.contrib import admin
from djcelery.models import TaskMeta from django.utils.html import format_html
from django_db_logger.admin import StatusLogAdmin
from django_db_logger.models import StatusLog
from djcelery import models as djcelery_models
from .models import (BackupData, DBMSCatalog, KnobCatalog, from .models import (BackupData, DBMSCatalog, KnobCatalog,
KnobData, MetricCatalog, MetricData, KnobData, MetricCatalog, MetricData,
@ -45,12 +48,12 @@ class SessionAdmin(admin.ModelAdmin):
class SessionKnobAdmin(admin.ModelAdmin): class SessionKnobAdmin(admin.ModelAdmin):
list_display = ('knob', 'dbms', 'session', 'minval', 'maxval', 'tunable') list_display = ('knob', 'dbms', 'session', 'minval', 'maxval', 'tunable')
list_filter = (('session__dbms', admin.RelatedOnlyFieldListFilter), list_filter = (('session__dbms', admin.RelatedOnlyFieldListFilter),
('session', admin.RelatedOnlyFieldListFilter), ('tunable')) ('session', admin.RelatedOnlyFieldListFilter),
('tunable', admin.FieldListFilter))
ordering = ('session__dbms', 'session__name', '-tunable', 'knob__name') ordering = ('session__dbms', 'session__name', '-tunable', 'knob__name')
@staticmethod def dbms(self, instance): # pylint: disable=no-self-use
def dbms(obj): return instance.session.dbms
return obj.session.dbms
class HardwareAdmin(admin.ModelAdmin): class HardwareAdmin(admin.ModelAdmin):
@ -71,22 +74,6 @@ class MetricDataAdmin(admin.ModelAdmin):
ordering = ('creation_time',) ordering = ('creation_time',)
class TaskMetaAdmin(admin.ModelAdmin):
list_display = ('id', 'status', 'task_result', 'date_done')
readonly_fields = ('id', 'task_id', 'status', 'result', 'date_done',
'traceback', 'hidden', 'meta')
fields = readonly_fields
list_filter = ('status',)
ordering = ('date_done',)
@staticmethod
def task_result(obj, maxlen=300):
res = obj.result
if res and len(res) > maxlen:
res = res[:maxlen] + '...'
return res
class ResultAdmin(admin.ModelAdmin): class ResultAdmin(admin.ModelAdmin):
readonly_fields = ('dbms', 'knob_data', 'metric_data', 'session', 'workload') readonly_fields = ('dbms', 'knob_data', 'metric_data', 'session', 'workload')
list_display = ('id', 'dbms', 'session', 'workload', 'creation_time') list_display = ('id', 'dbms', 'session', 'workload', 'creation_time')
@ -120,6 +107,55 @@ class WorkloadAdmin(admin.ModelAdmin):
('hardware', admin.RelatedOnlyFieldListFilter)) ('hardware', admin.RelatedOnlyFieldListFilter))
class TaskMetaAdmin(admin.ModelAdmin):
list_display = ('colored_status', 'task_result', 'date_done', 'task_traceback')
list_display_links = ('colored_status', 'task_result')
readonly_fields = ('id', 'task_id', 'status', 'result', 'date_done',
'traceback', 'hidden', 'meta')
fields = readonly_fields
list_filter = ('status',)
list_per_page = 10
ordering = ('date_done',)
max_field_length = 1000
@staticmethod
def color_field(text, status):
if status == 'SUCCESS':
color = 'green'
elif status in ('PENDING', 'RECEIVED', 'STARTED'):
color = 'orange'
else:
color = 'red'
return format_html('<span style="color: {};">{}</span>'.format(color, text))
def format_field(self, field):
text = str(field) if field else ''
if len(text) > self.max_field_length:
text = text[:self.max_field_length] + '...'
return text
def colored_status(self, instance):
return self.color_field(instance.status, instance.status)
colored_status.short_description = 'Status'
def task_traceback(self, instance):
text = self.format_field(instance.traceback)
return format_html('<pre><code>{}</code></pre>'.format(text))
task_traceback.short_description = 'Traceback'
def task_result(self, instance):
res = self.format_field(instance.result)
return self.color_field(res, instance.status)
task_result.short_description = 'Result'
class CustomStatusLogAdmin(StatusLogAdmin):
list_display = ('logger_name', 'colored_msg', 'traceback', 'create_datetime_format')
list_display_links = ('logger_name',)
list_filter = ('logger_name', 'level')
# Admin classes for website models
admin.site.register(DBMSCatalog, DBMSCatalogAdmin) admin.site.register(DBMSCatalog, DBMSCatalogAdmin)
admin.site.register(KnobCatalog, KnobCatalogAdmin) admin.site.register(KnobCatalog, KnobCatalogAdmin)
admin.site.register(MetricCatalog, MetricCatalogAdmin) admin.site.register(MetricCatalog, MetricCatalogAdmin)
@ -127,7 +163,6 @@ admin.site.register(Session, SessionAdmin)
admin.site.register(Project, ProjectAdmin) admin.site.register(Project, ProjectAdmin)
admin.site.register(KnobData, KnobDataAdmin) admin.site.register(KnobData, KnobDataAdmin)
admin.site.register(MetricData, MetricDataAdmin) admin.site.register(MetricData, MetricDataAdmin)
admin.site.register(TaskMeta, TaskMetaAdmin)
admin.site.register(Result, ResultAdmin) admin.site.register(Result, ResultAdmin)
admin.site.register(BackupData, BackupDataAdmin) admin.site.register(BackupData, BackupDataAdmin)
admin.site.register(PipelineData, PipelineDataAdmin) admin.site.register(PipelineData, PipelineDataAdmin)
@ -135,3 +170,21 @@ admin.site.register(PipelineRun, PipelineRunAdmin)
admin.site.register(Workload, WorkloadAdmin) admin.site.register(Workload, WorkloadAdmin)
admin.site.register(SessionKnob, SessionKnobAdmin) admin.site.register(SessionKnob, SessionKnobAdmin)
admin.site.register(Hardware, HardwareAdmin) admin.site.register(Hardware, HardwareAdmin)
# Admin classes for 3rd party models
admin.site.unregister(StatusLog)
admin.site.register(StatusLog, CustomStatusLogAdmin)
admin.site.register(djcelery_models.TaskMeta, TaskMetaAdmin)
# Unregister empty djcelery models
UNUSED_DJCELERY_MODELS = (
djcelery_models.CrontabSchedule,
djcelery_models.IntervalSchedule,
djcelery_models.PeriodicTask,
djcelery_models.TaskState,
djcelery_models.WorkerState,
)
for model in UNUSED_DJCELERY_MODELS:
if model.objects.count() == 0:
admin.site.unregister(model)

View File

@ -0,0 +1,16 @@
#
# OtterTune - cleardblog.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
from django.core.management.base import BaseCommand
from django_db_logger.models import StatusLog
class Command(BaseCommand):
help = 'Clear all log entries from the django_db_logger table.'
def handle(self, *args, **options):
StatusLog.objects.all().delete()
self.stdout.write(self.style.SUCCESS(
"Successfully cleared the django_db_logger table."))

View File

@ -0,0 +1,116 @@
#
# OtterTune - startcelery.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
import os
import time
from django.conf import settings
from django.core.management.base import BaseCommand
from fabric.api import hide, lcd, local
class Command(BaseCommand):
help = 'Start celery and celerybeat in the background.'
celery_cmd = 'python3 manage.py {cmd} {opts} {pipe} &'.format
max_wait_sec = 15
def add_arguments(self, parser):
parser.add_argument(
'--celery-only',
action='store_true',
help='Start celery only (skip celerybeat).')
parser.add_argument(
'--celerybeat-only',
action='store_true',
help='Start celerybeat only (skip celery).')
parser.add_argument(
'--loglevel',
metavar='LOGLEVEL',
help='Logging level, choose between DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL. '
'Defaults to DEBUG if settings.DEBUG is true, otherwise INFO.')
parser.add_argument(
'--pool',
metavar='POOL_CLS',
default='threads',
help='Pool implementation: prefork (default), eventlet, gevent, solo or threads.'
'Default: threads.')
parser.add_argument(
'--celery-pidfile',
metavar='PIDFILE',
default='celery.pid',
help='File used to store the process pid. The program will not start if this '
'file already exists and the pid is still alive. Default: celery.pid.')
parser.add_argument(
'--celerybeat-pidfile',
metavar='PIDFILE',
default='celerybeat.pid',
help='File used to store the process pid. The program will not start if this '
'file already exists and the pid is still alive. Default: celerybeat.pid.')
parser.add_argument(
'--celery-options',
metavar='OPTIONS',
help="A comma-separated list of additional options to pass to celery, "
"see 'python manage.py celery worker --help' for all available options. "
"IMPORTANT: the option's initial -/-- must be omitted. "
"Example: '--celery-options purge,include=foo.tasks,q'.")
parser.add_argument(
'--celerybeat-options',
metavar='OPTIONS',
help="A comma-separated list of additional options to pass to celerybeat, "
"see 'python manage.py celerybeat --help' for all available options. "
"IMPORTANT: the option's initial -/-- must be omitted. "
"Example: '--celerybeat-options uid=123,q'.")
def _parse_suboptions(self, suboptions):
suboptions = suboptions or ''
parsed = []
for opt in suboptions.split(','):
if opt:
opt = ('-{}' if len(opt) == 1 else '--{}').format(opt)
parsed.append(opt)
return parsed
def handle(self, *args, **options):
loglevel = options['loglevel'] or ('DEBUG' if settings.DEBUG else 'INFO')
celery_options = [
'--loglevel={}'.format(loglevel),
'--pool={}'.format(options['pool']),
'--pidfile={}'.format(options['celery_pidfile']),
] + self._parse_suboptions(options['celery_options'])
celerybeat_options = [
'--loglevel={}'.format(loglevel),
'--pidfile={}'.format(options['celerybeat_pidfile']),
] + self._parse_suboptions(options['celerybeat_options'])
pipe = '' if 'console' in settings.LOGGING['loggers']['celery']['handlers'] \
else '> /dev/null 2>&1'
with lcd(settings.PROJECT_ROOT), hide('command'):
if not options['celerybeat_only']:
local(self.celery_cmd(
cmd='celery worker', opts=' '.join(celery_options), pipe=pipe))
if not options['celery_only']:
local(self.celery_cmd(
cmd='celerybeat', opts=' '.join(celerybeat_options), pipe=pipe))
pidfiles = [options['celery_pidfile'], options['celerybeat_pidfile']]
wait_sec = 0
while wait_sec < self.max_wait_sec and len(pidfiles) > 0:
time.sleep(1)
wait_sec += 1
for i in range(len(pidfiles)):
if os.path.exists(pidfiles[i]):
pidfiles.pop(i)
for name in ('celery', 'celerybeat'):
if os.path.exists(options[name + '_pidfile']):
self.stdout.write(self.style.SUCCESS(
"Successfully started '{}'.".format(name)))
else:
self.stdout.write(self.style.NOTICE(
"Failed to start '{}'.".format(name)))

View File

@ -0,0 +1,56 @@
#
# OtterTune - stopcelery.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
import os
import time
from django.core.management.base import BaseCommand
from fabric.api import local
class Command(BaseCommand):
help = 'Start celery and celerybeat in the background.'
celery_cmd = 'python3 manage.py {cmd} {opts} &'.format
max_wait_sec = 15
def add_arguments(self, parser):
parser.add_argument(
'--celery-pidfile',
metavar='PIDFILE',
default='celery.pid',
help="Alternate path to the process' pid file if not located at ./celery.pid.")
parser.add_argument(
'--celerybeat-pidfile',
metavar='PIDFILE',
default='celerybeat.pid',
help="Alternate path to the process' pid file if not located at ./celerybeat.pid.")
def handle(self, *args, **options):
check_pidfiles = []
for name in ('celery', 'celerybeat'):
try:
pidfile = options[name + '_pidfile']
with open(pidfile, 'r') as f:
pid = f.read()
local('kill {}'.format(pid))
check_pidfiles.append((name, pidfile))
except Exception as e:
self.stdout.write(self.style.NOTICE(
"ERROR: an exception occurred while stopping '{}':\n{}\n".format(name, e)))
if check_pidfiles:
self.stdout.write("Waiting for processes to shutdown...\n")
for name, pidfile in check_pidfiles:
wait_sec = 0
while os.path.exists(pidfile) and wait_sec < self.max_wait_sec:
time.sleep(1)
wait_sec += 1
if os.path.exists(pidfile):
self.stdout.write(self.style.NOTICE(
"WARNING: file '{}' still exists after stopping {}.".format(
pidfile, name)))
else:
self.stdout.write(self.style.SUCCESS(
"Successfully stopped '{}'.".format(name)))

View File

@ -194,6 +194,7 @@ INSTALLED_APPS = (
'djcelery', 'djcelery',
# 'django_celery_beat', # 'django_celery_beat',
'website', 'website',
'django_db_logger',
) )
# ============================================== # ==============================================
@ -253,6 +254,14 @@ LOGGING = {
'backupCount': 2, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'dblog': {
'level': 'DEBUG',
'class': 'django_db_logger.db_log_handler.DatabaseLogHandler',
},
'dblog_warn': {
'level': 'WARN',
'class': 'django_db_logger.db_log_handler.DatabaseLogHandler',
},
'console': { 'console': {
'level': 'DEBUG', 'level': 'DEBUG',
'class': 'logging.StreamHandler', 'class': 'logging.StreamHandler',
@ -271,26 +280,26 @@ LOGGING = {
}, },
'loggers': { 'loggers': {
'django': { 'django': {
'handlers': ['console', 'logfile'], 'handlers': ['console', 'logfile', 'dblog_warn'],
'propagate': True, 'propagate': True,
'level': 'WARN', 'level': 'WARN',
}, },
'django.db.backends': { 'django.db.backends': {
'handlers': ['console', 'logfile'], 'handlers': ['console', 'logfile', 'dblog_warn'],
'level': 'WARN', 'level': 'WARN',
'propagate': False, 'propagate': False,
}, },
'website': { 'website': {
'handlers': ['console', 'logfile'], 'handlers': ['console', 'logfile', 'dblog'],
'level': 'DEBUG', 'level': 'DEBUG',
}, },
'django.request': { 'django.request': {
'handlers': ['console'], 'handlers': ['console', 'dblog_warn'],
'level': 'DEBUG', 'level': 'INFO',
'propagate': False, 'propagate': False,
}, },
'celery': { 'celery': {
'handlers': ['console', 'celery'], 'handlers': ['celery', 'dblog'],
'level': 'DEBUG', 'level': 'DEBUG',
'propogate': True, 'propogate': True,
}, },