Added django-db-logger for storing log messages in the database, removed random *.sh scripts from website code then extended manage.py with same commands

client/controller/.gitignore

@@ -26,3 +26,8 @@ lib/
 
 # log file
 *.log
+
+# controller configuration files
+config/*
+!config/sample_*_config.json
+

server/website/.gitignore

@@ -4,9 +4,10 @@ log/
 *.log
 local_settings.py
 
-# celery beat schedule file #
+# celery/celerybeat #
-#############################
+#####################
 celerybeat-schedule
+*.pid
 
 # Raw data files #
 ##################

server/website/config/.gitignore

@@ -1,2 +0,0 @@
-*.bak
-prod_supervisord.conf

server/website/config/postgresql.conf

@@ -1,33 +0,0 @@
-# -----------------------------
-# PostgreSQL configuration file
-# -----------------------------
-#
-# This file consists of lines of the form:
-#
-#   name = value
-#
-# (The "=" is optional.)  Whitespace may be used.  Comments are introduced with
-# "#" anywhere on a line.  The complete list of parameter names and allowed
-# values can be found in the PostgreSQL documentation.
-#
-# The commented-out settings shown in this file represent the default values.
-# Re-commenting a setting is NOT sufficient to revert it to the default value;
-# you need to reload the server.
-#
-# This file is read on server startup and when the server receives a SIGHUP
-# signal.  If you edit the file on a running system, you have to SIGHUP the
-# server for the changes to take effect, or use "pg_ctl reload".  Some
-# parameters, which are marked below, require a server shutdown and restart to
-# take effect.
-#
-# Any parameter can also be given as a command-line option to the server, e.g.,
-# "postgres -c log_connections=on".  Some parameters can be changed at run time
-# with the "SET" SQL command.
-#
-# Memory units:  kB = kilobytes        Time units:  ms  = milliseconds
-#                MB = megabytes                     s   = seconds
-#                GB = gigabytes                     min = minutes
-#                                                   h   = hours
-#                                                   d   = days
-
-

server/website/website/admin.py

@@ -4,7 +4,10 @@
 # Copyright (c) 2017-18, Carnegie Mellon University Database Group
 #
 from django.contrib import admin
-from djcelery.models import TaskMeta
+from django.utils.html import format_html
+from django_db_logger.admin import StatusLogAdmin
+from django_db_logger.models import StatusLog
+from djcelery import models as djcelery_models
 
 from .models import (BackupData, DBMSCatalog, KnobCatalog,
                      KnobData, MetricCatalog, MetricData,
@@ -45,12 +48,12 @@ class SessionAdmin(admin.ModelAdmin):
 class SessionKnobAdmin(admin.ModelAdmin):
     list_display = ('knob', 'dbms', 'session', 'minval', 'maxval', 'tunable')
     list_filter = (('session__dbms', admin.RelatedOnlyFieldListFilter),
-                   ('session', admin.RelatedOnlyFieldListFilter), ('tunable'))
+                   ('session', admin.RelatedOnlyFieldListFilter),
+                   ('tunable', admin.FieldListFilter))
     ordering = ('session__dbms', 'session__name', '-tunable', 'knob__name')
 
-    @staticmethod
+    def dbms(self, instance):  # pylint: disable=no-self-use
-    def dbms(obj):
+        return instance.session.dbms
-        return obj.session.dbms
 
 
 class HardwareAdmin(admin.ModelAdmin):
@@ -71,22 +74,6 @@ class MetricDataAdmin(admin.ModelAdmin):
     ordering = ('creation_time',)
 
 
-class TaskMetaAdmin(admin.ModelAdmin):
-    list_display = ('id', 'status', 'task_result', 'date_done')
-    readonly_fields = ('id', 'task_id', 'status', 'result', 'date_done',
-                       'traceback', 'hidden', 'meta')
-    fields = readonly_fields
-    list_filter = ('status',)
-    ordering = ('date_done',)
-
-    @staticmethod
-    def task_result(obj, maxlen=300):
-        res = obj.result
-        if res and len(res) > maxlen:
-            res = res[:maxlen] + '...'
-        return res
-
-
 class ResultAdmin(admin.ModelAdmin):
     readonly_fields = ('dbms', 'knob_data', 'metric_data', 'session', 'workload')
     list_display = ('id', 'dbms', 'session', 'workload', 'creation_time')
@@ -120,6 +107,55 @@ class WorkloadAdmin(admin.ModelAdmin):
                    ('hardware', admin.RelatedOnlyFieldListFilter))
 
 
+class TaskMetaAdmin(admin.ModelAdmin):
+    list_display = ('colored_status', 'task_result', 'date_done', 'task_traceback')
+    list_display_links = ('colored_status', 'task_result')
+    readonly_fields = ('id', 'task_id', 'status', 'result', 'date_done',
+                       'traceback', 'hidden', 'meta')
+    fields = readonly_fields
+    list_filter = ('status',)
+    list_per_page = 10
+    ordering = ('date_done',)
+    max_field_length = 1000
+
+    @staticmethod
+    def color_field(text, status):
+        if status == 'SUCCESS':
+            color = 'green'
+        elif status in ('PENDING', 'RECEIVED', 'STARTED'):
+            color = 'orange'
+        else:
+            color = 'red'
+        return format_html('<span style="color: {};">{}</span>'.format(color, text))
+
+    def format_field(self, field):
+        text = str(field) if field else ''
+        if len(text) > self.max_field_length:
+            text = text[:self.max_field_length] + '...'
+        return text
+
+    def colored_status(self, instance):
+        return self.color_field(instance.status, instance.status)
+    colored_status.short_description = 'Status'
+
+    def task_traceback(self, instance):
+        text = self.format_field(instance.traceback)
+        return format_html('<pre><code>{}</code></pre>'.format(text))
+    task_traceback.short_description = 'Traceback'
+
+    def task_result(self, instance):
+        res = self.format_field(instance.result)
+        return self.color_field(res, instance.status)
+    task_result.short_description = 'Result'
+
+
+class CustomStatusLogAdmin(StatusLogAdmin):
+    list_display = ('logger_name', 'colored_msg', 'traceback', 'create_datetime_format')
+    list_display_links = ('logger_name',)
+    list_filter = ('logger_name', 'level')
+
+
+# Admin classes for website models
 admin.site.register(DBMSCatalog, DBMSCatalogAdmin)
 admin.site.register(KnobCatalog, KnobCatalogAdmin)
 admin.site.register(MetricCatalog, MetricCatalogAdmin)
@@ -127,7 +163,6 @@ admin.site.register(Session, SessionAdmin)
 admin.site.register(Project, ProjectAdmin)
 admin.site.register(KnobData, KnobDataAdmin)
 admin.site.register(MetricData, MetricDataAdmin)
-admin.site.register(TaskMeta, TaskMetaAdmin)
 admin.site.register(Result, ResultAdmin)
 admin.site.register(BackupData, BackupDataAdmin)
 admin.site.register(PipelineData, PipelineDataAdmin)
@@ -135,3 +170,21 @@ admin.site.register(PipelineRun, PipelineRunAdmin)
 admin.site.register(Workload, WorkloadAdmin)
 admin.site.register(SessionKnob, SessionKnobAdmin)
 admin.site.register(Hardware, HardwareAdmin)
+
+# Admin classes for 3rd party models
+admin.site.unregister(StatusLog)
+admin.site.register(StatusLog, CustomStatusLogAdmin)
+admin.site.register(djcelery_models.TaskMeta, TaskMetaAdmin)
+
+# Unregister empty djcelery models
+UNUSED_DJCELERY_MODELS = (
+    djcelery_models.CrontabSchedule,
+    djcelery_models.IntervalSchedule,
+    djcelery_models.PeriodicTask,
+    djcelery_models.TaskState,
+    djcelery_models.WorkerState,
+)
+
+for model in UNUSED_DJCELERY_MODELS:
+    if model.objects.count() == 0:
+        admin.site.unregister(model)

server/website/website/management/commands/cleardblog.py

@@ -0,0 +1,16 @@
+#
+# OtterTune - cleardblog.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+from django.core.management.base import BaseCommand
+from django_db_logger.models import StatusLog
+
+
+class Command(BaseCommand):
+    help = 'Clear all log entries from the django_db_logger table.'
+
+    def handle(self, *args, **options):
+        StatusLog.objects.all().delete()
+        self.stdout.write(self.style.SUCCESS(
+            "Successfully cleared the django_db_logger table."))

server/website/website/management/commands/startcelery.py

@@ -0,0 +1,116 @@
+#
+# OtterTune - startcelery.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+import os
+import time
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from fabric.api import hide, lcd, local
+
+
+class Command(BaseCommand):
+    help = 'Start celery and celerybeat in the background.'
+    celery_cmd = 'python3 manage.py {cmd} {opts} {pipe} &'.format
+    max_wait_sec = 15
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--celery-only',
+            action='store_true',
+            help='Start celery only (skip celerybeat).')
+        parser.add_argument(
+            '--celerybeat-only',
+            action='store_true',
+            help='Start celerybeat only (skip celery).')
+        parser.add_argument(
+            '--loglevel',
+            metavar='LOGLEVEL',
+            help='Logging level, choose between DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL. '
+                 'Defaults to DEBUG if settings.DEBUG is true, otherwise INFO.')
+        parser.add_argument(
+            '--pool',
+            metavar='POOL_CLS',
+            default='threads',
+            help='Pool implementation: prefork (default), eventlet, gevent, solo or threads.'
+                 'Default: threads.')
+        parser.add_argument(
+            '--celery-pidfile',
+            metavar='PIDFILE',
+            default='celery.pid',
+            help='File used to store the process pid. The program will not start if this '
+                 'file already exists and the pid is still alive. Default: celery.pid.')
+        parser.add_argument(
+            '--celerybeat-pidfile',
+            metavar='PIDFILE',
+            default='celerybeat.pid',
+            help='File used to store the process pid. The program will not start if this '
+                 'file already exists and the pid is still alive. Default: celerybeat.pid.')
+        parser.add_argument(
+            '--celery-options',
+            metavar='OPTIONS',
+            help="A comma-separated list of additional options to pass to celery, "
+                 "see 'python manage.py celery worker --help' for all available options. "
+                 "IMPORTANT: the option's initial -/-- must be omitted. "
+                 "Example: '--celery-options purge,include=foo.tasks,q'.")
+        parser.add_argument(
+            '--celerybeat-options',
+            metavar='OPTIONS',
+            help="A comma-separated list of additional options to pass to celerybeat, "
+                 "see 'python manage.py celerybeat --help' for all available options. "
+                 "IMPORTANT: the option's initial -/-- must be omitted. "
+                 "Example: '--celerybeat-options uid=123,q'.")
+
+    def _parse_suboptions(self, suboptions):
+        suboptions = suboptions or ''
+        parsed = []
+        for opt in suboptions.split(','):
+            if opt:
+                opt = ('-{}' if len(opt) == 1 else '--{}').format(opt)
+                parsed.append(opt)
+        return parsed
+
+    def handle(self, *args, **options):
+        loglevel = options['loglevel'] or ('DEBUG' if settings.DEBUG else 'INFO')
+        celery_options = [
+            '--loglevel={}'.format(loglevel),
+            '--pool={}'.format(options['pool']),
+            '--pidfile={}'.format(options['celery_pidfile']),
+        ] + self._parse_suboptions(options['celery_options'])
+        celerybeat_options = [
+            '--loglevel={}'.format(loglevel),
+            '--pidfile={}'.format(options['celerybeat_pidfile']),
+        ] + self._parse_suboptions(options['celerybeat_options'])
+
+        pipe = '' if 'console' in settings.LOGGING['loggers']['celery']['handlers'] \
+            else '> /dev/null 2>&1'
+
+        with lcd(settings.PROJECT_ROOT), hide('command'):
+            if not options['celerybeat_only']:
+                local(self.celery_cmd(
+                    cmd='celery worker', opts=' '.join(celery_options), pipe=pipe))
+
+            if not options['celery_only']:
+                local(self.celery_cmd(
+                    cmd='celerybeat', opts=' '.join(celerybeat_options), pipe=pipe))
+
+        pidfiles = [options['celery_pidfile'], options['celerybeat_pidfile']]
+        wait_sec = 0
+
+        while wait_sec < self.max_wait_sec and len(pidfiles) > 0:
+            time.sleep(1)
+            wait_sec += 1
+
+            for i in range(len(pidfiles)):
+                if os.path.exists(pidfiles[i]):
+                    pidfiles.pop(i)
+
+        for name in ('celery', 'celerybeat'):
+            if os.path.exists(options[name + '_pidfile']):
+                self.stdout.write(self.style.SUCCESS(
+                    "Successfully started '{}'.".format(name)))
+            else:
+                self.stdout.write(self.style.NOTICE(
+                    "Failed to start '{}'.".format(name)))

server/website/website/management/commands/stopcelery.py

@@ -0,0 +1,56 @@
+#
+# OtterTune - stopcelery.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+import os
+import time
+
+from django.core.management.base import BaseCommand
+from fabric.api import local
+
+
+class Command(BaseCommand):
+    help = 'Start celery and celerybeat in the background.'
+    celery_cmd = 'python3 manage.py {cmd} {opts} &'.format
+    max_wait_sec = 15
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--celery-pidfile',
+            metavar='PIDFILE',
+            default='celery.pid',
+            help="Alternate path to the process' pid file if not located at ./celery.pid.")
+        parser.add_argument(
+            '--celerybeat-pidfile',
+            metavar='PIDFILE',
+            default='celerybeat.pid',
+            help="Alternate path to the process' pid file if not located at ./celerybeat.pid.")
+
+    def handle(self, *args, **options):
+        check_pidfiles = []
+        for name in ('celery', 'celerybeat'):
+            try:
+                pidfile = options[name + '_pidfile']
+                with open(pidfile, 'r') as f:
+                    pid = f.read()
+                local('kill {}'.format(pid))
+                check_pidfiles.append((name, pidfile))
+            except Exception as e:
+                self.stdout.write(self.style.NOTICE(
+                    "ERROR: an exception occurred while stopping '{}':\n{}\n".format(name, e)))
+
+        if check_pidfiles:
+            self.stdout.write("Waiting for processes to shutdown...\n")
+        for name, pidfile in check_pidfiles:
+            wait_sec = 0
+            while os.path.exists(pidfile) and wait_sec < self.max_wait_sec:
+                time.sleep(1)
+                wait_sec += 1
+            if os.path.exists(pidfile):
+                self.stdout.write(self.style.NOTICE(
+                    "WARNING: file '{}' still exists after stopping {}.".format(
+                        pidfile, name)))
+            else:
+                self.stdout.write(self.style.SUCCESS(
+                    "Successfully stopped '{}'.".format(name)))

server/website/website/settings/common.py

@@ -194,6 +194,7 @@ INSTALLED_APPS = (
     'djcelery',
     # 'django_celery_beat',
     'website',
+    'django_db_logger',
 )
 
 # ==============================================
@@ -253,6 +254,14 @@ LOGGING = {
             'backupCount': 2,
             'formatter': 'standard',
         },
+        'dblog': {
+            'level': 'DEBUG',
+            'class': 'django_db_logger.db_log_handler.DatabaseLogHandler',
+        },
+        'dblog_warn': {
+            'level': 'WARN',
+            'class': 'django_db_logger.db_log_handler.DatabaseLogHandler',
+        },
         'console': {
             'level': 'DEBUG',
             'class': 'logging.StreamHandler',
@@ -271,26 +280,26 @@ LOGGING = {
     },
     'loggers': {
         'django': {
-            'handlers': ['console', 'logfile'],
+            'handlers': ['console', 'logfile', 'dblog_warn'],
             'propagate': True,
             'level': 'WARN',
         },
         'django.db.backends': {
-            'handlers': ['console', 'logfile'],
+            'handlers': ['console', 'logfile', 'dblog_warn'],
             'level': 'WARN',
             'propagate': False,
         },
         'website': {
-            'handlers': ['console', 'logfile'],
+            'handlers': ['console', 'logfile', 'dblog'],
             'level': 'DEBUG',
         },
         'django.request': {
-            'handlers': ['console'],
+            'handlers': ['console', 'dblog_warn'],
-            'level': 'DEBUG',
+            'level': 'INFO',
             'propagate': False,
         },
         'celery': {
-            'handlers': ['console', 'celery'],
+            'handlers': ['celery', 'dblog'],
             'level': 'DEBUG',
             'propogate': True,
         },