Added django-db-logger for storing log messages in the database, removed random *.sh scripts from website code then extended manage.py with same commands

server/website/website/admin.py

@@ -4,7 +4,10 @@
 # Copyright (c) 2017-18, Carnegie Mellon University Database Group
 #
 from django.contrib import admin
-from djcelery.models import TaskMeta
+from django.utils.html import format_html
+from django_db_logger.admin import StatusLogAdmin
+from django_db_logger.models import StatusLog
+from djcelery import models as djcelery_models
 
 from .models import (BackupData, DBMSCatalog, KnobCatalog,
                      KnobData, MetricCatalog, MetricData,
@@ -45,12 +48,12 @@ class SessionAdmin(admin.ModelAdmin):
 class SessionKnobAdmin(admin.ModelAdmin):
     list_display = ('knob', 'dbms', 'session', 'minval', 'maxval', 'tunable')
     list_filter = (('session__dbms', admin.RelatedOnlyFieldListFilter),
-                   ('session', admin.RelatedOnlyFieldListFilter), ('tunable'))
+                   ('session', admin.RelatedOnlyFieldListFilter),
+                   ('tunable', admin.FieldListFilter))
     ordering = ('session__dbms', 'session__name', '-tunable', 'knob__name')
 
-    @staticmethod
-    def dbms(obj):
-        return obj.session.dbms
+    def dbms(self, instance):  # pylint: disable=no-self-use
+        return instance.session.dbms
 
 
 class HardwareAdmin(admin.ModelAdmin):
@@ -71,22 +74,6 @@ class MetricDataAdmin(admin.ModelAdmin):
     ordering = ('creation_time',)
 
 
-class TaskMetaAdmin(admin.ModelAdmin):
-    list_display = ('id', 'status', 'task_result', 'date_done')
-    readonly_fields = ('id', 'task_id', 'status', 'result', 'date_done',
-                       'traceback', 'hidden', 'meta')
-    fields = readonly_fields
-    list_filter = ('status',)
-    ordering = ('date_done',)
-
-    @staticmethod
-    def task_result(obj, maxlen=300):
-        res = obj.result
-        if res and len(res) > maxlen:
-            res = res[:maxlen] + '...'
-        return res
-
-
 class ResultAdmin(admin.ModelAdmin):
     readonly_fields = ('dbms', 'knob_data', 'metric_data', 'session', 'workload')
     list_display = ('id', 'dbms', 'session', 'workload', 'creation_time')
@@ -120,6 +107,55 @@ class WorkloadAdmin(admin.ModelAdmin):
                    ('hardware', admin.RelatedOnlyFieldListFilter))
 
 
+class TaskMetaAdmin(admin.ModelAdmin):
+    list_display = ('colored_status', 'task_result', 'date_done', 'task_traceback')
+    list_display_links = ('colored_status', 'task_result')
+    readonly_fields = ('id', 'task_id', 'status', 'result', 'date_done',
+                       'traceback', 'hidden', 'meta')
+    fields = readonly_fields
+    list_filter = ('status',)
+    list_per_page = 10
+    ordering = ('date_done',)
+    max_field_length = 1000
+
+    @staticmethod
+    def color_field(text, status):
+        if status == 'SUCCESS':
+            color = 'green'
+        elif status in ('PENDING', 'RECEIVED', 'STARTED'):
+            color = 'orange'
+        else:
+            color = 'red'
+        return format_html('<span style="color: {};">{}</span>'.format(color, text))
+
+    def format_field(self, field):
+        text = str(field) if field else ''
+        if len(text) > self.max_field_length:
+            text = text[:self.max_field_length] + '...'
+        return text
+
+    def colored_status(self, instance):
+        return self.color_field(instance.status, instance.status)
+    colored_status.short_description = 'Status'
+
+    def task_traceback(self, instance):
+        text = self.format_field(instance.traceback)
+        return format_html('<pre><code>{}</code></pre>'.format(text))
+    task_traceback.short_description = 'Traceback'
+
+    def task_result(self, instance):
+        res = self.format_field(instance.result)
+        return self.color_field(res, instance.status)
+    task_result.short_description = 'Result'
+
+
+class CustomStatusLogAdmin(StatusLogAdmin):
+    list_display = ('logger_name', 'colored_msg', 'traceback', 'create_datetime_format')
+    list_display_links = ('logger_name',)
+    list_filter = ('logger_name', 'level')
+
+
+# Admin classes for website models
 admin.site.register(DBMSCatalog, DBMSCatalogAdmin)
 admin.site.register(KnobCatalog, KnobCatalogAdmin)
 admin.site.register(MetricCatalog, MetricCatalogAdmin)
@@ -127,7 +163,6 @@ admin.site.register(Session, SessionAdmin)
 admin.site.register(Project, ProjectAdmin)
 admin.site.register(KnobData, KnobDataAdmin)
 admin.site.register(MetricData, MetricDataAdmin)
-admin.site.register(TaskMeta, TaskMetaAdmin)
 admin.site.register(Result, ResultAdmin)
 admin.site.register(BackupData, BackupDataAdmin)
 admin.site.register(PipelineData, PipelineDataAdmin)
@@ -135,3 +170,21 @@ admin.site.register(PipelineRun, PipelineRunAdmin)
 admin.site.register(Workload, WorkloadAdmin)
 admin.site.register(SessionKnob, SessionKnobAdmin)
 admin.site.register(Hardware, HardwareAdmin)
+
+# Admin classes for 3rd party models
+admin.site.unregister(StatusLog)
+admin.site.register(StatusLog, CustomStatusLogAdmin)
+admin.site.register(djcelery_models.TaskMeta, TaskMetaAdmin)
+
+# Unregister empty djcelery models
+UNUSED_DJCELERY_MODELS = (
+    djcelery_models.CrontabSchedule,
+    djcelery_models.IntervalSchedule,
+    djcelery_models.PeriodicTask,
+    djcelery_models.TaskState,
+    djcelery_models.WorkerState,
+)
+
+for model in UNUSED_DJCELERY_MODELS:
+    if model.objects.count() == 0:
+        admin.site.unregister(model)