diff --git a/client/controller/.gitignore b/client/controller/.gitignore index 8f3a7e5..5b86e7d 100644 --- a/client/controller/.gitignore +++ b/client/controller/.gitignore @@ -26,3 +26,8 @@ lib/ # log file *.log + +# controller configuration files +config/* +!config/sample_*_config.json + diff --git a/server/website/.gitignore b/server/website/.gitignore index bcaf137..3399987 100644 --- a/server/website/.gitignore +++ b/server/website/.gitignore @@ -4,9 +4,10 @@ log/ *.log local_settings.py -# celery beat schedule file # -############################# +# celery/celerybeat # +##################### celerybeat-schedule +*.pid # Raw data files # ################## diff --git a/server/website/config/.gitignore b/server/website/config/.gitignore deleted file mode 100644 index 16443c5..0000000 --- a/server/website/config/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.bak -prod_supervisord.conf diff --git a/server/website/config/postgresql.conf b/server/website/config/postgresql.conf deleted file mode 100644 index 422b34f..0000000 --- a/server/website/config/postgresql.conf +++ /dev/null @@ -1,33 +0,0 @@ -# ----------------------------- -# PostgreSQL configuration file -# ----------------------------- -# -# This file consists of lines of the form: -# -# name = value -# -# (The "=" is optional.) Whitespace may be used. Comments are introduced with -# "#" anywhere on a line. The complete list of parameter names and allowed -# values can be found in the PostgreSQL documentation. -# -# The commented-out settings shown in this file represent the default values. -# Re-commenting a setting is NOT sufficient to revert it to the default value; -# you need to reload the server. -# -# This file is read on server startup and when the server receives a SIGHUP -# signal. If you edit the file on a running system, you have to SIGHUP the -# server for the changes to take effect, or use "pg_ctl reload". Some -# parameters, which are marked below, require a server shutdown and restart to -# take effect. -# -# Any parameter can also be given as a command-line option to the server, e.g., -# "postgres -c log_connections=on". Some parameters can be changed at run time -# with the "SET" SQL command. -# -# Memory units: kB = kilobytes Time units: ms = milliseconds -# MB = megabytes s = seconds -# GB = gigabytes min = minutes -# h = hours -# d = days - - diff --git a/server/website/beat.sh b/server/website/script/management/beat.sh similarity index 100% rename from server/website/beat.sh rename to server/website/script/management/beat.sh diff --git a/server/website/celery.sh b/server/website/script/management/celery.sh similarity index 100% rename from server/website/celery.sh rename to server/website/script/management/celery.sh diff --git a/server/website/django.sh b/server/website/script/management/django.sh similarity index 100% rename from server/website/django.sh rename to server/website/script/management/django.sh diff --git a/server/website/fabfile.py b/server/website/script/management/fabfile.py similarity index 100% rename from server/website/fabfile.py rename to server/website/script/management/fabfile.py diff --git a/server/website/website/admin.py b/server/website/website/admin.py index 90715ba..54cb093 100644 --- a/server/website/website/admin.py +++ b/server/website/website/admin.py @@ -4,7 +4,10 @@ # Copyright (c) 2017-18, Carnegie Mellon University Database Group # from django.contrib import admin -from djcelery.models import TaskMeta +from django.utils.html import format_html +from django_db_logger.admin import StatusLogAdmin +from django_db_logger.models import StatusLog +from djcelery import models as djcelery_models from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog, MetricData, @@ -45,12 +48,12 @@ class SessionAdmin(admin.ModelAdmin): class SessionKnobAdmin(admin.ModelAdmin): list_display = ('knob', 'dbms', 'session', 'minval', 'maxval', 'tunable') list_filter = (('session__dbms', admin.RelatedOnlyFieldListFilter), - ('session', admin.RelatedOnlyFieldListFilter), ('tunable')) + ('session', admin.RelatedOnlyFieldListFilter), + ('tunable', admin.FieldListFilter)) ordering = ('session__dbms', 'session__name', '-tunable', 'knob__name') - @staticmethod - def dbms(obj): - return obj.session.dbms + def dbms(self, instance): # pylint: disable=no-self-use + return instance.session.dbms class HardwareAdmin(admin.ModelAdmin): @@ -71,22 +74,6 @@ class MetricDataAdmin(admin.ModelAdmin): ordering = ('creation_time',) -class TaskMetaAdmin(admin.ModelAdmin): - list_display = ('id', 'status', 'task_result', 'date_done') - readonly_fields = ('id', 'task_id', 'status', 'result', 'date_done', - 'traceback', 'hidden', 'meta') - fields = readonly_fields - list_filter = ('status',) - ordering = ('date_done',) - - @staticmethod - def task_result(obj, maxlen=300): - res = obj.result - if res and len(res) > maxlen: - res = res[:maxlen] + '...' - return res - - class ResultAdmin(admin.ModelAdmin): readonly_fields = ('dbms', 'knob_data', 'metric_data', 'session', 'workload') list_display = ('id', 'dbms', 'session', 'workload', 'creation_time') @@ -120,6 +107,55 @@ class WorkloadAdmin(admin.ModelAdmin): ('hardware', admin.RelatedOnlyFieldListFilter)) +class TaskMetaAdmin(admin.ModelAdmin): + list_display = ('colored_status', 'task_result', 'date_done', 'task_traceback') + list_display_links = ('colored_status', 'task_result') + readonly_fields = ('id', 'task_id', 'status', 'result', 'date_done', + 'traceback', 'hidden', 'meta') + fields = readonly_fields + list_filter = ('status',) + list_per_page = 10 + ordering = ('date_done',) + max_field_length = 1000 + + @staticmethod + def color_field(text, status): + if status == 'SUCCESS': + color = 'green' + elif status in ('PENDING', 'RECEIVED', 'STARTED'): + color = 'orange' + else: + color = 'red' + return format_html('{}'.format(color, text)) + + def format_field(self, field): + text = str(field) if field else '' + if len(text) > self.max_field_length: + text = text[:self.max_field_length] + '...' + return text + + def colored_status(self, instance): + return self.color_field(instance.status, instance.status) + colored_status.short_description = 'Status' + + def task_traceback(self, instance): + text = self.format_field(instance.traceback) + return format_html('
{}
'.format(text))
+ task_traceback.short_description = 'Traceback'
+
+ def task_result(self, instance):
+ res = self.format_field(instance.result)
+ return self.color_field(res, instance.status)
+ task_result.short_description = 'Result'
+
+
+class CustomStatusLogAdmin(StatusLogAdmin):
+ list_display = ('logger_name', 'colored_msg', 'traceback', 'create_datetime_format')
+ list_display_links = ('logger_name',)
+ list_filter = ('logger_name', 'level')
+
+
+# Admin classes for website models
admin.site.register(DBMSCatalog, DBMSCatalogAdmin)
admin.site.register(KnobCatalog, KnobCatalogAdmin)
admin.site.register(MetricCatalog, MetricCatalogAdmin)
@@ -127,7 +163,6 @@ admin.site.register(Session, SessionAdmin)
admin.site.register(Project, ProjectAdmin)
admin.site.register(KnobData, KnobDataAdmin)
admin.site.register(MetricData, MetricDataAdmin)
-admin.site.register(TaskMeta, TaskMetaAdmin)
admin.site.register(Result, ResultAdmin)
admin.site.register(BackupData, BackupDataAdmin)
admin.site.register(PipelineData, PipelineDataAdmin)
@@ -135,3 +170,21 @@ admin.site.register(PipelineRun, PipelineRunAdmin)
admin.site.register(Workload, WorkloadAdmin)
admin.site.register(SessionKnob, SessionKnobAdmin)
admin.site.register(Hardware, HardwareAdmin)
+
+# Admin classes for 3rd party models
+admin.site.unregister(StatusLog)
+admin.site.register(StatusLog, CustomStatusLogAdmin)
+admin.site.register(djcelery_models.TaskMeta, TaskMetaAdmin)
+
+# Unregister empty djcelery models
+UNUSED_DJCELERY_MODELS = (
+ djcelery_models.CrontabSchedule,
+ djcelery_models.IntervalSchedule,
+ djcelery_models.PeriodicTask,
+ djcelery_models.TaskState,
+ djcelery_models.WorkerState,
+)
+
+for model in UNUSED_DJCELERY_MODELS:
+ if model.objects.count() == 0:
+ admin.site.unregister(model)
diff --git a/server/website/website/management/commands/cleardblog.py b/server/website/website/management/commands/cleardblog.py
new file mode 100644
index 0000000..182d61d
--- /dev/null
+++ b/server/website/website/management/commands/cleardblog.py
@@ -0,0 +1,16 @@
+#
+# OtterTune - cleardblog.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+from django.core.management.base import BaseCommand
+from django_db_logger.models import StatusLog
+
+
+class Command(BaseCommand):
+ help = 'Clear all log entries from the django_db_logger table.'
+
+ def handle(self, *args, **options):
+ StatusLog.objects.all().delete()
+ self.stdout.write(self.style.SUCCESS(
+ "Successfully cleared the django_db_logger table."))
diff --git a/server/website/website/management/commands/startcelery.py b/server/website/website/management/commands/startcelery.py
new file mode 100644
index 0000000..c2a2ced
--- /dev/null
+++ b/server/website/website/management/commands/startcelery.py
@@ -0,0 +1,116 @@
+#
+# OtterTune - startcelery.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+import os
+import time
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from fabric.api import hide, lcd, local
+
+
+class Command(BaseCommand):
+ help = 'Start celery and celerybeat in the background.'
+ celery_cmd = 'python3 manage.py {cmd} {opts} {pipe} &'.format
+ max_wait_sec = 15
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ '--celery-only',
+ action='store_true',
+ help='Start celery only (skip celerybeat).')
+ parser.add_argument(
+ '--celerybeat-only',
+ action='store_true',
+ help='Start celerybeat only (skip celery).')
+ parser.add_argument(
+ '--loglevel',
+ metavar='LOGLEVEL',
+ help='Logging level, choose between DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL. '
+ 'Defaults to DEBUG if settings.DEBUG is true, otherwise INFO.')
+ parser.add_argument(
+ '--pool',
+ metavar='POOL_CLS',
+ default='threads',
+ help='Pool implementation: prefork (default), eventlet, gevent, solo or threads.'
+ 'Default: threads.')
+ parser.add_argument(
+ '--celery-pidfile',
+ metavar='PIDFILE',
+ default='celery.pid',
+ help='File used to store the process pid. The program will not start if this '
+ 'file already exists and the pid is still alive. Default: celery.pid.')
+ parser.add_argument(
+ '--celerybeat-pidfile',
+ metavar='PIDFILE',
+ default='celerybeat.pid',
+ help='File used to store the process pid. The program will not start if this '
+ 'file already exists and the pid is still alive. Default: celerybeat.pid.')
+ parser.add_argument(
+ '--celery-options',
+ metavar='OPTIONS',
+ help="A comma-separated list of additional options to pass to celery, "
+ "see 'python manage.py celery worker --help' for all available options. "
+ "IMPORTANT: the option's initial -/-- must be omitted. "
+ "Example: '--celery-options purge,include=foo.tasks,q'.")
+ parser.add_argument(
+ '--celerybeat-options',
+ metavar='OPTIONS',
+ help="A comma-separated list of additional options to pass to celerybeat, "
+ "see 'python manage.py celerybeat --help' for all available options. "
+ "IMPORTANT: the option's initial -/-- must be omitted. "
+ "Example: '--celerybeat-options uid=123,q'.")
+
+ def _parse_suboptions(self, suboptions):
+ suboptions = suboptions or ''
+ parsed = []
+ for opt in suboptions.split(','):
+ if opt:
+ opt = ('-{}' if len(opt) == 1 else '--{}').format(opt)
+ parsed.append(opt)
+ return parsed
+
+ def handle(self, *args, **options):
+ loglevel = options['loglevel'] or ('DEBUG' if settings.DEBUG else 'INFO')
+ celery_options = [
+ '--loglevel={}'.format(loglevel),
+ '--pool={}'.format(options['pool']),
+ '--pidfile={}'.format(options['celery_pidfile']),
+ ] + self._parse_suboptions(options['celery_options'])
+ celerybeat_options = [
+ '--loglevel={}'.format(loglevel),
+ '--pidfile={}'.format(options['celerybeat_pidfile']),
+ ] + self._parse_suboptions(options['celerybeat_options'])
+
+ pipe = '' if 'console' in settings.LOGGING['loggers']['celery']['handlers'] \
+ else '> /dev/null 2>&1'
+
+ with lcd(settings.PROJECT_ROOT), hide('command'):
+ if not options['celerybeat_only']:
+ local(self.celery_cmd(
+ cmd='celery worker', opts=' '.join(celery_options), pipe=pipe))
+
+ if not options['celery_only']:
+ local(self.celery_cmd(
+ cmd='celerybeat', opts=' '.join(celerybeat_options), pipe=pipe))
+
+ pidfiles = [options['celery_pidfile'], options['celerybeat_pidfile']]
+ wait_sec = 0
+
+ while wait_sec < self.max_wait_sec and len(pidfiles) > 0:
+ time.sleep(1)
+ wait_sec += 1
+
+ for i in range(len(pidfiles)):
+ if os.path.exists(pidfiles[i]):
+ pidfiles.pop(i)
+
+ for name in ('celery', 'celerybeat'):
+ if os.path.exists(options[name + '_pidfile']):
+ self.stdout.write(self.style.SUCCESS(
+ "Successfully started '{}'.".format(name)))
+ else:
+ self.stdout.write(self.style.NOTICE(
+ "Failed to start '{}'.".format(name)))
diff --git a/server/website/website/management/commands/stopcelery.py b/server/website/website/management/commands/stopcelery.py
new file mode 100644
index 0000000..0eef50f
--- /dev/null
+++ b/server/website/website/management/commands/stopcelery.py
@@ -0,0 +1,56 @@
+#
+# OtterTune - stopcelery.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+import os
+import time
+
+from django.core.management.base import BaseCommand
+from fabric.api import local
+
+
+class Command(BaseCommand):
+ help = 'Start celery and celerybeat in the background.'
+ celery_cmd = 'python3 manage.py {cmd} {opts} &'.format
+ max_wait_sec = 15
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ '--celery-pidfile',
+ metavar='PIDFILE',
+ default='celery.pid',
+ help="Alternate path to the process' pid file if not located at ./celery.pid.")
+ parser.add_argument(
+ '--celerybeat-pidfile',
+ metavar='PIDFILE',
+ default='celerybeat.pid',
+ help="Alternate path to the process' pid file if not located at ./celerybeat.pid.")
+
+ def handle(self, *args, **options):
+ check_pidfiles = []
+ for name in ('celery', 'celerybeat'):
+ try:
+ pidfile = options[name + '_pidfile']
+ with open(pidfile, 'r') as f:
+ pid = f.read()
+ local('kill {}'.format(pid))
+ check_pidfiles.append((name, pidfile))
+ except Exception as e:
+ self.stdout.write(self.style.NOTICE(
+ "ERROR: an exception occurred while stopping '{}':\n{}\n".format(name, e)))
+
+ if check_pidfiles:
+ self.stdout.write("Waiting for processes to shutdown...\n")
+ for name, pidfile in check_pidfiles:
+ wait_sec = 0
+ while os.path.exists(pidfile) and wait_sec < self.max_wait_sec:
+ time.sleep(1)
+ wait_sec += 1
+ if os.path.exists(pidfile):
+ self.stdout.write(self.style.NOTICE(
+ "WARNING: file '{}' still exists after stopping {}.".format(
+ pidfile, name)))
+ else:
+ self.stdout.write(self.style.SUCCESS(
+ "Successfully stopped '{}'.".format(name)))
diff --git a/server/website/website/settings/common.py b/server/website/website/settings/common.py
index d24688b..cd0feb0 100644
--- a/server/website/website/settings/common.py
+++ b/server/website/website/settings/common.py
@@ -194,6 +194,7 @@ INSTALLED_APPS = (
'djcelery',
# 'django_celery_beat',
'website',
+ 'django_db_logger',
)
# ==============================================
@@ -253,6 +254,14 @@ LOGGING = {
'backupCount': 2,
'formatter': 'standard',
},
+ 'dblog': {
+ 'level': 'DEBUG',
+ 'class': 'django_db_logger.db_log_handler.DatabaseLogHandler',
+ },
+ 'dblog_warn': {
+ 'level': 'WARN',
+ 'class': 'django_db_logger.db_log_handler.DatabaseLogHandler',
+ },
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
@@ -271,26 +280,26 @@ LOGGING = {
},
'loggers': {
'django': {
- 'handlers': ['console', 'logfile'],
+ 'handlers': ['console', 'logfile', 'dblog_warn'],
'propagate': True,
'level': 'WARN',
},
'django.db.backends': {
- 'handlers': ['console', 'logfile'],
+ 'handlers': ['console', 'logfile', 'dblog_warn'],
'level': 'WARN',
'propagate': False,
},
'website': {
- 'handlers': ['console', 'logfile'],
+ 'handlers': ['console', 'logfile', 'dblog'],
'level': 'DEBUG',
},
'django.request': {
- 'handlers': ['console'],
- 'level': 'DEBUG',
+ 'handlers': ['console', 'dblog_warn'],
+ 'level': 'INFO',
'propagate': False,
},
'celery': {
- 'handlers': ['console', 'celery'],
+ 'handlers': ['celery', 'dblog'],
'level': 'DEBUG',
'propogate': True,
},