Added management function similar to 'startcelery' except that it auto-reloads modified website files.

This commit is contained in:
dvanaken 2020-01-07 02:45:32 -05:00 committed by Dana Van Aken
parent 9fced864fe
commit 8001b658c9
2 changed files with 91 additions and 7 deletions

View File

@ -0,0 +1,70 @@
#
# OtterTune - runcelery.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
import os
from django.core.management import call_command
from django.core.management.base import BaseCommand
from django.utils import autoreload
from fabric.api import hide, local, settings
class Command(BaseCommand):
help = 'Start celery and celerybeat using the auto-reloader.'
def add_arguments(self, parser):
parser.add_argument(
'--loglevel',
metavar='LOGLEVEL',
help='Logging level, choose between DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL. '
'Defaults to DEBUG if settings.DEBUG is true, otherwise INFO.')
parser.add_argument(
'--pool',
metavar='POOL_CLS',
default='threads',
help='Pool implementation: prefork (default), eventlet, gevent, solo or threads. '
'Default: threads.')
parser.add_argument(
'--celery-pidfile',
metavar='PIDFILE',
default='celery.pid',
help='File used to store the process pid. The program will not start if this '
'file already exists and the pid is still alive. Default: celery.pid.')
parser.add_argument(
'--celerybeat-pidfile',
metavar='PIDFILE',
default='celerybeat.pid',
help='File used to store the process pid. The program will not start if this '
'file already exists and the pid is still alive. Default: celerybeat.pid.')
parser.add_argument(
'--celery-options',
metavar='OPTIONS',
help="A comma-separated list of additional options to pass to celery, "
"see 'python manage.py celery worker --help' for all available options. "
"IMPORTANT: the option's initial -/-- must be omitted. "
"Example: '--celery-options purge,include=foo.tasks,q'.")
parser.add_argument(
'--celerybeat-options',
metavar='OPTIONS',
help="A comma-separated list of additional options to pass to celerybeat, "
"see 'python manage.py celerybeat --help' for all available options. "
"IMPORTANT: the option's initial -/-- must be omitted. "
"Example: '--celerybeat-options uid=123,q'.")
def inner_run(self, *args, **options): # pylint: disable=unused-argument
autoreload.raise_last_exception()
for pidfile in (options['celery_pidfile'], options['celerybeat_pidfile']):
if os.path.exists(pidfile):
with open(pidfile, 'r') as f:
pid = f.read().strip()
with settings(warn_only=True), hide('commands'): # pylint: disable=not-context-manager
local('kill -9 {}'.format(pid))
local('rm -f {}'.format(pidfile))
call_command('startcelery', silent=True, pipe='', **options)
self.stdout.write(self.style.SUCCESS("Successfully reloaded celery and celerybeat."))
def handle(self, *args, **options):
autoreload.main(self.inner_run, None, options)

View File

@ -17,11 +17,12 @@ class Command(BaseCommand):
max_wait_sec = 15
def add_arguments(self, parser):
parser.add_argument(
group = parser.add_mutually_exclusive_group()
group.add_argument(
'--celery-only',
action='store_true',
help='Start celery only (skip celerybeat).')
parser.add_argument(
group.add_argument(
'--celerybeat-only',
action='store_true',
help='Start celerybeat only (skip celery).')
@ -63,7 +64,8 @@ class Command(BaseCommand):
"IMPORTANT: the option's initial -/-- must be omitted. "
"Example: '--celerybeat-options uid=123,q'.")
def _parse_suboptions(self, suboptions):
@staticmethod
def _parse_suboptions(suboptions):
suboptions = suboptions or ''
parsed = []
for opt in suboptions.split(','):
@ -73,6 +75,21 @@ class Command(BaseCommand):
return parsed
def handle(self, *args, **options):
# Stealth option to disable stdout
if options.get('silent', False):
self.stdout = open(os.devnull, 'w')
# Stealth option that assigns where to pipe initial output
pipe = options.get('pipe', None)
if pipe is None:
pipe = '> /dev/null 2>&1'
try:
if 'celery' in settings.LOGGING['loggers']['celery']['handlers']:
logfile = settings.LOGGING['handlers']['celery']['filename']
pipe = '>> {} 2>&1'.format(logfile)
except KeyError:
pass
loglevel = options['loglevel'] or ('DEBUG' if settings.DEBUG else 'INFO')
celery_options = [
'--loglevel={}'.format(loglevel),
@ -84,10 +101,7 @@ class Command(BaseCommand):
'--pidfile={}'.format(options['celerybeat_pidfile']),
] + self._parse_suboptions(options['celerybeat_options'])
pipe = '' if 'console' in settings.LOGGING['loggers']['celery']['handlers'] \
else '> /dev/null 2>&1'
with lcd(settings.PROJECT_ROOT), hide('commands'):
with lcd(settings.PROJECT_ROOT), hide('commands'): # pylint: disable=not-context-manager
if not options['celerybeat_only']:
local(self.celery_cmd(
cmd='celery worker', opts=' '.join(celery_options), pipe=pipe))