diff --git a/server/website/website/migrations/0002_enable_compression.py b/server/website/website/migrations/0002_enable_compression.py index ddf23b8..a9ef5eb 100644 --- a/server/website/website/migrations/0002_enable_compression.py +++ b/server/website/website/migrations/0002_enable_compression.py @@ -2,9 +2,7 @@ import logging -from django.conf import settings -#from django.core.exceptions import ProgrammingError -from django.db import connection, migrations, ProgrammingError +from django.db import migrations, ProgrammingError LOG = logging.getLogger(__name__) @@ -15,6 +13,58 @@ TABLES_TO_COMPRESS = [ "website_pipelinedata", ] +MYSQL_MIN_VERSION = (5, 7, 0) + +ALTER_SQL = "ALTER TABLE %s COMPRESSION='%s'" +OPTIMIZE_SQL = "OPTIMIZE TABLE %s" + + +def compression_supported(schema_editor): + supported = False + dbms = schema_editor.connection.vendor + + if dbms == 'mysql': + with schema_editor.connection.cursor() as cursor: + cursor.execute('SELECT VERSION()') + res = cursor.fetchone()[0] + + version_str = res.split('-')[0] + version = tuple(int(v) for v in version_str.split('.')) + assert len(version) == len(MYSQL_MIN_VERSION), \ + 'MySQL - current version: {}, min version: {}'.format(version, MYSQL_MIN_VERSION) + + if version >= MYSQL_MIN_VERSION: + supported = True + LOG.debug("%s %s: table compression supported.", dbms.upper(), version_str) + else: + LOG.debug("%s %s: table compression NOT supported.", dbms.upper(), version_str) + else: + LOG.debug("%s: table compression NOT supported.", dbms.upper()) + + return supported + + +def enable_compression(apps, schema_editor): + # try: + if compression_supported(schema_editor): + for table in TABLES_TO_COMPRESS: + schema_editor.execute(ALTER_SQL % (table, 'zlib')) + schema_editor.execute(OPTIMIZE_SQL % table) + + # except ProgrammingError: + # LOG.warning("Error applying forward migration '0002_enable_compression'... Skipping.") + + +def disable_compression(apps, schema_editor): + try: + if compression_supported(schema_editor): + for table in TABLES_TO_COMPRESS: + schema_editor.execute(ALTER_SQL % (table, 'none')) + schema_editor.execute(OPTIMIZE_SQL % table) + + except ProgrammingError: + LOG.warning("Error applying reverse migration '0002_enable_compression'... Skipping.") + class Migration(migrations.Migration): @@ -22,37 +72,6 @@ class Migration(migrations.Migration): ('website', '0001_initial'), ] - try: - if connection.vendor == 'mysql': + operations = [migrations.RunPython(enable_compression, disable_compression)] - version = (0, 0, 0) - with connection.cursor() as cursor: - cursor.execute('SELECT VERSION()') - version = cursor.fetchone()[0] - - version_str = version.split('-')[0] - version = version_str.split('.') - version = tuple(int(v) for v in version) - - if version >= (5, 7, 0): - operations = [ - migrations.RunSQL(["ALTER TABLE " + table_name + " COMPRESSION='zlib';", - "OPTIMIZE TABLE " + table_name + ";"], - ["ALTER TABLE " + table_name + " COMPRESSION='none';", - "OPTIMIZE TABLE " + table_name + ";"]) - for table_name in TABLES_TO_COMPRESS - ] - LOG.debug("Enabled compression for '%s %s'", connection.vendor, version_str) - - else: - operations = [] - LOG.debug("Disabled compression for '%s %s': version not supported", - connection.vendor, version_str) - - else: - LOG.debug("Disabled compression for '%s': vendor not supported", connection.vendor) - - except ProgrammingError as err: - LOG.warning("Error applying migration '0002_enable_compression'... Skipping") - operations = [] diff --git a/server/website/website/models.py b/server/website/website/models.py index f472816..bca4d2c 100644 --- a/server/website/website/models.py +++ b/server/website/website/models.py @@ -123,8 +123,8 @@ class MetricManager(models.Manager): @staticmethod def get_metric_meta(dbms, target_objective=None): - numeric_metric_names = MetricCatalog.objects.filter( - dbms=dbms, metric_type=MetricType.COUNTER).values_list('name', flat=True) + numeric_metric_names = MetricCatalog.objects.filter(dbms=dbms).exclude( + metric_type=MetricType.INFO).values_list('name', flat=True) numeric_metrics = {} for metname in numeric_metric_names: numeric_metrics[metname] = MetricMeta( diff --git a/server/website/website/settings/common.py b/server/website/website/settings/common.py index 93e140c..d24688b 100644 --- a/server/website/website/settings/common.py +++ b/server/website/website/settings/common.py @@ -33,10 +33,6 @@ CONFIG_DIR = join(PROJECT_ROOT, 'config') # Where the log files are stored LOG_DIR = join(PROJECT_ROOT, 'log') -# File/directory upload permissions -FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o664 -FILE_UPLOAD_PERMISSIONS = 0o664 - # Path to OtterTune's website and ML modules OTTERTUNE_LIBS = dirname(PROJECT_ROOT) @@ -111,7 +107,7 @@ MEDIA_URL = '/media/' # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/var/www/example.com/static/" -STATIC_ROOT = join(PROJECT_ROOT, 'website', 'static') +STATIC_ROOT = join(PROJECT_ROOT, 'static') # URL prefix for static files. # Example: "http://example.com/static/", "http://static.example.com/" @@ -218,6 +214,8 @@ CELERYD_MAX_TASKS_PER_CHILD = 50 # Number of concurrent workers. CELERYD_CONCURRENCY = 8 +CELERYD_HIJACK_ROOT_LOGGER = False + djcelery.setup_loader() # ============================================== @@ -247,8 +245,16 @@ LOGGING = { 'backupCount': 2, 'formatter': 'standard', }, + 'celery': { + 'level': 'DEBUG', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': join(LOG_DIR, 'celery.log'), + 'maxBytes': 50000, + 'backupCount': 2, + 'formatter': 'standard', + }, 'console': { - 'level': 'INFO', + 'level': 'DEBUG', 'class': 'logging.StreamHandler', 'formatter': 'standard' }, @@ -276,7 +282,6 @@ LOGGING = { }, 'website': { 'handlers': ['console', 'logfile'], - 'propagate': False, 'level': 'DEBUG', }, 'django.request': { @@ -284,6 +289,11 @@ LOGGING = { 'level': 'DEBUG', 'propagate': False, }, + 'celery': { + 'handlers': ['console', 'celery'], + 'level': 'DEBUG', + 'propogate': True, + }, # Uncomment to email admins after encountering an error (and debug=False) # 'django.request': { # 'handlers': ['mail_admins'],