From 0da1b724cc7b4a14c2d674b88c0ca004b9793aef Mon Sep 17 00:00:00 2001 From: Dana Van Aken Date: Tue, 8 Oct 2019 07:21:08 -0400 Subject: [PATCH] Added postgresql client package to Dockerfiles and psycopg2-binary to requirements.txt, removed unused matplotlib code/package from analysis/cluster.py and requirements.txt, replaced commands in docker start.sh file with new management commands. --- docker/Dockerfile | 1 - docker/Dockerfile.base-centos-7 | 6 +- docker/Dockerfile.base-ubuntu-18.04 | 4 +- docker/createadmin.py | 26 ------ docker/docker-compose.yml | 1 - docker/start.sh | 6 +- script/validators/source_validator.py | 3 - server/analysis/cluster.py | 120 -------------------------- server/website/requirements.txt | 15 ++-- 9 files changed, 14 insertions(+), 168 deletions(-) delete mode 100644 docker/createadmin.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 83fa0d7..af1c7d1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,7 +11,6 @@ WORKDIR /app/server/website COPY ./docker/credentials.py ./website/settings COPY ./docker/start.sh . -COPY ./docker/createadmin.py . COPY ./docker/wait-for-it.sh . RUN chmod +x ./*.sh diff --git a/docker/Dockerfile.base-centos-7 b/docker/Dockerfile.base-centos-7 index e1d9748..9fc22e8 100644 --- a/docker/Dockerfile.base-centos-7 +++ b/docker/Dockerfile.base-centos-7 @@ -8,12 +8,12 @@ ENV PATH=${GRADLE_HOME}/bin:${PATH} COPY ./server/website/requirements.txt / RUN yum update -y \ - && yum install -y mariadb mariadb-devel \ + && yum install -y mariadb mariadb-devel postgresql \ https://centos7.iuscommunity.org/ius-release.rpm \ && yum install -y gcc git MySQL-python openldap-devel \ parallel python36u python36u-devel python36u-libs \ - python36u-pip python36u-tkinter rabbitmq-server \ - java-11-openjdk-devel wget which unzip curl \ + python36u-pip rabbitmq-server java-11-openjdk-devel \ + wget which unzip curl \ && yum -y autoremove \ && yum clean metadata \ && yum clean all \ diff --git a/docker/Dockerfile.base-ubuntu-18.04 b/docker/Dockerfile.base-ubuntu-18.04 index eaaeec4..e2f5406 100644 --- a/docker/Dockerfile.base-ubuntu-18.04 +++ b/docker/Dockerfile.base-ubuntu-18.04 @@ -9,8 +9,8 @@ ENV PATH=${GRADLE_HOME}/bin:${PATH} COPY ./server/website/requirements.txt / RUN apt-get update \ - && apt-get install -y python3.6 python3-pip python3-tk \ - mysql-client libmysqlclient-dev python-mysqldb \ + && apt-get install -y python3.6 python3-pip mysql-client \ + libmysqlclient-dev python-mysqldb postgresql-client \ openjdk-11-jdk git unzip wget curl \ && wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \ && unzip ${GRADLE_VERSION}-bin.zip -d /opt \ diff --git a/docker/createadmin.py b/docker/createadmin.py deleted file mode 100644 index 197b2ec..0000000 --- a/docker/createadmin.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python - -import os -import django - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "website.settings") -django.setup() - -from django.contrib.auth.models import User - -username = os.environ.get('ADMIN_USER', 'admin') -password = os.environ.get('ADMIN_PASSWORD') -email = os.environ.get('ADMIN_EMAIL', 'admin@example.com') - -if password: - if not User.objects.filter(username=username).exists(): - print(f"Creating '{username}' user...") - User.objects.create_superuser(username=username, - password=password, - email=email) - print(f"'{username}' user created!") - else: - print(f"'{username}' user already exists! Setting '{username}' password") - u = User.objects.get(username=username) - u.set_password(password) - u.save() diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 56a0d1c..c121c00 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -29,7 +29,6 @@ services: environment: DEBUG: 'True' ADMIN_PASSWORD: 'changeme' - ADMIN_EMAIL: 'admin@example.com' MYSQL_USER: 'root' MYSQL_PASSWORD: 'ottertune' MYSQL_HOST: 'mysql' diff --git a/docker/start.sh b/docker/start.sh index 5c4f3eb..9e5ac35 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -6,9 +6,7 @@ ## Needs a connection to a DB so migrations go here python3 manage.py makemigrations website python3 manage.py migrate -python3 createadmin.py +python3 manage.py createuser admin $ADMIN_PASSWORD --superuser -python3 manage.py celery worker --loglevel=info --pool=threads & -python3 manage.py celerybeat --verbosity=2 --loglevel=info & +python3 manage.py startcelery python3 manage.py runserver 0.0.0.0:8000 - diff --git a/script/validators/source_validator.py b/script/validators/source_validator.py index a70f29b..15b99c0 100644 --- a/script/validators/source_validator.py +++ b/script/validators/source_validator.py @@ -75,9 +75,6 @@ EXCLUDE_DIRECTORIES = [ EXCLUDE_FILES = [ # Django-generated files os.path.join(OTTERTUNE_DIR, 'server/website/manage.py'), - - # Docker files - os.path.join(OTTERTUNE_DIR, 'server/website/createadmin.py'), ] CHECKSTYLE_JAR_PATH = os.path.join(OTTERTUNE_DIR, diff --git a/server/analysis/cluster.py b/server/analysis/cluster.py index 39a7594..f9033f5 100644 --- a/server/analysis/cluster.py +++ b/server/analysis/cluster.py @@ -3,11 +3,6 @@ # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # -''' -Created on Jul 4, 2016 - -@author: dva -''' from abc import ABCMeta, abstractproperty from collections import OrderedDict @@ -15,7 +10,6 @@ import os import json import copy import numpy as np -import matplotlib.pyplot as plt from scipy.spatial.distance import cdist from sklearn.metrics import silhouette_score @@ -274,46 +268,6 @@ class KMeansClusters(ModelBase): return self - def save(self, savedir): - """Saves the KMeans model results - - Parameters - ---------- - savedir : string - Path to the directory to save the results in. - """ - if self.cluster_map_ is None: - raise Exception("No models have been fitted yet!") - - cluster_map = OrderedDict() - inertias = [] - for K, model in sorted(self.cluster_map_.items()): - cluster_map[K] = { - "cluster_inertia": model.cluster_inertia_, - "cluster_labels": model.cluster_labels_, - "cluster_centers": model.cluster_centers_, - } - inertias.append(model.cluster_inertia_) - - # Save sum of squares plot (elbow curve) - fig = plt.figure() - plt.plot(list(cluster_map.keys()), inertias, '--o') - plt.xlabel("Number of clusters (K)") - plt.ylabel("Within sum of squares W_k") - plt.title("Within Sum of Squares vs. Number of Clusters") - fig.canvas.set_window_title(os.path.basename(savedir)) - savepath = os.path.join(savedir, "kmeans_sum_of_squares.pdf") - plt.savefig(savepath, bbox_inches="tight") - plt.close() - - # save cluster memberships - for K in range(self.min_cluster_, self.max_cluster_ + 1): - savepath = os.path.join(savedir, - "memberships_{}-clusters.json".format(K)) - members = self.cluster_map_[K].get_memberships() - with open(savepath, "w") as f: - f.write(members) - class KSelection(ModelBase, metaclass=ABCMeta): """KSelection: @@ -529,38 +483,6 @@ class GapStatistic(KSelection): for i in range(K) for x in X[cluster_labels == i]]) - def save(self, savedir): - """Saves the estimation results of the optimal # of clusters. - - Parameters - ---------- - savedir : string - Path to the directory to save the results in. - """ - super(GapStatistic, self).save(savedir) - - # Plot the calculated gap - gaps = self.log_wkbs_ - self.log_wks_ - fig = plt.figure() - plt.plot(self.clusters_, gaps, '--o') - plt.title("Gap vs. Number of Clusters") - plt.xlabel("Number of clusters (K)") - plt.ylabel("gap_K") - fig.canvas.set_window_title(os.path.basename(savedir)) - plt.savefig(os.path.join(savedir, self.name_ + ".pdf"), bbox_inches="tight") - plt.close() - - # Plot the gap statistic - fig = plt.figure() - plt.bar(self.clusters_, self.khats_) - plt.title("Gap Statistic vs. Number of Clusters") - plt.xlabel("Number of clusters (K)") - plt.ylabel("gap(K)-(gap(K+1)-s(K+1))") - fig.canvas.set_window_title(os.path.basename(savedir)) - plt.savefig(os.path.join(savedir, self.name_ + "_final.pdf"), - bbox_inches="tight") - plt.close() - class DetK(KSelection): """DetK: @@ -649,27 +571,6 @@ class DetK(KSelection): self.fs_ = fs return self - def save(self, savedir): - """Saves the estimation results of the optimal # of clusters. - - Parameters - ---------- - savedir : string - Path to the directory to save the results in. - """ - super(DetK, self).save(savedir) - - # Plot the evaluation function - fig = plt.figure() - plt.plot(self.clusters_, self.fs_, '--o') - plt.xlabel("Number of clusters (K)") - plt.ylabel("Evaluation function (F_k)") - plt.title("Evaluation Function vs. Number of Clusters") - fig.canvas.set_window_title(os.path.basename(savedir)) - savepath = os.path.join(savedir, self.name_ + "_eval_function.pdf") - plt.savefig(savepath, bbox_inches="tight") - plt.close() - class Silhouette(KSelection): """Det: @@ -746,27 +647,6 @@ class Silhouette(KSelection): self.scores_ = scores return self - def save(self, savedir): - """Saves the estimation results of the optimal # of clusters. - - Parameters - ---------- - savedir : string - Path to the directory to save the results in. - """ - super(Silhouette, self).save(savedir) - - # Plot the evaluation function - fig = plt.figure() - plt.plot(self.clusters_, self.scores_, '--o') - plt.xlabel("Number of clusters (K)") - plt.ylabel("Silhouette scores") - plt.title("Silhouette Scores vs. Number of Clusters") - fig.canvas.set_window_title(os.path.basename(savedir)) - savepath = os.path.join(savedir, self.name_ + "_eval_function.pdf") - plt.savefig(savepath, bbox_inches="tight") - plt.close() - def create_kselection_model(model_name): """Constructs the KSelection model object with the given name diff --git a/server/website/requirements.txt b/server/website/requirements.txt index b428a2a..432d875 100644 --- a/server/website/requirements.txt +++ b/server/website/requirements.txt @@ -6,20 +6,19 @@ django-debug-toolbar==1.5 django-db-logger>=0.1.7 django-request-logging==0.4.6 mock==2.0.0 -Fabric3==1.13.1.post1 -hurry.filesize==0.9 -matplotlib==2.0.0 +Fabric3>=1.13.1.post1 +hurry.filesize>=0.9 numpy==1.13.1 requests==2.18.4 pycodestyle==2.3.1 astroid==1.5.1 -psycopg2>=2.5.4 +psycopg2-binary>=2.5.4 pylint==1.5.2 -pyDOE==0.3.8 +pyDOE>=0.3.8 mysqlclient==1.3.12 scikit-learn==0.19.1 scipy==1.0.0 tensorflow==1.10 -threadpool==1.3.2 -torch==1.2.0 -torchvision==0.4.0 +threadpool>=1.3.2 +torch>=1.2.0 +torchvision>=0.4.0