Added postgresql client package to Dockerfiles and psycopg2-binary to requirements.txt, removed unused matplotlib code/package from analysis/cluster.py and requirements.txt, replaced commands in docker start.sh file with new management commands.
This commit is contained in:
Dana Van Aken
parent
e8f28ebef0
commit
0da1b724cc
|
|
@ -11,7 +11,6 @@ WORKDIR /app/server/website
|
|||
|
||||
COPY ./docker/credentials.py ./website/settings
|
||||
COPY ./docker/start.sh .
|
||||
COPY ./docker/createadmin.py .
|
||||
COPY ./docker/wait-for-it.sh .
|
||||
|
||||
RUN chmod +x ./*.sh
|
||||
|
|
|
|||
|
|
@ -8,12 +8,12 @@ ENV PATH=${GRADLE_HOME}/bin:${PATH}
|
|||
COPY ./server/website/requirements.txt /
|
||||
|
||||
RUN yum update -y \
|
||||
&& yum install -y mariadb mariadb-devel \
|
||||
&& yum install -y mariadb mariadb-devel postgresql \
|
||||
https://centos7.iuscommunity.org/ius-release.rpm \
|
||||
&& yum install -y gcc git MySQL-python openldap-devel \
|
||||
parallel python36u python36u-devel python36u-libs \
|
||||
python36u-pip python36u-tkinter rabbitmq-server \
|
||||
java-11-openjdk-devel wget which unzip curl \
|
||||
python36u-pip rabbitmq-server java-11-openjdk-devel \
|
||||
wget which unzip curl \
|
||||
&& yum -y autoremove \
|
||||
&& yum clean metadata \
|
||||
&& yum clean all \
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ ENV PATH=${GRADLE_HOME}/bin:${PATH}
|
|||
COPY ./server/website/requirements.txt /
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y python3.6 python3-pip python3-tk \
|
||||
mysql-client libmysqlclient-dev python-mysqldb \
|
||||
&& apt-get install -y python3.6 python3-pip mysql-client \
|
||||
libmysqlclient-dev python-mysqldb postgresql-client \
|
||||
openjdk-11-jdk git unzip wget curl \
|
||||
&& wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \
|
||||
&& unzip ${GRADLE_VERSION}-bin.zip -d /opt \
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
import django
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "website.settings")
|
||||
django.setup()
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
username = os.environ.get('ADMIN_USER', 'admin')
|
||||
password = os.environ.get('ADMIN_PASSWORD')
|
||||
email = os.environ.get('ADMIN_EMAIL', 'admin@example.com')
|
||||
|
||||
if password:
|
||||
if not User.objects.filter(username=username).exists():
|
||||
print(f"Creating '{username}' user...")
|
||||
User.objects.create_superuser(username=username,
|
||||
password=password,
|
||||
email=email)
|
||||
print(f"'{username}' user created!")
|
||||
else:
|
||||
print(f"'{username}' user already exists! Setting '{username}' password")
|
||||
u = User.objects.get(username=username)
|
||||
u.set_password(password)
|
||||
u.save()
|
||||
|
|
@ -29,7 +29,6 @@ services:
|
|||
environment:
|
||||
DEBUG: 'True'
|
||||
ADMIN_PASSWORD: 'changeme'
|
||||
ADMIN_EMAIL: 'admin@example.com'
|
||||
MYSQL_USER: 'root'
|
||||
MYSQL_PASSWORD: 'ottertune'
|
||||
MYSQL_HOST: 'mysql'
|
||||
|
|
|
|||
|
|
@ -6,9 +6,7 @@
|
|||
## Needs a connection to a DB so migrations go here
|
||||
python3 manage.py makemigrations website
|
||||
python3 manage.py migrate
|
||||
python3 createadmin.py
|
||||
python3 manage.py createuser admin $ADMIN_PASSWORD --superuser
|
||||
|
||||
python3 manage.py celery worker --loglevel=info --pool=threads &
|
||||
python3 manage.py celerybeat --verbosity=2 --loglevel=info &
|
||||
python3 manage.py startcelery
|
||||
python3 manage.py runserver 0.0.0.0:8000
|
||||
|
||||
|
|
|
|||
|
|
@ -75,9 +75,6 @@ EXCLUDE_DIRECTORIES = [
|
|||
EXCLUDE_FILES = [
|
||||
# Django-generated files
|
||||
os.path.join(OTTERTUNE_DIR, 'server/website/manage.py'),
|
||||
|
||||
# Docker files
|
||||
os.path.join(OTTERTUNE_DIR, 'server/website/createadmin.py'),
|
||||
]
|
||||
|
||||
CHECKSTYLE_JAR_PATH = os.path.join(OTTERTUNE_DIR,
|
||||
|
|
|
|||
|
|
@ -3,11 +3,6 @@
|
|||
#
|
||||
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||
#
|
||||
'''
|
||||
Created on Jul 4, 2016
|
||||
|
||||
@author: dva
|
||||
'''
|
||||
from abc import ABCMeta, abstractproperty
|
||||
from collections import OrderedDict
|
||||
|
||||
|
|
@ -15,7 +10,6 @@ import os
|
|||
import json
|
||||
import copy
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
from scipy.spatial.distance import cdist
|
||||
from sklearn.metrics import silhouette_score
|
||||
|
|
@ -274,46 +268,6 @@ class KMeansClusters(ModelBase):
|
|||
|
||||
return self
|
||||
|
||||
def save(self, savedir):
|
||||
"""Saves the KMeans model results
|
||||
|
||||
Parameters
|
||||
----------
|
||||
savedir : string
|
||||
Path to the directory to save the results in.
|
||||
"""
|
||||
if self.cluster_map_ is None:
|
||||
raise Exception("No models have been fitted yet!")
|
||||
|
||||
cluster_map = OrderedDict()
|
||||
inertias = []
|
||||
for K, model in sorted(self.cluster_map_.items()):
|
||||
cluster_map[K] = {
|
||||
"cluster_inertia": model.cluster_inertia_,
|
||||
"cluster_labels": model.cluster_labels_,
|
||||
"cluster_centers": model.cluster_centers_,
|
||||
}
|
||||
inertias.append(model.cluster_inertia_)
|
||||
|
||||
# Save sum of squares plot (elbow curve)
|
||||
fig = plt.figure()
|
||||
plt.plot(list(cluster_map.keys()), inertias, '--o')
|
||||
plt.xlabel("Number of clusters (K)")
|
||||
plt.ylabel("Within sum of squares W_k")
|
||||
plt.title("Within Sum of Squares vs. Number of Clusters")
|
||||
fig.canvas.set_window_title(os.path.basename(savedir))
|
||||
savepath = os.path.join(savedir, "kmeans_sum_of_squares.pdf")
|
||||
plt.savefig(savepath, bbox_inches="tight")
|
||||
plt.close()
|
||||
|
||||
# save cluster memberships
|
||||
for K in range(self.min_cluster_, self.max_cluster_ + 1):
|
||||
savepath = os.path.join(savedir,
|
||||
"memberships_{}-clusters.json".format(K))
|
||||
members = self.cluster_map_[K].get_memberships()
|
||||
with open(savepath, "w") as f:
|
||||
f.write(members)
|
||||
|
||||
|
||||
class KSelection(ModelBase, metaclass=ABCMeta):
|
||||
"""KSelection:
|
||||
|
|
@ -529,38 +483,6 @@ class GapStatistic(KSelection):
|
|||
for i in range(K)
|
||||
for x in X[cluster_labels == i]])
|
||||
|
||||
def save(self, savedir):
|
||||
"""Saves the estimation results of the optimal # of clusters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
savedir : string
|
||||
Path to the directory to save the results in.
|
||||
"""
|
||||
super(GapStatistic, self).save(savedir)
|
||||
|
||||
# Plot the calculated gap
|
||||
gaps = self.log_wkbs_ - self.log_wks_
|
||||
fig = plt.figure()
|
||||
plt.plot(self.clusters_, gaps, '--o')
|
||||
plt.title("Gap vs. Number of Clusters")
|
||||
plt.xlabel("Number of clusters (K)")
|
||||
plt.ylabel("gap_K")
|
||||
fig.canvas.set_window_title(os.path.basename(savedir))
|
||||
plt.savefig(os.path.join(savedir, self.name_ + ".pdf"), bbox_inches="tight")
|
||||
plt.close()
|
||||
|
||||
# Plot the gap statistic
|
||||
fig = plt.figure()
|
||||
plt.bar(self.clusters_, self.khats_)
|
||||
plt.title("Gap Statistic vs. Number of Clusters")
|
||||
plt.xlabel("Number of clusters (K)")
|
||||
plt.ylabel("gap(K)-(gap(K+1)-s(K+1))")
|
||||
fig.canvas.set_window_title(os.path.basename(savedir))
|
||||
plt.savefig(os.path.join(savedir, self.name_ + "_final.pdf"),
|
||||
bbox_inches="tight")
|
||||
plt.close()
|
||||
|
||||
|
||||
class DetK(KSelection):
|
||||
"""DetK:
|
||||
|
|
@ -649,27 +571,6 @@ class DetK(KSelection):
|
|||
self.fs_ = fs
|
||||
return self
|
||||
|
||||
def save(self, savedir):
|
||||
"""Saves the estimation results of the optimal # of clusters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
savedir : string
|
||||
Path to the directory to save the results in.
|
||||
"""
|
||||
super(DetK, self).save(savedir)
|
||||
|
||||
# Plot the evaluation function
|
||||
fig = plt.figure()
|
||||
plt.plot(self.clusters_, self.fs_, '--o')
|
||||
plt.xlabel("Number of clusters (K)")
|
||||
plt.ylabel("Evaluation function (F_k)")
|
||||
plt.title("Evaluation Function vs. Number of Clusters")
|
||||
fig.canvas.set_window_title(os.path.basename(savedir))
|
||||
savepath = os.path.join(savedir, self.name_ + "_eval_function.pdf")
|
||||
plt.savefig(savepath, bbox_inches="tight")
|
||||
plt.close()
|
||||
|
||||
|
||||
class Silhouette(KSelection):
|
||||
"""Det:
|
||||
|
|
@ -746,27 +647,6 @@ class Silhouette(KSelection):
|
|||
self.scores_ = scores
|
||||
return self
|
||||
|
||||
def save(self, savedir):
|
||||
"""Saves the estimation results of the optimal # of clusters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
savedir : string
|
||||
Path to the directory to save the results in.
|
||||
"""
|
||||
super(Silhouette, self).save(savedir)
|
||||
|
||||
# Plot the evaluation function
|
||||
fig = plt.figure()
|
||||
plt.plot(self.clusters_, self.scores_, '--o')
|
||||
plt.xlabel("Number of clusters (K)")
|
||||
plt.ylabel("Silhouette scores")
|
||||
plt.title("Silhouette Scores vs. Number of Clusters")
|
||||
fig.canvas.set_window_title(os.path.basename(savedir))
|
||||
savepath = os.path.join(savedir, self.name_ + "_eval_function.pdf")
|
||||
plt.savefig(savepath, bbox_inches="tight")
|
||||
plt.close()
|
||||
|
||||
|
||||
def create_kselection_model(model_name):
|
||||
"""Constructs the KSelection model object with the given name
|
||||
|
|
|
|||
|
|
@ -6,20 +6,19 @@ django-debug-toolbar==1.5
|
|||
django-db-logger>=0.1.7
|
||||
django-request-logging==0.4.6
|
||||
mock==2.0.0
|
||||
Fabric3==1.13.1.post1
|
||||
hurry.filesize==0.9
|
||||
matplotlib==2.0.0
|
||||
Fabric3>=1.13.1.post1
|
||||
hurry.filesize>=0.9
|
||||
numpy==1.13.1
|
||||
requests==2.18.4
|
||||
pycodestyle==2.3.1
|
||||
astroid==1.5.1
|
||||
psycopg2>=2.5.4
|
||||
psycopg2-binary>=2.5.4
|
||||
pylint==1.5.2
|
||||
pyDOE==0.3.8
|
||||
pyDOE>=0.3.8
|
||||
mysqlclient==1.3.12
|
||||
scikit-learn==0.19.1
|
||||
scipy==1.0.0
|
||||
tensorflow==1.10
|
||||
threadpool==1.3.2
|
||||
torch==1.2.0
|
||||
torchvision==0.4.0
|
||||
threadpool>=1.3.2
|
||||
torch>=1.2.0
|
||||
torchvision>=0.4.0
|
||||
|
|
|
|||
Loading…
Reference in New Issue