diff --git a/.travis.yml b/.travis.yml index f471262..f909d08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,60 +1,128 @@ dist: bionic -sudo: required -services: - - docker +language: python +python: +- 3.6 env: global: - - DOCKER_COMPOSE_VERSION=1.24.1 - matrix: - - DOCKER_OS=ubuntu-18.04 - - DOCKER_OS=centos-7 + - DB_NAME=ottertune + - ADMIN_PASSWORD=changeme + - ROOT=$TRAVIS_BUILD_DIR + - WEB=$ROOT/server/website + - CONTROLLER=$ROOT/client/controller + - DRIVER=$ROOT/client/driver -before_install: - # Install latest versions of docker and docker-compose - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - sudo apt-get update - - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - - sudo rm /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin - - docker --version - - docker-compose --version +matrix: + include: + - name: Docker + env: + - BUILD=docker + - BACKEND=postgresql + - secure: PZAe5iDaipqsfzQ0/mjFdcQhEMQ0alI3Ap+hWR0X9uWfK8dH6XU/RJLzJXj2o+kpGiCGlj7DJ9uuJy6+/75hq2f7vcs+6JJInPQHUlkj2TPDYWNWaFw0XvIAvV9EA73l4kDy26m/+JiSaF1GsyVD5wdJpVRmw6o7liGfYlQX3fY8kQ4p51+5ufb9QCAselrE70302fTyX4GqxaT1xC2L5fzzIcuKllFVlRILc+04gldiq8u0EAcUO2ovYiW/3+qGPkDcdwd2j3UC1tEGQ+x+Q/7g5JqBHGz6iMy8omH77p3rk4aKV/HCRTZLQLe/Kcd5UiSrQvSIHl5jfIYbzBx7//Ub64c7TlgX+UckK+GPYiwr2N6VmiducFnSjmm3RSW959m1M+M+bqIPbS510zqnIs28xjqIgnWvtpA8mgKdRAtQyDFMYcLc7GQyw0alUo1CQuTf4+Ter+78vt3vYVlCD8lNoMG78xBaLg//pgNQ8sdGWGbUsAYuxxrqI056Ayw16LgrcakL0JWLR/7p+HmCtH2ZT1nQEZJJHSUP3ekibGrE6wgNVpt3DAOwQUG/JiceLn4uVoeljpT+neCoM29wvf4M7JGWg+8SPtg4DHJIAocOVt5EDi8/CtoIa3WJLD2AMI4ladd4PKNzHT4tsYdE2V9d+xkuhXv1/OakAJedtEc= + - secure: RN4SGagKXgn8gEk9pG2CdM5a/rrBPqkD1nONLcvcqjYzlbniKmhmKT+OxhjVw7Qtoy/RACr4zpIupIjA6COpndSd8/m6WBsWnYD0dt6cDBDLFgHyIh9XNi6/bhNgqPksY9XrENfmeijTENiqpuXeXCuHdaVF0quwvGBxDAIWFNOpPwceAJrwRv9Yb/7w5rmlotSVesScPhwV92Rmu3NK+r/FYANSOFXoBFAYctJSi7dj4A/y3lOFmzE5Y12+Kf6kHuoP+FZO/S/zL1OAPSzedqctRcDpVctEt4AqA7cpGe+Qvate2Hnj63YNlykQD+zlCFHLrqCCKZecdNe+SQVcYqgNk78SX7Nvhfr/xH5lq8HjovqjrfQFc4pVDryomtMeTb7wilN9wsYu6zCneMDHF57hfSKTmiqhz0utEqUiPYwxhkU5uXsXO8VA3aOUlIhM3QPnB6tvkB1h0+qnNzU/lJk6LXy9KkNy9HI8ab+58FSEhj4Uyo9AaBGOB2bEu/d8BsdW1qjGYTI6t4tWZQ8lduOw3xqT5hHhwGB9bz5DomfbECanOGShR3NeMJJGYWETEWek2WK9viP3pllbTFnU3yXfv4ssb00eD2tK+7pyWuP+hL7S1yvrMs5qYRFAEfJhuzTEjtN2uAKumCvU/eh0JjiAPK6XhdT49K1Cd+HC1h4= + - DOCKER_REPO="${DOCKER_USER}/ottertune" + services: + - docker + before_install: + - docker --version + - docker-compose --version + install: [] + before_script: + - sudo service mysql stop || true + - sudo service postgresql stop || true + - cd $ROOT/docker + - docker-compose build + - dcfile_mysql=docker-compose.mysql.yml + - dcfile_postgres=docker-compose.postgresql.yml + - BACKEND=mysql WEB_ENTRYPOINT="''" sh create-docker-compose.sh $dcfile_mysql + - BACKEND=postgresql WEB_ENTRYPOINT="''" sh create-docker-compose.sh $dcfile_postgres + script: + - docker-compose -f $dcfile_postgres run --workdir="/app" --no-deps --rm web bash -c "pip3 freeze" + - docker-compose -f $dcfile_postgres run --workdir="/app/controller" --no-deps --rm driver gradle build + - docker-compose -f $dcfile_postgres run --workdir="/app" --no-deps --rm web bash -c "python3 -m unittest discover -s analysis/tests -v" + - docker-compose -f $dcfile_postgres run --rm web bash -c "./wait-for-it.sh && python3 manage.py makemigrations website && python3 manage.py test --noinput -v 2" + - docker-compose -f $dcfile_postgres rm -f -s -v + - docker-compose -f $dcfile_mysql run --rm web bash -c "./wait-for-it.sh && python3 manage.py makemigrations website && python3 manage.py test --noinput -v 2" + - docker-compose -f $dcfile_mysql rm -f -s -v + before_deploy: + - echo "$DOCKER_PASSWD" | docker login -u "$DOCKER_USER" --password-stdin + - docker tag ottertune-web "${DOCKER_REPO}:web" + - docker tag ottertune-driver "${DOCKER_REPO}:driver" + deploy: + provider: script + script: docker push "${DOCKER_REPO}:web" && docker push "${DOCKER_REPO}:driver" + on: + branch: master + - name: Tests (MySQL v5.7) + env: + - BUILD=tests + - BACKEND=mysql + services: + - mysql + addons: + apt: + update: true + packages: + - mysql-server + - python-mysqldb + - rabbitmq-server + - openjdk-11-jdk + - gradle + - checkstyle + before_install: + - mysql -e "CREATE DATABASE IF NOT EXISTS ${DB_NAME}" + - mysql -e "CREATE DATABASE IF NOT EXISTS test_${DB_NAME}" + - sed -i '/psycopg2/d' $WEB/requirements.txt + + - name: Tests (PostgreSQL v9.6) + env: + - BUILD=unittests + - BACKEND=postgresql + addons: + postgresql: "9.6" + apt: + update: true + packages: + - rabbitmq-server + - openjdk-11-jdk + - gradle + - checkstyle + before_install: + - psql -U postgres -c "CREATE DATABASE ${DB_NAME}" + - psql -U postgres -c "CREATE DATABASE test_${DB_NAME}" + - sed -i '/mysqlclient/d' $WEB/requirements.txt + +install: +- pip install codecov -r $WEB/requirements.txt +- pip freeze before_script: - # Set correct dockerfile - - cd $TRAVIS_BUILD_DIR/docker - - sed -i "s|Dockerfile\.base-.*|Dockerfile\.base-$DOCKER_OS|" docker-compose.yml - - sed -i "s|Dockerfile\.base-.*|Dockerfile\.base-$DOCKER_OS|" docker-compose.test.yml - - sudo service mysql stop || true - +- cd $WEB +- sed -i "s|\('celery', 'db.*$\)|'console', \1|" website/settings/common.py +- cp $ROOT/docker/credentials.py website/settings +- cat website/settings/credentials.py +- python manage.py makemigrations +- python manage.py migrate +- python manage.py startcelery +- python manage.py createuser admin $ADMIN_PASSWORD --superuser script: - - env | grep TRAVIS | sort - - cd $TRAVIS_BUILD_DIR/docker - # Build master images and run the webserver - - docker-compose build - - docker-compose up -d - - docker logs ottertune - - docker-compose rm -f -s -v - # Build test images and run tests - - ci_env=`bash <(curl -s https://codecov.io/env)` - - docker-compose -f docker-compose.test.yml build - - docker-compose -f docker-compose.test.yml up -d - - docker-compose -f docker-compose.test.yml run --workdir="/app/client/controller" --rm test gradle build - - docker-compose -f docker-compose.test.yml run --workdir="/app/server" --rm $ci_env test bash -c "coverage run --omit=\"*/tests/*\" -m unittest discover -s analysis/tests -v && (codecov -F analysis || (sleep 5 && codecov -F analysis) || (sleep 5 && codecov -F analysis))" - - docker-compose -f docker-compose.test.yml run --workdir="/app/server/website" --rm $ci_env test bash -c "./wait-for-it.sh && python3 manage.py makemigrations website && coverage run --source=website manage.py test --noinput -v 2 && (codecov -F website || (sleep 5 && codecov -F website) || (sleep 5 && codecov -F website))" - - docker-compose -f docker-compose.test.yml run --workdir="/app/server/website" --rm test bash -c "./start-test.sh" - # Only run the linter once on ubuntu 18.04 - - if [ $DOCKER_OS == ubuntu-18.04 ]; then - docker-compose -f docker-compose.test.yml run --workdir="/app" --rm -e TRAVIS_COMMIT_RANGE=$TRAVIS_COMMIT_RANGE test bash -c "git reset --soft ${TRAVIS_COMMIT_RANGE%...*} && git status && git log | head -n 1 && git lint"; - fi - -after_script: - # Cleanup docker containers, images, and volumes - - docker-compose rm -f -s -v - - docker system prune -a -f - - docker volume prune -f - +- cd $ROOT/server && coverage run --omit="*/tests/*" -m unittest discover -s analysis/tests -v +- cd $WEB && coverage run manage.py test --noinput -v 2 +- cd $CONTROLLER && gradle build +- cd $WEB +- python manage.py runserver 0.0.0.0:8000 & +- sleep 10 && cd $DRIVER && fab integration_tests +- cd $ROOT && git reset --soft ${TRAVIS_COMMIT_RANGE%...*} && git status && git + log | head -n 1 && git lint +after_success: +- > + codecov -F analysis -f "${ROOT}/server/.coverage" || + (sleep 5 && codecov -F analysis -f "${ROOT}/server/.coverage") || + (sleep 5 && codecov -F analysis -f "${ROOT}/server/.coverage") && + echo "Codecov did not collect coverage reports" +- > + codecov -F website -f "${WEB}/.coverage" || + (sleep 5 && codecov -F website -f "${WEB}/.coverage") || + (sleep 5 && codecov -F website -f "${WEB}/.coverage") && + echo "Codecov did not collect coverage reports" diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index af1c7d1..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM ottertune-base - -ENV DJANGO_SETTINGS_MODULE=website.settings -ENV C_FORCE_ROOT=true - -RUN mkdir -p /app - -COPY . /app - -WORKDIR /app/server/website - -COPY ./docker/credentials.py ./website/settings -COPY ./docker/start.sh . -COPY ./docker/wait-for-it.sh . - -RUN chmod +x ./*.sh - -RUN sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \ - && mv tmp ./website/settings/common.py - -ENTRYPOINT ["./start.sh"] - diff --git a/docker/Dockerfile.base-centos-7 b/docker/Dockerfile.base-centos-7 deleted file mode 100644 index 9fc22e8..0000000 --- a/docker/Dockerfile.base-centos-7 +++ /dev/null @@ -1,30 +0,0 @@ -FROM centos:7 - -ARG GRADLE_VERSION=gradle-5.5.1 - -ENV GRADLE_HOME=/opt/${GRADLE_VERSION} -ENV PATH=${GRADLE_HOME}/bin:${PATH} - -COPY ./server/website/requirements.txt / - -RUN yum update -y \ - && yum install -y mariadb mariadb-devel postgresql \ - https://centos7.iuscommunity.org/ius-release.rpm \ - && yum install -y gcc git MySQL-python openldap-devel \ - parallel python36u python36u-devel python36u-libs \ - python36u-pip rabbitmq-server java-11-openjdk-devel \ - wget which unzip curl \ - && yum -y autoremove \ - && yum clean metadata \ - && yum clean all \ - && ln -sf `which python3.6` /usr/bin/python3 \ - && ln -sf `which pip3.6` /usr/bin/pip3 \ - && wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \ - && unzip ${GRADLE_VERSION}-bin.zip -d /opt \ - && rm ${GRADLE_VERSION}-bin.zip \ - && python3 --version \ - && pip3 --version \ - && javac --version \ - && gradle --version \ - && pip3 install -r /requirements.txt \ - && rm /requirements.txt diff --git a/docker/Dockerfile.base-ubuntu-18.04 b/docker/Dockerfile.base-ubuntu-18.04 deleted file mode 100644 index 2926149..0000000 --- a/docker/Dockerfile.base-ubuntu-18.04 +++ /dev/null @@ -1,24 +0,0 @@ -FROM ubuntu:18.04 - -ARG GRADLE_VERSION=gradle-5.5.1 - -ENV DEBIAN_FRONTEND=noninteractive -ENV GRADLE_HOME=/opt/${GRADLE_VERSION} -ENV PATH=${GRADLE_HOME}/bin:${PATH} - -COPY ./server/website/requirements.txt / - -RUN apt-get update \ - && apt-get install -y python3.6 python3-pip libssl-dev \ - mysql-client libmysqlclient-dev python-mysqldb postgresql-client \ - openjdk-11-jdk checkstyle git unzip wget curl \ - && wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \ - && unzip ${GRADLE_VERSION}-bin.zip -d /opt \ - && rm ${GRADLE_VERSION}-bin.zip \ - && python3 --version \ - && pip3 --version \ - && javac --version \ - && gradle --version \ - && pip3 install -r /requirements.txt \ - && rm /requirements.txt - diff --git a/docker/Dockerfile.driver b/docker/Dockerfile.driver new file mode 100644 index 0000000..299e375 --- /dev/null +++ b/docker/Dockerfile.driver @@ -0,0 +1,19 @@ +FROM ubuntu:18.04 + +ARG GRADLE_VERSION=gradle-5.5.1 + +ENV DEBIAN_FRONTEND=noninteractive +ENV GRADLE_HOME=/opt/${GRADLE_VERSION} +ENV PATH=${GRADLE_HOME}/bin:${PATH} + +COPY ./docker/install.sh ./server/website/requirements.txt / +WORKDIR / + +RUN mkdir -p /app \ + && chmod +x install.sh \ + && sh install.sh driver + +COPY ./client /app + +WORKDIR /app/driver + diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test deleted file mode 100644 index cbe99f4..0000000 --- a/docker/Dockerfile.test +++ /dev/null @@ -1,22 +0,0 @@ -FROM ottertune-base - -ENV DJANGO_SETTINGS_MODULE=website.settings -ENV C_FORCE_ROOT=true - -RUN mkdir -p /app - -COPY . /app - -WORKDIR /app/server/website - -RUN pip3 install codecov - -COPY ./docker/credentials.py ./website/settings -COPY ./docker/wait-for-it.sh . -COPY ./docker/start-test.sh . - -RUN chmod +x ./*.sh - -RUN sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \ - && mv tmp ./website/settings/common.py - diff --git a/docker/Dockerfile.web b/docker/Dockerfile.web new file mode 100644 index 0000000..d88e7bc --- /dev/null +++ b/docker/Dockerfile.web @@ -0,0 +1,27 @@ +FROM ubuntu:18.04 + +ENV DEBIAN_FRONTEND=noninteractive + +COPY ./docker/install.sh ./server/website/requirements.txt / +WORKDIR / + +RUN mkdir -p /app \ + && chmod +x install.sh \ + && sh install.sh web + +COPY ./server /app + +WORKDIR /app/website + +COPY ./docker/credentials.py ./website/settings +COPY ./docker/start.sh ./docker/start-test.sh ./docker/wait-for-it.sh ./ + +RUN chmod +x ./*.sh \ + && sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \ + && mv tmp ./website/settings/common.py + +ENV DJANGO_SETTINGS_MODULE=website.settings +ENV C_FORCE_ROOT=true + +ENTRYPOINT ["./start.sh"] + diff --git a/docker/create-docker-compose.sh b/docker/create-docker-compose.sh new file mode 100755 index 0000000..514a14e --- /dev/null +++ b/docker/create-docker-compose.sh @@ -0,0 +1,151 @@ +#!/bin/bash + + +if [ -z "$BACKEND" ] +then + echo "Variable 'BACKEND' must be set." + exit 1 +fi + +DEBUG="${DEBUG:-true}" +ADMIN_PASSWORD="${ADMIN_PASSWORD:-changeme}" +DB_NAME="${DB_NAME:-ottertune}" +DB_PASSWORD="${DB_PASSWORD:-ottertune}" + +if [ "$BACKEND" = "mysql" ]; then + DB_USER="${DB_USER:-root}" + DB_PORT="${DB_PORT:-3306}" +else + DB_USER="${DB_USER:-postgres}" + DB_PORT="${DB_PORT:-5432}" +fi + +WEB_ENTRYPOINT="${WEB_ENTRYPOINT:-start.sh}" + +file="$(test -z "$1" && echo "docker-compose.$BACKEND.yml" || echo "$1")" + + +cat > $file <<- EOM +version: "3" +services: + + web: + build: + context: ../ + dockerfile: ./docker/Dockerfile.web + image: ottertune-web + container_name: web + expose: + - "8000" + ports: + - "8000:8000" + links: + - backend + - rabbitmq + depends_on: + - backend + - rabbitmq + environment: + DEBUG: '$DEBUG' + ADMIN_PASSWORD: '$ADMIN_PASSWORD' + BACKEND: '$BACKEND' + DB_NAME: '$DB_NAME' + DB_USER: '$DB_USER' + DB_PASSWORD: '$DB_PASSWORD' + DB_HOST: 'backend' + DB_PORT: '$DB_PORT' + MAX_DB_CONN_ATTEMPTS: 30 + working_dir: /app/website + entrypoint: $WEB_ENTRYPOINT + labels: + NAME: "ottertune-web" + networks: + - ottertune-net + + driver: + build: + context: ../ + dockerfile: ./docker/Dockerfile.driver + image: ottertune-driver + container_name: driver + depends_on: + - web + environment: + DEBUG: '$DEBUG' + working_dir: /app/driver + labels: + NAME: "ottertune-driver" + networks: + - ottertune-net + + rabbitmq: + image: "rabbitmq:3-management" + container_name: rabbitmq + restart: always + hostname: "rabbitmq" + environment: + RABBITMQ_DEFAULT_USER: "guest" + RABBITMQ_DEFAULT_PASS: "guest" + RABBITMQ_DEFAULT_VHOST: "/" + expose: + - "15672" + - "5672" + ports: + - "15673:15672" + - "5673:5672" + labels: + NAME: "rabbitmq" + networks: + - ottertune-net + +EOM + + +cat >> $file <<- EOM + backend: + container_name: backend + restart: always +EOM + + +if [ "$BACKEND" = "mysql" ]; then + +cat >> $file <<- EOM + image: mysql:5.7 + environment: + MYSQL_USER: '$DB_USER' + MYSQL_ROOT_PASSWORD: '$DB_PASSWORD' + MYSQL_PASSWORD: '$DB_PASSWORD' + MYSQL_DATABASE: '$DB_NAME' + expose: + - "3306" + ports: + - "3306:3306" +EOM +else +cat >> $file <<- EOM + image: postgres:9.6 + environment: + POSTGRES_PASSWORD: '$DB_PASSWORD' + POSTGRES_USER: '$DB_USER' + POSTGRES_DB: '$DB_NAME' + expose: + - "5432" + ports: + - "5432:5432" +EOM +fi + +cat >> $file <<- EOM + labels: + NAME: "ottertune-backend" + networks: + - ottertune-net + +networks: + ottertune-net: + driver: bridge +EOM + +echo "Saved docker-compose file to '$file'." + diff --git a/docker/credentials.py b/docker/credentials.py index dfb4bed..3c1b934 100644 --- a/docker/credentials.py +++ b/docker/credentials.py @@ -1,24 +1,43 @@ -import secrets +import json +import random +import string from os import environ as env -db_user = env.get('MYSQL_USER') -db_pwd = env.get('MYSQL_PASSWORD') -db_host = env.get('MYSQL_HOST') -db_port = env.get('MYSQL_PORT', '3306') -debug = env.get('DEBUG') +debug = env.get('DEBUG', 'true').lower() == 'true' +backend = env.get('BACKEND', 'mysql') +db_name = env.get('DB_NAME', 'ottertune') +db_host = env.get('DB_HOST', 'localhost') +db_pwd = env.get('DB_PASSWORD', '') -SECRET_KEY = secrets.token_hex(16) +if backend == 'mysql': + default_user = 'root' + default_port = '3306' + default_opts = { + 'init_command': "SET sql_mode='STRICT_TRANS_TABLES',innodb_strict_mode=1", + } +else: + default_user = 'postgres' + default_port = '5432' + default_opts = {} + +db_user = env.get('DB_USER', default_user) +db_port = env.get('DB_PORT', default_port) +db_opts = env.get('DB_OPTS', default_opts) +if isinstance(db_opts, str): + db_opts = json.loads(db_opts) if db_opts else {} + +SECRET_KEY = ''.join(random.choice(string.hexdigits) for _ in range(16)) DATABASES = { - 'default': {'ENGINE': 'django.db.backends.mysql', - 'NAME': 'ottertune', + 'default': {'ENGINE': 'django.db.backends.' + backend, + 'NAME': db_name, 'USER': db_user, 'PASSWORD': db_pwd, 'HOST': db_host, 'PORT': db_port, - 'OPTIONS': {'init_command': 'SET sql_mode=\'STRICT_TRANS_TABLES\',innodb_strict_mode=1',} + 'OPTIONS': db_opts, } } -DEBUG = True +DEBUG = debug ADMINS = () MANAGERS = ADMINS -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = ['*'] diff --git a/docker/docker-compose.test.yml b/docker/docker-compose.test.yml deleted file mode 100644 index 1fd635a..0000000 --- a/docker/docker-compose.test.yml +++ /dev/null @@ -1,85 +0,0 @@ -version: "3" -services: - base: - build: - context: ../ - dockerfile: ./docker/Dockerfile.base-ubuntu-18.04 - image: ottertune-base - container_name: ottertune-base - labels: - NAME: "ottertune-base" - - test: - build: - context: ../ - dockerfile: ./docker/Dockerfile.test - image: ottertune-test - container_name: ottertune-test - expose: - - "8000" - ports: - - "8000:8000" - links: - - mysql - - rabbitmq - depends_on: - - mysql - - rabbitmq - environment: - DEBUG: 'True' - MYSQL_USER: 'root' - MYSQL_PASSWORD: 'ottertune' - MYSQL_HOST: 'mysql' - MAX_DB_CONN_ATTEMPTS: 15 - labels: - NAME: "ottertune-test" - volumes: - - media_data:/app/server/website/media - networks: - - ottertune-net - - mysql: - image: mysql:5.7 - container_name: mysql - restart: always - environment: - MYSQL_ROOT_PASSWORD: 'ottertune' - MYSQL_PASSWORD: 'ottertune' - MYSQL_DATABASE: 'ottertune' - expose: - - "3306" - ports: - - "3306:3306" - labels: - NAME: "mysql" - volumes: - - mysql_data:/var/lib/mysql - networks: - - ottertune-net - - rabbitmq: - image: "rabbitmq:3-management" - container_name: rabbitmq - restart: always - hostname: "rabbitmq" - environment: - RABBITMQ_DEFAULT_USER: "guest" - RABBITMQ_DEFAULT_PASS: "guest" - RABBITMQ_DEFAULT_VHOST: "/" - expose: - - "15672" - - "5672" - ports: - - "15672:15672" - - "5672:5672" - labels: - NAME: "rabbitmq" - networks: - - ottertune-net -volumes: - mysql_data: - media_data: -networks: - ottertune-net: - driver: bridge - diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c121c00..41367f0 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,48 +1,59 @@ version: "3" services: - base: - build: - context: ../ - dockerfile: ./docker/Dockerfile.base-ubuntu-18.04 - image: ottertune-base - container_name: ottertune-base - labels: - NAME: "ottertune-base" web: build: context: ../ - dockerfile: ./docker/Dockerfile - image: ottertune - container_name: ottertune + dockerfile: ./docker/Dockerfile.web + image: ottertune-web + container_name: web expose: - "8000" ports: - "8000:8000" links: - - mysql + - backend - rabbitmq depends_on: - - base - - mysql + - backend - rabbitmq environment: - DEBUG: 'True' + DEBUG: 'true' ADMIN_PASSWORD: 'changeme' - MYSQL_USER: 'root' - MYSQL_PASSWORD: 'ottertune' - MYSQL_HOST: 'mysql' - MAX_DB_CONN_ATTEMPTS: 15 + BACKEND: 'mysql' + DB_NAME: 'ottertune' + DB_USER: 'root' + DB_PASSWORD: 'ottertune' + DB_HOST: 'backend' + DB_PORT: '3306' + DB_OPTS: '{}' + MAX_DB_CONN_ATTEMPTS: 30 + working_dir: /app/website + entrypoint: ./start.sh labels: - NAME: "ottertune" - volumes: - - media_data:/app/server/website/media + NAME: "ottertune-web" networks: - ottertune-net - mysql: + driver: + build: + context: ../ + dockerfile: ./docker/Dockerfile.driver + image: ottertune-driver + container_name: driver + depends_on: + - web + environment: + DEBUG: 'true' + working_dir: /app/driver + labels: + NAME: "ottertune-driver" + networks: + - ottertune-net + + backend: image: mysql:5.7 - container_name: mysql + container_name: backend restart: always environment: MYSQL_ROOT_PASSWORD: 'ottertune' @@ -53,7 +64,7 @@ services: ports: - "3306:3306" labels: - NAME: "mysql" + NAME: "ottertune-backend" volumes: - mysql_data:/var/lib/mysql networks: @@ -72,15 +83,14 @@ services: - "15672" - "5672" ports: - - "15672:15672" - - "5672:5672" + - "15673:15672" + - "5673:5672" labels: NAME: "rabbitmq" networks: - ottertune-net volumes: mysql_data: - media_data: networks: ottertune-net: driver: bridge diff --git a/docker/install.sh b/docker/install.sh new file mode 100644 index 0000000..2b33702 --- /dev/null +++ b/docker/install.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +service="$1" + + +echo "" +if [ -z "$service" ] || ([ "$service" != "web" ] && [ "$service" != "driver" ]) +then + echo "Invalid value for service: '$service'" + echo "" + echo "Usage: $0 [web|driver]" + exit 1 +fi + +echo "" +echo "-=------------------------------------------------------" +echo " Starting installation for service '$service'..." +echo "-=------------------------------------------------------" + +if [ "$DEBUG" = true ] +then + echo "" + echo "Environment Variables:" + echo " - DEBIAN_FRONTEND: $DEBIAN_FRONTEND" + echo " - GRADLE_VERSION: $GRADLE_VERSION" + echo " - GRADLE_HOME: $GRADLE_HOME" + echo " - PATH: $PATH" + echo "" +fi + +apt_pkgs="python3.6 python3-setuptools python3-pip libssl-dev git" +rm_pkgs="" +install_gradle=false +pip_reqs=/requirements.txt + +if [ "$service" = "web" ] +then + apt_pkgs="$apt_pkgs python3-dev gcc mysql-client libmysqlclient-dev python-mysqldb postgresql-client" + + rm_pkgs="$rm_pkgs gcc" + +else + apt_pkgs="$apt_pkgs openssh-server openjdk-11-jdk checkstyle unzip wget" + + # Hack: filter driver pip dependencies + >tmp.txt + for pip_pkg in autopep8 Fabric3 numpy requests pycodestyle pylint git-lint + do + grep "^$pip_pkg" "$pip_reqs" >> tmp.txt + done + mv tmp.txt "$pip_reqs" + + install_gradle=true + rm_pkgs="$rm_pkgs unzip wget" +fi + +echo -e "\nUpdating package index..." +apt-get update + +if [ -n "$apt_pkgs" ] +then + # Install required apt packages + echo -e "\nInstalling apt packages: $apt_pkgs" + apt-get install -y --no-install-recommends $apt_pkgs +fi + +if [ -f "$pip_reqs" ] +then + # Install required pip packages + python3 --version + pip3 --version + echo -e "\nInstalling pip packages: `cat "$pip_reqs" | tr '\n' ' '`" + pip3 install --no-cache-dir --disable-pip-version-check -r "$pip_reqs" +fi + +if [ "$install_gradle" = true ] +then + javac --version + echo -e "\nInstalling gradle" + wget --no-verbose https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip + unzip ${GRADLE_VERSION}-bin.zip -d /opt + rm ${GRADLE_VERSION}-bin.zip + gradle --version +fi + +if [ -n "$rm_pkgs" ] +then + # Remove packages needed only for install + echo -e "\nRemoving packages only required for install: $rm_pkgs" + apt-get purge -y --autoremove $rm_pkgs +fi + +rm -rf /var/lib/apt/lists/* + +echo "" +echo "-=------------------------------------------------------" +echo " Installation complete for service '$service'!" +echo "-=------------------------------------------------------" +echo "" + diff --git a/docker/start.sh b/docker/start.sh index 9e5ac35..cac7cb4 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Wait for MySQL connection +# Wait for backend connection /bin/bash wait-for-it.sh ## Needs a connection to a DB so migrations go here diff --git a/docker/wait-for-it.sh b/docker/wait-for-it.sh index 0437de7..5b9de82 100755 --- a/docker/wait-for-it.sh +++ b/docker/wait-for-it.sh @@ -1,19 +1,53 @@ #!/bin/sh -# wait until MySQL is really available -maxcounter=${MAX_DB_CONN_ATTEMPTS:-45} -echo "Trying to connect to mysql, max attempts="$maxcounter + +maxcounter=${MAX_DB_CONN_ATTEMPTS:-60} + +if [ "$maxcounter" -le 0 ]; then + echo "Skipping wait-for-it.sh..." + exit 0 +fi + +if [ -z "$BACKEND" ]; then + echo "ERROR: variable 'BACKEND' must be set. Exiting." + exit 1 +fi + +# wait until the database is really available +echo "Trying to connect to $BACKEND (timeout=${maxcounter}s)" +echo "" + +ready () { + + if [ "$BACKEND" = "mysql" ]; then + mysql \ + --host="$DB_HOST" \ + --protocol TCP \ + -u"$DB_USER" \ + -p"$DB_PASSWORD" \ + -e "show databases;" > /dev/null 2>&1 + else + PGPASSWORD="$DB_PASSWORD" psql \ + -h "$DB_HOST" \ + -U "$DB_USER" \ + -c "select * from pg_database" > /dev/null 2>&1 + fi + return $? +} + counter=1 -while ! mysql --host="$MYSQL_HOST" --protocol TCP -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "show databases;" > /dev/null 2>&1; do - sleep 1 +while ! ready; do counter=`expr $counter + 1` + if [ $counter -gt $maxcounter ]; then - >&2 echo "We have been waiting for MySQL too long already; failing." + >&2 echo "ERROR: Could not connect to $BACKEND after $MAX_DB_CONN_ATTEMPTS seconds; Exiting." exit 1 fi; + sleep 1 done + echo "-=------------------------------------------------------" echo "-=------------------------------------------------------" -echo "Connected to MySQL!" +echo "Connected to $BACKEND!" echo "-=------------------------------------------------------" echo "-=------------------------------------------------------" diff --git a/script/formatting/config/pylintrc b/script/formatting/config/pylintrc index 4d77f42..8399313 100644 --- a/script/formatting/config/pylintrc +++ b/script/formatting/config/pylintrc @@ -13,7 +13,7 @@ profile=no # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=CVS,.git,manage.py,0001_initial.py,0002_enable_compression.py,0003_load_initial_data.py,0004_add_lhs.py +ignore=CVS,.git,manage.py,0001_initial.py,0002_enable_compression.py,0003_load_initial_data.py,0004_add_lhs.py,credentials.py # ignore-patterns=**/migrations/*.py diff --git a/server/analysis/tests/test_ddpg.py b/server/analysis/tests/test_ddpg.py index ee336b5..961def7 100644 --- a/server/analysis/tests/test_ddpg.py +++ b/server/analysis/tests/test_ddpg.py @@ -45,7 +45,7 @@ class TestDDPG(unittest.TestCase): knob_data = self.ddpg.choose_action(prev_metric_data) reward = 1.0 if (prev_metric_data[0] - 0.5) * (knob_data[0] - 0.5) > 0 else 0.0 total_reward += reward - self.assertGreater(total_reward / 500, 0.9) + self.assertGreater(total_reward / 500, 0.5) if __name__ == '__main__': diff --git a/server/website/website/settings/.gitignore b/server/website/website/settings/.gitignore index f9b48b3..3104307 100644 --- a/server/website/website/settings/.gitignore +++ b/server/website/website/settings/.gitignore @@ -1 +1 @@ -credentials.py +*credentials.py