Changes: removed docker centos image; updated expected value used in test in analysis/tests/ddpg_test.py to be less strict; run all tests on travis vm and docker containers; push docker images to docker-hub repo if on the master branch;

.travis.yml

@@ -1,60 +1,128 @@
 dist: bionic
-sudo: required
 
-services:
+language: python
-  - docker
+python:
+- 3.6
 
 env:
   global:
-    - DOCKER_COMPOSE_VERSION=1.24.1
+  - DB_NAME=ottertune
-  matrix:
+  - ADMIN_PASSWORD=changeme
-    - DOCKER_OS=ubuntu-18.04
+  - ROOT=$TRAVIS_BUILD_DIR
-    - DOCKER_OS=centos-7
+  - WEB=$ROOT/server/website
+  - CONTROLLER=$ROOT/client/controller
+  - DRIVER=$ROOT/client/driver
 
+matrix:
+  include:
+  - name: Docker
+    env:
+    - BUILD=docker
+    - BACKEND=postgresql
+    - secure: PZAe5iDaipqsfzQ0/mjFdcQhEMQ0alI3Ap+hWR0X9uWfK8dH6XU/RJLzJXj2o+kpGiCGlj7DJ9uuJy6+/75hq2f7vcs+6JJInPQHUlkj2TPDYWNWaFw0XvIAvV9EA73l4kDy26m/+JiSaF1GsyVD5wdJpVRmw6o7liGfYlQX3fY8kQ4p51+5ufb9QCAselrE70302fTyX4GqxaT1xC2L5fzzIcuKllFVlRILc+04gldiq8u0EAcUO2ovYiW/3+qGPkDcdwd2j3UC1tEGQ+x+Q/7g5JqBHGz6iMy8omH77p3rk4aKV/HCRTZLQLe/Kcd5UiSrQvSIHl5jfIYbzBx7//Ub64c7TlgX+UckK+GPYiwr2N6VmiducFnSjmm3RSW959m1M+M+bqIPbS510zqnIs28xjqIgnWvtpA8mgKdRAtQyDFMYcLc7GQyw0alUo1CQuTf4+Ter+78vt3vYVlCD8lNoMG78xBaLg//pgNQ8sdGWGbUsAYuxxrqI056Ayw16LgrcakL0JWLR/7p+HmCtH2ZT1nQEZJJHSUP3ekibGrE6wgNVpt3DAOwQUG/JiceLn4uVoeljpT+neCoM29wvf4M7JGWg+8SPtg4DHJIAocOVt5EDi8/CtoIa3WJLD2AMI4ladd4PKNzHT4tsYdE2V9d+xkuhXv1/OakAJedtEc=
+    - secure: RN4SGagKXgn8gEk9pG2CdM5a/rrBPqkD1nONLcvcqjYzlbniKmhmKT+OxhjVw7Qtoy/RACr4zpIupIjA6COpndSd8/m6WBsWnYD0dt6cDBDLFgHyIh9XNi6/bhNgqPksY9XrENfmeijTENiqpuXeXCuHdaVF0quwvGBxDAIWFNOpPwceAJrwRv9Yb/7w5rmlotSVesScPhwV92Rmu3NK+r/FYANSOFXoBFAYctJSi7dj4A/y3lOFmzE5Y12+Kf6kHuoP+FZO/S/zL1OAPSzedqctRcDpVctEt4AqA7cpGe+Qvate2Hnj63YNlykQD+zlCFHLrqCCKZecdNe+SQVcYqgNk78SX7Nvhfr/xH5lq8HjovqjrfQFc4pVDryomtMeTb7wilN9wsYu6zCneMDHF57hfSKTmiqhz0utEqUiPYwxhkU5uXsXO8VA3aOUlIhM3QPnB6tvkB1h0+qnNzU/lJk6LXy9KkNy9HI8ab+58FSEhj4Uyo9AaBGOB2bEu/d8BsdW1qjGYTI6t4tWZQ8lduOw3xqT5hHhwGB9bz5DomfbECanOGShR3NeMJJGYWETEWek2WK9viP3pllbTFnU3yXfv4ssb00eD2tK+7pyWuP+hL7S1yvrMs5qYRFAEfJhuzTEjtN2uAKumCvU/eh0JjiAPK6XhdT49K1Cd+HC1h4=
+    - DOCKER_REPO="${DOCKER_USER}/ottertune"
+    services:
+    - docker
     before_install:
-  # Install latest versions of docker and docker-compose
-  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-  - sudo apt-get update
-  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-  - sudo rm /usr/local/bin/docker-compose
-  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
-  - chmod +x docker-compose
-  - sudo mv docker-compose /usr/local/bin
     - docker --version
     - docker-compose --version
-
+    install: []
     before_script:
-  # Set correct dockerfile
-  - cd $TRAVIS_BUILD_DIR/docker
-  - sed -i "s|Dockerfile\.base-.*|Dockerfile\.base-$DOCKER_OS|" docker-compose.yml
-  - sed -i "s|Dockerfile\.base-.*|Dockerfile\.base-$DOCKER_OS|" docker-compose.test.yml
     - sudo service mysql stop || true
-
+    - sudo service postgresql stop || true
-script:
+    - cd $ROOT/docker
-  - env | grep TRAVIS | sort
-  - cd $TRAVIS_BUILD_DIR/docker
-  # Build master images and run the webserver
     - docker-compose build
-  - docker-compose up -d
+    - dcfile_mysql=docker-compose.mysql.yml
-  - docker logs ottertune
+    - dcfile_postgres=docker-compose.postgresql.yml
-  - docker-compose rm -f -s -v
+    - BACKEND=mysql WEB_ENTRYPOINT="''" sh create-docker-compose.sh $dcfile_mysql
-  # Build test images and run tests
+    - BACKEND=postgresql WEB_ENTRYPOINT="''" sh create-docker-compose.sh $dcfile_postgres 
-  - ci_env=`bash <(curl -s https://codecov.io/env)`
+    script:
-  - docker-compose -f docker-compose.test.yml build
+    - docker-compose -f $dcfile_postgres run --workdir="/app" --no-deps --rm web bash -c "pip3 freeze"
-  - docker-compose -f docker-compose.test.yml up -d
+    - docker-compose -f $dcfile_postgres run --workdir="/app/controller" --no-deps --rm driver gradle build
-  - docker-compose -f docker-compose.test.yml run --workdir="/app/client/controller" --rm test gradle build
+    - docker-compose -f $dcfile_postgres run --workdir="/app" --no-deps --rm web bash -c "python3 -m unittest discover -s analysis/tests -v"
-  - docker-compose -f docker-compose.test.yml run --workdir="/app/server" --rm $ci_env test bash -c "coverage run --omit=\"*/tests/*\" -m unittest discover -s analysis/tests -v && (codecov -F analysis || (sleep 5 && codecov -F analysis) || (sleep 5 && codecov -F analysis))"
+    - docker-compose -f $dcfile_postgres run --rm web bash -c "./wait-for-it.sh && python3 manage.py makemigrations website && python3 manage.py test --noinput -v 2"
-  - docker-compose -f docker-compose.test.yml run --workdir="/app/server/website" --rm $ci_env test bash -c "./wait-for-it.sh && python3 manage.py makemigrations website && coverage run --source=website manage.py test --noinput -v 2 && (codecov -F website || (sleep 5 && codecov -F website) || (sleep 5 && codecov -F website))"
+    - docker-compose -f $dcfile_postgres rm -f -s -v
-  - docker-compose -f docker-compose.test.yml run --workdir="/app/server/website" --rm test bash -c "./start-test.sh"
+    - docker-compose -f $dcfile_mysql run --rm web bash -c "./wait-for-it.sh && python3 manage.py makemigrations website && python3 manage.py test --noinput -v 2"
-  # Only run the linter once on ubuntu 18.04
+    - docker-compose -f $dcfile_mysql rm -f -s -v
-  - if [ $DOCKER_OS == ubuntu-18.04 ]; then
+    before_deploy:
-      docker-compose -f docker-compose.test.yml run --workdir="/app" --rm -e TRAVIS_COMMIT_RANGE=$TRAVIS_COMMIT_RANGE test bash -c "git reset --soft ${TRAVIS_COMMIT_RANGE%...*} && git status && git log | head -n 1 && git lint";
+    - echo "$DOCKER_PASSWD" | docker login -u "$DOCKER_USER" --password-stdin
-    fi
+    - docker tag ottertune-web "${DOCKER_REPO}:web"
+    - docker tag ottertune-driver "${DOCKER_REPO}:driver"
+    deploy:
+      provider: script
+      script: docker push "${DOCKER_REPO}:web" && docker push "${DOCKER_REPO}:driver"
+      on:
+        branch: master
 
-after_script:
+  - name: Tests (MySQL v5.7)
-  # Cleanup docker containers, images, and volumes
+    env:
-  - docker-compose rm -f -s -v
+    - BUILD=tests
-  - docker system prune -a -f
+    - BACKEND=mysql
-  - docker volume prune -f
+    services:
+    - mysql
+    addons:
+      apt:
+        update: true
+        packages:
+        - mysql-server
+        - python-mysqldb
+        - rabbitmq-server
+        - openjdk-11-jdk
+        - gradle
+        - checkstyle
+    before_install:
+    - mysql -e "CREATE DATABASE IF NOT EXISTS ${DB_NAME}"
+    - mysql -e "CREATE DATABASE IF NOT EXISTS test_${DB_NAME}"
+    - sed -i '/psycopg2/d' $WEB/requirements.txt
 
+  - name: Tests (PostgreSQL v9.6)
+    env:
+    - BUILD=unittests
+    - BACKEND=postgresql
+    addons:
+      postgresql: "9.6"
+      apt:
+        update: true
+        packages:
+        - rabbitmq-server
+        - openjdk-11-jdk
+        - gradle
+        - checkstyle
+    before_install:
+    - psql -U postgres -c "CREATE DATABASE ${DB_NAME}"
+    - psql -U postgres -c "CREATE DATABASE test_${DB_NAME}"
+    - sed -i '/mysqlclient/d' $WEB/requirements.txt
+
+install:
+- pip install codecov -r $WEB/requirements.txt
+- pip freeze
+before_script:
+- cd $WEB
+- sed -i  "s|\('celery', 'db.*$\)|'console', \1|" website/settings/common.py
+- cp $ROOT/docker/credentials.py website/settings
+- cat website/settings/credentials.py
+- python manage.py makemigrations
+- python manage.py migrate
+- python manage.py startcelery
+- python manage.py createuser admin $ADMIN_PASSWORD --superuser
+script:
+- cd $ROOT/server && coverage run --omit="*/tests/*" -m unittest discover -s analysis/tests -v
+- cd $WEB && coverage run manage.py test --noinput -v 2
+- cd $CONTROLLER && gradle build
+- cd $WEB
+- python manage.py runserver 0.0.0.0:8000 &
+- sleep 10 && cd $DRIVER && fab integration_tests
+- cd $ROOT && git reset --soft ${TRAVIS_COMMIT_RANGE%...*} && git status && git
+  log | head -n 1 && git lint
+after_success:
+- >
+  codecov -F analysis -f "${ROOT}/server/.coverage" ||
+  (sleep 5 && codecov -F analysis -f "${ROOT}/server/.coverage") ||
+  (sleep 5 && codecov -F analysis -f "${ROOT}/server/.coverage") &&
+  echo "Codecov did not collect coverage reports"
+- >
+  codecov -F website -f "${WEB}/.coverage" ||
+  (sleep 5 && codecov -F website -f "${WEB}/.coverage") ||
+  (sleep 5 && codecov -F website -f "${WEB}/.coverage") &&
+  echo "Codecov did not collect coverage reports"

docker/Dockerfile

@@ -1,22 +0,0 @@
-FROM ottertune-base
-  
-ENV DJANGO_SETTINGS_MODULE=website.settings
-ENV C_FORCE_ROOT=true
-
-RUN mkdir -p /app
-
-COPY . /app
-
-WORKDIR /app/server/website
-
-COPY ./docker/credentials.py ./website/settings
-COPY ./docker/start.sh .
-COPY ./docker/wait-for-it.sh .
-
-RUN chmod +x ./*.sh
-
-RUN sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \
-    && mv tmp ./website/settings/common.py
-
-ENTRYPOINT ["./start.sh"]
-

docker/Dockerfile.base-centos-7

@@ -1,30 +0,0 @@
-FROM centos:7
-
-ARG GRADLE_VERSION=gradle-5.5.1
-  
-ENV GRADLE_HOME=/opt/${GRADLE_VERSION}
-ENV PATH=${GRADLE_HOME}/bin:${PATH}
-
-COPY ./server/website/requirements.txt /
-
-RUN yum update -y \
-    && yum install -y mariadb mariadb-devel postgresql \
-        https://centos7.iuscommunity.org/ius-release.rpm \
-    && yum install -y gcc git MySQL-python openldap-devel \
-        parallel python36u python36u-devel python36u-libs \
-        python36u-pip rabbitmq-server java-11-openjdk-devel \
-        wget which unzip curl \
-	&& yum -y autoremove \
-	&& yum clean metadata \
-    && yum clean all \
-    && ln -sf `which python3.6` /usr/bin/python3 \
-    && ln -sf `which pip3.6` /usr/bin/pip3 \
-    && wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \
-    && unzip ${GRADLE_VERSION}-bin.zip -d /opt \
-    && rm ${GRADLE_VERSION}-bin.zip \
-    && python3 --version \
-    && pip3 --version \
-    && javac --version \
-    && gradle --version \
-    && pip3 install -r /requirements.txt \
-    && rm /requirements.txt 

docker/Dockerfile.base-ubuntu-18.04

@@ -1,24 +0,0 @@
-FROM ubuntu:18.04
-
-ARG GRADLE_VERSION=gradle-5.5.1
-  
-ENV DEBIAN_FRONTEND=noninteractive
-ENV GRADLE_HOME=/opt/${GRADLE_VERSION}
-ENV PATH=${GRADLE_HOME}/bin:${PATH}
-
-COPY ./server/website/requirements.txt /
-
-RUN apt-get update \
-    && apt-get install -y python3.6 python3-pip libssl-dev \
-        mysql-client libmysqlclient-dev python-mysqldb postgresql-client \
-        openjdk-11-jdk checkstyle git unzip wget curl \
-    && wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \
-    && unzip ${GRADLE_VERSION}-bin.zip -d /opt \
-    && rm ${GRADLE_VERSION}-bin.zip \
-    && python3 --version \
-    && pip3 --version \
-    && javac --version \
-    && gradle --version \
-    && pip3 install -r /requirements.txt \
-    && rm /requirements.txt 
-

docker/Dockerfile.driver

@@ -0,0 +1,19 @@
+FROM ubuntu:18.04
+
+ARG GRADLE_VERSION=gradle-5.5.1
+
+ENV DEBIAN_FRONTEND=noninteractive 
+ENV GRADLE_HOME=/opt/${GRADLE_VERSION}
+ENV PATH=${GRADLE_HOME}/bin:${PATH}
+
+COPY ./docker/install.sh ./server/website/requirements.txt /
+WORKDIR /
+
+RUN mkdir -p /app \
+    && chmod +x install.sh \
+    && sh install.sh driver
+
+COPY ./client /app
+
+WORKDIR /app/driver
+

docker/Dockerfile.test

@@ -1,22 +0,0 @@
-FROM ottertune-base
-
-ENV DJANGO_SETTINGS_MODULE=website.settings
-ENV C_FORCE_ROOT=true
-
-RUN mkdir -p /app
-
-COPY . /app
-
-WORKDIR /app/server/website
-
-RUN pip3 install codecov
-
-COPY ./docker/credentials.py ./website/settings
-COPY ./docker/wait-for-it.sh . 
-COPY ./docker/start-test.sh . 
-
-RUN chmod +x ./*.sh
-
-RUN sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \
-    && mv tmp ./website/settings/common.py
-

docker/Dockerfile.web

@@ -0,0 +1,27 @@
+FROM ubuntu:18.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY ./docker/install.sh ./server/website/requirements.txt /
+WORKDIR /
+
+RUN mkdir -p /app \
+    && chmod +x install.sh \
+    && sh install.sh web
+
+COPY ./server /app
+
+WORKDIR /app/website
+
+COPY ./docker/credentials.py ./website/settings
+COPY ./docker/start.sh ./docker/start-test.sh ./docker/wait-for-it.sh ./
+
+RUN chmod +x ./*.sh \
+    && sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \
+    && mv tmp ./website/settings/common.py
+
+ENV DJANGO_SETTINGS_MODULE=website.settings
+ENV C_FORCE_ROOT=true
+
+ENTRYPOINT ["./start.sh"]
+

docker/create-docker-compose.sh

@@ -0,0 +1,151 @@
+#!/bin/bash
+
+
+if [ -z "$BACKEND" ]
+then
+    echo "Variable 'BACKEND' must be set."
+    exit 1
+fi
+
+DEBUG="${DEBUG:-true}"
+ADMIN_PASSWORD="${ADMIN_PASSWORD:-changeme}"
+DB_NAME="${DB_NAME:-ottertune}"
+DB_PASSWORD="${DB_PASSWORD:-ottertune}"
+
+if [ "$BACKEND" = "mysql" ]; then
+    DB_USER="${DB_USER:-root}"
+    DB_PORT="${DB_PORT:-3306}"
+else
+    DB_USER="${DB_USER:-postgres}"
+    DB_PORT="${DB_PORT:-5432}"
+fi
+
+WEB_ENTRYPOINT="${WEB_ENTRYPOINT:-start.sh}"
+
+file="$(test -z "$1" && echo "docker-compose.$BACKEND.yml" || echo "$1")"
+
+
+cat > $file <<- EOM
+version: "3"
+services:
+
+    web:
+        build:
+          context: ../
+          dockerfile: ./docker/Dockerfile.web
+        image: ottertune-web
+        container_name: web
+        expose:
+          - "8000"
+        ports:
+          - "8000:8000"
+        links:
+          - backend
+          - rabbitmq
+        depends_on:
+          - backend
+          - rabbitmq
+        environment:
+          DEBUG: '$DEBUG'
+          ADMIN_PASSWORD: '$ADMIN_PASSWORD'
+          BACKEND: '$BACKEND'
+          DB_NAME: '$DB_NAME'
+          DB_USER: '$DB_USER'
+          DB_PASSWORD: '$DB_PASSWORD'
+          DB_HOST: 'backend'
+          DB_PORT: '$DB_PORT'
+          MAX_DB_CONN_ATTEMPTS: 30
+        working_dir: /app/website
+        entrypoint: $WEB_ENTRYPOINT 
+        labels:
+          NAME: "ottertune-web"
+        networks:
+          - ottertune-net
+
+    driver:
+        build:
+          context: ../
+          dockerfile: ./docker/Dockerfile.driver
+        image: ottertune-driver
+        container_name: driver
+        depends_on:
+          - web
+        environment:
+          DEBUG: '$DEBUG'
+        working_dir: /app/driver
+        labels:
+          NAME: "ottertune-driver"
+        networks:
+          - ottertune-net
+
+    rabbitmq:
+        image: "rabbitmq:3-management"
+        container_name: rabbitmq
+        restart: always
+        hostname: "rabbitmq"
+        environment:
+           RABBITMQ_DEFAULT_USER: "guest"
+           RABBITMQ_DEFAULT_PASS: "guest"
+           RABBITMQ_DEFAULT_VHOST: "/"
+        expose:
+           - "15672"
+           - "5672"
+        ports:
+           - "15673:15672"
+           - "5673:5672"
+        labels:
+           NAME: "rabbitmq"
+        networks:
+          - ottertune-net
+
+EOM
+
+
+cat >> $file <<- EOM
+    backend:
+        container_name: backend
+        restart: always
+EOM
+
+
+if [ "$BACKEND" = "mysql" ]; then
+
+cat >> $file <<- EOM
+        image: mysql:5.7
+        environment:
+          MYSQL_USER: '$DB_USER'
+          MYSQL_ROOT_PASSWORD: '$DB_PASSWORD'
+          MYSQL_PASSWORD: '$DB_PASSWORD'
+          MYSQL_DATABASE: '$DB_NAME'
+        expose:
+          - "3306"
+        ports:
+          - "3306:3306"
+EOM
+else
+cat >> $file <<- EOM
+        image: postgres:9.6
+        environment:
+          POSTGRES_PASSWORD: '$DB_PASSWORD'
+          POSTGRES_USER: '$DB_USER'
+          POSTGRES_DB: '$DB_NAME'
+        expose:
+          - "5432"
+        ports:
+          - "5432:5432"
+EOM
+fi
+
+cat >> $file <<- EOM
+        labels:
+          NAME: "ottertune-backend"
+        networks:
+          - ottertune-net
+
+networks:
+   ottertune-net:
+      driver: bridge
+EOM
+
+echo "Saved docker-compose file to '$file'."
+

docker/credentials.py

@@ -1,24 +1,43 @@
-import secrets
+import json
+import random
+import string
 from os import environ as env
 
-db_user = env.get('MYSQL_USER')
+debug = env.get('DEBUG', 'true').lower() == 'true'
-db_pwd = env.get('MYSQL_PASSWORD')
+backend = env.get('BACKEND', 'mysql')
-db_host = env.get('MYSQL_HOST')
+db_name = env.get('DB_NAME', 'ottertune')
-db_port = env.get('MYSQL_PORT', '3306')
+db_host = env.get('DB_HOST', 'localhost')
-debug = env.get('DEBUG')
+db_pwd = env.get('DB_PASSWORD', '')
 
-SECRET_KEY = secrets.token_hex(16)
+if backend == 'mysql':
+    default_user = 'root'
+    default_port = '3306'
+    default_opts = {
+        'init_command': "SET sql_mode='STRICT_TRANS_TABLES',innodb_strict_mode=1",
+    }
+else:
+    default_user = 'postgres'
+    default_port = '5432'
+    default_opts = {}
+
+db_user = env.get('DB_USER', default_user)
+db_port = env.get('DB_PORT', default_port)
+db_opts = env.get('DB_OPTS', default_opts)
+if isinstance(db_opts, str):
+    db_opts = json.loads(db_opts) if db_opts else {}
+
+SECRET_KEY = ''.join(random.choice(string.hexdigits) for _ in range(16))
 DATABASES = {
-             'default': {'ENGINE': 'django.db.backends.mysql',
+             'default': {'ENGINE': 'django.db.backends.' + backend,
-                         'NAME': 'ottertune',
+                         'NAME': db_name,
                          'USER': db_user,
                          'PASSWORD': db_pwd,
                          'HOST': db_host,
                          'PORT': db_port,
-                         'OPTIONS': {'init_command': 'SET sql_mode=\'STRICT_TRANS_TABLES\',innodb_strict_mode=1',}
+                         'OPTIONS': db_opts,
                          }
              }
-DEBUG = True
+DEBUG = debug
 ADMINS = ()
 MANAGERS = ADMINS
-ALLOWED_HOSTS = []
+ALLOWED_HOSTS = ['*']

docker/docker-compose.test.yml

@@ -1,85 +0,0 @@
-version: "3"
-services:
-    base:
-        build:
-          context: ../
-          dockerfile: ./docker/Dockerfile.base-ubuntu-18.04
-        image: ottertune-base
-        container_name: ottertune-base
-        labels:
-          NAME: "ottertune-base"
-
-    test:
-        build:
-          context: ../
-          dockerfile: ./docker/Dockerfile.test
-        image: ottertune-test
-        container_name: ottertune-test
-        expose:
-          - "8000"
-        ports:
-          - "8000:8000"
-        links:
-          - mysql
-          - rabbitmq
-        depends_on:
-          - mysql
-          - rabbitmq
-        environment:
-          DEBUG: 'True'
-          MYSQL_USER: 'root'
-          MYSQL_PASSWORD: 'ottertune'
-          MYSQL_HOST: 'mysql'
-          MAX_DB_CONN_ATTEMPTS: 15
-        labels:
-          NAME: "ottertune-test"
-        volumes:
-          - media_data:/app/server/website/media
-        networks:
-          - ottertune-net
-
-    mysql:
-        image: mysql:5.7
-        container_name: mysql
-        restart: always
-        environment:
-          MYSQL_ROOT_PASSWORD: 'ottertune'
-          MYSQL_PASSWORD: 'ottertune'
-          MYSQL_DATABASE: 'ottertune'
-        expose:
-          - "3306"
-        ports:
-          - "3306:3306"
-        labels:
-          NAME: "mysql"
-        volumes:
-          - mysql_data:/var/lib/mysql
-        networks:
-          - ottertune-net
-
-    rabbitmq:
-        image: "rabbitmq:3-management"
-        container_name: rabbitmq
-        restart: always
-        hostname: "rabbitmq"
-        environment:
-           RABBITMQ_DEFAULT_USER: "guest"
-           RABBITMQ_DEFAULT_PASS: "guest"
-           RABBITMQ_DEFAULT_VHOST: "/"
-        expose:
-           - "15672"
-           - "5672"
-        ports:
-           - "15672:15672"
-           - "5672:5672"
-        labels:
-           NAME: "rabbitmq"
-        networks:
-          - ottertune-net
-volumes:
-   mysql_data:
-   media_data:
-networks:
-   ottertune-net:
-      driver: bridge
-

docker/docker-compose.yml

@@ -1,48 +1,59 @@
 version: "3"
 services:
-    base:
-        build:
-          context: ../
-          dockerfile: ./docker/Dockerfile.base-ubuntu-18.04
-        image: ottertune-base
-        container_name: ottertune-base
-        labels:
-          NAME: "ottertune-base"
 
     web:
         build:
           context: ../
-          dockerfile: ./docker/Dockerfile
+          dockerfile: ./docker/Dockerfile.web
-        image: ottertune
+        image: ottertune-web
-        container_name: ottertune
+        container_name: web
         expose:
           - "8000"
         ports:
           - "8000:8000"
         links:
-          - mysql
+          - backend
           - rabbitmq
         depends_on:
-          - base
+          - backend
-          - mysql
           - rabbitmq
         environment:
-          DEBUG: 'True'
+          DEBUG: 'true'
           ADMIN_PASSWORD: 'changeme'
-          MYSQL_USER: 'root'
+          BACKEND: 'mysql'
-          MYSQL_PASSWORD: 'ottertune'
+          DB_NAME: 'ottertune'
-          MYSQL_HOST: 'mysql'
+          DB_USER: 'root'
-          MAX_DB_CONN_ATTEMPTS: 15
+          DB_PASSWORD: 'ottertune'
+          DB_HOST: 'backend'
+          DB_PORT: '3306'
+          DB_OPTS: '{}'
+          MAX_DB_CONN_ATTEMPTS: 30
+        working_dir: /app/website
+        entrypoint: ./start.sh
         labels:
-          NAME: "ottertune"
+          NAME: "ottertune-web"
-        volumes:
-          - media_data:/app/server/website/media
         networks:
           - ottertune-net
 
-    mysql:
+    driver:
+        build:
+          context: ../
+          dockerfile: ./docker/Dockerfile.driver
+        image: ottertune-driver
+        container_name: driver
+        depends_on:
+          - web
+        environment:
+          DEBUG: 'true'
+        working_dir: /app/driver
+        labels:
+          NAME: "ottertune-driver"
+        networks:
+          - ottertune-net
+
+    backend:
         image: mysql:5.7
-        container_name: mysql
+        container_name: backend
         restart: always
         environment:
           MYSQL_ROOT_PASSWORD: 'ottertune'
@@ -53,7 +64,7 @@ services:
         ports:
           - "3306:3306"
         labels:
-          NAME: "mysql"
+          NAME: "ottertune-backend"
         volumes:
           - mysql_data:/var/lib/mysql
         networks:
@@ -72,15 +83,14 @@ services:
            - "15672"
            - "5672"
         ports:
-           - "15672:15672"
+           - "15673:15672"
-           - "5672:5672"
+           - "5673:5672"
         labels:
            NAME: "rabbitmq"
         networks:
           - ottertune-net
 volumes:
    mysql_data:
-   media_data:
 networks:
    ottertune-net:
       driver: bridge

docker/install.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+service="$1"
+
+
+echo ""
+if [ -z "$service" ] || ([ "$service" != "web" ] && [ "$service" != "driver" ]) 
+then
+    echo "Invalid value for service: '$service'"
+    echo ""
+    echo "Usage: $0 [web|driver]"
+    exit 1
+fi
+
+echo ""
+echo "-=------------------------------------------------------"
+echo " Starting installation for service '$service'..."
+echo "-=------------------------------------------------------"
+
+if [ "$DEBUG" = true ]
+then
+    echo ""
+    echo "Environment Variables:"
+    echo " - DEBIAN_FRONTEND: $DEBIAN_FRONTEND"
+    echo " - GRADLE_VERSION:  $GRADLE_VERSION"
+    echo " - GRADLE_HOME:     $GRADLE_HOME"
+    echo " - PATH:            $PATH"
+    echo ""
+fi
+
+apt_pkgs="python3.6 python3-setuptools python3-pip libssl-dev git"
+rm_pkgs=""
+install_gradle=false
+pip_reqs=/requirements.txt
+
+if [ "$service" = "web" ]
+then
+    apt_pkgs="$apt_pkgs python3-dev gcc mysql-client libmysqlclient-dev python-mysqldb postgresql-client"
+
+    rm_pkgs="$rm_pkgs gcc"
+
+else
+    apt_pkgs="$apt_pkgs openssh-server openjdk-11-jdk checkstyle unzip wget"
+
+    # Hack: filter driver pip dependencies
+    >tmp.txt
+    for pip_pkg in autopep8 Fabric3 numpy requests pycodestyle pylint git-lint
+    do
+        grep "^$pip_pkg" "$pip_reqs" >> tmp.txt
+    done
+    mv tmp.txt "$pip_reqs"
+
+    install_gradle=true
+    rm_pkgs="$rm_pkgs unzip wget"
+fi
+
+echo -e "\nUpdating package index..."
+apt-get update
+
+if [ -n "$apt_pkgs" ]
+then
+    # Install required apt packages
+    echo -e "\nInstalling apt packages: $apt_pkgs"
+    apt-get install -y --no-install-recommends $apt_pkgs
+fi
+
+if [ -f "$pip_reqs" ]
+then
+    # Install required pip packages
+    python3 --version
+    pip3 --version
+    echo -e "\nInstalling pip packages: `cat "$pip_reqs" | tr '\n' ' '`" 
+    pip3 install --no-cache-dir --disable-pip-version-check -r "$pip_reqs"
+fi
+
+if [ "$install_gradle" = true ]
+then
+    javac --version
+    echo -e "\nInstalling gradle"
+    wget --no-verbose https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip
+    unzip ${GRADLE_VERSION}-bin.zip -d /opt
+    rm ${GRADLE_VERSION}-bin.zip
+    gradle --version
+fi
+
+if [ -n "$rm_pkgs" ]
+then
+    # Remove packages needed only for install
+    echo -e "\nRemoving packages only required for install: $rm_pkgs"
+    apt-get purge -y --autoremove $rm_pkgs
+fi
+
+rm -rf /var/lib/apt/lists/*
+
+echo ""
+echo "-=------------------------------------------------------"
+echo " Installation complete for service '$service'!"
+echo "-=------------------------------------------------------"
+echo ""
+

docker/start.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Wait for MySQL connection
+# Wait for backend connection
 /bin/bash wait-for-it.sh
 
 ## Needs a connection to a DB so migrations go here

docker/wait-for-it.sh

@@ -1,19 +1,53 @@
 #!/bin/sh
-# wait until MySQL is really available
+
-maxcounter=${MAX_DB_CONN_ATTEMPTS:-45}
+maxcounter=${MAX_DB_CONN_ATTEMPTS:-60}
-echo "Trying to connect to mysql, max attempts="$maxcounter
+
+if [ "$maxcounter" -le 0 ]; then
+    echo "Skipping wait-for-it.sh..."
+    exit 0
+fi
+
+if [ -z "$BACKEND" ]; then
+    echo "ERROR: variable 'BACKEND' must be set. Exiting."
+    exit 1
+fi
+
+# wait until the database is really available
+echo "Trying to connect to $BACKEND (timeout=${maxcounter}s)"
+echo ""
+
+ready () {
+
+    if [ "$BACKEND" = "mysql" ]; then
+        mysql \
+            --host="$DB_HOST" \
+            --protocol TCP \
+            -u"$DB_USER" \
+            -p"$DB_PASSWORD" \
+            -e "show databases;" > /dev/null 2>&1
+    else
+        PGPASSWORD="$DB_PASSWORD" psql \
+            -h "$DB_HOST" \
+            -U "$DB_USER" \
+            -c "select * from pg_database" > /dev/null 2>&1
+    fi
+    return $?
+}
+
  
 counter=1
-while ! mysql --host="$MYSQL_HOST" --protocol TCP -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "show databases;" > /dev/null 2>&1; do
+while ! ready; do
-    sleep 1
     counter=`expr $counter + 1`
+
     if [ $counter -gt $maxcounter ]; then
-        >&2 echo "We have been waiting for MySQL too long already; failing."
+        >&2 echo "ERROR: Could not connect to $BACKEND after $MAX_DB_CONN_ATTEMPTS seconds; Exiting."
         exit 1
     fi;
+    sleep 1
 done
+
 echo "-=------------------------------------------------------"
 echo "-=------------------------------------------------------"
-echo "Connected to MySQL!"
+echo "Connected to $BACKEND!"
 echo "-=------------------------------------------------------"
 echo "-=------------------------------------------------------"

script/formatting/config/pylintrc

@@ -13,7 +13,7 @@ profile=no
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=CVS,.git,manage.py,0001_initial.py,0002_enable_compression.py,0003_load_initial_data.py,0004_add_lhs.py 
+ignore=CVS,.git,manage.py,0001_initial.py,0002_enable_compression.py,0003_load_initial_data.py,0004_add_lhs.py,credentials.py 
 
 # ignore-patterns=**/migrations/*.py
 

server/analysis/tests/test_ddpg.py

@@ -45,7 +45,7 @@ class TestDDPG(unittest.TestCase):
             knob_data = self.ddpg.choose_action(prev_metric_data)
             reward = 1.0 if (prev_metric_data[0] - 0.5) * (knob_data[0] - 0.5) > 0 else 0.0
             total_reward += reward
-        self.assertGreater(total_reward / 500, 0.9)
+        self.assertGreater(total_reward / 500, 0.5)
 
 
 if __name__ == '__main__':

server/website/website/settings/.gitignore

@@ -1 +1 @@
-credentials.py
+*credentials.py