Changes: removed docker centos image; updated expected value used in test in analysis/tests/ddpg_test.py to be less strict; run all tests on travis vm and docker containers; push docker images to docker-hub repo if on the master branch;

docker/Dockerfile

@@ -1,22 +0,0 @@
-FROM ottertune-base
-  
-ENV DJANGO_SETTINGS_MODULE=website.settings
-ENV C_FORCE_ROOT=true
-
-RUN mkdir -p /app
-
-COPY . /app
-
-WORKDIR /app/server/website
-
-COPY ./docker/credentials.py ./website/settings
-COPY ./docker/start.sh .
-COPY ./docker/wait-for-it.sh .
-
-RUN chmod +x ./*.sh
-
-RUN sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \
-    && mv tmp ./website/settings/common.py
-
-ENTRYPOINT ["./start.sh"]
-

docker/Dockerfile.base-centos-7

@@ -1,30 +0,0 @@
-FROM centos:7
-
-ARG GRADLE_VERSION=gradle-5.5.1
-  
-ENV GRADLE_HOME=/opt/${GRADLE_VERSION}
-ENV PATH=${GRADLE_HOME}/bin:${PATH}
-
-COPY ./server/website/requirements.txt /
-
-RUN yum update -y \
-    && yum install -y mariadb mariadb-devel postgresql \
-        https://centos7.iuscommunity.org/ius-release.rpm \
-    && yum install -y gcc git MySQL-python openldap-devel \
-        parallel python36u python36u-devel python36u-libs \
-        python36u-pip rabbitmq-server java-11-openjdk-devel \
-        wget which unzip curl \
-	&& yum -y autoremove \
-	&& yum clean metadata \
-    && yum clean all \
-    && ln -sf `which python3.6` /usr/bin/python3 \
-    && ln -sf `which pip3.6` /usr/bin/pip3 \
-    && wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \
-    && unzip ${GRADLE_VERSION}-bin.zip -d /opt \
-    && rm ${GRADLE_VERSION}-bin.zip \
-    && python3 --version \
-    && pip3 --version \
-    && javac --version \
-    && gradle --version \
-    && pip3 install -r /requirements.txt \
-    && rm /requirements.txt 

docker/Dockerfile.base-ubuntu-18.04

@@ -1,24 +0,0 @@
-FROM ubuntu:18.04
-
-ARG GRADLE_VERSION=gradle-5.5.1
-  
-ENV DEBIAN_FRONTEND=noninteractive
-ENV GRADLE_HOME=/opt/${GRADLE_VERSION}
-ENV PATH=${GRADLE_HOME}/bin:${PATH}
-
-COPY ./server/website/requirements.txt /
-
-RUN apt-get update \
-    && apt-get install -y python3.6 python3-pip libssl-dev \
-        mysql-client libmysqlclient-dev python-mysqldb postgresql-client \
-        openjdk-11-jdk checkstyle git unzip wget curl \
-    && wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \
-    && unzip ${GRADLE_VERSION}-bin.zip -d /opt \
-    && rm ${GRADLE_VERSION}-bin.zip \
-    && python3 --version \
-    && pip3 --version \
-    && javac --version \
-    && gradle --version \
-    && pip3 install -r /requirements.txt \
-    && rm /requirements.txt 
-

docker/Dockerfile.driver

@@ -0,0 +1,19 @@
+FROM ubuntu:18.04
+
+ARG GRADLE_VERSION=gradle-5.5.1
+
+ENV DEBIAN_FRONTEND=noninteractive 
+ENV GRADLE_HOME=/opt/${GRADLE_VERSION}
+ENV PATH=${GRADLE_HOME}/bin:${PATH}
+
+COPY ./docker/install.sh ./server/website/requirements.txt /
+WORKDIR /
+
+RUN mkdir -p /app \
+    && chmod +x install.sh \
+    && sh install.sh driver
+
+COPY ./client /app
+
+WORKDIR /app/driver
+

docker/Dockerfile.test

@@ -1,22 +0,0 @@
-FROM ottertune-base
-
-ENV DJANGO_SETTINGS_MODULE=website.settings
-ENV C_FORCE_ROOT=true
-
-RUN mkdir -p /app
-
-COPY . /app
-
-WORKDIR /app/server/website
-
-RUN pip3 install codecov
-
-COPY ./docker/credentials.py ./website/settings
-COPY ./docker/wait-for-it.sh . 
-COPY ./docker/start-test.sh . 
-
-RUN chmod +x ./*.sh
-
-RUN sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \
-    && mv tmp ./website/settings/common.py
-

docker/Dockerfile.web

@@ -0,0 +1,27 @@
+FROM ubuntu:18.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY ./docker/install.sh ./server/website/requirements.txt /
+WORKDIR /
+
+RUN mkdir -p /app \
+    && chmod +x install.sh \
+    && sh install.sh web
+
+COPY ./server /app
+
+WORKDIR /app/website
+
+COPY ./docker/credentials.py ./website/settings
+COPY ./docker/start.sh ./docker/start-test.sh ./docker/wait-for-it.sh ./
+
+RUN chmod +x ./*.sh \
+    && sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \
+    && mv tmp ./website/settings/common.py
+
+ENV DJANGO_SETTINGS_MODULE=website.settings
+ENV C_FORCE_ROOT=true
+
+ENTRYPOINT ["./start.sh"]
+

docker/create-docker-compose.sh

@@ -0,0 +1,151 @@
+#!/bin/bash
+
+
+if [ -z "$BACKEND" ]
+then
+    echo "Variable 'BACKEND' must be set."
+    exit 1
+fi
+
+DEBUG="${DEBUG:-true}"
+ADMIN_PASSWORD="${ADMIN_PASSWORD:-changeme}"
+DB_NAME="${DB_NAME:-ottertune}"
+DB_PASSWORD="${DB_PASSWORD:-ottertune}"
+
+if [ "$BACKEND" = "mysql" ]; then
+    DB_USER="${DB_USER:-root}"
+    DB_PORT="${DB_PORT:-3306}"
+else
+    DB_USER="${DB_USER:-postgres}"
+    DB_PORT="${DB_PORT:-5432}"
+fi
+
+WEB_ENTRYPOINT="${WEB_ENTRYPOINT:-start.sh}"
+
+file="$(test -z "$1" && echo "docker-compose.$BACKEND.yml" || echo "$1")"
+
+
+cat > $file <<- EOM
+version: "3"
+services:
+
+    web:
+        build:
+          context: ../
+          dockerfile: ./docker/Dockerfile.web
+        image: ottertune-web
+        container_name: web
+        expose:
+          - "8000"
+        ports:
+          - "8000:8000"
+        links:
+          - backend
+          - rabbitmq
+        depends_on:
+          - backend
+          - rabbitmq
+        environment:
+          DEBUG: '$DEBUG'
+          ADMIN_PASSWORD: '$ADMIN_PASSWORD'
+          BACKEND: '$BACKEND'
+          DB_NAME: '$DB_NAME'
+          DB_USER: '$DB_USER'
+          DB_PASSWORD: '$DB_PASSWORD'
+          DB_HOST: 'backend'
+          DB_PORT: '$DB_PORT'
+          MAX_DB_CONN_ATTEMPTS: 30
+        working_dir: /app/website
+        entrypoint: $WEB_ENTRYPOINT 
+        labels:
+          NAME: "ottertune-web"
+        networks:
+          - ottertune-net
+
+    driver:
+        build:
+          context: ../
+          dockerfile: ./docker/Dockerfile.driver
+        image: ottertune-driver
+        container_name: driver
+        depends_on:
+          - web
+        environment:
+          DEBUG: '$DEBUG'
+        working_dir: /app/driver
+        labels:
+          NAME: "ottertune-driver"
+        networks:
+          - ottertune-net
+
+    rabbitmq:
+        image: "rabbitmq:3-management"
+        container_name: rabbitmq
+        restart: always
+        hostname: "rabbitmq"
+        environment:
+           RABBITMQ_DEFAULT_USER: "guest"
+           RABBITMQ_DEFAULT_PASS: "guest"
+           RABBITMQ_DEFAULT_VHOST: "/"
+        expose:
+           - "15672"
+           - "5672"
+        ports:
+           - "15673:15672"
+           - "5673:5672"
+        labels:
+           NAME: "rabbitmq"
+        networks:
+          - ottertune-net
+
+EOM
+
+
+cat >> $file <<- EOM
+    backend:
+        container_name: backend
+        restart: always
+EOM
+
+
+if [ "$BACKEND" = "mysql" ]; then
+
+cat >> $file <<- EOM
+        image: mysql:5.7
+        environment:
+          MYSQL_USER: '$DB_USER'
+          MYSQL_ROOT_PASSWORD: '$DB_PASSWORD'
+          MYSQL_PASSWORD: '$DB_PASSWORD'
+          MYSQL_DATABASE: '$DB_NAME'
+        expose:
+          - "3306"
+        ports:
+          - "3306:3306"
+EOM
+else
+cat >> $file <<- EOM
+        image: postgres:9.6
+        environment:
+          POSTGRES_PASSWORD: '$DB_PASSWORD'
+          POSTGRES_USER: '$DB_USER'
+          POSTGRES_DB: '$DB_NAME'
+        expose:
+          - "5432"
+        ports:
+          - "5432:5432"
+EOM
+fi
+
+cat >> $file <<- EOM
+        labels:
+          NAME: "ottertune-backend"
+        networks:
+          - ottertune-net
+
+networks:
+   ottertune-net:
+      driver: bridge
+EOM
+
+echo "Saved docker-compose file to '$file'."
+

docker/credentials.py

@@ -1,24 +1,43 @@
-import secrets
+import json
+import random
+import string
 from os import environ as env
 
-db_user = env.get('MYSQL_USER')
-db_pwd = env.get('MYSQL_PASSWORD')
-db_host = env.get('MYSQL_HOST')
-db_port = env.get('MYSQL_PORT', '3306')
-debug = env.get('DEBUG')
+debug = env.get('DEBUG', 'true').lower() == 'true'
+backend = env.get('BACKEND', 'mysql')
+db_name = env.get('DB_NAME', 'ottertune')
+db_host = env.get('DB_HOST', 'localhost')
+db_pwd = env.get('DB_PASSWORD', '')
 
-SECRET_KEY = secrets.token_hex(16)
+if backend == 'mysql':
+    default_user = 'root'
+    default_port = '3306'
+    default_opts = {
+        'init_command': "SET sql_mode='STRICT_TRANS_TABLES',innodb_strict_mode=1",
+    }
+else:
+    default_user = 'postgres'
+    default_port = '5432'
+    default_opts = {}
+
+db_user = env.get('DB_USER', default_user)
+db_port = env.get('DB_PORT', default_port)
+db_opts = env.get('DB_OPTS', default_opts)
+if isinstance(db_opts, str):
+    db_opts = json.loads(db_opts) if db_opts else {}
+
+SECRET_KEY = ''.join(random.choice(string.hexdigits) for _ in range(16))
 DATABASES = {
-             'default': {'ENGINE': 'django.db.backends.mysql',
-                         'NAME': 'ottertune',
+             'default': {'ENGINE': 'django.db.backends.' + backend,
+                         'NAME': db_name,
                          'USER': db_user,
                          'PASSWORD': db_pwd,
                          'HOST': db_host,
                          'PORT': db_port,
-                         'OPTIONS': {'init_command': 'SET sql_mode=\'STRICT_TRANS_TABLES\',innodb_strict_mode=1',}
+                         'OPTIONS': db_opts,
                          }
              }
-DEBUG = True
+DEBUG = debug
 ADMINS = ()
 MANAGERS = ADMINS
-ALLOWED_HOSTS = []
+ALLOWED_HOSTS = ['*']

docker/docker-compose.test.yml

@@ -1,85 +0,0 @@
-version: "3"
-services:
-    base:
-        build:
-          context: ../
-          dockerfile: ./docker/Dockerfile.base-ubuntu-18.04
-        image: ottertune-base
-        container_name: ottertune-base
-        labels:
-          NAME: "ottertune-base"
-
-    test:
-        build:
-          context: ../
-          dockerfile: ./docker/Dockerfile.test
-        image: ottertune-test
-        container_name: ottertune-test
-        expose:
-          - "8000"
-        ports:
-          - "8000:8000"
-        links:
-          - mysql
-          - rabbitmq
-        depends_on:
-          - mysql
-          - rabbitmq
-        environment:
-          DEBUG: 'True'
-          MYSQL_USER: 'root'
-          MYSQL_PASSWORD: 'ottertune'
-          MYSQL_HOST: 'mysql'
-          MAX_DB_CONN_ATTEMPTS: 15
-        labels:
-          NAME: "ottertune-test"
-        volumes:
-          - media_data:/app/server/website/media
-        networks:
-          - ottertune-net
-
-    mysql:
-        image: mysql:5.7
-        container_name: mysql
-        restart: always
-        environment:
-          MYSQL_ROOT_PASSWORD: 'ottertune'
-          MYSQL_PASSWORD: 'ottertune'
-          MYSQL_DATABASE: 'ottertune'
-        expose:
-          - "3306"
-        ports:
-          - "3306:3306"
-        labels:
-          NAME: "mysql"
-        volumes:
-          - mysql_data:/var/lib/mysql
-        networks:
-          - ottertune-net
-
-    rabbitmq:
-        image: "rabbitmq:3-management"
-        container_name: rabbitmq
-        restart: always
-        hostname: "rabbitmq"
-        environment:
-           RABBITMQ_DEFAULT_USER: "guest"
-           RABBITMQ_DEFAULT_PASS: "guest"
-           RABBITMQ_DEFAULT_VHOST: "/"
-        expose:
-           - "15672"
-           - "5672"
-        ports:
-           - "15672:15672"
-           - "5672:5672"
-        labels:
-           NAME: "rabbitmq"
-        networks:
-          - ottertune-net
-volumes:
-   mysql_data:
-   media_data:
-networks:
-   ottertune-net:
-      driver: bridge
-

docker/docker-compose.yml

@@ -1,48 +1,59 @@
 version: "3"
 services:
-    base:
-        build:
-          context: ../
-          dockerfile: ./docker/Dockerfile.base-ubuntu-18.04
-        image: ottertune-base
-        container_name: ottertune-base
-        labels:
-          NAME: "ottertune-base"
 
     web:
         build:
           context: ../
-          dockerfile: ./docker/Dockerfile
-        image: ottertune
-        container_name: ottertune
+          dockerfile: ./docker/Dockerfile.web
+        image: ottertune-web
+        container_name: web
         expose:
           - "8000"
         ports:
           - "8000:8000"
         links:
-          - mysql
+          - backend
           - rabbitmq
         depends_on:
-          - base
-          - mysql
+          - backend
           - rabbitmq
         environment:
-          DEBUG: 'True'
+          DEBUG: 'true'
           ADMIN_PASSWORD: 'changeme'
-          MYSQL_USER: 'root'
-          MYSQL_PASSWORD: 'ottertune'
-          MYSQL_HOST: 'mysql'
-          MAX_DB_CONN_ATTEMPTS: 15
+          BACKEND: 'mysql'
+          DB_NAME: 'ottertune'
+          DB_USER: 'root'
+          DB_PASSWORD: 'ottertune'
+          DB_HOST: 'backend'
+          DB_PORT: '3306'
+          DB_OPTS: '{}'
+          MAX_DB_CONN_ATTEMPTS: 30
+        working_dir: /app/website
+        entrypoint: ./start.sh
         labels:
-          NAME: "ottertune"
-        volumes:
-          - media_data:/app/server/website/media
+          NAME: "ottertune-web"
         networks:
           - ottertune-net
 
-    mysql:
+    driver:
+        build:
+          context: ../
+          dockerfile: ./docker/Dockerfile.driver
+        image: ottertune-driver
+        container_name: driver
+        depends_on:
+          - web
+        environment:
+          DEBUG: 'true'
+        working_dir: /app/driver
+        labels:
+          NAME: "ottertune-driver"
+        networks:
+          - ottertune-net
+
+    backend:
         image: mysql:5.7
-        container_name: mysql
+        container_name: backend
         restart: always
         environment:
           MYSQL_ROOT_PASSWORD: 'ottertune'
@@ -53,7 +64,7 @@ services:
         ports:
           - "3306:3306"
         labels:
-          NAME: "mysql"
+          NAME: "ottertune-backend"
         volumes:
           - mysql_data:/var/lib/mysql
         networks:
@@ -72,15 +83,14 @@ services:
            - "15672"
            - "5672"
         ports:
-           - "15672:15672"
-           - "5672:5672"
+           - "15673:15672"
+           - "5673:5672"
         labels:
            NAME: "rabbitmq"
         networks:
           - ottertune-net
 volumes:
    mysql_data:
-   media_data:
 networks:
    ottertune-net:
       driver: bridge

docker/install.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+service="$1"
+
+
+echo ""
+if [ -z "$service" ] || ([ "$service" != "web" ] && [ "$service" != "driver" ]) 
+then
+    echo "Invalid value for service: '$service'"
+    echo ""
+    echo "Usage: $0 [web|driver]"
+    exit 1
+fi
+
+echo ""
+echo "-=------------------------------------------------------"
+echo " Starting installation for service '$service'..."
+echo "-=------------------------------------------------------"
+
+if [ "$DEBUG" = true ]
+then
+    echo ""
+    echo "Environment Variables:"
+    echo " - DEBIAN_FRONTEND: $DEBIAN_FRONTEND"
+    echo " - GRADLE_VERSION:  $GRADLE_VERSION"
+    echo " - GRADLE_HOME:     $GRADLE_HOME"
+    echo " - PATH:            $PATH"
+    echo ""
+fi
+
+apt_pkgs="python3.6 python3-setuptools python3-pip libssl-dev git"
+rm_pkgs=""
+install_gradle=false
+pip_reqs=/requirements.txt
+
+if [ "$service" = "web" ]
+then
+    apt_pkgs="$apt_pkgs python3-dev gcc mysql-client libmysqlclient-dev python-mysqldb postgresql-client"
+
+    rm_pkgs="$rm_pkgs gcc"
+
+else
+    apt_pkgs="$apt_pkgs openssh-server openjdk-11-jdk checkstyle unzip wget"
+
+    # Hack: filter driver pip dependencies
+    >tmp.txt
+    for pip_pkg in autopep8 Fabric3 numpy requests pycodestyle pylint git-lint
+    do
+        grep "^$pip_pkg" "$pip_reqs" >> tmp.txt
+    done
+    mv tmp.txt "$pip_reqs"
+
+    install_gradle=true
+    rm_pkgs="$rm_pkgs unzip wget"
+fi
+
+echo -e "\nUpdating package index..."
+apt-get update
+
+if [ -n "$apt_pkgs" ]
+then
+    # Install required apt packages
+    echo -e "\nInstalling apt packages: $apt_pkgs"
+    apt-get install -y --no-install-recommends $apt_pkgs
+fi
+
+if [ -f "$pip_reqs" ]
+then
+    # Install required pip packages
+    python3 --version
+    pip3 --version
+    echo -e "\nInstalling pip packages: `cat "$pip_reqs" | tr '\n' ' '`" 
+    pip3 install --no-cache-dir --disable-pip-version-check -r "$pip_reqs"
+fi
+
+if [ "$install_gradle" = true ]
+then
+    javac --version
+    echo -e "\nInstalling gradle"
+    wget --no-verbose https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip
+    unzip ${GRADLE_VERSION}-bin.zip -d /opt
+    rm ${GRADLE_VERSION}-bin.zip
+    gradle --version
+fi
+
+if [ -n "$rm_pkgs" ]
+then
+    # Remove packages needed only for install
+    echo -e "\nRemoving packages only required for install: $rm_pkgs"
+    apt-get purge -y --autoremove $rm_pkgs
+fi
+
+rm -rf /var/lib/apt/lists/*
+
+echo ""
+echo "-=------------------------------------------------------"
+echo " Installation complete for service '$service'!"
+echo "-=------------------------------------------------------"
+echo ""
+

docker/start.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Wait for MySQL connection
+# Wait for backend connection
 /bin/bash wait-for-it.sh
 
 ## Needs a connection to a DB so migrations go here

docker/wait-for-it.sh

@@ -1,19 +1,53 @@
 #!/bin/sh
-# wait until MySQL is really available
-maxcounter=${MAX_DB_CONN_ATTEMPTS:-45}
-echo "Trying to connect to mysql, max attempts="$maxcounter
+
+maxcounter=${MAX_DB_CONN_ATTEMPTS:-60}
+
+if [ "$maxcounter" -le 0 ]; then
+    echo "Skipping wait-for-it.sh..."
+    exit 0
+fi
+
+if [ -z "$BACKEND" ]; then
+    echo "ERROR: variable 'BACKEND' must be set. Exiting."
+    exit 1
+fi
+
+# wait until the database is really available
+echo "Trying to connect to $BACKEND (timeout=${maxcounter}s)"
+echo ""
+
+ready () {
+
+    if [ "$BACKEND" = "mysql" ]; then
+        mysql \
+            --host="$DB_HOST" \
+            --protocol TCP \
+            -u"$DB_USER" \
+            -p"$DB_PASSWORD" \
+            -e "show databases;" > /dev/null 2>&1
+    else
+        PGPASSWORD="$DB_PASSWORD" psql \
+            -h "$DB_HOST" \
+            -U "$DB_USER" \
+            -c "select * from pg_database" > /dev/null 2>&1
+    fi
+    return $?
+}
+
  
 counter=1
-while ! mysql --host="$MYSQL_HOST" --protocol TCP -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "show databases;" > /dev/null 2>&1; do
-    sleep 1
+while ! ready; do
     counter=`expr $counter + 1`
+
     if [ $counter -gt $maxcounter ]; then
-        >&2 echo "We have been waiting for MySQL too long already; failing."
+        >&2 echo "ERROR: Could not connect to $BACKEND after $MAX_DB_CONN_ATTEMPTS seconds; Exiting."
         exit 1
     fi;
+    sleep 1
 done
+
 echo "-=------------------------------------------------------"
 echo "-=------------------------------------------------------"
-echo "Connected to MySQL!"
+echo "Connected to $BACKEND!"
 echo "-=------------------------------------------------------"
 echo "-=------------------------------------------------------"