Changes: when we deploy our docker images we now also build/deploy the internal driver image; fixed .dockerignore; moved integration test data into the driver; fixed cast in oracle db_time target objective.

.dockerignore

@@ -1,6 +1,29 @@
-*.swp
-*~
-*.pyc
-*.md
-.travis.yml
-LICENSE
+*
+!client
+client/controller/.gradle
+client/controller/build
+client/controller/output
+client/driver/log
+client/driver/results
+!docker
+docker/Dockerfile*
+docker/docker-compose*.yml
+docker/create-docker-compose.sh
+!server
+server/website/celerybeat-schedule*
+server/website/log
+server/website/*.pid
+server/website/debug_*.tar.gz
+server/website/session_knobs.json
+server/website/dump_website.json
+server/website/script
+server/website/website/settings/*credentials.py
+**/.git*
+**/*.swp
+**/*.swo
+**/*~
+**/*.pyc
+**/*.log
+**/*.bak
+**/.DS_Store
+**/__pycache__

.travis.yml

@@ -17,22 +17,25 @@ matrix:
   include:
   - name: Docker
     env:
-    - BUILD=docker
     - BACKEND=postgresql
     - secure: PZAe5iDaipqsfzQ0/mjFdcQhEMQ0alI3Ap+hWR0X9uWfK8dH6XU/RJLzJXj2o+kpGiCGlj7DJ9uuJy6+/75hq2f7vcs+6JJInPQHUlkj2TPDYWNWaFw0XvIAvV9EA73l4kDy26m/+JiSaF1GsyVD5wdJpVRmw6o7liGfYlQX3fY8kQ4p51+5ufb9QCAselrE70302fTyX4GqxaT1xC2L5fzzIcuKllFVlRILc+04gldiq8u0EAcUO2ovYiW/3+qGPkDcdwd2j3UC1tEGQ+x+Q/7g5JqBHGz6iMy8omH77p3rk4aKV/HCRTZLQLe/Kcd5UiSrQvSIHl5jfIYbzBx7//Ub64c7TlgX+UckK+GPYiwr2N6VmiducFnSjmm3RSW959m1M+M+bqIPbS510zqnIs28xjqIgnWvtpA8mgKdRAtQyDFMYcLc7GQyw0alUo1CQuTf4+Ter+78vt3vYVlCD8lNoMG78xBaLg//pgNQ8sdGWGbUsAYuxxrqI056Ayw16LgrcakL0JWLR/7p+HmCtH2ZT1nQEZJJHSUP3ekibGrE6wgNVpt3DAOwQUG/JiceLn4uVoeljpT+neCoM29wvf4M7JGWg+8SPtg4DHJIAocOVt5EDi8/CtoIa3WJLD2AMI4ladd4PKNzHT4tsYdE2V9d+xkuhXv1/OakAJedtEc=
     - secure: RN4SGagKXgn8gEk9pG2CdM5a/rrBPqkD1nONLcvcqjYzlbniKmhmKT+OxhjVw7Qtoy/RACr4zpIupIjA6COpndSd8/m6WBsWnYD0dt6cDBDLFgHyIh9XNi6/bhNgqPksY9XrENfmeijTENiqpuXeXCuHdaVF0quwvGBxDAIWFNOpPwceAJrwRv9Yb/7w5rmlotSVesScPhwV92Rmu3NK+r/FYANSOFXoBFAYctJSi7dj4A/y3lOFmzE5Y12+Kf6kHuoP+FZO/S/zL1OAPSzedqctRcDpVctEt4AqA7cpGe+Qvate2Hnj63YNlykQD+zlCFHLrqCCKZecdNe+SQVcYqgNk78SX7Nvhfr/xH5lq8HjovqjrfQFc4pVDryomtMeTb7wilN9wsYu6zCneMDHF57hfSKTmiqhz0utEqUiPYwxhkU5uXsXO8VA3aOUlIhM3QPnB6tvkB1h0+qnNzU/lJk6LXy9KkNy9HI8ab+58FSEhj4Uyo9AaBGOB2bEu/d8BsdW1qjGYTI6t4tWZQ8lduOw3xqT5hHhwGB9bz5DomfbECanOGShR3NeMJJGYWETEWek2WK9viP3pllbTFnU3yXfv4ssb00eD2tK+7pyWuP+hL7S1yvrMs5qYRFAEfJhuzTEjtN2uAKumCvU/eh0JjiAPK6XhdT49K1Cd+HC1h4=
+    - secure: "il9BGo4uQhjWOb/+TDdAkXhl4n22DQgmp9F1zP/dFVm/ZgdqNZnIzA9Libe4CiQEUQ+JSEXJqeJCyj657rNUntFZUM6bEzjCSpkDDzkArY4R7ZQQsMOmI5Pv1aKsU+qO899URe9hxmbqnuM23Ke42aY7vT+2qnRTSaC9N6c+4gRXzrcUeAdWU92nt+XPED05LNCr6nTgqrEVAh1434yQsRZZlB54iZKd7n6Plpldr6md7UMe3EmToaOpOHCcEWZmHkHVjWtDxUqb77upTknL2sf83hlhtX0Z3u8o0VQ6wzb6P5dH0jgm08JC+APT03q+iyF87xS7R8BvnkjLhf9iLUpdAfbGzRM98y3hgATkR++E2oSXoqyK4ArMjMLkhntemfMtjMYHYAvXo7IZ9Gh+5JL0fRQvb6XRUX2jGOvtCujPZjMbwCPcWkP9qBL6JRMkrSECzW8yNReW4VR+67blhb9CPT/OVIJYZT69QA+QcWcyU/+tGbBuw/F0vXP6GvntHU/ybwCIu17JoyaBuQJ/YwYR2se1pojYhFX8kUVpRMn1w+a0D/8UdmEJpwwTZuEsNu6y86IUO6B+Tc+jQbNS/nAWfHrsSiFqPjntLThjRr+Uf+1g0qoyKfK54GHuAAEX9xxOZhA3kfeDlyBaTKaU1efT9/rHGQ1PVcOLPaGzrrQ="
+    - secure: "Qi5rZ/CSWspEa1JNAPbq9YU7d8ql4JEJs5pQ+Mo+O8SyoMAJzkWGKKk98zTIWfCASZEnckgXMe8jqrhtPq9kmWWulHOR4P9FrO0+JX7Z47hkeXcqJN4LxP3R4BQgZZQCtYpXYfMAEvPs+nJp7IOQ6cTe80Xiz9eAD3385Hpcmz7tiZcgnnc/Eys90RqjK1GZyWRfeekXVvz39e08goW2y4W5DXZWGQ6QDqLr3Wrg7aK6A9e01otNxJGs8eqd12wzkeSiYi8l96taUnLKW9uUIMhhdoDPrHt4o6BjeWmW7OWzWbppcd3oV1IrUo2WngBsamUohL8PYilt/aEDItsdgLb7RBkFoP0prAlOUIPW6Plk34rk4Sk03KFxeBWSXULE+DgGlJZCFobj8MgxW9IQnJBg3Gk6nmadWAA831Aa3LzmC3dhShwQzFrGAxw/xeVHxodUDD0lwVb1UA4EZJuuNCr4/oyDqgjUcSiTHvR+wLdMFzG0+ySnxtoYt+hmTZJaV6ZffFMn8MIdR1tz+W56H+BwJeDxCpGhBuzyYDFDWH3u5Q08lBK3icgpG6wqfCCaZpjCOuGL8GIMkOEtznmGrrYzqsgJ0AbGizahXIQ57Qh/Wob0qVKGXnDqz6jxPH99aRugNa+snp+SaChz6Q5ojY1bbe4jIW03iOY1m51394A="
     - DOCKER_REPO="${DOCKER_USER}/ottertune"
+    - GIT_REPO="ottertune-driver-internal"
     services:
     - docker
     before_install:
     - docker --version
     - docker-compose --version
+    - env | sort
     install: []
     before_script:
     - sudo service mysql stop || true
     - sudo service postgresql stop || true
     - cd $ROOT/docker
-    - docker-compose build
+    - docker-compose -f docker-compose.build.yml build
     - dcfile_mysql=docker-compose.mysql.yml
     - dcfile_postgres=docker-compose.postgresql.yml
     - BACKEND=mysql WEB_ENTRYPOINT="''" sh create-docker-compose.sh $dcfile_mysql
@@ -45,19 +48,18 @@ matrix:
     - docker-compose -f $dcfile_postgres rm -f -s -v
     - docker-compose -f $dcfile_mysql run --rm web bash -c "./wait-for-it.sh && python3 manage.py makemigrations website && python3 manage.py test --noinput -v 2"
     - docker-compose -f $dcfile_mysql rm -f -s -v
+    after_success: []
     before_deploy:
-    - echo "$DOCKER_PASSWD" | docker login -u "$DOCKER_USER" --password-stdin
-    - docker tag ottertune-web "${DOCKER_REPO}:web"
-    - docker tag ottertune-driver "${DOCKER_REPO}:driver"
+    - /$ROOT/.travis/before_deploy.sh
     deploy:
       provider: script
-      script: docker push "${DOCKER_REPO}:web" && docker push "${DOCKER_REPO}:driver"
+      script: /$ROOT/.travis/deploy.sh 
       on:
+        repo: cmu-db/ottertune
         branch: master
 
   - name: Tests (MySQL v5.7)
     env:
-    - BUILD=tests
     - BACKEND=mysql
     services:
     - mysql
@@ -73,12 +75,10 @@ matrix:
         - checkstyle
     before_install:
     - mysql -e "CREATE DATABASE IF NOT EXISTS ${DB_NAME}"
-    - mysql -e "CREATE DATABASE IF NOT EXISTS test_${DB_NAME}"
     - sed -i '/psycopg2/d' $WEB/requirements.txt
 
   - name: Tests (PostgreSQL v9.6)
     env:
-    - BUILD=unittests
     - BACKEND=postgresql
     addons:
       postgresql: "9.6"
@@ -91,13 +91,13 @@ matrix:
         - checkstyle
     before_install:
     - psql -U postgres -c "CREATE DATABASE ${DB_NAME}"
-    - psql -U postgres -c "CREATE DATABASE test_${DB_NAME}"
     - sed -i '/mysqlclient/d' $WEB/requirements.txt
 
 install:
 - pip install codecov -r $WEB/requirements.txt
 - pip freeze
 before_script:
+- env | sort
 - cd $WEB
 - sed -i  "s|\('celery', 'db.*$\)|'console', \1|" website/settings/common.py
 - cp $ROOT/docker/credentials.py website/settings

.travis/before_deploy.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+cd $ROOT/docker
+mkdir tmp
+cd tmp
+git clone "https://${GIT_TOKEN}@github.com/${GIT_ORG}/${GIT_REPO}"
+cd $GIT_REPO
+cp $ROOT/docker/install.sh $WEB/requirements.txt .
+docker-compose -f docker-compose.build.yml build
+
+docker tag ottertune-web "${DOCKER_REPO}:web"
+docker tag ottertune-driver "${DOCKER_REPO}:driver"
+docker tag ottertune-driver-internal "${DOCKER_REPO}:driver-internal"
+echo "$DOCKER_PASSWD" | docker login -u "$DOCKER_USER" --password-stdin
+

.travis/deploy.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+images="base web driver driver-internal"
+
+for img in $images
+do
+    docker push "${DOCKER_REPO}:${img}"
+done
+

client/driver/fabfile.py

@@ -744,21 +744,21 @@ def integration_tests():
 
     # Upload training data
     LOG.info('Upload training data to no tuning session')
-    upload_batch(result_dir='../../integrationTests/data/', upload_code='ottertuneTestNoTuning')
+    upload_batch(result_dir='./integrationTests/data/', upload_code='ottertuneTestNoTuning')
 
     # wait celery periodic task finishes
     assert wait_pipeline_data_ready(), "Pipeline data failed"
 
     # Test DNN
     LOG.info('Test DNN (deep neural network)')
-    upload_result(result_dir='../../integrationTests/data/', prefix='0__',
+    upload_result(result_dir='./integrationTests/data/', prefix='0__',
                   upload_code='ottertuneTestTuningDNN')
     response = get_result(upload_code='ottertuneTestTuningDNN')
     assert response['status'] == 'good'
 
     # Test GPR
     LOG.info('Test GPR (gaussian process regression)')
-    upload_result(result_dir='../../integrationTests/data/', prefix='0__',
+    upload_result(result_dir='./integrationTests/data/', prefix='0__',
                   upload_code='ottertuneTestTuningGPR')
     response = get_result(upload_code='ottertuneTestTuningGPR')
     assert response['status'] == 'good'

docker/.dockerignore

@@ -1 +0,0 @@
-*.swp

docker/.gitignore

@@ -0,0 +1,3 @@
+docker-compose*.yml
+!docker-compose.build.yml
+!docker-compose.up.yml

docker/Dockerfile.base

@@ -0,0 +1,14 @@
+FROM ubuntu:18.04
+
+ARG DEBUG=true
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY ./docker/install.sh ./server/website/requirements.txt /
+WORKDIR /
+
+RUN mkdir -p /app \
+    && chmod +x install.sh \
+    && ./install.sh base
+
+ENV DEBUG=$DEBUG

docker/Dockerfile.driver

@@ -1,17 +1,11 @@
-FROM ubuntu:18.04
+FROM ottertune-base
 
 ARG GRADLE_VERSION=gradle-5.5.1
 
-ENV DEBIAN_FRONTEND=noninteractive 
 ENV GRADLE_HOME=/opt/${GRADLE_VERSION}
 ENV PATH=${GRADLE_HOME}/bin:${PATH}
 
-COPY ./docker/install.sh ./server/website/requirements.txt /
-WORKDIR /
-
-RUN mkdir -p /app \
-    && chmod +x install.sh \
-    && sh install.sh driver
+RUN /install.sh driver
 
 COPY ./client /app
 

docker/Dockerfile.web

@@ -1,13 +1,4 @@
-FROM ubuntu:18.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-COPY ./docker/install.sh ./server/website/requirements.txt /
-WORKDIR /
-
-RUN mkdir -p /app \
-    && chmod +x install.sh \
-    && sh install.sh web
+FROM ottertune-base
 
 COPY ./server /app
 
@@ -16,7 +7,8 @@ WORKDIR /app/website
 COPY ./docker/credentials.py ./website/settings
 COPY ./docker/start.sh ./docker/start-test.sh ./docker/wait-for-it.sh ./
 
-RUN chmod +x ./*.sh \
+RUN /install.sh web \
+    && chmod +x ./*.sh \
     && sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \
     && mv tmp ./website/settings/common.py
 

docker/create-docker-compose.sh

@@ -20,7 +20,7 @@ else
     DB_PORT="${DB_PORT:-5432}"
 fi
 
-WEB_ENTRYPOINT="${WEB_ENTRYPOINT:-start.sh}"
+WEB_ENTRYPOINT="${WEB_ENTRYPOINT:-./start.sh}"
 
 file="$(test -z "$1" && echo "docker-compose.$BACKEND.yml" || echo "$1")"
 
@@ -30,9 +30,6 @@ version: "3"
 services:
 
     web:
-        build:
-          context: ../
-          dockerfile: ./docker/Dockerfile.web
         image: ottertune-web
         container_name: web
         expose:
@@ -63,9 +60,6 @@ services:
           - ottertune-net
 
     driver:
-        build:
-          context: ../
-          dockerfile: ./docker/Dockerfile.driver
         image: ottertune-driver
         container_name: driver
         depends_on:

docker/credentials.py

@@ -4,7 +4,7 @@ import string
 from os import environ as env
 
 debug = env.get('DEBUG', 'true').lower() == 'true'
-backend = env.get('BACKEND', 'mysql')
+backend = env.get('BACKEND', 'postgresql')
 db_name = env.get('DB_NAME', 'ottertune')
 db_host = env.get('DB_HOST', 'localhost')
 db_pwd = env.get('DB_PASSWORD', '')

docker/docker-compose.build.yml

@@ -0,0 +1,32 @@
+version: "3"
+services:
+
+    base:
+        build:
+          context: ../
+          dockerfile: ./docker/Dockerfile.base
+          args:
+            DEBUG: "true"
+        image: ottertune-base
+        labels:
+          NAME: "ottertune-base"
+
+    web:
+        build:
+          context: ../
+          dockerfile: ./docker/Dockerfile.web
+        image: ottertune-web
+        depends_on:
+          - base
+        labels:
+          NAME: "ottertune-web"
+
+    driver:
+        build:
+          context: ../
+          dockerfile: ./docker/Dockerfile.driver
+        image: ottertune-driver
+        depends_on:
+          - base
+        labels:
+          NAME: "ottertune-driver"

docker/docker-compose.up.yml

@@ -2,9 +2,6 @@ version: "3"
 services:
 
     web:
-        build:
-          context: ../
-          dockerfile: ./docker/Dockerfile.web
         image: ottertune-web
         container_name: web
         expose:
@@ -20,12 +17,12 @@ services:
         environment:
           DEBUG: 'true'
           ADMIN_PASSWORD: 'changeme'
-          BACKEND: 'mysql'
+          BACKEND: 'postgresql'
           DB_NAME: 'ottertune'
-          DB_USER: 'root'
+          DB_USER: 'postgres'
           DB_PASSWORD: 'ottertune'
           DB_HOST: 'backend'
-          DB_PORT: '3306'
+          DB_PORT: '5432'
           DB_OPTS: '{}'
           MAX_DB_CONN_ATTEMPTS: 30
         working_dir: /app/website
@@ -36,9 +33,6 @@ services:
           - ottertune-net
 
     driver:
-        build:
-          context: ../
-          dockerfile: ./docker/Dockerfile.driver
         image: ottertune-driver
         container_name: driver
         depends_on:
@@ -51,25 +45,6 @@ services:
         networks:
           - ottertune-net
 
-    backend:
-        image: mysql:5.7
-        container_name: backend
-        restart: always
-        environment:
-          MYSQL_ROOT_PASSWORD: 'ottertune'
-          MYSQL_PASSWORD: 'ottertune'
-          MYSQL_DATABASE: 'ottertune'
-        expose:
-          - "3306"
-        ports:
-          - "3306:3306"
-        labels:
-          NAME: "ottertune-backend"
-        volumes:
-          - mysql_data:/var/lib/mysql
-        networks:
-          - ottertune-net
-
     rabbitmq:
         image: "rabbitmq:3-management"
         container_name: rabbitmq
@@ -89,9 +64,24 @@ services:
            NAME: "rabbitmq"
         networks:
           - ottertune-net
-volumes:
-   mysql_data:
+
+    backend:
+        container_name: backend
+        restart: always
+        image: postgres:9.6
+        environment:
+          POSTGRES_USER: 'postgres'
+          POSTGRES_PASSWORD: 'ottertune'
+          POSTGRES_DB: 'ottertune'
+        expose:
+          - "5432"
+        ports:
+          - "5432:5432"
+        labels:
+          NAME: "ottertune-backend"
+        networks:
+          - ottertune-net
+
 networks:
    ottertune-net:
       driver: bridge
-

docker/install.sh

@@ -2,16 +2,6 @@
 
 service="$1"
 
-
-echo ""
-if [ -z "$service" ] || ([ "$service" != "web" ] && [ "$service" != "driver" ]) 
-then
-    echo "Invalid value for service: '$service'"
-    echo ""
-    echo "Usage: $0 [web|driver]"
-    exit 1
-fi
-
 echo ""
 echo "-=------------------------------------------------------"
 echo " Starting installation for service '$service'..."
@@ -19,39 +9,65 @@ echo "-=------------------------------------------------------"
 
 if [ "$DEBUG" = true ]
 then
+    echo ""
+    echo "Command-line Args:"
+    echo " - service: $service"
     echo ""
     echo "Environment Variables:"
     echo " - DEBIAN_FRONTEND: $DEBIAN_FRONTEND"
+    echo " - PATH:            $PATH"
     echo " - GRADLE_VERSION:  $GRADLE_VERSION"
     echo " - GRADLE_HOME:     $GRADLE_HOME"
-    echo " - PATH:            $PATH"
     echo ""
 fi
 
-apt_pkgs="python3.6 python3-setuptools python3-pip libssl-dev git"
+apt_pkgs=""
 rm_pkgs=""
 install_gradle=false
-pip_reqs=/requirements.txt
+pip_common_pkgs="Fabric3 numpy requests"
+master_pip_reqs_file=/requirements.txt
+pip_reqs_file="/${service}-requirements.txt"
 
-if [ "$service" = "web" ]
+if [ "$service" = "base" ]
 then
-    apt_pkgs="$apt_pkgs python3-dev gcc mysql-client libmysqlclient-dev python-mysqldb postgresql-client"
+    apt_pkgs="python3.6 python3-setuptools python3-pip libssl-dev"
 
-    rm_pkgs="$rm_pkgs gcc"
+    # Filter common pip packages
+    for pip_pkg in $pip_common_pkgs
+    do
+        grep "^$pip_pkg" "$master_pip_reqs_file" >> "$pip_reqs_file"
+    done
+
+elif [ "$service" = "web" ]
+then
+    apt_pkgs="python3-dev gcc mysql-client libmysqlclient-dev python-mysqldb postgresql-client"
+
+    rm_pkgs="gcc"
+
+    pip_skip_pkgs="$pip_common_pkgs astroid autopep8 git-lint pycodestyle pylint"
+    cp "$master_pip_reqs_file" "$pip_reqs_file"
+
+    for pip_pkg in $pip_skip_pkgs
+    do
+        sed -i "/$pip_pkg/d" "$pip_reqs_file"
+    done
+
+elif [ "$service" = "driver" ]
+then
+    apt_pkgs="openssh-server openjdk-11-jdk unzip wget"
+    rm_pkgs="unzip wget"
+    install_gradle=true
+
+elif [ "$service" = "driver-internal" ]
+then
+    apt_pkgs="openssh-server vim"
 
 else
-    apt_pkgs="$apt_pkgs openssh-server openjdk-11-jdk checkstyle unzip wget"
-
-    # Hack: filter driver pip dependencies
-    >tmp.txt
-    for pip_pkg in autopep8 Fabric3 numpy requests pycodestyle pylint git-lint
-    do
-        grep "^$pip_pkg" "$pip_reqs" >> tmp.txt
-    done
-    mv tmp.txt "$pip_reqs"
-
-    install_gradle=true
-    rm_pkgs="$rm_pkgs unzip wget"
+    echo ""
+    echo "ERROR: Invalid value for service: '$service'"
+    echo ""
+    echo "Usage: $0 [base|web|driver|driver-internal]"
+    exit 1
 fi
 
 echo -e "\nUpdating package index..."
@@ -64,13 +80,13 @@ then
     apt-get install -y --no-install-recommends $apt_pkgs
 fi
 
-if [ -f "$pip_reqs" ]
+if [ -f "$pip_reqs_file" ] && [ -s "$pip_reqs_file" ]
 then
     # Install required pip packages
     python3 --version
     pip3 --version
-    echo -e "\nInstalling pip packages: `cat "$pip_reqs" | tr '\n' ' '`" 
-    pip3 install --no-cache-dir --disable-pip-version-check -r "$pip_reqs"
+    echo -e "\nInstalling pip packages: `cat "$pip_reqs_file" | tr '\n' ' '`"
+    pip3 install --no-cache-dir --disable-pip-version-check -r "$pip_reqs_file"
 fi
 
 if [ "$install_gradle" = true ]

docker/start.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+addrport="0.0.0.0:8000"
+
 # Wait for backend connection
 /bin/bash wait-for-it.sh
 
@@ -7,6 +9,10 @@
 python3 manage.py makemigrations website
 python3 manage.py migrate
 python3 manage.py createuser admin $ADMIN_PASSWORD --superuser
-
 python3 manage.py startcelery
-python3 manage.py runserver 0.0.0.0:8000
+
+echo ""
+echo "-=------------------------------------------------------"
+echo " Starting the web server on '$addrport'..."
+echo "-=------------------------------------------------------"
+python3 manage.py runserver "$addrport"

server/website/website/db/oracle/target_objective.py

@@ -15,7 +15,7 @@ class DBTime(BaseTargetObjective):
                          improvement=LESS_IS_BETTER)
 
     def compute(self, metrics, observation_time):
-        return float('global.sys_time_model.db time')
+        return float(metrics['global.sys_time_model.db time'])
 
 
 target_objective_list = tuple((DBMSType.ORACLE, target_obj) for target_obj in [  # pylint: disable=invalid-name