Changes: when we deploy our docker images we now also build/deploy the internal driver image; fixed .dockerignore; moved integration test data into the driver; fixed cast in oracle db_time target objective.

This commit is contained in:
dvanaken 2019-12-03 21:26:59 -05:00 committed by Dana Van Aken
parent c4433181f7
commit 7be5b89975
57 changed files with 200 additions and 113 deletions

View File

@ -1,6 +1,29 @@
*.swp
*~
*.pyc
*.md
.travis.yml
LICENSE
*
!client
client/controller/.gradle
client/controller/build
client/controller/output
client/driver/log
client/driver/results
!docker
docker/Dockerfile*
docker/docker-compose*.yml
docker/create-docker-compose.sh
!server
server/website/celerybeat-schedule*
server/website/log
server/website/*.pid
server/website/debug_*.tar.gz
server/website/session_knobs.json
server/website/dump_website.json
server/website/script
server/website/website/settings/*credentials.py
**/.git*
**/*.swp
**/*.swo
**/*~
**/*.pyc
**/*.log
**/*.bak
**/.DS_Store
**/__pycache__

View File

@ -17,22 +17,25 @@ matrix:
include:
- name: Docker
env:
- BUILD=docker
- BACKEND=postgresql
- secure: PZAe5iDaipqsfzQ0/mjFdcQhEMQ0alI3Ap+hWR0X9uWfK8dH6XU/RJLzJXj2o+kpGiCGlj7DJ9uuJy6+/75hq2f7vcs+6JJInPQHUlkj2TPDYWNWaFw0XvIAvV9EA73l4kDy26m/+JiSaF1GsyVD5wdJpVRmw6o7liGfYlQX3fY8kQ4p51+5ufb9QCAselrE70302fTyX4GqxaT1xC2L5fzzIcuKllFVlRILc+04gldiq8u0EAcUO2ovYiW/3+qGPkDcdwd2j3UC1tEGQ+x+Q/7g5JqBHGz6iMy8omH77p3rk4aKV/HCRTZLQLe/Kcd5UiSrQvSIHl5jfIYbzBx7//Ub64c7TlgX+UckK+GPYiwr2N6VmiducFnSjmm3RSW959m1M+M+bqIPbS510zqnIs28xjqIgnWvtpA8mgKdRAtQyDFMYcLc7GQyw0alUo1CQuTf4+Ter+78vt3vYVlCD8lNoMG78xBaLg//pgNQ8sdGWGbUsAYuxxrqI056Ayw16LgrcakL0JWLR/7p+HmCtH2ZT1nQEZJJHSUP3ekibGrE6wgNVpt3DAOwQUG/JiceLn4uVoeljpT+neCoM29wvf4M7JGWg+8SPtg4DHJIAocOVt5EDi8/CtoIa3WJLD2AMI4ladd4PKNzHT4tsYdE2V9d+xkuhXv1/OakAJedtEc=
- secure: RN4SGagKXgn8gEk9pG2CdM5a/rrBPqkD1nONLcvcqjYzlbniKmhmKT+OxhjVw7Qtoy/RACr4zpIupIjA6COpndSd8/m6WBsWnYD0dt6cDBDLFgHyIh9XNi6/bhNgqPksY9XrENfmeijTENiqpuXeXCuHdaVF0quwvGBxDAIWFNOpPwceAJrwRv9Yb/7w5rmlotSVesScPhwV92Rmu3NK+r/FYANSOFXoBFAYctJSi7dj4A/y3lOFmzE5Y12+Kf6kHuoP+FZO/S/zL1OAPSzedqctRcDpVctEt4AqA7cpGe+Qvate2Hnj63YNlykQD+zlCFHLrqCCKZecdNe+SQVcYqgNk78SX7Nvhfr/xH5lq8HjovqjrfQFc4pVDryomtMeTb7wilN9wsYu6zCneMDHF57hfSKTmiqhz0utEqUiPYwxhkU5uXsXO8VA3aOUlIhM3QPnB6tvkB1h0+qnNzU/lJk6LXy9KkNy9HI8ab+58FSEhj4Uyo9AaBGOB2bEu/d8BsdW1qjGYTI6t4tWZQ8lduOw3xqT5hHhwGB9bz5DomfbECanOGShR3NeMJJGYWETEWek2WK9viP3pllbTFnU3yXfv4ssb00eD2tK+7pyWuP+hL7S1yvrMs5qYRFAEfJhuzTEjtN2uAKumCvU/eh0JjiAPK6XhdT49K1Cd+HC1h4=
- secure: "il9BGo4uQhjWOb/+TDdAkXhl4n22DQgmp9F1zP/dFVm/ZgdqNZnIzA9Libe4CiQEUQ+JSEXJqeJCyj657rNUntFZUM6bEzjCSpkDDzkArY4R7ZQQsMOmI5Pv1aKsU+qO899URe9hxmbqnuM23Ke42aY7vT+2qnRTSaC9N6c+4gRXzrcUeAdWU92nt+XPED05LNCr6nTgqrEVAh1434yQsRZZlB54iZKd7n6Plpldr6md7UMe3EmToaOpOHCcEWZmHkHVjWtDxUqb77upTknL2sf83hlhtX0Z3u8o0VQ6wzb6P5dH0jgm08JC+APT03q+iyF87xS7R8BvnkjLhf9iLUpdAfbGzRM98y3hgATkR++E2oSXoqyK4ArMjMLkhntemfMtjMYHYAvXo7IZ9Gh+5JL0fRQvb6XRUX2jGOvtCujPZjMbwCPcWkP9qBL6JRMkrSECzW8yNReW4VR+67blhb9CPT/OVIJYZT69QA+QcWcyU/+tGbBuw/F0vXP6GvntHU/ybwCIu17JoyaBuQJ/YwYR2se1pojYhFX8kUVpRMn1w+a0D/8UdmEJpwwTZuEsNu6y86IUO6B+Tc+jQbNS/nAWfHrsSiFqPjntLThjRr+Uf+1g0qoyKfK54GHuAAEX9xxOZhA3kfeDlyBaTKaU1efT9/rHGQ1PVcOLPaGzrrQ="
- secure: "Qi5rZ/CSWspEa1JNAPbq9YU7d8ql4JEJs5pQ+Mo+O8SyoMAJzkWGKKk98zTIWfCASZEnckgXMe8jqrhtPq9kmWWulHOR4P9FrO0+JX7Z47hkeXcqJN4LxP3R4BQgZZQCtYpXYfMAEvPs+nJp7IOQ6cTe80Xiz9eAD3385Hpcmz7tiZcgnnc/Eys90RqjK1GZyWRfeekXVvz39e08goW2y4W5DXZWGQ6QDqLr3Wrg7aK6A9e01otNxJGs8eqd12wzkeSiYi8l96taUnLKW9uUIMhhdoDPrHt4o6BjeWmW7OWzWbppcd3oV1IrUo2WngBsamUohL8PYilt/aEDItsdgLb7RBkFoP0prAlOUIPW6Plk34rk4Sk03KFxeBWSXULE+DgGlJZCFobj8MgxW9IQnJBg3Gk6nmadWAA831Aa3LzmC3dhShwQzFrGAxw/xeVHxodUDD0lwVb1UA4EZJuuNCr4/oyDqgjUcSiTHvR+wLdMFzG0+ySnxtoYt+hmTZJaV6ZffFMn8MIdR1tz+W56H+BwJeDxCpGhBuzyYDFDWH3u5Q08lBK3icgpG6wqfCCaZpjCOuGL8GIMkOEtznmGrrYzqsgJ0AbGizahXIQ57Qh/Wob0qVKGXnDqz6jxPH99aRugNa+snp+SaChz6Q5ojY1bbe4jIW03iOY1m51394A="
- DOCKER_REPO="${DOCKER_USER}/ottertune"
- GIT_REPO="ottertune-driver-internal"
services:
- docker
before_install:
- docker --version
- docker-compose --version
- env | sort
install: []
before_script:
- sudo service mysql stop || true
- sudo service postgresql stop || true
- cd $ROOT/docker
- docker-compose build
- docker-compose -f docker-compose.build.yml build
- dcfile_mysql=docker-compose.mysql.yml
- dcfile_postgres=docker-compose.postgresql.yml
- BACKEND=mysql WEB_ENTRYPOINT="''" sh create-docker-compose.sh $dcfile_mysql
@ -45,19 +48,18 @@ matrix:
- docker-compose -f $dcfile_postgres rm -f -s -v
- docker-compose -f $dcfile_mysql run --rm web bash -c "./wait-for-it.sh && python3 manage.py makemigrations website && python3 manage.py test --noinput -v 2"
- docker-compose -f $dcfile_mysql rm -f -s -v
after_success: []
before_deploy:
- echo "$DOCKER_PASSWD" | docker login -u "$DOCKER_USER" --password-stdin
- docker tag ottertune-web "${DOCKER_REPO}:web"
- docker tag ottertune-driver "${DOCKER_REPO}:driver"
- /$ROOT/.travis/before_deploy.sh
deploy:
provider: script
script: docker push "${DOCKER_REPO}:web" && docker push "${DOCKER_REPO}:driver"
script: /$ROOT/.travis/deploy.sh
on:
repo: cmu-db/ottertune
branch: master
- name: Tests (MySQL v5.7)
env:
- BUILD=tests
- BACKEND=mysql
services:
- mysql
@ -73,12 +75,10 @@ matrix:
- checkstyle
before_install:
- mysql -e "CREATE DATABASE IF NOT EXISTS ${DB_NAME}"
- mysql -e "CREATE DATABASE IF NOT EXISTS test_${DB_NAME}"
- sed -i '/psycopg2/d' $WEB/requirements.txt
- name: Tests (PostgreSQL v9.6)
env:
- BUILD=unittests
- BACKEND=postgresql
addons:
postgresql: "9.6"
@ -91,13 +91,13 @@ matrix:
- checkstyle
before_install:
- psql -U postgres -c "CREATE DATABASE ${DB_NAME}"
- psql -U postgres -c "CREATE DATABASE test_${DB_NAME}"
- sed -i '/mysqlclient/d' $WEB/requirements.txt
install:
- pip install codecov -r $WEB/requirements.txt
- pip freeze
before_script:
- env | sort
- cd $WEB
- sed -i "s|\('celery', 'db.*$\)|'console', \1|" website/settings/common.py
- cp $ROOT/docker/credentials.py website/settings

15
.travis/before_deploy.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash
cd $ROOT/docker
mkdir tmp
cd tmp
git clone "https://${GIT_TOKEN}@github.com/${GIT_ORG}/${GIT_REPO}"
cd $GIT_REPO
cp $ROOT/docker/install.sh $WEB/requirements.txt .
docker-compose -f docker-compose.build.yml build
docker tag ottertune-web "${DOCKER_REPO}:web"
docker tag ottertune-driver "${DOCKER_REPO}:driver"
docker tag ottertune-driver-internal "${DOCKER_REPO}:driver-internal"
echo "$DOCKER_PASSWD" | docker login -u "$DOCKER_USER" --password-stdin

9
.travis/deploy.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/bash
images="base web driver driver-internal"
for img in $images
do
docker push "${DOCKER_REPO}:${img}"
done

View File

@ -744,21 +744,21 @@ def integration_tests():
# Upload training data
LOG.info('Upload training data to no tuning session')
upload_batch(result_dir='../../integrationTests/data/', upload_code='ottertuneTestNoTuning')
upload_batch(result_dir='./integrationTests/data/', upload_code='ottertuneTestNoTuning')
# wait celery periodic task finishes
assert wait_pipeline_data_ready(), "Pipeline data failed"
# Test DNN
LOG.info('Test DNN (deep neural network)')
upload_result(result_dir='../../integrationTests/data/', prefix='0__',
upload_result(result_dir='./integrationTests/data/', prefix='0__',
upload_code='ottertuneTestTuningDNN')
response = get_result(upload_code='ottertuneTestTuningDNN')
assert response['status'] == 'good'
# Test GPR
LOG.info('Test GPR (gaussian process regression)')
upload_result(result_dir='../../integrationTests/data/', prefix='0__',
upload_result(result_dir='./integrationTests/data/', prefix='0__',
upload_code='ottertuneTestTuningGPR')
response = get_result(upload_code='ottertuneTestTuningGPR')
assert response['status'] == 'good'

View File

@ -1 +0,0 @@
*.swp

3
docker/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
docker-compose*.yml
!docker-compose.build.yml
!docker-compose.up.yml

14
docker/Dockerfile.base Normal file
View File

@ -0,0 +1,14 @@
FROM ubuntu:18.04
ARG DEBUG=true
ENV DEBIAN_FRONTEND=noninteractive
COPY ./docker/install.sh ./server/website/requirements.txt /
WORKDIR /
RUN mkdir -p /app \
&& chmod +x install.sh \
&& ./install.sh base
ENV DEBUG=$DEBUG

View File

@ -1,17 +1,11 @@
FROM ubuntu:18.04
FROM ottertune-base
ARG GRADLE_VERSION=gradle-5.5.1
ENV DEBIAN_FRONTEND=noninteractive
ENV GRADLE_HOME=/opt/${GRADLE_VERSION}
ENV PATH=${GRADLE_HOME}/bin:${PATH}
COPY ./docker/install.sh ./server/website/requirements.txt /
WORKDIR /
RUN mkdir -p /app \
&& chmod +x install.sh \
&& sh install.sh driver
RUN /install.sh driver
COPY ./client /app

View File

@ -1,13 +1,4 @@
FROM ubuntu:18.04
ENV DEBIAN_FRONTEND=noninteractive
COPY ./docker/install.sh ./server/website/requirements.txt /
WORKDIR /
RUN mkdir -p /app \
&& chmod +x install.sh \
&& sh install.sh web
FROM ottertune-base
COPY ./server /app
@ -16,7 +7,8 @@ WORKDIR /app/website
COPY ./docker/credentials.py ./website/settings
COPY ./docker/start.sh ./docker/start-test.sh ./docker/wait-for-it.sh ./
RUN chmod +x ./*.sh \
RUN /install.sh web \
&& chmod +x ./*.sh \
&& sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \
&& mv tmp ./website/settings/common.py

View File

@ -20,7 +20,7 @@ else
DB_PORT="${DB_PORT:-5432}"
fi
WEB_ENTRYPOINT="${WEB_ENTRYPOINT:-start.sh}"
WEB_ENTRYPOINT="${WEB_ENTRYPOINT:-./start.sh}"
file="$(test -z "$1" && echo "docker-compose.$BACKEND.yml" || echo "$1")"
@ -30,9 +30,6 @@ version: "3"
services:
web:
build:
context: ../
dockerfile: ./docker/Dockerfile.web
image: ottertune-web
container_name: web
expose:
@ -63,9 +60,6 @@ services:
- ottertune-net
driver:
build:
context: ../
dockerfile: ./docker/Dockerfile.driver
image: ottertune-driver
container_name: driver
depends_on:

View File

@ -4,7 +4,7 @@ import string
from os import environ as env
debug = env.get('DEBUG', 'true').lower() == 'true'
backend = env.get('BACKEND', 'mysql')
backend = env.get('BACKEND', 'postgresql')
db_name = env.get('DB_NAME', 'ottertune')
db_host = env.get('DB_HOST', 'localhost')
db_pwd = env.get('DB_PASSWORD', '')

View File

@ -0,0 +1,32 @@
version: "3"
services:
base:
build:
context: ../
dockerfile: ./docker/Dockerfile.base
args:
DEBUG: "true"
image: ottertune-base
labels:
NAME: "ottertune-base"
web:
build:
context: ../
dockerfile: ./docker/Dockerfile.web
image: ottertune-web
depends_on:
- base
labels:
NAME: "ottertune-web"
driver:
build:
context: ../
dockerfile: ./docker/Dockerfile.driver
image: ottertune-driver
depends_on:
- base
labels:
NAME: "ottertune-driver"

View File

@ -2,9 +2,6 @@ version: "3"
services:
web:
build:
context: ../
dockerfile: ./docker/Dockerfile.web
image: ottertune-web
container_name: web
expose:
@ -20,12 +17,12 @@ services:
environment:
DEBUG: 'true'
ADMIN_PASSWORD: 'changeme'
BACKEND: 'mysql'
BACKEND: 'postgresql'
DB_NAME: 'ottertune'
DB_USER: 'root'
DB_USER: 'postgres'
DB_PASSWORD: 'ottertune'
DB_HOST: 'backend'
DB_PORT: '3306'
DB_PORT: '5432'
DB_OPTS: '{}'
MAX_DB_CONN_ATTEMPTS: 30
working_dir: /app/website
@ -36,9 +33,6 @@ services:
- ottertune-net
driver:
build:
context: ../
dockerfile: ./docker/Dockerfile.driver
image: ottertune-driver
container_name: driver
depends_on:
@ -51,25 +45,6 @@ services:
networks:
- ottertune-net
backend:
image: mysql:5.7
container_name: backend
restart: always
environment:
MYSQL_ROOT_PASSWORD: 'ottertune'
MYSQL_PASSWORD: 'ottertune'
MYSQL_DATABASE: 'ottertune'
expose:
- "3306"
ports:
- "3306:3306"
labels:
NAME: "ottertune-backend"
volumes:
- mysql_data:/var/lib/mysql
networks:
- ottertune-net
rabbitmq:
image: "rabbitmq:3-management"
container_name: rabbitmq
@ -89,9 +64,24 @@ services:
NAME: "rabbitmq"
networks:
- ottertune-net
volumes:
mysql_data:
backend:
container_name: backend
restart: always
image: postgres:9.6
environment:
POSTGRES_USER: 'postgres'
POSTGRES_PASSWORD: 'ottertune'
POSTGRES_DB: 'ottertune'
expose:
- "5432"
ports:
- "5432:5432"
labels:
NAME: "ottertune-backend"
networks:
- ottertune-net
networks:
ottertune-net:
driver: bridge

View File

@ -2,16 +2,6 @@
service="$1"
echo ""
if [ -z "$service" ] || ([ "$service" != "web" ] && [ "$service" != "driver" ])
then
echo "Invalid value for service: '$service'"
echo ""
echo "Usage: $0 [web|driver]"
exit 1
fi
echo ""
echo "-=------------------------------------------------------"
echo " Starting installation for service '$service'..."
@ -19,39 +9,65 @@ echo "-=------------------------------------------------------"
if [ "$DEBUG" = true ]
then
echo ""
echo "Command-line Args:"
echo " - service: $service"
echo ""
echo "Environment Variables:"
echo " - DEBIAN_FRONTEND: $DEBIAN_FRONTEND"
echo " - PATH: $PATH"
echo " - GRADLE_VERSION: $GRADLE_VERSION"
echo " - GRADLE_HOME: $GRADLE_HOME"
echo " - PATH: $PATH"
echo ""
fi
apt_pkgs="python3.6 python3-setuptools python3-pip libssl-dev git"
apt_pkgs=""
rm_pkgs=""
install_gradle=false
pip_reqs=/requirements.txt
pip_common_pkgs="Fabric3 numpy requests"
master_pip_reqs_file=/requirements.txt
pip_reqs_file="/${service}-requirements.txt"
if [ "$service" = "web" ]
if [ "$service" = "base" ]
then
apt_pkgs="$apt_pkgs python3-dev gcc mysql-client libmysqlclient-dev python-mysqldb postgresql-client"
apt_pkgs="python3.6 python3-setuptools python3-pip libssl-dev"
rm_pkgs="$rm_pkgs gcc"
# Filter common pip packages
for pip_pkg in $pip_common_pkgs
do
grep "^$pip_pkg" "$master_pip_reqs_file" >> "$pip_reqs_file"
done
elif [ "$service" = "web" ]
then
apt_pkgs="python3-dev gcc mysql-client libmysqlclient-dev python-mysqldb postgresql-client"
rm_pkgs="gcc"
pip_skip_pkgs="$pip_common_pkgs astroid autopep8 git-lint pycodestyle pylint"
cp "$master_pip_reqs_file" "$pip_reqs_file"
for pip_pkg in $pip_skip_pkgs
do
sed -i "/$pip_pkg/d" "$pip_reqs_file"
done
elif [ "$service" = "driver" ]
then
apt_pkgs="openssh-server openjdk-11-jdk unzip wget"
rm_pkgs="unzip wget"
install_gradle=true
elif [ "$service" = "driver-internal" ]
then
apt_pkgs="openssh-server vim"
else
apt_pkgs="$apt_pkgs openssh-server openjdk-11-jdk checkstyle unzip wget"
# Hack: filter driver pip dependencies
>tmp.txt
for pip_pkg in autopep8 Fabric3 numpy requests pycodestyle pylint git-lint
do
grep "^$pip_pkg" "$pip_reqs" >> tmp.txt
done
mv tmp.txt "$pip_reqs"
install_gradle=true
rm_pkgs="$rm_pkgs unzip wget"
echo ""
echo "ERROR: Invalid value for service: '$service'"
echo ""
echo "Usage: $0 [base|web|driver|driver-internal]"
exit 1
fi
echo -e "\nUpdating package index..."
@ -64,13 +80,13 @@ then
apt-get install -y --no-install-recommends $apt_pkgs
fi
if [ -f "$pip_reqs" ]
if [ -f "$pip_reqs_file" ] && [ -s "$pip_reqs_file" ]
then
# Install required pip packages
python3 --version
pip3 --version
echo -e "\nInstalling pip packages: `cat "$pip_reqs" | tr '\n' ' '`"
pip3 install --no-cache-dir --disable-pip-version-check -r "$pip_reqs"
echo -e "\nInstalling pip packages: `cat "$pip_reqs_file" | tr '\n' ' '`"
pip3 install --no-cache-dir --disable-pip-version-check -r "$pip_reqs_file"
fi
if [ "$install_gradle" = true ]

View File

@ -1,5 +1,7 @@
#!/bin/bash
addrport="0.0.0.0:8000"
# Wait for backend connection
/bin/bash wait-for-it.sh
@ -7,6 +9,10 @@
python3 manage.py makemigrations website
python3 manage.py migrate
python3 manage.py createuser admin $ADMIN_PASSWORD --superuser
python3 manage.py startcelery
python3 manage.py runserver 0.0.0.0:8000
echo ""
echo "-=------------------------------------------------------"
echo " Starting the web server on '$addrport'..."
echo "-=------------------------------------------------------"
python3 manage.py runserver "$addrport"

View File

@ -15,7 +15,7 @@ class DBTime(BaseTargetObjective):
improvement=LESS_IS_BETTER)
def compute(self, metrics, observation_time):
return float('global.sys_time_model.db time')
return float(metrics['global.sys_time_model.db time'])
target_objective_list = tuple((DBMSType.ORACLE, target_obj) for target_obj in [ # pylint: disable=invalid-name