From 2fa97149d589a0cb2c59e46c5725091574238e6e Mon Sep 17 00:00:00 2001 From: bohanjason Date: Thu, 24 Oct 2019 11:59:36 -0400 Subject: [PATCH] add integration test --- client/driver/fabfile.py | 33 ++++++++++--- server/website/website/models.py | 9 +++- server/website/website/tasks/async_tasks.py | 2 +- server/website/website/urls.py | 4 ++ server/website/website/views.py | 53 ++++++++++++++++++++- 5 files changed, 89 insertions(+), 12 deletions(-) diff --git a/client/driver/fabfile.py b/client/driver/fabfile.py index ac8b831..ff7fb97 100644 --- a/client/driver/fabfile.py +++ b/client/driver/fabfile.py @@ -647,24 +647,38 @@ def run_loops(max_iter=1): LOG.info('The %s-th Loop Ends / Total Loops %s', i + 1, max_iter) -@task +@task def rename_batch(result_dir=None): result_dir = result_dir or CONF['save_path'] results = glob.glob(os.path.join(result_dir, '*__summary.json')) results = sorted(results) - count = len(results) for i, result in enumerate(results): prefix = os.path.basename(result) prefix_len = os.path.basename(result).find('_') + 2 prefix = prefix[:prefix_len] new_prefix = str(i) + '__' - files = {} for base in ('summary', 'knobs', 'metrics_before', 'metrics_after'): fpath = os.path.join(result_dir, prefix + base + '.json') rename_path = os.path.join(result_dir, new_prefix + base + '.json') os.rename(fpath, rename_path) +def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10): + max_time_sec = int(max_time_sec) + interval_sec = int(interval_sec) + elapsed = 0 + + while elapsed <= max_time_sec: + response = requests.get(CONF['upload_url'] + '/test/pipeline/') + response = response.content + LOG.info(response) + if 'False' in response: + time.sleep(interval_sec) + elapsed += interval_sec + else: + return + + @task def integration_tests(): @@ -674,17 +688,22 @@ def integration_tests(): # Upload training data LOG.info('Upload training data to no tuning session') - upload_batch(result_dir='../../integrationTests/data/',upload_code='ottertuneTestNoTuning') + upload_batch(result_dir='../../integrationTests/data/', upload_code='ottertuneTestNoTuning') # TO DO: BG ready + response = requests.get(CONF['upload_url'] + '/test/pipeline/') + LOG.info(response.content) + # Test DNN LOG.info('Test DNN (deep neural network)') - upload_result(result_dir='../../integrationTests/data/', prefix='0__', upload_code='ottertuneTestTuningDNN') + upload_result(result_dir='../../integrationTests/data/', prefix='0__', + upload_code='ottertuneTestTuningDNN') response = get_result(upload_code='ottertuneTestTuningDNN') assert response['status'] == 'good' # Test GPR LOG.info('Test GPR (gaussian process regression)') - upload_result(result_dir='../../integrationTests/data/', prefix='0__', upload_code='ottertuneTestTuningGPR') + upload_result(result_dir='../../integrationTests/data/', prefix='0__', + upload_code='ottertuneTestTuningGPR') response = get_result(upload_code='ottertuneTestTuningGPR') - assert response['status'] == 'good' \ No newline at end of file + assert response['status'] == 'good' diff --git a/server/website/website/models.py b/server/website/website/models.py index 01db1a7..afe1cc9 100644 --- a/server/website/website/models.py +++ b/server/website/website/models.py @@ -182,17 +182,22 @@ class Session(BaseModel): class SessionKnobManager(models.Manager): @staticmethod - def get_knobs_for_session(session): + def get_knobs_for_session(session, only_session_knobs=False): # Returns a dict of the knob knobs = KnobCatalog.objects.filter(dbms=session.dbms) knob_dicts = list(knobs.values()) + session_knob_dicts = [] for i, _ in enumerate(knob_dicts): if SessionKnob.objects.filter(session=session, knob=knobs[i]).exists(): new_knob = SessionKnob.objects.filter(session=session, knob=knobs[i])[0] knob_dicts[i]["minval"] = new_knob.minval knob_dicts[i]["maxval"] = new_knob.maxval knob_dicts[i]["tunable"] = new_knob.tunable - knob_dicts = [knob for knob in knob_dicts if knob["tunable"]] + session_knob_dicts.append(new_knob) + if only_session_knobs: + knob_dicts = session_knob_dicts + else: + knob_dicts = [knob for knob in knob_dicts if knob["tunable"]] return knob_dicts @staticmethod diff --git a/server/website/website/tasks/async_tasks.py b/server/website/website/tasks/async_tasks.py index 322348d..0026064 100644 --- a/server/website/website/tasks/async_tasks.py +++ b/server/website/website/tasks/async_tasks.py @@ -544,7 +544,7 @@ def configuration_recommendation(recommendation_input): X_max = np.empty(X_scaled.shape[1]) X_scaler_matrix = np.zeros([1, X_scaled.shape[1]]) - session_knobs = SessionKnob.objects.get_knobs_for_session(newest_result.session) + session_knobs = SessionKnob.objects.get_knobs_for_session(newest_result.session, only_session_knobs=True) # Set min/max for knob values for i in range(X_scaled.shape[1]): diff --git a/server/website/website/urls.py b/server/website/website/urls.py index f638fdf..30c5025 100644 --- a/server/website/website/urls.py +++ b/server/website/website/urls.py @@ -70,6 +70,10 @@ urlpatterns = [ # train ddpg with results in the given session url(r'^train_ddpg/sessions/(?P[0-9]+)$', website_views.train_ddpg_loops, name='train_ddpg_loops'), + + # Test + url(r'^test/create/', website_views.create_test_website, name='create_test_website'), + url(r'^test/pipeline/', website_views.pipeline_data_ready, name='pipeline_data_ready') ] if settings.DEBUG: diff --git a/server/website/website/views.py b/server/website/website/views.py index 284dc75..0b55b63 100644 --- a/server/website/website/views.py +++ b/server/website/website/views.py @@ -29,8 +29,8 @@ from pytz import timezone from .db import parser, target_objectives from .forms import NewResultForm, ProjectForm, SessionForm, SessionKnobForm -from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog, - MetricData, Project, Result, Session, Workload, SessionKnob) +from .models import (BackupData, DBMSCatalog, KnobCatalog, KnobData, MetricCatalog, User, Hardware, + MetricData, Project, Result, Session, Workload, SessionKnob, PipelineRun) from .tasks import (aggregate_target_results, map_workload, train_ddpg, configuration_recommendation, configuration_recommendation_ddpg) from .types import (DBMSType, KnobUnitType, MetricType, @@ -1041,3 +1041,52 @@ def train_ddpg_loops(request, session_id): # pylint: disable=unused-argument for result in results: train_ddpg(result.pk) return HttpResponse() + + +# integration test +@csrf_exempt +def pipeline_data_ready(request): # pylint: disable=unused-argument + LOG.info(PipelineRun.objects.get_latest()) + if PipelineRun.objects.get_latest() is None: + response = "Pipeline data ready: False" + else: + response = "Pipeline data ready: True" + return HttpResponse(response) + + +# integration test +@csrf_exempt +def create_test_website(request): # pylint: disable=unused-argument + if User.objects.filter(username='ottertune_test_user').exists(): + User.objects.filter(username='ottertune_test_user').delete() + if Hardware.objects.filter(pk=1).exists(): + test_hardware = Hardware.objects.get(pk=1) + else: + test_hardware = Hardware.objects.create(pk=1) + + test_user = User.objects.create_user(username='ottertune_test_user', + password='ottertune_test_user') + test_project = Project.objects.create(user=test_user, name='ottertune_test_project', + creation_time=now(), last_update=now()) + # create no tuning session + Session.objects.create(name='test_session_no_tuning', tuning_session='no_tuning_session', + dbms_id=1, hardware=test_hardware, project=test_project, + creation_time=now(), last_update=now(), user=test_user, + upload_code='ottertuneTestNoTuning') + # create gpr session + Session.objects.create(name='test_session_gpr', user=test_user, tuning_session='tuning_session', + dbms_id=1, hardware=test_hardware, project=test_project, + creation_time=now(), last_update=now(), algorithm=AlgorithmType.GPR, + upload_code='ottertuneTestTuningGPR') + # create dnn session + Session.objects.create(name='test_session_dnn', user=test_user, tuning_session='tuning_session', + dbms_id=1, hardware=test_hardware, project=test_project, + creation_time=now(), last_update=now(), algorithm=AlgorithmType.DNN, + upload_code='ottertuneTestTuningDNN') + # create ddpg session + Session.objects.create(name='test_session_ddpg', tuning_session='tuning_session', + dbms_id=1, hardware=test_hardware, project=test_project, + creation_time=now(), last_update=now(), user=test_user, + upload_code='ottertuneTestTuningDDPG', algorithm=AlgorithmType.DDPG) + response = HttpResponse("Success: create test website successfully") + return response