diff --git a/client/driver/fabfile.py b/client/driver/fabfile.py index 65678e6..0f09a3a 100644 --- a/client/driver/fabfile.py +++ b/client/driver/fabfile.py @@ -23,7 +23,7 @@ import requests from fabric.api import env, lcd, local, settings, show, task from fabric.state import output as fabric_output -from utils import (file_exists, get, load_driver_conf, parse_bool, +from utils import (file_exists, get, get_content, load_driver_conf, parse_bool, put, run, run_sql_script, sudo, FabricException) # Loads the driver config file (defaults to driver_config.py) @@ -306,12 +306,12 @@ def upload_result(result_dir=None, prefix=None, upload_code=None): data={'upload_code': upload_code}) if response.status_code != 200: raise Exception('Error uploading result.\nStatus: {}\nMessage: {}\n'.format( - response.status_code, response.content)) + response.status_code, get_content(response))) for f in files.values(): # pylint: disable=not-an-iterable f.close() - LOG.info(response.content) + LOG.info(get_content(response)) return response @@ -328,7 +328,7 @@ def get_result(max_time_sec=180, interval_sec=5, upload_code=None): while elapsed <= max_time_sec: rsp = requests.get(url) - response = rsp.content.decode() + response = get_content(rsp) assert response != 'null' LOG.debug('%s [status code: %d, content_type: %s, elapsed: %ds]', response, @@ -693,16 +693,20 @@ def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10): max_time_sec = int(max_time_sec) interval_sec = int(interval_sec) elapsed = 0 + ready = False while elapsed <= max_time_sec: response = requests.get(dconf.WEBSITE_URL + '/test/pipeline/') - response = response.content - LOG.info(response) - if 'False' in str(response): + content = get_content(response) + LOG.info("%s (elapsed: %ss)", content, interval_sec) + if 'False' in content: time.sleep(interval_sec) elapsed += interval_sec else: - return + ready = True + break + + return ready @task @@ -710,14 +714,14 @@ def integration_tests(): # Create test website response = requests.get(dconf.WEBSITE_URL + '/test/create/') - LOG.info(response.content) + LOG.info(get_content(response)) # Upload training data LOG.info('Upload training data to no tuning session') upload_batch(result_dir='../../integrationTests/data/', upload_code='ottertuneTestNoTuning') # wait celery periodic task finishes - wait_pipeline_data_ready() + assert wait_pipeline_data_ready(), "Pipeline data failed" # Test DNN LOG.info('Test DNN (deep neural network)') diff --git a/client/driver/utils.py b/client/driver/utils.py index 67b5b8d..af66ca1 100644 --- a/client/driver/utils.py +++ b/client/driver/utils.py @@ -53,6 +53,13 @@ def parse_bool(value): return value +def get_content(response): + content = response.content + if isinstance(content, bytes): + content = content.decode('utf-8') + return content + + @task def run(cmd, capture=True, **kwargs): capture = parse_bool(capture) diff --git a/server/website/website/views.py b/server/website/website/views.py index eb64925..500db8d 100644 --- a/server/website/website/views.py +++ b/server/website/website/views.py @@ -761,7 +761,7 @@ def tuner_status_view(request, project_id, session_id, result_id): # pylint: di tasks = TaskUtil.get_tasks(res.task_ids) overall_status, num_completed = TaskUtil.get_task_status(tasks) - if overall_status in ['PENDING', 'RECEIVED', 'STARTED']: + if overall_status in ['PENDING', 'RECEIVED', 'STARTED', None]: completion_time = 'N/A' total_runtime = 'N/A' else: @@ -1001,13 +1001,18 @@ def give_result(request, upload_code): # pylint: disable=unused-argument overall_status, num_completed = TaskUtil.get_task_status(tasks) if overall_status == 'SUCCESS': - next_config = latest_result.next_configuration - if not next_config: + if not latest_result.next_configuration: + # If the task status was incomplete when we first queried latest_result + # but succeeded before the call to TaskUtil.get_task_status() finished + # then latest_result is stale and must be updated. + latest_result = Result.objects.get(id=latest_result.pk) + + if not latest_result.next_configuration: overall_status = 'FAILURE' response = _failed_response(latest_result, tasks, num_completed, overall_status, 'Failed to get the next configuration.') else: - response = HttpResponse(JSONUtil.dumps(next_config), + response = HttpResponse(JSONUtil.dumps(latest_result.next_configuration), content_type='application/json') elif overall_status in ('FAILURE', 'REVOKED', 'RETRY'): @@ -1252,7 +1257,7 @@ def alt_create_or_edit_session(request): # integration test @csrf_exempt def pipeline_data_ready(request): # pylint: disable=unused-argument - LOG.info(PipelineRun.objects.get_latest()) + LOG.debug("Latest pipeline run: %s", PipelineRun.objects.get_latest()) if PipelineRun.objects.get_latest() is None: response = "Pipeline data ready: False" else: