public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Upgrade integration tests for testing algorithm performance

This commit is contained in:
yangdsh 2020-05-25 04:58:34 +00:00 committed by Dana Van Aken
parent 4b0d452719
commit 96347e217c
6 changed files with 680 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
client/driver/fabfile.py vendored
View File

@ -943,7 +943,7 @@ def wait_pipeline_data_ready(max_time_sec=800, interval_sec=10):
@task
def integration_tests():
def integration_tests_simple():
# Create test website
response = requests.get(dconf.WEBSITE_URL + '/test/create/')
@ -1029,3 +1029,93 @@ def task_status_ui_test():
assert 'Success:' in get_content(response)
LOG.info("\n\nTask Status UI Tests: PASSED!!\n")
def simulate_db_run(i, next_conf):
# Using 1 knob to decide performance; simple but effective
gain = int(next_conf['effective_cache_size'].replace('kB', ''))
with open('./integrationTests/data/x__metrics_after.json', 'r') as fin:
metrics_after = json.load(fin)
metrics_after['local']['database']['pg_stat_database']['tpcc']['xact_commit'] = gain
with open('./integrationTests/data/x__metrics_after.json', 'w') as fout:
json.dump(metrics_after, fout)
with open('./integrationTests/data/x__knobs.json', 'r') as fin:
knobs = json.load(fin)
for knob in next_conf:
knobs['global']['global'][knob] = next_conf[knob]
with open('./integrationTests/data/x__knobs.json', 'w') as fout:
json.dump(knobs, fout)
with open('./integrationTests/data/x__summary.json', 'r') as fin:
summary = json.load(fin)
summary['start_time'] = i * 20000000
summary['observation_time'] = 100
summary['end_time'] = (i + 1) * 20000000
with open('./integrationTests/data/x__summary.json', 'w') as fout:
json.dump(summary, fout)
return gain
@task
def integration_tests():
# Create test website
response = requests.get(dconf.WEBSITE_URL + '/test/create/')
LOG.info(get_content(response))
# Upload training data
LOG.info('Upload training data to no tuning session')
upload_batch(result_dir='./integrationTests/data/', upload_code='ottertuneTestNoTuning')
# periodic tasks haven't ran, lhs result returns.
LOG.info('Test no pipeline data, LHS returned')
upload_result(result_dir='./integrationTests/data/', prefix='0__',
upload_code='ottertuneTestTuningGPR')
response = get_result(upload_code='ottertuneTestTuningGPR')
assert response['status'] == 'lhs'
# wait celery periodic task finishes
assert wait_pipeline_data_ready(), "Pipeline data failed"
total_n = 30
first_n = 5
last_n = 5
average = 0
simulate_db_run(1, {'effective_cache_size': '0kB'})
for i in range(2, total_n + 2):
LOG.info('Test GPR (gaussian process regression)')
upload_result(result_dir='./integrationTests/data/', prefix='x__',
upload_code='ottertuneTestTuningGPR')
response = get_result(upload_code='ottertuneTestTuningGPR')
assert response['status'] == 'good'
gain = simulate_db_run(i, response['recommendation'])
if i < first_n + 2:
average += gain / first_n
elif i > total_n - last_n + 2:
assert gain > average
average = 0
simulate_db_run(1, {'effective_cache_size': '0kB'})
for i in range(2, total_n + 2):
LOG.info('Test DNN (deep neural network)')
upload_result(result_dir='./integrationTests/data/', prefix='x__',
upload_code='ottertuneTestTuningDNN')
response = get_result(upload_code='ottertuneTestTuningDNN')
assert response['status'] == 'good'
gain = simulate_db_run(i, response['recommendation'])
if i < first_n + 2:
average += gain / first_n
elif i > total_n - last_n + 2:
assert gain > average
average = 0
simulate_db_run(1, {'effective_cache_size': '0kB'})
for i in range(2, total_n + 2):
upload_result(result_dir='./integrationTests/data/', prefix='x__',
upload_code='ottertuneTestTuningDDPG')
response = get_result(upload_code='ottertuneTestTuningDDPG')
assert response['status'] == 'good'
gain = simulate_db_run(i, response['recommendation'])
if i < first_n + 2:
average += gain / first_n
elif i > total_n - last_n + 2:
assert gain > average

1
client/driver/integrationTests/data/x__knobs.json Normal file
View File

File diff suppressed because one or more lines are too long

1
client/driver/integrationTests/data/x__metrics_after.json Normal file
View File

File diff suppressed because one or more lines are too long

581
client/driver/integrationTests/data/x__metrics_before.json Normal file
View File

@ -0,0 +1,581 @@
{
"global": {
"pg_stat_archiver": {
"archived_count": "0",
"failed_count": "0",
"stats_reset": "2019-09-11 21:28:47.239843-04"
},
"pg_stat_bgwriter": {
"buffers_alloc": "3692959832",
"buffers_backend": "1741292048",
"buffers_backend_fsync": "0",
"buffers_checkpoint": "1838672434",
"buffers_clean": "90148655",
"checkpoint_sync_time": "13751757",
"checkpoint_write_time": "214665311",
"checkpoints_req": "9948",
"checkpoints_timed": "2925",
"maxwritten_clean": "290648",
"stats_reset": "2019-09-11 21:28:47.239843-04"
}
},
"local": {
"table": {
"pg_stat_user_tables": {
"customer": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "1",
"idx_tup_fetch": "6000000",
"last_autoanalyze": "2019-10-01 01:58:54.728893-04",
"n_dead_tup": "0",
"n_live_tup": "5999969",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "6000000",
"n_tup_upd": "0",
"relid": "48704",
"relname": "customer",
"schemaname": "public",
"seq_scan": "4",
"seq_tup_read": "24000000",
"vacuum_count": "0"
},
"warehouse": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "0",
"idx_tup_fetch": "0",
"last_autoanalyze": "2019-10-01 01:52:28.446885-04",
"n_dead_tup": "0",
"n_live_tup": "200",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "200",
"n_tup_upd": "0",
"relid": "48734",
"relname": "warehouse",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "600",
"vacuum_count": "0"
},
"new_order": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "1",
"idx_tup_fetch": "1692083",
"last_autoanalyze": "2019-10-01 01:52:27.655021-04",
"n_dead_tup": "0",
"n_live_tup": "1692083",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "1692083",
"n_tup_upd": "0",
"relid": "48721",
"relname": "new_order",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "1692083",
"vacuum_count": "0"
},
"order_line": {
"analyze_count": "0",
"autoanalyze_count": "0",
"autovacuum_count": "0",
"idx_scan": "1",
"idx_tup_fetch": "59995443",
"n_dead_tup": "0",
"n_live_tup": "59995443",
"n_mod_since_analyze": "59995443",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "59995443",
"n_tup_upd": "0",
"relid": "48728",
"relname": "order_line",
"schemaname": "public",
"seq_scan": "2",
"seq_tup_read": "119990886",
"vacuum_count": "0"
},
"history": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"last_autoanalyze": "2019-10-01 01:53:35.074571-04",
"n_dead_tup": "0",
"n_live_tup": "6000000",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "6000000",
"n_tup_upd": "0",
"relid": "48714",
"relname": "history",
"schemaname": "public",
"seq_scan": "2",
"seq_tup_read": "12000000",
"vacuum_count": "0"
},
"item": {
"analyze_count": "0",
"autoanalyze_count": "0",
"autovacuum_count": "0",
"idx_scan": "0",
"idx_tup_fetch": "0",
"n_dead_tup": "0",
"n_live_tup": "100000",
"n_mod_since_analyze": "100000",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "100000",
"n_tup_upd": "0",
"relid": "48718",
"relname": "item",
"schemaname": "public",
"seq_scan": "2",
"seq_tup_read": "200000",
"vacuum_count": "0"
},
"stock": {
"analyze_count": "0",
"autoanalyze_count": "0",
"autovacuum_count": "0",
"idx_scan": "59995443",
"idx_tup_fetch": "59995443",
"n_dead_tup": "0",
"n_live_tup": "20000000",
"n_mod_since_analyze": "20000000",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "20000000",
"n_tup_upd": "0",
"relid": "48731",
"relname": "stock",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "60000000",
"vacuum_count": "0"
},
"oorder": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "3",
"idx_tup_fetch": "18000000",
"last_autoanalyze": "2019-10-01 01:53:45.816842-04",
"n_dead_tup": "0",
"n_live_tup": "6000000",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "6000000",
"n_tup_upd": "0",
"relid": "48724",
"relname": "oorder",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "18000000",
"vacuum_count": "0"
},
"district": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "0",
"idx_tup_fetch": "0",
"last_autoanalyze": "2019-10-01 01:52:29.056627-04",
"n_dead_tup": "0",
"n_live_tup": "2000",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "2000",
"n_tup_upd": "0",
"relid": "48711",
"relname": "district",
"schemaname": "public",
"seq_scan": "4",
"seq_tup_read": "8000",
"vacuum_count": "0"
}
},
"pg_statio_user_tables": {
"order_line": {
"heap_blks_hit": "842040",
"heap_blks_read": "2957855",
"idx_blks_hit": "0",
"idx_blks_read": "229872",
"relid": "48728",
"relname": "order_line",
"schemaname": "public"
},
"oorder": {
"heap_blks_hit": "6241021",
"heap_blks_read": "97952",
"idx_blks_hit": "2051",
"idx_blks_read": "66928",
"relid": "48724",
"relname": "oorder",
"schemaname": "public"
},
"item": {
"heap_blks_hit": "2542",
"heap_blks_read": "1271",
"idx_blks_hit": "0",
"idx_blks_read": "1",
"relid": "48718",
"relname": "item",
"schemaname": "public"
},
"history": {
"heap_blks_hit": "128440",
"heap_blks_read": "124344",
"relid": "48714",
"relname": "history",
"schemaname": "public"
},
"warehouse": {
"heap_blks_hit": "12",
"heap_blks_read": "3",
"idx_blks_hit": "0",
"idx_blks_read": "1",
"relid": "48734",
"relname": "warehouse",
"schemaname": "public"
},
"stock": {
"heap_blks_hit": "59117416",
"heap_blks_read": "4360259",
"idx_blks_hit": "180095972",
"idx_blks_read": "54841",
"relid": "48731",
"relname": "stock",
"schemaname": "public"
},
"new_order": {
"heap_blks_hit": "29895",
"heap_blks_read": "16246",
"idx_blks_hit": "0",
"idx_blks_read": "6487",
"relid": "48721",
"relname": "new_order",
"schemaname": "public"
},
"customer": {
"heap_blks_hit": "248380",
"heap_blks_read": "2891409",
"idx_blks_hit": "0",
"idx_blks_read": "22993",
"relid": "48704",
"relname": "customer",
"schemaname": "public",
"tidx_blks_hit": "0",
"tidx_blks_read": "0",
"toast_blks_hit": "0",
"toast_blks_read": "0"
},
"district": {
"heap_blks_hit": "150",
"heap_blks_read": "30",
"idx_blks_hit": "1",
"idx_blks_read": "1",
"relid": "48711",
"relname": "district",
"schemaname": "public"
}
}
},
"database": {
"pg_stat_database": {
"postgres": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "4037127",
"blks_read": "254822",
"conflicts": "0",
"datid": "12439",
"datname": "postgres",
"deadlocks": "0",
"numbackends": "0",
"stats_reset": "2019-09-11 21:28:47.953315-04",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "1796046",
"tup_inserted": "0",
"tup_returned": "25033610",
"tup_updated": "14",
"xact_commit": "66610",
"xact_rollback": "11"
},
"tpcc": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "246720695",
"blks_read": "10830975",
"conflicts": "0",
"datid": "48703",
"datname": "tpcc",
"deadlocks": "0",
"numbackends": "1",
"stats_reset": "2019-10-01 01:52:19.906323-04",
"temp_bytes": "2081841152",
"temp_files": "3",
"tup_deleted": "0",
"tup_fetched": "145689968",
"tup_inserted": "99790620",
"tup_returned": "381590889",
"tup_updated": "55",
"xact_commit": "256",
"xact_rollback": "1"
},
"template1": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "2793953",
"blks_read": "109736",
"conflicts": "0",
"datid": "1",
"datname": "template1",
"deadlocks": "0",
"numbackends": "0",
"stats_reset": "2019-09-11 22:05:20.533143-04",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "953195",
"tup_inserted": "0",
"tup_returned": "21935277",
"tup_updated": "0",
"xact_commit": "52248",
"xact_rollback": "0"
},
"template0": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "1774960",
"blks_read": "66988",
"conflicts": "0",
"datid": "12438",
"datname": "template0",
"deadlocks": "0",
"numbackends": "0",
"stats_reset": "2019-09-17 02:25:02.20631-04",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "570334",
"tup_inserted": "0",
"tup_returned": "15667447",
"tup_updated": "13",
"xact_commit": "31727",
"xact_rollback": "0"
}
},
"pg_stat_database_conflicts": {
"postgres": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "12439",
"datname": "postgres"
},
"tpcc": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "48703",
"datname": "tpcc"
},
"template1": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "1",
"datname": "template1"
},
"template0": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "12438",
"datname": "template0"
}
}
},
"indexes": {
"pg_stat_user_indexes": {
"district": {
"idx_scan": "0",
"idx_tup_fetch": "0",
"idx_tup_read": "0",
"indexrelid": "48748",
"indexrelname": "district_pkey",
"relid": "48711",
"relname": "district",
"schemaname": "public"
},
"warehouse": {
"idx_scan": "0",
"idx_tup_fetch": "0",
"idx_tup_read": "0",
"indexrelid": "48750",
"indexrelname": "warehouse_pkey",
"relid": "48734",
"relname": "warehouse",
"schemaname": "public"
},
"item": {
"idx_scan": "0",
"idx_tup_fetch": "0",
"idx_tup_read": "0",
"indexrelid": "48757",
"indexrelname": "item_pkey",
"relid": "48718",
"relname": "item",
"schemaname": "public"
},
"new_order": {
"idx_scan": "1",
"idx_tup_fetch": "1692083",
"idx_tup_read": "1692083",
"indexrelid": "48759",
"indexrelname": "new_order_pkey",
"relid": "48721",
"relname": "new_order",
"schemaname": "public"
},
"oorder": {
"idx_scan": "1",
"idx_tup_fetch": "6000000",
"idx_tup_read": "6000000",
"indexrelid": "48781",
"indexrelname": "idx_order",
"relid": "48724",
"relname": "oorder",
"schemaname": "public"
},
"customer": {
"idx_scan": "0",
"idx_tup_fetch": "0",
"idx_tup_read": "0",
"indexrelid": "48797",
"indexrelname": "idx_customer_name",
"relid": "48704",
"relname": "customer",
"schemaname": "public"
},
"order_line": {
"idx_scan": "1",
"idx_tup_fetch": "59995443",
"idx_tup_read": "59995443",
"indexrelid": "48813",
"indexrelname": "order_line_pkey",
"relid": "48728",
"relname": "order_line",
"schemaname": "public"
},
"stock": {
"idx_scan": "59995443",
"idx_tup_fetch": "59995443",
"idx_tup_read": "59995443",
"indexrelid": "48815",
"indexrelname": "stock_pkey",
"relid": "48731",
"relname": "stock",
"schemaname": "public"
}
},
"pg_statio_user_indexes": {
"district": {
"idx_blks_hit": "1",
"idx_blks_read": "1",
"indexrelid": "48748",
"indexrelname": "district_pkey",
"relid": "48711",
"relname": "district",
"schemaname": "public"
},
"warehouse": {
"idx_blks_hit": "0",
"idx_blks_read": "1",
"indexrelid": "48750",
"indexrelname": "warehouse_pkey",
"relid": "48734",
"relname": "warehouse",
"schemaname": "public"
},
"item": {
"idx_blks_hit": "0",
"idx_blks_read": "1",
"indexrelid": "48757",
"indexrelname": "item_pkey",
"relid": "48718",
"relname": "item",
"schemaname": "public"
},
"new_order": {
"idx_blks_hit": "0",
"idx_blks_read": "6487",
"indexrelid": "48759",
"indexrelname": "new_order_pkey",
"relid": "48721",
"relname": "new_order",
"schemaname": "public"
},
"oorder": {
"idx_blks_hit": "1",
"idx_blks_read": "22992",
"indexrelid": "48781",
"indexrelname": "idx_order",
"relid": "48724",
"relname": "oorder",
"schemaname": "public"
},
"customer": {
"idx_blks_hit": "0",
"idx_blks_read": "1",
"indexrelid": "48797",
"indexrelname": "idx_customer_name",
"relid": "48704",
"relname": "customer",
"schemaname": "public"
},
"order_line": {
"idx_blks_hit": "0",
"idx_blks_read": "229872",
"indexrelid": "48813",
"indexrelname": "order_line_pkey",
"relid": "48728",
"relname": "order_line",
"schemaname": "public"
},
"stock": {
"idx_blks_hit": "180095972",
"idx_blks_read": "54841",
"indexrelid": "48815",
"indexrelname": "stock_pkey",
"relid": "48731",
"relname": "stock",
"schemaname": "public"
}
}
}
}
}

1
client/driver/integrationTests/data/x__summary.json Normal file
View File

@ -0,0 +1 @@
{"start_time": 620000000, "end_time": 640000000, "observation_time": 100, "database_type": "postgres", "database_version": "9.6", "workload_name": "tpcc"}

8
server/website/website/views.py
View File

@ -1784,10 +1784,12 @@ def alt_create_or_edit_session(request):
@csrf_exempt
def pipeline_data_ready(request): # pylint: disable=unused-argument
LOG.debug("Latest pipeline run: %s", PipelineRun.objects.get_latest())
if PipelineRun.objects.get_latest() is None:
response = "Pipeline data ready: False"
else:
newest_result = Result.objects.filter().order_by('-pk')[0]
pipeline_data = PipelineData.objects.filter(workload=newest_result.workload).order_by('-pk')
if pipeline_data.exists() and pipeline_data[0].pipeline_run.end_time is not None:
response = "Pipeline data ready: True"
else:
response = "Pipeline data ready: False"
return HttpResponse(response)