Initial code to send/manage DB error

This commit is contained in:
arifiorino 2019-12-02 23:02:20 +00:00 committed by Dana Van Aken
parent e80639c859
commit 0dff040012
2 changed files with 123 additions and 71 deletions

View File

@ -114,6 +114,7 @@ def restart_database():
run_sql_script('restartOracle.sh')
else:
raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
return True
@task
@ -518,7 +519,19 @@ def loop(i):
clean_logs()
# restart database
restart_database()
restart_succeeded = restart_database()
if not restart_succeeded:
files = {'summary':b'{error:"DB_RESTART_ERROR"}',
'knobs':b'',
'metrics_before':b'',
'metrics_after':b''}
response = requests.post(dconf.WEBSITE_URL + '/new_result/', files=files,
data={'upload_code': dconf.UPLOAD_CODE})
response = get_result()
save_next_config(response, t=result_timestamp)
change_conf(response['recommendation'])
return
time.sleep(dconf.RESTART_SLEEP_SEC)
# check disk usage

View File

@ -455,6 +455,45 @@ def handle_result_files(session, files):
# Load the contents of the controller's summary file
summary = JSONUtil.loads(files['summary'])
# If database crashed on restart, pull latest result and worst throughput so far
if 'error' in summary and summary['error']=="DB_RESTART_ERROR":
LOG.debug("Error in restarting database")
# Find worst throughput
past_configs = MetricData.objects.filter(session=session)
worst_throughput = None
for curr_config in past_configs:
throughput = JSONUtil.loads(curr_config.data)["throughput_txn_per_sec"]
if worst_throughput is None or throughput < worst_throughput:
worst_throughput = throughput
LOG.debug("Worst throughput so far is:%d",worst_throughput)
# Copy latest data and modify
knob_data = KnobData.objects.filter(session=session).order_by("-id").first()
knob_data.pk = None
knob_data.save()
metric_data = MetricData.objects.filter(session=session).order_by("-id").first()
metric_cpy = JSONUtil.loads(metric_data.data)
metric_cpy["throughput_txn_per_sec"]=worst_throughput
metric_cpy = JSONUtil.dumps(metric_cpy)
metric_data.pk = None
metric_data.data = metric_cpy
metric_data.save()
result = Result.objects.filter(session=session).order_by("-id").first()
result.pk = None
result.knob_data = knob_data
result.metric_data = metric_data
result.save()
backup_data = BackupData.objects.filter(result=result).first()
backup_data.pk = None
backup_data.result = result
backup_data.save()
else:
dbms_type = DBMSType.type(summary['database_type'])
dbms_version = summary['database_version'] # TODO: fix parse_version_string
workload_name = summary['workload_name']