Initial code to send/manage DB error

client/driver/fabfile.py

@@ -114,6 +114,7 @@ def restart_database():
         run_sql_script('restartOracle.sh')
     else:
         raise Exception("Database Type {} Not Implemented !".format(dconf.DB_TYPE))
+    return True
 
 
 @task
@@ -518,7 +519,19 @@ def loop(i):
     clean_logs()
 
     # restart database
-    restart_database()
+    restart_succeeded = restart_database()
+    if not restart_succeeded:
+        files = {'summary':b'{error:"DB_RESTART_ERROR"}',
+                 'knobs':b'',
+                 'metrics_before':b'',
+                 'metrics_after':b''}
+        response = requests.post(dconf.WEBSITE_URL + '/new_result/', files=files,
+                             data={'upload_code': dconf.UPLOAD_CODE})
+        response = get_result()
+        save_next_config(response, t=result_timestamp)
+        change_conf(response['recommendation'])
+        return
+
     time.sleep(dconf.RESTART_SLEEP_SEC)
 
     # check disk usage

server/website/website/views.py

@@ -455,6 +455,45 @@ def handle_result_files(session, files):
 
     # Load the contents of the controller's summary file
     summary = JSONUtil.loads(files['summary'])
+
+    # If database crashed on restart, pull latest result and worst throughput so far
+    if 'error' in summary and summary['error']=="DB_RESTART_ERROR":
+
+        LOG.debug("Error in restarting database")
+        # Find worst throughput
+        past_configs = MetricData.objects.filter(session=session)
+        worst_throughput = None
+        for curr_config in past_configs:
+            throughput = JSONUtil.loads(curr_config.data)["throughput_txn_per_sec"]
+            if worst_throughput is None or throughput < worst_throughput:
+                worst_throughput = throughput
+        LOG.debug("Worst throughput so far is:%d",worst_throughput)
+
+        # Copy latest data and modify
+        knob_data = KnobData.objects.filter(session=session).order_by("-id").first()
+        knob_data.pk = None
+        knob_data.save()
+
+        metric_data = MetricData.objects.filter(session=session).order_by("-id").first()
+        metric_cpy = JSONUtil.loads(metric_data.data)
+        metric_cpy["throughput_txn_per_sec"]=worst_throughput
+        metric_cpy = JSONUtil.dumps(metric_cpy)
+        metric_data.pk = None
+        metric_data.data = metric_cpy
+        metric_data.save()
+
+        result = Result.objects.filter(session=session).order_by("-id").first()
+        result.pk = None
+        result.knob_data = knob_data
+        result.metric_data = metric_data
+        result.save()
+
+        backup_data = BackupData.objects.filter(result=result).first()
+        backup_data.pk = None
+        backup_data.result = result
+        backup_data.save()
+
+    else:
         dbms_type = DBMSType.type(summary['database_type'])
         dbms_version = summary['database_version']  # TODO: fix parse_version_string
         workload_name = summary['workload_name']