From dceee6e0ba3e36475df829ea35eb433839bde081 Mon Sep 17 00:00:00 2001
From: yangdsh <dongshen@andrew.cmu.edu>
Date: Sat, 23 Nov 2019 20:21:51 +0000
Subject: [PATCH] support Oracle flash back; auto-fix metric type; compile
 ojdbc

---
 client/controller/build.gradle               |  1 +
 client/driver/driver_config.py               |  4 +++
 client/driver/fabfile.py                     | 37 ++++++++++++++------
 client/driver/oracleScripts/archiveLog.sh    | 11 ++++++
 client/driver/oracleScripts/createRestore.sh | 33 +++++++++++++++++
 client/driver/oracleScripts/flashBack.sh     | 13 +++++++
 client/driver/oracleScripts/removeRestore.sh | 10 ++++++
 server/website/website/db/base/parser.py     | 13 ++++++-
 8 files changed, 111 insertions(+), 11 deletions(-)
 create mode 100644 client/driver/oracleScripts/archiveLog.sh
 create mode 100644 client/driver/oracleScripts/createRestore.sh
 create mode 100644 client/driver/oracleScripts/flashBack.sh
 create mode 100644 client/driver/oracleScripts/removeRestore.sh

diff --git a/client/controller/build.gradle b/client/controller/build.gradle
index 703db99..2c898d0 100644
--- a/client/controller/build.gradle
+++ b/client/controller/build.gradle
@@ -52,6 +52,7 @@ dependencies {
 
     // For Oracle, create the directory client/controller/libs and copy the driver
     // (e.g., ojdbc8.jar) into it. The driver must be manually downloaded from Oracle.
+    dependencies {compile files('lib/ojdbc8.jar')}
 }
 
 run {
diff --git a/client/driver/driver_config.py b/client/driver/driver_config.py
index f8fa089..d17f60b 100644
--- a/client/driver/driver_config.py
+++ b/client/driver/driver_config.py
@@ -68,6 +68,10 @@ PG_DATADIR = '/var/lib/postgresql/9.6/main'
 
 # ORACLE-SPECIFIC OPTIONS >>>
 ORACLE_AWR_ENABLED = False
+ORACLE_FLASH_BACK = True
+RESTORE_POINT = 'tpcc_point'
+RECOVERY_FILE_DEST = '/opt/oracle/oradata/ORCL'
+RECOVERY_FILE_DEST_SIZE = '15G'
 
 
 #==========================================================
diff --git a/client/driver/fabfile.py b/client/driver/fabfile.py
index a579d6c..65678e6 100644
--- a/client/driver/fabfile.py
+++ b/client/driver/fabfile.py
@@ -415,15 +415,23 @@ def upload_batch(result_dir=None, sort=True, upload_code=None):
 @task
 def dump_database():
     dumpfile = os.path.join(dconf.DB_DUMP_DIR, dconf.DB_NAME + '.dump')
-    if file_exists(dumpfile):
+    if not dconf.ORACLE_FLASH_BACK and file_exists(dumpfile):
         LOG.info('%s already exists ! ', dumpfile)
         return False
 
-    LOG.info('Dump database %s to %s', dconf.DB_NAME, dumpfile)
+    if dconf.ORACLE_FLASH_BACK:
+        LOG.info('create restore point %s for database %s in %s', dconf.RESTORE_POINT,
+                 dconf.DB_NAME, dconf.RECOVERY_FILE_DEST)
+    else:
+        LOG.info('Dump database %s to %s', dconf.DB_NAME, dumpfile)
 
     if dconf.DB_TYPE == 'oracle':
-        run_sql_script('dumpOracle.sh', dconf.DB_USER, dconf.DB_PASSWORD,
-                       dconf.DB_NAME, dconf.DB_DUMP_DIR)
+        if dconf.ORACLE_FLASH_BACK:
+            run_sql_script('createRestore.sh', dconf.RESTORE_POINT,
+                           dconf.RECOVERY_FILE_DEST_SIZE, dconf.RECOVERY_FILE_DEST)
+        else:
+            run_sql_script('dumpOracle.sh', dconf.DB_USER, dconf.DB_PASSWORD,
+                           dconf.DB_NAME, dconf.DB_DUMP_DIR)
 
     elif dconf.DB_TYPE == 'postgres':
         run('PGPASSWORD={} pg_dump -U {} -h {} -F c -d {} > {}'.format(
@@ -434,19 +442,28 @@ def dump_database():
     return True
 
 
+@task
+def clean_recovery():
+    run_sql_script('removeRestore.sh', dconf.RESTORE_POINT)
+    cmds = ("""rman TARGET / <<EOF\nDELETE ARCHIVELOG ALL;\nexit\nEOF""")
+    run(cmds)
+
+
 @task
 def restore_database():
     dumpfile = os.path.join(dconf.DB_DUMP_DIR, dconf.DB_NAME + '.dump')
-    if not file_exists(dumpfile):
+    if not dconf.ORACLE_FLASH_BACK and not file_exists(dumpfile):
         raise FileNotFoundError("Database dumpfile '{}' does not exist!".format(dumpfile))
 
     LOG.info('Start restoring database')
     if dconf.DB_TYPE == 'oracle':
-        # You must create a directory named dpdata through sqlplus in your Oracle database
-        # The following script assumes such directory exists.
-        drop_user()
-        create_user()
-        run_sql_script('restoreOracle.sh', dconf.DB_USER, dconf.DB_NAME)
+        if dconf.ORACLE_FLASH_BACK:
+            run_sql_script('flashBack.sh', dconf.RESTORE_POINT)
+            clean_recovery()
+        else:
+            drop_user()
+            create_user()
+            run_sql_script('restoreOracle.sh', dconf.DB_USER, dconf.DB_NAME)
     elif dconf.DB_TYPE == 'postgres':
         drop_database()
         create_database()
diff --git a/client/driver/oracleScripts/archiveLog.sh b/client/driver/oracleScripts/archiveLog.sh
new file mode 100644
index 0000000..663f053
--- /dev/null
+++ b/client/driver/oracleScripts/archiveLog.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# Change log mode
+sqlplus / as sysdba <<EOF
+    shutdown immediate
+    startup mount
+    ALTER DATABASE ARCHIVELOG;
+    ALTER DATABASE OPEN;
+quit
+EOF
+
diff --git a/client/driver/oracleScripts/createRestore.sh b/client/driver/oracleScripts/createRestore.sh
new file mode 100644
index 0000000..1a931b3
--- /dev/null
+++ b/client/driver/oracleScripts/createRestore.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+RESTORE_POINT="$1"
+SIZE="$2"
+RECOVERY_FILE="$3"
+
+# Make sure the physical directory exists
+mkdir -p "$DP_PATH"
+
+# Set recovery file and restore point
+sqlplus / as sysdba <<EOF
+    DECLARE
+      rfdexists INTEGER;
+    BEGIN
+      SELECT COUNT(*) INTO rfdexists FROM V\$RECOVERY_FILE_DEST WHERE name='$RECOVERY_FILE';
+      IF (rfdexists = 0) then
+        EXECUTE IMMEDIATE 'ALTER SYSTEM SET DB_RECOVERY_FILE_DEST_SIZE = $SIZE';
+        EXECUTE IMMEDIATE 'ALTER SYSTEM SET DB_RECOVERY_FILE_DEST = $RECOVERY_FILE';
+      END IF;
+      DBMS_OUTPUT.PUT_LINE(rfdexists);
+    END;
+    DECLARE
+      rpexists INTEGER;
+    BEGIN
+      SELECT COUNT(*) INTO rpexists FROM v\\\\\$restore_point WHERE name='$RESTORE_POINT';
+      IF (rpexists = 0) then
+        EXECUTE IMMEDIATE 'CREATE RESTORE POINT $RESTORE_POINT GUARANTEE FLASHBACK DATABASE';
+      END IF;
+      DBMS_OUTPUT.PUT_LINE(rpexists);
+    END;
+quit
+EOF
+
diff --git a/client/driver/oracleScripts/flashBack.sh b/client/driver/oracleScripts/flashBack.sh
new file mode 100644
index 0000000..f7c2502
--- /dev/null
+++ b/client/driver/oracleScripts/flashBack.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+RESTORE_POINT="$1"
+
+# Flash back
+sqlplus / as sysdba <<EOF
+    SHUTDOWN IMMEDIATE
+    STARTUP MOUNT
+    FLASHBACK DATABASE TO RESTORE POINT $RESTORE_POINT;
+    ALTER DATABASE OPEN RESETLOGS;
+quit
+EOF
+
diff --git a/client/driver/oracleScripts/removeRestore.sh b/client/driver/oracleScripts/removeRestore.sh
new file mode 100644
index 0000000..af3e116
--- /dev/null
+++ b/client/driver/oracleScripts/removeRestore.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+RESTORE_POINT="$1"
+
+# Remove RESTORE Point
+sqlplus / as sysdba <<EOF
+    'DROP RESTORE POINT $RESTORE_POINT
+quit
+EOF
+
diff --git a/server/website/website/db/base/parser.py b/server/website/website/db/base/parser.py
index 2234edc..dd8c9f5 100644
--- a/server/website/website/db/base/parser.py
+++ b/server/website/website/db/base/parser.py
@@ -3,6 +3,8 @@
 #
 # Copyright (c) 2017-18, Carnegie Mellon University Database Group
 #
+import logging
+
 from collections import OrderedDict
 
 from website.models import KnobCatalog, KnobUnitType, MetricCatalog
@@ -10,6 +12,8 @@ from website.types import BooleanType, MetricType, VarType
 from website.utils import ConversionUtil
 from .. import target_objectives
 
+LOG = logging.getLogger(__name__)
+
 
 # pylint: disable=no-self-use
 class BaseParser:
@@ -300,7 +304,7 @@ class BaseParser:
                     'Invalid metric type: {}'.format(metric.metric_type))
         return valid_metrics, diffs
 
-    def calculate_change_in_metrics(self, metrics_start, metrics_end):
+    def calculate_change_in_metrics(self, metrics_start, metrics_end, fix_metric_type=True):
         adjusted_metrics = {}
         for met_name, start_val in list(metrics_start.items()):
             end_val = metrics_end[met_name]
@@ -316,6 +320,13 @@ class BaseParser:
                     adj_val = end_val - start_val
                 else:  # MetricType.STATISTICS or MetricType.INFO
                     adj_val = end_val
+                if fix_metric_type:
+                    if adj_val < 0:
+                        adj_val = end_val
+                        LOG.debug("Changing metric %s from COUNTER to STATISTICS", met_name)
+                        metric_fixed = self.metric_catalog_[met_name]
+                        metric_fixed.metric_type = MetricType.STATISTICS
+                        metric_fixed.save()
                 assert adj_val >= 0, \
                     '{} wrong metric type: {} (start={}, end={}, diff={})'.format(
                         met_name, MetricType.name(met_info.metric_type), start_val,