Added support for Oracle v12.1c.

client/controller/sample_output/oracle/v12.1/knobs.json

@@ -0,0 +1,285 @@
+{
+ "global": {"global": {
+  "aq_tm_processes": "1",
+  "archive_lag_target": "0",
+  "asm_power_limit": "1",
+  "audit_file_dest": "/opt/oracle/admin/ORCLCDB/adump",
+  "audit_sys_operations": "FALSE",
+  "audit_trail": "NONE",
+  "awr_snapshot_time_offset": "0",
+  "background_core_dump": "partial",
+  "background_dump_dest": "/opt/oracle/product/12.1.0.2/dbhome_1/rdbms/log",
+  "backup_tape_io_slaves": "FALSE",
+  "bitmap_merge_area_size": "1048576",
+  "blank_trimming": "FALSE",
+  "cell_offload_compaction": "ADAPTIVE",
+  "cell_offload_decryption": "TRUE",
+  "cell_offload_plan_display": "AUTO",
+  "cell_offload_processing": "TRUE",
+  "client_result_cache_lag": "3000",
+  "client_result_cache_size": "0",
+  "clonedb": "FALSE",
+  "cluster_database": "FALSE",
+  "cluster_database_instances": "1",
+  "commit_point_strength": "1",
+  "common_user_prefix": "C##",
+  "compatible": "12.1.0.2.0",
+  "connection_brokers": "((TYPE=DEDICATED)(BROKERS=1)), ((TYPE=EMON)(BROKERS=1))",
+  "control_file_record_keep_time": "7",
+  "control_files": "/opt/oracle/oradata/ORCLCDB/control01.ctl",
+  "control_management_pack_access": "DIAGNOSTIC+TUNING",
+  "core_dump_dest": "/opt/oracle/diag/rdbms/orclcdb/ORCLCDB/cdump",
+  "cpu_count": "4",
+  "create_bitmap_area_size": "8388608",
+  "cursor_bind_capture_destination": "memory+disk",
+  "cursor_sharing": "EXACT",
+  "cursor_space_for_time": "FALSE",
+  "db_16k_cache_size": "0",
+  "db_2k_cache_size": "0",
+  "db_32k_cache_size": "0",
+  "db_4k_cache_size": "0",
+  "db_8k_cache_size": "0",
+  "db_big_table_cache_percent_target": "0",
+  "db_block_buffers": "0",
+  "db_block_checking": "FALSE",
+  "db_block_checksum": "TYPICAL",
+  "db_block_size": "8192",
+  "db_cache_advice": "ON",
+  "db_cache_size": "0",
+  "db_file_multiblock_read_count": "128",
+  "db_files": "200",
+  "db_flash_cache_size": "0",
+  "db_flashback_retention_target": "1440",
+  "db_index_compression_inheritance": "NONE",
+  "db_keep_cache_size": "0",
+  "db_lost_write_protect": "NONE",
+  "db_name": "ORCLCDB",
+  "db_recovery_file_dest": "/opt/oracle/oradata/fast_recovery_area",
+  "db_recovery_file_dest_size": "4781506560",
+  "db_recycle_cache_size": "0",
+  "db_securefile": "PREFERRED",
+  "db_ultra_safe": "OFF",
+  "db_unique_name": "ORCLCDB",
+  "db_unrecoverable_scn_tracking": "TRUE",
+  "db_writer_processes": "1",
+  "dbfips_140": "FALSE",
+  "dbwr_io_slaves": "0",
+  "ddl_lock_timeout": "0",
+  "deferred_segment_creation": "TRUE",
+  "dg_broker_config_file1": "/opt/oracle/product/12.1.0.2/dbhome_1/dbs/dr1ORCLCDB.dat",
+  "dg_broker_config_file2": "/opt/oracle/product/12.1.0.2/dbhome_1/dbs/dr2ORCLCDB.dat",
+  "dg_broker_start": "FALSE",
+  "diagnostic_dest": "/opt/oracle",
+  "disk_asynch_io": "TRUE",
+  "dispatchers": "(PROTOCOL=TCP) (SERVICE=ORCLCDBXDB)",
+  "distributed_lock_timeout": "60",
+  "dml_locks": "2076",
+  "dnfs_batch_size": "4096",
+  "dst_upgrade_insert_conv": "TRUE",
+  "enable_ddl_logging": "FALSE",
+  "enable_goldengate_replication": "FALSE",
+  "enable_pluggable_database": "TRUE",
+  "exclude_seed_cdb_view": "TRUE",
+  "fast_start_io_target": "0",
+  "fast_start_mttr_target": "0",
+  "fast_start_parallel_rollback": "LOW",
+  "file_mapping": "FALSE",
+  "filesystemio_options": "none",
+  "gcs_server_processes": "0",
+  "global_names": "FALSE",
+  "global_txn_processes": "1",
+  "hash_area_size": "131072",
+  "heat_map": "OFF",
+  "hi_shared_memory_address": "0",
+  "hs_autoregister": "TRUE",
+  "inmemory_force": "DEFAULT",
+  "inmemory_max_populate_servers": "0",
+  "inmemory_query": "ENABLE",
+  "inmemory_size": "0",
+  "inmemory_trickle_repopulate_servers_percent": "1",
+  "instance_name": "ORCLCDB",
+  "instance_number": "0",
+  "instance_type": "RDBMS",
+  "instant_restore": "FALSE",
+  "java_jit_enabled": "TRUE",
+  "java_max_sessionspace_size": "0",
+  "java_pool_size": "0",
+  "java_restrict": "none",
+  "java_soft_sessionspace_limit": "0",
+  "job_queue_processes": "1000",
+  "large_pool_size": "0",
+  "ldap_directory_access": "NONE",
+  "ldap_directory_sysauth": "no",
+  "license_max_sessions": "0",
+  "license_max_users": "0",
+  "license_sessions_warning": "0",
+  "lock_sga": "FALSE",
+  "log_archive_dest_state_1": "enable",
+  "log_archive_dest_state_10": "enable",
+  "log_archive_dest_state_11": "enable",
+  "log_archive_dest_state_12": "enable",
+  "log_archive_dest_state_13": "enable",
+  "log_archive_dest_state_14": "enable",
+  "log_archive_dest_state_15": "enable",
+  "log_archive_dest_state_16": "enable",
+  "log_archive_dest_state_17": "enable",
+  "log_archive_dest_state_18": "enable",
+  "log_archive_dest_state_19": "enable",
+  "log_archive_dest_state_2": "enable",
+  "log_archive_dest_state_20": "enable",
+  "log_archive_dest_state_21": "enable",
+  "log_archive_dest_state_22": "enable",
+  "log_archive_dest_state_23": "enable",
+  "log_archive_dest_state_24": "enable",
+  "log_archive_dest_state_25": "enable",
+  "log_archive_dest_state_26": "enable",
+  "log_archive_dest_state_27": "enable",
+  "log_archive_dest_state_28": "enable",
+  "log_archive_dest_state_29": "enable",
+  "log_archive_dest_state_3": "enable",
+  "log_archive_dest_state_30": "enable",
+  "log_archive_dest_state_31": "enable",
+  "log_archive_dest_state_4": "enable",
+  "log_archive_dest_state_5": "enable",
+  "log_archive_dest_state_6": "enable",
+  "log_archive_dest_state_7": "enable",
+  "log_archive_dest_state_8": "enable",
+  "log_archive_dest_state_9": "enable",
+  "log_archive_format": "%t_%s_%r.dbf",
+  "log_archive_max_processes": "4",
+  "log_archive_min_succeed_dest": "1",
+  "log_archive_start": "FALSE",
+  "log_archive_trace": "0",
+  "log_buffer": "13328384",
+  "log_checkpoint_interval": "0",
+  "log_checkpoint_timeout": "1800",
+  "log_checkpoints_to_alert": "FALSE",
+  "max_dump_file_size": "unlimited",
+  "max_enabled_roles": "150",
+  "max_string_size": "STANDARD",
+  "memory_max_target": "0",
+  "memory_target": "0",
+  "nls_comp": "BINARY",
+  "nls_language": "AMERICAN",
+  "nls_length_semantics": "BYTE",
+  "nls_nchar_conv_excp": "FALSE",
+  "nls_territory": "AMERICA",
+  "noncdb_compatible": "FALSE",
+  "o7_dictionary_accessibility": "FALSE",
+  "object_cache_max_size_percent": "10",
+  "object_cache_optimal_size": "102400",
+  "olap_page_pool_size": "0",
+  "open_cursors": "300",
+  "open_links": "4",
+  "open_links_per_instance": "4",
+  "optimizer_adaptive_features": "TRUE",
+  "optimizer_adaptive_reporting_only": "FALSE",
+  "optimizer_capture_sql_plan_baselines": "FALSE",
+  "optimizer_dynamic_sampling": "2",
+  "optimizer_features_enable": "12.1.0.2",
+  "optimizer_index_caching": "0",
+  "optimizer_index_cost_adj": "100",
+  "optimizer_inmemory_aware": "TRUE",
+  "optimizer_mode": "ALL_ROWS",
+  "optimizer_secure_view_merging": "TRUE",
+  "optimizer_use_invisible_indexes": "FALSE",
+  "optimizer_use_pending_statistics": "FALSE",
+  "optimizer_use_sql_plan_baselines": "TRUE",
+  "os_authent_prefix": "ops$",
+  "os_roles": "FALSE",
+  "parallel_adaptive_multi_user": "TRUE",
+  "parallel_automatic_tuning": "FALSE",
+  "parallel_degree_level": "100",
+  "parallel_degree_limit": "CPU",
+  "parallel_degree_policy": "MANUAL",
+  "parallel_execution_message_size": "16384",
+  "parallel_force_local": "FALSE",
+  "parallel_io_cap_enabled": "FALSE",
+  "parallel_max_servers": "160",
+  "parallel_min_percent": "0",
+  "parallel_min_servers": "16",
+  "parallel_min_time_threshold": "AUTO",
+  "parallel_server": "FALSE",
+  "parallel_server_instances": "1",
+  "parallel_servers_target": "64",
+  "parallel_threads_per_cpu": "2",
+  "permit_92_wrap_format": "TRUE",
+  "pga_aggregate_limit": "2147483648",
+  "pga_aggregate_target": "536870912",
+  "plscope_settings": "IDENTIFIERS:NONE",
+  "plsql_code_type": "INTERPRETED",
+  "plsql_debug": "FALSE",
+  "plsql_optimize_level": "2",
+  "plsql_v2_compatibility": "FALSE",
+  "plsql_warnings": "DISABLE:ALL",
+  "pre_page_sga": "TRUE",
+  "processes": "300",
+  "query_rewrite_enabled": "TRUE",
+  "query_rewrite_integrity": "enforced",
+  "read_only_open_delayed": "FALSE",
+  "recovery_parallelism": "0",
+  "recyclebin": "on",
+  "remote_dependencies_mode": "TIMESTAMP",
+  "remote_login_passwordfile": "EXCLUSIVE",
+  "remote_os_authent": "FALSE",
+  "remote_os_roles": "FALSE",
+  "replication_dependency_tracking": "TRUE",
+  "resource_limit": "TRUE",
+  "resource_manager_cpu_allocation": "4",
+  "result_cache_max_result": "5",
+  "result_cache_max_size": "8060928",
+  "result_cache_mode": "MANUAL",
+  "result_cache_remote_expiration": "0",
+  "resumable_timeout": "0",
+  "sec_case_sensitive_logon": "TRUE",
+  "sec_max_failed_login_attempts": "3",
+  "sec_protocol_error_further_action": "(DROP,3)",
+  "sec_protocol_error_trace_action": "TRACE",
+  "sec_return_server_release_banner": "FALSE",
+  "serial_reuse": "disable",
+  "service_names": "ORCLCDB",
+  "session_cached_cursors": "50",
+  "session_max_open_files": "10",
+  "sessions": "472",
+  "sga_max_size": "1610612736",
+  "sga_target": "1610612736",
+  "shadow_core_dump": "partial",
+  "shared_memory_address": "0",
+  "shared_pool_reserved_size": "17616076",
+  "shared_pool_size": "0",
+  "shared_servers": "1",
+  "skip_unusable_indexes": "TRUE",
+  "sort_area_retained_size": "0",
+  "sort_area_size": "65536",
+  "spatial_vector_acceleration": "FALSE",
+  "spfile": "/opt/oracle/product/12.1.0.2/dbhome_1/dbs/spfileORCLCDB.ora",
+  "sql92_security": "FALSE",
+  "sql_trace": "FALSE",
+  "sqltune_category": "DEFAULT",
+  "standby_archive_dest": "?/dbs/arch",
+  "standby_file_management": "MANUAL",
+  "star_transformation_enabled": "FALSE",
+  "statistics_level": "TYPICAL",
+  "streams_pool_size": "0",
+  "tape_asynch_io": "TRUE",
+  "temp_undo_enabled": "FALSE",
+  "thread": "0",
+  "threaded_execution": "FALSE",
+  "timed_os_statistics": "0",
+  "timed_statistics": "TRUE",
+  "trace_enabled": "TRUE",
+  "transactions": "519",
+  "transactions_per_rollback_segment": "5",
+  "undo_management": "AUTO",
+  "undo_retention": "900",
+  "undo_tablespace": "UNDOTBS1",
+  "unified_audit_sga_queue_size": "1048576",
+  "use_dedicated_broker": "FALSE",
+  "use_indirect_data_buffers": "FALSE",
+  "use_large_pages": "TRUE",
+  "user_dump_dest": "/opt/oracle/product/12.1.0.2/dbhome_1/rdbms/log",
+  "workarea_size_policy": "AUTO",
+  "xml_db_events": "enable"
+ }},
+ "local": null
+}

client/controller/sample_output/oracle/v12.1/summary.json

@@ -0,0 +1,9 @@
+{
+ "start_time": 1572978787444,
+ "end_time": 1572978819060,
+ "observation_time": 31,
+ "database_type": "oracle",
+ "database_version": "12.2.0.1.0",
+ "workload_name": "tpcc",
+ "real_database_version": "12.1.0.2.0"
+}

server/website/script/fixture_generators/knob_settings/oracle/.gitignore

@@ -0,0 +1 @@
+oracle-*_knobs.json

server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py

@@ -6,7 +6,6 @@
 import csv
 import json
 import shutil
-from collections import OrderedDict
 from operator import itemgetter
 
 # Oracle Type:
@@ -161,19 +160,12 @@ def set_field(fields):
 
 COLNAMES = ("NAME", "TYPE", "DEFAULT_VALUE", "DESCRIPTION")
 
-# DB_VERSIONS = {
-#     '19': 19,
-#     '12': 12,
-#     '12.1': 121,
-# }
 
-
+def process_version(version, delim=','):
-def process_version(version, dbms_id, delim=','):
     fields_list = []
     with open('oracle{}.csv'.format(version), 'r', newline='') as f:
         reader = csv.reader(f, delimiter=delim)
         header = [h.upper() for h in next(reader)]
-        # header = ["NUM","NAME","TYPE","VALUE","DISPLAY_VALUE","DEFAULT_VALUE","ISDEFAULT","ISSES_MODIFIABLE","ISSYS_MODIFIABLE","ISPDB_MODIFIABLE","ISINSTANCE_MODIFIABLE","ISMODIFIED","ISADJUSTED","ISDEPRECATED","ISBASIC","DESCRIPTION","UPDATE_COMMENT","HASH","CON_ID"]
         idxs = [header.index(c) for c in COLNAMES]
         ncols = len(header)
 
@@ -188,7 +180,6 @@ def process_version(version, dbms_id, delim=','):
                 if cname == 'NAME':
                     fields['name'] = value.upper()
                 elif cname == 'TYPE':
-                    print('TYPE: {}'.format(value))
                     value = int(value)
                     if value == 1:
                         fields['vartype'] = 4  # Boolean
@@ -203,7 +194,7 @@ def process_version(version, dbms_id, delim=','):
 
                 fields.update(
                     scope='global',
-                    dbms=dbms_id,
+                    dbms=version,
                     category='',
                     enumvals=None,
                     context='',
@@ -224,12 +215,13 @@ def process_version(version, dbms_id, delim=','):
     filename = 'oracle-{}_knobs.json'.format(version)
     with open(filename, 'w') as f:
         json.dump(final_metrics, f, indent=4)
-    # shutil.copy(filename, "../../../../website/fixtures/{}".format(filename))
+    shutil.copy(filename, "../../../../website/fixtures/{}".format(filename))
+
 
 def main():
-    process_version('19', 19)
+    process_version(19)
-    # process_version('12', 12)
+    # process_version(12)
-    process_version('12.1', 121, delim='|')
+    process_version(121, delim='|')
 
 
 if __name__ == '__main__':

server/website/script/fixture_generators/metric_settings/oracle/.gitignore

@@ -0,0 +1 @@
+oracle-*_metrics.json

server/website/script/fixture_generators/metric_settings/oracle/create_metric_settings.py

@@ -5,16 +5,10 @@
 #
 import json
 import os
+import shutil
 import sys
 from collections import OrderedDict
 
-ROOT = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
-while os.path.basename(ROOT) != 'website':
-    ROOT = os.path.dirname(ROOT)
-print('WEBSITE ROOT: {}'.format(ROOT))
-sys.path.insert(0, ROOT)
-
-from website.types import MetricType, VarType
 
 # Metric catalog fields:
 # dbms
@@ -24,42 +18,18 @@ from website.types import MetricType, VarType
 # scope
 # metric_type
 
-# Constants
+# Ottertune Type:
-MODEL = 'website.MetricCatalog'
+# STRING = 1
-SCOPE = 'global'
+# INTEGER = 2
-VERSIONS = (12, 19)
+# REAL = 3
+# BOOL = 4
+# ENUM = 5
+# TIMESTAMP = 6
 
-
+# Metric Types:
-# def main():
+# COUNTER = 1
-#     final_metrics = []
+# INFO = 2
-#     with open('oracle12.txt', 'r') as f:
+# STATISTICS = 3
-#         odd = 0
-#         entry = {}
-#         fields = {}
-#         lines = f.readlines()
-#         for line in lines:
-#             line = line.strip().replace("\n", "")
-#             if not line:
-#                 continue
-#             if line == 'NAME' or line.startswith('-'):
-#                 continue
-#             if odd == 0:
-#                 entry = {}
-#                 entry['model'] = 'website.MetricCatalog'
-#                 fields = {}
-#                 fields['name'] = "global." + line
-#                 fields['summary'] = line
-#                 fields['vartype'] = 2	 # int
-#                 fields['scope'] = 'global'
-#                 fields['metric_type'] = 3	 # stat
-#                 if fields['name'] == "global.user commits":
-#                     fields['metric_type'] = 1	 # counter
-#                 fields['dbms'] = 12  # oracle
-#                 entry['fields'] = fields
-#                 final_metrics.append(entry)
-#     with open('oracle-12_metrics.json', 'w') as f:
-#         json.dump(final_metrics, f, indent=4)
-#     shutil.copy('oracle-12_metrics.json', '../../../../website/fixtures/oracle-12_metrics.json')
 
 
 def check_type(value):
@@ -72,32 +42,31 @@ def check_type(value):
         except ValueError:
             pass
     if isinstance(value, int):
-        vtype = VarType.INTEGER
+        vtype = 2  # Integer
     elif isinstance(value, float):
-        vtype = VarType.REAL
+        vtype = 3  # Real
     else:
-        vtype = VarType.STRING
+        vtype = 1  # String
 
     return vtype
 
-
 def create_settings(metric_data, dbms):
     metrics = []
     for name, value in metric_data.items():
         vartype = check_type(value)
 
-        if vartype in (VarType.INTEGER, VarType.REAL):
+        if vartype in (2, 3):  # Numeric (integer/real)
             if 'average' in name or name.endswith('current') or \
-                    name.startswith('session pga memory'):
+                    name.startswith('sysstat.session pga memory'):
-                mettype = MetricType.STATISTICS
+                mettype = 3  # Statistic
             else:
-                mettype = MetricType.COUNTER  # Most int/float metrics are counters
+                mettype = 1  # Counter - most common type of numeric metric
         else:
-            mettype = MetricType.INFO
+            mettype = 2  # Info (anything that's not numeric)
         summary = '{}: {}'.format(name, value)
 
-        if name == 'user commits':
+        if name == 'sysstat.user commits':
-            assert vartype == VarType.INTEGER and mettype == MetricType.COUNTER
+            assert vartype == 2 and 1  # Check it's an int/counter
 
         entry = OrderedDict([
             ('dbms', dbms),
@@ -107,13 +76,17 @@ def create_settings(metric_data, dbms):
             ('scope', 'global'),
             ('metric_type', mettype),
         ])
-        metrics.append(OrderedDict([('fields', entry), ('model', MODEL)]))
+        metrics.append(OrderedDict([('fields', entry), ('model', 'website.MetricCatalog')]))
 
     return metrics
 
 
+VERSIONS = (121, 19)
+
+
 def usage():
-    print('python3 create_metric_settings.py [version] (valid versions: 12, 19)')
+    print('python3 create_metric_settings.py [version] (valid versions: {})'.format(
+        ', '.join(VERSIONS)))
     sys.exit(1)
 
 
@@ -132,10 +105,10 @@ def main():
 
         metrics = metrics['global']['global']
         meta = create_settings(metrics, version)
-        savepath = os.path.join(
+        filename = 'oracle-{}_metrics.json'.format(version)
-            ROOT, 'website', 'fixtures', 'oracle-{}_metrics.json'.format(version))
+        with open (filename, 'w') as f:
-        with open(savepath, 'w') as f:
             json.dump(meta, f, indent=4)
+        shutil.copy(filename, "../../../../website/fixtures/{}".format(filename))
 
 
 if __name__ == '__main__':

server/website/website/db/oracle/target_objective.py

@@ -15,13 +15,15 @@ class DBTime(BaseTargetObjective):
                          improvement=LESS_IS_BETTER)
 
     def compute(self, metrics, observation_time):
-        metric_names = ('global.db cpu', 'global.cursor: pin s wait on x.time_waited',
+        metric_names = (
-                        'global.user i/o wait time')
+            'global.sys_time_model.db cpu',
+            'global.system_event.cursor: pin s wait on x.time_waited',
+            'global.sysstat.user i/o wait time')
         db_time = float(sum(metrics[mname] for mname in metric_names)) / observation_time
         return db_time
 
 
 target_objective_list = tuple((DBMSType.ORACLE, target_obj) for target_obj in [  # pylint: disable=invalid-name
-    BaseThroughput(transactions_counter='global.user commits'),
+    BaseThroughput(transactions_counter='global.sysstat.user commits'),
     DBTime(),
 ])

server/website/website/migrations/0003_load_initial_data.py

@@ -21,6 +21,8 @@ def load_initial_data(apps, schema_editor):
         "myrocks-5.6_metrics.json",
         "oracle-12_knobs.json",
         "oracle-12_metrics.json",
+        "oracle-121_knobs.json",
+        "oracle-121_metrics.json",
         "oracle-19_knobs.json",
         "oracle-19_metrics.json",
     ]