public/ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit with BSL

Browse Source
This commit is contained in:
Andy Pavlo
2019-08-23 11:47:19 -04:00
commit 3e564ce922
286 changed files with 177642 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
client/controller/.gitignore vendored Normal file
View File

@@ -0,0 +1,28 @@
# Mac OS X hidden file
.DS_Store
# workspace configuration files
.settings/
.metadata/
# Gradle
.gradle/
/gradlew.bat
/gradlew
/settings.gradle
# Intellij
.idea
/dbcollector.iml
# generated files
bin/
build/
out/
output/
# lib
lib/
# log file
*.log

12
client/controller/README.md Normal file
View File

@@ -0,0 +1,12 @@
## OtterTune Controller
The controller is responsible for collecting database metrics and knobs information during an experiment.</br>
#### Usage:
To build the project, run `gradle build`.</br>
To run the controller, you need to provide a configuration file and provide command line arguments (command line arguments are optional). Then run `gradle run`.
* Command line arguments:
* time (flag : `-t`) </br>
The duration of the experiment in `seconds`. The default time is set to 300 seconds.
* configuration file path (flag : `-c`) </br>
The path of the input configuration file (required). Sample config files are under the directory `config`.

90
client/controller/build.gradle Normal file
View File

@@ -0,0 +1,90 @@
plugins {
id "de.undercouch.download" version "3.3.0"
id "com.github.spotbugs" version "2.0.0"
}
apply plugin: 'java'
apply plugin: 'maven'
apply plugin: 'application'
apply plugin: 'project-report'
mainClassName = "com.controller.Main"
test.testLogging { exceptionFormat "full"; events "failed", "passed", "skipped" }
repositories {
mavenCentral()
}
dependencies {
testCompile group: 'junit', name: 'junit', version: '4.11'
runtime fileTree(dir: 'lib', include: '*.jar')
compile group: 'net.sourceforge.collections', name: 'collections-generic', version: '4.01'
compile group: 'commons-lang', name: 'commons-lang', version: '2.6'
compile group: 'log4j', name: 'log4j', version: '1.2.17'
compile group: 'com.googlecode.json-simple', name: 'json-simple', version: '1.1.1'
compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3'
compile group: 'org.apache.httpcomponents', name: 'httpmime', version: '4.5.3'
// https://mvnrepository.com/artifact/com.github.fge/json-schema-validator
compile group: 'com.github.fge', name: 'json-schema-validator', version: '2.2.6'
// https://mvnrepository.com/artifact/com.github.fge/jackson-coreutils
compile group: 'com.github.fge', name: 'jackson-coreutils', version: '1.8'
// https://mvnrepository.com/artifact/com.github.fge/json-schema-core
compile group: 'com.github.fge', name: 'json-schema-core', version: '1.2.5'
// https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations
compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.9.0'
// https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core
compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.8.0'
// https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind
compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.8.0'
compile(group: "com.github.java-json-tools", name: "json-schema-validator", version: "2.2.8");
compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.10.0'
compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.10.0'
// https://mvnrepository.com/artifact/commons-cli/commons-cli
compile group: 'commons-cli', name: 'commons-cli', version: '1.2'
// https://mvnrepository.com/artifact/mysql/mysql-connector-java
compile group: 'mysql', name: 'mysql-connector-java', version: '5.1.6'
// https://mvnrepository.com/artifact/org.postgresql/postgresql
compile group: 'org.postgresql', name: 'postgresql', version: '9.4-1201-jdbc41'
// This lib has to be manually downloaded from Oracle
dependencies {compile files('lib/ojdbc8.jar')}
}
run {
if (project.hasProperty("appArgs")) {
args(appArgs.split(' '))
}
// Below is another way to add args but this causes gradle to
// always invoke the run application task
// args = ["-c","$config","-t","$time","-d","$dir"]
}
spotbugs {
toolVersion = '4.0.0-beta3'
ignoreFailures = true
}
// Note (07-29-2019): the HTML report for spotbugs is currently broken
tasks.withType(com.github.spotbugs.SpotBugsTask) {
reports {
xml.enabled true
html.enabled false
}
}
import de.undercouch.gradle.tasks.download.Download
task downloadJars(type: Download) {
src ([
'https://github.com/google/google-java-format/releases/download/google-java-format-1.5/google-java-format-1.5-all-deps.jar',
'https://github.com/checkstyle/checkstyle/releases/download/checkstyle-8.8/checkstyle-8.8-all.jar'
])
dest libsDir
overwrite false
}
build.finalizedBy(downloadJars)

9
client/controller/config/sample_mysql_config.json Normal file
View File

@@ -0,0 +1,9 @@
{
"database_type" : "mysql",
"database_url" : "jdbc:mysql://localhost:3306/mysqldb",
"username" : "MY_DATABASE_USERNAME",
"password" : "MY_DATABASE_PASSWORD",
"upload_code" : "DEPRECATED",
"upload_url" : "DEPRECATED",
"workload_name" : "workload_name"
}

9
client/controller/config/sample_oracle_config.json Normal file
View File

@@ -0,0 +1,9 @@
{
"database_type" : "oracle",
"database_url" : "jdbc:oracle:thin:@localhost:1521:orcldb",
"username" : "sys as sysdba",
"password" : "oracle",
"upload_code" : "DEPRECATED",
"upload_url" : "DEPRECATED",
"workload_name" : "tpcc"
}

9
client/controller/config/sample_postgres_config.json Normal file
View File

@@ -0,0 +1,9 @@
{
"database_type" : "postgres",
"database_url" : "jdbc:postgresql://localhost:5432/postgres",
"username" : "MY_DATABASE_USERNAME",
"password" : "MY_DATABASE_PASSWORD",
"upload_code" : "DEPRECATED",
"upload_url" : "DEPRECATED",
"workload_name" : "workload_name"
}

9
client/controller/config/sample_saphana_config.json Normal file
View File

@@ -0,0 +1,9 @@
{
"database_type" : "saphana",
"database_url" : "jdbc:sap://localhost:39015",
"username" : "MY_DATABASE_USERNAME",
"password" : "MY_DATABASE_PASSWORD",
"upload_code" : "DEPRECATED",
"upload_url" : "DEPRECATED",
"workload_name" : "workload_name"
}

6
client/controller/gradle/wrapper/gradle-wrapper.properties vendored Normal file
View File

@@ -0,0 +1,6 @@
#Thu Nov 30 15:45:05 EST 2017
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip

28
client/controller/log4j.properties Normal file
View File

@@ -0,0 +1,28 @@
# Set root logger level to DEBUG and its only appender to A1.
log4j.rootLogger=INFO, A1, FILE
log4j.rootLogger.layout=org.apache.log4j.PatternLayout
# A1 is set to be a ConsoleAppender.
log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%d{ABSOLUTE} (%F:%L) %-5p - %m%n
# Redirect log messages to a log file, support file rolling.
# Define the file appender
log4j.appender.FILE=org.apache.log4j.FileAppender
# Set the name of the file
log4j.appender.FILE.file=controller.log
# Set the immediate flush to true (default)
log4j.appender.FILE.immediateFlush=true
# Set the threshold to debug mode
log4j.appender.FILE.Threshold=debug
# Set the append to false, overwrite
log4j.appender.FILE.append=true
# Define the layout for file appender
log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%d{ABSOLUTE} (%F:%L) %-5p - %m%n

517
client/controller/sample_output/mysql/knobs.json Normal file
View File

@@ -0,0 +1,517 @@
{
"global": {"global": {
"auto_generate_certs": "ON",
"auto_increment_increment": "1",
"auto_increment_offset": "1",
"autocommit": "ON",
"automatic_sp_privileges": "ON",
"avoid_temporal_upgrade": "OFF",
"back_log": "80",
"basedir": "/usr/local/Cellar/mysql/5.7.20/",
"big_tables": "OFF",
"bind_address": "127.0.0.1",
"binlog_cache_size": "32768",
"binlog_checksum": "CRC32",
"binlog_direct_non_transactional_updates": "OFF",
"binlog_error_action": "ABORT_SERVER",
"binlog_format": "ROW",
"binlog_group_commit_sync_delay": "0",
"binlog_group_commit_sync_no_delay_count": "0",
"binlog_gtid_simple_recovery": "ON",
"binlog_max_flush_queue_time": "0",
"binlog_order_commits": "ON",
"binlog_row_image": "FULL",
"binlog_rows_query_log_events": "OFF",
"binlog_stmt_cache_size": "32768",
"block_encryption_mode": "aes-128-ecb",
"bulk_insert_buffer_size": "8388608",
"character_set_client": "utf8",
"character_set_connection": "utf8",
"character_set_database": "utf8",
"character_set_filesystem": "binary",
"character_set_results": "",
"character_set_server": "utf8",
"character_set_system": "utf8",
"character_sets_dir": "/usr/local/Cellar/mysql/5.7.20/share/mysql/charsets/",
"check_proxy_users": "OFF",
"collation_connection": "utf8_general_ci",
"collation_database": "utf8_general_ci",
"collation_server": "utf8_general_ci",
"completion_type": "NO_CHAIN",
"concurrent_insert": "AUTO",
"connect_timeout": "10",
"core_file": "OFF",
"datadir": "/usr/local/var/mysql/",
"date_format": "%Y-%m-%d",
"datetime_format": "%Y-%m-%d %H:%i:%s",
"default_authentication_plugin": "mysql_native_password",
"default_password_lifetime": "0",
"default_storage_engine": "InnoDB",
"default_tmp_storage_engine": "InnoDB",
"default_week_format": "0",
"delay_key_write": "ON",
"delayed_insert_limit": "100",
"delayed_insert_timeout": "300",
"delayed_queue_size": "1000",
"disabled_storage_engines": "",
"disconnect_on_expired_password": "ON",
"div_precision_increment": "4",
"end_markers_in_json": "OFF",
"enforce_gtid_consistency": "OFF",
"eq_range_index_dive_limit": "200",
"error_count": "0",
"event_scheduler": "OFF",
"expire_logs_days": "0",
"explicit_defaults_for_timestamp": "OFF",
"external_user": "",
"flush": "OFF",
"flush_time": "0",
"foreign_key_checks": "ON",
"ft_boolean_syntax": "+ -><()~*:\"\"&|",
"ft_max_word_len": "84",
"ft_min_word_len": "4",
"ft_query_expansion_limit": "20",
"ft_stopword_file": "(built-in)",
"general_log": "OFF",
"general_log_file": "/usr/local/var/mysql/JiangShuli.log",
"group_concat_max_len": "1024",
"gtid_executed_compression_period": "1000",
"gtid_mode": "OFF",
"gtid_next": "AUTOMATIC",
"gtid_owned": "",
"gtid_purged": "",
"have_compress": "YES",
"have_crypt": "YES",
"have_dynamic_loading": "YES",
"have_geometry": "YES",
"have_openssl": "YES",
"have_profiling": "YES",
"have_query_cache": "YES",
"have_rtree_keys": "YES",
"have_ssl": "YES",
"have_statement_timeout": "YES",
"have_symlink": "YES",
"host_cache_size": "279",
"hostname": "JiangShuli.local",
"identity": "0",
"ignore_builtin_innodb": "OFF",
"ignore_db_dirs": "",
"init_connect": "",
"init_file": "",
"init_slave": "",
"innodb_adaptive_flushing": "ON",
"innodb_adaptive_flushing_lwm": "10",
"innodb_adaptive_hash_index": "ON",
"innodb_adaptive_hash_index_parts": "8",
"innodb_adaptive_max_sleep_delay": "150000",
"innodb_api_bk_commit_interval": "5",
"innodb_api_disable_rowlock": "OFF",
"innodb_api_enable_binlog": "OFF",
"innodb_api_enable_mdl": "OFF",
"innodb_api_trx_level": "0",
"innodb_autoextend_increment": "64",
"innodb_autoinc_lock_mode": "1",
"innodb_buffer_pool_chunk_size": "134217728",
"innodb_buffer_pool_dump_at_shutdown": "ON",
"innodb_buffer_pool_dump_now": "OFF",
"innodb_buffer_pool_dump_pct": "25",
"innodb_buffer_pool_filename": "ib_buffer_pool",
"innodb_buffer_pool_instances": "1",
"innodb_buffer_pool_load_abort": "OFF",
"innodb_buffer_pool_load_at_startup": "ON",
"innodb_buffer_pool_load_now": "OFF",
"innodb_buffer_pool_size": "134217728",
"innodb_change_buffer_max_size": "25",
"innodb_change_buffering": "all",
"innodb_checksum_algorithm": "crc32",
"innodb_checksums": "ON",
"innodb_cmp_per_index_enabled": "OFF",
"innodb_commit_concurrency": "0",
"innodb_compression_failure_threshold_pct": "5",
"innodb_compression_level": "6",
"innodb_compression_pad_pct_max": "50",
"innodb_concurrency_tickets": "5000",
"innodb_data_file_path": "ibdata1:12M:autoextend",
"innodb_data_home_dir": "",
"innodb_deadlock_detect": "ON",
"innodb_default_row_format": "dynamic",
"innodb_disable_sort_file_cache": "OFF",
"innodb_doublewrite": "ON",
"innodb_fast_shutdown": "1",
"innodb_file_format": "Barracuda",
"innodb_file_format_check": "ON",
"innodb_file_format_max": "Barracuda",
"innodb_file_per_table": "ON",
"innodb_fill_factor": "100",
"innodb_flush_log_at_timeout": "1",
"innodb_flush_log_at_trx_commit": "1",
"innodb_flush_method": "",
"innodb_flush_neighbors": "1",
"innodb_flush_sync": "ON",
"innodb_flushing_avg_loops": "30",
"innodb_force_load_corrupted": "OFF",
"innodb_force_recovery": "0",
"innodb_ft_aux_table": "",
"innodb_ft_cache_size": "8000000",
"innodb_ft_enable_diag_print": "OFF",
"innodb_ft_enable_stopword": "ON",
"innodb_ft_max_token_size": "84",
"innodb_ft_min_token_size": "3",
"innodb_ft_num_word_optimize": "2000",
"innodb_ft_result_cache_limit": "2000000000",
"innodb_ft_server_stopword_table": "",
"innodb_ft_sort_pll_degree": "2",
"innodb_ft_total_cache_size": "640000000",
"innodb_ft_user_stopword_table": "",
"innodb_io_capacity": "200",
"innodb_io_capacity_max": "2000",
"innodb_large_prefix": "ON",
"innodb_lock_wait_timeout": "50",
"innodb_locks_unsafe_for_binlog": "OFF",
"innodb_log_buffer_size": "16777216",
"innodb_log_checksums": "ON",
"innodb_log_compressed_pages": "ON",
"innodb_log_file_size": "50331648",
"innodb_log_files_in_group": "2",
"innodb_log_group_home_dir": "./",
"innodb_log_write_ahead_size": "8192",
"innodb_lru_scan_depth": "1024",
"innodb_max_dirty_pages_pct": "75.000000",
"innodb_max_dirty_pages_pct_lwm": "0.000000",
"innodb_max_purge_lag": "0",
"innodb_max_purge_lag_delay": "0",
"innodb_max_undo_log_size": "1073741824",
"innodb_monitor_disable": "",
"innodb_monitor_enable": "",
"innodb_monitor_reset": "",
"innodb_monitor_reset_all": "",
"innodb_old_blocks_pct": "37",
"innodb_old_blocks_time": "1000",
"innodb_online_alter_log_max_size": "134217728",
"innodb_open_files": "2000",
"innodb_optimize_fulltext_only": "OFF",
"innodb_page_cleaners": "1",
"innodb_page_size": "16384",
"innodb_print_all_deadlocks": "OFF",
"innodb_purge_batch_size": "300",
"innodb_purge_rseg_truncate_frequency": "128",
"innodb_purge_threads": "4",
"innodb_random_read_ahead": "OFF",
"innodb_read_ahead_threshold": "56",
"innodb_read_io_threads": "4",
"innodb_read_only": "OFF",
"innodb_replication_delay": "0",
"innodb_rollback_on_timeout": "OFF",
"innodb_rollback_segments": "128",
"innodb_sort_buffer_size": "1048576",
"innodb_spin_wait_delay": "6",
"innodb_stats_auto_recalc": "ON",
"innodb_stats_include_delete_marked": "OFF",
"innodb_stats_method": "nulls_equal",
"innodb_stats_on_metadata": "OFF",
"innodb_stats_persistent": "ON",
"innodb_stats_persistent_sample_pages": "20",
"innodb_stats_sample_pages": "8",
"innodb_stats_transient_sample_pages": "8",
"innodb_status_output": "OFF",
"innodb_status_output_locks": "OFF",
"innodb_strict_mode": "ON",
"innodb_support_xa": "ON",
"innodb_sync_array_size": "1",
"innodb_sync_spin_loops": "30",
"innodb_table_locks": "ON",
"innodb_temp_data_file_path": "ibtmp1:12M:autoextend",
"innodb_thread_concurrency": "0",
"innodb_thread_sleep_delay": "10000",
"innodb_tmpdir": "",
"innodb_undo_directory": "./",
"innodb_undo_log_truncate": "OFF",
"innodb_undo_logs": "128",
"innodb_undo_tablespaces": "0",
"innodb_use_native_aio": "OFF",
"innodb_version": "5.7.20",
"innodb_write_io_threads": "4",
"insert_id": "0",
"interactive_timeout": "28800",
"internal_tmp_disk_storage_engine": "InnoDB",
"join_buffer_size": "262144",
"keep_files_on_create": "OFF",
"key_buffer_size": "8388608",
"key_cache_age_threshold": "300",
"key_cache_block_size": "1024",
"key_cache_division_limit": "100",
"large_files_support": "ON",
"large_page_size": "0",
"large_pages": "OFF",
"last_insert_id": "0",
"lc_messages": "en_US",
"lc_messages_dir": "/usr/local/Cellar/mysql/5.7.20/share/mysql/",
"lc_time_names": "en_US",
"license": "GPL",
"local_infile": "ON",
"lock_wait_timeout": "31536000",
"locked_in_memory": "OFF",
"log_bin": "OFF",
"log_bin_basename": "",
"log_bin_index": "",
"log_bin_trust_function_creators": "OFF",
"log_bin_use_v1_row_events": "OFF",
"log_builtin_as_identified_by_password": "OFF",
"log_error": "./JiangShuli.local.err",
"log_error_verbosity": "3",
"log_output": "FILE",
"log_queries_not_using_indexes": "OFF",
"log_slave_updates": "OFF",
"log_slow_admin_statements": "OFF",
"log_slow_slave_statements": "OFF",
"log_statements_unsafe_for_binlog": "ON",
"log_syslog": "OFF",
"log_syslog_facility": "daemon",
"log_syslog_include_pid": "ON",
"log_syslog_tag": "",
"log_throttle_queries_not_using_indexes": "0",
"log_timestamps": "UTC",
"log_warnings": "2",
"long_query_time": "10.000000",
"low_priority_updates": "OFF",
"lower_case_file_system": "ON",
"lower_case_table_names": "2",
"master_info_repository": "FILE",
"master_verify_checksum": "OFF",
"max_allowed_packet": "4194304",
"max_binlog_cache_size": "18446744073709547520",
"max_binlog_size": "1073741824",
"max_binlog_stmt_cache_size": "18446744073709547520",
"max_connect_errors": "100",
"max_connections": "151",
"max_delayed_threads": "20",
"max_digest_length": "1024",
"max_error_count": "64",
"max_execution_time": "0",
"max_heap_table_size": "16777216",
"max_insert_delayed_threads": "20",
"max_join_size": "18446744073709551615",
"max_length_for_sort_data": "1024",
"max_points_in_geometry": "65536",
"max_prepared_stmt_count": "16382",
"max_relay_log_size": "0",
"max_seeks_for_key": "18446744073709551615",
"max_sort_length": "1024",
"max_sp_recursion_depth": "0",
"max_tmp_tables": "32",
"max_user_connections": "0",
"max_write_lock_count": "18446744073709551615",
"metadata_locks_cache_size": "1024",
"metadata_locks_hash_instances": "8",
"min_examined_row_limit": "0",
"multi_range_count": "256",
"myisam_data_pointer_size": "6",
"myisam_max_sort_file_size": "9223372036853727232",
"myisam_mmap_size": "18446744073709551615",
"myisam_recover_options": "OFF",
"myisam_repair_threads": "1",
"myisam_sort_buffer_size": "8388608",
"myisam_stats_method": "nulls_unequal",
"myisam_use_mmap": "OFF",
"mysql_native_password_proxy_users": "OFF",
"net_buffer_length": "16384",
"net_read_timeout": "30",
"net_retry_count": "10",
"net_write_timeout": "60",
"new": "OFF",
"ngram_token_size": "2",
"offline_mode": "OFF",
"old": "OFF",
"old_alter_table": "OFF",
"old_passwords": "0",
"open_files_limit": "5000",
"optimizer_prune_level": "1",
"optimizer_search_depth": "62",
"optimizer_switch": "index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,engine_condition_pushdown=on,index_condition_pushdown=on,mrr=on,mrr_cost_based=on,block_nested_loop=on,batched_key_access=off,materialization=on,semijoin=on,loosescan=on,firstmatch=on,duplicateweedout=on,subquery_materialization_cost_based=on,use_index_extensions=on,condition_fanout_filter=on,derived_merge=on",
"optimizer_trace": "enabled=off,one_line=off",
"optimizer_trace_features": "greedy_search=on,range_optimizer=on,dynamic_range=on,repeated_subselect=on",
"optimizer_trace_limit": "1",
"optimizer_trace_max_mem_size": "16384",
"optimizer_trace_offset": "-1",
"parser_max_mem_size": "18446744073709551615",
"performance_schema": "ON",
"performance_schema_accounts_size": "-1",
"performance_schema_digests_size": "10000",
"performance_schema_events_stages_history_long_size": "10000",
"performance_schema_events_stages_history_size": "10",
"performance_schema_events_statements_history_long_size": "10000",
"performance_schema_events_statements_history_size": "10",
"performance_schema_events_transactions_history_long_size": "10000",
"performance_schema_events_transactions_history_size": "10",
"performance_schema_events_waits_history_long_size": "10000",
"performance_schema_events_waits_history_size": "10",
"performance_schema_hosts_size": "-1",
"performance_schema_max_cond_classes": "80",
"performance_schema_max_cond_instances": "-1",
"performance_schema_max_digest_length": "1024",
"performance_schema_max_file_classes": "80",
"performance_schema_max_file_handles": "32768",
"performance_schema_max_file_instances": "-1",
"performance_schema_max_index_stat": "-1",
"performance_schema_max_memory_classes": "320",
"performance_schema_max_metadata_locks": "-1",
"performance_schema_max_mutex_classes": "210",
"performance_schema_max_mutex_instances": "-1",
"performance_schema_max_prepared_statements_instances": "-1",
"performance_schema_max_program_instances": "-1",
"performance_schema_max_rwlock_classes": "40",
"performance_schema_max_rwlock_instances": "-1",
"performance_schema_max_socket_classes": "10",
"performance_schema_max_socket_instances": "-1",
"performance_schema_max_sql_text_length": "1024",
"performance_schema_max_stage_classes": "150",
"performance_schema_max_statement_classes": "193",
"performance_schema_max_statement_stack": "10",
"performance_schema_max_table_handles": "-1",
"performance_schema_max_table_instances": "-1",
"performance_schema_max_table_lock_stat": "-1",
"performance_schema_max_thread_classes": "50",
"performance_schema_max_thread_instances": "-1",
"performance_schema_session_connect_attrs_size": "512",
"performance_schema_setup_actors_size": "-1",
"performance_schema_setup_objects_size": "-1",
"performance_schema_users_size": "-1",
"pid_file": "/usr/local/var/mysql/JiangShuli.local.pid",
"plugin_dir": "/usr/local/Cellar/mysql/5.7.20/lib/plugin/",
"port": "3306",
"preload_buffer_size": "32768",
"profiling": "OFF",
"profiling_history_size": "15",
"protocol_version": "10",
"proxy_user": "",
"pseudo_slave_mode": "OFF",
"pseudo_thread_id": "3",
"query_alloc_block_size": "8192",
"query_cache_limit": "1048576",
"query_cache_min_res_unit": "4096",
"query_cache_size": "1048576",
"query_cache_type": "OFF",
"query_cache_wlock_invalidate": "OFF",
"query_prealloc_size": "8192",
"rand_seed1": "0",
"rand_seed2": "0",
"range_alloc_block_size": "4096",
"range_optimizer_max_mem_size": "8388608",
"rbr_exec_mode": "STRICT",
"read_buffer_size": "131072",
"read_only": "OFF",
"read_rnd_buffer_size": "262144",
"relay_log": "",
"relay_log_basename": "/usr/local/var/mysql/JiangShuli-relay-bin",
"relay_log_index": "/usr/local/var/mysql/JiangShuli-relay-bin.index",
"relay_log_info_file": "relay-log.info",
"relay_log_info_repository": "FILE",
"relay_log_purge": "ON",
"relay_log_recovery": "OFF",
"relay_log_space_limit": "0",
"report_host": "",
"report_password": "",
"report_port": "3306",
"report_user": "",
"require_secure_transport": "OFF",
"rpl_stop_slave_timeout": "31536000",
"secure_auth": "ON",
"secure_file_priv": "NULL",
"server_id": "0",
"server_id_bits": "32",
"server_uuid": "89286c98-d4ae-11e7-bb02-defe5cd105a2",
"session_track_gtids": "OFF",
"session_track_schema": "ON",
"session_track_state_change": "OFF",
"session_track_system_variables": "time_zone,autocommit,character_set_client,character_set_results,character_set_connection",
"session_track_transaction_info": "OFF",
"sha256_password_auto_generate_rsa_keys": "ON",
"sha256_password_private_key_path": "private_key.pem",
"sha256_password_proxy_users": "OFF",
"sha256_password_public_key_path": "public_key.pem",
"show_compatibility_56": "OFF",
"show_old_temporals": "OFF",
"skip_external_locking": "ON",
"skip_name_resolve": "OFF",
"skip_networking": "OFF",
"skip_show_database": "OFF",
"slave_allow_batching": "OFF",
"slave_checkpoint_group": "512",
"slave_checkpoint_period": "300",
"slave_compressed_protocol": "OFF",
"slave_exec_mode": "STRICT",
"slave_load_tmpdir": "/var/folders/hd/lw_b86v52pdfg323g_jn76k80000gn/T/",
"slave_max_allowed_packet": "1073741824",
"slave_net_timeout": "60",
"slave_parallel_type": "DATABASE",
"slave_parallel_workers": "0",
"slave_pending_jobs_size_max": "16777216",
"slave_preserve_commit_order": "OFF",
"slave_rows_search_algorithms": "TABLE_SCAN,INDEX_SCAN",
"slave_skip_errors": "OFF",
"slave_sql_verify_checksum": "ON",
"slave_transaction_retries": "10",
"slave_type_conversions": "",
"slow_launch_time": "2",
"slow_query_log": "OFF",
"slow_query_log_file": "/usr/local/var/mysql/JiangShuli-slow.log",
"socket": "/tmp/mysql.sock",
"sort_buffer_size": "262144",
"sql_auto_is_null": "OFF",
"sql_big_selects": "ON",
"sql_buffer_result": "OFF",
"sql_log_bin": "ON",
"sql_log_off": "OFF",
"sql_mode": "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION",
"sql_notes": "ON",
"sql_quote_show_create": "ON",
"sql_safe_updates": "OFF",
"sql_select_limit": "18446744073709551615",
"sql_slave_skip_counter": "0",
"sql_warnings": "OFF",
"ssl_ca": "ca.pem",
"ssl_capath": "",
"ssl_cert": "server-cert.pem",
"ssl_cipher": "",
"ssl_crl": "",
"ssl_crlpath": "",
"ssl_key": "server-key.pem",
"stored_program_cache": "256",
"super_read_only": "OFF",
"sync_binlog": "1",
"sync_frm": "ON",
"sync_master_info": "10000",
"sync_relay_log": "10000",
"sync_relay_log_info": "10000",
"system_time_zone": "EST",
"table_definition_cache": "1400",
"table_open_cache": "2000",
"table_open_cache_instances": "16",
"thread_cache_size": "9",
"thread_handling": "one-thread-per-connection",
"thread_stack": "262144",
"time_format": "%H:%i:%s",
"time_zone": "SYSTEM",
"timestamp": "1512532880.980241",
"tls_version": "TLSv1,TLSv1.1,TLSv1.2",
"tmp_table_size": "16777216",
"tmpdir": "/var/folders/hd/lw_b86v52pdfg323g_jn76k80000gn/T/",
"transaction_alloc_block_size": "8192",
"transaction_allow_batching": "OFF",
"transaction_isolation": "REPEATABLE-READ",
"transaction_prealloc_size": "4096",
"transaction_read_only": "OFF",
"transaction_write_set_extraction": "OFF",
"tx_isolation": "REPEATABLE-READ",
"tx_read_only": "OFF",
"unique_checks": "ON",
"updatable_views_with_limit": "YES",
"version": "5.7.20",
"version_comment": "Homebrew",
"version_compile_machine": "x86_64",
"version_compile_os": "osx10.12",
"wait_timeout": "28800",
"warning_count": "0"
}},
"local": null
}

362
client/controller/sample_output/mysql/metrics_after.json Normal file
View File

@@ -0,0 +1,362 @@
{
"global": {"global": {
"aborted_clients": "0",
"aborted_connects": "0",
"binlog_cache_disk_use": "0",
"binlog_cache_use": "0",
"binlog_stmt_cache_disk_use": "0",
"binlog_stmt_cache_use": "0",
"bytes_received": "1227",
"bytes_sent": "29713",
"com_admin_commands": "0",
"com_alter_db": "0",
"com_alter_db_upgrade": "0",
"com_alter_event": "0",
"com_alter_function": "0",
"com_alter_instance": "0",
"com_alter_procedure": "0",
"com_alter_server": "0",
"com_alter_table": "0",
"com_alter_tablespace": "0",
"com_alter_user": "0",
"com_analyze": "0",
"com_assign_to_keycache": "0",
"com_begin": "0",
"com_binlog": "0",
"com_call_procedure": "0",
"com_change_db": "0",
"com_change_master": "0",
"com_change_repl_filter": "0",
"com_check": "0",
"com_checksum": "0",
"com_commit": "0",
"com_create_db": "0",
"com_create_event": "0",
"com_create_function": "0",
"com_create_index": "0",
"com_create_procedure": "0",
"com_create_server": "0",
"com_create_table": "0",
"com_create_trigger": "0",
"com_create_udf": "0",
"com_create_user": "0",
"com_create_view": "0",
"com_dealloc_sql": "0",
"com_delete": "0",
"com_delete_multi": "0",
"com_do": "0",
"com_drop_db": "0",
"com_drop_event": "0",
"com_drop_function": "0",
"com_drop_index": "0",
"com_drop_procedure": "0",
"com_drop_server": "0",
"com_drop_table": "0",
"com_drop_trigger": "0",
"com_drop_user": "0",
"com_drop_view": "0",
"com_empty_query": "0",
"com_execute_sql": "0",
"com_explain_other": "0",
"com_flush": "0",
"com_get_diagnostics": "0",
"com_grant": "0",
"com_group_replication_start": "0",
"com_group_replication_stop": "0",
"com_ha_close": "0",
"com_ha_open": "0",
"com_ha_read": "0",
"com_help": "0",
"com_insert": "0",
"com_insert_select": "0",
"com_install_plugin": "0",
"com_kill": "0",
"com_load": "0",
"com_lock_tables": "0",
"com_optimize": "0",
"com_preload_keys": "0",
"com_prepare_sql": "0",
"com_purge": "0",
"com_purge_before_date": "0",
"com_release_savepoint": "0",
"com_rename_table": "0",
"com_rename_user": "0",
"com_repair": "0",
"com_replace": "0",
"com_replace_select": "0",
"com_reset": "0",
"com_resignal": "0",
"com_revoke": "0",
"com_revoke_all": "0",
"com_rollback": "0",
"com_rollback_to_savepoint": "0",
"com_savepoint": "0",
"com_select": "2",
"com_set_option": "2",
"com_show_binlog_events": "0",
"com_show_binlogs": "0",
"com_show_charsets": "1",
"com_show_collations": "1",
"com_show_create_db": "0",
"com_show_create_event": "0",
"com_show_create_func": "0",
"com_show_create_proc": "0",
"com_show_create_table": "0",
"com_show_create_trigger": "0",
"com_show_create_user": "0",
"com_show_databases": "0",
"com_show_engine_logs": "0",
"com_show_engine_mutex": "0",
"com_show_engine_status": "0",
"com_show_errors": "0",
"com_show_events": "0",
"com_show_fields": "0",
"com_show_function_code": "0",
"com_show_function_status": "0",
"com_show_grants": "0",
"com_show_keys": "0",
"com_show_master_status": "0",
"com_show_open_tables": "0",
"com_show_plugins": "0",
"com_show_privileges": "0",
"com_show_procedure_code": "0",
"com_show_procedure_status": "0",
"com_show_processlist": "0",
"com_show_profile": "0",
"com_show_profiles": "0",
"com_show_relaylog_events": "0",
"com_show_slave_hosts": "0",
"com_show_slave_status": "0",
"com_show_status": "1",
"com_show_storage_engines": "0",
"com_show_table_status": "0",
"com_show_tables": "0",
"com_show_triggers": "0",
"com_show_variables": "2",
"com_show_warnings": "0",
"com_shutdown": "0",
"com_signal": "0",
"com_slave_start": "0",
"com_slave_stop": "0",
"com_stmt_close": "0",
"com_stmt_execute": "0",
"com_stmt_fetch": "0",
"com_stmt_prepare": "0",
"com_stmt_reprepare": "0",
"com_stmt_reset": "0",
"com_stmt_send_long_data": "0",
"com_truncate": "0",
"com_uninstall_plugin": "0",
"com_unlock_tables": "0",
"com_update": "0",
"com_update_multi": "0",
"com_xa_commit": "0",
"com_xa_end": "0",
"com_xa_prepare": "0",
"com_xa_recover": "0",
"com_xa_rollback": "0",
"com_xa_start": "0",
"compression": "OFF",
"connection_errors_accept": "0",
"connection_errors_internal": "0",
"connection_errors_max_connections": "0",
"connection_errors_peer_address": "0",
"connection_errors_select": "0",
"connection_errors_tcpwrap": "0",
"connections": "4",
"created_tmp_disk_tables": "0",
"created_tmp_files": "5",
"created_tmp_tables": "4",
"delayed_errors": "0",
"delayed_insert_threads": "0",
"delayed_writes": "0",
"flush_commands": "1",
"handler_commit": "0",
"handler_delete": "0",
"handler_discover": "0",
"handler_external_lock": "4",
"handler_mrr_init": "0",
"handler_prepare": "0",
"handler_read_first": "0",
"handler_read_key": "0",
"handler_read_last": "0",
"handler_read_next": "0",
"handler_read_prev": "0",
"handler_read_rnd": "0",
"handler_read_rnd_next": "2317",
"handler_rollback": "0",
"handler_savepoint": "0",
"handler_savepoint_rollback": "0",
"handler_update": "0",
"handler_write": "1287",
"innodb_available_undo_logs": "128",
"innodb_buffer_pool_bytes_data": "11288576",
"innodb_buffer_pool_bytes_dirty": "0",
"innodb_buffer_pool_dump_status": "Dumping of buffer pool not started",
"innodb_buffer_pool_load_status": "Buffer pool(s) load completed at 171205 23:01:14",
"innodb_buffer_pool_pages_data": "689",
"innodb_buffer_pool_pages_dirty": "0",
"innodb_buffer_pool_pages_flushed": "37",
"innodb_buffer_pool_pages_free": "7502",
"innodb_buffer_pool_pages_misc": "0",
"innodb_buffer_pool_pages_total": "8191",
"innodb_buffer_pool_read_ahead": "0",
"innodb_buffer_pool_read_ahead_evicted": "0",
"innodb_buffer_pool_read_ahead_rnd": "0",
"innodb_buffer_pool_read_requests": "1833",
"innodb_buffer_pool_reads": "655",
"innodb_buffer_pool_resize_status": "",
"innodb_buffer_pool_wait_free": "0",
"innodb_buffer_pool_write_requests": "515",
"innodb_data_fsyncs": "7",
"innodb_data_pending_fsyncs": "0",
"innodb_data_pending_reads": "0",
"innodb_data_pending_writes": "0",
"innodb_data_read": "10801664",
"innodb_data_reads": "683",
"innodb_data_writes": "54",
"innodb_data_written": "641024",
"innodb_dblwr_pages_written": "2",
"innodb_dblwr_writes": "1",
"innodb_log_waits": "0",
"innodb_log_write_requests": "0",
"innodb_log_writes": "2",
"innodb_num_open_files": "20",
"innodb_os_log_fsyncs": "4",
"innodb_os_log_pending_fsyncs": "0",
"innodb_os_log_pending_writes": "0",
"innodb_os_log_written": "1024",
"innodb_page_size": "16384",
"innodb_pages_created": "35",
"innodb_pages_read": "654",
"innodb_pages_written": "37",
"innodb_row_lock_current_waits": "0",
"innodb_row_lock_time": "0",
"innodb_row_lock_time_avg": "0",
"innodb_row_lock_time_max": "0",
"innodb_row_lock_waits": "0",
"innodb_rows_deleted": "0",
"innodb_rows_inserted": "0",
"innodb_rows_read": "8",
"innodb_rows_updated": "0",
"innodb_truncated_status_writes": "0",
"key_blocks_not_flushed": "0",
"key_blocks_unused": "6695",
"key_blocks_used": "3",
"key_read_requests": "6",
"key_reads": "3",
"key_write_requests": "0",
"key_writes": "0",
"last_query_cost": "0.000000",
"last_query_partial_plans": "2",
"locked_connects": "0",
"max_execution_time_exceeded": "0",
"max_execution_time_set": "0",
"max_execution_time_set_failed": "0",
"max_used_connections": "1",
"max_used_connections_time": "2017-12-05 23:01:20",
"not_flushed_delayed_rows": "0",
"ongoing_anonymous_transaction_count": "0",
"open_files": "14",
"open_streams": "0",
"open_table_definitions": "228",
"open_tables": "100",
"opened_files": "357",
"opened_table_definitions": "0",
"opened_tables": "1",
"performance_schema_accounts_lost": "0",
"performance_schema_cond_classes_lost": "0",
"performance_schema_cond_instances_lost": "0",
"performance_schema_digest_lost": "0",
"performance_schema_file_classes_lost": "0",
"performance_schema_file_handles_lost": "0",
"performance_schema_file_instances_lost": "0",
"performance_schema_hosts_lost": "0",
"performance_schema_index_stat_lost": "0",
"performance_schema_locker_lost": "0",
"performance_schema_memory_classes_lost": "0",
"performance_schema_metadata_lock_lost": "0",
"performance_schema_mutex_classes_lost": "0",
"performance_schema_mutex_instances_lost": "0",
"performance_schema_nested_statement_lost": "0",
"performance_schema_prepared_statements_lost": "0",
"performance_schema_program_lost": "0",
"performance_schema_rwlock_classes_lost": "0",
"performance_schema_rwlock_instances_lost": "0",
"performance_schema_session_connect_attrs_lost": "0",
"performance_schema_socket_classes_lost": "0",
"performance_schema_socket_instances_lost": "0",
"performance_schema_stage_classes_lost": "0",
"performance_schema_statement_classes_lost": "0",
"performance_schema_table_handles_lost": "0",
"performance_schema_table_instances_lost": "0",
"performance_schema_table_lock_stat_lost": "0",
"performance_schema_thread_classes_lost": "0",
"performance_schema_thread_instances_lost": "0",
"performance_schema_users_lost": "0",
"prepared_stmt_count": "0",
"qcache_free_blocks": "1",
"qcache_free_memory": "1031832",
"qcache_hits": "0",
"qcache_inserts": "0",
"qcache_lowmem_prunes": "0",
"qcache_not_cached": "2",
"qcache_queries_in_cache": "0",
"qcache_total_blocks": "1",
"queries": "11",
"questions": "9",
"rsa_public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApBHbP3hArKih5/R7ffR9\n8ZUpK1VCGmD/7PCjOMQiChcb+KYr7PdgokSw/D95Ua84u36bvRHiB+7a6uGSYHZv\nOMB/h+zP0pSdmucCF5CNRuFgA6D0SHP7lql8O1t4x0dBYGkI0bkUgCxZeeRZshLm\nG6rGMXh79j3WlZP2Ny87TbCcgpbvW0tYxM9/pc1em/8y3+hDd3V0tVSBk36TEuJd\nJRr8Of09H0paSNjPdnvaZYdpXyhZUD0frhLSfnCj52OqFcypeelArR/ntEQffqif\n7CQvVm06mD5a5BlflgIVvnCLSH5pTBcn5tnPACFwbxBMYeRzfmRu+TFqFMBaxEhl\nVQIDAQAB\n-----END PUBLIC KEY-----\n",
"select_full_join": "0",
"select_full_range_join": "0",
"select_range": "0",
"select_range_check": "0",
"select_scan": "6",
"slave_open_temp_tables": "0",
"slow_launch_threads": "0",
"slow_queries": "0",
"sort_merge_passes": "0",
"sort_range": "0",
"sort_rows": "0",
"sort_scan": "0",
"ssl_accept_renegotiates": "0",
"ssl_accepts": "0",
"ssl_callback_cache_hits": "0",
"ssl_cipher": "",
"ssl_cipher_list": "",
"ssl_client_connects": "0",
"ssl_connect_renegotiates": "0",
"ssl_ctx_verify_depth": "18446744073709551615",
"ssl_ctx_verify_mode": "5",
"ssl_default_timeout": "0",
"ssl_finished_accepts": "0",
"ssl_finished_connects": "0",
"ssl_server_not_after": "Nov 27 02:39:35 2027 GMT",
"ssl_server_not_before": "Nov 29 02:39:35 2017 GMT",
"ssl_session_cache_hits": "0",
"ssl_session_cache_misses": "0",
"ssl_session_cache_mode": "SERVER",
"ssl_session_cache_overflows": "0",
"ssl_session_cache_size": "128",
"ssl_session_cache_timeouts": "0",
"ssl_sessions_reused": "0",
"ssl_used_session_cache_entries": "0",
"ssl_verify_depth": "0",
"ssl_verify_mode": "0",
"ssl_version": "",
"table_locks_immediate": "101",
"table_locks_waited": "0",
"table_open_cache_hits": "1",
"table_open_cache_misses": "1",
"table_open_cache_overflows": "0",
"tc_log_max_pages_used": "0",
"tc_log_page_size": "0",
"tc_log_page_waits": "0",
"threads_cached": "0",
"threads_connected": "1",
"threads_created": "1",
"threads_running": "1",
"uptime": "8",
"uptime_since_flush_status": "8"
}},
"local": null
}

362
client/controller/sample_output/mysql/metrics_before.json Normal file
View File

@@ -0,0 +1,362 @@
{
"global": {"global": {
"aborted_clients": "0",
"aborted_connects": "0",
"binlog_cache_disk_use": "0",
"binlog_cache_use": "0",
"binlog_stmt_cache_disk_use": "0",
"binlog_stmt_cache_use": "0",
"bytes_received": "1227",
"bytes_sent": "29713",
"com_admin_commands": "0",
"com_alter_db": "0",
"com_alter_db_upgrade": "0",
"com_alter_event": "0",
"com_alter_function": "0",
"com_alter_instance": "0",
"com_alter_procedure": "0",
"com_alter_server": "0",
"com_alter_table": "0",
"com_alter_tablespace": "0",
"com_alter_user": "0",
"com_analyze": "0",
"com_assign_to_keycache": "0",
"com_begin": "0",
"com_binlog": "0",
"com_call_procedure": "0",
"com_change_db": "0",
"com_change_master": "0",
"com_change_repl_filter": "0",
"com_check": "0",
"com_checksum": "0",
"com_commit": "0",
"com_create_db": "0",
"com_create_event": "0",
"com_create_function": "0",
"com_create_index": "0",
"com_create_procedure": "0",
"com_create_server": "0",
"com_create_table": "0",
"com_create_trigger": "0",
"com_create_udf": "0",
"com_create_user": "0",
"com_create_view": "0",
"com_dealloc_sql": "0",
"com_delete": "0",
"com_delete_multi": "0",
"com_do": "0",
"com_drop_db": "0",
"com_drop_event": "0",
"com_drop_function": "0",
"com_drop_index": "0",
"com_drop_procedure": "0",
"com_drop_server": "0",
"com_drop_table": "0",
"com_drop_trigger": "0",
"com_drop_user": "0",
"com_drop_view": "0",
"com_empty_query": "0",
"com_execute_sql": "0",
"com_explain_other": "0",
"com_flush": "0",
"com_get_diagnostics": "0",
"com_grant": "0",
"com_group_replication_start": "0",
"com_group_replication_stop": "0",
"com_ha_close": "0",
"com_ha_open": "0",
"com_ha_read": "0",
"com_help": "0",
"com_insert": "0",
"com_insert_select": "0",
"com_install_plugin": "0",
"com_kill": "0",
"com_load": "0",
"com_lock_tables": "0",
"com_optimize": "0",
"com_preload_keys": "0",
"com_prepare_sql": "0",
"com_purge": "0",
"com_purge_before_date": "0",
"com_release_savepoint": "0",
"com_rename_table": "0",
"com_rename_user": "0",
"com_repair": "0",
"com_replace": "0",
"com_replace_select": "0",
"com_reset": "0",
"com_resignal": "0",
"com_revoke": "0",
"com_revoke_all": "0",
"com_rollback": "0",
"com_rollback_to_savepoint": "0",
"com_savepoint": "0",
"com_select": "2",
"com_set_option": "2",
"com_show_binlog_events": "0",
"com_show_binlogs": "0",
"com_show_charsets": "1",
"com_show_collations": "1",
"com_show_create_db": "0",
"com_show_create_event": "0",
"com_show_create_func": "0",
"com_show_create_proc": "0",
"com_show_create_table": "0",
"com_show_create_trigger": "0",
"com_show_create_user": "0",
"com_show_databases": "0",
"com_show_engine_logs": "0",
"com_show_engine_mutex": "0",
"com_show_engine_status": "0",
"com_show_errors": "0",
"com_show_events": "0",
"com_show_fields": "0",
"com_show_function_code": "0",
"com_show_function_status": "0",
"com_show_grants": "0",
"com_show_keys": "0",
"com_show_master_status": "0",
"com_show_open_tables": "0",
"com_show_plugins": "0",
"com_show_privileges": "0",
"com_show_procedure_code": "0",
"com_show_procedure_status": "0",
"com_show_processlist": "0",
"com_show_profile": "0",
"com_show_profiles": "0",
"com_show_relaylog_events": "0",
"com_show_slave_hosts": "0",
"com_show_slave_status": "0",
"com_show_status": "1",
"com_show_storage_engines": "0",
"com_show_table_status": "0",
"com_show_tables": "0",
"com_show_triggers": "0",
"com_show_variables": "2",
"com_show_warnings": "0",
"com_shutdown": "0",
"com_signal": "0",
"com_slave_start": "0",
"com_slave_stop": "0",
"com_stmt_close": "0",
"com_stmt_execute": "0",
"com_stmt_fetch": "0",
"com_stmt_prepare": "0",
"com_stmt_reprepare": "0",
"com_stmt_reset": "0",
"com_stmt_send_long_data": "0",
"com_truncate": "0",
"com_uninstall_plugin": "0",
"com_unlock_tables": "0",
"com_update": "0",
"com_update_multi": "0",
"com_xa_commit": "0",
"com_xa_end": "0",
"com_xa_prepare": "0",
"com_xa_recover": "0",
"com_xa_rollback": "0",
"com_xa_start": "0",
"compression": "OFF",
"connection_errors_accept": "0",
"connection_errors_internal": "0",
"connection_errors_max_connections": "0",
"connection_errors_peer_address": "0",
"connection_errors_select": "0",
"connection_errors_tcpwrap": "0",
"connections": "4",
"created_tmp_disk_tables": "0",
"created_tmp_files": "5",
"created_tmp_tables": "4",
"delayed_errors": "0",
"delayed_insert_threads": "0",
"delayed_writes": "0",
"flush_commands": "1",
"handler_commit": "0",
"handler_delete": "0",
"handler_discover": "0",
"handler_external_lock": "4",
"handler_mrr_init": "0",
"handler_prepare": "0",
"handler_read_first": "0",
"handler_read_key": "0",
"handler_read_last": "0",
"handler_read_next": "0",
"handler_read_prev": "0",
"handler_read_rnd": "0",
"handler_read_rnd_next": "2317",
"handler_rollback": "0",
"handler_savepoint": "0",
"handler_savepoint_rollback": "0",
"handler_update": "0",
"handler_write": "1287",
"innodb_available_undo_logs": "128",
"innodb_buffer_pool_bytes_data": "11288576",
"innodb_buffer_pool_bytes_dirty": "0",
"innodb_buffer_pool_dump_status": "Dumping of buffer pool not started",
"innodb_buffer_pool_load_status": "Buffer pool(s) load completed at 171205 23:01:14",
"innodb_buffer_pool_pages_data": "689",
"innodb_buffer_pool_pages_dirty": "0",
"innodb_buffer_pool_pages_flushed": "37",
"innodb_buffer_pool_pages_free": "7502",
"innodb_buffer_pool_pages_misc": "0",
"innodb_buffer_pool_pages_total": "8191",
"innodb_buffer_pool_read_ahead": "0",
"innodb_buffer_pool_read_ahead_evicted": "0",
"innodb_buffer_pool_read_ahead_rnd": "0",
"innodb_buffer_pool_read_requests": "1833",
"innodb_buffer_pool_reads": "655",
"innodb_buffer_pool_resize_status": "",
"innodb_buffer_pool_wait_free": "0",
"innodb_buffer_pool_write_requests": "515",
"innodb_data_fsyncs": "7",
"innodb_data_pending_fsyncs": "0",
"innodb_data_pending_reads": "0",
"innodb_data_pending_writes": "0",
"innodb_data_read": "10801664",
"innodb_data_reads": "683",
"innodb_data_writes": "54",
"innodb_data_written": "641024",
"innodb_dblwr_pages_written": "2",
"innodb_dblwr_writes": "1",
"innodb_log_waits": "0",
"innodb_log_write_requests": "0",
"innodb_log_writes": "2",
"innodb_num_open_files": "20",
"innodb_os_log_fsyncs": "4",
"innodb_os_log_pending_fsyncs": "0",
"innodb_os_log_pending_writes": "0",
"innodb_os_log_written": "1024",
"innodb_page_size": "16384",
"innodb_pages_created": "35",
"innodb_pages_read": "654",
"innodb_pages_written": "37",
"innodb_row_lock_current_waits": "0",
"innodb_row_lock_time": "0",
"innodb_row_lock_time_avg": "0",
"innodb_row_lock_time_max": "0",
"innodb_row_lock_waits": "0",
"innodb_rows_deleted": "0",
"innodb_rows_inserted": "0",
"innodb_rows_read": "8",
"innodb_rows_updated": "0",
"innodb_truncated_status_writes": "0",
"key_blocks_not_flushed": "0",
"key_blocks_unused": "6695",
"key_blocks_used": "3",
"key_read_requests": "6",
"key_reads": "3",
"key_write_requests": "0",
"key_writes": "0",
"last_query_cost": "0.000000",
"last_query_partial_plans": "2",
"locked_connects": "0",
"max_execution_time_exceeded": "0",
"max_execution_time_set": "0",
"max_execution_time_set_failed": "0",
"max_used_connections": "1",
"max_used_connections_time": "2017-12-05 23:01:20",
"not_flushed_delayed_rows": "0",
"ongoing_anonymous_transaction_count": "0",
"open_files": "14",
"open_streams": "0",
"open_table_definitions": "228",
"open_tables": "100",
"opened_files": "357",
"opened_table_definitions": "0",
"opened_tables": "1",
"performance_schema_accounts_lost": "0",
"performance_schema_cond_classes_lost": "0",
"performance_schema_cond_instances_lost": "0",
"performance_schema_digest_lost": "0",
"performance_schema_file_classes_lost": "0",
"performance_schema_file_handles_lost": "0",
"performance_schema_file_instances_lost": "0",
"performance_schema_hosts_lost": "0",
"performance_schema_index_stat_lost": "0",
"performance_schema_locker_lost": "0",
"performance_schema_memory_classes_lost": "0",
"performance_schema_metadata_lock_lost": "0",
"performance_schema_mutex_classes_lost": "0",
"performance_schema_mutex_instances_lost": "0",
"performance_schema_nested_statement_lost": "0",
"performance_schema_prepared_statements_lost": "0",
"performance_schema_program_lost": "0",
"performance_schema_rwlock_classes_lost": "0",
"performance_schema_rwlock_instances_lost": "0",
"performance_schema_session_connect_attrs_lost": "0",
"performance_schema_socket_classes_lost": "0",
"performance_schema_socket_instances_lost": "0",
"performance_schema_stage_classes_lost": "0",
"performance_schema_statement_classes_lost": "0",
"performance_schema_table_handles_lost": "0",
"performance_schema_table_instances_lost": "0",
"performance_schema_table_lock_stat_lost": "0",
"performance_schema_thread_classes_lost": "0",
"performance_schema_thread_instances_lost": "0",
"performance_schema_users_lost": "0",
"prepared_stmt_count": "0",
"qcache_free_blocks": "1",
"qcache_free_memory": "1031832",
"qcache_hits": "0",
"qcache_inserts": "0",
"qcache_lowmem_prunes": "0",
"qcache_not_cached": "2",
"qcache_queries_in_cache": "0",
"qcache_total_blocks": "1",
"queries": "11",
"questions": "9",
"rsa_public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApBHbP3hArKih5/R7ffR9\n8ZUpK1VCGmD/7PCjOMQiChcb+KYr7PdgokSw/D95Ua84u36bvRHiB+7a6uGSYHZv\nOMB/h+zP0pSdmucCF5CNRuFgA6D0SHP7lql8O1t4x0dBYGkI0bkUgCxZeeRZshLm\nG6rGMXh79j3WlZP2Ny87TbCcgpbvW0tYxM9/pc1em/8y3+hDd3V0tVSBk36TEuJd\nJRr8Of09H0paSNjPdnvaZYdpXyhZUD0frhLSfnCj52OqFcypeelArR/ntEQffqif\n7CQvVm06mD5a5BlflgIVvnCLSH5pTBcn5tnPACFwbxBMYeRzfmRu+TFqFMBaxEhl\nVQIDAQAB\n-----END PUBLIC KEY-----\n",
"select_full_join": "0",
"select_full_range_join": "0",
"select_range": "0",
"select_range_check": "0",
"select_scan": "6",
"slave_open_temp_tables": "0",
"slow_launch_threads": "0",
"slow_queries": "0",
"sort_merge_passes": "0",
"sort_range": "0",
"sort_rows": "0",
"sort_scan": "0",
"ssl_accept_renegotiates": "0",
"ssl_accepts": "0",
"ssl_callback_cache_hits": "0",
"ssl_cipher": "",
"ssl_cipher_list": "",
"ssl_client_connects": "0",
"ssl_connect_renegotiates": "0",
"ssl_ctx_verify_depth": "18446744073709551615",
"ssl_ctx_verify_mode": "5",
"ssl_default_timeout": "0",
"ssl_finished_accepts": "0",
"ssl_finished_connects": "0",
"ssl_server_not_after": "Nov 27 02:39:35 2027 GMT",
"ssl_server_not_before": "Nov 29 02:39:35 2017 GMT",
"ssl_session_cache_hits": "0",
"ssl_session_cache_misses": "0",
"ssl_session_cache_mode": "SERVER",
"ssl_session_cache_overflows": "0",
"ssl_session_cache_size": "128",
"ssl_session_cache_timeouts": "0",
"ssl_sessions_reused": "0",
"ssl_used_session_cache_entries": "0",
"ssl_verify_depth": "0",
"ssl_verify_mode": "0",
"ssl_version": "",
"table_locks_immediate": "101",
"table_locks_waited": "0",
"table_open_cache_hits": "1",
"table_open_cache_misses": "1",
"table_open_cache_overflows": "0",
"tc_log_max_pages_used": "0",
"tc_log_page_size": "0",
"tc_log_page_waits": "0",
"threads_cached": "0",
"threads_connected": "1",
"threads_created": "1",
"threads_running": "1",
"uptime": "8",
"uptime_since_flush_status": "8"
}},
"local": null
}

8
client/controller/sample_output/mysql/summary.json Normal file
View File

@@ -0,0 +1,8 @@
{
"start_time": 1535653369274,
"end_time": 1535653559607,
"observation_time": 190,
"database_type": "mysql",
"database_version": "5.7.20",
"workload_name": "workload_name"
}

4902
client/controller/sample_output/oracle/knobs.json Normal file
View File

File diff suppressed because it is too large Load Diff

2041
client/controller/sample_output/oracle/metrics_after.json Normal file
View File

File diff suppressed because it is too large Load Diff

2041
client/controller/sample_output/oracle/metrics_before.json Normal file
View File

File diff suppressed because it is too large Load Diff

8
client/controller/sample_output/oracle/summary.json Normal file
View File

@@ -0,0 +1,8 @@
{
"start_time": 1563264802685,
"end_time": 1563265002918,
"observation_time": 200,
"database_type": "oracle",
"database_version": "19.0.0.0.0",
"workload_name": "tpcc"
}

274
client/controller/sample_output/postgres/knobs.json Normal file
View File

@@ -0,0 +1,274 @@
{
"global": {"global": {
"DateStyle": "ISO, MDY",
"IntervalStyle": "postgres",
"TimeZone": "America/New_York",
"allow_system_table_mods": "off",
"application_name": "",
"archive_command": "(disabled)",
"archive_mode": "off",
"archive_timeout": "0",
"array_nulls": "on",
"authentication_timeout": "1min",
"autovacuum": "on",
"autovacuum_analyze_scale_factor": "0.1",
"autovacuum_analyze_threshold": "50",
"autovacuum_freeze_max_age": "200000000",
"autovacuum_max_workers": "3",
"autovacuum_multixact_freeze_max_age": "400000000",
"autovacuum_naptime": "1min",
"autovacuum_vacuum_cost_delay": "20ms",
"autovacuum_vacuum_cost_limit": "-1",
"autovacuum_vacuum_scale_factor": "0.2",
"autovacuum_vacuum_threshold": "50",
"autovacuum_work_mem": "-1",
"backend_flush_after": "0",
"backslash_quote": "safe_encoding",
"bgwriter_delay": "200ms",
"bgwriter_flush_after": "0",
"bgwriter_lru_maxpages": "100",
"bgwriter_lru_multiplier": "2",
"block_size": "8192",
"bonjour": "off",
"bonjour_name": "",
"bytea_output": "hex",
"check_function_bodies": "on",
"checkpoint_completion_target": "0.5",
"checkpoint_flush_after": "0",
"checkpoint_timeout": "5min",
"checkpoint_warning": "30s",
"client_encoding": "UTF8",
"client_min_messages": "notice",
"cluster_name": "",
"commit_delay": "0",
"commit_siblings": "5",
"config_file": "/Users/MacadamiaKitten/Desktop/psql_db/postgresql.conf",
"constraint_exclusion": "partition",
"cpu_index_tuple_cost": "0.005",
"cpu_operator_cost": "0.0025",
"cpu_tuple_cost": "0.01",
"cursor_tuple_fraction": "0.1",
"data_checksums": "off",
"data_directory": "/Users/MacadamiaKitten/Desktop/psql_db",
"db_user_namespace": "off",
"deadlock_timeout": "1s",
"debug_assertions": "off",
"debug_pretty_print": "on",
"debug_print_parse": "off",
"debug_print_plan": "off",
"debug_print_rewritten": "off",
"default_statistics_target": "100",
"default_tablespace": "",
"default_text_search_config": "pg_catalog.english",
"default_transaction_deferrable": "off",
"default_transaction_isolation": "read committed",
"default_transaction_read_only": "off",
"default_with_oids": "off",
"dynamic_library_path": "$libdir",
"dynamic_shared_memory_type": "posix",
"effective_cache_size": "4GB",
"effective_io_concurrency": "0",
"enable_bitmapscan": "on",
"enable_gathermerge": "on",
"enable_hashagg": "on",
"enable_hashjoin": "on",
"enable_indexonlyscan": "on",
"enable_indexscan": "on",
"enable_material": "on",
"enable_mergejoin": "on",
"enable_nestloop": "on",
"enable_seqscan": "on",
"enable_sort": "on",
"enable_tidscan": "on",
"escape_string_warning": "on",
"event_source": "PostgreSQL",
"exit_on_error": "off",
"external_pid_file": "",
"extra_float_digits": "3",
"force_parallel_mode": "off",
"from_collapse_limit": "8",
"fsync": "on",
"full_page_writes": "on",
"geqo": "on",
"geqo_effort": "5",
"geqo_generations": "0",
"geqo_pool_size": "0",
"geqo_seed": "0",
"geqo_selection_bias": "2",
"geqo_threshold": "12",
"gin_fuzzy_search_limit": "0",
"gin_pending_list_limit": "4MB",
"hba_file": "/Users/MacadamiaKitten/Desktop/psql_db/pg_hba.conf",
"hot_standby": "on",
"hot_standby_feedback": "off",
"huge_pages": "try",
"ident_file": "/Users/MacadamiaKitten/Desktop/psql_db/pg_ident.conf",
"idle_in_transaction_session_timeout": "0",
"ignore_checksum_failure": "off",
"ignore_system_indexes": "off",
"integer_datetimes": "on",
"join_collapse_limit": "8",
"krb_caseins_users": "off",
"krb_server_keyfile": "FILE:/usr/local/etc/postgresql/krb5.keytab",
"lc_collate": "en_US.UTF-8",
"lc_ctype": "en_US.UTF-8",
"lc_messages": "en_US.UTF-8",
"lc_monetary": "en_US.UTF-8",
"lc_numeric": "en_US.UTF-8",
"lc_time": "en_US.UTF-8",
"listen_addresses": "localhost",
"lo_compat_privileges": "off",
"local_preload_libraries": "",
"lock_timeout": "0",
"log_autovacuum_min_duration": "-1",
"log_checkpoints": "off",
"log_connections": "off",
"log_destination": "stderr",
"log_directory": "log",
"log_disconnections": "off",
"log_duration": "off",
"log_error_verbosity": "default",
"log_executor_stats": "off",
"log_file_mode": "0600",
"log_filename": "postgresql-%Y-%m-%d_%H%M%S.log",
"log_hostname": "off",
"log_line_prefix": "%m [%p] ",
"log_lock_waits": "off",
"log_min_duration_statement": "-1",
"log_min_error_statement": "error",
"log_min_messages": "warning",
"log_parser_stats": "off",
"log_planner_stats": "off",
"log_replication_commands": "off",
"log_rotation_age": "1d",
"log_rotation_size": "10MB",
"log_statement": "none",
"log_statement_stats": "off",
"log_temp_files": "-1",
"log_timezone": "US/Eastern",
"log_truncate_on_rotation": "off",
"logging_collector": "off",
"maintenance_work_mem": "64MB",
"max_connections": "100",
"max_files_per_process": "1000",
"max_function_args": "100",
"max_identifier_length": "63",
"max_index_keys": "32",
"max_locks_per_transaction": "64",
"max_logical_replication_workers": "4",
"max_parallel_workers": "8",
"max_parallel_workers_per_gather": "2",
"max_pred_locks_per_page": "2",
"max_pred_locks_per_relation": "-2",
"max_pred_locks_per_transaction": "64",
"max_prepared_transactions": "0",
"max_replication_slots": "10",
"max_stack_depth": "2MB",
"max_standby_archive_delay": "30s",
"max_standby_streaming_delay": "30s",
"max_sync_workers_per_subscription": "2",
"max_wal_senders": "10",
"max_wal_size": "1GB",
"max_worker_processes": "8",
"min_parallel_index_scan_size": "512kB",
"min_parallel_table_scan_size": "8MB",
"min_wal_size": "80MB",
"old_snapshot_threshold": "-1",
"operator_precedence_warning": "off",
"parallel_setup_cost": "1000",
"parallel_tuple_cost": "0.1",
"password_encryption": "md5",
"port": "5432",
"post_auth_delay": "0",
"pre_auth_delay": "0",
"quote_all_identifiers": "off",
"random_page_cost": "4",
"replacement_sort_tuples": "150000",
"restart_after_crash": "on",
"row_security": "on",
"search_path": "\"$user\", public",
"segment_size": "1GB",
"seq_page_cost": "1",
"server_encoding": "UTF8",
"server_version": "10.1",
"server_version_num": "100001",
"session_preload_libraries": "",
"session_replication_role": "origin",
"shared_buffers": "128MB",
"shared_preload_libraries": "",
"ssl": "off",
"ssl_ca_file": "",
"ssl_cert_file": "server.crt",
"ssl_ciphers": "HIGH:MEDIUM:+3DES:!aNULL",
"ssl_crl_file": "",
"ssl_dh_params_file": "",
"ssl_ecdh_curve": "prime256v1",
"ssl_key_file": "server.key",
"ssl_prefer_server_ciphers": "on",
"standard_conforming_strings": "on",
"statement_timeout": "0",
"stats_temp_directory": "pg_stat_tmp",
"superuser_reserved_connections": "3",
"synchronize_seqscans": "on",
"synchronous_commit": "on",
"synchronous_standby_names": "",
"syslog_facility": "local0",
"syslog_ident": "postgres",
"syslog_sequence_numbers": "on",
"syslog_split_messages": "on",
"tcp_keepalives_count": "8",
"tcp_keepalives_idle": "7200",
"tcp_keepalives_interval": "75",
"temp_buffers": "8MB",
"temp_file_limit": "-1",
"temp_tablespaces": "",
"timezone_abbreviations": "Default",
"trace_notify": "off",
"trace_recovery_messages": "log",
"trace_sort": "off",
"track_activities": "on",
"track_activity_query_size": "1024",
"track_commit_timestamp": "off",
"track_counts": "on",
"track_functions": "none",
"track_io_timing": "off",
"transaction_deferrable": "off",
"transaction_isolation": "read committed",
"transaction_read_only": "off",
"transform_null_equals": "off",
"unix_socket_directories": "/tmp",
"unix_socket_group": "",
"unix_socket_permissions": "0777",
"update_process_title": "on",
"vacuum_cost_delay": "0",
"vacuum_cost_limit": "200",
"vacuum_cost_page_dirty": "20",
"vacuum_cost_page_hit": "1",
"vacuum_cost_page_miss": "10",
"vacuum_defer_cleanup_age": "0",
"vacuum_freeze_min_age": "50000000",
"vacuum_freeze_table_age": "150000000",
"vacuum_multixact_freeze_min_age": "5000000",
"vacuum_multixact_freeze_table_age": "150000000",
"wal_block_size": "8192",
"wal_buffers": "4MB",
"wal_compression": "off",
"wal_consistency_checking": "",
"wal_keep_segments": "0",
"wal_level": "replica",
"wal_log_hints": "off",
"wal_receiver_status_interval": "10s",
"wal_receiver_timeout": "1min",
"wal_retrieve_retry_interval": "5s",
"wal_segment_size": "16MB",
"wal_sender_timeout": "1min",
"wal_sync_method": "open_datasync",
"wal_writer_delay": "200ms",
"wal_writer_flush_after": "1MB",
"work_mem": "4MB",
"xmlbinary": "base64",
"xmloption": "content",
"zero_damaged_pages": "off"
}},
"local": null
}

582
client/controller/sample_output/postgres/metrics_after.json Normal file
View File

@@ -0,0 +1,582 @@
{
"global": {
"pg_stat_archiver": {
"archived_count": "0",
"failed_count": "0",
"stats_reset": "2017-11-10 10:59:47.397075-05"
},
"pg_stat_bgwriter": {
"buffers_alloc": "87670",
"buffers_backend": "81032",
"buffers_backend_fsync": "0",
"buffers_checkpoint": "33250",
"buffers_clean": "49590",
"checkpoint_sync_time": "19",
"checkpoint_write_time": "597851",
"checkpoints_req": "2",
"checkpoints_timed": "1277",
"maxwritten_clean": "325",
"stats_reset": "2017-11-10 10:59:47.397075-05"
}
},
"local": {
"table": {
"pg_stat_user_tables": {
"history": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"last_autoanalyze": "2017-11-20 15:59:02.567618-05",
"n_dead_tup": "0",
"n_live_tup": "60854",
"n_mod_since_analyze": "854",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "60854",
"n_tup_upd": "0",
"relid": "16536",
"relname": "history",
"schemaname": "public",
"seq_scan": "2",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"warehouse": {
"analyze_count": "0",
"autoanalyze_count": "2",
"autovacuum_count": "2",
"idx_scan": "202634",
"idx_tup_fetch": "202634",
"last_autoanalyze": "2017-11-20 19:23:34.236294-05",
"last_autovacuum": "2017-11-20 19:23:34.235793-05",
"n_dead_tup": "0",
"n_live_tup": "2",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "854",
"n_tup_ins": "2",
"n_tup_upd": "854",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"stock": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "644561",
"idx_tup_fetch": "644561",
"last_autoanalyze": "2017-11-20 15:59:01.368483-05",
"n_dead_tup": "4364",
"n_live_tup": "200000",
"n_mod_since_analyze": "8901",
"n_tup_del": "0",
"n_tup_hot_upd": "5305",
"n_tup_ins": "200000",
"n_tup_upd": "8901",
"relid": "16523",
"relname": "stock",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"customer": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "125261",
"idx_tup_fetch": "85299628",
"last_autoanalyze": "2017-11-20 15:59:18.824212-05",
"n_dead_tup": "1510",
"n_live_tup": "60000",
"n_mod_since_analyze": "1594",
"n_tup_del": "0",
"n_tup_hot_upd": "262",
"n_tup_ins": "60000",
"n_tup_upd": "1594",
"relid": "16540",
"relname": "customer",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"order_line": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "1655",
"idx_tup_fetch": "33762",
"last_autoanalyze": "2017-11-20 16:00:11.017507-05",
"n_dead_tup": "2550",
"n_live_tup": "608373",
"n_mod_since_analyze": "16230",
"n_tup_del": "0",
"n_tup_hot_upd": "5393",
"n_tup_ins": "608373",
"n_tup_upd": "7329",
"relid": "16513",
"relname": "order_line",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"oorder": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "627652",
"idx_tup_fetch": "627652",
"last_autoanalyze": "2017-11-20 15:59:54.690984-05",
"n_dead_tup": "117",
"n_live_tup": "60889",
"n_mod_since_analyze": "1629",
"n_tup_del": "0",
"n_tup_hot_upd": "662",
"n_tup_ins": "60900",
"n_tup_upd": "740",
"relid": "16528",
"relname": "oorder",
"schemaname": "public",
"seq_scan": "4",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"new_order": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "1481",
"idx_tup_fetch": "1480",
"last_autoanalyze": "2017-11-20 16:00:11.217111-05",
"n_dead_tup": "751",
"n_live_tup": "16964",
"n_mod_since_analyze": "1629",
"n_tup_del": "740",
"n_tup_hot_upd": "0",
"n_tup_ins": "17715",
"n_tup_upd": "0",
"relid": "16518",
"relname": "new_order",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"district": {
"analyze_count": "0",
"autoanalyze_count": "2",
"autovacuum_count": "0",
"idx_scan": "122234",
"idx_tup_fetch": "122234",
"last_autoanalyze": "2017-11-20 19:23:34.201509-05",
"n_dead_tup": "33",
"n_live_tup": "20",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "1754",
"n_tup_ins": "20",
"n_tup_upd": "1754",
"relid": "16549",
"relname": "district",
"schemaname": "public",
"seq_scan": "2221",
"seq_tup_read": "41522",
"vacuum_count": "0"
},
"item": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "209020",
"idx_tup_fetch": "209009",
"last_autoanalyze": "2017-11-20 15:59:26.613728-05",
"n_dead_tup": "0",
"n_live_tup": "102000",
"n_mod_since_analyze": "2000",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "100000",
"n_tup_upd": "0",
"relid": "16554",
"relname": "item",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
}
},
"pg_statio_user_tables": {
"history": {
"heap_blks_hit": "184380",
"heap_blks_read": "746",
"relid": "16536",
"relname": "history",
"schemaname": "public"
},
"order_line": {
"heap_blks_hit": "1869417",
"heap_blks_read": "12419",
"idx_blks_hit": "1788651",
"idx_blks_read": "3708",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"warehouse": {
"heap_blks_hit": "404486",
"heap_blks_read": "80",
"idx_blks_hit": "202643",
"idx_blks_read": "6",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
},
"new_order": {
"heap_blks_hit": "37856",
"heap_blks_read": "192",
"idx_blks_hit": "38225",
"idx_blks_read": "134",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"heap_blks_hit": "1920817",
"heap_blks_read": "11757",
"idx_blks_hit": "2447522",
"idx_blks_read": "1530",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"heap_blks_hit": "1378399",
"heap_blks_read": "928",
"idx_blks_hit": "3979052",
"idx_blks_read": "1881",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"district": {
"heap_blks_hit": "249754",
"heap_blks_read": "3",
"idx_blks_hit": "122259",
"idx_blks_read": "5",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"heap_blks_hit": "509702",
"heap_blks_read": "4542",
"idx_blks_hit": "617914",
"idx_blks_read": "877",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"customer": {
"heap_blks_hit": "70136669",
"heap_blks_read": "13826",
"idx_blks_hit": "1411491",
"idx_blks_read": "2716",
"relid": "16540",
"relname": "customer",
"schemaname": "public",
"tidx_blks_hit": "0",
"tidx_blks_read": "0",
"toast_blks_hit": "0",
"toast_blks_read": "0"
}
}
},
"database": {
"pg_stat_database": {
"postgres": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "115229324",
"blks_read": "104188",
"conflicts": "0",
"datid": "12558",
"datname": "postgres",
"deadlocks": "0",
"numbackends": "1",
"stats_reset": "2017-11-10 11:14:57.116228-05",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "1818",
"tup_fetched": "103355344",
"tup_inserted": "2210752",
"tup_returned": "110741743",
"tup_updated": "32675",
"xact_commit": "19082",
"xact_rollback": "17"
},
"tpcc": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "16384",
"datname": "tpcc",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
},
"template1": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "1",
"datname": "template1",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
},
"template0": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "12557",
"datname": "template0",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
}
},
"pg_stat_database_conflicts": {
"postgres": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "12558",
"datname": "postgres"
},
"tpcc": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "16384",
"datname": "tpcc"
},
"template1": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "1",
"datname": "template1"
},
"template0": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "12557",
"datname": "template0"
}
}
},
"indexes": {
"pg_stat_user_indexes": {
"order_line": {
"idx_scan": "1655",
"idx_tup_fetch": "33762",
"idx_tup_read": "35698",
"indexrelid": "16516",
"indexrelname": "order_line_pkey",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"new_order": {
"idx_scan": "1481",
"idx_tup_fetch": "1480",
"idx_tup_read": "2200",
"indexrelid": "16521",
"indexrelname": "new_order_pkey",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"idx_scan": "644561",
"idx_tup_fetch": "644561",
"idx_tup_read": "647319",
"indexrelid": "16526",
"indexrelname": "stock_pkey",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"idx_scan": "616371",
"idx_tup_fetch": "616371",
"idx_tup_read": "616371",
"indexrelid": "16565",
"indexrelname": "idx_order",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"customer": {
"idx_scan": "82442",
"idx_tup_fetch": "85256809",
"idx_tup_read": "85256841",
"indexrelid": "16564",
"indexrelname": "idx_customer_name",
"relid": "16540",
"relname": "customer",
"schemaname": "public"
},
"district": {
"idx_scan": "122234",
"idx_tup_fetch": "122234",
"idx_tup_read": "122234",
"indexrelid": "16552",
"indexrelname": "district_pkey",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"idx_scan": "209020",
"idx_tup_fetch": "209009",
"idx_tup_read": "209009",
"indexrelid": "16557",
"indexrelname": "item_pkey",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"warehouse": {
"idx_scan": "202634",
"idx_tup_fetch": "201331",
"idx_tup_read": "202634",
"indexrelid": "16562",
"indexrelname": "warehouse_pkey",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
}
},
"pg_statio_user_indexes": {
"order_line": {
"idx_blks_hit": "1788651",
"idx_blks_read": "3708",
"indexrelid": "16516",
"indexrelname": "order_line_pkey",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"new_order": {
"idx_blks_hit": "38225",
"idx_blks_read": "134",
"indexrelid": "16521",
"indexrelname": "new_order_pkey",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"idx_blks_hit": "2447522",
"idx_blks_read": "1530",
"indexrelid": "16526",
"indexrelname": "stock_pkey",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"idx_blks_hit": "3689479",
"idx_blks_read": "733",
"indexrelid": "16565",
"indexrelname": "idx_order",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"customer": {
"idx_blks_hit": "1151523",
"idx_blks_read": "1589",
"indexrelid": "16564",
"indexrelname": "idx_customer_name",
"relid": "16540",
"relname": "customer",
"schemaname": "public"
},
"district": {
"idx_blks_hit": "122259",
"idx_blks_read": "5",
"indexrelid": "16552",
"indexrelname": "district_pkey",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"idx_blks_hit": "617914",
"idx_blks_read": "877",
"indexrelid": "16557",
"indexrelname": "item_pkey",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"warehouse": {
"idx_blks_hit": "202643",
"idx_blks_read": "6",
"indexrelid": "16562",
"indexrelname": "warehouse_pkey",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
}
}
}
}
}

582
client/controller/sample_output/postgres/metrics_before.json Normal file
View File

@@ -0,0 +1,582 @@
{
"global": {
"pg_stat_archiver": {
"archived_count": "0",
"failed_count": "0",
"stats_reset": "2017-11-10 10:59:47.397075-05"
},
"pg_stat_bgwriter": {
"buffers_alloc": "87670",
"buffers_backend": "81032",
"buffers_backend_fsync": "0",
"buffers_checkpoint": "33250",
"buffers_clean": "49590",
"checkpoint_sync_time": "19",
"checkpoint_write_time": "597851",
"checkpoints_req": "2",
"checkpoints_timed": "1277",
"maxwritten_clean": "325",
"stats_reset": "2017-11-10 10:59:47.397075-05"
}
},
"local": {
"table": {
"pg_stat_user_tables": {
"history": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"last_autoanalyze": "2017-11-20 15:59:02.567618-05",
"n_dead_tup": "0",
"n_live_tup": "60854",
"n_mod_since_analyze": "854",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "60854",
"n_tup_upd": "0",
"relid": "16536",
"relname": "history",
"schemaname": "public",
"seq_scan": "2",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"warehouse": {
"analyze_count": "0",
"autoanalyze_count": "2",
"autovacuum_count": "2",
"idx_scan": "202634",
"idx_tup_fetch": "202634",
"last_autoanalyze": "2017-11-20 19:23:34.236294-05",
"last_autovacuum": "2017-11-20 19:23:34.235793-05",
"n_dead_tup": "0",
"n_live_tup": "2",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "854",
"n_tup_ins": "2",
"n_tup_upd": "854",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"stock": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "644561",
"idx_tup_fetch": "644561",
"last_autoanalyze": "2017-11-20 15:59:01.368483-05",
"n_dead_tup": "4364",
"n_live_tup": "200000",
"n_mod_since_analyze": "8901",
"n_tup_del": "0",
"n_tup_hot_upd": "5305",
"n_tup_ins": "200000",
"n_tup_upd": "8901",
"relid": "16523",
"relname": "stock",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"customer": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "125261",
"idx_tup_fetch": "85299628",
"last_autoanalyze": "2017-11-20 15:59:18.824212-05",
"n_dead_tup": "1510",
"n_live_tup": "60000",
"n_mod_since_analyze": "1594",
"n_tup_del": "0",
"n_tup_hot_upd": "262",
"n_tup_ins": "60000",
"n_tup_upd": "1594",
"relid": "16540",
"relname": "customer",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"order_line": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "1655",
"idx_tup_fetch": "33762",
"last_autoanalyze": "2017-11-20 16:00:11.017507-05",
"n_dead_tup": "2550",
"n_live_tup": "608373",
"n_mod_since_analyze": "16230",
"n_tup_del": "0",
"n_tup_hot_upd": "5393",
"n_tup_ins": "608373",
"n_tup_upd": "7329",
"relid": "16513",
"relname": "order_line",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"oorder": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "627652",
"idx_tup_fetch": "627652",
"last_autoanalyze": "2017-11-20 15:59:54.690984-05",
"n_dead_tup": "117",
"n_live_tup": "60889",
"n_mod_since_analyze": "1629",
"n_tup_del": "0",
"n_tup_hot_upd": "662",
"n_tup_ins": "60900",
"n_tup_upd": "740",
"relid": "16528",
"relname": "oorder",
"schemaname": "public",
"seq_scan": "4",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"new_order": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "1481",
"idx_tup_fetch": "1480",
"last_autoanalyze": "2017-11-20 16:00:11.217111-05",
"n_dead_tup": "751",
"n_live_tup": "16964",
"n_mod_since_analyze": "1629",
"n_tup_del": "740",
"n_tup_hot_upd": "0",
"n_tup_ins": "17715",
"n_tup_upd": "0",
"relid": "16518",
"relname": "new_order",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"district": {
"analyze_count": "0",
"autoanalyze_count": "2",
"autovacuum_count": "0",
"idx_scan": "122234",
"idx_tup_fetch": "122234",
"last_autoanalyze": "2017-11-20 19:23:34.201509-05",
"n_dead_tup": "33",
"n_live_tup": "20",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "1754",
"n_tup_ins": "20",
"n_tup_upd": "1754",
"relid": "16549",
"relname": "district",
"schemaname": "public",
"seq_scan": "2221",
"seq_tup_read": "41522",
"vacuum_count": "0"
},
"item": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "209020",
"idx_tup_fetch": "209009",
"last_autoanalyze": "2017-11-20 15:59:26.613728-05",
"n_dead_tup": "0",
"n_live_tup": "102000",
"n_mod_since_analyze": "2000",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "100000",
"n_tup_upd": "0",
"relid": "16554",
"relname": "item",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
}
},
"pg_statio_user_tables": {
"history": {
"heap_blks_hit": "184380",
"heap_blks_read": "746",
"relid": "16536",
"relname": "history",
"schemaname": "public"
},
"order_line": {
"heap_blks_hit": "1869417",
"heap_blks_read": "12419",
"idx_blks_hit": "1788651",
"idx_blks_read": "3708",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"warehouse": {
"heap_blks_hit": "404486",
"heap_blks_read": "80",
"idx_blks_hit": "202643",
"idx_blks_read": "6",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
},
"new_order": {
"heap_blks_hit": "37856",
"heap_blks_read": "192",
"idx_blks_hit": "38225",
"idx_blks_read": "134",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"heap_blks_hit": "1920817",
"heap_blks_read": "11757",
"idx_blks_hit": "2447522",
"idx_blks_read": "1530",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"heap_blks_hit": "1378399",
"heap_blks_read": "928",
"idx_blks_hit": "3979052",
"idx_blks_read": "1881",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"district": {
"heap_blks_hit": "249754",
"heap_blks_read": "3",
"idx_blks_hit": "122259",
"idx_blks_read": "5",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"heap_blks_hit": "509702",
"heap_blks_read": "4542",
"idx_blks_hit": "617914",
"idx_blks_read": "877",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"customer": {
"heap_blks_hit": "70136669",
"heap_blks_read": "13826",
"idx_blks_hit": "1411491",
"idx_blks_read": "2716",
"relid": "16540",
"relname": "customer",
"schemaname": "public",
"tidx_blks_hit": "0",
"tidx_blks_read": "0",
"toast_blks_hit": "0",
"toast_blks_read": "0"
}
}
},
"database": {
"pg_stat_database": {
"postgres": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "115229324",
"blks_read": "104188",
"conflicts": "0",
"datid": "12558",
"datname": "postgres",
"deadlocks": "0",
"numbackends": "1",
"stats_reset": "2017-11-10 11:14:57.116228-05",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "1818",
"tup_fetched": "103355344",
"tup_inserted": "2210752",
"tup_returned": "110741743",
"tup_updated": "32675",
"xact_commit": "19082",
"xact_rollback": "17"
},
"tpcc": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "16384",
"datname": "tpcc",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
},
"template1": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "1",
"datname": "template1",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
},
"template0": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "12557",
"datname": "template0",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
}
},
"pg_stat_database_conflicts": {
"postgres": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "12558",
"datname": "postgres"
},
"tpcc": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "16384",
"datname": "tpcc"
},
"template1": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "1",
"datname": "template1"
},
"template0": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "12557",
"datname": "template0"
}
}
},
"indexes": {
"pg_stat_user_indexes": {
"order_line": {
"idx_scan": "1655",
"idx_tup_fetch": "33762",
"idx_tup_read": "35698",
"indexrelid": "16516",
"indexrelname": "order_line_pkey",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"new_order": {
"idx_scan": "1481",
"idx_tup_fetch": "1480",
"idx_tup_read": "2200",
"indexrelid": "16521",
"indexrelname": "new_order_pkey",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"idx_scan": "644561",
"idx_tup_fetch": "644561",
"idx_tup_read": "647319",
"indexrelid": "16526",
"indexrelname": "stock_pkey",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"idx_scan": "616371",
"idx_tup_fetch": "616371",
"idx_tup_read": "616371",
"indexrelid": "16565",
"indexrelname": "idx_order",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"customer": {
"idx_scan": "82442",
"idx_tup_fetch": "85256809",
"idx_tup_read": "85256841",
"indexrelid": "16564",
"indexrelname": "idx_customer_name",
"relid": "16540",
"relname": "customer",
"schemaname": "public"
},
"district": {
"idx_scan": "122234",
"idx_tup_fetch": "122234",
"idx_tup_read": "122234",
"indexrelid": "16552",
"indexrelname": "district_pkey",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"idx_scan": "209020",
"idx_tup_fetch": "209009",
"idx_tup_read": "209009",
"indexrelid": "16557",
"indexrelname": "item_pkey",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"warehouse": {
"idx_scan": "202634",
"idx_tup_fetch": "201331",
"idx_tup_read": "202634",
"indexrelid": "16562",
"indexrelname": "warehouse_pkey",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
}
},
"pg_statio_user_indexes": {
"order_line": {
"idx_blks_hit": "1788651",
"idx_blks_read": "3708",
"indexrelid": "16516",
"indexrelname": "order_line_pkey",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"new_order": {
"idx_blks_hit": "38225",
"idx_blks_read": "134",
"indexrelid": "16521",
"indexrelname": "new_order_pkey",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"idx_blks_hit": "2447522",
"idx_blks_read": "1530",
"indexrelid": "16526",
"indexrelname": "stock_pkey",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"idx_blks_hit": "3689479",
"idx_blks_read": "733",
"indexrelid": "16565",
"indexrelname": "idx_order",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"customer": {
"idx_blks_hit": "1151523",
"idx_blks_read": "1589",
"indexrelid": "16564",
"indexrelname": "idx_customer_name",
"relid": "16540",
"relname": "customer",
"schemaname": "public"
},
"district": {
"idx_blks_hit": "122259",
"idx_blks_read": "5",
"indexrelid": "16552",
"indexrelname": "district_pkey",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"idx_blks_hit": "617914",
"idx_blks_read": "877",
"indexrelid": "16557",
"indexrelname": "item_pkey",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"warehouse": {
"idx_blks_hit": "202643",
"idx_blks_read": "6",
"indexrelid": "16562",
"indexrelname": "warehouse_pkey",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
}
}
}
}
}

8
client/controller/sample_output/postgres/summary.json Normal file
View File

@@ -0,0 +1,8 @@
{
"start_time": 1535653369274,
"end_time": 1535653559607,
"observation_time": 190,
"database_type": "postgres",
"database_version": "9.3",
"workload_name": "workload_name"
}

9440
client/controller/sample_output/saphana/knobs.json Normal file
View File

File diff suppressed because it is too large Load Diff

8919
client/controller/sample_output/saphana/metrics_after.json Normal file
View File

File diff suppressed because it is too large Load Diff

8919
client/controller/sample_output/saphana/metrics_before.json Normal file
View File

File diff suppressed because it is too large Load Diff

8
client/controller/sample_output/saphana/summary.json Normal file
View File

@@ -0,0 +1,8 @@
{
"start_time": 1535653369274,
"end_time": 1535653559607,
"observation_time": 190,
"database_type": "saphana",
"database_version": "2.00.023.00.1513691289",
"workload_name": "workload_name"
}

105
client/controller/src/main/java/com/controller/ControllerConfiguration.java Normal file
View File

@@ -0,0 +1,105 @@
/*
* OtterTune - ControllerConfiguration.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller;
/** Controller Configuration. */
public class ControllerConfiguration {
private DatabaseType dbType;
private String dbName;
private String dbUsername;
private String dbPassword;
private String dbURL;
private String uploadCode;
private String uploadURL;
private String workloadName;
public ControllerConfiguration() {}
public ControllerConfiguration(
String dbName,
String dbUsername,
String dbPassword,
String dbURL,
String uploadCode,
String uploadURL,
String workloadName) {
this.dbType = DatabaseType.get(dbName);
this.dbName = dbName;
this.dbUsername = dbUsername;
this.dbPassword = dbPassword;
this.dbURL = dbURL;
this.uploadCode = uploadCode;
this.uploadURL = uploadURL;
this.workloadName = workloadName;
}
/* Mutators */
public void setDBType(DatabaseType dbType) {
this.dbType = dbType;
}
public void setDBName(String dbName) {
this.dbName = dbName;
}
public void setDBUsername(String dbUsername) {
this.dbUsername = dbUsername;
}
public void setPassword(String dbPassword) {
this.dbPassword = dbPassword;
}
public void setDBURL(String dbURL) {
this.dbURL = dbURL;
}
public void setUploadCode(String uploadCode) {
this.uploadCode = uploadCode;
}
public void setUploadURL(String uploadURL) {
this.uploadURL = uploadURL;
}
public void setWorkloadName(String workloadName) {
this.workloadName = workloadName;
}
/* Getters */
public DatabaseType getDBType() {
return this.dbType;
}
public String getDBName() {
return this.dbName;
}
public String getDBUsername() {
return this.dbUsername;
}
public String getDBPassword() {
return this.dbPassword;
}
public String getDBURL() {
return this.dbURL;
}
public String getUploadCode() {
return this.uploadCode;
}
public String getUploadURL() {
return this.uploadURL;
}
public String getWorkloadName() {
return this.workloadName;
}
}

296
client/controller/src/main/java/com/controller/Main.java Normal file
View File

@@ -0,0 +1,296 @@
/*
* OtterTune - Main.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller;
import com.controller.collectors.DBCollector;
import com.controller.collectors.MySQLCollector;
import com.controller.collectors.OracleCollector;
import com.controller.collectors.PostgresCollector;
import com.controller.collectors.SAPHanaCollector;
import com.controller.types.JSONSchemaType;
import com.controller.util.FileUtil;
import com.controller.util.JSONUtil;
import com.controller.util.json.JSONException;
import com.controller.util.json.JSONObject;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.lang.management.ManagementFactory;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import sun.misc.Signal;
/**
* Controller main.
*
* @author Shuli
*/
public class Main {
static final Logger LOG = Logger.getLogger(Main.class);
// Default output directory name
private static final String DEFAULT_DIRECTORY = "output";
// Default observation period time (5 minutes)
private static final int DEFAULT_TIME_SECONDS = -1;
// Path to JSON schema directory
private static final String SCHEMA_PATH = "src/main/java/com/controller/json_validation_schema";
private static final int TO_MILLISECONDS = 1000;
private static boolean keepRunning = true;
private static boolean firstCollecting = false;
public static void main(String[] args) {
// Initialize log4j
PropertyConfigurator.configure("log4j.properties");
// Initialize keepRunning
keepRunning = true;
// Initialize firstCollecting
firstCollecting = false;
// Create the command line parser
CommandLineParser parser = new PosixParser();
Options options = new Options();
options.addOption("c", "config", true, "[required] Controller configuration file");
options.addOption("t", "time", true, "The observation time in seconds, default is 300s");
options.addOption(
"d", "directory", true, "Base directory for the result files, default is 'output'");
options.addOption("h", "help", true, "Print this help");
String configFilePath = null;
// Parse the command line arguments
CommandLine argsLine;
try {
argsLine = parser.parse(options, args);
} catch (ParseException e) {
LOG.error("Unable to Parse command line arguments");
printUsage(options);
return;
}
if (argsLine.hasOption("h")) {
printUsage(options);
return;
} else if (argsLine.hasOption("c") == false) {
LOG.error("Missing configuration file");
printUsage(options);
return;
}
int time = DEFAULT_TIME_SECONDS;
if (argsLine.hasOption("t")) {
time = Integer.parseInt(argsLine.getOptionValue("t"));
}
LOG.info("Experiment time is set to: " + time);
String outputDirectory = DEFAULT_DIRECTORY;
if (argsLine.hasOption("d")) {
outputDirectory = argsLine.getOptionValue("d");
}
LOG.info("Experiment output directory is set to: " + outputDirectory);
FileUtil.makeDirIfNotExists(outputDirectory);
// Parse controller configuration file
String configPath = argsLine.getOptionValue("c");
File configFile = new File(configPath);
// Check config format
if (!JSONSchemaType.isValidJson(JSONSchemaType.CONFIG, configFile)) {
LOG.error("Invalid configuration JSON format");
return;
}
// Load configuration file
ControllerConfiguration config = null;
try {
JSONObject input = new JSONObject(FileUtil.readFile(configFile));
config =
new ControllerConfiguration(
input.getString("database_type"),
input.getString("username"),
input.getString("password"),
input.getString("database_url"),
input.getString("upload_code"),
input.getString("upload_url"),
input.getString("workload_name"));
} catch (JSONException e) {
e.printStackTrace();
}
DBCollector collector = getCollector(config);
try {
// add a signal handler
Signal.handle(new Signal("INT"), signal -> firstCollecting = true);
File f = new File("pid.txt");
// get pid of this process and write the pid to a file before recording the start time
if (time < 0) {
String vmName = ManagementFactory.getRuntimeMXBean().getName();
int p = vmName.indexOf("@");
int pid = Integer.valueOf(vmName.substring(0, p));
try {
f.createNewFile();
PrintWriter pidWriter = new PrintWriter(f);
pidWriter.println(pid);
pidWriter.flush();
pidWriter.close();
} catch (IOException ioe) {
ioe.printStackTrace();
}
}
LOG.info("Output the process pid to pid.txt");
while (!firstCollecting) {
Thread.sleep(1);
}
// first collection (before queries)
LOG.info("First collection of metrics before experiment");
String metricsBefore = collector.collectMetrics();
if (!JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, metricsBefore)) {
LOG.error("Invalid output JSON format (metrics_before)");
return;
}
PrintWriter metricsWriter =
new PrintWriter(FileUtil.joinPath(outputDirectory, "metrics_before.json"), "UTF-8");
metricsWriter.println(metricsBefore);
metricsWriter.close();
String knobs = collector.collectParameters();
if (!JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, knobs)) {
LOG.error("Invalid output JSON format (knobs)");
return;
}
PrintWriter knobsWriter =
new PrintWriter(FileUtil.joinPath(outputDirectory, "knobs.json"), "UTF-8");
knobsWriter.println(knobs);
knobsWriter.close();
// add a signal handler
Signal.handle(new Signal("INT"), signal -> keepRunning = false);
// record start time
long startTime = System.currentTimeMillis();
LOG.info("Starting the experiment ...");
// go to sleep
if (time >= 0) {
Thread.sleep(time * TO_MILLISECONDS);
} else {
while (keepRunning) {
Thread.sleep(1);
}
f.delete();
}
long endTime = System.currentTimeMillis();
long observationTime = time >= 0 ? time : (endTime - startTime) / TO_MILLISECONDS;
LOG.info("Done running the experiment");
// summary json obj
JSONObject summary = null;
try {
summary = new JSONObject();
summary.put("start_time", startTime);
summary.put("end_time", endTime);
summary.put("observation_time", observationTime);
summary.put("database_type", config.getDBName());
summary.put("database_version", collector.collectVersion());
summary.put("workload_name", config.getWorkloadName());
} catch (JSONException e) {
e.printStackTrace();
}
if (!JSONSchemaType.isValidJson(JSONSchemaType.SUMMARY, summary.toString())) {
LOG.error("Invalid summary JSON format");
return;
}
// write summary JSONObject into a JSON file
PrintWriter summaryout =
new PrintWriter(FileUtil.joinPath(outputDirectory, "summary.json"), "UTF-8");
summaryout.println(JSONUtil.format(summary.toString()));
summaryout.close();
// second collection (after workload execution)
LOG.info("Second collection of metrics after experiment");
collector = getCollector(config);
String metricsAfter = collector.collectMetrics();
if (!JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, metricsAfter)) {
LOG.error("Invalid output JSON format (metrics_after)");
return;
}
PrintWriter metricsWriterFinal =
new PrintWriter(FileUtil.joinPath(outputDirectory, "metrics_after.json"), "UTF-8");
metricsWriterFinal.println(metricsAfter);
metricsWriterFinal.close();
} catch (FileNotFoundException | UnsupportedEncodingException | InterruptedException e) {
LOG.error("Failed to produce output files");
e.printStackTrace();
}
if (config.getUploadURL() != null && !config.getUploadURL().equals("")) {
Map<String, String> outfiles = new HashMap<>();
outfiles.put("knobs", FileUtil.joinPath(outputDirectory, "knobs.json"));
outfiles.put("metrics_before", FileUtil.joinPath(outputDirectory, "metrics_before.json"));
outfiles.put("metrics_after", FileUtil.joinPath(outputDirectory, "metrics_after.json"));
outfiles.put("summary", FileUtil.joinPath(outputDirectory, "summary.json"));
try {
ResultUploader.upload(config.getUploadURL(), config.getUploadCode(), outfiles);
} catch (IOException ioe) {
LOG.warn("Failed to upload results from the controller");
}
} else {
LOG.warn("Empty upload URL. Skipping upload...");
}
}
private static void printUsage(Options options) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("controller", options);
}
private static DBCollector getCollector(ControllerConfiguration config) {
DBCollector collector = null;
switch (config.getDBType()) {
case POSTGRES:
collector =
new PostgresCollector(
config.getDBURL(), config.getDBUsername(), config.getDBPassword());
break;
case MYSQL:
collector =
new MySQLCollector(config.getDBURL(), config.getDBUsername(), config.getDBPassword());
break;
case SAPHANA:
collector =
new SAPHanaCollector(config.getDBURL(), config.getDBUsername(), config.getDBPassword());
break;
case ORACLE:
collector =
new OracleCollector(config.getDBURL(), config.getDBUsername(), config.getDBPassword());
break;
default:
LOG.error("Invalid database type");
throw new RuntimeException("Invalid database type");
}
return collector;
}
}

62
client/controller/src/main/java/com/controller/ResultUploader.java Normal file
View File

@@ -0,0 +1,62 @@
/*
* OtterTune - ResultUploader.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.FileBody;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
/**
* Uploading the result.
*
* @author Shuli
*/
public class ResultUploader {
public static void upload(String uploadURL, String uploadCode,
Map<String, String> files) throws IOException {
try {
List<String> filesToSendNames = new ArrayList<>();
List<File> filesToSend = new ArrayList<>();
for (String fileName : files.keySet()) {
String path = files.get(fileName);
filesToSendNames.add(fileName);
File f = new File(path);
filesToSend.add(f);
}
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpPost httppost = new HttpPost(uploadURL);
MultipartEntityBuilder mb =
MultipartEntityBuilder.create().addTextBody("upload_code", uploadCode);
for (int i = 0; i < filesToSendNames.size(); i++) {
mb.addPart(filesToSendNames.get(i), new FileBody(filesToSend.get(i)));
}
HttpEntity reqEntity = mb.build();
httppost.setEntity(reqEntity);
CloseableHttpResponse response = httpclient.execute(httppost);
try {
HttpEntity resEntity = response.getEntity();
EntityUtils.consume(resEntity);
} finally {
response.close();
}
} catch (IOException e) {
throw new IOException();
}
}
}

51
client/controller/src/main/java/com/controller/collectors/DBCollector.java Normal file
View File

@@ -0,0 +1,51 @@
/*
* OtterTune - DBCollector.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
import com.controller.util.JSONUtil;
import java.util.Map;
import java.util.TreeMap;
import org.apache.log4j.Logger;
public class DBCollector implements DBParameterCollector {
private static final Logger LOG = Logger.getLogger(DBCollector.class);
protected static final String JSON_GLOBAL_KEY = "global";
protected static final String JSON_LOCAL_KEY = "local";
protected final Map<String, String> dbParameters = new TreeMap<String, String>();
protected final Map<String, String> dbMetrics = new TreeMap<String, String>();
protected final StringBuilder version = new StringBuilder();
@Override
public boolean hasParameters() {
return (dbParameters.isEmpty() == false);
}
@Override
public boolean hasMetrics() {
return (dbMetrics.isEmpty() == false);
}
@Override
public String collectParameters() {
return JSONUtil.format(JSONUtil.toJSONString(dbParameters));
}
@Override
public String collectMetrics() {
return JSONUtil.format(JSONUtil.toJSONString(dbMetrics));
}
@Override
public String collectVersion() {
return version.toString();
}
}

19
client/controller/src/main/java/com/controller/collectors/DBParameterCollector.java Normal file
View File

@@ -0,0 +1,19 @@
/*
* OtterTune - DBParameterCollector.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
public interface DBParameterCollector {
boolean hasParameters();
boolean hasMetrics();
String collectParameters();
String collectMetrics();
String collectVersion();
}

105
client/controller/src/main/java/com/controller/collectors/MySQLCollector.java Normal file
View File

@@ -0,0 +1,105 @@
/*
* OtterTune - MySQLCollector.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
import com.controller.util.JSONUtil;
import com.controller.util.json.JSONException;
import com.controller.util.json.JSONObject;
import com.controller.util.json.JSONStringer;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import org.apache.log4j.Logger;
/** */
public class MySQLCollector extends DBCollector {
private static final Logger LOG = Logger.getLogger(MySQLCollector.class);
private static final String VERSION_SQL = "SELECT @@GLOBAL.version;";
private static final String PARAMETERS_SQL = "SHOW VARIABLES;";
private static final String METRICS_SQL = "SHOW STATUS";
public MySQLCollector(String oriDBUrl, String username, String password) {
try {
Connection conn = DriverManager.getConnection(oriDBUrl, username, password);
Statement s = conn.createStatement();
// Collect DBMS version
ResultSet out = s.executeQuery(VERSION_SQL);
if (out.next()) {
this.version.append(out.getString(1));
}
// Collect DBMS parameters
out = s.executeQuery(PARAMETERS_SQL);
while (out.next()) {
dbParameters.put(out.getString(1).toLowerCase(), out.getString(2));
}
// Collect DBMS internal metrics
out = s.executeQuery(METRICS_SQL);
while (out.next()) {
dbMetrics.put(out.getString(1).toLowerCase(), out.getString(2));
}
conn.close();
} catch (SQLException e) {
LOG.error("Error while collecting DB parameters: " + e.getMessage());
e.printStackTrace();
}
}
@Override
public String collectParameters() {
JSONStringer stringer = new JSONStringer();
try {
stringer.object();
stringer.key(JSON_GLOBAL_KEY);
JSONObject jobLocal = new JSONObject();
JSONObject job = new JSONObject();
for (String k : dbParameters.keySet()) {
job.put(k, dbParameters.get(k));
}
// "global is a fake view_name (a placeholder)"
jobLocal.put("global", job);
stringer.value(jobLocal);
stringer.key(JSON_LOCAL_KEY);
stringer.value(null);
stringer.endObject();
} catch (JSONException jsonexn) {
jsonexn.printStackTrace();
}
return JSONUtil.format(stringer.toString());
}
@Override
public String collectMetrics() {
JSONStringer stringer = new JSONStringer();
try {
stringer.object();
stringer.key(JSON_GLOBAL_KEY);
JSONObject jobGlobal = new JSONObject();
JSONObject job = new JSONObject();
for (Map.Entry<String, String> entry : dbMetrics.entrySet()) {
job.put(entry.getKey(), entry.getValue());
}
// "global" is a a placeholder
jobGlobal.put("global", job);
stringer.value(jobGlobal);
stringer.key(JSON_LOCAL_KEY);
stringer.value(null);
stringer.endObject();
} catch (JSONException e) {
e.printStackTrace();
}
return JSONUtil.format(stringer.toString());
}
}

108
client/controller/src/main/java/com/controller/collectors/OracleCollector.java Normal file
View File

@@ -0,0 +1,108 @@
/*
* OtterTune - OracleCollector.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
import com.controller.util.JSONUtil;
import com.controller.util.json.JSONException;
import com.controller.util.json.JSONObject;
import com.controller.util.json.JSONStringer;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import org.apache.log4j.Logger;
/** */
public class OracleCollector extends DBCollector {
private static final Logger LOG = Logger.getLogger(MySQLCollector.class);
private static final String VERSION_SQL = "select VERSION from product_component_version";
private static final String PARAMETERS_SQL = "select name, value from v$parameter";
private static final String PARAMETERS_SQL_WITH_HIDDEN =
"select x.ksppinm name, y.ksppstvl value from sys.x$ksppi x, sys.x$ksppcv y where"
+ " x.inst_id = userenv('Instance') and y.inst_id = userenv('Instance') and x.indx = y.indx";
private static final String METRICS_SQL = "select name, value from v$sysstat";
public OracleCollector(String oriDBUrl, String username, String password) {
try {
Connection conn = DriverManager.getConnection(oriDBUrl, username, password);
Statement statement = conn.createStatement();
// Collect DBMS version
ResultSet out = statement.executeQuery(VERSION_SQL);
if (out.next()) {
this.version.append(out.getString(1));
}
// Collect DBMS parameters
out = statement.executeQuery(PARAMETERS_SQL_WITH_HIDDEN);
while (out.next()) {
dbParameters.put(out.getString(1).toLowerCase(), out.getString(2));
}
// Collect DBMS internal metrics
out = statement.executeQuery(METRICS_SQL);
while (out.next()) {
dbMetrics.put(out.getString(1).toLowerCase(), out.getString(2));
}
conn.close();
} catch (SQLException e) {
LOG.error("Error while collecting DB parameters: " + e.getMessage());
e.printStackTrace();
}
}
@Override
public String collectParameters() {
JSONStringer stringer = new JSONStringer();
try {
stringer.object();
stringer.key(JSON_GLOBAL_KEY);
JSONObject jobLocal = new JSONObject();
JSONObject job = new JSONObject();
for (Map.Entry<String, String> entry : dbParameters.entrySet()) {
job.put(entry.getKey(), entry.getValue());
}
// "global is a fake view_name (a placeholder)"
jobLocal.put("global", job);
stringer.value(jobLocal);
stringer.key(JSON_LOCAL_KEY);
stringer.value(null);
stringer.endObject();
} catch (JSONException jsonexn) {
jsonexn.printStackTrace();
}
return JSONUtil.format(stringer.toString());
}
@Override
public String collectMetrics() {
JSONStringer stringer = new JSONStringer();
try {
stringer.object();
stringer.key(JSON_GLOBAL_KEY);
JSONObject jobGlobal = new JSONObject();
JSONObject job = new JSONObject();
for (Map.Entry<String, String> entry : dbMetrics.entrySet()) {
job.put(entry.getKey(), entry.getValue());
}
// "global" is a placeholder
jobGlobal.put("global", job);
stringer.value(jobGlobal);
stringer.key(JSON_LOCAL_KEY);
stringer.value(null);
stringer.endObject();
} catch (JSONException e) {
e.printStackTrace();
}
return JSONUtil.format(stringer.toString());
}
}

237
client/controller/src/main/java/com/controller/collectors/PostgresCollector.java Normal file
View File

@@ -0,0 +1,237 @@
/*
* OtterTune - PostgresCollector.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
import com.controller.util.JSONUtil;
import com.controller.util.json.JSONException;
import com.controller.util.json.JSONObject;
import com.controller.util.json.JSONStringer;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.apache.log4j.Logger;
public class PostgresCollector extends DBCollector {
private static final Logger LOG = Logger.getLogger(PostgresCollector.class);
private static final String VERSION_SQL = "SELECT version();";
private static final String PARAMETERS_SQL = "SHOW ALL;";
private boolean oldVersion = false;
private static final String[] PG_STAT_VIEWS = {
"pg_stat_archiver", "pg_stat_bgwriter", "pg_stat_database",
"pg_stat_database_conflicts", "pg_stat_user_tables", "pg_statio_user_tables",
"pg_stat_user_indexes", "pg_statio_user_indexes"
};
private static final String[] PG_STAT_VIEWS_OLD_VERSION = {
"pg_stat_bgwriter", "pg_stat_database",
"pg_stat_database_conflicts", "pg_stat_user_tables", "pg_statio_user_tables",
"pg_stat_user_indexes", "pg_statio_user_indexes"
};
private static final String[] PG_STAT_VIEWS_LOCAL_DATABASE = {
"pg_stat_database", "pg_stat_database_conflicts"
};
private static final String PG_STAT_VIEWS_LOCAL_DATABASE_KEY = "datname";
private static final String[] PG_STAT_VIEWS_LOCAL_TABLE = {
"pg_stat_user_tables", "pg_statio_user_tables"
};
private static final String PG_STAT_VIEWS_LOCAL_TABLE_KEY = "relname";
private static final String[] PG_STAT_VIEWS_LOCAL_INDEXES = {
"pg_stat_user_indexes", "pg_statio_user_indexes"
};
private static final String PG_STAT_VIEWS_LOCAL_INDEXES_KEY = "relname";
private final Map<String, List<Map<String, String>>> pgMetrics;
public PostgresCollector(String oriDBUrl, String username, String password) {
pgMetrics = new HashMap<>();
try {
Connection conn = DriverManager.getConnection(oriDBUrl, username, password);
Statement s = conn.createStatement();
// Collect DBMS version
ResultSet out = s.executeQuery(VERSION_SQL);
if (out.next()) {
String[] outStr = out.getString(1).split(" ");
String[] verStr = outStr[1].split("\\.");
this.version.append(verStr[0]);
this.version.append(".");
this.version.append(verStr[1]);
}
// Collect DBMS parameters
out = s.executeQuery(PARAMETERS_SQL);
while (out.next()) {
dbParameters.put(out.getString("name"), out.getString("setting"));
}
// Collect DBMS internal metrics
String[] pgStatViews = PG_STAT_VIEWS;
if (Float.parseFloat(this.version.toString()) < 9.4) {
this.oldVersion = true;
pgStatViews = PG_STAT_VIEWS_OLD_VERSION;
}
for (String viewName : pgStatViews) {
out = s.executeQuery("SELECT * FROM " + viewName);
pgMetrics.put(viewName, getMetrics(out));
}
conn.close();
} catch (SQLException e) {
e.printStackTrace();
LOG.error("Error while collecting DB parameters: " + e.getMessage());
}
}
@Override
public boolean hasMetrics() {
return (pgMetrics.isEmpty() == false);
}
private JSONObject genMapJSONObj(Map<String, String> mapin) {
JSONObject res = new JSONObject();
try {
for (String key : mapin.keySet()) {
res.put(key, mapin.get(key));
}
} catch (JSONException je) {
LOG.error(je);
}
return res;
}
private JSONObject genLocalJSONObj(String viewName, String jsonKeyName) {
JSONObject thisViewObj = new JSONObject();
List<Map<String, String>> thisViewList = pgMetrics.get(viewName);
try {
for (Map<String, String> dbmap : thisViewList) {
String jsonkey = dbmap.get(jsonKeyName);
thisViewObj.put(jsonkey, genMapJSONObj(dbmap));
}
} catch (JSONException je) {
LOG.error(je);
}
return thisViewObj;
}
@Override
public String collectMetrics() {
JSONStringer stringer = new JSONStringer();
try {
stringer.object();
stringer.key(JSON_GLOBAL_KEY);
// create global objects for two views: "pg_stat_archiver" and "pg_stat_bgwriter"
JSONObject jobGlobal = new JSONObject();
// "pg_stat_archiver" (only one instance in the list) >= version 9.4
if (!this.oldVersion) {
Map<String, String> archiverList = pgMetrics.get("pg_stat_archiver").get(0);
jobGlobal.put("pg_stat_archiver", genMapJSONObj(archiverList));
}
// "pg_stat_bgwriter" (only one instance in the list)
Map<String, String> bgwriterList = pgMetrics.get("pg_stat_bgwriter").get(0);
jobGlobal.put("pg_stat_bgwriter", genMapJSONObj(bgwriterList));
// add global json object
stringer.value(jobGlobal);
stringer.key(JSON_LOCAL_KEY);
// create local objects for the rest of the views
JSONObject jobLocal = new JSONObject();
// "table"
JSONObject jobTable = new JSONObject();
for (int i = 0; i < PG_STAT_VIEWS_LOCAL_TABLE.length; i++) {
String viewName = PG_STAT_VIEWS_LOCAL_TABLE[i];
String jsonKeyName = PG_STAT_VIEWS_LOCAL_TABLE_KEY;
jobTable.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
}
jobLocal.put("table", jobTable);
// "database"
JSONObject jobDatabase = new JSONObject();
for (int i = 0; i < PG_STAT_VIEWS_LOCAL_DATABASE.length; i++) {
String viewName = PG_STAT_VIEWS_LOCAL_DATABASE[i];
String jsonKeyName = PG_STAT_VIEWS_LOCAL_DATABASE_KEY;
jobDatabase.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
}
jobLocal.put("database", jobDatabase);
// "indexes"
JSONObject jobIndexes = new JSONObject();
for (int i = 0; i < PG_STAT_VIEWS_LOCAL_INDEXES.length; i++) {
String viewName = PG_STAT_VIEWS_LOCAL_INDEXES[i];
String jsonKeyName = PG_STAT_VIEWS_LOCAL_INDEXES_KEY;
jobIndexes.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
}
jobLocal.put("indexes", jobIndexes);
// add local json object
stringer.value(jobLocal);
stringer.endObject();
} catch (JSONException jsonexn) {
jsonexn.printStackTrace();
}
return JSONUtil.format(stringer.toString());
}
private static List<Map<String, String>> getMetrics(ResultSet out) throws SQLException {
ResultSetMetaData metadata = out.getMetaData();
int numColumns = metadata.getColumnCount();
String[] columnNames = new String[numColumns];
for (int i = 0; i < numColumns; ++i) {
columnNames[i] = metadata.getColumnName(i + 1).toLowerCase();
}
List<Map<String, String>> metrics = new ArrayList<Map<String, String>>();
while (out.next()) {
Map<String, String> metricMap = new TreeMap<String, String>();
for (int i = 0; i < numColumns; ++i) {
metricMap.put(columnNames[i], out.getString(i + 1));
}
metrics.add(metricMap);
}
return metrics;
}
@Override
public String collectParameters() {
JSONStringer stringer = new JSONStringer();
try {
stringer.object();
stringer.key(JSON_GLOBAL_KEY);
JSONObject jobLocal = new JSONObject();
JSONObject job = new JSONObject();
for (String k : dbParameters.keySet()) {
job.put(k, dbParameters.get(k));
}
// "global is a fake view_name (a placeholder)"
jobLocal.put("global", job);
stringer.value(jobLocal);
stringer.key(JSON_LOCAL_KEY);
stringer.value(null);
stringer.endObject();
} catch (JSONException jsonexn) {
jsonexn.printStackTrace();
}
return JSONUtil.format(stringer.toString());
}
}

231
client/controller/src/main/java/com/controller/collectors/SAPHanaCollector.java Normal file
View File

@@ -0,0 +1,231 @@
/*
* OtterTune - SAPHanaCollector.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
import com.controller.util.JSONUtil;
import com.controller.util.json.JSONException;
import com.controller.util.json.JSONObject;
import com.controller.util.json.JSONStringer;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.apache.log4j.Logger;
public class SAPHanaCollector extends DBCollector {
private static final Logger LOG = Logger.getLogger(SAPHanaCollector.class);
private static final String VERSION_SQL = "SELECT VERSION from M_DATABASE";
private static final String PARAMETERS_SQL = "Select * from M_INIFILE_CONTENTS";
private static final String[] SAP_SYS_VIEW = {
"m_host_agent_metrics",
"m_caches",
"m_disk_usage",
"m_garbage_collection_statistics",
"m_host_resource_utilization",
"m_data_volumes"
};
private static final String[] SAP_SYS_LOCAL_VIEW = {"m_table_statistics", "m_rs_indexes"};
private static final String[] SAP_SYS_VIEW_GLOBAL = {
"m_host_agent_metrics",
"m_caches",
"m_disk_usage",
"m_garbage_collection_statistics",
"m_host_resource_utilization",
"m_data_volumes"
};
private static final String[] SAP_SYS_VIEW_GLOBAL_KEY = {
"instance_id", "cache_id", "usage_type", "store_type", "host", "volume_id"
};
private static final String[] SAP_SYS_VIEW_LOCAL_TABLE = {"m_table_statistics"};
private static final String[] SAP_SYS_VIEW_LOCAL_TABLE_KEY = {"table_name"};
private static final String[] SAP_SYS_VIEW_LOCAL_INDEXES = {"m_rs_indexes"};
private static final String[] SAP_SYS_VIEW_LOCAL_INDEXES_KEY = {"index_name"};
private final Map<String, List<Map<String, String>>> pgMetrics;
public SAPHanaCollector(String oriDBUrl, String username, String password) {
pgMetrics = new HashMap<>();
try {
Connection conn = DriverManager.getConnection(oriDBUrl, username, password);
Statement s = conn.createStatement();
// Collect DBMS version
ResultSet out = s.executeQuery(VERSION_SQL);
if (out.next()) {
this.version.append(out.getString(1));
}
// Collect DBMS parameters
out = s.executeQuery(PARAMETERS_SQL);
while (out.next()) {
dbParameters.put(
"("
+ out.getString("FILE_NAME")
+ ","
+ out.getString("LAYER_NAME")
+ ","
+ out.getString("TENANT_NAME")
+ ","
+ out.getString("HOST")
+ ","
+ out.getString("SECTION")
+ ","
+ out.getString("KEY")
+ ")",
out.getString("VALUE"));
}
// Collect DBMS internal metrics
for (String viewName : SAP_SYS_VIEW) {
out = s.executeQuery("SELECT * FROM " + viewName);
pgMetrics.put(viewName, getMetrics(out));
}
for (String viewName : SAP_SYS_LOCAL_VIEW) {
out = s.executeQuery("SELECT * FROM " + viewName + " where schema_name = 'SYSTEM' ");
pgMetrics.put(viewName, getMetrics(out));
}
conn.close();
} catch (SQLException e) {
e.printStackTrace();
LOG.error("Error while collecting DB parameters: " + e.getMessage());
}
}
@Override
public boolean hasMetrics() {
return (pgMetrics.isEmpty() == false);
}
private JSONObject genMapJSONObj(Map<String, String> mapin) {
JSONObject res = new JSONObject();
try {
for (String key : mapin.keySet()) {
res.put(key, mapin.get(key));
}
} catch (JSONException je) {
LOG.error(je);
}
return res;
}
private JSONObject genLocalJSONObj(String viewName, String jsonKeyName) {
JSONObject thisViewObj = new JSONObject();
List<Map<String, String>> thisViewList = pgMetrics.get(viewName);
try {
for (Map<String, String> dbmap : thisViewList) {
String jsonkey = dbmap.get(jsonKeyName);
thisViewObj.put(jsonkey, genMapJSONObj(dbmap));
}
} catch (JSONException je) {
LOG.error(je);
}
return thisViewObj;
}
@Override
public String collectMetrics() {
JSONStringer stringer = new JSONStringer();
try {
stringer.object();
stringer.key(JSON_GLOBAL_KEY);
JSONObject jobGlobal = new JSONObject();
JSONObject jobMetric = new JSONObject();
for (int i = 0; i < SAP_SYS_VIEW_GLOBAL.length; i++) {
String viewName = SAP_SYS_VIEW_GLOBAL[i];
String jsonKeyName = SAP_SYS_VIEW_GLOBAL_KEY[i];
jobGlobal.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
}
// add global json object
stringer.value(jobGlobal);
stringer.key(JSON_LOCAL_KEY);
// create local objects for the rest of the views
JSONObject jobLocal = new JSONObject();
// "table"
JSONObject jobTable = new JSONObject();
for (int i = 0; i < SAP_SYS_VIEW_LOCAL_TABLE.length; i++) {
String viewName = SAP_SYS_VIEW_LOCAL_TABLE[i];
String jsonKeyName = SAP_SYS_VIEW_LOCAL_TABLE_KEY[i];
jobTable.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
}
jobLocal.put("table", jobTable);
// "indexes"
JSONObject jobIndexes = new JSONObject();
for (int i = 0; i < SAP_SYS_VIEW_LOCAL_INDEXES.length; i++) {
String viewName = SAP_SYS_VIEW_LOCAL_INDEXES[i];
String jsonKeyName = SAP_SYS_VIEW_LOCAL_INDEXES_KEY[i];
jobIndexes.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
}
jobLocal.put("indexes", jobIndexes);
// add local json object
stringer.value(jobLocal);
stringer.endObject();
} catch (JSONException jsonexn) {
jsonexn.printStackTrace();
}
return JSONUtil.format(stringer.toString());
}
private static List<Map<String, String>> getMetrics(ResultSet out) throws SQLException {
ResultSetMetaData metadata = out.getMetaData();
int numColumns = metadata.getColumnCount();
String[] columnNames = new String[numColumns];
for (int i = 0; i < numColumns; ++i) {
columnNames[i] = metadata.getColumnName(i + 1).toLowerCase();
}
List<Map<String, String>> metrics = new ArrayList<Map<String, String>>();
while (out.next()) {
Map<String, String> metricMap = new TreeMap<String, String>();
for (int i = 0; i < numColumns; ++i) {
metricMap.put(columnNames[i], out.getString(i + 1));
}
metrics.add(metricMap);
}
return metrics;
}
@Override
public String collectParameters() {
JSONStringer stringer = new JSONStringer();
try {
stringer.object();
stringer.key(JSON_GLOBAL_KEY);
JSONObject jobLocal = new JSONObject();
JSONObject job = new JSONObject();
for (String k : dbParameters.keySet()) {
job.put(k, dbParameters.get(k));
}
// "global is a fake view_name (a placeholder)"
jobLocal.put("global", job);
stringer.value(jobLocal);
stringer.key(JSON_LOCAL_KEY);
stringer.value(null);
stringer.endObject();
} catch (JSONException jsonexn) {
jsonexn.printStackTrace();
}
return JSONUtil.format(stringer.toString());
}
}

38
client/controller/src/main/java/com/controller/json_validation_schema/config_schema.json Normal file
View File

@@ -0,0 +1,38 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "summary of collector output",
"description": "config files: user input",
"type": "object",
"properties": {
"database_type": {
"type" : "string"
},
"database_url": {
"type" : "string"
},
"username" :{
"type" : "string"
},
"password": {
"type" : "string"
},
"upload_code": {
"type" : "string"
},
"upload_url": {
"type" : "string"
},
"workload_name": {
"type" : "string"
}
},
"required": [
"database_type",
"database_url",
"username",
"password",
"upload_code",
"upload_url",
"workload_name"
]
}

47
client/controller/src/main/java/com/controller/json_validation_schema/schema.json Normal file
View File

@@ -0,0 +1,47 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "collector output",
"description": "Collected metrics or knobs info",
"type": "object",
"properties": {
"global" : {
"type": ["object", "null"],
"patternProperties" : {
"^[A-Za-z0-9 -_]" : {
"type" : "object",
"patternProperties" : {
"^[A-Za-z0-9 -_]" : {
"type" : "string"
}
},
"additionalProperties": false
}
},
"additionalProperties": false
},
"local" : {
"type": ["object", "null"],
"patternProperties" : {
"^[A-Za-z0-9 -_]" : {
"type" : "object",
"patternProperties" : {
"^[A-Za-z0-9 -_]" : {
"type" : "object",
"patternProperties" : {
"^[A-Za-z0-9 -_]" : {
"type" : "object",
"patternProperties" : {
"^[A-Za-z0-9 -_]" : {
"type" : "string"
}
}
}
}
}
}
}
}
}
},
"required": ["global", "local"]
}

30
client/controller/src/main/java/com/controller/json_validation_schema/summary_schema.json Normal file
View File

@@ -0,0 +1,30 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "summary of collector output",
"description": "output summary",
"type": "object",
"properties": {
"database_type": {
"type" : "string"
},
"start_time": {
"type" : "number"
},
"observation_time" :{
"type" : "number"
},
"end_time": {
"type" : "number"
},
"database_version": {
"type" : "string"
}
},
"required": [
"database_type",
"start_time",
"observation_time",
"end_time",
"database_version"
]
}

66
client/controller/src/main/java/com/controller/types/DatabaseType.java Normal file
View File

@@ -0,0 +1,66 @@
/*
* OtterTune - DatabaseType.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
/** Database Type. */
public enum DatabaseType {
/** Parameters: (1) JDBC Driver String */
MYSQL("com.mysql.jdbc.Driver"),
MYROCKS("com.mysql.jdbc.Driver"),
POSTGRES("org.postgresql.Driver"),
SAPHANA("com.sap.db.jdbc.Driver"),
ORACLE("oracle.jdbc.driver.OracleDriver");
private DatabaseType(String driver) {
this.driver = driver;
}
/**
* This is the suggested driver string to use in the configuration xml This corresponds to the
* <B>'driver'</b> attribute.
*/
private final String driver;
// ---------------------------------------------------------------
// ACCESSORS
// ----------------------------------------------------------------
/**
* Returns the suggested driver string to use for the given database type
*
* @return
*/
public String getSuggestedDriver() {
return (this.driver);
}
// ----------------------------------------------------------------
// STATIC METHODS + MEMBERS
// ----------------------------------------------------------------
protected static final Map<Integer, DatabaseType> idx_lookup =
new HashMap<Integer, DatabaseType>();
protected static final Map<String, DatabaseType> name_lookup =
new HashMap<String, DatabaseType>();
static {
for (DatabaseType vt : EnumSet.allOf(DatabaseType.class)) {
DatabaseType.idx_lookup.put(vt.ordinal(), vt);
DatabaseType.name_lookup.put(vt.name().toUpperCase(), vt);
}
}
public static DatabaseType get(String name) {
DatabaseType ret = DatabaseType.name_lookup.get(name.toUpperCase());
return (ret);
}
}

63
client/controller/src/main/java/com/controller/types/JSONSchemaType.java Normal file
View File

@@ -0,0 +1,63 @@
/*
* OtterTune - JSONSchemaType.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.types;
import com.controller.util.FileUtil;
import com.controller.util.ValidationUtils;
import com.fasterxml.jackson.databind.JsonNode;
import com.github.fge.jsonschema.core.exceptions.ProcessingException;
import com.github.fge.jsonschema.main.JsonSchema;
import java.io.File;
import java.io.IOException;
public enum JSONSchemaType {
/** Parameters: (1) schema filename */
OUTPUT("schema.json"),
CONFIG("config_schema.json"),
SUMMARY("summary_schema.json");
// Path to JSON schema directory
private static final String SCHEMA_PATH = "src/main/java/com/controller/json_validation_schema";
private final JsonSchema schema;
private JSONSchemaType(String fileName) {
JsonSchema newSchema = null;
String configPath = FileUtil.joinPath(SCHEMA_PATH, fileName);
try {
newSchema = ValidationUtils.getSchemaNode(new File(configPath));
} catch (IOException | ProcessingException e) {
e.printStackTrace();
}
this.schema = newSchema;
}
public JsonSchema getSchema() {
return this.schema;
}
public static boolean isValidJson(JSONSchemaType schemaType, String jsonString) {
try {
JsonNode jsonNode = ValidationUtils.getJsonNode(jsonString);
return ValidationUtils.isJsonValid(schemaType.getSchema(), jsonNode);
} catch (IOException | ProcessingException e) {
e.printStackTrace();
}
return false;
}
public static boolean isValidJson(JSONSchemaType schemaType, File jsonFile) {
try {
JsonNode jsonNode = ValidationUtils.getJsonNode(jsonFile);
return ValidationUtils.isJsonValid(schemaType.getSchema(), jsonNode);
} catch (IOException | ProcessingException e) {
e.printStackTrace();
}
return false;
}
}

297
client/controller/src/main/java/com/controller/util/ClassUtil.java Normal file
View File

@@ -0,0 +1,297 @@
/*
* OtterTune - ClassUtil.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.util;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.collections15.CollectionUtils;
import org.apache.commons.lang.ClassUtils;
import org.apache.log4j.Logger;
/** @author pavlo */
public abstract class ClassUtil {
private static final Logger LOG = Logger.getLogger(ClassUtil.class);
private static final Class<?>[] EMPTY_ARRAY = new Class[] {};
private static final Map<Class<?>, List<Class<?>>> CACHE_getSuperClasses =
new HashMap<Class<?>, List<Class<?>>>();
private static final Map<Class<?>, Set<Class<?>>> CACHE_getInterfaceClasses =
new HashMap<Class<?>, Set<Class<?>>>();
/**
* Check if the given object is an array (primitve or native).
* http://www.java2s.com/Code/Java/Reflection/Checkifthegivenobjectisanarrayprimitveornative.htm
*
* @param obj Object to test.
* @return True of the object is an array.
*/
public static boolean isArray(final Object obj) {
return (obj != null ? obj.getClass().isArray() : false);
}
public static boolean[] isArray(final Object[] objs) {
boolean[] isArray = new boolean[objs.length];
for (int i = 0; i < objs.length; i++) {
isArray[i] = ClassUtil.isArray(objs[i]);
} // FOR
return (isArray);
}
/**
* Convert a Enum array to a Field array This assumes that the name of each Enum element
* corresponds to a data member in the clas
*
* @param <E>
* @param clazz
* @param members
* @return
* @throws NoSuchFieldException
*/
public static <E extends Enum<?>> Field[] getFieldsFromMembersEnum(Class<?> clazz, E[] members)
throws NoSuchFieldException {
Field[] fields = new Field[members.length];
for (int i = 0; i < members.length; i++) {
fields[i] = clazz.getDeclaredField(members[i].name().toLowerCase());
} // FOR
return (fields);
}
/**
* Get the generic types for the given field
*
* @param field
* @return
*/
public static List<Class<?>> getGenericTypes(Field field) {
ArrayList<Class<?>> genericClasses = new ArrayList<Class<?>>();
Type gtype = field.getGenericType();
if (gtype instanceof ParameterizedType) {
ParameterizedType ptype = (ParameterizedType) gtype;
getGenericTypesImpl(ptype, genericClasses);
}
return (genericClasses);
}
private static void getGenericTypesImpl(ParameterizedType ptype, List<Class<?>> classes) {
// list the actual type arguments
for (Type t : ptype.getActualTypeArguments()) {
if (t instanceof Class) {
// System.err.println("C: " + t);
classes.add((Class<?>) t);
} else if (t instanceof ParameterizedType) {
ParameterizedType next = (ParameterizedType) t;
// System.err.println("PT: " + next);
classes.add((Class<?>) next.getRawType());
getGenericTypesImpl(next, classes);
}
} // FOR
return;
}
/**
* Return an ordered list of all the sub-classes for a given class Useful when dealing with
* generics
*
* @param elementClass
* @return
*/
public static List<Class<?>> getSuperClasses(Class<?> elementClass) {
List<Class<?>> ret = ClassUtil.CACHE_getSuperClasses.get(elementClass);
if (ret == null) {
ret = new ArrayList<Class<?>>();
while (elementClass != null) {
ret.add(elementClass);
elementClass = elementClass.getSuperclass();
} // WHILE
ret = Collections.unmodifiableList(ret);
ClassUtil.CACHE_getSuperClasses.put(elementClass, ret);
}
return (ret);
}
/**
* Get a set of all of the interfaces that the element_class implements
*
* @param elementClass
* @return
*/
@SuppressWarnings("unchecked")
public static Collection<Class<?>> getInterfaces(Class<?> elementClass) {
Set<Class<?>> ret = ClassUtil.CACHE_getInterfaceClasses.get(elementClass);
if (ret == null) {
// ret = new HashSet<Class<?>>();
// Queue<Class<?>> queue = new LinkedList<Class<?>>();
// queue.add(element_class);
// while (!queue.isEmpty()) {
// Class<?> current = queue.poll();
// for (Class<?> i : current.getInterfaces()) {
// ret.add(i);
// queue.add(i);
// } // FOR
// } // WHILE
ret = new HashSet<Class<?>>(ClassUtils.getAllInterfaces(elementClass));
if (elementClass.isInterface()) {
ret.add(elementClass);
}
ret = Collections.unmodifiableSet(ret);
ClassUtil.CACHE_getInterfaceClasses.put(elementClass, ret);
}
return (ret);
}
@SuppressWarnings("unchecked")
public static <T> T newInstance(String className, Object[] params, Class<?>[] classes) {
return ((T) ClassUtil.newInstance(ClassUtil.getClass(className), params, classes));
}
public static <T> T newInstance(Class<T> targetClass, Object[] params, Class<?>[] classes) {
// Class<?> const_params[] = new Class<?>[params.length];
// for (int i = 0; i < params.length; i++) {
// const_params[i] = params[i].getClass();
// System.err.println("[" + i + "] " + params[i] + " " + params[i].getClass());
// } // FOR
Constructor<T> constructor = ClassUtil.getConstructor(targetClass, classes);
T ret = null;
try {
ret = constructor.newInstance(params);
} catch (Exception ex) {
throw new RuntimeException(
"Failed to create new instance of " + targetClass.getSimpleName(), ex);
}
return (ret);
}
/**
* Create an object for the given class and initialize it from conf
*
* @param theClass class of which an object is created
* @param expected the expected parent class or interface
* @return a new object
*/
public static <T> T newInstance(Class<?> theClass, Class<T> expected) {
T result;
try {
if (!expected.isAssignableFrom(theClass)) {
throw new Exception(
"Specified class "
+ theClass.getName()
+ ""
+ "does not extend/implement "
+ expected.getName());
}
Class<? extends T> clazz = (Class<? extends T>) theClass;
Constructor<? extends T> meth = clazz.getDeclaredConstructor(EMPTY_ARRAY);
meth.setAccessible(true);
result = meth.newInstance();
} catch (Exception e) {
throw new RuntimeException(e);
}
return result;
}
public static <T> T newInstance(String className, Class<T> expected)
throws ClassNotFoundException {
return newInstance(getClass(className), expected);
}
/**
* @param <T>
* @param targetClass
* @param params
* @return
*/
@SuppressWarnings("unchecked")
public static <T> Constructor<T> getConstructor(Class<T> targetClass, Class<?>... params) {
NoSuchMethodException error = null;
try {
return (targetClass.getConstructor(params));
} catch (NoSuchMethodException ex) {
// The first time we get this it can be ignored
// We'll try to be nice and find a match for them
error = ex;
}
assert (error != null);
if (LOG.isDebugEnabled()) {
LOG.debug("TARGET CLASS: " + targetClass);
LOG.debug("TARGET PARAMS: " + Arrays.toString(params));
}
List<Class<?>>[] paramSuper = (List<Class<?>>[]) new List[params.length];
for (int i = 0; i < params.length; i++) {
paramSuper[i] = ClassUtil.getSuperClasses(params[i]);
if (LOG.isDebugEnabled()) {
LOG.debug(" SUPER[" + params[i].getSimpleName() + "] => " + paramSuper[i]);
}
} // FOR
for (Constructor<?> c : targetClass.getConstructors()) {
Class<?>[] ctypes = c.getParameterTypes();
if (LOG.isDebugEnabled()) {
LOG.debug("CANDIDATE: " + c);
LOG.debug("CANDIDATE PARAMS: " + Arrays.toString(ctypes));
}
if (params.length != ctypes.length) {
continue;
}
for (int i = 0; i < params.length; i++) {
List<Class<?>> csuper = ClassUtil.getSuperClasses(ctypes[i]);
if (LOG.isDebugEnabled()) {
LOG.debug(" SUPER[" + ctypes[i].getSimpleName() + "] => " + csuper);
}
if (CollectionUtils.intersection(paramSuper[i], csuper).isEmpty() == false) {
return ((Constructor<T>) c);
}
} // FOR (param)
} // FOR (constructors)
throw new RuntimeException(
"Failed to retrieve constructor for " + targetClass.getSimpleName(), error);
}
/**
* @param className
* @return
*/
public static Class<?> getClass(String className) {
Class<?> targetClass = null;
try {
ClassLoader loader = ClassLoader.getSystemClassLoader();
targetClass = (Class<?>) loader.loadClass(className);
} catch (Exception ex) {
throw new RuntimeException("Failed to retrieve class for " + className, ex);
}
return (targetClass);
}
/**
* Returns true if asserts are enabled. This assumes that we're always using the default system
* ClassLoader
*/
public static boolean isAssertsEnabled() {
boolean ret = false;
try {
assert (false);
} catch (AssertionError ex) {
ret = true;
}
return (ret);
}
}

491
client/controller/src/main/java/com/controller/util/CollectionUtil.java Normal file
View File

@@ -0,0 +1,491 @@
/*
* OtterTune - CollectionUtil.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.util;
import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
import java.util.Random;
import java.util.Set;
import java.util.SortedMap;
import org.apache.commons.collections15.set.ListOrderedSet;
import org.apache.commons.lang.NotImplementedException;
/** @author pavlo */
public abstract class CollectionUtil {
private static final Random RANDOM = new Random();
/**
* Put all of the elements in items into the given array This assumes that the array has been
* pre-allocated
*
* @param <T>
* @param items
* @param array
*/
public static <T> void toArray(
Collection<T> items, Object[] array, boolean convertToPrimitive) {
assert (items.size() == array.length);
int i = 0;
for (T t : items) {
if (convertToPrimitive) {
if (t instanceof Long) {
array[i] = ((Long) t).longValue();
} else if (t instanceof Integer) {
array[i] = ((Integer) t).intValue();
} else if (t instanceof Double) {
array[i] = ((Double) t).doubleValue();
} else if (t instanceof Boolean) {
array[i] = ((Boolean) t).booleanValue();
}
} else {
array[i] = t;
}
}
return;
}
public static int[] toIntArray(Collection<Integer> items) {
int[] ret = new int[items.size()];
int idx = 0;
for (Integer i : items) {
assert (i != null);
ret[idx++] = i.intValue();
} // FOR
return (ret);
}
/**
* Put all the values of an Iterator into a List
*
* @param <T>
* @param it
* @return
*/
public static <T> List<T> list(Iterator<T> it) {
List<T> list = new ArrayList<T>();
CollectionUtil.addAll(list, it);
return (list);
}
/**
* Put all of the values of an Enumeration into a new List
*
* @param <T>
* @param e
* @return
*/
public static <T> List<T> list(Enumeration<T> e) {
return (list(iterable(e)));
}
/**
* Put all of the values of an Iterable into a new List
*
* @param <T>
* @param it
* @return
*/
public static <T> List<T> list(Iterable<T> it) {
return (list(it.iterator()));
}
/**
* Put all the values of an Iterator into a Set
*
* @param <T>
* @param it
* @return
*/
public static <T> Set<T> set(Iterator<T> it) {
Set<T> set = new HashSet<T>();
CollectionUtil.addAll(set, it);
return (set);
}
/**
* Put all of the values of an Iterable into a new Set
*
* @param <T>
* @param it
* @return
*/
public static <T> Set<T> set(Iterable<T> it) {
return (set(it.iterator()));
}
/**
* Returns a list containing the string representations of the elements in the collection
*
* @param <T>
* @param data
* @return
*/
public static <T> List<String> toStringList(Collection<T> data) {
List<String> ret = new ArrayList<String>();
for (T t : data) {
ret.add(t.toString());
}
return (ret);
}
/**
* Returns a set containing the string representations of the elements in the collection
*
* @param <T>
* @param data
* @return
*/
public static <T> Set<String> toStringSet(Collection<T> data) {
Set<String> ret = new HashSet<String>();
for (T t : data) {
ret.add(t.toString());
}
return (ret);
}
/**
* Return a random value from the given Collection
*
* @param <T>
* @param items
*/
public static <T> T random(Collection<T> items) {
return (CollectionUtil.random(items, RANDOM));
}
/**
* Return a random value from the given Collection
*
* @param <T>
* @param items
* @param rand
* @return
*/
public static <T> T random(Collection<T> items, Random rand) {
int idx = rand.nextInt(items.size());
return (CollectionUtil.get(items, idx));
}
/**
* Return a random value from the given Iterable
*
* @param <T>
* @param it
* @return
*/
public static <T> T random(Iterable<T> it) {
return (CollectionUtil.random(it, RANDOM));
}
/**
* Return a random value from the given Iterable
*
* @param <T>
* @param it
* @param rand
* @return
*/
public static <T> T random(Iterable<T> it, Random rand) {
List<T> list = new ArrayList<T>();
for (T t : it) {
list.add(t);
} // FOR
return (CollectionUtil.random(list, rand));
}
public static <E extends Enum<?>> Set<E> getAllExcluding(E[] elements, E... excluding) {
Set<E> excludeSet = new HashSet<E>();
for (E e : excluding) {
excludeSet.add(e);
}
Set<E> elementsSet = new HashSet<E>();
for (int i = 0; i < elements.length; i++) {
if (!excludeSet.contains(elements[i])) {
elementsSet.add(elements[i]);
}
} // FOR
return (elementsSet);
// Crappy java....
// Object new_elements[] = new Object[elements_set.size()];
// elements_set.toArray(new_elements);
// return ((E[])new_elements);
}
/**
* Add all the items in the array to a Collection
*
* @param <T>
* @param data
* @param items
*/
public static <T> Collection<T> addAll(Collection<T> data, T... items) {
for (T i : (T[]) items) {
data.add(i);
}
return (data);
}
/**
* Add all the items in the Enumeration into a Collection
*
* @param <T>
* @param data
* @param items
*/
public static <T> Collection<T> addAll(Collection<T> data, Enumeration<T> items) {
while (items.hasMoreElements()) {
data.add(items.nextElement());
} // WHILE
return (data);
}
/**
* Add all of the items from the Iterable into the given collection
*
* @param <T>
* @param data
* @param items
*/
public static <T> Collection<T> addAll(Collection<T> data, Iterable<T> items) {
return (CollectionUtil.addAll(data, items.iterator()));
}
/**
* Add all of the items from the Iterator into the given collection
*
* @param <T>
* @param data
* @param items
*/
public static <T> Collection<T> addAll(Collection<T> data, Iterator<T> items) {
while (items.hasNext()) {
data.add(items.next());
} // WHILE
return (data);
}
/**
* @param <T>
* @param <U>
* @param map
* @return
*/
public static <T, U extends Comparable<U>> T getGreatest(Map<T, U> map) {
T maxKey = null;
U maxValue = null;
for (Entry<T, U> e : map.entrySet()) {
T key = e.getKey();
U value = e.getValue();
if (maxValue == null || value.compareTo(maxValue) > 0) {
maxValue = value;
maxKey = key;
}
} // FOR
return (maxKey);
}
/**
* Return the first item in a Iterable
*
* @param <T>
* @param items
* @return
*/
public static <T> T first(Iterable<T> items) {
return (CollectionUtil.get(items, 0));
}
/**
* Return the first item in a Iterator
*
* @param <T>
* @param items
* @return
*/
public static <T> T first(Iterator<T> items) {
return (items.hasNext() ? items.next() : null);
}
/**
* Returns the first item in an Enumeration
*
* @param <T>
* @param items
* @return
*/
public static <T> T first(Enumeration<T> items) {
return (items.hasMoreElements() ? items.nextElement() : null);
}
/**
* Return the ith element of a set. Super lame
*
* @param <T>
* @param items
* @param idx
* @return
*/
public static <T> T get(Iterable<T> items, int idx) {
if (items instanceof AbstractList<?>) {
return ((AbstractList<T>) items).get(idx);
} else if (items instanceof ListOrderedSet<?>) {
return ((ListOrderedSet<T>) items).get(idx);
}
int ctr = 0;
for (T t : items) {
if (ctr++ == idx) {
return (t);
}
}
return (null);
}
/**
* Return the last item in an Iterable
*
* @param <T>
* @param items
* @return
*/
public static <T> T last(Iterable<T> items) {
T last = null;
if (items instanceof AbstractList<?>) {
AbstractList<T> list = (AbstractList<T>) items;
last = (list.isEmpty() ? null : list.get(list.size() - 1));
} else {
for (T t : items) {
last = t;
}
}
return (last);
}
/**
* Return the last item in an array
*
* @param <T>
* @param items
* @return
*/
public static <T> T last(T... items) {
if (items != null && items.length > 0) {
return (items[items.length - 1]);
}
return (null);
}
/**
* @param <K>
* @param <V>
* @param map
* @return
*/
public static <K extends Comparable<?>, V> List<V> getSortedList(
SortedMap<K, Collection<V>> map) {
List<V> ret = new ArrayList<V>();
for (Collection<V> col : map.values()) {
ret.addAll(col);
} // FOR
return (ret);
}
/**
* Wrap an Iterable around an Iterator
*
* @param <T>
* @param it
* @return
*/
public static <T> Iterable<T> iterable(final Iterator<T> it) {
return (new Iterable<T>() {
@Override
public Iterator<T> iterator() {
return (it);
}
});
}
public static <T> Iterable<T> iterable(final T[] values) {
return (new Iterable<T>() {
@Override
public Iterator<T> iterator() {
return new Iterator<T>() {
private int idx = 0;
@Override
public boolean hasNext() {
return (this.idx < values.length);
}
@Override
public T next() {
if (this.idx == values.length) {
throw new NoSuchElementException();
}
return values[this.idx++];
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
});
}
/**
* Wrap an Iterable around an Enumeration
*
* @param <T>
* @param e
* @return
*/
public static <T> Iterable<T> iterable(final Enumeration<T> e) {
return (new Iterable<T>() {
@Override
public Iterator<T> iterator() {
return new Iterator<T>() {
@Override
public boolean hasNext() {
return (e.hasMoreElements());
}
@Override
public T next() {
return (e.nextElement());
}
@Override
public void remove() {
throw new NotImplementedException();
}
};
}
});
}
public static <T> T pop(Collection<T> items) {
T t = CollectionUtil.first(items);
if (t != null) {
boolean ret = items.remove(t);
assert (ret);
}
return (t);
}
}

383
client/controller/src/main/java/com/controller/util/FileUtil.java Normal file
View File

@@ -0,0 +1,383 @@
/*
* OtterTune - FileUtil.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.util;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import org.apache.log4j.Logger;
/** @author pavlo */
public abstract class FileUtil {
private static final Logger LOG = Logger.getLogger(FileUtil.class);
private static final Pattern EXT_SPLIT = Pattern.compile("\\.");
/**
* Join path components
*
* @param args
* @return
*/
public static String joinPath(String... args) {
StringBuilder result = new StringBuilder();
boolean first = true;
for (String a : args) {
if (a != null && a.length() > 0) {
if (!first) {
result.append("/");
}
result.append(a);
first = false;
}
}
return result.toString();
}
/**
* Given a basename for a file, find the next possible filename if this file already exists. For
* example, if the file test.res already exists, create a file called, test.1.res
*
* @param basename
* @return
*/
public static String getNextFilename(String basename) {
if (!exists(basename)) {
return basename;
}
File f = new File(basename);
if (f != null && f.isFile()) {
String[] parts = EXT_SPLIT.split(basename);
// Check how many files already exist
int counter = 1;
String nextName = parts[0] + "." + counter + "." + parts[1];
while (exists(nextName)) {
++counter;
nextName = parts[0] + "." + counter + "." + parts[1];
}
return nextName;
}
// Should we throw instead??
return null;
}
public static boolean exists(String path) {
return (new File(path).exists());
}
public static String realpath(String path) {
File f = new File(path);
String ret = null;
try {
ret = f.getCanonicalPath();
} catch (Exception ex) {
LOG.warn(ex);
}
return (ret);
}
public static String basename(String path) {
return (new File(path)).getName();
}
public static String getExtension(File f) {
if (f != null && f.isFile()) {
String[] parts = EXT_SPLIT.split(f.getName());
if (parts.length > 1) {
return (parts[parts.length - 1]);
}
}
return (null);
}
/**
* Create any directory in the list paths if it doesn't exist
*
* @param paths
*/
public static void makeDirIfNotExists(String... paths) {
for (String p : paths) {
if (p == null) {
continue;
}
File f = new File(p);
if (f.exists() == false) {
f.mkdirs();
}
} // FOR
}
/**
* Return a File handle to a temporary file location
*
* @param ext the suffix of the filename
* @param deleteOnExit whether to delete this file after the JVM exits
* @return
*/
public static File getTempFile(String ext, boolean deleteOnExit) {
return getTempFile(null, ext, deleteOnExit);
}
public static File getTempFile(String ext) {
return (FileUtil.getTempFile(null, ext, false));
}
public static File getTempFile(String prefix, String suffix, boolean deleteOnExit) {
File tempFile;
if (suffix != null && suffix.startsWith(".") == false) {
suffix = "." + suffix;
}
if (prefix == null) {
prefix = "hstore";
}
try {
tempFile = File.createTempFile(prefix, suffix);
if (deleteOnExit) {
tempFile.deleteOnExit();
}
} catch (Exception ex) {
throw new RuntimeException(ex);
}
return (tempFile);
}
/**
* Unsafely create a temporary directory Yes I said that this was unsafe. I don't care...
*
* @return
*/
public static File getTempDirectory() {
final File temp = FileUtil.getTempFile(null);
if (!(temp.delete())) {
throw new RuntimeException("Could not delete temp file: " + temp.getAbsolutePath());
} else if (!(temp.mkdir())) {
throw new RuntimeException("Could not create temp directory: " + temp.getAbsolutePath());
}
return (temp);
}
public static File writeStringToFile(String filePath, String content) throws IOException {
return (FileUtil.writeStringToFile(new File(filePath), content));
}
public static File writeStringToFile(File file, String content) throws IOException {
FileWriter writer = new FileWriter(file);
writer.write(content);
writer.flush();
writer.close();
return (file);
}
/**
* Write the given string to a temporary file Will not delete the file after the JVM exits
*
* @param content
* @return
*/
public static File writeStringToTempFile(String content) {
return (writeStringToTempFile(content, "tmp", false));
}
/**
* Write the given string to a temporary file with the given extension as the suffix Will not
* delete the file after the JVM exits
*
* @param content
* @param ext
* @return
*/
public static File writeStringToTempFile(String content, String ext) {
return (writeStringToTempFile(content, ext, false));
}
/**
* Write the given string to a temporary file with the given extension as the suffix If
* deleteOnExit is true, then the file will be removed when the JVM exits
*
* @param content
* @param ext
* @param deleteOnExit
* @return
*/
public static File writeStringToTempFile(String content, String ext, boolean deleteOnExit) {
File tempFile = FileUtil.getTempFile(ext, deleteOnExit);
try {
FileUtil.writeStringToFile(tempFile, content);
} catch (Exception e) {
throw new RuntimeException(e);
}
return tempFile;
}
public static String readFile(File path) {
LOG.debug(path);
return (readFile(path.getAbsolutePath()));
}
public static String readFile(String path) {
StringBuilder buffer = new StringBuilder();
try {
BufferedReader in = FileUtil.getReader(path);
while (in.ready()) {
buffer.append(in.readLine()).append("\n");
} // WHILE
in.close();
} catch (IOException ex) {
throw new RuntimeException("Failed to read file contents from '" + path + "'", ex);
}
return (buffer.toString());
}
/**
* Creates a BufferedReader for the given input path Can handle both gzip and plain text files
*
* @param path
* @return
* @throws IOException
*/
public static BufferedReader getReader(String path) throws IOException {
return (FileUtil.getReader(new File(path)));
}
/**
* Creates a BufferedReader for the given input path Can handle both gzip and plain text files
*
* @param file
* @return
* @throws IOException
*/
public static BufferedReader getReader(File file) throws IOException {
if (!file.exists()) {
throw new IOException("The file '" + file + "' does not exist");
}
BufferedReader in = null;
if (file.getPath().endsWith(".gz")) {
FileInputStream fin = new FileInputStream(file);
GZIPInputStream gzis = new GZIPInputStream(fin);
in = new BufferedReader(new InputStreamReader(gzis));
LOG.debug("Reading in the zipped contents of '" + file.getName() + "'");
} else {
in = new BufferedReader(new FileReader(file));
LOG.debug("Reading in the contents of '" + file.getName() + "'");
}
return (in);
}
public static byte[] readBytesFromFile(String path) throws IOException {
File file = new File(path);
FileInputStream in = new FileInputStream(file);
// Create the byte array to hold the data
long length = file.length();
byte[] bytes = new byte[(int) length];
LOG.debug("Reading in the contents of '" + file.getAbsolutePath() + "'");
// Read in the bytes
int offset = 0;
int numRead = 0;
while ((offset < bytes.length)
&& ((numRead = in.read(bytes, offset, bytes.length - offset)) >= 0)) {
offset += numRead;
} // WHILE
if (offset < bytes.length) {
throw new IOException("Failed to read the entire contents of '" + file.getName() + "'");
}
in.close();
return (bytes);
}
/**
* Find the path to a directory below our current location in the source tree Throws a
* RuntimeException if we go beyond our repository checkout
*
* @param dirName
* @return
* @throws IOException
*/
public static File findDirectory(String dirName) throws IOException {
return (FileUtil.find(dirName, new File(".").getCanonicalFile(), true).getCanonicalFile());
}
/**
* Find the path to a directory below our current location in the source tree Throws a
* RuntimeException if we go beyond our repository checkout
*
* @param dirName
* @return
* @throws IOException
*/
public static File findFile(String fileName) throws IOException {
return (FileUtil.find(fileName, new File(".").getCanonicalFile(), false).getCanonicalFile());
}
private static final File find(String name, File current, boolean isdir) throws IOException {
LOG.debug("Find Current Location = " + current);
boolean hasSvn = false;
for (File file : current.listFiles()) {
if (file.getCanonicalPath().endsWith(File.separator + name) && file.isDirectory() == isdir) {
return (file);
// Make sure that we don't go to far down...
} else if (file.getCanonicalPath().endsWith(File.separator + ".svn")) {
hasSvn = true;
}
} // FOR
// If we didn't see an .svn directory, then we went too far down
if (!hasSvn) {
throw new RuntimeException(
"Unable to find directory '" + name + "' [last_dir=" + current.getAbsolutePath() + "]");
}
File next = new File(current.getCanonicalPath() + File.separator + "");
return (FileUtil.find(name, next, isdir));
}
/**
* Returns a list of all the files in a directory whose name starts with the provided prefix
*
* @param dir
* @param filePrefix
* @return
* @throws IOException
*/
public static List<File> getFilesInDirectory(final File dir, final String filePrefix)
throws IOException {
assert (dir.isDirectory()) : "Invalid search directory path: " + dir;
FilenameFilter filter =
new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return (name.startsWith(filePrefix));
}
};
return (Arrays.asList(dir.listFiles(filter)));
}
public static String getAbsPath(String dirPath) {
String current = null;
try {
current = new java.io.File(dirPath).getCanonicalPath();
} catch (IOException e) {
e.printStackTrace();
}
return current;
}
}

23
client/controller/src/main/java/com/controller/util/JSONSerializable.java Normal file
View File

@@ -0,0 +1,23 @@
/*
* OtterTune - JSONSerializable.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.util;
import com.controller.util.json.JSONException;
import com.controller.util.json.JSONObject;
import com.controller.util.json.JSONString;
import com.controller.util.json.JSONStringer;
import java.io.IOException;
public interface JSONSerializable extends JSONString {
public void save(String outputPath) throws IOException;
public void load(String inputPath) throws IOException;
public void toJSON(JSONStringer stringer) throws JSONException;
public void fromJSON(JSONObject jsonObject) throws JSONException;
}

800
client/controller/src/main/java/com/controller/util/JSONUtil.java Normal file
View File

@@ -0,0 +1,800 @@
/*
* OtterTune - JSONUtil.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.util;
import com.controller.util.json.JSONArray;
import com.controller.util.json.JSONException;
import com.controller.util.json.JSONObject;
import com.controller.util.json.JSONStringer;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.Stack;
import org.apache.log4j.Logger;
/** @author pavlo */
public abstract class JSONUtil {
private static final Logger LOG = Logger.getLogger(JSONUtil.class.getName());
private static final String JSON_CLASS_SUFFIX = "_class";
private static final Map<Class<?>, Field[]> SERIALIZABLE_FIELDS =
new HashMap<Class<?>, Field[]>();
/**
* @param clazz
* @return
*/
public static Field[] getSerializableFields(Class<?> clazz, String... fieldsToExclude) {
Field[] ret = SERIALIZABLE_FIELDS.get(clazz);
if (ret == null) {
Collection<String> exclude = CollectionUtil.addAll(new HashSet<String>(), fieldsToExclude);
synchronized (SERIALIZABLE_FIELDS) {
ret = SERIALIZABLE_FIELDS.get(clazz);
if (ret == null) {
List<Field> fields = new ArrayList<Field>();
for (Field f : clazz.getFields()) {
int modifiers = f.getModifiers();
if (Modifier.isTransient(modifiers) == false
&& Modifier.isPublic(modifiers) == true
&& Modifier.isStatic(modifiers) == false
&& exclude.contains(f.getName()) == false) {
fields.add(f);
}
} // FOR
ret = fields.toArray(new Field[0]);
SERIALIZABLE_FIELDS.put(clazz, ret);
}
} // SYNCH
}
return (ret);
}
/**
* JSON Pretty Print
*
* @param json
* @return
* @throws JSONException
*/
public static String format(String json) {
try {
return (JSONUtil.format(new JSONObject(json)));
} catch (RuntimeException ex) {
throw ex;
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
/**
* JSON Pretty Print
*
* @param <T>
* @param object
* @return
*/
public static <T extends JSONSerializable> String format(T object) {
JSONStringer stringer = new JSONStringer();
try {
if (object instanceof JSONObject) {
return ((JSONObject) object).toString(2);
}
stringer.object();
object.toJSON(stringer);
stringer.endObject();
} catch (JSONException ex) {
throw new RuntimeException(ex);
}
return (JSONUtil.format(stringer.toString()));
}
public static String format(JSONObject o) {
try {
return o.toString(1);
} catch (JSONException ex) {
throw new RuntimeException(ex);
}
}
/**
* @param <T>
* @param object
* @return
*/
public static String toJSONString(Object object) {
JSONStringer stringer = new JSONStringer();
try {
if (object instanceof JSONSerializable) {
stringer.object();
((JSONSerializable) object).toJSON(stringer);
stringer.endObject();
} else if (object != null) {
Class<?> clazz = object.getClass();
// stringer.key(clazz.getSimpleName());
JSONUtil.writeFieldValue(stringer, clazz, object);
}
} catch (JSONException e) {
throw new RuntimeException(e);
}
return (stringer.toString());
}
public static <T extends JSONSerializable> T fromJSONString(T t, String json) {
try {
JSONObject jsonObject = new JSONObject(json);
t.fromJSON(jsonObject);
} catch (JSONException ex) {
throw new RuntimeException("Failed to deserialize object " + t, ex);
}
return (t);
}
/**
* Write the contents of a JSONSerializable object out to a file on the local disk
*
* @param <T>
* @param object
* @param outputPath
* @throws IOException
*/
public static <T extends JSONSerializable> void save(T object, String outputPath)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Writing out contents of "
+ object.getClass().getSimpleName()
+ " to '"
+ outputPath
+ "'");
}
File f = new File(outputPath);
try {
FileUtil.makeDirIfNotExists(f.getParent());
String json = object.toJSONString();
FileUtil.writeStringToFile(f, format(json));
} catch (Exception ex) {
LOG.error(
"Failed to serialize the " + object.getClass().getSimpleName() + " file '" + f + "'", ex);
throw new IOException(ex);
}
}
/**
* Load in a JSONSerialable stored in a file
*
* @param <T>
* @param object
* @param inputPath
* @throws Exception
*/
public static <T extends JSONSerializable> void load(T object, String inputPath)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Loading in serialized "
+ object.getClass().getSimpleName()
+ " from '"
+ inputPath
+ "'");
}
String contents = FileUtil.readFile(inputPath);
if (contents.isEmpty()) {
throw new IOException(
"The " + object.getClass().getSimpleName() + " file '" + inputPath + "' is empty");
}
try {
object.fromJSON(new JSONObject(contents));
} catch (Exception ex) {
if (LOG.isDebugEnabled()) {
LOG.error(
"Failed to deserialize the "
+ object.getClass().getSimpleName()
+ " from file '"
+ inputPath
+ "'",
ex);
}
throw new IOException(ex);
}
if (LOG.isDebugEnabled()) {
LOG.debug("The loading of the " + object.getClass().getSimpleName() + " is complete");
}
}
/**
* For a given Enum, write out the contents of the corresponding field to the JSONObject We assume
* that the given object has matching fields that correspond to the Enum members, except that
* their names are lower case.
*
* @param <E>
* @param <T>
* @param stringer
* @param object
* @param baseClass
* @param members
* @throws JSONException
*/
public static <E extends Enum<?>, T> void fieldsToJSON(
JSONStringer stringer, T object, Class<? extends T> baseClass, E[] members)
throws JSONException {
try {
fieldsToJSON(
stringer, object, baseClass, ClassUtil.getFieldsFromMembersEnum(baseClass, members));
} catch (NoSuchFieldException ex) {
throw new JSONException(ex);
}
}
/**
* For a given list of Fields, write out the contents of the corresponding field to the JSONObject
* The each of the JSONObject's elements will be the upper case version of the Field's name
*
* @param <T>
* @param stringer
* @param object
* @param baseClass
* @param fields
* @throws JSONException
*/
public static <T> void fieldsToJSON(
JSONStringer stringer, T object, Class<? extends T> baseClass, Field[] fields)
throws JSONException {
if (LOG.isDebugEnabled()) {
LOG.debug("Serializing out " + fields.length + " elements for " + baseClass.getSimpleName());
}
for (Field f : fields) {
String jsonKey = f.getName().toUpperCase();
stringer.key(jsonKey);
try {
Class<?> fclass = f.getType();
Object fvalue = f.get(object);
// Null
if (fvalue == null) {
writeFieldValue(stringer, fclass, fvalue);
// Maps
} else if (fvalue instanceof Map) {
writeFieldValue(stringer, fclass, fvalue);
// Everything else
} else {
writeFieldValue(stringer, fclass, fvalue);
}
} catch (Exception ex) {
throw new JSONException(ex);
}
} // FOR
}
/**
* @param stringer
* @param fieldClass
* @param fieldValue
* @throws JSONException
*/
public static void writeFieldValue(
JSONStringer stringer, Class<?> fieldClass, Object fieldValue) throws JSONException {
// Null
if (fieldValue == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("writeNullFieldValue(" + fieldClass + ", " + fieldValue + ")");
}
stringer.value(null);
// Collections
} else if (ClassUtil.getInterfaces(fieldClass).contains(Collection.class)) {
if (LOG.isDebugEnabled()) {
LOG.debug("writeCollectionFieldValue(" + fieldClass + ", " + fieldValue + ")");
}
stringer.array();
for (Object value : (Collection<?>) fieldValue) {
if (value == null) {
stringer.value(null);
} else {
writeFieldValue(stringer, value.getClass(), value);
}
} // FOR
stringer.endArray();
// Maps
} else if (fieldValue instanceof Map) {
if (LOG.isDebugEnabled()) {
LOG.debug("writeMapFieldValue(" + fieldClass + ", " + fieldValue + ")");
}
stringer.object();
for (Entry<?, ?> e : ((Map<?, ?>) fieldValue).entrySet()) {
// We can handle null keys
String keyValue = null;
if (e.getKey() != null) {
// deserialize it on the other side
Class<?> keyClass = e.getKey().getClass();
keyValue = makePrimitiveValue(keyClass, e.getKey()).toString();
}
stringer.key(keyValue);
// We can also handle null values. Where is your god now???
if (e.getValue() == null) {
stringer.value(null);
} else {
writeFieldValue(stringer, e.getValue().getClass(), e.getValue());
}
} // FOR
stringer.endObject();
// Primitive
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("writePrimitiveFieldValue(" + fieldClass + ", " + fieldValue + ")");
}
stringer.value(makePrimitiveValue(fieldClass, fieldValue));
}
return;
}
/**
* Read data from the given JSONObject and populate the given Map
*
* @param jsonObject
* @param map
* @param innerClasses
* @throws Exception
*/
@SuppressWarnings("unchecked")
protected static void readMapField(
final JSONObject jsonObject, final Map map, final Stack<Class> innerClasses)
throws Exception {
Class<?> keyClass = innerClasses.pop();
Class<?> valClass = innerClasses.pop();
Collection<Class<?>> valInterfaces = ClassUtil.getInterfaces(valClass);
assert (jsonObject != null);
for (String jsonKey : CollectionUtil.iterable(jsonObject.keys())) {
final Stack<Class> nextInnerClasses = new Stack<Class>();
nextInnerClasses.addAll(innerClasses);
assert (nextInnerClasses.equals(innerClasses));
// KEY
Object key = JSONUtil.getPrimitiveValue(jsonKey, keyClass);
// VALUE
Object object = null;
if (jsonObject.isNull(jsonKey)) {
// Nothing...
} else if (valInterfaces.contains(List.class)) {
object = new ArrayList();
readCollectionField(
jsonObject.getJSONArray(jsonKey), (Collection) object, nextInnerClasses);
} else if (valInterfaces.contains(Set.class)) {
object = new HashSet();
readCollectionField(
jsonObject.getJSONArray(jsonKey), (Collection) object, nextInnerClasses);
} else if (valInterfaces.contains(Map.class)) {
object = new HashMap();
readMapField(jsonObject.getJSONObject(jsonKey), (Map) object, nextInnerClasses);
} else {
String jsonString = jsonObject.getString(jsonKey);
try {
object = JSONUtil.getPrimitiveValue(jsonString, valClass);
} catch (Exception ex) {
System.err.println("val_interfaces: " + valInterfaces);
LOG.error(
"Failed to deserialize value '"
+ jsonString
+ "' from inner map key '"
+ jsonKey
+ "'");
throw ex;
}
}
map.put(key, object);
}
}
/**
* Read data from the given JSONArray and populate the given Collection
*
* @param jsonArray
* @param collection
* @param innerClasses
* @throws Exception
*/
@SuppressWarnings("unchecked")
protected static void readCollectionField(
final JSONArray jsonArray, final Collection collection, final Stack<Class> innerClasses)
throws Exception {
// We need to figure out what the inner type of the collection is
// If it's a Collection or a Map, then we need to instantiate it before
// we can call readFieldValue() again for it.
Class innerClass = innerClasses.pop();
Collection<Class<?>> innerInterfaces = ClassUtil.getInterfaces(innerClass);
for (int i = 0, cnt = jsonArray.length(); i < cnt; i++) {
final Stack<Class> nextInnerClasses = new Stack<Class>();
nextInnerClasses.addAll(innerClasses);
assert (nextInnerClasses.equals(innerClasses));
Object value = null;
// Null
if (jsonArray.isNull(i)) {
value = null;
// Lists
} else if (innerInterfaces.contains(List.class)) {
value = new ArrayList();
readCollectionField(jsonArray.getJSONArray(i), (Collection) value, nextInnerClasses);
// Sets
} else if (innerInterfaces.contains(Set.class)) {
value = new HashSet();
readCollectionField(jsonArray.getJSONArray(i), (Collection) value, nextInnerClasses);
// Maps
} else if (innerInterfaces.contains(Map.class)) {
value = new HashMap();
readMapField(jsonArray.getJSONObject(i), (Map) value, nextInnerClasses);
// Values
} else {
String jsonString = jsonArray.getString(i);
value = JSONUtil.getPrimitiveValue(jsonString, innerClass);
}
collection.add(value);
} // FOR
return;
}
/**
* @param jsonObject
* @param jsonKey
* @param fieldHandle
* @param object
* @throws Exception
*/
@SuppressWarnings("unchecked")
public static void readFieldValue(
final JSONObject jsonObject, final String jsonKey, Field fieldHandle, Object object)
throws Exception {
assert (jsonObject.has(jsonKey)) : "No entry exists for '" + jsonKey + "'";
Class<?> fieldClass = fieldHandle.getType();
Object fieldObject = fieldHandle.get(object);
// String field_name = field_handle.getName();
// Null
if (jsonObject.isNull(jsonKey)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Field " + jsonKey + " is null");
}
fieldHandle.set(object, null);
// Collections
} else if (ClassUtil.getInterfaces(fieldClass).contains(Collection.class)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Field " + jsonKey + " is a collection");
}
assert (fieldObject != null);
Stack<Class> innerClasses = new Stack<Class>();
innerClasses.addAll(ClassUtil.getGenericTypes(fieldHandle));
Collections.reverse(innerClasses);
JSONArray jsonInner = jsonObject.getJSONArray(jsonKey);
if (jsonInner == null) {
throw new JSONException("No array exists for '" + jsonKey + "'");
}
readCollectionField(jsonInner, (Collection) fieldObject, innerClasses);
// Maps
} else if (fieldObject instanceof Map) {
if (LOG.isDebugEnabled()) {
LOG.debug("Field " + jsonKey + " is a map");
}
assert (fieldObject != null);
Stack<Class> innerClasses = new Stack<Class>();
innerClasses.addAll(ClassUtil.getGenericTypes(fieldHandle));
Collections.reverse(innerClasses);
JSONObject jsonInner = jsonObject.getJSONObject(jsonKey);
if (jsonInner == null) {
throw new JSONException("No object exists for '" + jsonKey + "'");
}
readMapField(jsonInner, (Map) fieldObject, innerClasses);
// Everything else...
} else {
Class explicitFieldClass = JSONUtil.getClassForField(jsonObject, jsonKey);
if (explicitFieldClass != null) {
fieldClass = explicitFieldClass;
if (LOG.isDebugEnabled()) {
LOG.debug(
"Found explict field class " + fieldClass.getSimpleName() + " for " + jsonKey);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("Field " + jsonKey + " is primitive type " + fieldClass.getSimpleName());
}
Object value = JSONUtil.getPrimitiveValue(jsonObject.getString(jsonKey), fieldClass);
fieldHandle.set(object, value);
if (LOG.isDebugEnabled()) {
LOG.debug("Set field " + jsonKey + " to '" + value + "'");
}
}
}
/**
* For the given enum, load in the values from the JSON object into the current object This will
* throw errors if a field is missing
*
* @param <E>
* @param jsonObject
* @param members
* @throws JSONException
*/
public static <E extends Enum<?>, T> void fieldsFromJSON(
JSONObject jsonObject, T object, Class<? extends T> baseClass, E... members)
throws JSONException {
JSONUtil.fieldsFromJSON(jsonObject, object, baseClass, false, members);
}
/**
* For the given enum, load in the values from the JSON object into the current object If
* ignore_missing is false, then JSONUtil will not throw an error if a field is missing
*
* @param <E>
* @param <T>
* @param jsonObject
* @param object
* @param baseClass
* @param ignoreMissing
* @param members
* @throws JSONException
*/
public static <E extends Enum<?>, T> void fieldsFromJSON(
JSONObject jsonObject,
T object,
Class<? extends T> baseClass,
boolean ignoreMissing,
E... members)
throws JSONException {
try {
fieldsFromJSON(
jsonObject,
object,
baseClass,
ignoreMissing,
ClassUtil.getFieldsFromMembersEnum(baseClass, members));
} catch (NoSuchFieldException ex) {
throw new JSONException(ex);
}
}
/**
* For the given list of Fields, load in the values from the JSON object into the current object
* If ignore_missing is false, then JSONUtil will not throw an error if a field is missing
*
* @param <E>
* @param <T>
* @param jsonObject
* @param object
* @param baseClass
* @param ignoreMissing
* @param fields
* @throws JSONException
*/
public static <E extends Enum<?>, T> void fieldsFromJSON(
JSONObject jsonObject,
T object,
Class<? extends T> baseClass,
boolean ignoreMissing,
Field... fields)
throws JSONException {
for (Field fieldHandle : fields) {
String jsonKey = fieldHandle.getName().toUpperCase();
if (LOG.isDebugEnabled()) {
LOG.debug("Retreiving value for field '" + jsonKey + "'");
}
if (!jsonObject.has(jsonKey)) {
String msg =
"JSONObject for "
+ baseClass.getSimpleName()
+ " does not have key '"
+ jsonKey
+ "': "
+ CollectionUtil.list(jsonObject.keys());
if (ignoreMissing) {
if (LOG.isDebugEnabled()) {
LOG.warn(msg);
}
continue;
} else {
throw new JSONException(msg);
}
}
try {
readFieldValue(jsonObject, jsonKey, fieldHandle, object);
} catch (Exception ex) {
// System.err.println(field_class + ": " + ClassUtil.getSuperClasses(field_class));
LOG.error(
"Unable to deserialize field '" + jsonKey + "' from " + baseClass.getSimpleName(),
ex);
throw new JSONException(ex);
}
} // FOR
}
/**
* Return the class of a field if it was stored in the JSONObject along with the value If there is
* no class information, then this will return null
*
* @param jsonObject
* @param jsonKey
* @return
* @throws JSONException
*/
private static Class<?> getClassForField(JSONObject jsonObject, String jsonKey)
throws JSONException {
Class<?> fieldClass = null;
// Check whether we also stored the class
if (jsonObject.has(jsonKey + JSON_CLASS_SUFFIX)) {
try {
fieldClass = ClassUtil.getClass(jsonObject.getString(jsonKey + JSON_CLASS_SUFFIX));
} catch (Exception ex) {
LOG.error("Failed to include class for field '" + jsonKey + "'", ex);
throw new JSONException(ex);
}
}
return (fieldClass);
}
/**
* Return the proper serialization string for the given value
*
* @param fieldClass
* @param fieldValue
* @return
*/
private static Object makePrimitiveValue(Class<?> fieldClass, Object fieldValue) {
Object value = null;
// Class
if (fieldClass.equals(Class.class)) {
value = ((Class<?>) fieldValue).getName();
// JSONSerializable
} else if (ClassUtil.getInterfaces(fieldClass).contains(JSONSerializable.class)) {
// Just return the value back. The JSON library will take care of it
// System.err.println(field_class + ": " + field_value);
value = fieldValue;
// Everything else
} else {
value = fieldValue; // .toString();
}
return (value);
}
/**
* For the given JSON string, figure out what kind of object it is and return it
*
* @param jsonValue
* @param fieldClass
* @return
* @throws Exception
*/
public static Object getPrimitiveValue(String jsonValue, Class<?> fieldClass) throws Exception {
Object value = null;
// Class
if (fieldClass.equals(Class.class)) {
value = ClassUtil.getClass(jsonValue);
if (value == null) {
throw new JSONException("Failed to get class from '" + jsonValue + "'");
}
// Enum
} else if (fieldClass.isEnum()) {
for (Object o : fieldClass.getEnumConstants()) {
Enum<?> e = (Enum<?>) o;
if (jsonValue.equals(e.name())) {
return (e);
}
} // FOR
throw new JSONException(
"Invalid enum value '"
+ jsonValue
+ "': "
+ Arrays.toString(fieldClass.getEnumConstants()));
// JSONSerializable
} else if (ClassUtil.getInterfaces(fieldClass).contains(JSONSerializable.class)) {
value = ClassUtil.newInstance(fieldClass, null, null);
((JSONSerializable) value).fromJSON(new JSONObject(jsonValue));
// Boolean
} else if (fieldClass.equals(Boolean.class) || fieldClass.equals(boolean.class)) {
// We have to use field_class.equals() because the value may be null
value = Boolean.parseBoolean(jsonValue);
// Short
} else if (fieldClass.equals(Short.class) || fieldClass.equals(short.class)) {
value = Short.parseShort(jsonValue);
// Integer
} else if (fieldClass.equals(Integer.class) || fieldClass.equals(int.class)) {
value = Integer.parseInt(jsonValue);
// Long
} else if (fieldClass.equals(Long.class) || fieldClass.equals(long.class)) {
value = Long.parseLong(jsonValue);
// Float
} else if (fieldClass.equals(Float.class) || fieldClass.equals(float.class)) {
value = Float.parseFloat(jsonValue);
// Double
} else if (fieldClass.equals(Double.class) || fieldClass.equals(double.class)) {
value = Double.parseDouble(jsonValue);
// String
} else if (fieldClass.equals(String.class)) {
value = jsonValue.toString();
}
return (value);
}
public static Class<?> getPrimitiveType(String jsonValue) {
Object value = null;
// CHECKSTYLE:OFF
// Class
try {
value = ClassUtil.getClass(jsonValue);
if (value != null) return (Class.class);
} catch (Throwable ex) {
} // IGNORE
// Short
try {
value = Short.parseShort(jsonValue);
return (Short.class);
} catch (NumberFormatException ex) {
} // IGNORE
// Integer
try {
value = Integer.parseInt(jsonValue);
return (Integer.class);
} catch (NumberFormatException ex) {
} // IGNORE
// Long
try {
value = Long.parseLong(jsonValue);
return (Long.class);
} catch (NumberFormatException ex) {
} // IGNORE
// Float
try {
value = Float.parseFloat(jsonValue);
return (Float.class);
} catch (NumberFormatException ex) {
} // IGNORE
// Double
try {
value = Double.parseDouble(jsonValue);
return (Double.class);
} catch (NumberFormatException ex) {
} // IGNORE
// CHECKSTYLE:ON
// Boolean
if (jsonValue.equalsIgnoreCase("true") || jsonValue.equalsIgnoreCase("false")) {
return (Boolean.class);
}
// Default: String
return (String.class);
}
}

138
client/controller/src/main/java/com/controller/util/ValidationUtils.java Normal file
View File

@@ -0,0 +1,138 @@
/*
* OtterTune - ValidationUtils.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.util;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.github.fge.jackson.JsonLoader;
import com.github.fge.jsonschema.core.exceptions.ProcessingException;
import com.github.fge.jsonschema.core.report.ProcessingMessage;
import com.github.fge.jsonschema.core.report.ProcessingReport;
import com.github.fge.jsonschema.main.JsonSchema;
import com.github.fge.jsonschema.main.JsonSchemaFactory;
import java.io.File;
import java.io.IOException;
import java.net.URL;
public class ValidationUtils {
public static final String JSON_V4_SCHEMA_IDENTIFIER = "http://json-schema.org/draft-04/schema#";
public static final String JSON_SCHEMA_IDENTIFIER_ELEMENT = "$schema";
public static JsonNode getJsonNode(String jsonText) throws IOException {
return JsonLoader.fromString(jsonText);
} // getJsonNode(text) ends
public static JsonNode getJsonNode(File jsonFile) throws IOException {
return JsonLoader.fromFile(jsonFile);
} // getJsonNode(File) ends
public static JsonNode getJsonNode(URL url) throws IOException {
return JsonLoader.fromURL(url);
} // getJsonNode(URL) ends
public static JsonNode getJsonNodeFromResource(String resource) throws IOException {
return JsonLoader.fromResource(resource);
} // getJsonNode(Resource) ends
public static JsonSchema getSchemaNode(String schemaText)
throws IOException, ProcessingException {
final JsonNode schemaNode = getJsonNode(schemaText);
return getSchemaNodeHelper(schemaNode);
} // getSchemaNode(text) ends
public static JsonSchema getSchemaNode(File schemaFile) throws IOException, ProcessingException {
final JsonNode schemaNode = getJsonNode(schemaFile);
return getSchemaNodeHelper(schemaNode);
} // getSchemaNode(File) ends
public static JsonSchema getSchemaNode(URL schemaFile) throws IOException, ProcessingException {
final JsonNode schemaNode = getJsonNode(schemaFile);
return getSchemaNodeHelper(schemaNode);
} // getSchemaNode(URL) ends
public static JsonSchema getSchemaNodeFromResource(String resource)
throws IOException, ProcessingException {
final JsonNode schemaNode = getJsonNodeFromResource(resource);
return getSchemaNodeHelper(schemaNode);
} // getSchemaNode() ends
public static void validateJson(String schemaText, String jsonText)
throws IOException, ProcessingException {
final JsonSchema schemaNode = getSchemaNode(schemaText);
final JsonNode jsonNode = getJsonNode(jsonText);
validateJson(schemaNode, jsonNode);
} // validateJson(text) ends
public static void validateJson(File schemaFile, File jsonFile)
throws IOException, ProcessingException {
final JsonSchema schemaNode = getSchemaNode(schemaFile);
final JsonNode jsonNode = getJsonNode(jsonFile);
validateJson(schemaNode, jsonNode);
} // validateJson(File) ends
public static void validateJson(JsonSchema jsonSchemaNode, JsonNode jsonNode)
throws ProcessingException {
ProcessingReport report = jsonSchemaNode.validate(jsonNode);
if (!report.isSuccess()) {
for (ProcessingMessage processingMessage : report) {
throw new ProcessingException(processingMessage);
}
}
} // validateJson(Node) ends
public static void validateJson(URL schemaDocument, URL jsonDocument)
throws IOException, ProcessingException {
final JsonSchema schemaNode = getSchemaNode(schemaDocument);
final JsonNode jsonNode = getJsonNode(jsonDocument);
validateJson(schemaNode, jsonNode);
} // validateJson(URL) ends
public static boolean isJsonValid(JsonSchema jsonSchemaNode, JsonNode jsonNode)
throws ProcessingException {
ProcessingReport report = jsonSchemaNode.validate(jsonNode);
return report.isSuccess();
} // validateJson(Node) ends
public static boolean isJsonValid(String schemaText, String jsonText)
throws ProcessingException, IOException {
final JsonSchema schemaNode = getSchemaNode(schemaText);
final JsonNode jsonNode = getJsonNode(jsonText);
return isJsonValid(schemaNode, jsonNode);
} // validateJson(Node) ends
public static boolean isJsonValid(File schemaFile, File jsonFile)
throws ProcessingException, IOException {
final JsonSchema schemaNode = getSchemaNode(schemaFile);
final JsonNode jsonNode = getJsonNode(jsonFile);
return isJsonValid(schemaNode, jsonNode);
} // validateJson(Node) ends
public static boolean isJsonValid(URL schemaURL, URL jsonURL)
throws ProcessingException, IOException {
final JsonSchema schemaNode = getSchemaNode(schemaURL);
final JsonNode jsonNode = getJsonNode(jsonURL);
return isJsonValid(schemaNode, jsonNode);
} // validateJson(Node) ends
public static void validateJsonResource(String schemaResource, String jsonResource)
throws IOException, ProcessingException {
final JsonSchema schemaNode = getSchemaNode(schemaResource);
final JsonNode jsonNode = getJsonNodeFromResource(jsonResource);
validateJson(schemaNode, jsonNode);
} // validateJsonResource() ends
private static JsonSchema getSchemaNodeHelper(JsonNode jsonNode) throws ProcessingException {
final JsonNode schemaIdentifier = jsonNode.get(JSON_SCHEMA_IDENTIFIER_ELEMENT);
if (null == schemaIdentifier) {
((ObjectNode) jsonNode).put(JSON_SCHEMA_IDENTIFIER_ELEMENT, JSON_V4_SCHEMA_IDENTIFIER);
}
final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
return factory.getJsonSchema(jsonNode);
} // getSchemaNodeHelper() ends
}

862
client/controller/src/main/java/com/controller/util/json/JSONArray.java Normal file
View File

@@ -0,0 +1,862 @@
/*
Copyright (c) 2002 JSON.org
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
The Software shall be used for Good, not Evil.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package com.controller.util.json;
import java.io.IOException;
import java.io.Writer;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
/**
* A JSONArray is an ordered sequence of values. Its external text form is a string wrapped in
* square brackets with commas separating the values. The internal form is an object having <code>
* get</code> and <code>opt</code> methods for accessing the values by index, and <code>put</code>
* methods for adding or replacing values. The values can be any of these types: <code>Boolean
* </code>, <code>JSONArray</code>, <code>JSONObject</code>, <code>Number</code>, <code>String
* </code>, or the <code>JSONObject.NULL object</code>.
*
* <p>The constructor can convert a JSON text into a Java object. The <code>toString</code> method
* converts to JSON text.
*
* <p>A <code>get</code> method returns a value if one can be found, and throws an exception if one
* cannot be found. An <code>opt</code> method returns a default value instead of throwing an
* exception, and so is useful for obtaining optional values.
*
* <p>The generic <code>get()</code> and <code>opt()</code> methods return an object which you can
* cast or query for type. There are also typed <code>get</code> and <code>opt</code> methods that
* do type checking and type coersion for you.
*
* <p>The texts produced by the <code>toString</code> methods strictly conform to JSON syntax rules.
* The constructors are more forgiving in the texts they will accept:
*
* <ul>
* <li>An extra <code>,</code>&nbsp;<small>(comma)</small> may appear just before the closing
* bracket.
* <li>The <code>null</code> value will be inserted when there is <code>,</code>
* &nbsp;<small>(comma)</small> elision.
* <li>Strings may be quoted with <code>'</code>&nbsp;<small>(single quote)</small>.
* <li>Strings do not need to be quoted at all if they do not begin with a quote or single quote,
* and if they do not contain leading or trailing spaces, and if they do not contain any of
* these characters: <code>{ } [ ] / \ : , = ; #</code> and if they do not look like numbers
* and if they are not the reserved words <code>true</code>, <code>false</code>, or <code>null
* </code>.
* <li>Values can be separated by <code>;</code> <small>(semicolon)</small> as well as by <code>,
* </code> <small>(comma)</small>.
* <li>Numbers may have the <code>0-</code> <small>(octal)</small> or <code>0x-</code>
* <small>(hex)</small> prefix.
* <li>Comments written in the slashshlash, slashstar, and hash conventions will be ignored.
* </ul>
*
* @author JSON.org
* @version 2008-09-18
*/
public class JSONArray {
/** The arrayList where the JSONArray's properties are kept. */
private ArrayList<Object> myArrayList;
/** Construct an empty JSONArray. */
public JSONArray() {
this.myArrayList = new ArrayList<Object>();
}
/**
* Construct a JSONArray from a JSONTokener.
*
* @param x A JSONTokener
* @throws JSONException If there is a syntax error.
*/
public JSONArray(JSONTokener x) throws JSONException {
this();
char c = x.nextClean();
char q;
if (c == '[') {
q = ']';
} else if (c == '(') {
q = ')';
} else {
throw x.syntaxError("A JSONArray text must start with '['");
}
if (x.nextClean() == ']') {
return;
}
x.back();
for (; ; ) {
if (x.nextClean() == ',') {
x.back();
this.myArrayList.add(null);
} else {
x.back();
this.myArrayList.add(x.nextValue());
}
c = x.nextClean();
switch (c) {
case ';':
case ',':
if (x.nextClean() == ']') {
return;
}
x.back();
break;
case ']':
case ')':
if (q != c) {
throw x.syntaxError("Expected a '" + new Character(q) + "'");
}
return;
default:
throw x.syntaxError("Expected a ',' or ']'");
}
}
}
/**
* Construct a JSONArray from a source JSON text.
*
* @param source A string that begins with <code>[</code>&nbsp;<small>(left bracket)</small> and
* ends with <code>]</code>&nbsp;<small>(right bracket)</small>.
* @throws JSONException If there is a syntax error.
*/
public JSONArray(String source) throws JSONException {
this(new JSONTokener(source));
}
/**
* Construct a JSONArray from a Collection.
*
* @param collection A Collection.
*/
public JSONArray(Collection<?> collection) {
this.myArrayList =
(collection == null) ? new ArrayList<Object>() : new ArrayList<Object>(collection);
}
/** Construct a JSONArray from a collection of beans. The collection should have Java Beans. */
public JSONArray(Collection<?> collection, boolean includeSuperClass) {
this.myArrayList = new ArrayList<Object>();
if (collection != null) {
for (Iterator<?> iter = collection.iterator(); iter.hasNext(); ) {
this.myArrayList.add(new JSONObject(iter.next(), includeSuperClass));
}
}
}
/**
* Construct a JSONArray from an array
*
* @throws JSONException If not an array.
*/
public JSONArray(Object array) throws JSONException {
this();
if (array.getClass().isArray()) {
int length = Array.getLength(array);
for (int i = 0; i < length; i += 1) {
this.put(Array.get(array, i));
}
} else {
throw new JSONException("JSONArray initial value should be a string or collection or array.");
}
}
/**
* Construct a JSONArray from an array with a bean. The array should have Java Beans.
*
* @throws JSONException If not an array.
*/
public JSONArray(Object array, boolean includeSuperClass) throws JSONException {
this();
if (array.getClass().isArray()) {
int length = Array.getLength(array);
for (int i = 0; i < length; i += 1) {
this.put(new JSONObject(Array.get(array, i), includeSuperClass));
}
} else {
throw new JSONException("JSONArray initial value should be a string or collection or array.");
}
}
/**
* Get the object value associated with an index.
*
* @param index The index must be between 0 and length() - 1.
* @return An object value.
* @throws JSONException If there is no value for the index.
*/
public Object get(int index) throws JSONException {
Object o = opt(index);
if (o == null) {
throw new JSONException("JSONArray[" + index + "] not found.");
}
return o;
}
/**
* Get the boolean value associated with an index. The string values "true" and "false" are
* converted to boolean.
*
* @param index The index must be between 0 and length() - 1.
* @return The truth.
* @throws JSONException If there is no value for the index or if the value is not convertable to
* boolean.
*/
public boolean getBoolean(int index) throws JSONException {
Object o = get(index);
if (o.equals(Boolean.FALSE)
|| (o instanceof String && ((String) o).equalsIgnoreCase("false"))) {
return false;
} else if (o.equals(Boolean.TRUE)
|| (o instanceof String && ((String) o).equalsIgnoreCase("true"))) {
return true;
}
throw new JSONException("JSONArray[" + index + "] is not a Boolean.");
}
/**
* Get the double value associated with an index.
*
* @param index The index must be between 0 and length() - 1.
* @return The value.
* @throws JSONException If the key is not found or if the value cannot be converted to a number.
*/
public double getDouble(int index) throws JSONException {
Object o = get(index);
try {
return o instanceof Number
? ((Number) o).doubleValue()
: Double.valueOf((String) o).doubleValue();
} catch (Exception e) {
throw new JSONException("JSONArray[" + index + "] is not a number.");
}
}
/**
* Get the int value associated with an index.
*
* @param index The index must be between 0 and length() - 1.
* @return The value.
* @throws JSONException If the key is not found or if the value cannot be converted to a number.
* if the value cannot be converted to a number.
*/
public int getInt(int index) throws JSONException {
Object o = get(index);
return o instanceof Number ? ((Number) o).intValue() : (int) getDouble(index);
}
/**
* Get the JSONArray associated with an index.
*
* @param index The index must be between 0 and length() - 1.
* @return A JSONArray value.
* @throws JSONException If there is no value for the index. or if the value is not a JSONArray
*/
public JSONArray getJSONArray(int index) throws JSONException {
Object o = get(index);
if (o instanceof JSONArray) {
return (JSONArray) o;
}
throw new JSONException("JSONArray[" + index + "] is not a JSONArray.");
}
/**
* Get the JSONObject associated with an index.
*
* @param index subscript
* @return A JSONObject value.
* @throws JSONException If there is no value for the index or if the value is not a JSONObject
*/
public JSONObject getJSONObject(int index) throws JSONException {
Object o = get(index);
if (o instanceof JSONObject) {
return (JSONObject) o;
}
throw new JSONException("JSONArray[" + index + "] is not a JSONObject.");
}
/**
* Get the long value associated with an index.
*
* @param index The index must be between 0 and length() - 1.
* @return The value.
* @throws JSONException If the key is not found or if the value cannot be converted to a number.
*/
public long getLong(int index) throws JSONException {
Object o = get(index);
return o instanceof Number ? ((Number) o).longValue() : (long) getDouble(index);
}
/**
* Get the string associated with an index.
*
* @param index The index must be between 0 and length() - 1.
* @return A string value.
* @throws JSONException If there is no value for the index.
*/
public String getString(int index) throws JSONException {
return get(index).toString();
}
/**
* Determine if the value is null.
*
* @param index The index must be between 0 and length() - 1.
* @return true if the value at the index is null, or if there is no value.
*/
public boolean isNull(int index) {
return JSONObject.NULL.equals(opt(index));
}
/**
* Make a string from the contents of this JSONArray. The <code>separator</code> string is
* inserted between each element. Warning: This method assumes that the data structure is
* acyclical.
*
* @param separator A string that will be inserted between the elements.
* @return a string.
* @throws JSONException If the array contains an invalid number.
*/
public String join(String separator) throws JSONException {
int len = length();
StringBuffer sb = new StringBuffer();
for (int i = 0; i < len; i += 1) {
if (i > 0) {
sb.append(separator);
}
sb.append(JSONObject.valueToString(this.myArrayList.get(i)));
}
return sb.toString();
}
/**
* Get the number of elements in the JSONArray, included nulls.
*
* @return The length (or size).
*/
public int length() {
return this.myArrayList.size();
}
/**
* Get the optional object value associated with an index.
*
* @param index The index must be between 0 and length() - 1.
* @return An object value, or null if there is no object at that index.
*/
public Object opt(int index) {
return (index < 0 || index >= length()) ? null : this.myArrayList.get(index);
}
/**
* Get the optional boolean value associated with an index. It returns false if there is no value
* at that index, or if the value is not Boolean.TRUE or the String "true".
*
* @param index The index must be between 0 and length() - 1.
* @return The truth.
*/
public boolean optBoolean(int index) {
return optBoolean(index, false);
}
/**
* Get the optional boolean value associated with an index. It returns the defaultValue if there
* is no value at that index or if it is not a Boolean or the String "true" or "false" (case
* insensitive).
*
* @param index The index must be between 0 and length() - 1.
* @param defaultValue A boolean default.
* @return The truth.
*/
public boolean optBoolean(int index, boolean defaultValue) {
try {
return getBoolean(index);
} catch (Exception e) {
return defaultValue;
}
}
/**
* Get the optional double value associated with an index. NaN is returned if there is no value
* for the index, or if the value is not a number and cannot be converted to a number.
*
* @param index The index must be between 0 and length() - 1.
* @return The value.
*/
public double optDouble(int index) {
return optDouble(index, Double.NaN);
}
/**
* Get the optional double value associated with an index. The defaultValue is returned if there
* is no value for the index, or if the value is not a number and cannot be converted to a number.
*
* @param index subscript
* @param defaultValue The default value.
* @return The value.
*/
public double optDouble(int index, double defaultValue) {
try {
return getDouble(index);
} catch (Exception e) {
return defaultValue;
}
}
/**
* Get the optional int value associated with an index. Zero is returned if there is no value for
* the index, or if the value is not a number and cannot be converted to a number.
*
* @param index The index must be between 0 and length() - 1.
* @return The value.
*/
public int optInt(int index) {
return optInt(index, 0);
}
/**
* Get the optional int value associated with an index. The defaultValue is returned if there is
* no value for the index, or if the value is not a number and cannot be converted to a number.
*
* @param index The index must be between 0 and length() - 1.
* @param defaultValue The default value.
* @return The value.
*/
public int optInt(int index, int defaultValue) {
try {
return getInt(index);
} catch (Exception e) {
return defaultValue;
}
}
/**
* Get the optional JSONArray associated with an index.
*
* @param index subscript
* @return A JSONArray value, or null if the index has no value, or if the value is not a
* JSONArray.
*/
public JSONArray optJSONArray(int index) {
Object o = opt(index);
return o instanceof JSONArray ? (JSONArray) o : null;
}
/**
* Get the optional JSONObject associated with an index. Null is returned if the key is not found,
* or null if the index has no value, or if the value is not a JSONObject.
*
* @param index The index must be between 0 and length() - 1.
* @return A JSONObject value.
*/
public JSONObject optJSONObject(int index) {
Object o = opt(index);
return o instanceof JSONObject ? (JSONObject) o : null;
}
/**
* Get the optional long value associated with an index. Zero is returned if there is no value for
* the index, or if the value is not a number and cannot be converted to a number.
*
* @param index The index must be between 0 and length() - 1.
* @return The value.
*/
public long optLong(int index) {
return optLong(index, 0);
}
/**
* Get the optional long value associated with an index. The defaultValue is returned if there is
* no value for the index, or if the value is not a number and cannot be converted to a number.
*
* @param index The index must be between 0 and length() - 1.
* @param defaultValue The default value.
* @return The value.
*/
public long optLong(int index, long defaultValue) {
try {
return getLong(index);
} catch (Exception e) {
return defaultValue;
}
}
/**
* Get the optional string value associated with an index. It returns an empty string if there is
* no value at that index. If the value is not a string and is not null, then it is coverted to a
* string.
*
* @param index The index must be between 0 and length() - 1.
* @return A String value.
*/
public String optString(int index) {
return optString(index, "");
}
/**
* Get the optional string associated with an index. The defaultValue is returned if the key is
* not found.
*
* @param index The index must be between 0 and length() - 1.
* @param defaultValue The default value.
* @return A String value.
*/
public String optString(int index, String defaultValue) {
Object o = opt(index);
return o != null ? o.toString() : defaultValue;
}
/**
* Append a boolean value. This increases the array's length by one.
*
* @param value A boolean value.
* @return this.
*/
public JSONArray put(boolean value) {
put(value ? Boolean.TRUE : Boolean.FALSE);
return this;
}
/**
* Put a value in the JSONArray, where the value will be a JSONArray which is produced from a
* Collection.
*
* @param value A Collection value.
* @return this.
*/
public JSONArray put(Collection<?> value) {
put(new JSONArray(value));
return this;
}
/**
* Append a double value. This increases the array's length by one.
*
* @param value A double value.
* @throws JSONException if the value is not finite.
* @return this.
*/
public JSONArray put(double value) throws JSONException {
Double d = new Double(value);
JSONObject.testValidity(d);
put(d);
return this;
}
/**
* Append an int value. This increases the array's length by one.
*
* @param value An int value.
* @return this.
*/
public JSONArray put(int value) {
put(new Integer(value));
return this;
}
/**
* Append an long value. This increases the array's length by one.
*
* @param value A long value.
* @return this.
*/
public JSONArray put(long value) {
put(new Long(value));
return this;
}
/**
* Put a value in the JSONArray, where the value will be a JSONObject which is produced from a
* Map.
*
* @param value A Map value.
* @return this.
*/
public JSONArray put(Map<?, ?> value) {
put(new JSONObject(value));
return this;
}
/**
* Append an object value. This increases the array's length by one.
*
* @param value An object value. The value should be a Boolean, Double, Integer, JSONArray,
* JSONObject, Long, or String, or the JSONObject.NULL object.
* @return this.
*/
public JSONArray put(Object value) {
this.myArrayList.add(value);
return this;
}
/**
* Put or replace a boolean value in the JSONArray. If the index is greater than the length of the
* JSONArray, then null elements will be added as necessary to pad it out.
*
* @param index The subscript.
* @param value A boolean value.
* @return this.
* @throws JSONException If the index is negative.
*/
public JSONArray put(int index, boolean value) throws JSONException {
put(index, value ? Boolean.TRUE : Boolean.FALSE);
return this;
}
/**
* Put a value in the JSONArray, where the value will be a JSONArray which is produced from a
* Collection.
*
* @param index The subscript.
* @param value A Collection value.
* @return this.
* @throws JSONException If the index is negative or if the value is not finite.
*/
public JSONArray put(int index, Collection<?> value) throws JSONException {
put(index, new JSONArray(value));
return this;
}
/**
* Put or replace a double value. If the index is greater than the length of the JSONArray, then
* null elements will be added as necessary to pad it out.
*
* @param index The subscript.
* @param value A double value.
* @return this.
* @throws JSONException If the index is negative or if the value is not finite.
*/
public JSONArray put(int index, double value) throws JSONException {
put(index, new Double(value));
return this;
}
/**
* Put or replace an int value. If the index is greater than the length of the JSONArray, then
* null elements will be added as necessary to pad it out.
*
* @param index The subscript.
* @param value An int value.
* @return this.
* @throws JSONException If the index is negative.
*/
public JSONArray put(int index, int value) throws JSONException {
put(index, new Integer(value));
return this;
}
/**
* Put or replace a long value. If the index is greater than the length of the JSONArray, then
* null elements will be added as necessary to pad it out.
*
* @param index The subscript.
* @param value A long value.
* @return this.
* @throws JSONException If the index is negative.
*/
public JSONArray put(int index, long value) throws JSONException {
put(index, new Long(value));
return this;
}
/**
* Put a value in the JSONArray, where the value will be a JSONObject which is produced from a
* Map.
*
* @param index The subscript.
* @param value The Map value.
* @return this.
* @throws JSONException If the index is negative or if the the value is an invalid number.
*/
public JSONArray put(int index, Map<?, ?> value) throws JSONException {
put(index, new JSONObject(value));
return this;
}
/**
* Put or replace an object value in the JSONArray. If the index is greater than the length of the
* JSONArray, then null elements will be added as necessary to pad it out.
*
* @param index The subscript.
* @param value The value to put into the array. The value should be a Boolean, Double, Integer,
* JSONArray, JSONObject, Long, or String, or the JSONObject.NULL object.
* @return this.
* @throws JSONException If the index is negative or if the the value is an invalid number.
*/
public JSONArray put(int index, Object value) throws JSONException {
JSONObject.testValidity(value);
if (index < 0) {
throw new JSONException("JSONArray[" + index + "] not found.");
}
if (index < length()) {
this.myArrayList.set(index, value);
} else {
while (index != length()) {
put(JSONObject.NULL);
}
put(value);
}
return this;
}
/**
* Produce a JSONObject by combining a JSONArray of names with the values of this JSONArray.
*
* @param names A JSONArray containing a list of key strings. These will be paired with the
* values.
* @return A JSONObject, or null if there are no names or if this JSONArray has no values.
* @throws JSONException If any of the names are null.
*/
public JSONObject toJSONObject(JSONArray names) throws JSONException {
if (names == null || names.length() == 0 || length() == 0) {
return null;
}
JSONObject jo = new JSONObject();
for (int i = 0; i < names.length(); i += 1) {
jo.put(names.getString(i), this.opt(i));
}
return jo;
}
/**
* Make a JSON text of this JSONArray. For compactness, no unnecessary whitespace is added. If it
* is not possible to produce a syntactically correct JSON text then null will be returned
* instead. This could occur if the array contains an invalid number.
*
* <p>Warning: This method assumes that the data structure is acyclical.
*
* @return a printable, displayable, transmittable representation of the array.
*/
public String toString() {
try {
return '[' + join(",") + ']';
} catch (Exception e) {
return null;
}
}
/**
* Make a prettyprinted JSON text of this JSONArray. Warning: This method assumes that the data
* structure is acyclical.
*
* @param indentFactor The number of spaces to add to each level of indentation.
* @return a printable, displayable, transmittable representation of the object, beginning with
* <code>[</code>&nbsp;<small>(left bracket)</small> and ending with <code>]</code>
* &nbsp;<small>(right bracket)</small>.
* @throws JSONException
*/
public String toString(int indentFactor) throws JSONException {
return toString(indentFactor, 0);
}
/**
* Make a prettyprinted JSON text of this JSONArray. Warning: This method assumes that the data
* structure is acyclical.
*
* @param indentFactor The number of spaces to add to each level of indentation.
* @param indent The indention of the top level.
* @return a printable, displayable, transmittable representation of the array.
* @throws JSONException
*/
String toString(int indentFactor, int indent) throws JSONException {
int len = length();
if (len == 0) {
return "[]";
}
int i;
StringBuffer sb = new StringBuffer("[");
if (len == 1) {
sb.append(JSONObject.valueToString(this.myArrayList.get(0), indentFactor, indent));
} else {
int newindent = indent + indentFactor;
// sb.append('\n');
boolean intType = false;
for (i = 0; i < len; i += 1) {
if (this.myArrayList.get(i).getClass() != Integer.class) {
if (i == 0) {
sb.append('\n');
}
if (i > 0) {
sb.append(",\n");
}
for (int j = 0; j < newindent; j += 1) {
sb.append(' ');
}
} else {
intType = true;
if (i > 0) sb.append(", ");
}
sb.append(JSONObject.valueToString(this.myArrayList.get(i), indentFactor, newindent));
}
if (intType == false) {
sb.append('\n');
for (i = 0; i < indent; i += 1) {
sb.append(' ');
}
}
}
sb.append(']');
return sb.toString();
}
/**
* Write the contents of the JSONArray as JSON text to a writer. For compactness, no whitespace is
* added.
*
* <p>Warning: This method assumes that the data structure is acyclical.
*
* @return The writer.
* @throws JSONException
*/
public Writer write(Writer writer) throws JSONException {
try {
boolean b = false;
int len = length();
writer.write('[');
for (int i = 0; i < len; i += 1) {
if (b) {
writer.write(',');
}
Object v = this.myArrayList.get(i);
if (v instanceof JSONObject) {
((JSONObject) v).write(writer);
} else if (v instanceof JSONArray) {
((JSONArray) v).write(writer);
} else {
writer.write(JSONObject.valueToString(v));
}
b = true;
}
writer.write(']');
return writer;
} catch (IOException e) {
throw new JSONException(e);
}
}
}

30
client/controller/src/main/java/com/controller/util/json/JSONException.java Normal file
View File

@@ -0,0 +1,30 @@
package com.controller.util.json;
/**
* The JSONException is thrown by the JSON.org classes then things are amiss.
*
* @author JSON.org
* @version 2008-09-18
*/
public class JSONException extends Exception {
private static final long serialVersionUID = 1L;
private Throwable cause;
/**
* Constructs a JSONException with an explanatory message.
*
* @param message Detail about the reason for the exception.
*/
public JSONException(String message) {
super(message);
}
public JSONException(Throwable t) {
super(t.getMessage());
this.cause = t;
}
public Throwable getCause() {
return this.cause;
}
}

1417
client/controller/src/main/java/com/controller/util/json/JSONObject.java Normal file
View File

File diff suppressed because it is too large Load Diff

16
client/controller/src/main/java/com/controller/util/json/JSONString.java Normal file
View File

@@ -0,0 +1,16 @@
package com.controller.util.json;
/**
* The <code>JSONString</code> interface allows a <code>toJSONString()</code> method so that a class
* can change the behavior of <code>JSONObject.toString()</code>, <code>JSONArray.toString()</code>,
* and <code>JSONWriter.value(</code>Object<code>)</code>. The <code>toJSONString</code> method will
* be used instead of the default behavior of using the Object's <code>toString()</code> method and
* quoting the result.
*/
public interface JSONString {
/**
* The <code>toJSONString</code> method allows a class to produce its own JSON serialization.
*
* @return A strictly syntactically correct JSON text.
*/
public String toJSONString();
}

79
client/controller/src/main/java/com/controller/util/json/JSONStringer.java Normal file
View File

@@ -0,0 +1,79 @@
/*
Copyright (c) 2006 JSON.org
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
The Software shall be used for Good, not Evil.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package com.controller.util.json;
import java.io.StringWriter;
/**
* JSONStringer provides a quick and convenient way of producing JSON text. The texts produced
* strictly conform to JSON syntax rules. No whitespace is added, so the results are ready for
* transmission or storage. Each instance of JSONStringer can produce one JSON text.
*
* <p>A JSONStringer instance provides a <code>value</code> method for appending values to the text,
* and a <code>key</code> method for adding keys before values in objects. There are <code>array
* </code> and <code>endArray</code> methods that make and bound array values, and <code>object
* </code> and <code>endObject</code> methods which make and bound object values. All of these
* methods return the JSONWriter instance, permitting cascade style. For example,
*
* <pre>
* myString = new JSONStringer()
* .object()
* .key("JSON")
* .value("Hello, World!")
* .endObject()
* .toString();</pre>
*
* which produces the string
*
* <pre>
* {"JSON":"Hello, World!"}</pre>
*
* <p>The first method called must be <code>array</code> or <code>object</code>. There are no
* methods for adding commas or colons. JSONStringer adds them for you. Objects and arrays can be
* nested up to 20 levels deep.
*
* <p>This can sometimes be easier than using a JSONObject to build a string.
*
* @author JSON.org
* @version 2008-09-18
*/
public class JSONStringer extends JSONWriter {
/** Make a fresh JSONStringer. It can be used to build one JSON text. */
public JSONStringer() {
super(new StringWriter());
}
/**
* Return the JSON text. This method is used to obtain the product of the JSONStringer instance.
* It will return <code>null</code> if there was a problem in the construction of the JSON text
* (such as the calls to <code>array</code> were not properly balanced with calls to <code>
* endArray</code>).
*
* @return The JSON text.
*/
public String toString() {
return this.mode == 'd' ? this.writer.toString() : null;
}
}

403
client/controller/src/main/java/com/controller/util/json/JSONTokener.java Normal file
View File

@@ -0,0 +1,403 @@
/*
Copyright (c) 2002 JSON.org
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
The Software shall be used for Good, not Evil.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package com.controller.util.json;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
/**
* A JSONTokener takes a source string and extracts characters and tokens from it. It is used by the
* JSONObject and JSONArray constructors to parse JSON source strings.
*
* @author JSON.org
* @version 2008-09-18
*/
public class JSONTokener {
private int index;
private Reader reader;
private char lastChar;
private boolean useLastChar;
/**
* Construct a JSONTokener from a string.
*
* @param reader A reader.
*/
public JSONTokener(Reader reader) {
this.reader = reader.markSupported() ? reader : new BufferedReader(reader);
this.useLastChar = false;
this.index = 0;
}
/**
* Construct a JSONTokener from a string.
*
* @param s A source string.
*/
public JSONTokener(String s) {
this(new StringReader(s));
}
/**
* Back up one character. This provides a sort of lookahead capability, so that you can test for a
* digit or letter before attempting to parse the next number or identifier.
*/
public void back() throws JSONException {
if (useLastChar || index <= 0) {
throw new JSONException("Stepping back two steps is not supported");
}
index -= 1;
useLastChar = true;
}
/**
* Get the hex value of a character (base16).
*
* @param c A character between '0' and '9' or between 'A' and 'F' or between 'a' and 'f'.
* @return An int between 0 and 15, or -1 if c was not a hex digit.
*/
public static int dehexchar(char c) {
if (c >= '0' && c <= '9') {
return c - '0';
}
if (c >= 'A' && c <= 'F') {
return c - ('A' - 10);
}
if (c >= 'a' && c <= 'f') {
return c - ('a' - 10);
}
return -1;
}
/**
* Determine if the source string still contains characters that next() can consume.
*
* @return true if not yet at the end of the source.
*/
public boolean more() throws JSONException {
char nextChar = next();
if (nextChar == 0) {
return false;
}
back();
return true;
}
/**
* Get the next character in the source string.
*
* @return The next character, or 0 if past the end of the source string.
*/
public char next() throws JSONException {
if (this.useLastChar) {
this.useLastChar = false;
if (this.lastChar != 0) {
this.index += 1;
}
return this.lastChar;
}
int c;
try {
c = this.reader.read();
} catch (IOException exc) {
throw new JSONException(exc);
}
if (c <= 0) { // End of stream
this.lastChar = 0;
return 0;
}
this.index += 1;
this.lastChar = (char) c;
return this.lastChar;
}
/**
* Consume the next character, and check that it matches a specified character.
*
* @param c The character to match.
* @return The character.
* @throws JSONException if the character does not match.
*/
public char next(char c) throws JSONException {
char n = next();
if (n != c) {
throw syntaxError("Expected '" + c + "' and instead saw '" + n + "'");
}
return n;
}
/**
* Get the next n characters.
*
* @param n The number of characters to take.
* @return A string of n characters.
* @throws JSONException Substring bounds error if there are not n characters remaining in the
* source string.
*/
public String next(int n) throws JSONException {
if (n == 0) {
return "";
}
char[] buffer = new char[n];
int pos = 0;
if (this.useLastChar) {
this.useLastChar = false;
buffer[0] = this.lastChar;
pos = 1;
}
try {
int len;
while ((pos < n) && ((len = reader.read(buffer, pos, n - pos)) != -1)) {
pos += len;
}
} catch (IOException exc) {
throw new JSONException(exc);
}
this.index += pos;
if (pos < n) {
throw syntaxError("Substring bounds error");
}
this.lastChar = buffer[n - 1];
return new String(buffer);
}
/**
* Get the next char in the string, skipping whitespace.
*
* @throws JSONException
* @return A character, or 0 if there are no more characters.
*/
public char nextClean() throws JSONException {
for (; ; ) {
char c = next();
if (c == 0 || c > ' ') {
return c;
}
}
}
/**
* Return the characters up to the next close quote character. Backslash processing is done. The
* formal JSON format does not allow strings in single quotes, but an implementation is allowed to
* accept them.
*
* @param quote The quoting character, either <code>"</code>&nbsp;<small>(double quote)</small> or
* <code>'</code>&nbsp;<small>(single quote)</small>.
* @return A String.
* @throws JSONException Unterminated string.
*/
public String nextString(char quote) throws JSONException {
char c;
StringBuffer sb = new StringBuffer();
for (; ; ) {
c = next();
switch (c) {
case 0:
case '\n':
case '\r':
throw syntaxError("Unterminated string");
case '\\':
c = next();
switch (c) {
case 'b':
sb.append('\b');
break;
case 't':
sb.append('\t');
break;
case 'n':
sb.append('\n');
break;
case 'f':
sb.append('\f');
break;
case 'r':
sb.append('\r');
break;
case 'u':
sb.append((char) Integer.parseInt(next(4), 16));
break;
case 'x':
sb.append((char) Integer.parseInt(next(2), 16));
break;
default:
sb.append(c);
}
break;
default:
if (c == quote) {
return sb.toString();
}
sb.append(c);
}
}
}
/**
* Get the text up but not including the specified character or the end of line, whichever comes
* first.
*
* @param d A delimiter character.
* @return A string.
*/
public String nextTo(char d) throws JSONException {
StringBuffer sb = new StringBuffer();
for (; ; ) {
char c = next();
if (c == d || c == 0 || c == '\n' || c == '\r') {
if (c != 0) {
back();
}
return sb.toString().trim();
}
sb.append(c);
}
}
/**
* Get the text up but not including one of the specified delimiter characters or the end of line,
* whichever comes first.
*
* @param delimiters A set of delimiter characters.
* @return A string, trimmed.
*/
public String nextTo(String delimiters) throws JSONException {
char c;
StringBuffer sb = new StringBuffer();
for (; ; ) {
c = next();
if (delimiters.indexOf(c) >= 0 || c == 0 || c == '\n' || c == '\r') {
if (c != 0) {
back();
}
return sb.toString().trim();
}
sb.append(c);
}
}
/**
* Get the next value. The value can be a Boolean, Double, Integer, JSONArray, JSONObject, Long,
* or String, or the JSONObject.NULL object.
*
* @throws JSONException If syntax error.
* @return An object.
*/
public Object nextValue() throws JSONException {
char c = nextClean();
String s;
switch (c) {
case '"':
case '\'':
return nextString(c);
case '{':
back();
return new JSONObject(this);
case '[':
case '(':
back();
return new JSONArray(this);
}
/*
* Handle unquoted text. This could be the values true, false, or
* null, or it can be a number. An implementation (such as this one)
* is allowed to also accept non-standard forms.
*
* Accumulate characters until we reach the end of the text or a
* formatting character.
*/
StringBuffer sb = new StringBuffer();
while (c >= ' ' && ",:]}/\\\"[{;=#".indexOf(c) < 0) {
sb.append(c);
c = next();
}
back();
s = sb.toString().trim();
if (s.equals("")) {
throw syntaxError("Missing value");
}
return JSONObject.stringToValue(s);
}
/**
* Skip characters until the next character is the requested character. If the requested character
* is not found, no characters are skipped.
*
* @param to A character to skip to.
* @return The requested character, or zero if the requested character is not found.
*/
public char skipTo(char to) throws JSONException {
char c;
try {
int startIndex = this.index;
reader.mark(Integer.MAX_VALUE);
do {
c = next();
if (c == 0) {
reader.reset();
this.index = startIndex;
return c;
}
} while (c != to);
} catch (IOException exc) {
throw new JSONException(exc);
}
back();
return c;
}
/**
* Make a JSONException to signal a syntax error.
*
* @param message The error message.
* @return A JSONException object, suitable for throwing
*/
public JSONException syntaxError(String message) {
return new JSONException(message + toString());
}
/**
* Make a printable string of this JSONTokener.
*
* @return " at character [this.index]"
*/
public String toString() {
return " at character " + index;
}
}

309
client/controller/src/main/java/com/controller/util/json/JSONWriter.java Normal file
View File

@@ -0,0 +1,309 @@
/*
Copyright (c) 2006 JSON.org
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
The Software shall be used for Good, not Evil.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
package com.controller.util.json;
import java.io.IOException;
import java.io.Writer;
/**
* JSONWriter provides a quick and convenient way of producing JSON text. The texts produced
* strictly conform to JSON syntax rules. No whitespace is added, so the results are ready for
* transmission or storage. Each instance of JSONWriter can produce one JSON text.
*
* <p>A JSONWriter instance provides a <code>value</code> method for appending values to the text,
* and a <code>key</code> method for adding keys before values in objects. There are <code>array
* </code> and <code>endArray</code> methods that make and bound array values, and <code>object
* </code> and <code>endObject</code> methods which make and bound object values. All of these
* methods return the JSONWriter instance, permitting a cascade style. For example,
*
* <pre>
* new JSONWriter(myWriter)
* .object()
* .key("JSON")
* .value("Hello, World!")
* .endObject();</pre>
*
* which writes
*
* <pre>
* {"JSON":"Hello, World!"}</pre>
*
* <p>The first method called must be <code>array</code> or <code>object</code>. There are no
* methods for adding commas or colons. JSONWriter adds them for you. Objects and arrays can be
* nested up to 20 levels deep.
*
* <p>This can sometimes be easier than using a JSONObject to build a string.
*
* @author JSON.org
* @version 2008-09-18
*/
public class JSONWriter {
private static final int maxdepth = 20;
/** The comma flag determines if a comma should be output before the next value. */
private boolean comma;
/** The current mode. Values: 'a' (array), 'd' (done), 'i' (initial), 'k' (key), 'o' (object). */
protected char mode;
/** The object/array stack. */
private JSONObject stack[];
/** The stack top index. A value of 0 indicates that the stack is empty. */
private int top;
/** The writer that will receive the output. */
protected Writer writer;
/** Make a fresh JSONWriter. It can be used to build one JSON text. */
public JSONWriter(Writer w) {
this.comma = false;
this.mode = 'i';
this.stack = new JSONObject[maxdepth];
this.top = 0;
this.writer = w;
}
/**
* Append a value.
*
* @param s A string value.
* @return this
* @throws JSONException If the value is out of sequence.
*/
private JSONWriter append(String s) throws JSONException {
if (s == null) {
throw new JSONException("Null pointer");
}
if (this.mode == 'o' || this.mode == 'a') {
try {
if (this.comma && this.mode == 'a') {
this.writer.write(',');
}
this.writer.write(s);
} catch (IOException e) {
throw new JSONException(e);
}
if (this.mode == 'o') {
this.mode = 'k';
}
this.comma = true;
return this;
}
throw new JSONException("Value out of sequence.");
}
/**
* Begin appending a new array. All values until the balancing <code>endArray</code> will be
* appended to this array. The <code>endArray</code> method must be called to mark the array's
* end.
*
* @return this
* @throws JSONException If the nesting is too deep, or if the object is started in the wrong
* place (for example as a key or after the end of the outermost array or object).
*/
public JSONWriter array() throws JSONException {
if (this.mode == 'i' || this.mode == 'o' || this.mode == 'a') {
this.push(null);
this.append("[");
this.comma = false;
return this;
}
throw new JSONException("Misplaced array.");
}
/**
* End something.
*
* @param m Mode
* @param c Closing character
* @return this
* @throws JSONException If unbalanced.
*/
private JSONWriter end(char m, char c) throws JSONException {
if (this.mode != m) {
throw new JSONException(m == 'o' ? "Misplaced endObject." : "Misplaced endArray.");
}
this.pop(m);
try {
this.writer.write(c);
} catch (IOException e) {
throw new JSONException(e);
}
this.comma = true;
return this;
}
/**
* End an array. This method most be called to balance calls to <code>array</code>.
*
* @return this
* @throws JSONException If incorrectly nested.
*/
public JSONWriter endArray() throws JSONException {
return this.end('a', ']');
}
/**
* End an object. This method most be called to balance calls to <code>object</code>.
*
* @return this
* @throws JSONException If incorrectly nested.
*/
public JSONWriter endObject() throws JSONException {
return this.end('k', '}');
}
/**
* Append a key. The key will be associated with the next value. In an object, every value must be
* preceded by a key.
*
* @param s A key string.
* @return this
* @throws JSONException If the key is out of place. For example, keys do not belong in arrays or
* if the key is null.
*/
public JSONWriter key(String s) throws JSONException {
if (s == null) {
throw new JSONException("Null key.");
}
if (this.mode == 'k') {
try {
if (this.comma) {
this.writer.write(',');
}
stack[top - 1].putOnce(s, Boolean.TRUE);
this.writer.write(JSONObject.quote(s));
this.writer.write(':');
this.comma = false;
this.mode = 'o';
return this;
} catch (IOException e) {
throw new JSONException(e);
}
}
throw new JSONException("Misplaced key.");
}
/**
* Begin appending a new object. All keys and values until the balancing <code>endObject</code>
* will be appended to this object. The <code>endObject</code> method must be called to mark the
* object's end.
*
* @return this
* @throws JSONException If the nesting is too deep, or if the object is started in the wrong
* place (for example as a key or after the end of the outermost array or object).
*/
public JSONWriter object() throws JSONException {
if (this.mode == 'i') {
this.mode = 'o';
}
if (this.mode == 'o' || this.mode == 'a') {
this.append("{");
this.push(new JSONObject());
this.comma = false;
return this;
}
throw new JSONException("Misplaced object.");
}
/**
* Pop an array or object scope.
*
* @param c The scope to close.
* @throws JSONException If nesting is wrong.
*/
private void pop(char c) throws JSONException {
if (this.top <= 0) {
throw new JSONException("Nesting error.");
}
char m = this.stack[this.top - 1] == null ? 'a' : 'k';
if (m != c) {
throw new JSONException("Nesting error.");
}
this.top -= 1;
this.mode = this.top == 0 ? 'd' : this.stack[this.top - 1] == null ? 'a' : 'k';
}
/**
* Push an array or object scope.
*
* @param c The scope to open.
* @throws JSONException If nesting is too deep.
*/
private void push(JSONObject jo) throws JSONException {
if (this.top >= maxdepth) {
throw new JSONException("Nesting too deep.");
}
this.stack[this.top] = jo;
this.mode = jo == null ? 'a' : 'k';
this.top += 1;
}
/**
* Append either the value <code>true</code> or the value <code>false</code>.
*
* @param b A boolean.
* @return this
* @throws JSONException
*/
public JSONWriter value(boolean b) throws JSONException {
return this.append(b ? "true" : "false");
}
/**
* Append a double value.
*
* @param d A double.
* @return this
* @throws JSONException If the number is not finite.
*/
public JSONWriter value(double d) throws JSONException {
return this.value(new Double(d));
}
/**
* Append a long value.
*
* @param l A long.
* @return this
* @throws JSONException
*/
public JSONWriter value(long l) throws JSONException {
return this.append(Long.toString(l));
}
/**
* Append an object value.
*
* @param o The object to append. It can be null, or a Boolean, Number, String, JSONObject, or
* JSONArray, or an object with a toJSONString() method.
* @return this
* @throws JSONException If the value is out of sequence.
*/
public JSONWriter value(Object o) throws JSONException {
return this.append(JSONObject.valueToString(o));
}
}

440
client/controller/src/main/java/com/controller/util/json/Test.java Normal file
View File

@@ -0,0 +1,440 @@
package com.controller.util.json;
import java.io.StringWriter;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
/**
* Test class. This file is not formally a member of the com.controller.util.json library. It is
* just a casual test tool.
*/
public class Test {
/**
* Entry point.
*
* @param args
*/
public static void main(String args[]) {
Iterator<?> it;
JSONArray a;
JSONObject j;
JSONStringer jj;
String s;
/**
* Obj is a typical class that implements JSONString. It also provides some beanie methods that
* can be used to construct a JSONObject. It also demonstrates constructing a JSONObject with an
* array of names.
*/
class Obj implements JSONString {
public String aString;
public double aNumber;
public boolean aBoolean;
public Obj(String string, double n, boolean b) {
this.aString = string;
this.aNumber = n;
this.aBoolean = b;
}
public double getNumber() {
return this.aNumber;
}
public String getString() {
return this.aString;
}
public boolean isBoolean() {
return this.aBoolean;
}
public String getBENT() {
return "All uppercase key";
}
public String getX() {
return "x";
}
public String toJSONString() {
return "{"
+ JSONObject.quote(this.aString)
+ ":"
+ JSONObject.doubleToString(this.aNumber)
+ "}";
}
public String toString() {
return this.getString()
+ " "
+ this.getNumber()
+ " "
+ this.isBoolean()
+ "."
+ this.getBENT()
+ " "
+ this.getX();
}
}
Obj obj = new Obj("A beany object", 42, true);
try {
s =
"{ \"entity\": { \"imageURL\": \"\", \"name\": \"IXXXXXXXXXXXXX\", \"id\": 12336, \"ratingCount\": null, \"averageRating\": null } }";
j = new JSONObject(s);
System.out.println(j.toString(2));
jj = new JSONStringer();
s =
jj.object()
.key("single")
.value("MARIE HAA'S")
.key("Johnny")
.value("MARIE HAA\\'S")
.key("foo")
.value("bar")
.key("baz")
.array()
.object()
.key("quux")
.value("Thanks, Josh!")
.endObject()
.endArray()
.key("obj keys")
.value(JSONObject.getNames(obj))
.endObject()
.toString();
System.out.println(s);
System.out.println(
new JSONStringer()
.object()
.key("a")
.array()
.array()
.array()
.value("b")
.endArray()
.endArray()
.endArray()
.endObject()
.toString());
jj = new JSONStringer();
jj.array();
jj.value(1);
jj.array();
jj.value(null);
jj.array();
jj.object();
jj.key("empty-array").array().endArray();
jj.key("answer").value(42);
jj.key("null").value(null);
jj.key("false").value(false);
jj.key("true").value(true);
jj.key("big").value(123456789e+88);
jj.key("small").value(123456789e-88);
jj.key("empty-object").object().endObject();
jj.key("long");
jj.value(9223372036854775807L);
jj.endObject();
jj.value("two");
jj.endArray();
jj.value(true);
jj.endArray();
jj.value(98.6);
jj.value(-100.0);
jj.object();
jj.endObject();
jj.object();
jj.key("one");
jj.value(1.00);
jj.endObject();
jj.value(obj);
jj.endArray();
System.out.println(jj.toString());
System.out.println(new JSONArray(jj.toString()).toString(4));
int ar[] = {1, 2, 3};
JSONArray ja = new JSONArray(ar);
System.out.println(ja.toString());
String sa[] = {"aString", "aNumber", "aBoolean"};
j = new JSONObject(obj, sa);
j.put("Testing JSONString interface", obj);
System.out.println(j.toString(4));
j =
new JSONObject(
"{slashes: '///', closetag: '</script>', backslash:'\\\\', ei: {quotes: '\"\\''},eo: {a: '\"quoted\"', b:\"don't\"}, quotes: [\"'\", '\"']}");
System.out.println(j.toString(2));
System.out.println("");
j =
new JSONObject(
"{foo: [true, false,9876543210, 0.0, 1.00000001, 1.000000000001, 1.00000000000000001,"
+ " .00000000000000001, 2.00, 0.1, 2e100, -32,[],{}, \"string\"], "
+ " to : null, op : 'Good',"
+ "ten:10} postfix comment");
j.put("String", "98.6");
j.put("JSONObject", new JSONObject());
j.put("JSONArray", new JSONArray());
j.put("int", 57);
j.put("double", 123456789012345678901234567890.);
j.put("true", true);
j.put("false", false);
j.put("null", JSONObject.NULL);
j.put("bool", "true");
j.put("zero", -0.0);
j.put("\\u2028", "\u2028");
j.put("\\u2029", "\u2029");
a = j.getJSONArray("foo");
a.put(666);
a.put(2001.99);
a.put("so \"fine\".");
a.put("so <fine>.");
a.put(true);
a.put(false);
a.put(new JSONArray());
a.put(new JSONObject());
j.put("keys", JSONObject.getNames(j));
System.out.println(j.toString(4));
System.out.println("String: " + j.getDouble("String"));
System.out.println(" bool: " + j.getBoolean("bool"));
System.out.println(" to: " + j.getString("to"));
System.out.println(" true: " + j.getString("true"));
System.out.println(" foo: " + j.getJSONArray("foo"));
System.out.println(" op: " + j.getString("op"));
System.out.println(" ten: " + j.getInt("ten"));
System.out.println(" oops: " + j.optBoolean("oops"));
j =
new JSONObject(
"{nix: null, nux: false, null: 'null', 'Request-URI': '/', Method: 'GET', 'HTTP-Version': 'HTTP/1.0'}");
System.out.println(j.toString(2));
System.out.println("isNull: " + j.isNull("nix"));
System.out.println(" has: " + j.has("nix"));
System.out.println("");
j =
new JSONObject(
"{Envelope: {Body: {\"ns1:doGoogleSearch\": {oe: \"latin1\", filter: true, q: \"'+search+'\", key: \"GOOGLEKEY\", maxResults: 10, \"SOAP-ENV:encodingStyle\": \"http://schemas.xmlsoap.org/soap/encoding/\", start: 0, ie: \"latin1\", safeSearch:false, \"xmlns:ns1\": \"urn:GoogleSearch\"}}}}");
System.out.println(j.toString(2));
System.out.println("");
j =
new JSONObject(
"{script: 'It is not allowed in HTML to send a close script tag in a string<script>because it confuses browsers</script>so we insert a backslash before the /'}");
System.out.println(j.toString());
System.out.println("");
JSONTokener jt =
new JSONTokener("{op:'test', to:'session', pre:1}{op:'test', to:'session', pre:2}");
j = new JSONObject(jt);
System.out.println(j.toString());
System.out.println("pre: " + j.optInt("pre"));
int i = jt.skipTo('{');
System.out.println(i);
j = new JSONObject(jt);
System.out.println(j.toString());
System.out.println("");
a = new JSONArray(" [\"<escape>\", next is an implied null , , ok,] ");
System.out.println(a.toString());
System.out.println("");
System.out.println("");
j =
new JSONObject(
"{ fun => with non-standard forms ; forgiving => This package can be used to parse formats that are similar to but not stricting conforming to JSON; why=To make it easier to migrate existing data to JSON,one = [[1.00]]; uno=[[{1=>1}]];'+':+6e66 ;pluses=+++;empty = '' , 'double':0.666,true: TRUE, false: FALSE, null=NULL;[true] = [[!,@;*]]; string=> o. k. ; \r oct=0666; hex=0x666; dec=666; o=0999; noh=0x0x}");
System.out.println(j.toString(4));
System.out.println("");
if (j.getBoolean("true") && !j.getBoolean("false")) {
System.out.println("It's all good");
}
System.out.println("");
j = new JSONObject(j, new String[] {"dec", "oct", "hex", "missing"});
System.out.println(j.toString(4));
System.out.println("");
System.out.println(new JSONStringer().array().value(a).value(j).endArray());
j =
new JSONObject(
"{string: \"98.6\", long: 2147483648, int: 2147483647, longer: 9223372036854775807, double: 9223372036854775808}");
System.out.println(j.toString(4));
System.out.println("\ngetInt");
System.out.println("int " + j.getInt("int"));
System.out.println("long " + j.getInt("long"));
System.out.println("longer " + j.getInt("longer"));
System.out.println("double " + j.getInt("double"));
System.out.println("string " + j.getInt("string"));
System.out.println("\ngetLong");
System.out.println("int " + j.getLong("int"));
System.out.println("long " + j.getLong("long"));
System.out.println("longer " + j.getLong("longer"));
System.out.println("double " + j.getLong("double"));
System.out.println("string " + j.getLong("string"));
System.out.println("\ngetDouble");
System.out.println("int " + j.getDouble("int"));
System.out.println("long " + j.getDouble("long"));
System.out.println("longer " + j.getDouble("longer"));
System.out.println("double " + j.getDouble("double"));
System.out.println("string " + j.getDouble("string"));
j.put("good sized", 9223372036854775807L);
System.out.println(j.toString(4));
a = new JSONArray("[2147483647, 2147483648, 9223372036854775807, 9223372036854775808]");
System.out.println(a.toString(4));
System.out.println("\nKeys: ");
it = j.keys();
while (it.hasNext()) {
s = (String) it.next();
System.out.println(s + ": " + j.getString(s));
}
System.out.println("\naccumulate: ");
j = new JSONObject();
j.accumulate("stooge", "Curly");
j.accumulate("stooge", "Larry");
j.accumulate("stooge", "Moe");
a = j.getJSONArray("stooge");
a.put(5, "Shemp");
System.out.println(j.toString(4));
System.out.println("\nwrite:");
System.out.println(j.write(new StringWriter()));
Collection<?> c = null;
Map<?, ?> m = null;
j = new JSONObject(m);
a = new JSONArray(c);
j.append("stooge", "Joe DeRita");
j.append("stooge", "Shemp");
j.accumulate("stooges", "Curly");
j.accumulate("stooges", "Larry");
j.accumulate("stooges", "Moe");
j.accumulate("stoogearray", j.get("stooges"));
j.put("map", m);
j.put("collection", c);
j.put("array", a);
a.put(m);
a.put(c);
System.out.println(j.toString(4));
s =
"{plist=Apple; AnimalSmells = { pig = piggish; lamb = lambish; worm = wormy; }; AnimalSounds = { pig = oink; lamb = baa; worm = baa; Lisa = \"Why is the worm talking like a lamb?\" } ; AnimalColors = { pig = pink; lamb = black; worm = pink; } } ";
j = new JSONObject(s);
System.out.println(j.toString(4));
s = " (\"San Francisco\", \"New York\", \"Seoul\", \"London\", \"Seattle\", \"Shanghai\")";
a = new JSONArray(s);
System.out.println(a.toString());
System.out.println("\nTesting Exceptions: ");
System.out.print("Exception: ");
try {
a = new JSONArray();
a.put(Double.NEGATIVE_INFINITY);
a.put(Double.NaN);
System.out.println(a.toString());
} catch (Exception e) {
System.out.println(e);
}
System.out.print("Exception: ");
try {
System.out.println(j.getDouble("stooge"));
} catch (Exception e) {
System.out.println(e);
}
System.out.print("Exception: ");
try {
System.out.println(j.getDouble("howard"));
} catch (Exception e) {
System.out.println(e);
}
System.out.print("Exception: ");
try {
System.out.println(j.put(null, "howard"));
} catch (Exception e) {
System.out.println(e);
}
System.out.print("Exception: ");
try {
System.out.println(a.getDouble(0));
} catch (Exception e) {
System.out.println(e);
}
System.out.print("Exception: ");
try {
System.out.println(a.get(-1));
} catch (Exception e) {
System.out.println(e);
}
System.out.print("Exception: ");
try {
System.out.println(a.put(Double.NaN));
} catch (Exception e) {
System.out.println(e);
}
System.out.print("Exception: ");
try {
ja = new JSONArray(new Object());
System.out.println(ja.toString());
} catch (Exception e) {
System.out.println(e);
}
System.out.print("Exception: ");
try {
s = "[)";
a = new JSONArray(s);
System.out.println(a.toString());
} catch (Exception e) {
System.out.println(e);
}
System.out.print("Exception: ");
try {
s = "{\"koda\": true, \"koda\": true}";
j = new JSONObject(s);
System.out.println(j.toString(4));
} catch (Exception e) {
System.out.println(e);
}
System.out.print("Exception: ");
try {
jj = new JSONStringer();
s =
jj.object()
.key("bosanda")
.value("MARIE HAA'S")
.key("bosanda")
.value("MARIE HAA\\'S")
.endObject()
.toString();
System.out.println(j.toString(4));
} catch (Exception e) {
System.out.println(e);
}
} catch (Exception e) {
System.out.println(e.toString());
}
}
}

50
client/controller/src/test/java/com/controller/collectors/AbstractJSONValidationTestCase.java Normal file
View File

@@ -0,0 +1,50 @@
/*
* OtterTune - AbstractJSONValidationTestCase.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
import com.controller.types.JSONSchemaType;
import com.controller.util.FileUtil;
import java.io.File;
import junit.framework.TestCase;
public abstract class AbstractJSONValidationTestCase extends TestCase {
private static final String SAMPLE_OUTPUT_PATH = "sample_output";
private static final String SAMPLE_CONFIG_PATH = "config";
protected String dbName;
protected void setUp(String dbName) throws Exception {
super.setUp();
this.dbName = dbName;
}
public void testJsonKnobs() {
String jsonKnobsPath = FileUtil.joinPath(SAMPLE_OUTPUT_PATH, this.dbName, "knobs.json");
assertTrue(JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, new File(jsonKnobsPath)));
}
public void testJsonMetrics() {
String jsonMetricsBeforePath =
FileUtil.joinPath(SAMPLE_OUTPUT_PATH, this.dbName, "metrics_before.json");
String jsonMetricsAfterPath =
FileUtil.joinPath(SAMPLE_OUTPUT_PATH, this.dbName, "metrics_after.json");
assertTrue(JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, new File(jsonMetricsBeforePath)));
assertTrue(JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, new File(jsonMetricsAfterPath)));
}
public void testJsonSummary() {
String jsonSummaryPath = FileUtil.joinPath(SAMPLE_OUTPUT_PATH, this.dbName, "summary.json");
assertTrue(JSONSchemaType.isValidJson(JSONSchemaType.SUMMARY, new File(jsonSummaryPath)));
}
public void testJsonConfig() {
String jsonConfigPath =
FileUtil.joinPath(SAMPLE_CONFIG_PATH, "sample_" + this.dbName + "_config.json");
assertTrue(JSONSchemaType.isValidJson(JSONSchemaType.CONFIG, new File(jsonConfigPath)));
}
}

42
client/controller/src/test/java/com/controller/collectors/TestInvalidJSON.java Normal file
View File

@@ -0,0 +1,42 @@
/*
* OtterTune - TestInvalidJSON.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
import com.controller.types.JSONSchemaType;
import junit.framework.TestCase;
public class TestInvalidJSON extends TestCase {
// Wrong number of levels for "global"
private static final String BAD_JSON_TEXT_1 =
"{"
+ " \"global\" : {"
+ " \"global\" : {"
+ " \"auto_generate_certs\": {"
+ " \"auto_pram\" : \"NO\""
+ " }"
+ " }"
+ " },"
+ " \"local\" : {"
+ " }"
+ "}";
// Lacking "local"
private static final String BAD_JSON_TEXT_2 =
"{"
+ " \"global\" : {"
+ " \"global1\" : {"
+ " \"auto_generate_certs\": \"ON\""
+ " }"
+ " }"
+ "}";
public void testBadJSONOutput() {
assertFalse(JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, BAD_JSON_TEXT_1));
assertFalse(JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, BAD_JSON_TEXT_2));
}
}

15
client/controller/src/test/java/com/controller/collectors/TestMySQLJSON.java Normal file
View File

@@ -0,0 +1,15 @@
/*
* OtterTune - TestMySQLJSON.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
public class TestMySQLJSON extends AbstractJSONValidationTestCase {
@Override
protected void setUp() throws Exception {
super.setUp("mysql");
}
}

15
client/controller/src/test/java/com/controller/collectors/TestOracleJSON.java Normal file
View File

@@ -0,0 +1,15 @@
/*
* OtterTune - TestOracleJSON.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
public class TestOracleJSON extends AbstractJSONValidationTestCase {
@Override
protected void setUp() throws Exception {
super.setUp("oracle");
}
}

15
client/controller/src/test/java/com/controller/collectors/TestPostgresJSON.java Normal file
View File

@@ -0,0 +1,15 @@
/*
* OtterTune - TestPostgresJSON.java
*
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
*/
package com.controller.collectors;
public class TestPostgresJSON extends AbstractJSONValidationTestCase {
@Override
protected void setUp() throws Exception {
super.setUp("postgres");
}
}

69
client/driver/ConfParser.py Normal file
View File

@@ -0,0 +1,69 @@
#
# OtterTune - ConfParser.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
'''
Created on Mar 23, 2018
@author: Jacky, bohan, Dongsheng
'''
import sys
import json
from collections import OrderedDict
def change_postgres_conf(recommendation, postgresqlconf):
lines = postgresqlconf.readlines()
settings_idx = lines.index("# Add settings for extensions here\n")
postgresqlconf.seek(0)
postgresqlconf.truncate(0)
lines = lines[0:(settings_idx + 1)]
for line in lines:
postgresqlconf.write(line)
for (knob_name, knob_value) in list(recommendation.items()):
postgresqlconf.write(str(knob_name) + " = " + str(knob_value) + "\n")
def change_oracle_conf(recommendation, oracle_conf):
lines = oracle_conf.readlines()
signal = "# configurations recommended by ottertune:\n"
if signal not in lines:
oracle_conf.write('\n' + signal)
oracle_conf.flush()
oracle_conf.seek(0)
lines = oracle_conf.readlines()
settings_idx = lines.index(signal)
oracle_conf.seek(0)
oracle_conf.truncate(0)
lines = lines[0:(settings_idx + 1)]
for line in lines:
oracle_conf.write(line)
for (knob_name, knob_value) in list(recommendation.items()):
oracle_conf.write(str(knob_name) + " = " + str(knob_value).strip('B') + "\n")
def main():
if len(sys.argv) != 4:
raise Exception("Usage: python [DB type] ConfParser.py [Next Config] [Current Config]")
database_type = sys.argv[1]
next_config_name = sys.argv[2]
cur_config_name = sys.argv[3]
with open(next_config_name, 'r') as next_config, open(cur_config_name, 'r+') as cur_config:
config = json.load(next_config, encoding="UTF-8", object_pairs_hook=OrderedDict)
recommendation = config['recommendation']
if database_type == 'postgres':
change_postgres_conf(recommendation, cur_config)
elif database_type == 'oracle':
change_oracle_conf(recommendation, cur_config)
else:
raise Exception("Database Type {} Not Implemented !".format(database_type))
if __name__ == "__main__":
main()

43
client/driver/LatencyUDF.py Normal file
View File

@@ -0,0 +1,43 @@
#
# OtterTune - LatencyUDF.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
import sys
import json
from collections import OrderedDict
def main():
if (len(sys.argv) != 2):
raise Exception("Usage: python udf.py [Output Directory]")
with open(sys.argv[1] + "/summary.json", "r") as f:
conf = json.load(f,
encoding="UTF-8",
object_pairs_hook=OrderedDict)
start_time = conf['start_time']
end_time = conf['end_time']
with open(sys.argv[1] + "/metrics_before.json", "r") as f:
conf_before = json.load(f,
encoding="UTF-8",
object_pairs_hook=OrderedDict)
conf_before['global']['udf'] = OrderedDict([("latency", "0")])
with open(sys.argv[1] + "/metrics_after.json", "r") as f:
conf_after = json.load(f,
encoding="UTF-8",
object_pairs_hook=OrderedDict)
conf_after['global']['udf'] = OrderedDict([("latency", str(end_time - start_time))])
with open(sys.argv[1] + "/metrics_before.json", "w") as f:
f.write(json.dumps(conf_before, indent=4))
with open(sys.argv[1] + "/metrics_after.json", "w") as f:
f.write(json.dumps(conf_after, indent=4))
if __name__ == "__main__":
main()

21
client/driver/driver_config.json Normal file
View File

@@ -0,0 +1,21 @@
{
"database_type" : "postgres",
"database_name" : "tpcc",
"database_disk": "/dev/xvda1",
"database_conf": "/etc/postgresql/9.6/main/postgresql.conf",
"database_save_path": "/home/ubuntu/ottertune",
"username" : "bohan",
"password" : "bohan",
"oltpbench_home": "/home/ubuntu/oltpbench",
"oltpbench_config": "/home/ubuntu/oltpbench/config/tpcc_config_postgres.xml",
"oltpbench_workload": "tpcc",
"oltpbench_log" : "/home/ubuntu/ottertune/client/driver/oltp.log",
"controller_config": "/home/ubuntu/ottertune/client/controller/config/sample_postgres_config.json",
"controller_log" : "/home/ubuntu/ottertune/client/driver/controller.log",
"save_path": "/home/ubuntu/results",
"upload_url" : "http://127.0.0.1:8000",
"upload_code" : "I5I10PXK3PK27FM86YYS",
"lhs_knob_path" : "/home/ubuntu/ottertune/client/driver/knobs/postgres-96.json",
"lhs_save_path" : "/home/ubuntu/ottertune/client/driver/configs",
"oracle_awr_enabled": false
}

421
client/driver/fabfile.py vendored Normal file
View File

@@ -0,0 +1,421 @@
#
# OtterTune - fabfile.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
'''
Created on Mar 23, 2018
@author: bohan
'''
import sys
import json
import logging
import time
import os.path
import re
import glob
from multiprocessing import Process
from fabric.api import (env, local, task, lcd)
from fabric.state import output as fabric_output
LOG = logging.getLogger()
LOG.setLevel(logging.DEBUG)
Formatter = logging.Formatter( # pylint: disable=invalid-name
"%(asctime)s [%(levelname)s] %(message)s")
# print the log
ConsoleHandler = logging.StreamHandler(sys.stdout) # pylint: disable=invalid-name
ConsoleHandler.setFormatter(Formatter)
LOG.addHandler(ConsoleHandler)
# Fabric environment settings
env.hosts = ['localhost']
fabric_output.update({
'running': True,
'stdout': True,
})
# intervals of restoring the databse
RELOAD_INTERVAL = 10
# maximum disk usage
MAX_DISK_USAGE = 90
with open('driver_config.json', 'r') as f:
CONF = json.load(f)
@task
def check_disk_usage():
partition = CONF['database_disk']
disk_use = 0
cmd = "df -h {}".format(partition)
out = local(cmd, capture=True).splitlines()[1]
m = re.search('\d+(?=%)', out) # pylint: disable=anomalous-backslash-in-string
if m:
disk_use = int(m.group(0))
LOG.info("Current Disk Usage: %s%s", disk_use, '%')
return disk_use
@task
def check_memory_usage():
cmd = 'free -m -h'
local(cmd)
@task
def restart_database():
if CONF['database_type'] == 'postgres':
cmd = 'sudo service postgresql restart'
elif CONF['database_type'] == 'oracle':
cmd = 'sh oracleScripts/shutdownOracle.sh && sh oracleScripts/startupOracle.sh'
else:
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
local(cmd)
@task
def drop_database():
if CONF['database_type'] == 'postgres':
cmd = "PGPASSWORD={} dropdb -e --if-exists {} -U {}".\
format(CONF['password'], CONF['database_name'], CONF['username'])
else:
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
local(cmd)
@task
def create_database():
if CONF['database_type'] == 'postgres':
cmd = "PGPASSWORD={} createdb -e {} -U {}".\
format(CONF['password'], CONF['database_name'], CONF['username'])
else:
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
local(cmd)
@task
def change_conf():
next_conf = 'next_config'
cmd = "sudo python3 ConfParser.py {} {} {}".\
format(CONF['database_type'], next_conf, CONF['database_conf'])
local(cmd)
@task
def load_oltpbench():
cmd = "./oltpbenchmark -b {} -c {} --create=true --load=true".\
format(CONF['oltpbench_workload'], CONF['oltpbench_config'])
with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager
local(cmd)
@task
def run_oltpbench():
cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile".\
format(CONF['oltpbench_workload'], CONF['oltpbench_config'])
with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager
local(cmd)
@task
def run_oltpbench_bg():
cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile > {} 2>&1 &".\
format(CONF['oltpbench_workload'], CONF['oltpbench_config'], CONF['oltpbench_log'])
with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager
local(cmd)
@task
def run_controller():
cmd = 'sudo gradle run -PappArgs="-c {} -d output/" --no-daemon > {}'.\
format(CONF['controller_config'], CONF['controller_log'])
with lcd("../controller"): # pylint: disable=not-context-manager
local(cmd)
@task
def signal_controller():
pid = int(open('../controller/pid.txt').read())
cmd = 'sudo kill -2 {}'.format(pid)
with lcd("../controller"): # pylint: disable=not-context-manager
local(cmd)
@task
def save_dbms_result():
t = int(time.time())
files = ['knobs.json', 'metrics_after.json', 'metrics_before.json', 'summary.json']
for f_ in files:
f_prefix = f_.split('.')[0]
cmd = 'cp ../controller/output/{} {}/{}__{}.json'.\
format(f_, CONF['save_path'], t, f_prefix)
local(cmd)
@task
def free_cache():
cmd = 'sync; sudo bash -c "echo 1 > /proc/sys/vm/drop_caches"'
local(cmd)
@task
def upload_result():
cmd = 'python3 ../../server/website/script/upload/upload.py \
../controller/output/ {} {}/new_result/'.format(CONF['upload_code'],
CONF['upload_url'])
local(cmd)
@task
def get_result():
cmd = 'python3 ../../script/query_and_get.py {} {} 5'.\
format(CONF['upload_url'], CONF['upload_code'])
local(cmd)
@task
def add_udf():
cmd = 'sudo python3 ./LatencyUDF.py ../controller/output/'
local(cmd)
@task
def upload_batch():
cmd = 'python3 ./upload_batch.py {} {} {}/new_result/'.format(CONF['save_path'],
CONF['upload_code'],
CONF['upload_url'])
local(cmd)
@task
def dump_database():
db_file_path = '{}/{}.dump'.format(CONF['database_save_path'], CONF['database_name'])
if os.path.exists(db_file_path):
LOG.info('%s already exists ! ', db_file_path)
return False
else:
LOG.info('Dump database %s to %s', CONF['database_name'], db_file_path)
# You must create a directory named dpdata through sqlplus in your Oracle database
if CONF['database_type'] == 'oracle':
cmd = 'expdp {}/{}@{} schemas={} dumpfile={}.dump DIRECTORY=dpdata'.format(
'c##tpcc', 'oracle', 'orcldb', 'c##tpcc', 'orcldb')
elif CONF['database_type'] == 'postgres':
cmd = 'PGPASSWORD={} pg_dump -U {} -F c -d {} > {}'.format(CONF['password'],
CONF['username'],
CONF['database_name'],
db_file_path)
else:
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
local(cmd)
return True
@task
def restore_database():
if CONF['database_type'] == 'oracle':
# You must create a directory named dpdata through sqlplus in your Oracle database
# The following script assumes such directory exists.
# You may want to modify the username, password, and dump file name in the script
cmd = 'sh oracleScripts/restoreOracle.sh'
elif CONF['database_type'] == 'postgres':
db_file_path = '{}/{}.dump'.format(CONF['database_save_path'], CONF['database_name'])
drop_database()
create_database()
cmd = 'PGPASSWORD={} pg_restore -U {} -n public -j 8 -F c -d {} {}'.\
format(CONF['password'], CONF['username'], CONF['database_name'], db_file_path)
else:
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
LOG.info('Start restoring database')
local(cmd)
LOG.info('Finish restoring database')
def _ready_to_start_oltpbench():
return (os.path.exists(CONF['controller_log']) and
'Output the process pid to'
in open(CONF['controller_log']).read())
def _ready_to_start_controller():
return (os.path.exists(CONF['oltpbench_log']) and
'Warmup complete, starting measurements'
in open(CONF['oltpbench_log']).read())
def _ready_to_shut_down_controller():
pid_file_path = '../controller/pid.txt'
return (os.path.exists(pid_file_path) and os.path.exists(CONF['oltpbench_log']) and
'Output throughput samples into file' in open(CONF['oltpbench_log']).read())
def clean_logs():
# remove oltpbench log
cmd = 'rm -f {}'.format(CONF['oltpbench_log'])
local(cmd)
# remove controller log
cmd = 'rm -f {}'.format(CONF['controller_log'])
local(cmd)
@task
def lhs_samples(count=10):
cmd = 'python3 lhs.py {} {} {}'.format(count, CONF['lhs_knob_path'], CONF['lhs_save_path'])
local(cmd)
@task
def loop():
# free cache
free_cache()
# remove oltpbench log and controller log
clean_logs()
# restart database
restart_database()
# check disk usage
if check_disk_usage() > MAX_DISK_USAGE:
LOG.WARN('Exceeds max disk usage %s', MAX_DISK_USAGE)
# run controller from another process
p = Process(target=run_controller, args=())
p.start()
LOG.info('Run the controller')
# run oltpbench as a background job
while not _ready_to_start_oltpbench():
pass
run_oltpbench_bg()
LOG.info('Run OLTP-Bench')
# the controller starts the first collection
while not _ready_to_start_controller():
pass
signal_controller()
LOG.info('Start the first collection')
# stop the experiment
while not _ready_to_shut_down_controller():
pass
signal_controller()
LOG.info('Start the second collection, shut down the controller')
p.join()
# add user defined target objective
# add_udf()
# save result
save_dbms_result()
# upload result
upload_result()
# get result
get_result()
# change config
change_conf()
@task
def run_lhs():
datadir = CONF['lhs_save_path']
samples = glob.glob(os.path.join(datadir, 'config_*'))
# dump database if it's not done before.
dump = dump_database()
for i, sample in enumerate(samples):
# reload database periodically
if RELOAD_INTERVAL > 0:
if i % RELOAD_INTERVAL == 0:
if i == 0 and dump is False:
restore_database()
elif i > 0:
restore_database()
# free cache
free_cache()
LOG.info('\n\n Start %s-th sample %s \n\n', i, sample)
# check memory usage
# check_memory_usage()
# check disk usage
if check_disk_usage() > MAX_DISK_USAGE:
LOG.WARN('Exceeds max disk usage %s', MAX_DISK_USAGE)
# copy lhs-sampled config to the to-be-used config
cmd = 'cp {} next_config'.format(sample)
local(cmd)
# remove oltpbench log and controller log
clean_logs()
# change config
change_conf()
# restart database
restart_database()
if CONF.get('oracle_awr_enabled', False):
# create snapshot for oracle AWR report
if CONF['database_type'] == 'oracle':
local('sh snapshotOracle.sh')
# run controller from another process
p = Process(target=run_controller, args=())
p.start()
# run oltpbench as a background job
while not _ready_to_start_oltpbench():
pass
run_oltpbench_bg()
LOG.info('Run OLTP-Bench')
while not _ready_to_start_controller():
pass
signal_controller()
LOG.info('Start the first collection')
while not _ready_to_shut_down_controller():
pass
# stop the experiment
signal_controller()
LOG.info('Start the second collection, shut down the controller')
p.join()
# save result
save_dbms_result()
# upload result
upload_result()
if CONF.get('oracle_awr_enabled', False):
# create oracle AWR report for performance analysis
if CONF['database_type'] == 'oracle':
local('sh oracleScripts/snapshotOracle.sh && sh oracleScripts/awrOracle.sh')
@task
def run_loops(max_iter=1):
# dump database if it's not done before.
dump = dump_database()
for i in range(int(max_iter)):
if RELOAD_INTERVAL > 0:
if i % RELOAD_INTERVAL == 0:
if i == 0 and dump is False:
restore_database()
elif i > 0:
restore_database()
LOG.info('The %s-th Loop Starts / Total Loops %s', i + 1, max_iter)
loop()
LOG.info('The %s-th Loop Ends / Total Loops %s', i + 1, max_iter)

83
client/driver/knobs/oracle.json Normal file
View File

@@ -0,0 +1,83 @@
[
{
"name": "SHARED_POOL_SIZE",
"tuning_range": {
"minval": "500MB",
"maxval": "2500MB"
},
"default": "1500MB",
"type": "bytes"
},
{
"name": "DB_CACHE_SIZE",
"tuning_range": {
"minval": "10GB",
"maxval": "24GB"
},
"default": "14GB",
"type": "bytes"
},
{
"name": "LOG_BUFFER",
"tuning_range": {
"minval": "10MB",
"maxval": "1000MB"
},
"default": "20MB",
"type": "bytes"
},
{
"name": "LARGE_POOL_SIZE",
"tuning_range": {
"minval": "10MB",
"maxval": "1000MB"
},
"default": "100MB",
"type": "bytes"
},
{
"name": "STREAMS_POOL_SIZE",
"tuning_range": {
"minval": "10MB",
"maxval": "1000MB"
},
"default": "100MB",
"type": "bytes"
},
{
"name": "bitmap_merge_area_size",
"tuning_range": {
"minval": "1000000",
"maxval": "20000000"
},
"default": "1MB",
"type": "integer"
},
{
"name": "create_bitmap_area_size",
"tuning_range": {
"minval": "1000000",
"maxval": "100000000"
},
"default": "8MB",
"type": "integer"
},
{
"name": "hash_area_size",
"tuning_range": {
"minval": "65536",
"maxval": "1000000"
},
"default": "65536",
"type": "integer"
},
{
"name": "sort_area_size",
"tuning_range": {
"minval": "128000",
"maxval": "2000000"
},
"default": "128000",
"type": "integer"
}
]

110
client/driver/knobs/postgres-96.json Normal file
View File

@@ -0,0 +1,110 @@
[
{
"name": "effective_cache_size",
"tuning_range": {
"minval": "4GB",
"maxval": "16GB"
},
"default": "4GB",
"type": "bytes"
},
{
"name": "shared_buffers",
"tuning_range": {
"minval": "128MB",
"maxval": "12GB"
},
"default": "128MB",
"type": "bytes"
},
{
"name": "max_parallel_workers_per_gather",
"tuning_range": {
"minval": 0,
"maxval": 8
},
"default": 0,
"type": "integer"
},
{
"name": "default_statistics_target",
"tuning_range": {
"minval": 100,
"maxval": 2048
},
"default": 100,
"type": "integer"
},
{
"name": "bgwriter_lru_maxpages",
"tuning_range": {
"minval": 0,
"maxval": 1000
},
"default": 10,
"type": "integer"
},
{
"name": "checkpoint_timeout",
"tuning_range": {
"minval": "10ms",
"maxval": "1min"
},
"default": "200ms",
"type": "time"
},
{
"name": "random_page_cost",
"tuning_range": {
"minval": 1,
"maxval": 10
},
"default": 4.0,
"type": "float"
},
{
"name": "checkpoint_completion_target",
"tuning_range": {
"minval": 0.1,
"maxval": 0.9
},
"default": 0.5,
"type": "float"
},
{
"name": "checkpoint_timeout",
"tuning_range": {
"minval": "1min",
"maxval": "30min"
},
"default": "5min",
"type": "time"
},
{
"name": "max_wal_size",
"tuning_range": {
"minval": "256MB",
"maxval": "16GB"
},
"default": "1GB",
"type": "bytes"
},
{
"name": "temp_buffers",
"tuning_range": {
"minval": "8MB",
"maxval": "1GB"
},
"default": "8MB",
"type": "bytes"
},
{
"name": "work_mem",
"tuning_range": {
"minval": "4MB",
"maxval": "1GB"
},
"default": "4MB",
"type": "bytes"
}
]

126
client/driver/lhs.py Normal file
View File

@@ -0,0 +1,126 @@
#
# OtterTune - lhs.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
import sys
import json
import os
import numpy as np
from pyDOE import lhs
from scipy.stats import uniform
from hurry.filesize import size
BYTES_SYSTEM = [
(1024 ** 5, 'PB'),
(1024 ** 4, 'TB'),
(1024 ** 3, 'GB'),
(1024 ** 2, 'MB'),
(1024 ** 1, 'kB'),
(1024 ** 0, 'B'),
]
TIME_SYSTEM = [
(1000 * 60 * 60 * 24, 'd'),
(1000 * 60 * 60, 'h'),
(1000 * 60, 'min'),
(1000, 's'),
(1, 'ms'),
]
def get_raw_size(value, system):
for factor, suffix in system:
if value.endswith(suffix):
if len(value) == len(suffix):
amount = 1
else:
try:
amount = int(value[:-len(suffix)])
except ValueError:
continue
return amount * factor
return None
def get_knob_raw(value, knob_type):
if knob_type == 'integer':
return int(value)
elif knob_type == 'float':
return float(value)
elif knob_type == 'bytes':
return get_raw_size(value, BYTES_SYSTEM)
elif knob_type == 'time':
return get_raw_size(value, TIME_SYSTEM)
else:
raise Exception('Knob Type does not support')
def get_knob_readable(value, knob_type):
if knob_type == 'integer':
return int(round(value))
elif knob_type == 'float':
return float(value)
elif knob_type == 'bytes':
value = int(round(value))
return size(value, system=BYTES_SYSTEM)
elif knob_type == 'time':
value = int(round(value))
return size(value, system=TIME_SYSTEM)
else:
raise Exception('Knob Type does not support')
def get_knobs_readable(values, types):
result = []
for i, value in enumerate(values):
result.append(get_knob_readable(value, types[i]))
return result
def main(args):
if (len(sys.argv) != 4):
raise Exception("Usage: python3 lhs.py [Samples Count] [Knob Path] [Save Path]")
knob_path = args[2]
save_path = args[3]
with open(knob_path, "r") as f:
tuning_knobs = json.load(f)
names = []
maxvals = []
minvals = []
types = []
for knob in tuning_knobs:
names.append(knob['name'])
maxvals.append(get_knob_raw(knob['tuning_range']['maxval'], knob['type']))
minvals.append(get_knob_raw(knob['tuning_range']['minval'], knob['type']))
types.append(knob['type'])
nsamples = int(args[1])
nfeats = len(tuning_knobs)
samples = lhs(nfeats, samples=nsamples, criterion='maximin')
maxvals = np.array(maxvals)
minvals = np.array(minvals)
scales = maxvals - minvals
for fidx in range(nfeats):
samples[:, fidx] = uniform(loc=minvals[fidx], scale=scales[fidx]).ppf(samples[:, fidx])
samples_readable = []
for sample in samples:
samples_readable.append(get_knobs_readable(sample, types))
config = {'recommendation': {}}
for sidx in range(nsamples):
for fidx in range(nfeats):
config["recommendation"][names[fidx]] = samples_readable[sidx][fidx]
with open(os.path.join(save_path, 'config_' + str(sidx)), 'w+') as f:
f.write(json.dumps(config))
if __name__ == '__main__':
main(sys.argv)

1
client/driver/lhs.sh Executable file
View File

@@ -0,0 +1 @@
sudo -b nohup fab run_lhs > lhs.log 2>&1

9
client/driver/oracleScripts/awrOracle.sh Normal file
View File

@@ -0,0 +1,9 @@
#!/bin/sh
su - oracle <<EON
oracle
sqlplus / as sysdba <<EOF
@/home/oracle/ottertune/client/driver/autoawr.sql;
quit
EOF
exit
EON

23
client/driver/oracleScripts/restoreOracle.sh Normal file
View File

@@ -0,0 +1,23 @@
#!/bin/sh
su - oracle <<EON
oracle #system password
sqlplus / as sysdba <<EOF
drop user c##tpcc cascade;
# username
create user c##tpcc identified by oracle;
# username password
quit
EOF
impdp 'userid="/ as sysdba"' schemas=c##tpcc dumpfile=orcldb.dump DIRECTORY=dpdata
# username database_name db_directory
sqlplus / as sysdba <<EOF #restart the database
shutdown immediate
startup
quit
EOF
exit
EON

11
client/driver/oracleScripts/shutdownOracle.sh Normal file
View File

@@ -0,0 +1,11 @@
#!/bin/sh
su - oracle <<EON
oracle
sqlplus / as sysdba <<EOF
shutdown immediate
exit
EOF
exit
EON

11
client/driver/oracleScripts/snapshotOracle.sh Normal file
View File

@@ -0,0 +1,11 @@
#!/bin/sh
su - oracle <<EON
oracle
sqlplus / as sysdba <<EOF
exec dbms_workload_repository.create_snapshot;
quit
EOF
exit
EON

11
client/driver/oracleScripts/startupOracle.sh Normal file
View File

@@ -0,0 +1,11 @@
#!/bin/sh
su - oracle <<EON
oracle
sqlplus / as sysdba <<EOF
startup
quit
EOF
exit
EON

63
client/driver/upload_batch.py Normal file
View File

@@ -0,0 +1,63 @@
#
# OtterTune - upload_batch.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
import argparse
import glob
import logging
import os
import requests
# Logging
LOG = logging.getLogger(__name__)
LOG.addHandler(logging.StreamHandler())
LOG.setLevel(logging.INFO)
# Upload all files in the datadir to OtterTune's server side.
# You may want to upload your training data to the non-tuning session.
def upload_batch(datadir, upload_code, url):
samples = glob.glob(os.path.join(datadir, '*__summary.json'))
count = len(samples)
samples_prefix = []
LOG.info('Uploading %d samples in %s...', count, datadir)
for sample in samples:
prefix = sample.split('/')[-1].split('__')[0]
samples_prefix.append(prefix)
for i in range(count):
prefix = samples_prefix[i]
params = {
'summary': open(os.path.join(datadir, '{}__summary.json'.format(prefix)), 'rb'),
'knobs': open(os.path.join(datadir, '{}__knobs.json'.format(prefix)), 'rb'),
'metrics_before': open(os.path.join(datadir,
'{}__metrics_before.json'.format(prefix)), 'rb'),
'metrics_after': open(os.path.join(datadir,
'{}__metrics_after.json'.format(prefix)), 'rb'),
}
LOG.info('Upload %d-th sample %s__*.json', i + 1, prefix)
response = requests.post(url,
files=params,
data={'upload_code': upload_code})
LOG.info(response.content)
def main():
parser = argparse.ArgumentParser(description="Upload generated data to the website")
parser.add_argument('datadir', type=str, nargs=1,
help='Directory containing the generated data')
parser.add_argument('upload_code', type=str, nargs=1,
help='The website\'s upload code')
parser.add_argument('url', type=str, default='http://0.0.0.0:8000/new_result/',
nargs='?', help='The upload url: server_ip/new_result/')
args = parser.parse_args()
upload_batch(args.datadir[0], args.upload_code[0], args.url)
if __name__ == "__main__":
main()