public/ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit with BSL

Browse Source
This commit is contained in:
Andy Pavlo
2019-08-23 11:47:19 -04:00
commit 3e564ce922
286 changed files with 177642 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/website/script/controller_simulator/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
generated_data

132
server/website/script/controller_simulator/data_generator.py Normal file
View File

@@ -0,0 +1,132 @@
#
# OtterTune - data_generator.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
'''
Created on Nov 30, 2017
@author: dvanaken
'''
import copy
import datetime
import logging
import os
import shutil
import sys
import json
import numpy as np
LOG = logging.getLogger(__name__)
# Data generator configuration
OBSERVATION_TIME_SEC = 300 # 5 minutes
START_TIME = datetime.datetime.now() - datetime.timedelta(weeks=1)
START_FREQUENCY = datetime.timedelta(minutes=10)
END_FREQUENCY = datetime.timedelta(seconds=OBSERVATION_TIME_SEC)
EPOCH = datetime.datetime.utcfromtimestamp(0)
# Paths
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
SAMPLE_DIR = os.path.join(ROOT_DIR, 'samples')
OUTPUT_DIR = os.path.join(ROOT_DIR, 'generated_data')
def unix_time_millis(dt):
return int((dt - EPOCH).total_seconds() * 1000.0)
def generate_data(n_workloads, n_samples_per_workload):
with open(os.path.join(SAMPLE_DIR, 'knobs.json'), 'r') as f:
knob_sample = json.load(f)
with open(os.path.join(SAMPLE_DIR, 'metrics_before.json'), 'r') as f:
metrics_start_sample = json.load(f)
with open(os.path.join(SAMPLE_DIR, 'metrics_after.json'), 'r') as f:
metrics_end_sample = json.load(f)
with open(os.path.join(SAMPLE_DIR, 'summary.json'), 'r') as f:
summary_sample = json.load(f)
start_time = START_TIME
end_time = START_TIME + END_FREQUENCY
for i in range(n_workloads):
workload_name = 'workload-{}'.format(i)
wkld_dir = os.path.join(OUTPUT_DIR, workload_name)
os.mkdir(wkld_dir)
for j in range(n_samples_per_workload):
knob_data = copy.deepcopy(knob_sample)
metrics_start_data = copy.deepcopy(metrics_start_sample)
metrics_end_data = copy.deepcopy(metrics_end_sample)
summary_data = copy.deepcopy(summary_sample)
summary_data['workload_name'] = workload_name
summary_data['observation_time'] = OBSERVATION_TIME_SEC
summary_data['start_time'] = unix_time_millis(start_time)
summary_data['end_time'] = unix_time_millis(end_time)
start_time = start_time + START_FREQUENCY
end_time = start_time + END_FREQUENCY
knob_vals = np.random.randint(1, 11, 4)
global_knobs = knob_data['global']['global']
global_knobs['shared_buffers'] = str(knob_vals[0]) + 'GB'
global_knobs['work_mem'] = str(knob_vals[1]) + 'GB'
global_knobs['checkpoint_timing'] = str(knob_vals[2]) + 'min'
global_knobs['effective_io_concurrency'] = str(knob_vals[3])
metrics_start_data['global']['pg_stat_bgwriter']['buffers_alloc'] = np.random.randint(
3000, 7000)
metrics_end_data['global']['pg_stat_bgwriter']['buffers_alloc'] = np.random.randint(
7000, 10000)
locations = [
('xact_commit', metrics_start_data['local']['database']['pg_stat_database']),
('xact_commit', metrics_end_data['local']['database']['pg_stat_database']),
('n_tup_ins', metrics_start_data['local']['table']['pg_stat_user_tables']),
('n_tup_ins', metrics_end_data['local']['table']['pg_stat_user_tables']),
('idx_blks_hit', metrics_start_data['local']['indexes']['pg_statio_user_indexes']),
('idx_blks_hit', metrics_end_data['local']['indexes']['pg_statio_user_indexes']),
]
for k, (name, loc) in enumerate(locations):
for kvs in list(loc.values()):
if k % 2 == 0: # start time must be smaller value
met_val = np.random.randint(30000, 70000)
else:
met_val = np.random.randint(70000, 100000)
kvs[name] = met_val
basepath = os.path.join(wkld_dir, 'sample-{}'.format(j))
with open(basepath + "__knobs.json", 'w') as f:
json.dump(knob_data, f, indent=4)
with open(basepath + '__metrics_start.json', 'w') as f:
json.dump(metrics_start_data, f, indent=4)
with open(basepath + '__metrics_end.json', 'w') as f:
json.dump(metrics_end_data, f, indent=4)
with open(basepath + '__summary.json', 'w') as f:
json.dump(summary_data, f, indent=4)
def main():
if len(sys.argv) < 3:
LOG.error('Usage: python data_generator.py [n_workloads] [n_samples_per_workload] '
'[optional: random_seed]')
sys.exit(1)
if len(sys.argv) == 4:
random_seed = int(sys.argv[3])
LOG.info("Seeding the generator with value: %d", random_seed)
np.random.seed(seed=random_seed)
shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
os.mkdir(OUTPUT_DIR)
generate_data(int(sys.argv[1]), int(sys.argv[2]))
LOG.info("Finished. Generated data written to %s.", OUTPUT_DIR)
if __name__ == "__main__":
main()

274
server/website/script/controller_simulator/samples/knobs.json Normal file
View File

@@ -0,0 +1,274 @@
{
"global": {"global": {
"DateStyle": "ISO, MDY",
"IntervalStyle": "postgres",
"TimeZone": "America/New_York",
"allow_system_table_mods": "off",
"application_name": "",
"archive_command": "(disabled)",
"archive_mode": "off",
"archive_timeout": "0",
"array_nulls": "on",
"authentication_timeout": "1min",
"autovacuum": "on",
"autovacuum_analyze_scale_factor": "0.1",
"autovacuum_analyze_threshold": "50",
"autovacuum_freeze_max_age": "200000000",
"autovacuum_max_workers": "3",
"autovacuum_multixact_freeze_max_age": "400000000",
"autovacuum_naptime": "1min",
"autovacuum_vacuum_cost_delay": "20ms",
"autovacuum_vacuum_cost_limit": "-1",
"autovacuum_vacuum_scale_factor": "0.2",
"autovacuum_vacuum_threshold": "50",
"autovacuum_work_mem": "-1",
"backend_flush_after": "0",
"backslash_quote": "safe_encoding",
"bgwriter_delay": "200ms",
"bgwriter_flush_after": "0",
"bgwriter_lru_maxpages": "100",
"bgwriter_lru_multiplier": "2",
"block_size": "8192",
"bonjour": "off",
"bonjour_name": "",
"bytea_output": "hex",
"check_function_bodies": "on",
"checkpoint_completion_target": "0.5",
"checkpoint_flush_after": "0",
"checkpoint_timeout": "5min",
"checkpoint_warning": "30s",
"client_encoding": "UTF8",
"client_min_messages": "notice",
"cluster_name": "",
"commit_delay": "0",
"commit_siblings": "5",
"config_file": "/Users/MacadamiaKitten/Desktop/psql_db/postgresql.conf",
"constraint_exclusion": "partition",
"cpu_index_tuple_cost": "0.005",
"cpu_operator_cost": "0.0025",
"cpu_tuple_cost": "0.01",
"cursor_tuple_fraction": "0.1",
"data_checksums": "off",
"data_directory": "/Users/MacadamiaKitten/Desktop/psql_db",
"db_user_namespace": "off",
"deadlock_timeout": "1s",
"debug_assertions": "off",
"debug_pretty_print": "on",
"debug_print_parse": "off",
"debug_print_plan": "off",
"debug_print_rewritten": "off",
"default_statistics_target": "100",
"default_tablespace": "",
"default_text_search_config": "pg_catalog.english",
"default_transaction_deferrable": "off",
"default_transaction_isolation": "read committed",
"default_transaction_read_only": "off",
"default_with_oids": "off",
"dynamic_library_path": "$libdir",
"dynamic_shared_memory_type": "posix",
"effective_cache_size": "4GB",
"effective_io_concurrency": "0",
"enable_bitmapscan": "on",
"enable_gathermerge": "on",
"enable_hashagg": "on",
"enable_hashjoin": "on",
"enable_indexonlyscan": "on",
"enable_indexscan": "on",
"enable_material": "on",
"enable_mergejoin": "on",
"enable_nestloop": "on",
"enable_seqscan": "on",
"enable_sort": "on",
"enable_tidscan": "on",
"escape_string_warning": "on",
"event_source": "PostgreSQL",
"exit_on_error": "off",
"external_pid_file": "",
"extra_float_digits": "3",
"force_parallel_mode": "off",
"from_collapse_limit": "8",
"fsync": "on",
"full_page_writes": "on",
"geqo": "on",
"geqo_effort": "5",
"geqo_generations": "0",
"geqo_pool_size": "0",
"geqo_seed": "0",
"geqo_selection_bias": "2",
"geqo_threshold": "12",
"gin_fuzzy_search_limit": "0",
"gin_pending_list_limit": "4MB",
"hba_file": "/Users/MacadamiaKitten/Desktop/psql_db/pg_hba.conf",
"hot_standby": "on",
"hot_standby_feedback": "off",
"huge_pages": "try",
"ident_file": "/Users/MacadamiaKitten/Desktop/psql_db/pg_ident.conf",
"idle_in_transaction_session_timeout": "0",
"ignore_checksum_failure": "off",
"ignore_system_indexes": "off",
"integer_datetimes": "on",
"join_collapse_limit": "8",
"krb_caseins_users": "off",
"krb_server_keyfile": "FILE:/usr/local/etc/postgresql/krb5.keytab",
"lc_collate": "en_US.UTF-8",
"lc_ctype": "en_US.UTF-8",
"lc_messages": "en_US.UTF-8",
"lc_monetary": "en_US.UTF-8",
"lc_numeric": "en_US.UTF-8",
"lc_time": "en_US.UTF-8",
"listen_addresses": "localhost",
"lo_compat_privileges": "off",
"local_preload_libraries": "",
"lock_timeout": "0",
"log_autovacuum_min_duration": "-1",
"log_checkpoints": "off",
"log_connections": "off",
"log_destination": "stderr",
"log_directory": "log",
"log_disconnections": "off",
"log_duration": "off",
"log_error_verbosity": "default",
"log_executor_stats": "off",
"log_file_mode": "0600",
"log_filename": "postgresql-%Y-%m-%d_%H%M%S.log",
"log_hostname": "off",
"log_line_prefix": "%m [%p] ",
"log_lock_waits": "off",
"log_min_duration_statement": "-1",
"log_min_error_statement": "error",
"log_min_messages": "warning",
"log_parser_stats": "off",
"log_planner_stats": "off",
"log_replication_commands": "off",
"log_rotation_age": "1d",
"log_rotation_size": "10MB",
"log_statement": "none",
"log_statement_stats": "off",
"log_temp_files": "-1",
"log_timezone": "US/Eastern",
"log_truncate_on_rotation": "off",
"logging_collector": "off",
"maintenance_work_mem": "64MB",
"max_connections": "100",
"max_files_per_process": "1000",
"max_function_args": "100",
"max_identifier_length": "63",
"max_index_keys": "32",
"max_locks_per_transaction": "64",
"max_logical_replication_workers": "4",
"max_parallel_workers": "8",
"max_parallel_workers_per_gather": "2",
"max_pred_locks_per_page": "2",
"max_pred_locks_per_relation": "-2",
"max_pred_locks_per_transaction": "64",
"max_prepared_transactions": "0",
"max_replication_slots": "10",
"max_stack_depth": "2MB",
"max_standby_archive_delay": "30s",
"max_standby_streaming_delay": "30s",
"max_sync_workers_per_subscription": "2",
"max_wal_senders": "10",
"max_wal_size": "1GB",
"max_worker_processes": "8",
"min_parallel_index_scan_size": "512kB",
"min_parallel_table_scan_size": "8MB",
"min_wal_size": "80MB",
"old_snapshot_threshold": "-1",
"operator_precedence_warning": "off",
"parallel_setup_cost": "1000",
"parallel_tuple_cost": "0.1",
"password_encryption": "md5",
"port": "5432",
"post_auth_delay": "0",
"pre_auth_delay": "0",
"quote_all_identifiers": "off",
"random_page_cost": "4",
"replacement_sort_tuples": "150000",
"restart_after_crash": "on",
"row_security": "on",
"search_path": "\"$user\", public",
"segment_size": "1GB",
"seq_page_cost": "1",
"server_encoding": "UTF8",
"server_version": "10.1",
"server_version_num": "100001",
"session_preload_libraries": "",
"session_replication_role": "origin",
"shared_buffers": "128MB",
"shared_preload_libraries": "",
"ssl": "off",
"ssl_ca_file": "",
"ssl_cert_file": "server.crt",
"ssl_ciphers": "HIGH:MEDIUM:+3DES:!aNULL",
"ssl_crl_file": "",
"ssl_dh_params_file": "",
"ssl_ecdh_curve": "prime256v1",
"ssl_key_file": "server.key",
"ssl_prefer_server_ciphers": "on",
"standard_conforming_strings": "on",
"statement_timeout": "0",
"stats_temp_directory": "pg_stat_tmp",
"superuser_reserved_connections": "3",
"synchronize_seqscans": "on",
"synchronous_commit": "on",
"synchronous_standby_names": "",
"syslog_facility": "local0",
"syslog_ident": "postgres",
"syslog_sequence_numbers": "on",
"syslog_split_messages": "on",
"tcp_keepalives_count": "8",
"tcp_keepalives_idle": "7200",
"tcp_keepalives_interval": "75",
"temp_buffers": "8MB",
"temp_file_limit": "-1",
"temp_tablespaces": "",
"timezone_abbreviations": "Default",
"trace_notify": "off",
"trace_recovery_messages": "log",
"trace_sort": "off",
"track_activities": "on",
"track_activity_query_size": "1024",
"track_commit_timestamp": "off",
"track_counts": "on",
"track_functions": "none",
"track_io_timing": "off",
"transaction_deferrable": "off",
"transaction_isolation": "read committed",
"transaction_read_only": "off",
"transform_null_equals": "off",
"unix_socket_directories": "/tmp",
"unix_socket_group": "",
"unix_socket_permissions": "0777",
"update_process_title": "on",
"vacuum_cost_delay": "0",
"vacuum_cost_limit": "200",
"vacuum_cost_page_dirty": "20",
"vacuum_cost_page_hit": "1",
"vacuum_cost_page_miss": "10",
"vacuum_defer_cleanup_age": "0",
"vacuum_freeze_min_age": "50000000",
"vacuum_freeze_table_age": "150000000",
"vacuum_multixact_freeze_min_age": "5000000",
"vacuum_multixact_freeze_table_age": "150000000",
"wal_block_size": "8192",
"wal_buffers": "4MB",
"wal_compression": "off",
"wal_consistency_checking": "",
"wal_keep_segments": "0",
"wal_level": "replica",
"wal_log_hints": "off",
"wal_receiver_status_interval": "10s",
"wal_receiver_timeout": "1min",
"wal_retrieve_retry_interval": "5s",
"wal_segment_size": "16MB",
"wal_sender_timeout": "1min",
"wal_sync_method": "open_datasync",
"wal_writer_delay": "200ms",
"wal_writer_flush_after": "1MB",
"work_mem": "4MB",
"xmlbinary": "base64",
"xmloption": "content",
"zero_damaged_pages": "off"
}},
"local": null
}

582
server/website/script/controller_simulator/samples/metrics_after.json Normal file
View File

@@ -0,0 +1,582 @@
{
"global": {
"pg_stat_archiver": {
"archived_count": "0",
"failed_count": "0",
"stats_reset": "2017-11-10 10:59:47.397075-05"
},
"pg_stat_bgwriter": {
"buffers_alloc": "87670",
"buffers_backend": "81032",
"buffers_backend_fsync": "0",
"buffers_checkpoint": "33250",
"buffers_clean": "49590",
"checkpoint_sync_time": "19",
"checkpoint_write_time": "597851",
"checkpoints_req": "2",
"checkpoints_timed": "1277",
"maxwritten_clean": "325",
"stats_reset": "2017-11-10 10:59:47.397075-05"
}
},
"local": {
"table": {
"pg_stat_user_tables": {
"history": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"last_autoanalyze": "2017-11-20 15:59:02.567618-05",
"n_dead_tup": "0",
"n_live_tup": "60854",
"n_mod_since_analyze": "854",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "60854",
"n_tup_upd": "0",
"relid": "16536",
"relname": "history",
"schemaname": "public",
"seq_scan": "2",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"warehouse": {
"analyze_count": "0",
"autoanalyze_count": "2",
"autovacuum_count": "2",
"idx_scan": "202634",
"idx_tup_fetch": "202634",
"last_autoanalyze": "2017-11-20 19:23:34.236294-05",
"last_autovacuum": "2017-11-20 19:23:34.235793-05",
"n_dead_tup": "0",
"n_live_tup": "2",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "854",
"n_tup_ins": "2",
"n_tup_upd": "854",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"stock": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "644561",
"idx_tup_fetch": "644561",
"last_autoanalyze": "2017-11-20 15:59:01.368483-05",
"n_dead_tup": "4364",
"n_live_tup": "200000",
"n_mod_since_analyze": "8901",
"n_tup_del": "0",
"n_tup_hot_upd": "5305",
"n_tup_ins": "200000",
"n_tup_upd": "8901",
"relid": "16523",
"relname": "stock",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"customer": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "125261",
"idx_tup_fetch": "85299628",
"last_autoanalyze": "2017-11-20 15:59:18.824212-05",
"n_dead_tup": "1510",
"n_live_tup": "60000",
"n_mod_since_analyze": "1594",
"n_tup_del": "0",
"n_tup_hot_upd": "262",
"n_tup_ins": "60000",
"n_tup_upd": "1594",
"relid": "16540",
"relname": "customer",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"order_line": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "1655",
"idx_tup_fetch": "33762",
"last_autoanalyze": "2017-11-20 16:00:11.017507-05",
"n_dead_tup": "2550",
"n_live_tup": "608373",
"n_mod_since_analyze": "16230",
"n_tup_del": "0",
"n_tup_hot_upd": "5393",
"n_tup_ins": "608373",
"n_tup_upd": "7329",
"relid": "16513",
"relname": "order_line",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"oorder": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "627652",
"idx_tup_fetch": "627652",
"last_autoanalyze": "2017-11-20 15:59:54.690984-05",
"n_dead_tup": "117",
"n_live_tup": "60889",
"n_mod_since_analyze": "1629",
"n_tup_del": "0",
"n_tup_hot_upd": "662",
"n_tup_ins": "60900",
"n_tup_upd": "740",
"relid": "16528",
"relname": "oorder",
"schemaname": "public",
"seq_scan": "4",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"new_order": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "1481",
"idx_tup_fetch": "1480",
"last_autoanalyze": "2017-11-20 16:00:11.217111-05",
"n_dead_tup": "751",
"n_live_tup": "16964",
"n_mod_since_analyze": "1629",
"n_tup_del": "740",
"n_tup_hot_upd": "0",
"n_tup_ins": "17715",
"n_tup_upd": "0",
"relid": "16518",
"relname": "new_order",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"district": {
"analyze_count": "0",
"autoanalyze_count": "2",
"autovacuum_count": "0",
"idx_scan": "122234",
"idx_tup_fetch": "122234",
"last_autoanalyze": "2017-11-20 19:23:34.201509-05",
"n_dead_tup": "33",
"n_live_tup": "20",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "1754",
"n_tup_ins": "20",
"n_tup_upd": "1754",
"relid": "16549",
"relname": "district",
"schemaname": "public",
"seq_scan": "2221",
"seq_tup_read": "41522",
"vacuum_count": "0"
},
"item": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "209020",
"idx_tup_fetch": "209009",
"last_autoanalyze": "2017-11-20 15:59:26.613728-05",
"n_dead_tup": "0",
"n_live_tup": "102000",
"n_mod_since_analyze": "2000",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "100000",
"n_tup_upd": "0",
"relid": "16554",
"relname": "item",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
}
},
"pg_statio_user_tables": {
"history": {
"heap_blks_hit": "184380",
"heap_blks_read": "746",
"relid": "16536",
"relname": "history",
"schemaname": "public"
},
"order_line": {
"heap_blks_hit": "1869417",
"heap_blks_read": "12419",
"idx_blks_hit": "1788651",
"idx_blks_read": "3708",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"warehouse": {
"heap_blks_hit": "404486",
"heap_blks_read": "80",
"idx_blks_hit": "202643",
"idx_blks_read": "6",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
},
"new_order": {
"heap_blks_hit": "37856",
"heap_blks_read": "192",
"idx_blks_hit": "38225",
"idx_blks_read": "134",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"heap_blks_hit": "1920817",
"heap_blks_read": "11757",
"idx_blks_hit": "2447522",
"idx_blks_read": "1530",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"heap_blks_hit": "1378399",
"heap_blks_read": "928",
"idx_blks_hit": "3979052",
"idx_blks_read": "1881",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"district": {
"heap_blks_hit": "249754",
"heap_blks_read": "3",
"idx_blks_hit": "122259",
"idx_blks_read": "5",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"heap_blks_hit": "509702",
"heap_blks_read": "4542",
"idx_blks_hit": "617914",
"idx_blks_read": "877",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"customer": {
"heap_blks_hit": "70136669",
"heap_blks_read": "13826",
"idx_blks_hit": "1411491",
"idx_blks_read": "2716",
"relid": "16540",
"relname": "customer",
"schemaname": "public",
"tidx_blks_hit": "0",
"tidx_blks_read": "0",
"toast_blks_hit": "0",
"toast_blks_read": "0"
}
}
},
"database": {
"pg_stat_database": {
"postgres": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "115229324",
"blks_read": "104188",
"conflicts": "0",
"datid": "12558",
"datname": "postgres",
"deadlocks": "0",
"numbackends": "1",
"stats_reset": "2017-11-10 11:14:57.116228-05",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "1818",
"tup_fetched": "103355344",
"tup_inserted": "2210752",
"tup_returned": "110741743",
"tup_updated": "32675",
"xact_commit": "19082",
"xact_rollback": "17"
},
"tpcc": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "16384",
"datname": "tpcc",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
},
"template1": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "1",
"datname": "template1",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
},
"template0": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "12557",
"datname": "template0",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
}
},
"pg_stat_database_conflicts": {
"postgres": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "12558",
"datname": "postgres"
},
"tpcc": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "16384",
"datname": "tpcc"
},
"template1": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "1",
"datname": "template1"
},
"template0": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "12557",
"datname": "template0"
}
}
},
"indexes": {
"pg_stat_user_indexes": {
"order_line": {
"idx_scan": "1655",
"idx_tup_fetch": "33762",
"idx_tup_read": "35698",
"indexrelid": "16516",
"indexrelname": "order_line_pkey",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"new_order": {
"idx_scan": "1481",
"idx_tup_fetch": "1480",
"idx_tup_read": "2200",
"indexrelid": "16521",
"indexrelname": "new_order_pkey",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"idx_scan": "644561",
"idx_tup_fetch": "644561",
"idx_tup_read": "647319",
"indexrelid": "16526",
"indexrelname": "stock_pkey",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"idx_scan": "616371",
"idx_tup_fetch": "616371",
"idx_tup_read": "616371",
"indexrelid": "16565",
"indexrelname": "idx_order",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"customer": {
"idx_scan": "82442",
"idx_tup_fetch": "85256809",
"idx_tup_read": "85256841",
"indexrelid": "16564",
"indexrelname": "idx_customer_name",
"relid": "16540",
"relname": "customer",
"schemaname": "public"
},
"district": {
"idx_scan": "122234",
"idx_tup_fetch": "122234",
"idx_tup_read": "122234",
"indexrelid": "16552",
"indexrelname": "district_pkey",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"idx_scan": "209020",
"idx_tup_fetch": "209009",
"idx_tup_read": "209009",
"indexrelid": "16557",
"indexrelname": "item_pkey",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"warehouse": {
"idx_scan": "202634",
"idx_tup_fetch": "201331",
"idx_tup_read": "202634",
"indexrelid": "16562",
"indexrelname": "warehouse_pkey",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
}
},
"pg_statio_user_indexes": {
"order_line": {
"idx_blks_hit": "1788651",
"idx_blks_read": "3708",
"indexrelid": "16516",
"indexrelname": "order_line_pkey",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"new_order": {
"idx_blks_hit": "38225",
"idx_blks_read": "134",
"indexrelid": "16521",
"indexrelname": "new_order_pkey",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"idx_blks_hit": "2447522",
"idx_blks_read": "1530",
"indexrelid": "16526",
"indexrelname": "stock_pkey",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"idx_blks_hit": "3689479",
"idx_blks_read": "733",
"indexrelid": "16565",
"indexrelname": "idx_order",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"customer": {
"idx_blks_hit": "1151523",
"idx_blks_read": "1589",
"indexrelid": "16564",
"indexrelname": "idx_customer_name",
"relid": "16540",
"relname": "customer",
"schemaname": "public"
},
"district": {
"idx_blks_hit": "122259",
"idx_blks_read": "5",
"indexrelid": "16552",
"indexrelname": "district_pkey",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"idx_blks_hit": "617914",
"idx_blks_read": "877",
"indexrelid": "16557",
"indexrelname": "item_pkey",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"warehouse": {
"idx_blks_hit": "202643",
"idx_blks_read": "6",
"indexrelid": "16562",
"indexrelname": "warehouse_pkey",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
}
}
}
}
}

582
server/website/script/controller_simulator/samples/metrics_before.json Normal file
View File

@@ -0,0 +1,582 @@
{
"global": {
"pg_stat_archiver": {
"archived_count": "0",
"failed_count": "0",
"stats_reset": "2017-11-10 10:59:47.397075-05"
},
"pg_stat_bgwriter": {
"buffers_alloc": "87670",
"buffers_backend": "81032",
"buffers_backend_fsync": "0",
"buffers_checkpoint": "33250",
"buffers_clean": "49590",
"checkpoint_sync_time": "19",
"checkpoint_write_time": "597851",
"checkpoints_req": "2",
"checkpoints_timed": "1277",
"maxwritten_clean": "325",
"stats_reset": "2017-11-10 10:59:47.397075-05"
}
},
"local": {
"table": {
"pg_stat_user_tables": {
"history": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"last_autoanalyze": "2017-11-20 15:59:02.567618-05",
"n_dead_tup": "0",
"n_live_tup": "60854",
"n_mod_since_analyze": "854",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "60854",
"n_tup_upd": "0",
"relid": "16536",
"relname": "history",
"schemaname": "public",
"seq_scan": "2",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"warehouse": {
"analyze_count": "0",
"autoanalyze_count": "2",
"autovacuum_count": "2",
"idx_scan": "202634",
"idx_tup_fetch": "202634",
"last_autoanalyze": "2017-11-20 19:23:34.236294-05",
"last_autovacuum": "2017-11-20 19:23:34.235793-05",
"n_dead_tup": "0",
"n_live_tup": "2",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "854",
"n_tup_ins": "2",
"n_tup_upd": "854",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"stock": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "644561",
"idx_tup_fetch": "644561",
"last_autoanalyze": "2017-11-20 15:59:01.368483-05",
"n_dead_tup": "4364",
"n_live_tup": "200000",
"n_mod_since_analyze": "8901",
"n_tup_del": "0",
"n_tup_hot_upd": "5305",
"n_tup_ins": "200000",
"n_tup_upd": "8901",
"relid": "16523",
"relname": "stock",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"customer": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "125261",
"idx_tup_fetch": "85299628",
"last_autoanalyze": "2017-11-20 15:59:18.824212-05",
"n_dead_tup": "1510",
"n_live_tup": "60000",
"n_mod_since_analyze": "1594",
"n_tup_del": "0",
"n_tup_hot_upd": "262",
"n_tup_ins": "60000",
"n_tup_upd": "1594",
"relid": "16540",
"relname": "customer",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"order_line": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "1655",
"idx_tup_fetch": "33762",
"last_autoanalyze": "2017-11-20 16:00:11.017507-05",
"n_dead_tup": "2550",
"n_live_tup": "608373",
"n_mod_since_analyze": "16230",
"n_tup_del": "0",
"n_tup_hot_upd": "5393",
"n_tup_ins": "608373",
"n_tup_upd": "7329",
"relid": "16513",
"relname": "order_line",
"schemaname": "public",
"seq_scan": "3",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"oorder": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "627652",
"idx_tup_fetch": "627652",
"last_autoanalyze": "2017-11-20 15:59:54.690984-05",
"n_dead_tup": "117",
"n_live_tup": "60889",
"n_mod_since_analyze": "1629",
"n_tup_del": "0",
"n_tup_hot_upd": "662",
"n_tup_ins": "60900",
"n_tup_upd": "740",
"relid": "16528",
"relname": "oorder",
"schemaname": "public",
"seq_scan": "4",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"new_order": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "1481",
"idx_tup_fetch": "1480",
"last_autoanalyze": "2017-11-20 16:00:11.217111-05",
"n_dead_tup": "751",
"n_live_tup": "16964",
"n_mod_since_analyze": "1629",
"n_tup_del": "740",
"n_tup_hot_upd": "0",
"n_tup_ins": "17715",
"n_tup_upd": "0",
"relid": "16518",
"relname": "new_order",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
},
"district": {
"analyze_count": "0",
"autoanalyze_count": "2",
"autovacuum_count": "0",
"idx_scan": "122234",
"idx_tup_fetch": "122234",
"last_autoanalyze": "2017-11-20 19:23:34.201509-05",
"n_dead_tup": "33",
"n_live_tup": "20",
"n_mod_since_analyze": "0",
"n_tup_del": "0",
"n_tup_hot_upd": "1754",
"n_tup_ins": "20",
"n_tup_upd": "1754",
"relid": "16549",
"relname": "district",
"schemaname": "public",
"seq_scan": "2221",
"seq_tup_read": "41522",
"vacuum_count": "0"
},
"item": {
"analyze_count": "0",
"autoanalyze_count": "1",
"autovacuum_count": "0",
"idx_scan": "209020",
"idx_tup_fetch": "209009",
"last_autoanalyze": "2017-11-20 15:59:26.613728-05",
"n_dead_tup": "0",
"n_live_tup": "102000",
"n_mod_since_analyze": "2000",
"n_tup_del": "0",
"n_tup_hot_upd": "0",
"n_tup_ins": "100000",
"n_tup_upd": "0",
"relid": "16554",
"relname": "item",
"schemaname": "public",
"seq_scan": "1",
"seq_tup_read": "0",
"vacuum_count": "0"
}
},
"pg_statio_user_tables": {
"history": {
"heap_blks_hit": "184380",
"heap_blks_read": "746",
"relid": "16536",
"relname": "history",
"schemaname": "public"
},
"order_line": {
"heap_blks_hit": "1869417",
"heap_blks_read": "12419",
"idx_blks_hit": "1788651",
"idx_blks_read": "3708",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"warehouse": {
"heap_blks_hit": "404486",
"heap_blks_read": "80",
"idx_blks_hit": "202643",
"idx_blks_read": "6",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
},
"new_order": {
"heap_blks_hit": "37856",
"heap_blks_read": "192",
"idx_blks_hit": "38225",
"idx_blks_read": "134",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"heap_blks_hit": "1920817",
"heap_blks_read": "11757",
"idx_blks_hit": "2447522",
"idx_blks_read": "1530",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"heap_blks_hit": "1378399",
"heap_blks_read": "928",
"idx_blks_hit": "3979052",
"idx_blks_read": "1881",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"district": {
"heap_blks_hit": "249754",
"heap_blks_read": "3",
"idx_blks_hit": "122259",
"idx_blks_read": "5",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"heap_blks_hit": "509702",
"heap_blks_read": "4542",
"idx_blks_hit": "617914",
"idx_blks_read": "877",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"customer": {
"heap_blks_hit": "70136669",
"heap_blks_read": "13826",
"idx_blks_hit": "1411491",
"idx_blks_read": "2716",
"relid": "16540",
"relname": "customer",
"schemaname": "public",
"tidx_blks_hit": "0",
"tidx_blks_read": "0",
"toast_blks_hit": "0",
"toast_blks_read": "0"
}
}
},
"database": {
"pg_stat_database": {
"postgres": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "115229324",
"blks_read": "104188",
"conflicts": "0",
"datid": "12558",
"datname": "postgres",
"deadlocks": "0",
"numbackends": "1",
"stats_reset": "2017-11-10 11:14:57.116228-05",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "1818",
"tup_fetched": "103355344",
"tup_inserted": "2210752",
"tup_returned": "110741743",
"tup_updated": "32675",
"xact_commit": "19082",
"xact_rollback": "17"
},
"tpcc": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "16384",
"datname": "tpcc",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
},
"template1": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "1",
"datname": "template1",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
},
"template0": {
"blk_read_time": "0",
"blk_write_time": "0",
"blks_hit": "0",
"blks_read": "0",
"conflicts": "0",
"datid": "12557",
"datname": "template0",
"deadlocks": "0",
"numbackends": "0",
"temp_bytes": "0",
"temp_files": "0",
"tup_deleted": "0",
"tup_fetched": "0",
"tup_inserted": "0",
"tup_returned": "0",
"tup_updated": "0",
"xact_commit": "0",
"xact_rollback": "0"
}
},
"pg_stat_database_conflicts": {
"postgres": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "12558",
"datname": "postgres"
},
"tpcc": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "16384",
"datname": "tpcc"
},
"template1": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "1",
"datname": "template1"
},
"template0": {
"confl_bufferpin": "0",
"confl_deadlock": "0",
"confl_lock": "0",
"confl_snapshot": "0",
"confl_tablespace": "0",
"datid": "12557",
"datname": "template0"
}
}
},
"indexes": {
"pg_stat_user_indexes": {
"order_line": {
"idx_scan": "1655",
"idx_tup_fetch": "33762",
"idx_tup_read": "35698",
"indexrelid": "16516",
"indexrelname": "order_line_pkey",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"new_order": {
"idx_scan": "1481",
"idx_tup_fetch": "1480",
"idx_tup_read": "2200",
"indexrelid": "16521",
"indexrelname": "new_order_pkey",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"idx_scan": "644561",
"idx_tup_fetch": "644561",
"idx_tup_read": "647319",
"indexrelid": "16526",
"indexrelname": "stock_pkey",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"idx_scan": "616371",
"idx_tup_fetch": "616371",
"idx_tup_read": "616371",
"indexrelid": "16565",
"indexrelname": "idx_order",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"customer": {
"idx_scan": "82442",
"idx_tup_fetch": "85256809",
"idx_tup_read": "85256841",
"indexrelid": "16564",
"indexrelname": "idx_customer_name",
"relid": "16540",
"relname": "customer",
"schemaname": "public"
},
"district": {
"idx_scan": "122234",
"idx_tup_fetch": "122234",
"idx_tup_read": "122234",
"indexrelid": "16552",
"indexrelname": "district_pkey",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"idx_scan": "209020",
"idx_tup_fetch": "209009",
"idx_tup_read": "209009",
"indexrelid": "16557",
"indexrelname": "item_pkey",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"warehouse": {
"idx_scan": "202634",
"idx_tup_fetch": "201331",
"idx_tup_read": "202634",
"indexrelid": "16562",
"indexrelname": "warehouse_pkey",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
}
},
"pg_statio_user_indexes": {
"order_line": {
"idx_blks_hit": "1788651",
"idx_blks_read": "3708",
"indexrelid": "16516",
"indexrelname": "order_line_pkey",
"relid": "16513",
"relname": "order_line",
"schemaname": "public"
},
"new_order": {
"idx_blks_hit": "38225",
"idx_blks_read": "134",
"indexrelid": "16521",
"indexrelname": "new_order_pkey",
"relid": "16518",
"relname": "new_order",
"schemaname": "public"
},
"stock": {
"idx_blks_hit": "2447522",
"idx_blks_read": "1530",
"indexrelid": "16526",
"indexrelname": "stock_pkey",
"relid": "16523",
"relname": "stock",
"schemaname": "public"
},
"oorder": {
"idx_blks_hit": "3689479",
"idx_blks_read": "733",
"indexrelid": "16565",
"indexrelname": "idx_order",
"relid": "16528",
"relname": "oorder",
"schemaname": "public"
},
"customer": {
"idx_blks_hit": "1151523",
"idx_blks_read": "1589",
"indexrelid": "16564",
"indexrelname": "idx_customer_name",
"relid": "16540",
"relname": "customer",
"schemaname": "public"
},
"district": {
"idx_blks_hit": "122259",
"idx_blks_read": "5",
"indexrelid": "16552",
"indexrelname": "district_pkey",
"relid": "16549",
"relname": "district",
"schemaname": "public"
},
"item": {
"idx_blks_hit": "617914",
"idx_blks_read": "877",
"indexrelid": "16557",
"indexrelname": "item_pkey",
"relid": "16554",
"relname": "item",
"schemaname": "public"
},
"warehouse": {
"idx_blks_hit": "202643",
"idx_blks_read": "6",
"indexrelid": "16562",
"indexrelname": "warehouse_pkey",
"relid": "16559",
"relname": "warehouse",
"schemaname": "public"
}
}
}
}
}

8
server/website/script/controller_simulator/samples/summary.json Normal file
View File

@@ -0,0 +1,8 @@
{
"workload_name": "workload-0",
"database_type": "postgres",
"start_time": 1512076859887,
"observation_time": 300,
"end_time": 1512076864891,
"database_version": "9.6"
}

62
server/website/script/controller_simulator/upload_data.py Normal file
View File

@@ -0,0 +1,62 @@
#
# OtterTune - upload_data.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
'''
Created on Nov 30, 2017
@author: dvanaken
'''
import argparse
import glob
import logging
import os
import requests
# Logging
LOG = logging.getLogger(__name__)
LOG.addHandler(logging.StreamHandler())
LOG.setLevel(logging.INFO)
def upload(basedir, upload_code, upload_url):
for wkld_dir in sorted(glob.glob(os.path.join(basedir, '*'))):
LOG.info('Uploading sample for workload %s...', wkld_dir)
sample_idx = 0
while True:
samples = glob.glob(os.path.join(wkld_dir, 'sample-{}__*').format(sample_idx))
if len(samples) == 0:
break
assert len(samples) == 4
basename = samples[0].split('__')[0]
params = {
'summary': open(basename + '__summary.json', 'r'),
'knobs': open(basename + '__knobs.json', 'r'),
'metrics_before': open(basename + '__metrics_start.json', 'r'),
'metrics_after': open(basename + '__metrics_end.json', 'r'),
}
response = requests.post(upload_url + "/new_result/",
files=params,
data={'upload_code': upload_code})
LOG.info("Response: %s\n", response.content.decode())
sample_idx += 1
def main():
parser = argparse.ArgumentParser(description="Upload generated data to the website")
parser.add_argument('basedir', type=str, nargs=1,
help='Directory containing the generated data')
parser.add_argument('upload_code', type=str, nargs=1,
help='The website\'s upload code')
parser.add_argument('upload_url', type=str, default='http://0.0.0.0:8000',
nargs='?', help='The website\'s URL')
args = parser.parse_args()
upload(args.basedir[0], args.upload_code[0], args.upload_url)
if __name__ == "__main__":
main()