64 lines
2.2 KiB
Python
64 lines
2.2 KiB
Python
|
#
|
||
|
|
# OtterTune - upload_batch.py
|
||
|
|
#
|
||
|
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||
|
|
#
|
||
|
|
import argparse
|
||
|
|
import glob
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import requests
|
||
|
|
|
||
|
|
|
||
|
|
# Logging
|
||
|
|
LOG = logging.getLogger(__name__)
|
||
|
|
LOG.addHandler(logging.StreamHandler())
|
||
|
|
LOG.setLevel(logging.INFO)
|
||
|
|
|
||
|
|
|
||
|
|
# Upload all files in the datadir to OtterTune's server side.
|
||
|
|
# You may want to upload your training data to the non-tuning session.
|
||
|
|
def upload_batch(datadir, upload_code, url):
|
||
|
|
|
||
|
|
samples = glob.glob(os.path.join(datadir, '*__summary.json'))
|
||
|
|
count = len(samples)
|
||
|
|
samples_prefix = []
|
||
|
|
|
||
|
|
LOG.info('Uploading %d samples in %s...', count, datadir)
|
||
|
|
for sample in samples:
|
||
|
|
prefix = sample.split('/')[-1].split('__')[0]
|
||
|
|
samples_prefix.append(prefix)
|
||
|
|
|
||
|
|
for i in range(count):
|
||
|
|
prefix = samples_prefix[i]
|
||
|
|
params = {
|
||
|
|
'summary': open(os.path.join(datadir, '{}__summary.json'.format(prefix)), 'rb'),
|
||
|
|
'knobs': open(os.path.join(datadir, '{}__knobs.json'.format(prefix)), 'rb'),
|
||
|
|
'metrics_before': open(os.path.join(datadir,
|
||
|
|
'{}__metrics_before.json'.format(prefix)), 'rb'),
|
||
|
|
'metrics_after': open(os.path.join(datadir,
|
||
|
|
'{}__metrics_after.json'.format(prefix)), 'rb'),
|
||
|
|
}
|
||
|
|
|
||
|
|
LOG.info('Upload %d-th sample %s__*.json', i + 1, prefix)
|
||
|
|
response = requests.post(url,
|
||
|
|
files=params,
|
||
|
|
data={'upload_code': upload_code})
|
||
|
|
LOG.info(response.content)
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser(description="Upload generated data to the website")
|
||
|
|
parser.add_argument('datadir', type=str, nargs=1,
|
||
|
|
help='Directory containing the generated data')
|
||
|
|
parser.add_argument('upload_code', type=str, nargs=1,
|
||
|
|
help='The website\'s upload code')
|
||
|
|
parser.add_argument('url', type=str, default='http://0.0.0.0:8000/new_result/',
|
||
|
|
nargs='?', help='The upload url: server_ip/new_result/')
|
||
|
|
args = parser.parse_args()
|
||
|
|
upload_batch(args.datadir[0], args.upload_code[0], args.url)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|