# # OtterTune - simulation.py # # Copyright (c) 2017-18, Carnegie Mellon University Database Group # import heapq import random import os import sys try: import matplotlib.pyplot as plt except (ModuleNotFoundError, ImportError): plt = None import numpy as np import torch sys.path.append("../") from analysis.util import get_analysis_logger # noqa from analysis.ddpg.ddpg import DDPG # noqa from analysis.gp_tf import GPRGD # noqa from analysis.nn_tf import NeuralNet # noqa LOG = get_analysis_logger(__name__) class Environment(object): def __init__(self, n_knob, n_metric, mode=0): self.knob_dim = n_knob self.metric_dim = n_metric self.mode = mode def identity_sqrt(self, knob_data): n1 = self.knob_dim // 4 n2 = self.knob_dim // 4 part1 = np.sum(knob_data[0: n1]) part2 = np.sum(np.sqrt(knob_data[n1: n1 + n2])) reward = np.array([part1 + part2]) / (self.knob_dim // 2) metric_data = np.zeros(self.metric_dim) return reward, metric_data def borehole(self, knob_data): # ref: http://www.sfu.ca/~ssurjano/borehole.html # pylint: disable=invalid-name rw = knob_data[0] * (0.15 - 0.05) + 0.05 r = knob_data[1] * (50000 - 100) + 100 Tu = knob_data[2] * (115600 - 63070) + 63070 Hu = knob_data[3] * (1110 - 990) + 990 Tl = knob_data[4] * (116 - 63.1) + 63.1 Hl = knob_data[5] * (820 - 700) + 700 L = knob_data[6] * (1680 - 1120) + 1120 Kw = knob_data[7] * (12045 - 9855) + 9855 frac = 2 * L * Tu / (np.log(r / rw) * rw ** 2 * Kw) reward = 2 * np.pi * Tu * (Hu - Hl) / (np.log(r / rw) * (1 + frac + Tu / Tl)) return np.array([reward]), np.zeros(self.metric_dim) def threshold(self, knob_data): n1 = self.knob_dim // 4 n2 = self.knob_dim // 4 part1 = np.sum(knob_data[0: n1] > 0.9) part2 = np.sum(knob_data[n1: n1 + n2] < 0.1) reward = np.array([part1 + part2]) metric_data = np.zeros(self.metric_dim) return reward, metric_data def simulate(self, knob_data): if self.mode == 0: return self.identity_sqrt(knob_data) elif self.mode == 1: return self.threshold(knob_data) elif self.mode == 2: return self.borehole(knob_data) def ddpg(env, config, n_loops=1000): results = [] x_axis = [] gamma = config['gamma'] tau = config['tau'] lr = config['lr'] batch_size = config['batch_size'] n_epochs = config['n_epochs'] model_ddpg = DDPG(n_actions=env.knob_dim, n_states=env.metric_dim, gamma=gamma, tau=tau, clr=lr, alr=lr, batch_size=batch_size) knob_data = np.random.rand(env.knob_dim) prev_metric_data = np.zeros(env.metric_dim) for i in range(n_loops): reward, metric_data = env.simulate(knob_data) model_ddpg.add_sample(prev_metric_data, knob_data, reward, metric_data) for _ in range(n_epochs): model_ddpg.update() results.append(reward) x_axis.append(i) prev_metric_data = metric_data knob_data = model_ddpg.choose_action(prev_metric_data) return np.array(results), np.array(x_axis) class ReplayMemory(object): def __init__(self): self.actions = [] self.rewards = [] def push(self, action, reward): self.actions.append(action.tolist()) self.rewards.append(reward.tolist()) def get_all(self): return self.actions, self.rewards def dnn(env, config, n_loops=100): results = [] x_axis = [] memory = ReplayMemory() num_samples = config['num_samples'] Xmin = np.zeros(env.knob_dim) Xmax = np.ones(env.knob_dim) for i in range(n_loops): X_samples = np.random.rand(num_samples, env.knob_dim) if i >= 10: actions, rewards = memory.get_all() tuples = tuple(zip(actions, rewards)) top10 = heapq.nlargest(10, tuples, key=lambda e: e[1]) for entry in top10: X_samples = np.vstack((X_samples, np.array(entry[0]))) model_nn = NeuralNet(n_input=X_samples.shape[1], batch_size=X_samples.shape[0], explore_iters=500, noise_scale_begin=0.1, noise_scale_end=0.0, debug=False, debug_interval=100) if i >= 5: actions, rewards = memory.get_all() model_nn.fit(np.array(actions), -np.array(rewards), fit_epochs=500) res = model_nn.recommend(X_samples, Xmin, Xmax, explore=500, recommend_epochs=500) best_config_idx = np.argmin(res.minl.ravel()) best_config = res.minl_conf[best_config_idx, :] reward, _ = env.simulate(best_config) memory.push(best_config, reward) LOG.info('loop: %d reward: %f', i, reward[0]) results.append(reward) x_axis.append(i) return np.array(results), np.array(x_axis) def gprgd(env, config, n_loops=100): results = [] x_axis = [] memory = ReplayMemory() num_samples = config['num_samples'] X_min = np.zeros(env.knob_dim) X_max = np.ones(env.knob_dim) for _ in range(5): action = np.random.rand(env.knob_dim) reward, _ = env.simulate(action) memory.push(action, reward) for i in range(n_loops): X_samples = np.random.rand(num_samples, env.knob_dim) if i >= 5: actions, rewards = memory.get_all() tuples = tuple(zip(actions, rewards)) top10 = heapq.nlargest(10, tuples, key=lambda e: e[1]) for entry in top10: # Tensorflow get broken if we use the training data points as # starting points for GPRGD. X_samples = np.vstack((X_samples, np.array(entry[0]) * 0.97 + 0.01)) model = GPRGD(length_scale=1.0, magnitude=1.0, max_train_size=7000, batch_size=3000, num_threads=4, learning_rate=0.01, epsilon=1e-6, max_iter=500, sigma_multiplier=3.0, mu_multiplier=1.0) actions, rewards = memory.get_all() model.fit(np.array(actions), -np.array(rewards), X_min, X_max, ridge=0.01) res = model.predict(X_samples) best_config_idx = np.argmin(res.minl.ravel()) best_config = res.minl_conf[best_config_idx, :] reward, _ = env.simulate(best_config) memory.push(best_config, reward) LOG.info('loop: %d reward: %f', i, reward[0]) results.append(reward) x_axis.append(i) return np.array(results), np.array(x_axis) def plotlines(x_axis, results, labels, title, path): if plt: for result, label in zip(results, labels): plt.plot(x_axis, result, label=label) plt.legend() plt.xlabel("loops") plt.ylabel("rewards") plt.title(title) plt.savefig(path) plt.clf() def run(tuners, configs, labels, knob_dim, metric_dim, mode, n_loops, n_repeats): if not plt: LOG.info("Cannot import matplotlib. Will write results to files instead of figures.") random.seed(0) np.random.seed(0) torch.manual_seed(0) env = Environment(knob_dim, metric_dim, mode=mode) results = [] for i in range(n_repeats): for j, _ in enumerate(tuners): result, x_axis = tuners[j](env, configs[j], n_loops=n_loops) if i is 0: results.append(result / n_repeats) else: results[j] += result / n_repeats title = "mode_{}_knob_{}".format(mode, knob_dim) if plt: if not os.path.exists("figures"): os.mkdir("figures") filename = "figures/{}.pdf".format(title) plotlines(x_axis, results, labels, title, filename) for j in range(len(tuners)): with open(title + '_' + labels[j] + '.csv', 'w') as f: for i, result in zip(x_axis, results[j]): f.write(str(i) + ',' + str(result[0]) + '\n') def main(): knob_dim = 192 metric_dim = 60 mode = 0 n_loops = 2 n_repeats = 1 configs = [{'gamma': 0., 'tau': 0.002, 'lr': 0.001, 'batch_size': 32, 'n_epochs': 30}, {'gamma': 0.99, 'tau': 0.002, 'lr': 0.001, 'batch_size': 32, 'n_epochs': 30}, {'num_samples': 30}, {'num_samples': 30}] tuners = [ddpg, ddpg, dnn, gprgd] labels = [tuner.__name__ for tuner in tuners] labels[0] += '_gamma_0' labels[1] += '_gamma_99' run(tuners, configs, labels, knob_dim, metric_dim, mode, n_loops, n_repeats) if __name__ == '__main__': main()