public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

make session hyperparameters editable on the website

This commit is contained in:
yangdsh 2020-01-12 10:07:15 +00:00 committed by Dana Van Aken
parent 6bf50b892d
commit 24194293bc
8 changed files with 106 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/website/website/forms.py
View File

@ -134,11 +134,12 @@ class SessionForm(forms.ModelForm):
model = Session
fields = ('name', 'description', 'tuning_session', 'dbms', 'cpu', 'memory', 'storage',
'algorithm', 'target_objective')
'algorithm', 'target_objective', 'hyper_parameters')
widgets = {
'name': forms.TextInput(attrs={'required': True}),
'description': forms.Textarea(attrs={'maxlength': 500, 'rows': 5}),
'hyper_parameters': forms.Textarea(attrs={'maxlength': 2000, 'rows': 10}),
}
labels = {
'dbms': 'DBMS',

20
server/website/website/migrations/0006_session_hyper_parameters.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2020-01-12 07:29
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('website', '0005_add_workload_field'),
]
operations = [
migrations.AddField(
model_name='session',
name='hyper_parameters',
field=models.TextField(default='{}'),
),
]

1
server/website/website/models.py
View File

@ -162,6 +162,7 @@ class Session(BaseModel):
target_objective = models.CharField(
max_length=64, default=target_objectives.default())
hyper_parameters = models.TextField(default="{}")
def clean(self):
if self.target_objective is None:

18
server/website/website/settings/constants.py
View File

@ -4,10 +4,22 @@
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
# ---------------------------------------------
# These parameters are not specified for any session, so they can only be set here
# address categorical knobs (enum, boolean)
enable_dummy_encoder = False
# ---PIPELINE CONSTANTS---
# how often to run the background tests, in seconds
RUN_EVERY = 300
run_every = 300
# ---------------------------------------------
# The following parameters can be viewed and modified on the session page on the website
# ---SAMPLING CONSTANTS---
# the number of samples (staring points) in gradient descent
NUM_SAMPLES = 30
@ -20,10 +32,6 @@ IMPORTANT_KNOB_NUMBER = 10000
TOP_NUM_CONFIG = 10
# ---CONSTRAINTS CONSTANTS---
# address categorical knobs (enum, boolean)
ENABLE_DUMMY_ENCODER = False
# Initial probability to flip categorical feature in apply_constraints
# server/analysis/constraints.py
INIT_FLIP_PROB = 0.3

123
server/website/website/tasks/async_tasks.py
View File

@ -30,23 +30,7 @@ from website.models import (PipelineData, PipelineRun, Result, Workload, KnobCat
from website import db
from website.types import PipelineTaskType, AlgorithmType, VarType
from website.utils import DataUtil, JSONUtil
from website.settings import IMPORTANT_KNOB_NUMBER, NUM_SAMPLES, TOP_NUM_CONFIG # pylint: disable=no-name-in-module
from website.settings import (USE_GPFLOW, DEFAULT_LENGTH_SCALE, DEFAULT_MAGNITUDE,
MAX_TRAIN_SIZE, BATCH_SIZE, NUM_THREADS,
DEFAULT_RIDGE, DEFAULT_LEARNING_RATE,
DEFAULT_EPSILON, MAX_ITER, GPR_EPS,
DEFAULT_SIGMA_MULTIPLIER, DEFAULT_MU_MULTIPLIER,
DEFAULT_UCB_SCALE, HP_LEARNING_RATE, HP_MAX_ITER,
DDPG_SIMPLE_REWARD, DDPG_GAMMA, USE_DEFAULT,
DDPG_BATCH_SIZE, ACTOR_LEARNING_RATE,
CRITIC_LEARNING_RATE, UPDATE_EPOCHS,
ACTOR_HIDDEN_SIZES, CRITIC_HIDDEN_SIZES,
DNN_TRAIN_ITER, DNN_EXPLORE, DNN_EXPLORE_ITER,
DNN_NOISE_SCALE_BEGIN, DNN_NOISE_SCALE_END,
DNN_DEBUG, DNN_DEBUG_INTERVAL, GPR_DEBUG, UCB_BETA,
GPR_MODEL_NAME, ENABLE_DUMMY_ENCODER, DNN_GD_ITER)
from website.settings import INIT_FLIP_PROB, FLIP_PROB_DECAY
from website.settings import enable_dummy_encoder
LOG = get_task_logger(__name__)
@ -339,6 +323,7 @@ def train_ddpg(result_id):
LOG.info('Add training data to ddpg and train ddpg')
result = Result.objects.get(pk=result_id)
session = Result.objects.get(pk=result_id).session
params = JSONUtil.loads(session.hyper_parameters)
session_results = Result.objects.filter(session=session,
creation_time__lt=result.creation_time)
result_info = {}
@ -418,16 +403,16 @@ def train_ddpg(result_id):
LOG.info('reward: %f', reward)
# Update ddpg
ddpg = DDPG(n_actions=knob_num, n_states=metric_num, alr=ACTOR_LEARNING_RATE,
clr=CRITIC_LEARNING_RATE, gamma=DDPG_GAMMA, batch_size=DDPG_BATCH_SIZE,
a_hidden_sizes=ACTOR_HIDDEN_SIZES, c_hidden_sizes=CRITIC_HIDDEN_SIZES,
use_default=USE_DEFAULT)
ddpg = DDPG(n_actions=knob_num, n_states=metric_num, alr=params['ACTOR_LEARNING_RATE'],
clr=params['CRITIC_LEARNING_RATE'], gamma=params['DDPG_GAMMA'],
batch_size=params['DDPG_BATCH_SIZE'], a_hidden_sizes=params['ACTOR_HIDDEN_SIZES'],
c_hidden_sizes=params['CRITIC_HIDDEN_SIZES'], use_default=params['USE_DEFAULT'])
if session.ddpg_actor_model and session.ddpg_critic_model:
ddpg.set_model(session.ddpg_actor_model, session.ddpg_critic_model)
if session.ddpg_reply_memory:
ddpg.replay_memory.set(session.ddpg_reply_memory)
ddpg.add_sample(normalized_metric_data, knob_data, reward, normalized_metric_data)
for _ in range(UPDATE_EPOCHS):
for _ in range(params['UPDATE_EPOCHS']):
ddpg.update()
session.ddpg_actor_model, session.ddpg_critic_model = ddpg.get_model()
session.ddpg_reply_memory = ddpg.replay_memory.get()
@ -459,6 +444,7 @@ def configuration_recommendation_ddpg(result_info): # pylint: disable=invalid-n
result_list = Result.objects.filter(pk=result_id)
result = result_list.first()
session = result.session
params = JSONUtil.loads(session.hyper_parameters)
agg_data = DataUtil.aggregate_data(result_list)
metric_data, _ = clean_metric_data(agg_data['y_matrix'], agg_data['y_columnlabels'], session)
metric_data = metric_data.flatten()
@ -469,8 +455,9 @@ def configuration_recommendation_ddpg(result_info): # pylint: disable=invalid-n
knob_num = len(knob_labels)
metric_num = len(metric_data)
ddpg = DDPG(n_actions=knob_num, n_states=metric_num, a_hidden_sizes=ACTOR_HIDDEN_SIZES,
c_hidden_sizes=CRITIC_HIDDEN_SIZES, use_default=USE_DEFAULT)
ddpg = DDPG(n_actions=knob_num, n_states=metric_num,
a_hidden_sizes=params['ACTOR_HIDDEN_SIZES'],
c_hidden_sizes=params['CRITIC_HIDDEN_SIZES'], use_default=params['USE_DEFAULT'])
if session.ddpg_actor_model is not None and session.ddpg_critic_model is not None:
ddpg.set_model(session.ddpg_actor_model, session.ddpg_critic_model)
if session.ddpg_reply_memory is not None:
@ -505,6 +492,8 @@ def combine_workload(target_data):
workload_metric_data = JSONUtil.loads(workload_metric_data.data)
newest_result = Result.objects.get(pk=target_data['newest_result_id'])
session = newest_result.session
params = JSONUtil.loads(session.hyper_parameters)
cleaned_workload_knob_data = clean_knob_data(workload_knob_data["data"],
workload_knob_data["columnlabels"],
newest_result.session)
@ -535,7 +524,7 @@ def combine_workload(target_data):
pipeline_run=latest_pipeline_run,
workload=mapped_workload,
task_type=PipelineTaskType.RANKED_KNOBS)
ranked_knobs = JSONUtil.loads(ranked_knobs.data)[:IMPORTANT_KNOB_NUMBER]
ranked_knobs = JSONUtil.loads(ranked_knobs.data)[:params['IMPORTANT_KNOB_NUMBER']]
ranked_knob_idxs = [i for i, cl in enumerate(X_columnlabels) if cl in ranked_knobs]
X_workload = X_workload[:, ranked_knob_idxs]
X_target = X_target[:, ranked_knob_idxs]
@ -577,7 +566,7 @@ def combine_workload(target_data):
X_matrix = np.vstack([X_target, X_workload])
# Dummy encode categorial variables
if ENABLE_DUMMY_ENCODER:
if enable_dummy_encoder:
categorical_info = DataUtil.dummy_encoder_helper(X_columnlabels,
mapped_workload.dbms)
dummy_encoder = DummyEncoder(categorical_info['n_values'],
@ -632,8 +621,8 @@ def combine_workload(target_data):
constraint_helper = ParamConstraintHelper(scaler=X_scaler,
encoder=dummy_encoder,
binary_vars=binary_encoder,
init_flip_prob=INIT_FLIP_PROB,
flip_prob_decay=FLIP_PROB_DECAY)
init_flip_prob=params['INIT_FLIP_PROB'],
flip_prob_decay=params['FLIP_PROB_DECAY'])
# FIXME (dva): check if these are good values for the ridge
# ridge = np.empty(X_scaled.shape[0])
@ -671,6 +660,8 @@ def configuration_recommendation(recommendation_input):
target_data, algorithm = recommendation_input
LOG.info('configuration_recommendation called')
newest_result = Result.objects.get(pk=target_data['newest_result_id'])
session = newest_result.session
params = session.hyper_parameters
if target_data['bad'] is True:
target_data_res = create_and_save_recommendation(
@ -687,7 +678,7 @@ def configuration_recommendation(recommendation_input):
# FIXME: we should generate more samples and use a smarter sampling
# technique
num_samples = NUM_SAMPLES
num_samples = params['NUM_SAMPLES']
X_samples = np.empty((num_samples, X_scaled.shape[1]))
for i in range(X_scaled.shape[1]):
X_samples[:, i] = np.random.rand(num_samples) * (X_max[i] - X_min[i]) + X_min[i]
@ -697,7 +688,7 @@ def configuration_recommendation(recommendation_input):
q.put((y_scaled[x][0], x))
i = 0
while i < TOP_NUM_CONFIG:
while i < params['TOP_NUM_CONFIG']:
try:
item = q.get_nowait()
# Tensorflow get broken if we use the training data points as
@ -714,56 +705,58 @@ def configuration_recommendation(recommendation_input):
except queue.Empty:
break
session = newest_result.session
res = None
if algorithm == AlgorithmType.DNN:
# neural network model
model_nn = NeuralNet(n_input=X_samples.shape[1],
batch_size=X_samples.shape[0],
explore_iters=DNN_EXPLORE_ITER,
noise_scale_begin=DNN_NOISE_SCALE_BEGIN,
noise_scale_end=DNN_NOISE_SCALE_END,
debug=DNN_DEBUG,
debug_interval=DNN_DEBUG_INTERVAL)
explore_iters=params['DNN_EXPLORE_ITER'],
noise_scale_begin=params['DNN_NOISE_SCALE_BEGIN'],
noise_scale_end=params['DNN_NOISE_SCALE_END'],
debug=params['DNN_DEBUG'],
debug_interval=params['DNN_DEBUG_INTERVAL'])
if session.dnn_model is not None:
model_nn.set_weights_bin(session.dnn_model)
model_nn.fit(X_scaled, y_scaled, fit_epochs=DNN_TRAIN_ITER)
model_nn.fit(X_scaled, y_scaled, fit_epochs=params['DNN_TRAIN_ITER'])
res = model_nn.recommend(X_samples, X_min, X_max,
explore=DNN_EXPLORE, recommend_epochs=DNN_GD_ITER)
explore=params['DNN_EXPLORE'],
recommend_epochs=params['DNN_GD_ITER'])
session.dnn_model = model_nn.get_weights_bin()
session.save()
elif algorithm == AlgorithmType.GPR:
# default gpr model
if USE_GPFLOW:
if params['USE_GPFLOW']:
model_kwargs = {}
model_kwargs['model_learning_rate'] = HP_LEARNING_RATE
model_kwargs['model_maxiter'] = HP_MAX_ITER
model_kwargs['model_learning_rate'] = params['HP_LEARNING_RATE']
model_kwargs['model_maxiter'] = params['HP_MAX_ITER']
opt_kwargs = {}
opt_kwargs['learning_rate'] = DEFAULT_LEARNING_RATE
opt_kwargs['maxiter'] = MAX_ITER
opt_kwargs['learning_rate'] = params['DEFAULT_LEARNING_RATE']
opt_kwargs['maxiter'] = params['MAX_ITER']
opt_kwargs['bounds'] = [X_min, X_max]
opt_kwargs['debug'] = GPR_DEBUG
opt_kwargs['ucb_beta'] = ucb.get_ucb_beta(UCB_BETA, scale=DEFAULT_UCB_SCALE,
opt_kwargs['debug'] = params['GPR_DEBUG']
opt_kwargs['ucb_beta'] = ucb.get_ucb_beta(params['UCB_BETA'],
scale=params['DEFAULT_UCB_SCALE'],
t=i + 1., ndim=X_scaled.shape[1])
tf.reset_default_graph()
graph = tf.get_default_graph()
gpflow.reset_default_session(graph=graph)
m = gpr_models.create_model(GPR_MODEL_NAME, X=X_scaled, y=y_scaled, **model_kwargs)
m = gpr_models.create_model(params['GPR_MODEL_NAME'], X=X_scaled, y=y_scaled,
**model_kwargs)
res = tf_optimize(m.model, X_samples, **opt_kwargs)
else:
model = GPRGD(length_scale=DEFAULT_LENGTH_SCALE,
magnitude=DEFAULT_MAGNITUDE,
max_train_size=MAX_TRAIN_SIZE,
batch_size=BATCH_SIZE,
num_threads=NUM_THREADS,
learning_rate=DEFAULT_LEARNING_RATE,
epsilon=DEFAULT_EPSILON,
max_iter=MAX_ITER,
sigma_multiplier=DEFAULT_SIGMA_MULTIPLIER,
mu_multiplier=DEFAULT_MU_MULTIPLIER,
ridge=DEFAULT_RIDGE)
model = GPRGD(length_scale=params['DEFAULT_LENGTH_SCALE'],
magnitude=params['DEFAULT_MAGNITUDE'],
max_train_size=params['MAX_TRAIN_SIZE'],
batch_size=params['BATCH_SIZE'],
num_threads=params['NUM_THREADS'],
learning_rate=params['DEFAULT_LEARNING_RATE'],
epsilon=params['DEFAULT_EPSILON'],
max_iter=params['MAX_ITER'],
sigma_multiplier=params['DEFAULT_SIGMA_MULTIPLIER'],
mu_multiplier=params['DEFAULT_MU_MULTIPLIER'],
ridge=params['DEFAULT_RIDGE'])
model.fit(X_scaled, y_scaled, X_min, X_max)
res = model.predict(X_samples, constraint_helper=constraint_helper)
@ -771,7 +764,7 @@ def configuration_recommendation(recommendation_input):
best_config = res.minl_conf[best_config_idx, :]
best_config = X_scaler.inverse_transform(best_config)
if ENABLE_DUMMY_ENCODER:
if enable_dummy_encoder:
# Decode one-hot encoding into categorical knobs
best_config = dummy_encoder.inverse_transform(best_config)
@ -821,6 +814,8 @@ def map_workload(map_workload_input):
target_data['pipeline_run'] = latest_pipeline_run.pk
newest_result = Result.objects.get(pk=target_data['newest_result_id'])
session = newest_result.session
params = JSONUtil.loads(session.hyper_parameters)
target_workload = newest_result.workload
X_columnlabels = np.array(target_data['X_columnlabels'])
y_columnlabels = np.array(target_data['y_columnlabels'])
@ -870,7 +865,7 @@ def map_workload(map_workload_input):
# for the first workload
global_ranked_knobs = load_data_helper(
pipeline_data, unique_workload,
PipelineTaskType.RANKED_KNOBS)[:IMPORTANT_KNOB_NUMBER]
PipelineTaskType.RANKED_KNOBS)[:params['IMPORTANT_KNOB_NUMBER']]
global_pruned_metrics = load_data_helper(
pipeline_data, unique_workload, PipelineTaskType.PRUNED_METRICS)
ranked_knob_idxs = [i for i in range(X_matrix.shape[1]) if X_columnlabels[
@ -935,11 +930,11 @@ def map_workload(map_workload_input):
# and then predict the performance of each metric for each of
# the knob configurations attempted so far by the target.
y_col = y_col.reshape(-1, 1)
model = GPRNP(length_scale=DEFAULT_LENGTH_SCALE,
magnitude=DEFAULT_MAGNITUDE,
max_train_size=MAX_TRAIN_SIZE,
batch_size=BATCH_SIZE)
model.fit(X_scaled, y_col, ridge=DEFAULT_RIDGE)
model = GPRNP(length_scale=params['DEFAULT_LENGTH_SCALE'],
magnitude=params['DEFAULT_MAGNITUDE'],
max_train_size=params['MAX_TRAIN_SIZE'],
batch_size=params['BATCH_SIZE'])
model.fit(X_scaled, y_col, ridge=params['DEFAULT_RIDGE'])
predictions[:, j] = model.predict(X_target).ypreds.ravel()
# Bin each of the predicted metric columns by deciles and then
# compute the score (i.e., distance) between the target workload

6
server/website/website/tasks/periodic_tasks.py
View File

@ -18,7 +18,7 @@ from analysis.preprocessing import (Bin, get_shuffle_indices,
DummyEncoder,
consolidate_columnlabels)
from website.models import PipelineData, PipelineRun, Result, Workload
from website.settings import RUN_EVERY, ENABLE_DUMMY_ENCODER
from website.settings import run_every, enable_dummy_encoder
from website.types import PipelineTaskType, WorkloadStatusType
from website.utils import DataUtil, JSONUtil
@ -29,7 +29,7 @@ MIN_WORKLOAD_RESULTS_COUNT = 5
# Run the background tasks every 'RUN_EVERY' seconds
@periodic_task(run_every=RUN_EVERY, name="run_background_tasks")
@periodic_task(run_every=run_every, name="run_background_tasks")
def run_background_tasks():
LOG.debug("Starting background tasks")
# Find modified and not modified workloads, we only have to calculate for the
@ -296,7 +296,7 @@ def run_knob_identification(knob_data, metric_data, dbms):
nonconst_metric_columnlabels.append(cl)
nonconst_metric_matrix = np.hstack(nonconst_metric_matrix)
if ENABLE_DUMMY_ENCODER:
if enable_dummy_encoder:
# determine which knobs need encoding (enums with >2 possible values)
categorical_info = DataUtil.dummy_encoder_helper(nonconst_knob_columnlabels,

4
server/website/website/templates/edit_session.html
View File

@ -49,6 +49,10 @@
<td>{{ form.target_objective.label_tag }}</td>
<td>{{ form.target_objective }}</td>
</tr>
<tr id="target_obj_row">
<td>{{ form.hyper_parameters.label_tag }}</td>
<td>{{ form.hyper_parameters }}</td>
</tr>
<tr id="upload_code_row">
<td>{{ form.gen_upload_code.label_tag }}</td>
<td>{{ form.gen_upload_code }}</td>

4
server/website/website/views.py
View File

@ -40,7 +40,7 @@ from .tasks import (aggregate_target_results, map_workload, train_ddpg,
configuration_recommendation, configuration_recommendation_ddpg)
from .types import (DBMSType, KnobUnitType, MetricType,
TaskType, VarType, WorkloadStatusType, AlgorithmType)
from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil
from .utils import (JSONUtil, LabelUtil, MediaUtil, TaskUtil, ConversionUtil, get_constants)
from .settings import LOG_DIR, TIME_ZONE
from .set_default_knobs import set_default_knobs
@ -330,12 +330,14 @@ def create_or_edit_session(request, project_id, session_id=''):
else:
# Return a new form with defaults for creating a new session
session = None
hyper_parameters = JSONUtil.dumps(utils.get_constants())
form_kwargs.update(
initial={
'dbms': DBMSCatalog.objects.get(
type=DBMSType.POSTGRES, version='9.6'),
'algorithm': AlgorithmType.GPR,
'target_objective': target_objectives.default(),
'hyper_parameters': hyper_parameters
})
form = SessionForm(**form_kwargs)
context = {