public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

make session hyperparameters editable on the website

This commit is contained in:
yangdsh 2020-01-12 10:07:15 +00:00 committed by Dana Van Aken
parent 6bf50b892d
commit 24194293bc
8 changed files with 106 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/website/website/forms.py
View File

@ -134,11 +134,12 @@ class SessionForm(forms.ModelForm):
model = Session model = Session
fields = ('name', 'description', 'tuning_session', 'dbms', 'cpu', 'memory', 'storage', fields = ('name', 'description', 'tuning_session', 'dbms', 'cpu', 'memory', 'storage',
'algorithm', 'target_objective') 'algorithm', 'target_objective', 'hyper_parameters')
widgets = { widgets = {
'name': forms.TextInput(attrs={'required': True}), 'name': forms.TextInput(attrs={'required': True}),
'description': forms.Textarea(attrs={'maxlength': 500, 'rows': 5}), 'description': forms.Textarea(attrs={'maxlength': 500, 'rows': 5}),
'hyper_parameters': forms.Textarea(attrs={'maxlength': 2000, 'rows': 10}),
} }
labels = { labels = {
'dbms': 'DBMS', 'dbms': 'DBMS',

20
server/website/website/migrations/0006_session_hyper_parameters.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2020-01-12 07:29
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('website', '0005_add_workload_field'),
]
operations = [
migrations.AddField(
model_name='session',
name='hyper_parameters',
field=models.TextField(default='{}'),
),
]

1
server/website/website/models.py
View File

@ -162,6 +162,7 @@ class Session(BaseModel):
target_objective = models.CharField( target_objective = models.CharField(
max_length=64, default=target_objectives.default()) max_length=64, default=target_objectives.default())
hyper_parameters = models.TextField(default="{}")
def clean(self): def clean(self):
if self.target_objective is None: if self.target_objective is None:

18
server/website/website/settings/constants.py
View File

@ -4,10 +4,22 @@
# Copyright (c) 2017-18, Carnegie Mellon University Database Group # Copyright (c) 2017-18, Carnegie Mellon University Database Group
# #
# ---------------------------------------------
# These parameters are not specified for any session, so they can only be set here
# address categorical knobs (enum, boolean)
enable_dummy_encoder = False
# ---PIPELINE CONSTANTS--- # ---PIPELINE CONSTANTS---
# how often to run the background tests, in seconds # how often to run the background tests, in seconds
RUN_EVERY = 300 run_every = 300
# ---------------------------------------------
# The following parameters can be viewed and modified on the session page on the website
# ---SAMPLING CONSTANTS---
# the number of samples (staring points) in gradient descent # the number of samples (staring points) in gradient descent
NUM_SAMPLES = 30 NUM_SAMPLES = 30
@ -20,10 +32,6 @@ IMPORTANT_KNOB_NUMBER = 10000
TOP_NUM_CONFIG = 10 TOP_NUM_CONFIG = 10
# ---CONSTRAINTS CONSTANTS--- # ---CONSTRAINTS CONSTANTS---
# address categorical knobs (enum, boolean)
ENABLE_DUMMY_ENCODER = False
# Initial probability to flip categorical feature in apply_constraints # Initial probability to flip categorical feature in apply_constraints
# server/analysis/constraints.py # server/analysis/constraints.py
INIT_FLIP_PROB = 0.3 INIT_FLIP_PROB = 0.3

125
server/website/website/tasks/async_tasks.py
View File

@ -30,23 +30,7 @@ from website.models import (PipelineData, PipelineRun, Result, Workload, KnobCat
from website import db from website import db
from website.types import PipelineTaskType, AlgorithmType, VarType from website.types import PipelineTaskType, AlgorithmType, VarType
from website.utils import DataUtil, JSONUtil from website.utils import DataUtil, JSONUtil
from website.settings import IMPORTANT_KNOB_NUMBER, NUM_SAMPLES, TOP_NUM_CONFIG # pylint: disable=no-name-in-module from website.settings import enable_dummy_encoder
from website.settings import (USE_GPFLOW, DEFAULT_LENGTH_SCALE, DEFAULT_MAGNITUDE,
MAX_TRAIN_SIZE, BATCH_SIZE, NUM_THREADS,
DEFAULT_RIDGE, DEFAULT_LEARNING_RATE,
DEFAULT_EPSILON, MAX_ITER, GPR_EPS,
DEFAULT_SIGMA_MULTIPLIER, DEFAULT_MU_MULTIPLIER,
DEFAULT_UCB_SCALE, HP_LEARNING_RATE, HP_MAX_ITER,
DDPG_SIMPLE_REWARD, DDPG_GAMMA, USE_DEFAULT,
DDPG_BATCH_SIZE, ACTOR_LEARNING_RATE,
CRITIC_LEARNING_RATE, UPDATE_EPOCHS,
ACTOR_HIDDEN_SIZES, CRITIC_HIDDEN_SIZES,
DNN_TRAIN_ITER, DNN_EXPLORE, DNN_EXPLORE_ITER,
DNN_NOISE_SCALE_BEGIN, DNN_NOISE_SCALE_END,
DNN_DEBUG, DNN_DEBUG_INTERVAL, GPR_DEBUG, UCB_BETA,
GPR_MODEL_NAME, ENABLE_DUMMY_ENCODER, DNN_GD_ITER)
from website.settings import INIT_FLIP_PROB, FLIP_PROB_DECAY
LOG = get_task_logger(__name__) LOG = get_task_logger(__name__)
@ -339,6 +323,7 @@ def train_ddpg(result_id):
LOG.info('Add training data to ddpg and train ddpg') LOG.info('Add training data to ddpg and train ddpg')
result = Result.objects.get(pk=result_id) result = Result.objects.get(pk=result_id)
session = Result.objects.get(pk=result_id).session session = Result.objects.get(pk=result_id).session
params = JSONUtil.loads(session.hyper_parameters)
session_results = Result.objects.filter(session=session, session_results = Result.objects.filter(session=session,
creation_time__lt=result.creation_time) creation_time__lt=result.creation_time)
result_info = {} result_info = {}
@ -418,16 +403,16 @@ def train_ddpg(result_id):
LOG.info('reward: %f', reward) LOG.info('reward: %f', reward)
# Update ddpg # Update ddpg
ddpg = DDPG(n_actions=knob_num, n_states=metric_num, alr=ACTOR_LEARNING_RATE, ddpg = DDPG(n_actions=knob_num, n_states=metric_num, alr=params['ACTOR_LEARNING_RATE'],
clr=CRITIC_LEARNING_RATE, gamma=DDPG_GAMMA, batch_size=DDPG_BATCH_SIZE, clr=params['CRITIC_LEARNING_RATE'], gamma=params['DDPG_GAMMA'],
a_hidden_sizes=ACTOR_HIDDEN_SIZES, c_hidden_sizes=CRITIC_HIDDEN_SIZES, batch_size=params['DDPG_BATCH_SIZE'], a_hidden_sizes=params['ACTOR_HIDDEN_SIZES'],
use_default=USE_DEFAULT) c_hidden_sizes=params['CRITIC_HIDDEN_SIZES'], use_default=params['USE_DEFAULT'])
if session.ddpg_actor_model and session.ddpg_critic_model: if session.ddpg_actor_model and session.ddpg_critic_model:
ddpg.set_model(session.ddpg_actor_model, session.ddpg_critic_model) ddpg.set_model(session.ddpg_actor_model, session.ddpg_critic_model)
if session.ddpg_reply_memory: if session.ddpg_reply_memory:
ddpg.replay_memory.set(session.ddpg_reply_memory) ddpg.replay_memory.set(session.ddpg_reply_memory)
ddpg.add_sample(normalized_metric_data, knob_data, reward, normalized_metric_data) ddpg.add_sample(normalized_metric_data, knob_data, reward, normalized_metric_data)
for _ in range(UPDATE_EPOCHS): for _ in range(params['UPDATE_EPOCHS']):
ddpg.update() ddpg.update()
session.ddpg_actor_model, session.ddpg_critic_model = ddpg.get_model() session.ddpg_actor_model, session.ddpg_critic_model = ddpg.get_model()
session.ddpg_reply_memory = ddpg.replay_memory.get() session.ddpg_reply_memory = ddpg.replay_memory.get()
@ -459,6 +444,7 @@ def configuration_recommendation_ddpg(result_info): # pylint: disable=invalid-n
result_list = Result.objects.filter(pk=result_id) result_list = Result.objects.filter(pk=result_id)
result = result_list.first() result = result_list.first()
session = result.session session = result.session
params = JSONUtil.loads(session.hyper_parameters)
agg_data = DataUtil.aggregate_data(result_list) agg_data = DataUtil.aggregate_data(result_list)
metric_data, _ = clean_metric_data(agg_data['y_matrix'], agg_data['y_columnlabels'], session) metric_data, _ = clean_metric_data(agg_data['y_matrix'], agg_data['y_columnlabels'], session)
metric_data = metric_data.flatten() metric_data = metric_data.flatten()
@ -469,8 +455,9 @@ def configuration_recommendation_ddpg(result_info): # pylint: disable=invalid-n
knob_num = len(knob_labels) knob_num = len(knob_labels)
metric_num = len(metric_data) metric_num = len(metric_data)
ddpg = DDPG(n_actions=knob_num, n_states=metric_num, a_hidden_sizes=ACTOR_HIDDEN_SIZES, ddpg = DDPG(n_actions=knob_num, n_states=metric_num,
c_hidden_sizes=CRITIC_HIDDEN_SIZES, use_default=USE_DEFAULT) a_hidden_sizes=params['ACTOR_HIDDEN_SIZES'],
c_hidden_sizes=params['CRITIC_HIDDEN_SIZES'], use_default=params['USE_DEFAULT'])
if session.ddpg_actor_model is not None and session.ddpg_critic_model is not None: if session.ddpg_actor_model is not None and session.ddpg_critic_model is not None:
ddpg.set_model(session.ddpg_actor_model, session.ddpg_critic_model) ddpg.set_model(session.ddpg_actor_model, session.ddpg_critic_model)
if session.ddpg_reply_memory is not None: if session.ddpg_reply_memory is not None:
@ -505,6 +492,8 @@ def combine_workload(target_data):
workload_metric_data = JSONUtil.loads(workload_metric_data.data) workload_metric_data = JSONUtil.loads(workload_metric_data.data)
newest_result = Result.objects.get(pk=target_data['newest_result_id']) newest_result = Result.objects.get(pk=target_data['newest_result_id'])
session = newest_result.session
params = JSONUtil.loads(session.hyper_parameters)
cleaned_workload_knob_data = clean_knob_data(workload_knob_data["data"], cleaned_workload_knob_data = clean_knob_data(workload_knob_data["data"],
workload_knob_data["columnlabels"], workload_knob_data["columnlabels"],
newest_result.session) newest_result.session)
@ -516,7 +505,7 @@ def combine_workload(target_data):
rowlabels_workload = np.array(workload_metric_data['rowlabels']) rowlabels_workload = np.array(workload_metric_data['rowlabels'])
# Target workload data # Target workload data
newest_result = Result.objects.get(pk=target_data['newest_result_id']) newest_result = Result.objects.get(pk=target_data['newest_result_id'])
X_target = target_data['X_matrix'] X_target = target_data['X_matrix']
y_target = target_data['y_matrix'] y_target = target_data['y_matrix']
rowlabels_target = np.array(target_data['rowlabels']) rowlabels_target = np.array(target_data['rowlabels'])
@ -535,7 +524,7 @@ def combine_workload(target_data):
pipeline_run=latest_pipeline_run, pipeline_run=latest_pipeline_run,
workload=mapped_workload, workload=mapped_workload,
task_type=PipelineTaskType.RANKED_KNOBS) task_type=PipelineTaskType.RANKED_KNOBS)
ranked_knobs = JSONUtil.loads(ranked_knobs.data)[:IMPORTANT_KNOB_NUMBER] ranked_knobs = JSONUtil.loads(ranked_knobs.data)[:params['IMPORTANT_KNOB_NUMBER']]
ranked_knob_idxs = [i for i, cl in enumerate(X_columnlabels) if cl in ranked_knobs] ranked_knob_idxs = [i for i, cl in enumerate(X_columnlabels) if cl in ranked_knobs]
X_workload = X_workload[:, ranked_knob_idxs] X_workload = X_workload[:, ranked_knob_idxs]
X_target = X_target[:, ranked_knob_idxs] X_target = X_target[:, ranked_knob_idxs]
@ -577,7 +566,7 @@ def combine_workload(target_data):
X_matrix = np.vstack([X_target, X_workload]) X_matrix = np.vstack([X_target, X_workload])
# Dummy encode categorial variables # Dummy encode categorial variables
if ENABLE_DUMMY_ENCODER: if enable_dummy_encoder:
categorical_info = DataUtil.dummy_encoder_helper(X_columnlabels, categorical_info = DataUtil.dummy_encoder_helper(X_columnlabels,
mapped_workload.dbms) mapped_workload.dbms)
dummy_encoder = DummyEncoder(categorical_info['n_values'], dummy_encoder = DummyEncoder(categorical_info['n_values'],
@ -632,8 +621,8 @@ def combine_workload(target_data):
constraint_helper = ParamConstraintHelper(scaler=X_scaler, constraint_helper = ParamConstraintHelper(scaler=X_scaler,
encoder=dummy_encoder, encoder=dummy_encoder,
binary_vars=binary_encoder, binary_vars=binary_encoder,
init_flip_prob=INIT_FLIP_PROB, init_flip_prob=params['INIT_FLIP_PROB'],
flip_prob_decay=FLIP_PROB_DECAY) flip_prob_decay=params['FLIP_PROB_DECAY'])
# FIXME (dva): check if these are good values for the ridge # FIXME (dva): check if these are good values for the ridge
# ridge = np.empty(X_scaled.shape[0]) # ridge = np.empty(X_scaled.shape[0])
@ -671,6 +660,8 @@ def configuration_recommendation(recommendation_input):
target_data, algorithm = recommendation_input target_data, algorithm = recommendation_input
LOG.info('configuration_recommendation called') LOG.info('configuration_recommendation called')
newest_result = Result.objects.get(pk=target_data['newest_result_id']) newest_result = Result.objects.get(pk=target_data['newest_result_id'])
session = newest_result.session
params = session.hyper_parameters
if target_data['bad'] is True: if target_data['bad'] is True:
target_data_res = create_and_save_recommendation( target_data_res = create_and_save_recommendation(
@ -687,7 +678,7 @@ def configuration_recommendation(recommendation_input):
# FIXME: we should generate more samples and use a smarter sampling # FIXME: we should generate more samples and use a smarter sampling
# technique # technique
num_samples = NUM_SAMPLES num_samples = params['NUM_SAMPLES']
X_samples = np.empty((num_samples, X_scaled.shape[1])) X_samples = np.empty((num_samples, X_scaled.shape[1]))
for i in range(X_scaled.shape[1]): for i in range(X_scaled.shape[1]):
X_samples[:, i] = np.random.rand(num_samples) * (X_max[i] - X_min[i]) + X_min[i] X_samples[:, i] = np.random.rand(num_samples) * (X_max[i] - X_min[i]) + X_min[i]
@ -697,7 +688,7 @@ def configuration_recommendation(recommendation_input):
q.put((y_scaled[x][0], x)) q.put((y_scaled[x][0], x))
i = 0 i = 0
while i < TOP_NUM_CONFIG: while i < params['TOP_NUM_CONFIG']:
try: try:
item = q.get_nowait() item = q.get_nowait()
# Tensorflow get broken if we use the training data points as # Tensorflow get broken if we use the training data points as
@ -714,56 +705,58 @@ def configuration_recommendation(recommendation_input):
except queue.Empty: except queue.Empty:
break break
session = newest_result.session
res = None res = None
if algorithm == AlgorithmType.DNN: if algorithm == AlgorithmType.DNN:
# neural network model # neural network model
model_nn = NeuralNet(n_input=X_samples.shape[1], model_nn = NeuralNet(n_input=X_samples.shape[1],
batch_size=X_samples.shape[0], batch_size=X_samples.shape[0],
explore_iters=DNN_EXPLORE_ITER, explore_iters=params['DNN_EXPLORE_ITER'],
noise_scale_begin=DNN_NOISE_SCALE_BEGIN, noise_scale_begin=params['DNN_NOISE_SCALE_BEGIN'],
noise_scale_end=DNN_NOISE_SCALE_END, noise_scale_end=params['DNN_NOISE_SCALE_END'],
debug=DNN_DEBUG, debug=params['DNN_DEBUG'],
debug_interval=DNN_DEBUG_INTERVAL) debug_interval=params['DNN_DEBUG_INTERVAL'])
if session.dnn_model is not None: if session.dnn_model is not None:
model_nn.set_weights_bin(session.dnn_model) model_nn.set_weights_bin(session.dnn_model)
model_nn.fit(X_scaled, y_scaled, fit_epochs=DNN_TRAIN_ITER) model_nn.fit(X_scaled, y_scaled, fit_epochs=params['DNN_TRAIN_ITER'])
res = model_nn.recommend(X_samples, X_min, X_max, res = model_nn.recommend(X_samples, X_min, X_max,
explore=DNN_EXPLORE, recommend_epochs=DNN_GD_ITER) explore=params['DNN_EXPLORE'],
recommend_epochs=params['DNN_GD_ITER'])
session.dnn_model = model_nn.get_weights_bin() session.dnn_model = model_nn.get_weights_bin()
session.save() session.save()
elif algorithm == AlgorithmType.GPR: elif algorithm == AlgorithmType.GPR:
# default gpr model # default gpr model
if USE_GPFLOW: if params['USE_GPFLOW']:
model_kwargs = {} model_kwargs = {}
model_kwargs['model_learning_rate'] = HP_LEARNING_RATE model_kwargs['model_learning_rate'] = params['HP_LEARNING_RATE']
model_kwargs['model_maxiter'] = HP_MAX_ITER model_kwargs['model_maxiter'] = params['HP_MAX_ITER']
opt_kwargs = {} opt_kwargs = {}
opt_kwargs['learning_rate'] = DEFAULT_LEARNING_RATE opt_kwargs['learning_rate'] = params['DEFAULT_LEARNING_RATE']
opt_kwargs['maxiter'] = MAX_ITER opt_kwargs['maxiter'] = params['MAX_ITER']
opt_kwargs['bounds'] = [X_min, X_max] opt_kwargs['bounds'] = [X_min, X_max]
opt_kwargs['debug'] = GPR_DEBUG opt_kwargs['debug'] = params['GPR_DEBUG']
opt_kwargs['ucb_beta'] = ucb.get_ucb_beta(UCB_BETA, scale=DEFAULT_UCB_SCALE, opt_kwargs['ucb_beta'] = ucb.get_ucb_beta(params['UCB_BETA'],
scale=params['DEFAULT_UCB_SCALE'],
t=i + 1., ndim=X_scaled.shape[1]) t=i + 1., ndim=X_scaled.shape[1])
tf.reset_default_graph() tf.reset_default_graph()
graph = tf.get_default_graph() graph = tf.get_default_graph()
gpflow.reset_default_session(graph=graph) gpflow.reset_default_session(graph=graph)
m = gpr_models.create_model(GPR_MODEL_NAME, X=X_scaled, y=y_scaled, **model_kwargs) m = gpr_models.create_model(params['GPR_MODEL_NAME'], X=X_scaled, y=y_scaled,
**model_kwargs)
res = tf_optimize(m.model, X_samples, **opt_kwargs) res = tf_optimize(m.model, X_samples, **opt_kwargs)
else: else:
model = GPRGD(length_scale=DEFAULT_LENGTH_SCALE, model = GPRGD(length_scale=params['DEFAULT_LENGTH_SCALE'],
magnitude=DEFAULT_MAGNITUDE, magnitude=params['DEFAULT_MAGNITUDE'],
max_train_size=MAX_TRAIN_SIZE, max_train_size=params['MAX_TRAIN_SIZE'],
batch_size=BATCH_SIZE, batch_size=params['BATCH_SIZE'],
num_threads=NUM_THREADS, num_threads=params['NUM_THREADS'],
learning_rate=DEFAULT_LEARNING_RATE, learning_rate=params['DEFAULT_LEARNING_RATE'],
epsilon=DEFAULT_EPSILON, epsilon=params['DEFAULT_EPSILON'],
max_iter=MAX_ITER, max_iter=params['MAX_ITER'],
sigma_multiplier=DEFAULT_SIGMA_MULTIPLIER, sigma_multiplier=params['DEFAULT_SIGMA_MULTIPLIER'],
mu_multiplier=DEFAULT_MU_MULTIPLIER, mu_multiplier=params['DEFAULT_MU_MULTIPLIER'],
ridge=DEFAULT_RIDGE) ridge=params['DEFAULT_RIDGE'])
model.fit(X_scaled, y_scaled, X_min, X_max) model.fit(X_scaled, y_scaled, X_min, X_max)
res = model.predict(X_samples, constraint_helper=constraint_helper) res = model.predict(X_samples, constraint_helper=constraint_helper)
@ -771,7 +764,7 @@ def configuration_recommendation(recommendation_input):
best_config = res.minl_conf[best_config_idx, :] best_config = res.minl_conf[best_config_idx, :]
best_config = X_scaler.inverse_transform(best_config) best_config = X_scaler.inverse_transform(best_config)
if ENABLE_DUMMY_ENCODER: if enable_dummy_encoder:
# Decode one-hot encoding into categorical knobs # Decode one-hot encoding into categorical knobs
best_config = dummy_encoder.inverse_transform(best_config) best_config = dummy_encoder.inverse_transform(best_config)
@ -821,6 +814,8 @@ def map_workload(map_workload_input):
target_data['pipeline_run'] = latest_pipeline_run.pk target_data['pipeline_run'] = latest_pipeline_run.pk
newest_result = Result.objects.get(pk=target_data['newest_result_id']) newest_result = Result.objects.get(pk=target_data['newest_result_id'])
session = newest_result.session
params = JSONUtil.loads(session.hyper_parameters)
target_workload = newest_result.workload target_workload = newest_result.workload
X_columnlabels = np.array(target_data['X_columnlabels']) X_columnlabels = np.array(target_data['X_columnlabels'])
y_columnlabels = np.array(target_data['y_columnlabels']) y_columnlabels = np.array(target_data['y_columnlabels'])
@ -870,7 +865,7 @@ def map_workload(map_workload_input):
# for the first workload # for the first workload
global_ranked_knobs = load_data_helper( global_ranked_knobs = load_data_helper(
pipeline_data, unique_workload, pipeline_data, unique_workload,
PipelineTaskType.RANKED_KNOBS)[:IMPORTANT_KNOB_NUMBER] PipelineTaskType.RANKED_KNOBS)[:params['IMPORTANT_KNOB_NUMBER']]
global_pruned_metrics = load_data_helper( global_pruned_metrics = load_data_helper(
pipeline_data, unique_workload, PipelineTaskType.PRUNED_METRICS) pipeline_data, unique_workload, PipelineTaskType.PRUNED_METRICS)
ranked_knob_idxs = [i for i in range(X_matrix.shape[1]) if X_columnlabels[ ranked_knob_idxs = [i for i in range(X_matrix.shape[1]) if X_columnlabels[
@ -935,11 +930,11 @@ def map_workload(map_workload_input):
# and then predict the performance of each metric for each of # and then predict the performance of each metric for each of
# the knob configurations attempted so far by the target. # the knob configurations attempted so far by the target.
y_col = y_col.reshape(-1, 1) y_col = y_col.reshape(-1, 1)
model = GPRNP(length_scale=DEFAULT_LENGTH_SCALE, model = GPRNP(length_scale=params['DEFAULT_LENGTH_SCALE'],
magnitude=DEFAULT_MAGNITUDE, magnitude=params['DEFAULT_MAGNITUDE'],
max_train_size=MAX_TRAIN_SIZE, max_train_size=params['MAX_TRAIN_SIZE'],
batch_size=BATCH_SIZE) batch_size=params['BATCH_SIZE'])
model.fit(X_scaled, y_col, ridge=DEFAULT_RIDGE) model.fit(X_scaled, y_col, ridge=params['DEFAULT_RIDGE'])
predictions[:, j] = model.predict(X_target).ypreds.ravel() predictions[:, j] = model.predict(X_target).ypreds.ravel()
# Bin each of the predicted metric columns by deciles and then # Bin each of the predicted metric columns by deciles and then
# compute the score (i.e., distance) between the target workload # compute the score (i.e., distance) between the target workload

6
server/website/website/tasks/periodic_tasks.py
View File

@ -18,7 +18,7 @@ from analysis.preprocessing import (Bin, get_shuffle_indices,
DummyEncoder, DummyEncoder,
consolidate_columnlabels) consolidate_columnlabels)
from website.models import PipelineData, PipelineRun, Result, Workload from website.models import PipelineData, PipelineRun, Result, Workload
from website.settings import RUN_EVERY, ENABLE_DUMMY_ENCODER from website.settings import run_every, enable_dummy_encoder
from website.types import PipelineTaskType, WorkloadStatusType from website.types import PipelineTaskType, WorkloadStatusType
from website.utils import DataUtil, JSONUtil from website.utils import DataUtil, JSONUtil
@ -29,7 +29,7 @@ MIN_WORKLOAD_RESULTS_COUNT = 5
# Run the background tasks every 'RUN_EVERY' seconds # Run the background tasks every 'RUN_EVERY' seconds
@periodic_task(run_every=RUN_EVERY, name="run_background_tasks") @periodic_task(run_every=run_every, name="run_background_tasks")
def run_background_tasks(): def run_background_tasks():
LOG.debug("Starting background tasks") LOG.debug("Starting background tasks")
# Find modified and not modified workloads, we only have to calculate for the # Find modified and not modified workloads, we only have to calculate for the
@ -296,7 +296,7 @@ def run_knob_identification(knob_data, metric_data, dbms):
nonconst_metric_columnlabels.append(cl) nonconst_metric_columnlabels.append(cl)
nonconst_metric_matrix = np.hstack(nonconst_metric_matrix) nonconst_metric_matrix = np.hstack(nonconst_metric_matrix)
if ENABLE_DUMMY_ENCODER: if enable_dummy_encoder:
# determine which knobs need encoding (enums with >2 possible values) # determine which knobs need encoding (enums with >2 possible values)
categorical_info = DataUtil.dummy_encoder_helper(nonconst_knob_columnlabels, categorical_info = DataUtil.dummy_encoder_helper(nonconst_knob_columnlabels,

4
server/website/website/templates/edit_session.html
View File

@ -49,6 +49,10 @@
<td>{{ form.target_objective.label_tag }}</td> <td>{{ form.target_objective.label_tag }}</td>
<td>{{ form.target_objective }}</td> <td>{{ form.target_objective }}</td>
</tr> </tr>
<tr id="target_obj_row">
<td>{{ form.hyper_parameters.label_tag }}</td>
<td>{{ form.hyper_parameters }}</td>
</tr>
<tr id="upload_code_row"> <tr id="upload_code_row">
<td>{{ form.gen_upload_code.label_tag }}</td> <td>{{ form.gen_upload_code.label_tag }}</td>
<td>{{ form.gen_upload_code }}</td> <td>{{ form.gen_upload_code }}</td>

4
server/website/website/views.py
View File

@ -40,7 +40,7 @@ from .tasks import (aggregate_target_results, map_workload, train_ddpg,
configuration_recommendation, configuration_recommendation_ddpg) configuration_recommendation, configuration_recommendation_ddpg)
from .types import (DBMSType, KnobUnitType, MetricType, from .types import (DBMSType, KnobUnitType, MetricType,
TaskType, VarType, WorkloadStatusType, AlgorithmType) TaskType, VarType, WorkloadStatusType, AlgorithmType)
from .utils import JSONUtil, LabelUtil, MediaUtil, TaskUtil from .utils import (JSONUtil, LabelUtil, MediaUtil, TaskUtil, ConversionUtil, get_constants)
from .settings import LOG_DIR, TIME_ZONE from .settings import LOG_DIR, TIME_ZONE
from .set_default_knobs import set_default_knobs from .set_default_knobs import set_default_knobs
@ -330,12 +330,14 @@ def create_or_edit_session(request, project_id, session_id=''):
else: else:
# Return a new form with defaults for creating a new session # Return a new form with defaults for creating a new session
session = None session = None
hyper_parameters = JSONUtil.dumps(utils.get_constants())
form_kwargs.update( form_kwargs.update(
initial={ initial={
'dbms': DBMSCatalog.objects.get( 'dbms': DBMSCatalog.objects.get(
type=DBMSType.POSTGRES, version='9.6'), type=DBMSType.POSTGRES, version='9.6'),
'algorithm': AlgorithmType.GPR, 'algorithm': AlgorithmType.GPR,
'target_objective': target_objectives.default(), 'target_objective': target_objectives.default(),
'hyper_parameters': hyper_parameters
}) })
form = SessionForm(**form_kwargs) form = SessionForm(**form_kwargs)
context = { context = {