public
/
ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

skip workload mapping when there is no workload

This commit is contained in:
yangdsh 2020-02-18 16:44:25 +00:00 committed by Dana Van Aken
parent 59b723ee83
commit f7e22ff5bb
1 changed files with 51 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
server/website/website/tasks/async_tasks.py
View File

@ -65,7 +65,7 @@ class MapWorkloadTask(BaseTask): # pylint: disable=abstract-method
new_res = None new_res = None
# Replace result with formatted result # Replace result with formatted result
if not args[0][0]['bad']: if not args[0][0]['bad'] and args[0][0]['mapped_workload'] is not None:
new_res = { new_res = {
'scores': sorted(args[0][0]['scores'].items()), 'scores': sorted(args[0][0]['scores'].items()),
'mapped_workload_id': args[0][0]['mapped_workload'], 'mapped_workload_id': args[0][0]['mapped_workload'],
@ -469,34 +469,40 @@ def configuration_recommendation_ddpg(result_info): # pylint: disable=invalid-n
def combine_workload(target_data): def combine_workload(target_data):
# Load mapped workload data
mapped_workload_id = target_data['mapped_workload'][0]
latest_pipeline_run = PipelineRun.objects.get(pk=target_data['pipeline_run'])
mapped_workload = Workload.objects.get(pk=mapped_workload_id)
workload_knob_data = PipelineData.objects.get(
pipeline_run=latest_pipeline_run,
workload=mapped_workload,
task_type=PipelineTaskType.KNOB_DATA)
workload_knob_data = JSONUtil.loads(workload_knob_data.data)
workload_metric_data = PipelineData.objects.get(
pipeline_run=latest_pipeline_run,
workload=mapped_workload,
task_type=PipelineTaskType.METRIC_DATA)
workload_metric_data = JSONUtil.loads(workload_metric_data.data)
newest_result = Result.objects.get(pk=target_data['newest_result_id']) newest_result = Result.objects.get(pk=target_data['newest_result_id'])
latest_pipeline_run = PipelineRun.objects.get(pk=target_data['pipeline_run'])
session = newest_result.session session = newest_result.session
params = JSONUtil.loads(session.hyperparameters) params = JSONUtil.loads(session.hyperparameters)
cleaned_workload_knob_data = clean_knob_data(workload_knob_data["data"],
workload_knob_data["columnlabels"],
newest_result.session)
X_workload = np.array(cleaned_workload_knob_data[0]) # Load mapped workload data
X_columnlabels = np.array(cleaned_workload_knob_data[1]) if target_data['mapped_workload'] is not None:
y_workload = np.array(workload_metric_data['data']) mapped_workload_id = target_data['mapped_workload'][0]
y_columnlabels = np.array(workload_metric_data['columnlabels']) mapped_workload = Workload.objects.get(pk=mapped_workload_id)
rowlabels_workload = np.array(workload_metric_data['rowlabels']) workload_knob_data = PipelineData.objects.get(
pipeline_run=latest_pipeline_run,
workload=mapped_workload,
task_type=PipelineTaskType.KNOB_DATA)
workload_knob_data = JSONUtil.loads(workload_knob_data.data)
workload_metric_data = PipelineData.objects.get(
pipeline_run=latest_pipeline_run,
workload=mapped_workload,
task_type=PipelineTaskType.METRIC_DATA)
workload_metric_data = JSONUtil.loads(workload_metric_data.data)
cleaned_workload_knob_data = clean_knob_data(workload_knob_data["data"],
workload_knob_data["columnlabels"],
newest_result.session)
X_workload = np.array(cleaned_workload_knob_data[0])
X_columnlabels = np.array(cleaned_workload_knob_data[1])
y_workload = np.array(workload_metric_data['data'])
y_columnlabels = np.array(workload_metric_data['columnlabels'])
rowlabels_workload = np.array(workload_metric_data['rowlabels'])
else:
# combine the target_data with itself is actually adding nothing to the target_data
X_workload = np.array(target_data['X_matrix'])
X_columnlabels = np.array(target_data['X_columnlabels'])
y_workload = np.array(target_data['y_matrix'])
y_columnlabels = np.array(target_data['y_columnlabels'])
rowlabels_workload = np.array(target_data['rowlabels'])
# Target workload data # Target workload data
newest_result = Result.objects.get(pk=target_data['newest_result_id']) newest_result = Result.objects.get(pk=target_data['newest_result_id'])
@ -513,16 +519,17 @@ def combine_workload(target_data):
'identical y columnlabels (sorted metric names)'), 'identical y columnlabels (sorted metric names)'),
y_columnlabels, target_data['y_columnlabels']) y_columnlabels, target_data['y_columnlabels'])
# Filter Xs by top 10 ranked knobs if target_data['mapped_workload'] is not None:
ranked_knobs = PipelineData.objects.get( # Filter Xs by top 'IMPORTANT_KNOB_NUMBER' ranked knobs
pipeline_run=latest_pipeline_run, ranked_knobs = PipelineData.objects.get(
workload=mapped_workload, pipeline_run=latest_pipeline_run,
task_type=PipelineTaskType.RANKED_KNOBS) workload=mapped_workload,
ranked_knobs = JSONUtil.loads(ranked_knobs.data)[:params['IMPORTANT_KNOB_NUMBER']] task_type=PipelineTaskType.RANKED_KNOBS)
ranked_knob_idxs = [i for i, cl in enumerate(X_columnlabels) if cl in ranked_knobs] ranked_knobs = JSONUtil.loads(ranked_knobs.data)[:params['IMPORTANT_KNOB_NUMBER']]
X_workload = X_workload[:, ranked_knob_idxs] ranked_knob_idxs = [i for i, cl in enumerate(X_columnlabels) if cl in ranked_knobs]
X_target = X_target[:, ranked_knob_idxs] X_workload = X_workload[:, ranked_knob_idxs]
X_columnlabels = X_columnlabels[ranked_knob_idxs] X_target = X_target[:, ranked_knob_idxs]
X_columnlabels = X_columnlabels[ranked_knob_idxs]
# Filter ys by current target objective metric # Filter ys by current target objective metric
target_objective = newest_result.session.target_objective target_objective = newest_result.session.target_objective
@ -562,7 +569,7 @@ def combine_workload(target_data):
# Dummy encode categorial variables # Dummy encode categorial variables
if ENABLE_DUMMY_ENCODER: if ENABLE_DUMMY_ENCODER:
categorical_info = DataUtil.dummy_encoder_helper(X_columnlabels, categorical_info = DataUtil.dummy_encoder_helper(X_columnlabels,
mapped_workload.dbms) newest_result.dbms)
dummy_encoder = DummyEncoder(categorical_info['n_values'], dummy_encoder = DummyEncoder(categorical_info['n_values'],
categorical_info['categorical_features'], categorical_info['categorical_features'],
categorical_info['cat_columnlabels'], categorical_info['cat_columnlabels'],
@ -829,10 +836,16 @@ def map_workload(map_workload_input):
ranked_knob_idxs = None ranked_knob_idxs = None
pruned_metric_idxs = None pruned_metric_idxs = None
# Compute workload mapping data for each unique workload
unique_workloads = pipeline_data.values_list('workload', flat=True).distinct() unique_workloads = pipeline_data.values_list('workload', flat=True).distinct()
assert len(unique_workloads) > 0 if unique_workloads == 0:
# The background task that aggregates the data has not finished running yet
target_data.update(mapped_workload=None, scores=None)
LOG.debug('%s: Skipping workload mapping because there is no workload.\n',
AlgorithmType.name(algorithm))
return target_data, algorithm
workload_data = {} workload_data = {}
# Compute workload mapping data for each unique workload
for unique_workload in unique_workloads: for unique_workload in unique_workloads:
workload_obj = Workload.objects.get(pk=unique_workload) workload_obj = Workload.objects.get(pk=unique_workload)