How severe does this issue affect your experience of using Ray?
- High: It blocks me to complete my task.
Hi everyone, I’m trying to use Ray Tune and notice the same model training code is abysmally slow compared to when not using it. My XGBoost training takes <0.3s / iteration when running normally and 15s / iteration (!) when using Ray Tune.
Any help would be really appreciated! Here’s a reproducible example:
from time import time
import ray
import numpy as np
import pandas as pd
import xgboost as xgb
from ray.tune.search import ConcurrencyLimiter
from ray.tune.search.hyperopt import HyperOptSearch
def train_xgb(config, data=None, base_params=None):
# validation_0 refers to the first tuple of data passed to `eval_set`, in this case
# `validation`
tic = time()
model = (
xgb.XGBClassifier(**(base_params | config))
.fit(
data['train']['features'], data['train']['labels'],
eval_set=[(data['validation']['features'], data['validation']['labels'])],
verbose=True,
)
)
print(time() - tic)
results = model.evals_result()
return {f'validation-{metric}': values[-1] for metric, values in results['validation_0'].items()}
def train_model(data, tune_hyperparameters):
# Data
data = {
'train': {'features': np.random.rand(4000000, 30), 'labels': np.random.randint(0, 2, 4000000)},
'validation': {'features': np.random.rand(400000, 30), 'labels': np.random.randint(0, 2, 400000)},
}
base_params = {
'objective': 'binary:logistic',
'eval_metric': ['logloss', 'auc'],
'random_state': 0,
'n_jobs': -1,
"validate_parameters": True,
"verbosity": 1,
}
if tune_hyperparameters:
best_hparams = tune_model(
data,
base_params,
)
else:
best_hparams = {
'n_estimators': 10,
'learning_rate': 0.013921520897736126,
'grow_policy': 'depthwise',
'tree_method': 'hist',
'max_depth': 13,
'scale_pos_weight': 4.997334464166717,
}
best_model = train_xgb(config=best_hparams, data=data, base_params=base_params)
def tune_model(data, base_params):
run_config = ray.air.RunConfig(
verbose=0,
)
# Dummy
param_space = {
'n_estimators': ray.tune.randint(100, 400),
# 'learning_rate': ray.tune.loguniform(0.001, 0.1),
# 'grow_policy': ray.tune.choice(['depthwise', 'lossguide']),
# 'tree_method': ray.tune.choice(['approx', 'hist']),
# 'max_depth': ray.tune.randint(2, 15),
# 'scale_pos_weight': 1 / data['train']['labels'].mean() - 1,
}
tune_config = ray.tune.TuneConfig(
search_alg=ConcurrencyLimiter(
searcher=HyperOptSearch(
random_state_seed=0,
points_to_evaluate=[{
'n_estimators': 10,
'learning_rate': 0.013921520897736126,
'grow_policy': 'depthwise',
'tree_method': 'hist',
'max_depth': 13,
'scale_pos_weight': 4.997334464166717,
}],
),
max_concurrent=1,
),
num_samples=1,
metric='validation-auc',
mode='max',
reuse_actors=True,
)
# Note the difference in how parameters are passed into XGBoostTrainer when training
# and tuning
tuner = ray.tune.Tuner(
ray.tune.with_parameters(train_xgb, data=data, base_params=base_params),
param_space=param_space,
run_config=run_config,
tune_config=tune_config,
)
results = tuner.fit()
best_hparams = results.get_best_result().config
return best_hparams
# XGBoost - 0.5s / iteration
train_model(None, tune_hyperparameters=False)
# Ray Tune - 15s / iteration
train_model(None, tune_hyperparameters=True)