Hello all together,
i’ve been working on an hyperparametertuning with Ray and i am running into following error:
Following is the last cell of my ipynb:
def main(num_samples=10, max_num_epochs=10, gpus_per_trial=2):
config={
"dropout_rate": tune.sample_from(lambda _: np.random.randint(1, 450)*0.001),
"filter_size": tune.sample_from(lambda _: np.random.choice([3,5,7,9,11,15], (1,3))),
"nr_of_filters": tune.sample_from(lambda _: np.random.choice([2,4,6,8,12,16,32,48,64,96,128,132,148,196,224,256], (1,4))),
"LR": tune.choice([1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 5e-1, 5e-2, 5e-3, 5e-4, 5e-5]),
'batch_size' : tune.choice([1,2,3,5,7])
}
scheduler = ASHAScheduler(
max_t=max_num_epochs,
grace_period=1,
reduction_factor=2)
tuner = tune.Tuner(
tune.with_resources(
tune.with_parameters(train),
resources={"cpu": 8, "gpu": gpus_per_trial}
),
tune_config=tune.TuneConfig(
metric="loss",
mode="min",
scheduler=scheduler,
num_samples=num_samples,
),
param_space=config,
)
results = tuner.fit()
best_result = results.get_best_result("loss", "min")
print("Best trial config: {}".format(best_result.config))
print("Best trial final validation loss: {}".format(
best_result.metrics["loss"]))
print("Best trial final validation accuracy: {}".format(
best_result.metrics["accuracy"]))
test_best_model(best_result)
main(num_samples=1, max_num_epochs=2, gpus_per_trial=0)
Error:
2023-05-05 22:06:55,519 ERROR trial_runner.py:1450 -- Trial train_5ccca_00000: Error happened when processing _ExecutorEventType.TRAINING_RESULT.
ray.exceptions.RayTaskError(ValueError): ray::ImplicitFunc.train() (pid=3964, ip=127.0.0.1, repr=train)
File "python\ray\_raylet.pyx", line 877, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 881, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 821, in ray._raylet.execute_task.function_executor
File "c:\Users\usr\pyVenv\py38\lib\site-packages\ray\_private\function_manager.py", line 670, in actor_method_executor
return method(__ray_actor, *args, **kwargs)
File "c:\Users\usr\pyVenv\py38\lib\site-packages\ray\util\tracing\tracing_helper.py", line 460, in _resume_span
return method(self, *_args, **_kwargs)
File "c:\Users\usr\pyVenv\py38\lib\site-packages\ray\tune\trainable\trainable.py", line 384, in train
raise skipped from exception_cause(skipped)
File "c:\Users\usr\pyVenv\py38\lib\site-packages\ray\tune\trainable\function_trainable.py", line 336, in entrypoint
return self._trainable_func(
File "c:\Users\usr\pyVenv\py38\lib\site-packages\ray\util\tracing\tracing_helper.py", line 460, in _resume_span
return method(self, *_args, **_kwargs)
File "c:\Users\usr\pyVenv\py38\lib\site-packages\ray\tune\trainable\function_trainable.py", line 653, in _trainable_func
output = fn()
File "c:\Users\usr\pyVenv\py38\lib\site-packages\ray\tune\trainable\util.py", line 421, in _inner
return inner(config, checkpoint_dir=None)
File "c:\Users\usr\pyVenv\py38\lib\site-packages\ray\tune\trainable\util.py", line 413, in inner
return trainable(config, **fn_kwargs)
File "C:\Users\usr\AppData\Local\Temp\ipykernel_1896\1298047504.py", line 3, in train
File "C:\Users\usr\AppData\Local\Temp\ipykernel_1896\4257726157.py", line 6, in __init__
File "c:\Users\usr\pyVenv\py38\lib\site-packages\torch\nn\modules\conv.py", line 450, in __init__
super().__init__(
File "c:\Users\usr\pyVenv\py38\lib\site-packages\torch\nn\modules\conv.py", line 91, in __init__
if out_channels % groups != 0:
**ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()**
I wish someone can help me out.
Let me know if more code is needed
