# Issues with uniform/loguniform and batch\_size after adjustments

**URL:** <https://discuss.ray.io/t/issues-with-uniform-loguniform-and-batch-size-after-adjustments/15701>\
**Category:** Ray Serve\
**Created:** [August 30, 2024, 10:07am UTC](https://discuss.ray.io/t/issues-with-uniform-loguniform-and-batch-size-after-adjustments/15701 "2024-08-30T10:07:12Z")\
**Posts on this page:** 1\
**Page:** 1

<div class="post-metadata">

**Author:** ![gerlepper](https://avatars.discourse-cdn.com/v4/letter/g/41988e/32.png) [@gerlepper](https://discuss.ray.io/u/gerlepper)\
**Post date:** [August 30, 2024, 10:07am UTC](https://discuss.ray.io/t/issues-with-uniform-loguniform-and-batch-size-after-adjustments/15701/1 "2024-08-30T10:07:12Z")

</div>

Hi,

I used to work with this code couple of months ago and it worked perfectly. However, when I set up my new Colab workspace, there are many issues with tune. Full workflow:

```auto
!pip install ray[tune]
!pip install hpbandster ConfigSpace

import numpy as np
import os
import pandas as pd
import seaborn as sns
import torch
import torchvision

from functools import partial
from matplotlib import pyplot as plt
from random import randrange
from ray import train
from ray import tune
from ray.tune import CLIReporter
from ray.tune.schedulers import HyperBandForBOHB, ASHAScheduler
from ray.tune.search.bohb import TuneBOHB
from ray.tune.search.hyperopt import HyperOptSearch
from ray.train import Checkpoint, report, get_context
from tabulate import tabulate
from torch import nn, optim
from torch.nn import functional as F
from torch.utils.data import random_split
from torch.utils.data.sampler import SubsetRandomSampler
from torchsummary import summary

from ray.train import Checkpoint
import tempfile

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

# standard cast into Tensor and pixel values normalization in [-1, 1] range
transform = torchvision.transforms.Compose([
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])

# extra transfrom for the training data, in order to achieve better performance
train_transform = torchvision.transforms.Compose([
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
    torchvision.transforms.RandomCrop(32, padding=4, padding_mode='reflect'),
    torchvision.transforms.RandomHorizontalFlip(),
])

trainset = torchvision.datasets.CIFAR10(
    root='./data', train=True, download=True, transform=train_transform
)
validationset = torchvision.datasets.CIFAR10(
    root='./data', train=True, download=True, transform=transform
)
testset = torchvision.datasets.CIFAR10(
    root='./data', train=False, download=True, transform=transform
)

IDX_TO_LABEL = {v: k for k, v in trainset.class_to_idx.items()}

class ResNet(nn.Module):
    def __init__ (self):
        super(ResNet, self). __init__ ()

        self.conv1 = nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=64, kernel_size=3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True)
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2)
        )

        self.res1 = nn.Sequential(nn.Sequential(
            nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True)
        ), nn.Sequential(
            nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True))
        )

        self.conv3 = nn.Sequential(
            nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2)
        )
        self.conv4 = nn.Sequential(
            nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2)
        )

        self.res2 = nn.Sequential(nn.Sequential(
            nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True)
        ), nn.Sequential(
            nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True))
        )

        self.classifier = nn.Sequential(
            nn.MaxPool2d(4),
            nn.Flatten(),
            nn.Linear(512, 10)
        )

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.res1(x) + x
        x = self.conv3(x)
        x = self.conv4(x)
        x = self.res2(x) + x
        x = self.classifier(x)

        return x

# percentage of training set to use as validation
valid_size = 0.2

# obtain training indices that will be used for validation
num_train = len(trainset)
indices = list(range(num_train))
split = int(np.floor(valid_size * num_train))
train_idx, valid_idx = indices[split:], indices[:split]

# define samplers for obtaining training and validation batches
TRAIN_SAMPLER = SubsetRandomSampler(train_idx)
VALID_SAMPLER = SubsetRandomSampler(valid_idx)

# number of subprocesses to use for data loading
NUM_WORKERS = 2

def data_loaders(trainset, validationset, testset, size):
    trainloader = torch.utils.data.DataLoader(
        trainset, batch_size=size,
        sampler=TRAIN_SAMPLER, num_workers=NUM_WORKERS
    )
    validloader = torch.utils.data.DataLoader(
        validationset, batch_size=size,
        sampler=VALID_SAMPLER, num_workers=NUM_WORKERS
    )
    testloader = torch.utils.data.DataLoader(
        testset, batch_size=size, num_workers=NUM_WORKERS
    )

    return trainloader, validloader, testloader

def train_cifar(
    config, trainset, validationset, testset,
    epochs=10, tuning=False
):
    net = ResNet()
    net.to(device)

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(
        net.parameters(),
        lr=config['lr'],
        betas=(config['beta1'], config['beta2']),
        amsgrad=config['amsgrad'],
    )

    # The `checkpoint_dir` parameter gets passed by Ray Tune when a checkpoint
    # should be restored
    checkpoint: train.Checkpoint = train.get_checkpoint()
    if checkpoint:
        with checkpoint.as_directory() as checkpoint_dir:
            torch.load(os.path.join(checkpoint_dir, "checkpoint.pt"))

    trainloader, validloader, testloader = data_loaders(
        trainset, validationset, testset, config['batch_size']
    )

    train_loss_list = []
    accuracy_list = []

    # track minimum validation loss
    valid_loss_min = np.Inf

    for epoch in range(epochs):

        train_loss = 0.0
        net.train()
        for inputs, labels in trainloader:
            inputs, labels = inputs.to(device), labels.to(device)

            optimizer.zero_grad()

            outputs = net(inputs)
            loss = criterion(outputs, labels)

            loss.backward()
            optimizer.step()

            train_loss += loss.item() * inputs.size(0)

        correct = 0
        valid_loss = 0.0
        net.eval()
        for inputs, labels in validloader:
            with torch.no_grad():
                inputs, labels = inputs.to(device), labels.to(device)

                outputs = net(inputs)
                loss = criterion(outputs, labels)

                valid_loss += loss.item() * inputs.size(0)

                predicted = torch.max(outputs.data, 1)[1]

                correct += (predicted == labels).sum().item()

        train_loss = train_loss / len(TRAIN_SAMPLER)
        valid_loss = valid_loss / len(VALID_SAMPLER)
        accuracy = correct / len(VALID_SAMPLER)

        train_loss_list.append(train_loss)
        accuracy_list.append(accuracy)

        if not tuning:
            print(
                f'Epoch: {epoch} \tTraining Loss: {train_loss:.6f} \t'
                f'Validation Loss: {valid_loss:.6f} \t'
                f'Validation Accuracy: {accuracy:.6f}'
            )

            if valid_loss <= valid_loss_min:
                print(
                    'Validation loss decreased ('
                    f'{valid_loss_min:.6f} --> {valid_loss:.6f}). '
                    'Saving model ...'
                )

                torch.save(net.state_dict(), 'cnn.pt')
                valid_loss_min = valid_loss
        else:
            # Here we save a checkpoint. It is automatically registered with
            # Ray Tune and will potentially be passed as the `checkpoint_dir`
            # parameter in future iterations.
            #with tune.checkpoint_dir(step=epoch) as checkpoint_dir:
               # path = os.path.join(checkpoint_dir, 'checkpoint')
              # torch.save((net.state_dict(), optimizer.state_dict()), path)

            # tune.report(mean_loss=valid_loss, accuracy=accuracy

            with tempfile.TemporaryDirectory() as temp_checkpoint_dir:
                path = os.path.join(temp_checkpoint_dir, "checkpoint.pt")
                torch.save(
                    (net.state_dict(), optimizer.state_dict()), path
                )
                checkpoint = Checkpoint.from_directory(temp_checkpoint_dir)
                train.report(
                    {"mean_loss": valid_loss, "accuracy": accuracy},
                    checkpoint=checkpoint,
                )

    print(f'\n{color.bold}----------- Finished Training -----------{color.end}')

    if not tuning:
        return train_loss_list, accuracy_list

config = {
    'batch_size': tune.choice([16, 32, 128]),
    'lr': tune.loguniform(1e-6, 1e-2),
    'beta1': tune.uniform(0.8, 0.96),
    'beta2': tune.uniform(0.98, 0.999999),
    'amsgrad': tune.choice([True, False]),
}

search_alg = TuneBOHB(metric="mean_loss", mode="min")
scheduler = HyperBandForBOHB(
    time_attr='training_iteration',
    metric='mean_loss',
    mode='min',
    max_t=100
)
reporter = CLIReporter(
    metric_columns=['mean_loss', 'accuracy', 'training_iteration'],
    metric='mean_loss',
    mode='min',
    sort_by_metric=True
)

result = tune.run(
    tune.with_parameters(
        train_cifar,
        tuning=True,
        trainset=trainset,
        validationset=validationset,
        testset=testset
    ),
    num_samples=30,
    resources_per_trial={'gpu': 1},
    scheduler=scheduler,
    config=config,
    search_alg=search_alg,
    progress_reporter=reporter
)

```

After running the last piece I get:

> TypeError Traceback (most recent call last)  
> in \<cell line: 23\>()  
> 21 )  
> 22  
> —\> 23 result = tune.run(  
> 24 tune.with\_parameters(  
> 25 train\_cifar,

> 8 frames  
> /usr/local/lib/python3.10/dist-packages/ray/tune/search/bohb/bohb\_search.py in resolve\_value(par, domain)  
> 309 lower = math.ceil(domain.lower / quantize) \* quantize  
> 310 upper = math.floor(domain.upper / quantize) \* quantize  
> → 311 return ConfigSpace.UniformFloatHyperparameter(  
> 312 par, lower=lower, upper=upper, q=quantize, log=True  
> 313 )

> TypeError: UniformFloatHyperparameter. **init** () got an unexpected keyword argument ‘q’

Then, if I try to change uniform / loguniform spaces with, for example,

```auto
'lr': ray.tune.choice([0.0001,0.1])

```

etc., I get errors related to batch\_size:

> 2024-08-30 09:32:28,133 WARNING tune.py:902 – AIR\_VERBOSITY is set, ignoring passed-in ProgressReporter for now.  
> ±-------------------------------------------------------------------+  
> | Configuration for experiment train\_cifar\_2024-08-30\_09-32-28 |  
> ±-------------------------------------------------------------------+  
> | Search algorithm SearchGenerator |  
> | Scheduler HyperBandForBOHB |  
> | Number of trials 30 |  
> ±-------------------------------------------------------------------+

> View detailed results here: /root/ray\_results/train\_cifar\_2024-08-30\_09-32-28  
> To visualize your results with TensorBoard, run: tensorboard --logdir /tmp/ray/session\_2024-08-30\_07-55-06\_272177\_260/artifacts/2024-08-30\_09-32-28/train\_cifar\_2024-08-30\_09-32-28/driver\_artifacts

> Trial status: 1 PENDING  
> Current time: 2024-08-30 09:32:28. Total running time: 0s  
> Logical resource usage: 0/2 CPUs, 1.0/1 GPUs (0.0/1.0 accelerator\_type:T4)  
> ±--------------------------------+  
> | Trial name status |  
> ±--------------------------------+  
> | train\_cifar\_c4eccae3 PENDING |  
> ±--------------------------------+

> Trial train\_cifar\_c4eccae3 started with configuration:  
> ±----------------------------------------------+  
> | Trial train\_cifar\_c4eccae3 config |  
> ±----------------------------------------------+  
> | amsgrad 1 |  
> | batch\_size 16 |  
> | beta1 0.90686 |  
> | beta2 0.98488 |  
> | lr 0.06593 |  
> ±----------------------------------------------+  
> (train\_cifar pid=31271) Configuration received: {‘amsgrad’: 1, ‘batch\_size’: 16, ‘beta1’: 0.906863121917, ‘beta2’: 0.9848844711283, ‘lr’: 0.0659343170261}  
> 2024-08-30 09:32:34,367 ERROR tune\_controller.py:1331 – Trial task failed for trial train\_cifar\_c4eccae3  
> Traceback (most recent call last):  
> File “/usr/local/lib/python3.10/dist-packages/ray/air/execution/\_internal/event\_manager.py”, line 110, in resolve\_future  
> result = ray.get(future)  
> File “/usr/local/lib/python3.10/dist-packages/ray/\_private/auto\_init\_hook.py”, line 21, in auto\_init\_wrapper  
> return fn(\*args, \*\*kwargs)  
> File “/usr/local/lib/python3.10/dist-packages/ray/\_private/client\_mode\_hook.py”, line 103, in wrapper  
> return func(\*args, \*\*kwargs)  
> File “/usr/local/lib/python3.10/dist-packages/ray/\_private/worker.py”, line 2661, in get  
> values, debugger\_breakpoint = worker.get\_objects(object\_refs, timeout=timeout)  
> File “/usr/local/lib/python3.10/dist-packages/ray/\_private/worker.py”, line 871, in get\_objects  
> raise value.as\_instanceof\_cause()  
> ray.exceptions.RayTaskError(ValueError): ray::ImplicitFunc.train() (pid=31271, ip=172.28.0.12, actor\_id=812653cc902351e920cc5e6401000000, repr=train\_cifar)  
> File “/usr/local/lib/python3.10/dist-packages/ray/tune/trainable/trainable.py”, line 331, in train  
> raise skipped from exception\_cause(skipped)  
> File “/usr/local/lib/python3.10/dist-packages/ray/air/\_internal/util.py”, line 98, in run  
> self.\_ret = self.\_target(\*self.\_args, \*\*self.\_kwargs)  
> File “/usr/local/lib/python3.10/dist-packages/ray/tune/trainable/function\_trainable.py”, line 45, in   
> training\_func=lambda: self.\_trainable\_func(self.config),  
> File “/usr/local/lib/python3.10/dist-packages/ray/tune/trainable/function\_trainable.py”, line 250, in \_trainable\_func  
> output = fn()  
> File “/usr/local/lib/python3.10/dist-packages/ray/tune/trainable/util.py”, line 130, in inner  
> return trainable(config, \*\*fn\_kwargs)  
> File “”, line 29, in train\_cifar  
> File “”, line 5, in data\_loaders  
> File “/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py”, line 357, in **init**  
> batch\_sampler = BatchSampler(sampler, batch\_size, drop\_last)  
> File “/usr/local/lib/python3.10/dist-packages/torch/utils/data/sampler.py”, line 268, in **init**  
> raise ValueError(f"batch\_size should be a positive integer value, but got batch\_size={batch\_size}")  
> ValueError: batch\_size should be a positive integer value, but got batch\_size=16  
> (train\_cifar pid=31271) Batch size received in data\_loaders: 16  
> (train\_cifar pid=31271) TRAIN\_SAMPLER: \<torch.utils.data.sampler.SubsetRandomSampler object at 0x7903c59f1f90\>, VALID\_SAMPLER: \<torch.utils.data.sampler.SubsetRandomSampler object at 0x7903c59f1ed0\>

> Trial train\_cifar\_c4eccae3 errored after 0 iterations at 2024-08-30 09:32:34. Total running time: 6s  
> Error file: /tmp/ray/session\_2024-08-30\_07-55-06\_272177\_260/artifacts/2024-08-30\_09-32-28/train\_cifar\_2024-08-30\_09-32-28/driver\_artifacts/train\_cifar\_c4eccae3\_1\_amsgrad=1,batch\_size=16,beta1=0.9069,beta2=0.9849,lr=0.0659\_2024-08-30\_09-32-28/error.txt

I printed out the batch\_size and, well, it is an integer positive number.

Sorry for the chaos but I have no clue what might be wrong here…