RuntimeError: CUDA error: device-side assert triggered about deepviewagg HOT 10 CLOSED

Res16UNet34-PointPyramid-early-cityscapes-interpolate:
    path_pretrained: /path/to/your/project/torch-points3d/outputs/yyyy-mm-dd/hh-mm-ss/Res16UNet34-PointPyramid-early-cityscapes-interpolate.pt
    class: sparseconv3d.APIModel
    ...

in_coords = x_3d.cmaps[stride_in]
in_coords[:, :3] = ((in_coords[:, :3].float() / stride_out[0]).floor() * stride_out[0]).int()
out_coords = x_3d.cmaps[stride_out]
idx = sphashquery(sphash(in_coords), sphash(out_coords))

# Select you GPU
I_GPU = 0

# Uncomment to use autoreload
# %load_ext autoreload
# %autoreload 2

import os
import os.path as osp
import sys
import torch
import numpy as np
from time import time
from omegaconf import OmegaConf
start = time()
import warnings
warnings.filterwarnings('ignore')

torch.cuda.set_device(I_GPU)
DIR = os.path.dirname(os.getcwd())
ROOT = os.path.join(DIR, "..")
sys.path.insert(0, ROOT)
sys.path.insert(0, DIR)

from torch_points3d.utils.config import hydra_read
from torch_geometric.data import Data
from torch_points3d.core.multimodal.data import MMData, MMBatch
from torch_points3d.core.multimodal.image import SameSettingImageData, ImageData
from torch_points3d.datasets.segmentation.multimodal.kitti360 import KITTI360DatasetMM
from torch_points3d.core.multimodal.image import ImageData
from torch_points3d.models.model_factory import instantiate_model

# Set your dataset root directory, where the data was/will be downloaded
DATA_ROOT = '/path/to/your/dataset/root/directory'
mini = False                                                          # set to True to only load and play with a small portion of the KITTI-360 dataset
train_is_trainval = False                                             # set to True if you want to the Train set to be Train+Val
sample_per_epoch = 12000                                              # number of cylinders sampled in the Train set. Corrects class imbalance. Set to 0 for regularly-sampled cylinders

dataset_config = 'segmentation/multimodal/kitti360-sparse'   
models_config = 'segmentation/multimodal/sparseconv3d'                # model family
model_name = 'Res16UNet34-PointPyramid-early-cityscapes-interpolate'  # specific model

overrides = [
    'task=segmentation',
    f'data={dataset_config}',
    f'data.mini={mini}',
    f'models={models_config}',
    f'model_name={model_name}',
    f'data.dataroot={DATA_ROOT}',
    f'+train_is_trainval={train_is_trainval}',
    f'data.sample_per_epoch={sample_per_epoch}',
]

cfg = hydra_read(overrides)
# print(OmegaConf.to_yaml(cfg)

# Dataset instantiation
start = time()
dataset = KITTI360DatasetMM(cfg.data)
# print(dataset)
print(f"Time = {time() - start:0.1f} sec.")

# Create the model
print(f"Creating model: {cfg.model_name}")
model = instantiate_model(cfg, dataset)
# print(model)

# Prepare the model for inference
model = model.eval().cuda()
print('Model loaded')

# Specify the (idx_window, idx_center) of your problematic samples
problematic_cylinders = [(0, 76930), (0, 63780)]

# Convert (idx_window, idx_center) into global indices
cum_sizes = torch.cat((torch.LongTensor([0]), dataset.train_dataset.sampling_sizes.cumsum(0)))
problematic_cylinders_global = [cum_sizes[idx_window].item() + idx_center for idx_window, idx_center in problematic_cylinders]

# Create a batch with the samples
batch = MMBatch.from_mm_data_list([dataset.train_dataset[idx_global] for idx_global in problematic_cylinders_global])
print(batch)

# # Optionally, to see if number of points matters
# # Only keep points within k * voxel size of the firt point, to see if 
# # single voxel subsampling matters 
# k = 4
# pos = batch.data.pos
# closeby = torch.where(((pos[0] - pos)**2).sum(dim=1) < k * batch.data.grid_size.max())[0]
# batch = batch[closeby]
# print(batch)

# # Optionally, to see if number of views matters
# # Remove all images
# batch.modalities['image'] = batch.modalities['image'][None]
# print(batch)

# Run forward and backward
model = model.train().cuda()
model.set_input(batch, model.device)
model(batch)
model.backward()

Time = 14.7 sec.
Creating model: Res16UNet34-PointPyramid-early-cityscapes-interpolate
Model loaded
MMBatch(
    data = DataBatch(mapping_index=[75395], planarity=[75395], y=[75395], origin_id=[75395], linearity=[75395], num_raw_points=[2], norm=[75395, 3], scattering=[75395], pos=[75395, 3], grid_size=[2], idx_window=[2], idx_center=[2], x=[75395, 1], coords=[75395, 3], batch=[75395], ptr=[3])
    image = ImageBatch(num_settings=4, num_views=13, num_points=75395, device=cpu)
)
None
Traceback (most recent call last):
  File "/home/ws/project/lcfusion/DeepViewAgg/test.py", line 100, in <module>
    model(batch)
  File "/home/public/ws/semantic_kittenv/anaconda3/envs/segcontrast/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
    return forward_call(*input, **kwargs)
  File "/home/ws/project/lcfusion/DeepViewAgg/torch_points3d/models/segmentation/sparseconv3d.py", line 43, in forward
    features = self.backbone(self.input).x
  File "/home/public/ws/semantic_kittenv/anaconda3/envs/segcontrast/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
    return forward_call(*input, **kwargs)
  File "/home/ws/project/lcfusion/DeepViewAgg/torch_points3d/applications/sparseconv3d.py", line 228, in forward
    data = self.down_modules[i](data)
  File "/home/public/ws/semantic_kittenv/anaconda3/envs/segcontrast/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
    return forward_call(*input, **kwargs)
  File "/home/ws/project/lcfusion/DeepViewAgg/torch_points3d/modules/multimodal/modules.py", line 85, in forward
    mm_data_dict, self.block_1)
  File "/home/ws/project/lcfusion/DeepViewAgg/torch_points3d/modules/multimodal/modules.py", line 221, in forward_3d_block_down
    mm_data_dict['modalities'][m].select_points(idx, mode=mode)
  File "/home/ws/project/lcfusion/DeepViewAgg/torch_points3d/core/multimodal/image.py", line 1493, in select_points
    print(idx)
  File "/home/public/ws/semantic_kittenv/anaconda3/envs/segcontrast/lib/python3.7/site-packages/torch/_tensor.py", line 249, in __repr__
    return torch._tensor_str._str(self)
  File "/home/public/ws/semantic_kittenv/anaconda3/envs/segcontrast/lib/python3.7/site-packages/torch/_tensor_str.py", line 415, in _str
    return _str_intern(self)
  File "/home/public/ws/semantic_kittenv/anaconda3/envs/segcontrast/lib/python3.7/site-packages/torch/_tensor_str.py", line 390, in _str_intern
    tensor_str = _tensor_str(self, indent)
  File "/home/public/ws/semantic_kittenv/anaconda3/envs/segcontrast/lib/python3.7/site-packages/torch/_tensor_str.py", line 251, in _tensor_str
    formatter = _Formatter(get_summarized_data(self) if summarize else self)
  File "/home/public/ws/semantic_kittenv/anaconda3/envs/segcontrast/lib/python3.7/site-packages/torch/_tensor_str.py", line 86, in __init__
    value_str = '{}'.format(value)
  File "/home/public/ws/semantic_kittenv/anaconda3/envs/segcontrast/lib/python3.7/site-packages/torch/_tensor.py", line 571, in __format__
    return self.item().__format__(format_spec)
RuntimeError: CUDA error: device-side assert triggered
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.