Comments (1)
Hi @JunHyunjae
I've check your code above and it produces valid results. I had to modify it a little bit to make it run:
from nvidia.dali import Pipeline, fn, types
import numpy as np
class ExternalInputIterator(object):
def __init__(self, batch_size):
self.batch_size = batch_size
self.source_files = [
'DALI_extra/db/single/jpeg/450/bobsled-683993_1280.jpg',
'DALI_extra/db/single/jpeg/450/bobsled-643397_1280.jpg',
'DALI_extra/db/single/jpeg/450/bobsled-683995_1280.jpg',
'DALI_extra/db/single/jpeg/312/grasshopper-4357907_1280.jpg',
'DALI_extra/db/single/jpeg/312/grasshopper-4357903_1280.jpg',
'DALI_extra/db/single/jpeg/312/cricket-1345065_1280.jpg',
'DALI_extra/db/single/jpeg/372/baboon-3089012_1280.jpg',
'DALI_extra/db/single/jpeg/372/monkey-653705_1920.jpg',
'DALI_extra/db/single/jpeg/372/baboon-174073_1280.jpg',
]
def __iter__(self):
self.i = 0
self.n = self.batch_size
return self
def __next__(self):
source_batch = []
target_batch = []
name_batch = []
if self.i >= self.n:
self.__iter__()
raise StopIteration
for _ in range(self.batch_size):
filename = self.source_files[self.i % self.n]
# print("self.i : ", self.i, ", filename : ", filename)
source_batch.append(np.fromfile(filename, dtype=np.uint8)) # we can use numpy
target_batch.append(np.fromfile(filename, dtype=np.uint8)) # we can use numpy
# name_batch.append(filename)
name_batch.append(np.fromstring(filename, dtype=np.uint8))
self.i += 1
return (source_batch, target_batch, name_batch)
def __len__(self):
return self.data_set_len
next = __next__
class ExternalSourcePipeline(Pipeline):
def __init__(self, batch_size, num_threads, device_id):
super(ExternalSourcePipeline, self).__init__(batch_size, num_threads, device_id)
self.eii = ExternalInputIterator(batch_size)
self.iterator = iter(self.eii)
def define_graph(self):
source = fn.external_source(device="cpu", name="source_images")
target = fn.external_source(device="cpu", name="target_images")
filename = fn.external_source(device="cpu", name="filename")
source_images = fn.experimental.decoders.image(
source, device='mixed', output_type=types.GRAY, dtype=types.UINT16)
target_images = fn.experimental.decoders.image(
target, device='mixed', output_type=types.GRAY, dtype=types.UINT16)
return (source_images, target_images, filename)
def iter_setup(self):
source_images, target_images, filename = next(self.iterator)
self.feed_input("source_images", source_images)
self.feed_input("target_images", target_images)
self.feed_input("filename", filename)
p = ExternalSourcePipeline(1, 1, 0)
p.build()
print(p.run())
Producing:
(TensorListGPU(
[[[[16705]
[16191]
...
[ 6425]
[ 5911]]
[[15420]
[15934]
...
[ 3341]
[ 3855]]
...
[[54998]
[55769]
...
[61937]
[61680]]
[[54998]
[56026]
...
[61937]
[61937]]]],
dtype=DALIDataType.UINT16,
layout="HWC",
num_samples=1,
shape=[(988, 1280, 1)]), TensorListGPU(
[[[[16705]
[16191]
...
[ 6425]
[ 5911]]
[[15420]
[15934]
...
[ 3341]
[ 3855]]
...
[[54998]
[55769]
...
[61937]
[61680]]
[[54998]
[56026]
...
[61937]
[61937]]]],
dtype=DALIDataType.UINT16,
layout="HWC",
num_samples=1,
shape=[(988, 1280, 1)]), TensorListCPU(
[[ 47 104 111 109 101 47 106 97 110 116 111 110 47 103 105 116 47
68 65 76 73 95 101 120 116 114 97 47 100 98 47 115 105 110
103 108 101 47 106 112 101 103 47 52 53 48 47 98 111 98 115
108 101 100 45 54 56 51 57 57 51 95 49 50 56 48 46 106
112 103]],
dtype=DALIDataType.UINT8,
num_samples=1,
shape=[(70,)]))
If you still find issues with your code, please attach a full script that we can run on our end.
from dali.
Related Issues (20)
- Extracting properties from a list of DataNodes HOT 5
- A100 hardware decoder HOT 1
- Extract motion vectors HOT 7
- Segmentation fault when using 'mixed' HOT 5
- Bbox Pruning Too Aggressive? HOT 5
- Indexing video with binary mask HOT 1
- source_info tensor not guaranteed to contain correct data HOT 1
- COCO Reader pixelwise_masks Emtpy Output HOT 7
- Dali on Jetson: nvidia.dali.fn.readers.video_resize is missing HOT 4
- Numpy reader test (GDS) HOT 4
- How to add a scalar value to the loader? HOT 1
- Can DALI be integrated into HuggingFace Trainer? HOT 9
- Bug in creating `TensorGPU` when `stream` key is `None` in CUDA array interface HOT 2
- Configure max image size HOT 3
- Webdataset reader behavior with many sources HOT 1
- ModuleNotFoundError: No module named 'nvidia.dali.python_function_plugin' HOT 3
- Speed up Dino with DALI HOT 3
- error using webdataset
- webdataset cannot stop cycling at end of epoch HOT 11
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from dali.