Comments (3)
Some more details on this:
for the following code:
import xarray as xr
storage_options = {'account_name':'dasdata'}
fn = 'az://zarr/ooi_South_Tx.zarr/ooi_South_Tx.zarr/'
ds = xr.open_zarr(fn, storage_options)
This is the full print error
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/adlfs/spec.py:656, in AzureBlobFileSystem.do_connect(self)
655 else:
--> 656 raise ValueError(
657 "Must provide either a connection_string or account_name with credentials!!"
658 )
660 except RuntimeError:
ValueError: Must provide either a connection_string or account_name with credentials!!
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
Input In [3], in <cell line: 5>()
3 storage_options = {'account_name':'dasdata'}
4 fn = 'az://zarr/ooi_South_Tx.zarr/ooi_South_Tx.zarr/'
----> 5 ds = xr.open_zarr(fn, storage_options)
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/xarray/backends/zarr.py:792, in open_zarr(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, **kwargs)
779 raise TypeError(
780 "open_zarr() got unexpected keyword arguments " + ",".join(kwargs.keys())
781 )
783 backend_kwargs = {
784 "synchronizer": synchronizer,
785 "consolidated": consolidated,
(...)
789 "stacklevel": 4,
790 }
--> 792 ds = open_dataset(
793 filename_or_obj=store,
794 group=group,
795 decode_cf=decode_cf,
796 mask_and_scale=mask_and_scale,
797 decode_times=decode_times,
798 concat_characters=concat_characters,
799 decode_coords=decode_coords,
800 engine="zarr",
801 chunks=chunks,
802 drop_variables=drop_variables,
803 backend_kwargs=backend_kwargs,
804 decode_timedelta=decode_timedelta,
805 use_cftime=use_cftime,
806 )
807 return ds
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/xarray/backends/api.py:539, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, backend_kwargs, **kwargs)
527 decoders = _resolve_decoders_kwargs(
528 decode_cf,
529 open_backend_dataset_parameters=backend.open_dataset_parameters,
(...)
535 decode_coords=decode_coords,
536 )
538 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 539 backend_ds = backend.open_dataset(
540 filename_or_obj,
541 drop_variables=drop_variables,
542 **decoders,
543 **kwargs,
544 )
545 ds = _dataset_from_backend_dataset(
546 backend_ds,
547 filename_or_obj,
(...)
555 **kwargs,
556 )
557 return ds
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/xarray/backends/zarr.py:840, in ZarrBackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, stacklevel)
820 def open_dataset(
821 self,
822 filename_or_obj,
(...)
836 stacklevel=3,
837 ):
839 filename_or_obj = _normalize_path(filename_or_obj)
--> 840 store = ZarrStore.open_group(
841 filename_or_obj,
842 group=group,
843 mode=mode,
844 synchronizer=synchronizer,
845 consolidated=consolidated,
846 consolidate_on_close=False,
847 chunk_store=chunk_store,
848 storage_options=storage_options,
849 stacklevel=stacklevel + 1,
850 )
852 store_entrypoint = StoreBackendEntrypoint()
853 with close_on_error(store):
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/xarray/backends/zarr.py:384, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks, stacklevel)
382 if consolidated is None:
383 try:
--> 384 zarr_group = zarr.open_consolidated(store, **open_kwargs)
385 except KeyError:
386 try:
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/zarr/convenience.py:1283, in open_consolidated(store, metadata_key, mode, **kwargs)
1281 # normalize parameters
1282 zarr_version = kwargs.get('zarr_version')
-> 1283 store = normalize_store_arg(store, storage_options=kwargs.get("storage_options"), mode=mode,
1284 zarr_version=zarr_version)
1285 if mode not in {'r', 'r+'}:
1286 raise ValueError("invalid mode, expected either 'r' or 'r+'; found {!r}"
1287 .format(mode))
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/zarr/storage.py:171, in normalize_store_arg(store, storage_options, mode, zarr_version)
169 from zarr._storage.v3 import _normalize_store_arg_v3
170 normalize_store = _normalize_store_arg_v3
--> 171 return normalize_store(store, storage_options, mode)
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/zarr/storage.py:144, in _normalize_store_arg_v2(store, storage_options, mode)
142 if isinstance(store, str):
143 if "://" in store or "::" in store:
--> 144 return FSStore(store, mode=mode, **(storage_options or {}))
145 elif storage_options:
146 raise ValueError("storage_options passed with non-fsspec path")
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/zarr/storage.py:1328, in FSStore.__init__(self, url, normalize_keys, key_separator, mode, exceptions, dimension_separator, fs, check, create, missing_exceptions, **storage_options)
1326 if protocol in (None, "file") and not storage_options.get("auto_mkdir"):
1327 storage_options["auto_mkdir"] = True
-> 1328 self.map = fsspec.get_mapper(url, **{**mapper_options, **storage_options})
1329 self.fs = self.map.fs # for direct operations
1330 self.path = self.fs._strip_protocol(url)
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/fsspec/mapping.py:228, in get_mapper(url, check, create, missing_exceptions, alternate_root, **kwargs)
197 """Create key-value interface for given URL and options
198
199 The URL will be of the form "protocol://location" and point to the root
(...)
225 ``FSMap`` instance, the dict-like key-value store.
226 """
227 # Removing protocol here - could defer to each open() on the backend
--> 228 fs, urlpath = url_to_fs(url, **kwargs)
229 root = alternate_root if alternate_root is not None else urlpath
230 return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions)
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/fsspec/core.py:407, in url_to_fs(url, **kwargs)
405 options = cls._get_kwargs_from_urls(url)
406 update_storage_options(options, kwargs)
--> 407 fs = cls(**options)
408 urlpath = fs._strip_protocol(url)
409 return fs, urlpath
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/fsspec/spec.py:68, in _Cached.__call__(cls, *args, **kwargs)
66 return cls._cache[token]
67 else:
---> 68 obj = super().__call__(*args, **kwargs)
69 # Setting _fs_token here causes some static linters to complain.
70 obj._fs_token_ = token
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/adlfs/spec.py:494, in AzureBlobFileSystem.__init__(self, account_name, account_key, connection_string, credential, sas_token, request_session, socket_timeout, blocksize, client_id, client_secret, tenant_id, anon, location_mode, loop, asynchronous, default_fill_cache, default_cache_type, version_aware, **kwargs)
482 if (
483 self.credential is None
484 and self.anon is False
485 and self.sas_token is None
486 and self.account_key is None
487 ):
489 (
490 self.credential,
491 self.sync_credential,
492 ) = self._get_default_azure_credential(**kwargs)
--> 494 self.do_connect()
495 weakref.finalize(self, sync, self.loop, close_service_client, self)
497 if self.credential is not None:
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/adlfs/spec.py:666, in AzureBlobFileSystem.do_connect(self)
663 self.do_connect()
665 except Exception as e:
--> 666 raise ValueError(f"unable to connect to account for {e}")
ValueError: unable to connect to account for Must provide either a connection_string or account_name with credentials!!
I can successfully open the dataset using zarr.ABSStore and passing this to xarray.
from azure.storage.blob import BlobServiceClient
import zarr
import xarray as xr
store_url = "https://dasdata.blob.core.windows.net"
blob_service_client = BlobServiceClient(store_url)
container_client = blob_service_client.get_container_client("zarr")
store = zarr.ABSStore(client=container_client, prefix='SouthCable_Tx.zarr')
ds = xr.open_zarr(store)
When trying to use xarray directly, something is breaking. I'm not sure if it's in xarray, zarr, fsspec or adlfs. It would be nice for xarray to be able to directly access and support opening datasets from azure blob storage.
Thanks for the help!
from adlfs.
Can you try passing storage_options as a keyword?
ds = xr.open_zarr(fn, storage_options=storage_options)
from adlfs.
That was it, whoops...
thanks for help!
from adlfs.
Related Issues (20)
- Add use_emulator setting to better align with object_store crate HOT 2
- Current state of the library, milestones and current development HOT 1
- Concurrent download of multiple files HOT 1
- Support virtual directory stubs with uppercase "Hdi_isfolder" metadata HOT 1
- Feature Suggestion: Optional content type when for writing file HOT 2
- Support passing url in AzureBlobFileSystem HOT 1
- Add comment why `aiohttp` is required
- Fix typo in repo About
- Python 3.12 support blocked by aiohttp HOT 1
- Feature Request: Support for Adding Metadata to Blobs
- Runtime warning from missing await HOT 2
- `fs.info()` and `fs.ls(detail=True)` return different etag formats
- Issue with parallel uploads to the same blob HOT 1
- Can I use a bearer token / entra ID token for authentication? HOT 1
- Parameter anon ignored if set to False
- exists() is missing **kwargs
- object NoneType can't be used in 'await' expression HOT 3
- Does adlfs provide a way to set the content-type when writing a file to azure blob storage? HOT 1
- Does adlfs provide a way to get a file's public cloud url?
- Xarray Serialisation Issues reading NetCDF from AzureBlobFile HOT 3
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from adlfs.