Giter VIP home page Giter VIP logo

Comments (3)

John-Ragland avatar John-Ragland commented on September 26, 2024

Some more details on this:

for the following code:

import xarray as xr

storage_options = {'account_name':'dasdata'}
fn = 'az://zarr/ooi_South_Tx.zarr/ooi_South_Tx.zarr/'
ds = xr.open_zarr(fn, storage_options)

This is the full print error

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/adlfs/spec.py:656, in AzureBlobFileSystem.do_connect(self)
    655     else:
--> 656         raise ValueError(
    657             "Must provide either a connection_string or account_name with credentials!!"
    658         )
    660 except RuntimeError:

ValueError: Must provide either a connection_string or account_name with credentials!!

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
Input In [3], in <cell line: 5>()
      3 storage_options = {'account_name':'dasdata'}
      4 fn = 'az://zarr/ooi_South_Tx.zarr/ooi_South_Tx.zarr/'
----> 5 ds = xr.open_zarr(fn, storage_options)

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/xarray/backends/zarr.py:792, in open_zarr(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, **kwargs)
    779     raise TypeError(
    780         "open_zarr() got unexpected keyword arguments " + ",".join(kwargs.keys())
    781     )
    783 backend_kwargs = {
    784     "synchronizer": synchronizer,
    785     "consolidated": consolidated,
   (...)
    789     "stacklevel": 4,
    790 }
--> 792 ds = open_dataset(
    793     filename_or_obj=store,
    794     group=group,
    795     decode_cf=decode_cf,
    796     mask_and_scale=mask_and_scale,
    797     decode_times=decode_times,
    798     concat_characters=concat_characters,
    799     decode_coords=decode_coords,
    800     engine="zarr",
    801     chunks=chunks,
    802     drop_variables=drop_variables,
    803     backend_kwargs=backend_kwargs,
    804     decode_timedelta=decode_timedelta,
    805     use_cftime=use_cftime,
    806 )
    807 return ds

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/xarray/backends/api.py:539, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, backend_kwargs, **kwargs)
    527 decoders = _resolve_decoders_kwargs(
    528     decode_cf,
    529     open_backend_dataset_parameters=backend.open_dataset_parameters,
   (...)
    535     decode_coords=decode_coords,
    536 )
    538 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 539 backend_ds = backend.open_dataset(
    540     filename_or_obj,
    541     drop_variables=drop_variables,
    542     **decoders,
    543     **kwargs,
    544 )
    545 ds = _dataset_from_backend_dataset(
    546     backend_ds,
    547     filename_or_obj,
   (...)
    555     **kwargs,
    556 )
    557 return ds

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/xarray/backends/zarr.py:840, in ZarrBackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, stacklevel)
    820 def open_dataset(
    821     self,
    822     filename_or_obj,
   (...)
    836     stacklevel=3,
    837 ):
    839     filename_or_obj = _normalize_path(filename_or_obj)
--> 840     store = ZarrStore.open_group(
    841         filename_or_obj,
    842         group=group,
    843         mode=mode,
    844         synchronizer=synchronizer,
    845         consolidated=consolidated,
    846         consolidate_on_close=False,
    847         chunk_store=chunk_store,
    848         storage_options=storage_options,
    849         stacklevel=stacklevel + 1,
    850     )
    852     store_entrypoint = StoreBackendEntrypoint()
    853     with close_on_error(store):

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/xarray/backends/zarr.py:384, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks, stacklevel)
    382 if consolidated is None:
    383     try:
--> 384         zarr_group = zarr.open_consolidated(store, **open_kwargs)
    385     except KeyError:
    386         try:

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/zarr/convenience.py:1283, in open_consolidated(store, metadata_key, mode, **kwargs)
   1281 # normalize parameters
   1282 zarr_version = kwargs.get('zarr_version')
-> 1283 store = normalize_store_arg(store, storage_options=kwargs.get("storage_options"), mode=mode,
   1284                             zarr_version=zarr_version)
   1285 if mode not in {'r', 'r+'}:
   1286     raise ValueError("invalid mode, expected either 'r' or 'r+'; found {!r}"
   1287                      .format(mode))

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/zarr/storage.py:171, in normalize_store_arg(store, storage_options, mode, zarr_version)
    169     from zarr._storage.v3 import _normalize_store_arg_v3
    170     normalize_store = _normalize_store_arg_v3
--> 171 return normalize_store(store, storage_options, mode)

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/zarr/storage.py:144, in _normalize_store_arg_v2(store, storage_options, mode)
    142 if isinstance(store, str):
    143     if "://" in store or "::" in store:
--> 144         return FSStore(store, mode=mode, **(storage_options or {}))
    145     elif storage_options:
    146         raise ValueError("storage_options passed with non-fsspec path")

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/zarr/storage.py:1328, in FSStore.__init__(self, url, normalize_keys, key_separator, mode, exceptions, dimension_separator, fs, check, create, missing_exceptions, **storage_options)
   1326 if protocol in (None, "file") and not storage_options.get("auto_mkdir"):
   1327     storage_options["auto_mkdir"] = True
-> 1328 self.map = fsspec.get_mapper(url, **{**mapper_options, **storage_options})
   1329 self.fs = self.map.fs  # for direct operations
   1330 self.path = self.fs._strip_protocol(url)

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/fsspec/mapping.py:228, in get_mapper(url, check, create, missing_exceptions, alternate_root, **kwargs)
    197 """Create key-value interface for given URL and options
    198 
    199 The URL will be of the form "protocol://location" and point to the root
   (...)
    225 ``FSMap`` instance, the dict-like key-value store.
    226 """
    227 # Removing protocol here - could defer to each open() on the backend
--> 228 fs, urlpath = url_to_fs(url, **kwargs)
    229 root = alternate_root if alternate_root is not None else urlpath
    230 return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions)

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/fsspec/core.py:407, in url_to_fs(url, **kwargs)
    405     options = cls._get_kwargs_from_urls(url)
    406     update_storage_options(options, kwargs)
--> 407     fs = cls(**options)
    408     urlpath = fs._strip_protocol(url)
    409 return fs, urlpath

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/fsspec/spec.py:68, in _Cached.__call__(cls, *args, **kwargs)
     66     return cls._cache[token]
     67 else:
---> 68     obj = super().__call__(*args, **kwargs)
     69     # Setting _fs_token here causes some static linters to complain.
     70     obj._fs_token_ = token

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/adlfs/spec.py:494, in AzureBlobFileSystem.__init__(self, account_name, account_key, connection_string, credential, sas_token, request_session, socket_timeout, blocksize, client_id, client_secret, tenant_id, anon, location_mode, loop, asynchronous, default_fill_cache, default_cache_type, version_aware, **kwargs)
    482 if (
    483     self.credential is None
    484     and self.anon is False
    485     and self.sas_token is None
    486     and self.account_key is None
    487 ):
    489     (
    490         self.credential,
    491         self.sync_credential,
    492     ) = self._get_default_azure_credential(**kwargs)
--> 494 self.do_connect()
    495 weakref.finalize(self, sync, self.loop, close_service_client, self)
    497 if self.credential is not None:

File ~/anaconda3/envs/ooi_dev/lib/python3.10/site-packages/adlfs/spec.py:666, in AzureBlobFileSystem.do_connect(self)
    663     self.do_connect()
    665 except Exception as e:
--> 666     raise ValueError(f"unable to connect to account for {e}")

ValueError: unable to connect to account for Must provide either a connection_string or account_name with credentials!!

I can successfully open the dataset using zarr.ABSStore and passing this to xarray.

from azure.storage.blob import BlobServiceClient
import zarr
import xarray as xr

store_url = "https://dasdata.blob.core.windows.net"
blob_service_client = BlobServiceClient(store_url)
container_client = blob_service_client.get_container_client("zarr")
store = zarr.ABSStore(client=container_client, prefix='SouthCable_Tx.zarr')  
ds = xr.open_zarr(store)

When trying to use xarray directly, something is breaking. I'm not sure if it's in xarray, zarr, fsspec or adlfs. It would be nice for xarray to be able to directly access and support opening datasets from azure blob storage.

Thanks for the help!

from adlfs.

TomAugspurger avatar TomAugspurger commented on September 26, 2024

Can you try passing storage_options as a keyword?

ds = xr.open_zarr(fn, storage_options=storage_options)

from adlfs.

John-Ragland avatar John-Ragland commented on September 26, 2024

That was it, whoops...
thanks for help!

from adlfs.

Related Issues (20)

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.