Accessing opendap datasets about esgf-pyclient HOT 5 OPEN

@saeedvzf I have similar problems. It's probably because you have project= 'CORDEX'. You need special authorization to access that data via open_dap using the CORDEX project_id. I see that you have logged on. So you log on to one of the webportals of ESGF data nodes and check if you are part of the cordex project in your profile. If not, you can simply click something like join Cordex project in the top.

from esgf-pyclient.

Hi @jhamman, right now I don't have time to look into your issue but please see if this example sheds any light on your questions:
https://github.com/cehbrecht/demo-notebooks/blob/master/esgf-opendap.ipynb

from esgf-pyclient.

@agstephens - Indeed, I had seen this notebook. As far as I can tell, the problem seems to lie in the use of aggregation context urls to opendap datasets.

from esgf-pyclient.

@jhamman late answer ... there might be several issues but not related to esgf-pyclient. The aggregation might not work but it also looks like that pydap needs to be updated to work with ESGF.

I tried it with a CORDEX aggregation and I can't get pydap working:
https://github.com/cehbrecht/jupyterlab-notebooks/blob/master/esgf-examples/esgf-pydap.ipynb

See also:
https://pydap.readthedocs.io/en/latest/client.html?#earth-system-grid-federation-esgf

from esgf-pyclient.

Hi, I hope you all are doing well,

Can anyone help me to overcome this issue?

My OpenID is working and is connected to my ESGF acc.

Please let me know if you need more information.

Thank you,

Saeed

from pyesgf.search import SearchConnection conn = SearchConnection('https://esgf-index1.ceda.ac.uk/esg-search/',distrib=True)
ctx = conn.new_context(project= 'CORDEX', institute= 'KNMI', time_frequency= 'day', experiment= 'historical', variable= 'tas') ctx.hit_count
result = ctx.search()[14] result.dataset_id
ds = ctx.search()[14] files = ds.file_context().search() len(files)
for f in files: print(f.download_url);
from pyesgf.logon import LogonManager lm = LogonManager() lm.logoff() lm.is_logged_on()
OPENID = 'https://ceda.ac.uk/openid/xxx' lm.logon_with_openid(openid=OPENID, password=None, bootstrap=True) lm.is_logged_on()
password = 'xxx' username = 'xxx' myproxy_host = 'slcs1.ceda.ac.uk' lm.logon(username, password, hostname=myproxy_host, interactive=True, bootstrap=True) lm.is_logged_on()
import xarray as xr
ds = xr.open_dataset(f.download_url) print(ds)
`KeyError Traceback (most recent call last)
D:\Anaconda\envs\gdal\lib\site-packages\xarray\backends\file_manager.py in _acquire_with_cache_info(self, needs_lock)
197 try:
--> 198 file = self._cache[self._key]
199 except KeyError:

D:\Anaconda\envs\gdal\lib\site-packages\xarray\backends\lru_cache.py in getitem(self, key)
52 with self._lock:
---> 53 value = self._cache[key]
54 self._cache.move_to_end(key)

KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('http://esgf1.dkrz.de/thredds/fileServer/cordex/cordex/output/EUR-11/KNMI/ICHEC-EC-EARTH/historical/r3i1p1/KNMI-RACMO22E/v1/day/tas/v20190108/tas_EUR-11_ICHEC-EC-EARTH_historical_r3i1p1_KNMI-RACMO22E_v1_day_20010101-20051231.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False))]

During handling of the above exception, another exception occurred:

OSError Traceback (most recent call last)
in
----> 1 ds = xr.open_dataset(f.download_url)
2 print(ds)

D:\Anaconda\envs\gdal\lib\site-packages\xarray\backends\api.py in open_dataset(filename_or_obj, group, decode_cf, mask_and_scale, decode_times, autoclose, concat_characters, decode_coords, engine, chunks, lock, cache, drop_variables, backend_kwargs, use_cftime, decode_timedelta)
506 engine = _get_default_engine(filename_or_obj, allow_remote=True)
507 if engine == "netcdf4":
--> 508 store = backends.NetCDF4DataStore.open(
509 filename_or_obj, group=group, lock=lock, **backend_kwargs
510 )

D:\Anaconda\envs\gdal\lib\site-packages\xarray\backends\netCDF4_.py in open(cls, filename, mode, format, group, clobber, diskless, persist, lock, lock_maker, autoclose)
356 netCDF4.Dataset, filename, mode=mode, kwargs=kwargs
357 )
--> 358 return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
359
360 def _acquire(self, needs_lock=True):

D:\Anaconda\envs\gdal\lib\site-packages\xarray\backends\netCDF4_.py in init(self, manager, group, mode, lock, autoclose)
312 self._group = group
313 self._mode = mode
--> 314 self.format = self.ds.data_model
315 self._filename = self.ds.filepath()
316 self.is_remote = is_remote_uri(self._filename)

D:\Anaconda\envs\gdal\lib\site-packages\xarray\backends\netCDF4_.py in ds(self)
365 @Property
366 def ds(self):
--> 367 return self._acquire()
368
369 def open_store_variable(self, name, var):

D:\Anaconda\envs\gdal\lib\site-packages\xarray\backends\netCDF4_.py in _acquire(self, needs_lock)
359
360 def _acquire(self, needs_lock=True):
--> 361 with self._manager.acquire_context(needs_lock) as root:
362 ds = _nc4_require_group(root, self._group, self._mode)
363 return ds

D:\Anaconda\envs\gdal\lib\contextlib.py in enter(self)
111 del self.args, self.kwds, self.func
112 try:
--> 113 return next(self.gen)
114 except StopIteration:
115 raise RuntimeError("generator didn't yield") from None

D:\Anaconda\envs\gdal\lib\site-packages\xarray\backends\file_manager.py in acquire_context(self, needs_lock)
184 def acquire_context(self, needs_lock=True):
185 """Context manager for acquiring a file."""
--> 186 file, cached = self._acquire_with_cache_info(needs_lock)
187 try:
188 yield file

D:\Anaconda\envs\gdal\lib\site-packages\xarray\backends\file_manager.py in _acquire_with_cache_info(self, needs_lock)
202 kwargs = kwargs.copy()
203 kwargs["mode"] = self._mode
--> 204 file = self._opener(*self._args, **kwargs)
205 if self._mode == "w":
206 # ensure file doesn't get overriden when opened again

netCDF4_netCDF4.pyx in netCDF4._netCDF4.Dataset.init()

netCDF4_netCDF4.pyx in netCDF4._netCDF4._ensure_nc_success()

OSError: [Errno -78] NetCDF: Authorization failure: b'http://esgf1.dkrz.de/thredds/fileServer/cordex/cordex/output/EUR-11/KNMI/ICHEC-EC-EARTH/historical/r3i1p1/KNMI-RACMO22E/v1/day/tas/v20190108/tas_EUR-11_ICHEC-EC-EARTH_historical_r3i1p1_KNMI-RACMO22E_v1_day_20010101-20051231.nc'`

from esgf-pyclient.