Accessing remote files with earthaccess¶
When we search for data using earthaccess we get back a list of results from NASA's Common Metadata Repository or CMR for short. These results contain all the information
we need to access the files represented by the metadata. earthaccess offers 2 access methods that operate with these results, the first method is the well known, download()
where we copy the results from their location to our local disk, if we are running the code in AWS say on a Jupyterhub the files will be copied to the local VM disk.
The other method is open(), earthaccess uses fsspec to open remote files as if they were local. open has advantages and some disadvantages that we must know before using it.
The main advantage for open() is that we don't have to download the file, we can stream it into memory however depending on how we do it we may run into network performance issues. Again, if we run the code next to the data this would be fast, if we do it locally in our laptopts it will be slow.
import earthaccess
auth = earthaccess.login()
--------------------------------------------------------------------------- TypeError Traceback (most recent call last) Cell In[1], line 1 ----> 1 import earthaccess 3 auth = earthaccess.login() File ~/checkouts/readthedocs.org/user_builds/earthaccess/checkouts/1298/earthaccess/__init__.py:5 2 import threading 3 from typing import Optional ----> 5 from .api import ( 6 auth_environ, 7 collection_query, 8 download, 9 get_edl_token, 10 get_fsspec_https_session, 11 get_requests_https_session, 12 get_s3_credentials, 13 get_s3_filesystem, 14 get_s3fs_session, 15 granule_query, 16 login, 17 open, # noqa: A004 18 search_data, 19 search_datasets, 20 search_services, 21 status, 22 ) 23 from .auth import Auth 24 from .search import DataCollection, DataCollections, DataGranule, DataGranules File ~/checkouts/readthedocs.org/user_builds/earthaccess/checkouts/1298/earthaccess/api.py:20 18 from .auth import Auth 19 from .results import DataCollection, DataGranule, Results ---> 20 from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery 21 from .store import Store 22 from .system import PROD, System File ~/checkouts/readthedocs.org/user_builds/earthaccess/checkouts/1298/earthaccess/search.py:27 23 type FloatLike = str | SupportsFloat 24 type PointLike = tuple[FloatLike, FloatLike] ---> 27 class DataCollections(CollectionQuery): 28 """Query CMR for collection metadata. 29 30 ???+ Info (...) 33 the response has to be in umm_json to use the result classes. 34 """ 36 _fields: list[str] | None = None File ~/checkouts/readthedocs.org/user_builds/earthaccess/checkouts/1298/earthaccess/search.py:84, in DataCollections() 79 raise RuntimeError(ex.response.text) from ex 81 return int(response.headers["CMR-Hits"]) 83 @override ---> 84 def get(self, limit: int = 2000) -> Results[DataCollection]: 85 """Get all the collections (datasets) that match with our current parameters 86 up to some limit, even if spanning multiple pages. 87 (...) 101 RuntimeError: The CMR query failed. 102 """ 103 return Results( 104 [ 105 DataCollection(collection, self._fields) (...) 108 query=self, 109 ) TypeError: type 'Results' is not subscriptable
results = earthaccess.search_data(
short_name="ATL06",
cloud_hosted=False,
temporal=("2019-01", "2019-02"),
polygon=[(-100, 40), (-110, 40), (-105, 38), (-100, 40)],
)
results[0]
--------------------------------------------------------------------------- NameError Traceback (most recent call last) Cell In[2], line 1 ----> 1 results = earthaccess.search_data( 2 short_name="ATL06", 3 cloud_hosted=False, 4 temporal=("2019-01", "2019-02"), 5 polygon=[(-100, 40), (-110, 40), (-105, 38), (-100, 40)], 6 ) 7 results[0] NameError: name 'earthaccess' is not defined
nsidc_url = "https://n5eil01u.ecs.nsidc.org/DP7/ATLAS/ATL06.005/2019.02.21/ATL06_20190221121851_08410203_005_01.h5"
lpcloud_url = "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20220903T163129_2224611_012/EMIT_L2A_RFL_001_20220903T163129_2224611_012.nc"
session = earthaccess.get_requests_https_session()
--------------------------------------------------------------------------- NameError Traceback (most recent call last) Cell In[3], line 4 1 nsidc_url = "https://n5eil01u.ecs.nsidc.org/DP7/ATLAS/ATL06.005/2019.02.21/ATL06_20190221121851_08410203_005_01.h5" 2 lpcloud_url = "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20220903T163129_2224611_012/EMIT_L2A_RFL_001_20220903T163129_2224611_012.nc" ----> 4 session = earthaccess.get_requests_https_session() NameError: name 'earthaccess' is not defined
headers = {"Range": "bytes=0-100"}
r = session.get(lpcloud_url, headers=headers)
r
--------------------------------------------------------------------------- NameError Traceback (most recent call last) Cell In[4], line 2 1 headers = {"Range": "bytes=0-100"} ----> 2 r = session.get(lpcloud_url, headers=headers) 3 r NameError: name 'session' is not defined
fs = earthaccess.get_fsspec_https_session()
--------------------------------------------------------------------------- NameError Traceback (most recent call last) Cell In[5], line 1 ----> 1 fs = earthaccess.get_fsspec_https_session() NameError: name 'earthaccess' is not defined
with fs.open(lpcloud_url) as f:
data = f.read(100)
data
--------------------------------------------------------------------------- NameError Traceback (most recent call last) Cell In[6], line 1 ----> 1 with fs.open(lpcloud_url) as f: 2 data = f.read(100) 3 data NameError: name 'fs' is not defined
%%time
import xarray as xr
files = earthaccess.open(results[0:2])
ds = xr.open_dataset(files[0], group="/gt1r/land_ice_segments")
ds
CPU times: user 1.85 s, sys: 90.2 ms, total: 1.94 s Wall time: 1.84 s
--------------------------------------------------------------------------- NameError Traceback (most recent call last) Cell In[7], line 1 ----> 1 get_ipython().run_cell_magic('time', '', '\nimport xarray as xr\n\nfiles = earthaccess.open(results[0:2])\n\nds = xr.open_dataset(files[0], group="/gt1r/land_ice_segments")\nds\n') File ~/checkouts/readthedocs.org/user_builds/earthaccess/envs/1298/lib/python3.12/site-packages/IPython/core/interactiveshell.py:2565, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2563 with self.builtin_trap: 2564 args = (magic_arg_s, cell) -> 2565 result = fn(*args, **kwargs) 2567 # The code below prevents the output from being displayed 2568 # when using magics with decorator @output_can_be_silenced 2569 # when the last Python token in the expression is a ';'. 2570 if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): File ~/checkouts/readthedocs.org/user_builds/earthaccess/envs/1298/lib/python3.12/site-packages/IPython/core/magics/execution.py:1470, in ExecutionMagics.time(self, line, cell, local_ns) 1468 if interrupt_occured: 1469 if exit_on_interrupt and captured_exception: -> 1470 raise captured_exception 1471 return 1472 return out File ~/checkouts/readthedocs.org/user_builds/earthaccess/envs/1298/lib/python3.12/site-packages/IPython/core/magics/execution.py:1434, in ExecutionMagics.time(self, line, cell, local_ns) 1432 st = clock2() 1433 try: -> 1434 exec(code, glob, local_ns) 1435 out = None 1436 # multi-line %%time case File <timed exec>:3 NameError: name 'earthaccess' is not defined