cate icon indicating copy to clipboard operation
cate copied to clipboard
verified publisher icon CCI-Tools

ODP_landcover: Network Problebem&Data Access Error

Open Evadzi opened this issue 6 years ago • 0 comments

Expected behavior

Ingest esacci.LC.5-years.L4.... data using ODP

Actual behavior

Download failed and returns 2 different error types;(a)Network Problem: for ingestion of LC data without regional constraints and (b)Data Access Error: for ingestion of LC with regional Constraints

Steps to reproduce the problem

  1. for error(a) ingest esacci.LC.5-years.L4.... without regional constraint
  2. *for error(b) ingest esacci.LC.5-years.L4.... with regional constraint( lon -50,50: lat -50, 80)

### stacktrace for Error A: Network Problem ### stacktrace for Error B: Data Access Error(B-after A)

A

  File "c:\users\prosper\cci_toolbox_project\cate\cate\ds\esa_cci_odp.py", line 999, in _make_local
    urllib.request.urlretrieve(actual_url, filename=dataset_file, reporthook=reporthook)
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\urllib\request.py", line 288, in urlretrieve
    % (read, size), result)
urllib.error.ContentTooShortError: <urlopen error retrieval incomplete: got only 1079095485 out of 1176095468 bytes>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "c:\users\prosper\cci_toolbox_project\cate\cate\util\web\jsonrpchandler.py", line 209, in send_service_method_result
    result = future.result()
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\concurrent\futures\_base.py", line 425, in result
    return self.__get_result()
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\concurrent\futures\_base.py", line 384, in __get_result
    raise self._exception
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\concurrent\futures\thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\util\web\jsonrpchandler.py", line 306, in call_service_method
    result = method(*method_params, monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\webapi\websocket.py", line 295, in set_workspace_resource
    monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\wsmanag.py", line 320, in set_workspace_resource
    workspace.execute_workflow(res_name=res_name, monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\workspace.py", line 662, in execute_workflow
    self.workflow.invoke_steps(steps, context=self._new_context(), monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\workflow.py", line 627, in invoke_steps
    steps[0].invoke(context=context, monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\workflow.py", line 318, in invoke
    self._invoke_impl(_new_context(context, step=self), monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\workflow.py", line 980, in _invoke_impl
    return_value = self._op(monitor=monitor, **input_values)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\op.py", line 216, in _call_
    return_value = self._wrapped_op(**input_values)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\ops\io.py", line 83, in open_dataset
    monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\ds.py", line 675, in open_dataset
    monitor=monitor.child(80))
  File "c:\users\prosper\cci_toolbox_project\cate\cate\ds\esa_cci_odp.py", line 1077, in make_local
    raise e
  File "c:\users\prosper\cci_toolbox_project\cate\cate\ds\esa_cci_odp.py", line 1070, in make_local
    self._make_local(local_ds, time_range, region, var_names, monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\ds\esa_cci_odp.py", line 1013, in _make_local
    error_cls=NetworkError) from e
cate.core.ds.NetworkError: Failed to synchronize data source "esacci.LC.5-yrs.L4.LCCS.multi-sensor.multi-platform.Map.1-6-1.r1": <urlopen error retrieval incomplete: got only 1079095485 out of 1176095468 bytes>.```



### **B**

```Job: Opening data source "esacci.LC.5-yrs.L4.LCCS.multi-sensor.multi-platform.Map.1-6-1.r1"
Message: Failed to synchronize data source "esacci.LC.5-yrs.L4.LCCS.multi-sensor.multi-platform.Map.1-6-1.r1" for given region: [Errno -45] NetCDF: Not a valid data type or _FillValue type mismatch: b'https://cci-odp-data.ceda.ac.uk/thredds/dodsC/esg_esacci/land_cover/data/land_cover_maps/v1.6.1/ESACCI-LC-L4-LCCS-Map-300m-P5Y-2000-v1.6.1.nc'
Code: -32003
Method: set_workspace_resource
Exception: cate.core.ds.DataAccessError
Traceback (most recent call last):
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\site-packages\xarray\backends\file_manager.py", line 137, in acquire
    file = self._cache[self._key]
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\site-packages\xarray\backends\lru_cache.py", line 43, in __getitem__
    value = self._cache[key]
KeyError: [<function _open_netcdf4_group at 0x0000026833A05510>, ('https://cci-odp-data.ceda.ac.uk/thredds/dodsC/esg_esacci/land_cover/data/land_cover_maps/v1.6.1/ESACCI-LC-L4-LCCS-Map-300m-P5Y-2000-v1.6.1.nc', <SerializableLock: 7351a1b2-6860-441f-8065-0dd0113df436>), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('group', None), ('persist', False))]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "c:\users\prosper\cci_toolbox_project\cate\cate\ds\esa_cci_odp.py", line 920, in _make_local
    excluded_variables])
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\site-packages\xarray\backends\api.py", line 320, in open_dataset
    filename_or_obj, group=group, lock=lock, **backend_kwargs)
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\site-packages\xarray\backends\netCDF4_.py", line 355, in open
    return cls(manager, lock=lock, autoclose=autoclose)
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\site-packages\xarray\backends\netCDF4_.py", line 314, in __init__
    self.format = self.ds.data_model
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\site-packages\xarray\backends\netCDF4_.py", line 359, in ds
    return self._manager.acquire().value
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\site-packages\xarray\backends\file_manager.py", line 143, in acquire
    file = self._opener(*self._args, **kwargs)
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\site-packages\xarray\backends\netCDF4_.py", line 247, in _open_netcdf4_group
    ds = nc4.Dataset(filename, mode=mode, **kwargs)
  File "netCDF4\_netCDF4.pyx", line 2135, in netCDF4._netCDF4.Dataset.__init__
  File "netCDF4\_netCDF4.pyx", line 1752, in netCDF4._netCDF4._ensure_nc_success
OSError: [Errno -45] NetCDF: Not a valid data type or _FillValue type mismatch: b'https://cci-odp-data.ceda.ac.uk/thredds/dodsC/esg_esacci/land_cover/data/land_cover_maps/v1.6.1/ESACCI-LC-L4-LCCS-Map-300m-P5Y-2000-v1.6.1.nc'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "c:\users\prosper\cci_toolbox_project\cate\cate\util\web\jsonrpchandler.py", line 209, in send_service_method_result
    result = future.result()
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\concurrent\futures\_base.py", line 425, in result
    return self.__get_result()
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\concurrent\futures\_base.py", line 384, in __get_result
    raise self._exception
  File "C:\Users\prosper\Miniconda3\envs\cate-env\lib\concurrent\futures\thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\util\web\jsonrpchandler.py", line 306, in call_service_method
    result = method(*method_params, monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\webapi\websocket.py", line 295, in set_workspace_resource
    monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\wsmanag.py", line 320, in set_workspace_resource
    workspace.execute_workflow(res_name=res_name, monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\workspace.py", line 662, in execute_workflow
    self.workflow.invoke_steps(steps, context=self._new_context(), monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\workflow.py", line 627, in invoke_steps
    steps[0].invoke(context=context, monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\workflow.py", line 318, in invoke
    self._invoke_impl(_new_context(context, step=self), monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\workflow.py", line 980, in _invoke_impl
    return_value = self._op(monitor=monitor, **input_values)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\op.py", line 216, in __call__
    return_value = self._wrapped_op(**input_values)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\ops\io.py", line 83, in open_dataset
    monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\core\ds.py", line 675, in open_dataset
    monitor=monitor.child(80))
  File "c:\users\prosper\cci_toolbox_project\cate\cate\ds\esa_cci_odp.py", line 1077, in make_local
    raise e
  File "c:\users\prosper\cci_toolbox_project\cate\cate\ds\esa_cci_odp.py", line 1070, in make_local
    self._make_local(local_ds, time_range, region, var_names, monitor=monitor)
  File "c:\users\prosper\cci_toolbox_project\cate\cate\ds\esa_cci_odp.py", line 1016, in _make_local
    verb="synchronize", cause=e) from e
cate.core.ds.DataAccessError: Failed to synchronize data source "esacci.LC.5-yrs.L4.LCCS.multi-sensor.multi-platform.Map.1-6-1.r1" for given region: [Errno -45] NetCDF: Not a valid data type or _FillValue type mismatch: b'https://cci-odp-data.ceda.ac.uk/thredds/dodsC/esg_esacci/land_cover/data/land_cover_maps/v1.6.1/ESACCI-LC-L4-LCCS-Map-300m-P5Y-2000-v1.6.1.nc'.```

### Specifications 

*cate 2.0.0 dev 25, Windows 10*

Evadzi avatar Mar 14 '19 16:03 Evadzi