...
 
Commits (3)
This diff is collapsed.
This diff is collapsed.
......@@ -37,10 +37,10 @@ class NCDataset(Dataset):
def _convert_format(
self,
profile='global_attributes_profile.cfg',
attr_profile='global_attributes_profile.cfg',
**kwargs
):
dataset = super()._convert_format(**kwargs)
dataset = super()._convert_format(attr_profile, **kwargs)
# fill in some attributes
creation_date = datetime.datetime.now()
......
......@@ -137,58 +137,16 @@ class Feature(Dataset):
# all forms of time values
return self._make_coord('time', values, None, 'time', 'time')
def add_standard_attrs(self, *args, **kwargs):
"""Add standard global attributes to the feature
You can provide a custom profile of attributes here, using ``profile``
argument. Refer to :func:`~cerbere.cfconvention.default_global_attrs`
"""
defattrs = cfconvention.default_global_attrs(*args, **kwargs)
for att in defattrs:
if att not in self.dataset.attrs:
self.dataset.attrs[att] = defattrs[att]
@property
def xarray(self) -> xr.Dataset:
"""Internal xarray Dataset storage object.
Use with caution as dynamic transformations may not be applied (for
better performances or memory occupation). You may retrieve a feature
object that is no more generic. It is safer to use the
:meth:`~cerbere.dataset.dataset.as_dataset` method (future release).
"""
return self.dataset
def _dataset_class(self):
if isinstance(self.dataset, Dataset):
return self.dataset._dataset_class
else:
return 'Dataset'
def __str__(self):
result = 'Feature : %s\n' % self.__class__.__name__
result = result + 'Dims :\n'
for dim, dsize in self.geodims.items():
result = result + ' .\t{} : {}\n'.format(dim, dsize)
result = result + 'Coordinates :\n'
for coord in self.geocoords:
result = result + ' .\t{} ({})\n'.format(
coord,
', '.join(
[(k + ': ' + str(v))
for k, v in self.get_field_dims(coord).items()]
)
)
result = result + 'Fields :\n'
for field in self.fieldnames:
result = result + ' .\t{} ({})\n'.format(
field,
', '.join(
[(k + ': ' + str(v))
for k, v in self.get_field_dims(field).items()]
)
)
result = result + 'Global Attributes :\n'
for attr in self.attrs:
result = result + ' .\t{}\t{}\n'.format(
attr, str(self.attrs[attr])
)
result += super().__str__()
return result
@property
......@@ -198,11 +156,7 @@ class Feature(Dataset):
def has_coordinate(self, coord: str) -> bool:
"""Return True if the coordinate variable is defined"""
return coord in self.dataset.dataset.coords
# return (
# coord in self._geocoordinates
# and self._geocoordinates[coord] is not None
# )
return coord in self._std_dataset.coords
@property
@abstractmethod
......@@ -227,14 +181,6 @@ class Feature(Dataset):
(_, self.sizes[_]) for _ in list(self._feature_geodimnames)
])
@property
def geodimnames(self):
return tuple(self.geodims.keys())
@property
def geodimsizes(self):
return tuple(self.geodims.values())
@abstractmethod
def get_geocoord_dimnames(self, fieldname):
"""
......@@ -270,7 +216,7 @@ class Feature(Dataset):
newdims = geodims + [
_ for _ in fielddims if _ not in geodims
]
self.dataset[v] = self.dataset[v].transpose(
self._std_dataset[v] = self._std_dataset[v].transpose(
*newdims,
transpose_coords=True
)
......
......@@ -32,13 +32,29 @@ Attributes
Dataset.dims
Dataset.sizes
Dataset.geocoords
Dataset.dimnames
Dataset.dimsizes
Dataset.fieldnames
Dataset.coordnames
Dataset.geocoordnames
Dataset.geocoords
Dataset.get_field
Dataset.get_field_dims
Dataset.get_field_dimnames
Dataset.get_field_dimsizes
Dataset.add_field
Dataset.attrs
Dataset.bbox
Dataset.wkt_bbox
Dataset.time_coverage_start
Dataset.time_coverage_end
Dataset.url
Dataset.basename
Dataset.filesize
Dataset.file_creation_date
Dataset._raw_dataset
Dataset._std_dataset
Dataset contents
......@@ -46,6 +62,11 @@ Dataset contents
.. autosummary::
:toctree: generated
Dataset.exists
Dataset.has_field
Dataset.rename_field
Dataset.add_default_attrs
Other built-in datasets
-----------------------
......
......@@ -15,8 +15,8 @@ a typology of observation objects (called **features**).
Contents
========
Getting started
===============
.. toctree::
:maxdepth: 1
......@@ -25,6 +25,15 @@ Contents
working_with_data
behind_the_scene
User guide
==========
.. toctree::
:maxdepth: 1
creating_datasets
creating_features
Available dataset classes
=========================
......
This diff is collapsed.
"""
Test class for cerbere GHRSST files mapper
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import unittest
from .checker import Checker
class GHRSSTNCDatasetL2PChecker(Checker, unittest.TestCase):
"""Test class for GHRSSTNCFile swath files"""
def __init__(self, methodName="runTest"):
super(GHRSSTNCDatasetL2PChecker, self).__init__(methodName)
@classmethod
def dataset(cls):
"""Return the mapper class name"""
return 'GHRSSTNCDataset'
@classmethod
def feature(cls):
"""Return the related datamodel class name"""
return 'Swath'
@classmethod
def test_file(cls):
"""Return the name of the test file for this test"""
return "20190531130355-REMSS-L2P_GHRSST-SSTsubskin-AMSR2-L2B_v08_r37425-v02.0-fv01.0.nc"
@classmethod
def download_url(cls):
"""Return the URL of the data test repository where to get the test
files
"""
return "ftp://ftp.ifremer.fr/ifremer/cersat/projects/cerbere/test_data/"
def test_dim_swath(self):
ncf = self.datasetclass(self.testfile)
swath = self.featureclass(ncf)
self.assertEquals(tuple(swath.get_geocoord('lat').dims.keys()), ('row', 'cell',))
print("DMS SST ", swath.get_field_dimnames('sea_surface_temperature'))
self.assertEquals(swath.get_field_dimnames('sea_surface_temperature'), ('row', 'cell',))
\ No newline at end of file
......@@ -123,6 +123,33 @@ class TestCylindricalGridFeature(TestFeature):
print("Feature from: test_create_feature_from_dict_datetime64_2d")
print(feat)
def test_create_feature_from_dict_datetime64_2d_v2(self):
basefeat = self.define_base_feature()
times = np.full(
basefeat.get_field_dimsizes('myvar'),
np.datetime64('2018-01-01'),
dtype='datetime64[D]',
)
feat = self.get_feature_class()(
{
'lat': {'data': basefeat.get_lat(), 'dims': ('lat',)},
'lon': {'data': basefeat.get_lon(), 'dims': ('lon',)},
'time': {
'data': times,
'dims': ('lat', 'lon',)
},
'myvar': {
'data': basefeat.get_values('myvar'), 'dims': ('lat', 'lon')
}
}
)
self.assertIsInstance(feat, self.get_feature_class())
self.assertEqual(len(feat.get_field_dimnames('time')), 2)
print("Feature from: test_create_feature_from_dict_datetime64_1d")
print(feat)
def test_expanded_latlon(self):
basefeat = self.define_base_feature()
res = basefeat.get_values(
......@@ -169,4 +196,20 @@ class TestCylindricalGridFeature(TestFeature):
'lat',
expand=True
)
print("result: test_expanded_latlon ", res)
\ No newline at end of file
print("result: test_expanded_latlon ", res)
def test_expanded_latlon(self):
basefeat = self.define_base_feature()
res = basefeat.get_values(
'time',
index={'lat': slice(10, 15), 'lon': slice(50, 53)},
expand=True
)
self.assertEqual(res.shape, (5, 3,))
res = basefeat.get_values(
'time',
expand=True
)
print("result: test_expanded_time ", res)
\ No newline at end of file
from datetime import datetime
import numpy as np
import xarray as xr
from cerbere.feature.grid import CylindricalGrid
from .test_xarray_feature import TestFeature
class TestCylindricalGridFeature(TestFeature):
"""Test class for CylindricalGridFeature"""
def get_feature_class(self):
return CylindricalGrid
def define_base_feature(self):
# creates a test xarray object
lon = xr.DataArray(data=np.arange(-180, 180, 1), dims=['lon'])
lat = xr.DataArray(data=np.arange(-80, 80, 1), dims=['lat'])
time = xr.DataArray([datetime(2018, 1, 1)], dims=['time'])
z = xr.DataArray(data=np.arange(0, 10, 1), dims=['z'])
var = xr.DataArray(
data=np.ones(shape=(160, 360, 10)),
dims=['lat', 'lon', 'z'],
attrs={'myattr': 'test_attr_val'}
)
xrdataset = xr.Dataset(
coords={'lat': lat, 'lon': lon, 'time': time, 'z': z},
data_vars={'myvar': var},
attrs={'gattr1': 'gattr1_val', 'gattr2': 'gattr2_val'}
)
return CylindricalGrid(xrdataset)
def get_feature_dimnames(self):
return ('lat', 'lon',)
def get_feature_dimsizes(self):
return (160, 360)
def test_create_feature_with_incorrect_geodim_order(self):
basefeat = self.define_base_feature()
feat = self.get_feature_class()(
{
'lat': {'data': basefeat.get_lat(), 'dims': ('lat',)},
'lon': {'data': basefeat.get_lon(), 'dims': ('lon',)},
'z': {'data': basefeat.get_z(), 'dims': ('z',)},
'time': {
'data': np.array([datetime(2018, 1, 1)]), 'dims': ('time',)
},
'myvar': {
'data': np.ones(shape=(360, 160, 10)), 'dims': ('lon', 'lat', 'z')
}
}
)
self.assertEqual(feat.get_field_dimnames('myvar'), ('lat', 'lon', 'z'))
print("Feature from: test_create_feature_with_incorrect_geodim_order")
print(feat)
def test_create_feature_from_dict_datetime_1d(self):
basefeat = self.define_base_feature()
feat = self.get_feature_class()(
{
'lat': {'data': basefeat.get_lat(), 'dims': ('lat',)},
'lon': {'data': basefeat.get_lon(), 'dims': ('lon',)},
'z': {'data': basefeat.get_z(), 'dims': ('z',)},
'time': {
'data': np.array([datetime(2018, 1, 1)]), 'dims': ('time',)
},
'myvar': {
'data': basefeat.get_values('myvar'), 'dims': ('lat', 'lon', 'z')
}
}
)
self.assertIsInstance(feat, self.get_feature_class())
self.assertEqual(len(feat.get_field_dimnames('time')), 1)
print("Feature from: test_create_feature_from_dict_datetime_1d")
print(feat)
def test_create_feature_from_dict_datetime64_1d(self):
basefeat = self.define_base_feature()
feat = self.get_feature_class()(
{
'lat': {'data': basefeat.get_lat(), 'dims': ('lat',)},
'lon': {'data': basefeat.get_lon(), 'dims': ('lon',)},
'z': {'data': basefeat.get_z(), 'dims': ('z',)},
'time': {
'data': np.array([np.datetime64('2018-01-01')]),
'dims': ('time',)
},
'myvar': {
'data': basefeat.get_values('myvar'), 'dims': ('lat', 'lon', 'z')
}
}
)
self.assertIsInstance(feat, self.get_feature_class())
self.assertEqual(len(feat.get_field_dimnames('time')), 1)
print("Feature from: test_create_feature_from_dict_datetime64_1d")
print(feat)
def test_create_feature_from_dict_datetime64_2d(self):
basefeat = self.define_base_feature()
times = np.full(
(160, 360,),
np.datetime64('2018-01-01'),
dtype='datetime64[D]',
)
feat = self.get_feature_class()(
{
'coords': {
'lat': {'data': basefeat.get_lat(), 'dims': ('lat',)},
'lon': {'data': basefeat.get_lon(), 'dims': ('lon',)},
'z': {'data': basefeat.get_z(), 'dims': ('z',)},
'time': {'data': times, 'dims': ('lat', 'lon')}
},
'data_vars': {
'myvar': {
'data': basefeat.get_values('myvar'),
'dims': ('lat', 'lon', 'z')
}
},
}
)
self.assertIsInstance(feat, self.get_feature_class())
self.assertEqual(len(feat.get_field_dimnames('time')), 2)
print("Feature from: test_create_feature_from_dict_datetime64_2d")
print(feat)
def test_create_feature_from_dict_datetime64_2d_v2(self):
basefeat = self.define_base_feature()
times = np.full(
(160, 360,),
np.datetime64('2018-01-01'),
dtype='datetime64[D]',
)
feat = self.get_feature_class()(
{
'lat': {'data': basefeat.get_lat(), 'dims': ('lat',)},
'lon': {'data': basefeat.get_lon(), 'dims': ('lon',)},
'z': {'data': basefeat.get_z(), 'dims': ('z',)},
'time': {
'data': times,
'dims': ('lat', 'lon',)
},
'myvar': {
'data': basefeat.get_values('myvar'), 'dims': ('lat', 'lon', 'z')
}
}
)
self.assertIsInstance(feat, self.get_feature_class())
self.assertEqual(len(feat.get_field_dimnames('time')), 2)
print("Feature from: test_create_feature_from_dict_datetime64_1d")
print(feat)
def test_expanded_latlon(self):
basefeat = self.define_base_feature()
res = basefeat.get_values(
'lat',
index={'lat': slice(10, 15), 'lon': slice(50, 55)},
expand=False
)
self.assertEqual(res.shape, (5,))
self.assertTrue(np.equal(res, np.arange(-70, -65)).all())
res = basefeat.get_values(
'lat',
index={'lat': 10, 'lon': 50},
expand=True
)
self.assertEqual(res, -70)
res = basefeat.get_values(
'lon',
index={'lat': 10, 'lon': 50},
expand=True
)
self.assertEqual(res, -130)
res = basefeat.get_values(
'lat',
index={'lat': slice(10, 15), 'lon': slice(50, 53)},
expand=True
)
self.assertEqual(res.shape, (5, 3,))
self.assertTrue(np.equal(res[:, 0], np.arange(-70, -65)).all())
self.assertTrue(np.equal(res[0, :], np.ones((3,)) * -70).all())
res = basefeat.get_values(
'lon',
index={'lat': slice(10, 15), 'lon': slice(50, 53)},
expand=True
)
self.assertEqual(res.shape, (5, 3,))
self.assertTrue(np.equal(res[0, :], np.arange(-130, -127)).all())
self.assertTrue(np.equal(res[:, 0], np.ones((5,)) * -130).all())
res = basefeat.get_values(
'lat',
expand=True
)
print("result: test_expanded_latlon ", res)
def test_expanded_latlon(self):
basefeat = self.define_base_feature()
res = basefeat.get_values(
'time',
index={'lat': slice(10, 15), 'lon': slice(50, 53)},
expand=True
)
self.assertEqual(res.shape, (5, 3,))
res = basefeat.get_values(
'time',
expand=True
)
print("result: test_expanded_time ", res)
\ No newline at end of file
......@@ -336,4 +336,13 @@ class TestXArrayDataset(unittest.TestCase):
dst.get_field('test_var').to_dataarray()
)
self.assertEqual(subset.get_values('test_var').count(), 50)
self.assertEqual(subset.get_values('test_var').size, 100)
\ No newline at end of file
self.assertEqual(subset.get_values('test_var').size, 100)
def test_save_with_profile(self):
dst = Dataset(self.xrdataset)
if os.path.exists('test_profile.nc'):
os.remove("test_profile.nc")
dst.save(
dest='test_profile.nc',
attr_profile='global_attributes_profile.cfg'
)
\ No newline at end of file
......@@ -6,7 +6,7 @@ import numpy as np
import xarray as xr
from cerbere.dataset.ncdataset import NCDataset
from cerbere.dataset.field import Field, Variable
from cerbere.dataset.field import Field
TEST_FILE = 'test_feature_dataset.nc'
TEST_SAVE = 'test_saved_feature.nc'
......
......@@ -22,12 +22,12 @@ class GHRSSTNCDatasetL2PChecker(Checker, unittest.TestCase):
super(GHRSSTNCDatasetL2PChecker, self).__init__(methodName)
@classmethod
def mapper(cls):
def dataset(cls):
"""Return the mapper class name"""
return 'GHRSSTNCDataset'
@classmethod
def datamodel(cls):
def feature(cls):
"""Return the related datamodel class name"""
return 'Swath'
......@@ -44,8 +44,8 @@ class GHRSSTNCDatasetL2PChecker(Checker, unittest.TestCase):
return "ftp://ftp.ifremer.fr/ifremer/cersat/projects/cerbere/test_data/"
def test_dim_swath(self):
ncf = self.mapperclass(self.testfile)
swath = self.datamodelclass(ncf)
ncf = self.datasetclass(self.testfile)
swath = self.featureclass(ncf)
self.assertEquals(swath.get_geocoord('lat').dims, ('row', 'cell',))
print("DMS SST ", swath.get_field('sea_surface_temperature').dims)
self.assertEquals(
......
......@@ -5,7 +5,7 @@ import numpy as np
import xarray as xr
from cerbere.dataset.ncdataset import NCDataset
from cerbere.dataset.field import Field, Variable
from cerbere.dataset.field import Field
from cerbere.feature.grid import CylindricalGrid
from .test_xarray_feature import TestFeature, TEST_FILE, TEST_SAVE
......@@ -129,7 +129,7 @@ class TestGridFeature(TestFeature):
data=np.ones(shape=(360, 160))
)
newfield = Field(
variable=Variable('newfield'),
name='newfield',
data=da
)
grid.add_field(newfield)
......
"""
Test class for cerbere GHRSST files mapper
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from datetime import datetime
import os
import unittest
import numpy as np
from .checker import Checker
from cerbere.feature.grid import CylindricalGrid
from cerbere.dataset.ncdataset import NCDataset
class NCDatasetL2PChecker(Checker, unittest.TestCase):
"""Test class for NCDataset 3D Grid such as CMEMS model files"""
def __init__(self, methodName="runTest"):
super(NCDatasetL2PChecker, self).__init__(methodName)
@classmethod
def dataset(cls):
"""Return the mapper class name"""
return 'NCDataset'
@classmethod
def feature(cls):
"""Return the related datamodel class name"""
return 'GridTimeSeries'
@classmethod
def test_file(cls):
"""Return the name of the test file for this test"""
return "mercatorpsy4v3r1_gl12_hrly_20200219_R20200210.nc"
@classmethod
def download_url(cls):
"""Return the URL of the data test repository where to get the test
files
"""
return "ftp://ftp.ifremer.fr/ifremer/cersat/projects/cerbere/test_data/"
def test_dim_swath(self):
ncf = self.datasetclass(self.testfile)
feature = self.featureclass(ncf)
self.assertEquals(feature.get_geocoord('lat').dims, ('row', 'cell'))
self.assertEquals(
feature.get_field('thetao').dims,
('row', 'cell', 'time')
)