Commit 2725c6cc authored by PIOLLE's avatar PIOLLE

code changes following doc updates

parent 2939bde6
......@@ -11,6 +11,7 @@ import configparser
import inspect
import logging
import os
from typing import Mapping, Any
# Module level default values
DEFAULT_TIME_UNITS = 'seconds since 1981-01-01 00:00:00'
......@@ -18,8 +19,30 @@ DEFAULT_TIME_UNITS = 'seconds since 1981-01-01 00:00:00'
CF_AUTHORITY = 'CF-1.7'
def default_global_attrs(path=None, profile='global_attributes_profile.cfg'):
if path is None:
def default_global_attrs(
profile: str='global_attributes_profile.cfg'
) -> Mapping[str, Any]:
"""Returns a list of default global attributes
By default the function returns the built-in default dict of attributes
provided in the ``cerbere.share`` folder of the package.
This list can be customized by defining your own list of attributes in a new
attribute profile file and calling the function with ``profile`` argument.
If the profile file is stored in your home dir in ``~.cerbere`` folder, you
can just provide the file name, otherwise provide the full path to your
profile file.
Args:
profile: the path to or filename of the attribute profile configuration
file
Returns:
a dict of attributes and values
"""
if profile is None:
profile = 'global_attributes_profile.cfg'
if not os.path.exists(profile):
# get path from home dir or default
path = os.path.join(
os.path.expanduser("~"),
......@@ -34,6 +57,8 @@ def default_global_attrs(path=None, profile='global_attributes_profile.cfg'):
logging.warning(
'Using default global attribute file: {}'.format(path)
)
else:
path = profile
# read attributes
config = configparser.RawConfigParser()
......
This diff is collapsed.
......@@ -37,20 +37,11 @@ class NCDataset(Dataset):
def _convert_format(
self,
path=None,
profile='global_attributes_profile.cfg',
**kwargs
):
dataset = super()._convert_format(**kwargs)
# add CF attributes
attrs = default_global_attrs(
path=path, profile=profile
)
for att in attrs:
if att not in dataset.attrs:
dataset.attrs[att] = attrs[att]
# fill in some attributes
creation_date = datetime.datetime.now()
dataset.attrs['date_created'] = creation_date
......
......@@ -11,6 +11,7 @@ from collections import OrderedDict
import datetime
import logging
import warnings
from typing import Union
import numpy as np
import pandas as pd
......@@ -136,16 +137,26 @@ class Feature(Dataset):
# all forms of time values
return self._make_coord('time', values, None, 'time', 'time')
def add_standard_attrs(self):
"""Add standard global attributes"""
defattrs = cfconvention.default_global_attrs()
def add_standard_attrs(self, *args, **kwargs):
"""Add standard global attributes to the feature
You can provide a custom profile of attributes here, using ``profile``
argument. Refer to :func:`~cerbere.cfconvention.default_global_attrs`
"""
defattrs = cfconvention.default_global_attrs(*args, **kwargs)
for att in defattrs:
if att not in self.dataset.attrs:
self.dataset.attrs[att] = defattrs[att]
@property
def xarray(self) -> xr.Dataset:
"""Use with caution"""
"""Internal xarray Dataset storage object.
Use with caution as dynamic transformations may not be applied (for
better performances or memory occupation). You may retrieve a feature
object that is no more generic. It is safer to use the
:meth:`~cerbere.dataset.dataset.as_dataset` method (future release).
"""
return self.dataset
def __str__(self):
......@@ -185,7 +196,7 @@ class Feature(Dataset):
"""Return the type of the feature"""
return self.__class__.__name__
def has_coordinate(self, coord):
def has_coordinate(self, coord: str) -> bool:
"""Return True if the coordinate variable is defined"""
return coord in self.dataset.dataset.coords
# return (
......@@ -193,54 +204,6 @@ class Feature(Dataset):
# and self._geocoordinates[coord] is not None
# )
def set_bbox(self, bbox=None):
"""Set the bounding box of the feature
Args:
:class:`shapely.geometry.Polygon`: the bounding box. If None, it is
estimated from the feature lat/lon.
"""
if isinstance(bbox, tuple):
warnings.warn(
"bbox as a tuple is deprecated. Use shapely geometry.",
FutureWarning
)
self.dataset.attrs['bbox'] = shapely.geometry.box(bbox)
elif isinstance(bbox, shapely.geometry.Polygon):
self.dataset.attrs['bbox'] = bbox
elif bbox is None:
# estimate from the lat/lon
if self.has_coordinate('lat') and self.has_coordinate('lon'):
lats = self.get_lat()
lons = self.get_lon()
self.dataset.attrs['bbox'] = shapely.geometry.box(
lons.min(),
lats.min(),
lons.max(),
lats.max()
)
else:
raise TypeError("Wrong type for bbox: {}".format(type(bbox)))
def get_bbox(self):
'''
returns the bounding box, e.g. the south/north and east/west
limits of the feature.
Return:
:class:`shapely.geometry.Polygon`: the bounding box
'''
if 'bbox' not in self.attrs:
self.set_bbox()
return self._bbox
@property
def wkt_bbox(self) -> str:
"""The bounding box in WKT format."""
return self._bbox.wkt
@property
@abstractmethod
def _feature_geodimnames(self):
......@@ -371,7 +334,7 @@ class Feature(Dataset):
return Field._read_dataarray(field, **kwargs)
def append(self, feature, prefix, fields=None):
"""Append the fields of another feature
"""Append the fields from another feature
It is assumed the two features do not share any dimension. The appended
feature (child) is considered as ancillary fields and looses its
......@@ -715,7 +678,7 @@ class Feature(Dataset):
"""
if bbox is not None or footprint is not None:
raise NotImplementedError
return super().extract(**kwargs)
return self.__class__(super().extract(**kwargs))
def extract_field(
self,
......
......@@ -35,6 +35,17 @@ Attributes
Dataset.geocoords
Dataset.coordnames
Dataset.attrs
Dataset.bbox
Dataset.wkt_bbox
Dataset.time_coverage_start
Dataset.time_coverage_end
Dataset contents
----------------
.. autosummary::
:toctree: generated
Other built-in datasets
-----------------------
......@@ -89,11 +100,15 @@ Field contents
Field.set_values
Field.to_dataarray
Field.compute
Field.module
module
Feature
=======
A :class:`~cerbere.feature.feature.Feature` object inherits all the attributes
and methods of a :class:`~cerbere.dataset.dataset.Dataset` object. It provides
in addition the following methods and attributes.
Creating a feature
------------------
.. currentmodule:: cerbere.feature
......@@ -117,8 +132,8 @@ Attributes
Feature.geodims
Feature.geodimnames
Feature.geodimsizes
Feature.bbox
Feature.wkt_bbox
Feature.xarray
Feature contents
......@@ -126,6 +141,12 @@ Feature contents
.. autosummary::
:toctree: generated
get_lat
get_lon
get_times
\ No newline at end of file
Feature.get_lat
Feature.get_lon
Feature.get_times
Feature.extract
Feature.get_values
Feature.set_values
Feature.add_standard_attrs
Feature.xarray
Feature.append
\ No newline at end of file
.. |dataset| replace:: :mod:`~cerbere.dataset`
.. |feature| replace:: :mod:`~cerbere.feature`
=======================
Unit testing in cerbere
=======================
This section is for cerbere developers, in particular when implementing a new mapper.
This section is for cerbere developers, in particular when implementing a new
|dataset|.
mapper testing
==============
|dataset| testing
=================
Cerbere unit testing is based on :mod:`unittest` python package. This section
describes the procedure to test new or existing mappers.
describes the procedure to test new or existing |dataset| classes.
test data
---------
When providing a new mapper, test data files should be provided too. These
When providing a new |dataset|, test data files should be provided too. These
data files can not be stored on cerbere git server and must be made available
on some server.
As an example, the test data files for the mappers implemented by Ifremer are
available at ftp://ftp.ifremer.fr/ifremer/cersat/projects/cerbere/test_data/
As an example, the test data files for the |dataset| classes implemented by
Ifremer are available at
ftp://ftp.ifremer.fr/ifremer/cersat/projects/cerbere/test_data/
There should be one subfolder for each mapper module. For instance for the
Sentinel-3 SLSTR mappers (:mod:`cerberecontrib-s3` package, :mod:`safeslfile` module),
the test data files are to be found at:
There should be one subfolder for each |dataset| module. For instance for the
Sentinel-3 SLSTR |dataset| classes (:mod:`cerberecontrib-s3` package,
:mod:`safeslfile` module), the test data files are to be found at:
ftp://ftp.ifremer.fr/ifremer/cersat/projects/cerbere/test_data/safeslfile/
There should be at least one test file per class in the mapper module.
There should be at least one test file per class in the |dataset| module.
unit testing on mapper classes
------------------------------
An abtract test class is provided in :class:`cerbere.mapper.checker.Checker` package.
An abtract test class is provided in :class:`cerbere.test.checker.Checker`
module.
To unit test a mapper class, you basically need to create a new test class,
To unit test a |dataset| class, you basically need to create a new test class,
inheriting this class and :class:`unittest.TestCase` class. A test class must
be created for each class in your mapper module.
be created for each class in your |dataset| module.
This class file must be in the ``tests`` repository cerbere or your contrib
package.
For instance, for Sentinel-3 SLSTR SAFESLIRFIle mapper class for infra-red
For instance, for Sentinel-3 SLSTR SAFESLIRFIle |dataset| class for infra-red
products (RBT and WCT products), a simple test class would look as follow:
.. code-block:: python
"""
Test class for cerbere Sentinel-3 L1 mapper
Test class for cerbere Sentinel-3 L1 dataset
:copyright: Copyright 2016 Ifremer / Cersat.
:license: Released under GPL v3 license, see :ref:`license`.
......@@ -59,7 +65,7 @@ products (RBT and WCT products), a simple test class would look as follow:
"""
import unittest
from cerbere.mapper.checker import Checker
from cerbere.test.checker import Checker
class S3L1RBTChecker(Checker, unittest.TestCase):
......@@ -91,39 +97,38 @@ products (RBT and WCT products), a simple test class would look as follow:
return "ftp://ftp.ifremer.fr/ifremer/cersat/projects/cerbere/test_data/"
Your test class must inherit from both the abstract class :class:`cerbere.mapper.checker.Checker` and the
Your test class must inherit from both the abstract class :class:`cerbere.test.checker.Checker` and the
generic test class :class:`unittest.TestCase`:
.. code-block:: python
class S3L1RBTChecker(Checker, unittest.TestCase):
In the `:func:__init__` method of your test class, use the following code stub,
replacing ``S3L1RBTChecker`` with your test class name.
In the `:func:__init__` method of your test class, use the following code stub,
replacing ``S3L1RBTChecker`` with your test class name.
.. code-block:: python
def __init__(self, methodName="runTest"):
super(S3L1RBTChecker, self).__init__(methodName)
Then provide the main information needed to run the tests implemented in
:class:`cerbere.mapper.checker.Checker` class:
:class:`cerbere.test.checker.Checker` class:
* the mapper class name and associated datamodel class name to be passed
to the parent constructor, returned by :meth:`mapper` and :meth:`datamodel`
* the |dataset| class name and associated |feature| class name to be passed
to the parent constructor, returned by :meth:`dataset` and :meth:`feature`
class methods
.. code-block:: python
@classmethod
def mapper(cls):
"""Return the mapper class name"""
def dataset(cls):
"""Return the dataset class name"""
return 'safeslfile.SAFESLIRFile'
@classmethod
def datamodel(cls):
"""Return the related datamodel class name"""
def feature(cls):
"""Return the related feature class name"""
return 'Swath'
* which file to use for testing the class, returned by :meth:`test_file`
......
......@@ -10,7 +10,7 @@ in case you also want to use this |dataset| class to save data in a specific
format).
Creating a new |dataset| class
===============================
==============================
Writing a |dataset| class consists in writing a set of function that helps
cerbere to understand and access a file content. It must implements the
interface defined by :class:`~cerbere.dataset.dataset.Dataset` class, and
......@@ -47,12 +47,11 @@ follow:
dataset, **kwargs
)
The following functions of the :class:`~cerbere.dataset.dataset.Dataset` have
to be overriden, if necessary:
* __init__
* open()
* close
* :meth:`~cerbere.dataset.dataset.Dataset._open`
* :meth:`~cerbere.dataset.dataset.Dataset.close`
* get_matching_dimname
* get_standard_dimname
* get_geolocation_field
......@@ -68,7 +67,7 @@ to be overriden, if necessary:
* get_bbox
* get_spatial_resolution_in_deg
* get_start_time
* :meth:`~cerbere.dataset.dataset.Dataset.get_start_time`
* get_end_time
* read_fillvalue
......@@ -116,7 +115,7 @@ this file. The following operations must be performed:
"""
:func:`close()`
--------------
---------------
.. code-block:: python
......@@ -342,8 +341,9 @@ object.
"""
return {}
:func:`read_fillvalue()`
-------------------------------
------------------------
.. code-block:: python
......@@ -361,7 +361,7 @@ object.
:func:`read_global_attributes()`
-------------------------------
--------------------------------
.. code-block:: python
......
......@@ -94,7 +94,7 @@ setup(
),
install_requires=[
'netCDF4',
'xarray',
'xarray>=0.15',
'cftime',
'Shapely>=1.2.18',
'python-dateutil>=2.1'
......
......@@ -48,9 +48,8 @@ publisher_type = institution
creator_name = CERSAT
creator_url = http://cersat.ifremer.fr
creator_email = cersat@ifremer.fr
publisher_type = institution
creator_institution = Ifremer
creator_type = institution
creator_institution = Ifremer
acknowledgment =
contributor_name =
contributor_role =
......
......@@ -243,10 +243,12 @@ class Checker():
def test_read_time_coverage(self):
"""Test reading the coverage start end end time"""
mapperobj = self.open_file()
start = mapperobj.get_start_time()
start = mapperobj.time_coverage_start
print("...start time : ", start)
self.assertIsInstance(start, datetime.datetime)
end = mapperobj.get_end_time()
end = mapperobj.time_coverage_end
print("...end time : ", end)
self.assertIsInstance(end, datetime.datetime)
mapperobj.close()
......
......@@ -163,13 +163,13 @@ class TestXArrayDataset(unittest.TestCase):
def test_var_get_field_attributes(self):
print("...from xarray.Dataset")
dst = Dataset(self.xrdataset)
self.assertIsInstance(dst.get_field_attrs('test_var'), OrderedDict)
self.assertIsInstance(dst.get_field_attrs('test_var'), dict)
self.assertEqual(dst.get_field_attrs('test_var'),
{'test_attr': 'test_attr_val'})
print("...from file")
dst = Dataset('test_xarraydataset.nc')
self.assertIsInstance(dst.get_field_attrs('test_var'), OrderedDict)
self.assertIsInstance(dst.get_field_attrs('test_var'), dict)
self.assertEqual(dst.get_field_attrs('test_var'),
{'test_attr': 'test_attr_val'})
......@@ -181,12 +181,12 @@ class TestXArrayDataset(unittest.TestCase):
print("...from xarray.Dataset")
dst = Dataset(self.xrdataset)
self.assertIsInstance(dst.attrs, OrderedDict)
self.assertIsInstance(dst.attrs, dict)
self.assertDictEqual(dict(dst.attrs), attrs)
print("...from file")
dst = Dataset('test_xarraydataset.nc')
self.assertIsInstance(dst.attrs, OrderedDict)
self.assertIsInstance(dst.attrs, dict)
self.assertDictEqual(dict(dst.attrs), attrs)
def test_var_get_global_attribute(self):
......@@ -203,14 +203,14 @@ class TestXArrayDataset(unittest.TestCase):
print("...from xarray.Dataset")
dst = Dataset(self.xrdataset)
print(dst.dataset.attrs)
self.assertEqual(dst.get_start_time(), None)
self.assertEqual(dst.get_end_time(), None)
self.assertEqual(dst.time_coverage_start, None)
self.assertEqual(dst.time_coverage_end, None)
print("...from file")
dst = Dataset('test_xarraydataset.nc')
print(dst.dataset.attrs)
self.assertEqual(dst.get_start_time(), None)
self.assertEqual(dst.get_end_time(), None)
self.assertEqual(dst.time_coverage_start, None)
self.assertEqual(dst.time_coverage_end, None)
def test_print(self):
print("Test print")
......@@ -278,8 +278,8 @@ class TestXArrayDataset(unittest.TestCase):
subset.get_values('test_var').shape, (360, 160)
)
self.assertIsNot(
subset.get_field('test_var').to_dataarray,
dst.get_field('test_var').to_dataarray
subset.get_field('test_var').to_dataarray(),
dst.get_field('test_var').to_dataarray()
)
def test_extract_as_view(self):
......@@ -290,8 +290,8 @@ class TestXArrayDataset(unittest.TestCase):
subset.get_values('test_var').shape, (360, 160)
)
self.assertIs(
subset.get_field('test_var').to_dataarray,
dst.get_field('test_var').to_dataarray
subset.get_field('test_var').to_dataarray().data,
dst.get_field('test_var').to_dataarray().data
)
def test_extract_rename(self):
......@@ -302,8 +302,8 @@ class TestXArrayDataset(unittest.TestCase):
subset.get_values('new_test_var').shape, (360, 160)
)
self.assertIsNot(
subset.get_field('new_test_var').to_dataarray,
dst.get_field('test_var').to_dataarray
subset.get_field('new_test_var').to_dataarray(),
dst.get_field('test_var').to_dataarray()
)
def test_extract_subset(self):
......@@ -316,8 +316,8 @@ class TestXArrayDataset(unittest.TestCase):
subset.get_values('test_var').shape, (5, 10)
)
self.assertIsNot(
subset.get_field('test_var').to_dataarray,
dst.get_field('test_var').to_dataarray
subset.get_field('test_var').to_dataarray(),
dst.get_field('test_var').to_dataarray()
)
def test_extract_subset_padding(self):
......@@ -332,8 +332,8 @@ class TestXArrayDataset(unittest.TestCase):
subset.get_values('test_var').shape, (10, 10)
)
self.assertIsNot(
subset.get_field('test_var').to_dataarray,
dst.get_field('test_var').to_dataarray
subset.get_field('test_var').to_dataarray(),
dst.get_field('test_var').to_dataarray()
)
self.assertEqual(subset.get_values('test_var').count(), 50)
self.assertEqual(subset.get_values('test_var').size, 100)
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment