Commit 0ddf0b8b authored by Jeff Piollé's avatar Jeff Piollé

merge SLSTS mapper

parents dad4fa4f 09f1dcee
......@@ -39,6 +39,8 @@ nosetests.xml
DS_Store
*.sqlite3
VERSION.txt
migrations
*/migrations/*.py
......
......@@ -55,8 +55,9 @@ class AbstractFeature(object):
"""
"""
object.__init__(self)
if fields is not None:
assert (type(fields) is dict), "fields must be a dictionary"
if fields is not None and not isinstance(fields, dict):
raise Exception('fields must be a dictionary (use an OrderedDict '
'if you want to write fields in a specific order).')
self._datastorage = None
# geolocation fields (time,lat,lon)
self._geolocation_fields = {'time': None,
......@@ -110,7 +111,7 @@ class AbstractFeature(object):
self.metadata = {}
self._fields = fields
if fields is None:
self._fields = {}
self._fields = collections.OrderedDict()
return
def __str__(self):
......
......@@ -24,16 +24,6 @@ except DistributionNotFound, exception:
except ImportError:
logging.exception('A netCDF mapper failed to load, and is unavailable:')
try:
packages.require('pyhdf')
# from . import hdffile
# from . import qscathdffile
except DistributionNotFound, exception:
logging.warning('Python pyhdf package is required for HDF. '
'No HDF compatable modules are avaialble on this cerbere instance')
except ImportError:
logging.exception('An HDF mapper failed to load, and is unavailable:')
try:
packages.require('pygrib')
# from . import gribfile
......
......@@ -120,7 +120,7 @@ class KNMIL2IFRNCFile(NCFile):
# create a virtual field
variable = Variable(
shortname=fieldname,
description='best wind % solution'
description='best wind %s solution'
% VIRTUALFIELD_DESCR[fieldname],
authority=self.get_naming_authority(),
standardname='wind %s' % VIRTUALFIELD_DESCR[fieldname]
......
......@@ -166,12 +166,12 @@ class RapidScatPODAACNCFile(NCFile):
return field
def read_values(self, fieldname, slices=None):
if fieldname == 'time':
val = super(RapidScatPODAACNCFile, self).read_values('time')
# if fieldname == 'time':
# val = super(RapidScatPODAACNCFile, self).read_values('time')
# res = numpy.repeat(val,self.get_dimsize('cell'))
res = numpy.array([val,]*self.get_dimsize('cell')).transpose()
else:
res = super(RapidScatPODAACNCFile,self).read_values(fieldname)
# res = numpy.array([val,]*self.get_dimsize('cell')).transpose()
# else:
res = super(RapidScatPODAACNCFile,self).read_values(fieldname)
return res
......
This diff is collapsed.
......@@ -304,20 +304,36 @@ class SAFEMSIL1CStitchedFile(AbstractMapper):
else:
shp = slices.shape()
vals = np.zeros(shp, dtype=field.datatype)
count = np.zeros(shp, dtype='uint8')
for m, off, siz in zip(self._mappers, self._offsets, self._sizes):
outsub, insub = slices.outsub_insub_slices(off, siz)
if outsub is None:
continue
subvals = m.read_values(fieldname, slices=insub)
vals[outsub] += subvals
count[outsub] += np.uint8(1) - np.ma.getmask(subvals)
over = np.where(count > 1)
vals[over] /= count[over]
if count.min() == 0:
values = np.ma.array(vals, mask=count == 0)
if field.datatype != np.dtype(np.bool):
fillvalue = field.fillvalue
count = np.zeros(shp, dtype='uint8')
for m, off, siz in zip(self._mappers, self._offsets, self._sizes):
outsub, insub = slices.outsub_insub_slices(off, siz)
if outsub is None:
continue
subvals = m.read_values(fieldname, slices=insub)
if fillvalue is not None and fillvalue != 0 and \
subvals.mask is not np.ma.nomask:
subvals.data[subvals.mask] = 0
vals[outsub] += subvals
count[outsub] += np.uint8(1) - np.ma.getmask(subvals)
over = np.where(count > 1)
vals[over] /= count[over]
if count.min() == 0:
mask = count == 0
if fillvalue is not None and fillvalue != 0:
vals[mask] = fillvalue
values = np.ma.array(vals, mask=mask)
else:
values = np.ma.array(vals)
else:
values = np.ma.array(vals)
for m, off, siz in zip(self._mappers, self._offsets, self._sizes):
outsub, insub = slices.outsub_insub_slices(off, siz)
if outsub is None:
continue
subvals = m.read_values(fieldname, slices=insub)
vals[outsub] += subvals
values = vals
return values
def read_field(self, fieldname):
......
......@@ -732,18 +732,16 @@ class SAFESLFile(AbstractMapper):
+ (scan - first_scan_i.reshape((-1, 1)))\
* SCANSYNC + pixel * PIXSYNC_i
# mask wrong times (which occur in test data)
maxdate = date2num(datetime.datetime(2050, 1, 1),
maxdate = date2num(self.get_end_time(),
"microseconds since 2000-01-01T00:00:00Z")
mindate = date2num(datetime.datetime(2001, 1, 1),
mindate = date2num(self.get_start_time(),
"microseconds since 2000-01-01T00:00:00Z")
# time = ma.masked_where(
# ((time < mindate) | (time > maxdate)),
# time,
# copy=False
# )
time[(time < mindate) | (time > maxdate)] = maxdate
return ma.array(time, copy=False).filled(maxdate)
time = ma.masked_where(
((time < mindate) | (time > maxdate)),
time,
copy=False
)
return time
elif fieldname in ['lat', 'lon', 'z']:
return self.__geofieldlocator[native_name].read_values(native_name,
slices)
......
......@@ -216,8 +216,32 @@ def format_indices(indices, fielddims):
class Slices(tuple):
"""Class defining a Slices object.
A Slices object is a tuple of slice objects and therefore can be used
to slice a numpy array or a netCDF4 variable for instance. During
initialisation, input slice objects are checked and filled according to
the dimension sizes of the variable to slice. It also provides some methods
which aim at making the use of slices in cerbere easier.
Args:
slices (None, slice, list<slice>, tuple<slice>): input slice object(s).
dimsizes (None, int, list<int>, tuple<int>): dimension size(s) of the
variable to slice.
Example:
>>> values = numpy.arange(10)
>>> slices = Slices(slice(1, None, 4), values.shape)
>>> print slices
(slice(1, 10, 4),)
>>> print values[1::4]
[1 5 9]
>>> print values[slices]
[1 5 9]
"""
"""
def __new__(cls, slices=None, dimsizes=None):
"""
"""
......@@ -254,7 +278,7 @@ class Slices(tuple):
if (step > 0 and start >= stop) or (step < 0 and start <= stop):
raise Exception('Unexpected slice start Vs stop : {}'.format(_slice))
# Special case : negative step until the first element
# here : var[_slice] != var[slice(_slice.indices(dimsize))]
# here : var[_slice] != var[slice(*_slice.indices(dimsize))]
if stop == -1:
stop = None
new_slices.append(slice(start, stop, step))
......@@ -271,7 +295,21 @@ class Slices(tuple):
return Slices(add_slices, add_dimsizes)
def shape(self):
"""
"""Returns the expected shape of the sliced variable.
Returns:
tuple: numpy array like shape.
Example:
>>> values = numpy.zeros((10, 5))
>>> slices = Slices([slice(None, None, 3), slice(None, None, -2)], values.shape)
>>> print values[::3, ::-2].shape
(4, 3)
>>> print values[slices].shape
(4, 3)
>>> print slices.shape()
(4, 3)
"""
shp = []
for _slice in self:
......@@ -282,7 +320,40 @@ class Slices(tuple):
return tuple(shp)
def absolute_slices(self, view_slices):
"""
"""Returns absolute slices assuming this Slices object is relative to a view.
It can be seen as slices composition : slicing with the absolute
slices is equivalent to slice with the view slices then with the
relative to view slices (i.e. absolute = view o relative).
Args:
view_slices (Slices): the view slices as a Slices object.
Returns:
Slices: the absolute slices as a new Slices object.
Example:
If a view is defined in a cerbere mapper, the slices given by the user
to the mapper read_values() method are relative to this view. With this
method, we get the right slices to apply to original data.
>>> values = numpy.arange(100).reshape((2, 50))
>>> view_slices = Slices([slice(1, 2), slice(None, None, -4)], values.shape)
>>> print view_slices
(slice(1, 2, 1), slice(49, None, -4))
>>> rel_slices = Slices([slice(0, 1), slice(None, None, -2)], view_slices.shape())
>>> print rel_slices
(slice(0, 1, 1), slice(12, None, -2))
>>> abs_slices = rel_slices.absolute_slices(view_slices)
>>> print abs_slices
(slice(1, 2, 1), slice(1, 50, 8))
>>> print values[1:2, ::-4][0:1, ::-2]
[[51 59 67 75 83 91 99]]
>>> print values[view_slices][rel_slices]
[[51 59 67 75 83 91 99]]
>>> print values[abs_slices]
[[51 59 67 75 83 91 99]]
"""
assert isinstance(view_slices, Slices)
if len(self) != len(view_slices):
......@@ -311,7 +382,19 @@ class Slices(tuple):
return Slices(abs_slices, view_slices.dimsizes)
def indices_array(self, dtype=None):
"""
"""Returns arrays of indices corresponding to the slices.
Args:
dtype (dtype, optional): the type of the output indices arrays.
Returns:
list<ndarray>: indices array for each dimension.
Example:
>>> slices = Slices([slice(2, 12, 3), slice(9, 2, -2)], [15, 12])
>>> print slices.indices_array()
[array([ 2, 5, 8, 11]), array([9, 7, 5, 3])]
"""
ind_arr = []
for _slice in self:
......@@ -323,7 +406,16 @@ class Slices(tuple):
return ind_arr
def indices_minmax(self):
"""
"""Returns minimum and maximum of indices corresponding to the slices.
Returns:
list<list>: indices [min, max] for each dimension.
Example:
>>> slices = Slices([slice(2, 12, 3), slice(9, 2, -2)], [15, 12])
>>> print slices.indices_minmax()
[[2, 11], [3, 9]]
"""
ind_minmax = []
for _slice, size in zip(self, self.shape()):
......@@ -335,7 +427,16 @@ class Slices(tuple):
return ind_minmax
def indices_offsetextent(self):
"""
"""Returns offset and extent of indices corresponding to the slices.
Returns:
list<list>: indices [offset, extent] for each dimension.
Example:
>>> slices = Slices([slice(2, 12, 3), slice(9, 2, -2)], [15, 12])
>>> print slices.indices_offsetextent()
[[2, 10], [3, 7]]
"""
ind_offext = []
for minmax in self.indices_minmax():
......@@ -343,7 +444,61 @@ class Slices(tuple):
return ind_offext
def outsub_insub_slices(self, sub_offset, sub_size):
"""
"""Returns new Slices objects to deal with a sub area of a variable.
In 2D, for example, lets's assume an image is made up of subimages,
each one located with an offset and a size (for each dimension). And
let's assume also we want to build a sliced version of this image given
this Slices object and given the subimages. Then, this method returns
for a given subimage : an outsub Slices object defining where to fill
the sliced image we want to build and an insub Slices object defining
where to read in the subimage.
Args:
sub_offset (list<int>): offset of the sub area for each dimension.
sub_size (list<int>): size of the sub area for each dimension.
Returns:
list<Slices>, list<None>: [outsub, insub] as new Slice objects or
[None, None] if the sub area does not overlap with the slices
area.
Example:
With a 4x4 image made up of 4 2x2 subimages.
>>> # Subimages definition
>>> sub_offsets = [[0, 0], [0, 2], [2, 0], [2, 2]]
>>> sub_sizes = [[2, 2], [2, 2], [2, 2], [2, 2]]
>>> sub_images = [numpy.ones(sub_sizes[i]) * i for i in range(4)]
>>> # Image (that we don't want to build but shown for the example)
>>> _image = numpy.array([[0., 0., 1., 1.], [0., 0., 1., 1.],
... [2., 2., 3., 3.], [2., 2., 3., 3.]])
>>> print _image
[[ 0. 0. 1. 1.]
[ 0. 0. 1. 1.]
[ 2. 2. 3. 3.]
[ 2. 2. 3. 3.]]
>>> # Sliced image
>>> slices = Slices([slice(0, 2), slice(1, 3)], [4, 4])
>>> sliced_image = numpy.zeros(slices.shape())
>>> for i in range(4):
... outsub, insub = slices.outsub_insub_slices(sub_offsets[i], sub_sizes[i])
... if outsub is not None:
... sliced_image[outsub] = sub_images[i][insub]
... print outsub, insub
...
(slice(0, 2, 1), slice(0, 1, 1)) (slice(0, 2, 1), slice(1, 2, 1))
(slice(0, 2, 1), slice(1, 2, 1)) (slice(0, 2, 1), slice(0, 1, 1))
None None
None None
>>> print sliced_image
[[ 0. 1.]
[ 0. 1.]]
>>> print _image[slices]
[[ 0. 1.]
[ 0. 1.]]
"""
if len(self) != len(sub_offset) or len(self) != len(sub_size):
raise Exception('Mismatch between slices/ofssets/sizes lengths.')
......
......@@ -19,7 +19,7 @@ sudo apt-get install gcc \
# Numpy installation fails when it is done during "install_requires" processing
# of setup.py but it works fine when done this way.
pip install numpy==1.8
pip install numpy>=1.8
# the pyhdf project changed the version number convention they use for the
# python packages (X.Y-Z to X.Y.Z), which prevents pip from detecting the latest
......@@ -36,6 +36,7 @@ fi
export INCLUDE_DIRS=/usr/include/hdf
export LIBRARY_DIRS=/usr/lib
export NOSZIP=1
export CFLAGS=-D_FORTIFY_SOURCE=1
python setup.py install
# Now we can install cerbere
......@@ -11,15 +11,43 @@ Setup script for cerbere package. See package documentation for information.
import os
import sys
from distutils.core import setup
try:
import subprocess32 as subprocess
except ImportError:
import subprocess
from numpy import get_include
gdal_deps = 'gdal>=1.7.0'
try:
output = subprocess.check_output(['gdal-config', '--version'])
major, minor, _ = output.split('.')
gdal_deps = 'gdal>={}.{},<{}.{}'.format(major, minor, major, (int(minor) + 1))
except subprocess.CalledProcessError:
# Silenced
pass
with open("CONTRIBUTORS.txt", "r") as f:
authors_email = ', '.join(f.readlines())
# Update version when there is a new git commit
major_minor_version = '0.1'
package_dir = os.path.dirname(__file__)
version_path = os.path.join(package_dir, 'VERSION.txt')
if os.path.exists('.git') and os.path.isdir('.git'):
commits = subprocess.check_output([ '/usr/bin/git'
, 'rev-list'
, 'HEAD'
, '--count']).decode('utf-8').strip()
with open(version_path, 'w') as f:
f.write('{}.{}\n'.format(major_minor_version, commits))
with open(version_path, 'r') as f:
version = f.read()
setup(
name = "cerbere",
version = "0.1.1",
version = version,
author = "Felyx Project Collaborators: %s" % authors_email,
author_email = "jean.francois.piolle@ifremer.fr",
description = "",
......@@ -39,11 +67,11 @@ setup(
install_requires=[ 'numpy>=1.7.1'
, 'scipy>=0.12.0'
, 'netCDF4>=1.0.4'
, 'pyhdf==0.8.3'
, 'python-hdf4>=0.9'
, 'pygrib>=1.9.6'
, 'Shapely==1.2.18'
, 'python-dateutil>=2.1'
, 'GDAL>=1.7.0'
, gdal_deps
, 'pyresample>=1.1.0'
],
classifiers=[
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment