Commit 3ae7034c authored by W. Trevor King's avatar W. Trevor King
Browse files

Use an explicit .array attribute to determine if fields are arrays.

This improves on the old method of assuming they were scalar if
.item_count was 1.
parent b290d098
......@@ -63,6 +63,11 @@ complexUInt32 = _numpy.dtype(
class StaticStringField (_DynamicField):
_null_terminated = False
_array_size_field = None
def __init__(self, *args, **kwargs):
if 'array' not in kwargs:
kwargs['array'] = True
super(StaticStringField, self).__init__(*args, **kwargs)
def post_unpack(self, parents, data):
wave_structure = parents[-1]
wave_data = self._get_structure_data(parents, data, wave_structure)
......@@ -164,8 +169,8 @@ BinHeader5 = _Structure( # `version` field pulled out into Wave
_Field('l', 'formulaSize', help='The size of the dependency formula, if any.'),
_Field('l', 'noteSize', help='The size of the note text.'),
_Field('l', 'dataEUnitsSize', help='The size of optional extended data units.'),
_Field('l', 'dimEUnitsSize', help='The size of optional extended dimension units.', count=MAXDIMS),
_Field('l', 'dimLabelsSize', help='The size of optional dimension labels.', count=MAXDIMS),
_Field('l', 'dimEUnitsSize', help='The size of optional extended dimension units.', count=MAXDIMS, array=True),
_Field('l', 'dimLabelsSize', help='The size of optional dimension labels.', count=MAXDIMS, array=True),
_Field('l', 'sIndicesSize', help='The size of string indicies if this is a text wave.'),
_Field('l', 'optionsSize1', default=0, help='Reserved. Write zero. Ignore on read.'),
_Field('l', 'optionsSize2', default=0, help='Reserved. Write zero. Ignore on read.'),
......@@ -191,8 +196,8 @@ WaveHeader2 = _DynamicStructure(
_Field('h', 'whVersion', default=0, help='Write 0. Ignore on read.'),
_Field('h', 'srcFldr', default=0, help='Used in memory only. Write zero. Ignore on read.'),
_Field('P', 'fileName', default=0, help='Used in memory only. Write zero. Ignore on read.'),
_Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1),
_Field('c', 'xUnits', default=0, help='Natural x-axis units go here - null if none.', count=MAX_UNIT_CHARS+1),
_Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1, array=True),
_Field('c', 'xUnits', default=0, help='Natural x-axis units go here - null if none.', count=MAX_UNIT_CHARS+1, array=True),
_Field('l', 'npnts', help='Number of data points in wave.'),
_Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
_Field('d', 'hsA', help='X value for point p = hsA*p + hsB'),
......@@ -207,7 +212,7 @@ WaveHeader2 = _DynamicStructure(
_Field('P', 'formula', default=0, help='Used in memory only. Write zero. Ignore on read.'),
_Field('l', 'depID', default=0, help='Used in memory only. Write zero. Ignore on read.'),
_Field('L', 'creationDate', help='DateTime of creation. Not used in version 1 files.'),
_Field('c', 'wUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=2),
_Field('c', 'wUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=2, array=True),
_Field('L', 'modDate', help='DateTime of last modification.'),
_Field('P', 'waveNoteH', help='Used in memory only. Write zero. Ignore on read.'),
])
......@@ -224,27 +229,27 @@ WaveHeader5 = _DynamicStructure(
_Field('l', 'npnts', help='Total number of points (multiply dimensions up to first zero).'),
_Field('h', 'type', help='See types (e.g. NT_FP64) above. Zero for text waves.'),
_Field('h', 'dLock', default=0, help='Reserved. Write zero. Ignore on read.'),
_Field('c', 'whpad1', default=0, help='Reserved. Write zero. Ignore on read.', count=6),
_Field('c', 'whpad1', default=0, help='Reserved. Write zero. Ignore on read.', count=6, array=True),
_Field('h', 'whVersion', default=1, help='Write 1. Ignore on read.'),
NullStaticStringField('c', 'bname', help='Name of wave plus trailing null.', count=MAX_WAVE_NAME5+1),
_Field('l', 'whpad2', default=0, help='Reserved. Write zero. Ignore on read.'),
_Field('P', 'dFolder', default=0, help='Used in memory only. Write zero. Ignore on read.'),
# Dimensioning info. [0] == rows, [1] == cols etc
_Field('l', 'nDim', help='Number of of items in a dimension -- 0 means no data.', count=MAXDIMS),
_Field('d', 'sfA', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS),
_Field('d', 'sfB', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS),
_Field('l', 'nDim', help='Number of of items in a dimension -- 0 means no data.', count=MAXDIMS, array=True),
_Field('d', 'sfA', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS, array=True),
_Field('d', 'sfB', help='Index value for element e of dimension d = sfA[d]*e + sfB[d].', count=MAXDIMS, array=True),
# SI units
_Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1),
_Field('c', 'dimUnits', default=0, help='Natural dimension units go here - null if none.', count=(MAXDIMS, MAX_UNIT_CHARS+1)),
_Field('c', 'dataUnits', default=0, help='Natural data units go here - null if none.', count=MAX_UNIT_CHARS+1, array=True),
_Field('c', 'dimUnits', default=0, help='Natural dimension units go here - null if none.', count=(MAXDIMS, MAX_UNIT_CHARS+1), array=True),
_Field('h', 'fsValid', help='TRUE if full scale values have meaning.'),
_Field('h', 'whpad3', default=0, help='Reserved. Write zero. Ignore on read.'),
_Field('d', 'topFullScale', help='The max and max full scale value for wave'), # sic, probably "max and min"
_Field('d', 'botFullScale', help='The max and max full scale value for wave.'), # sic, probably "max and min"
_Field('P', 'dataEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.'),
_Field('P', 'dimEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS),
_Field('P', 'dimLabels', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS),
_Field('P', 'dimEUnits', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS, array=True),
_Field('P', 'dimLabels', default=0, help='Used in memory only. Write zero. Ignore on read.', count=MAXDIMS, array=True),
_Field('P', 'waveNoteH', default=0, help='Used in memory only. Write zero. Ignore on read.'),
_Field('l', 'whUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=16),
_Field('l', 'whUnused', default=0, help='Reserved. Write zero. Ignore on read.', count=16, array=True),
# The following stuff is considered private to Igor.
_Field('h', 'aModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
_Field('h', 'wModified', default=0, help='Used in memory only. Write zero. Ignore on read.'),
......@@ -577,7 +582,7 @@ Wave1 = _DynamicStructure(
fields=[
_Field(BinHeader1, 'bin_header', help='Binary wave header'),
_Field(WaveHeader2, 'wave_header', help='Wave header'),
DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0),
DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0, array=True),
])
Wave2 = _DynamicStructure(
......@@ -585,9 +590,9 @@ Wave2 = _DynamicStructure(
fields=[
_Field(BinHeader2, 'bin_header', help='Binary wave header'),
_Field(WaveHeader2, 'wave_header', help='Wave header'),
DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0),
_Field('x', 'padding', help='16 bytes of padding in versions 2 and 3.', count=16),
DynamicWaveNoteField('c', 'note', help='Optional wave note data', count=0),
DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0, array=True),
_Field('x', 'padding', help='16 bytes of padding in versions 2 and 3.', count=16, array=True),
DynamicWaveNoteField('c', 'note', help='Optional wave note data', count=0, array=True),
])
Wave3 = _DynamicStructure(
......@@ -595,10 +600,10 @@ Wave3 = _DynamicStructure(
fields=[
_Field(BinHeader3, 'bin_header', help='Binary wave header'),
_Field(WaveHeader2, 'wave_header', help='Wave header'),
DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0),
_Field('x', 'padding', help='16 bytes of padding in versions 2 and 3.', count=16),
DynamicWaveNoteField('c', 'note', help='Optional wave note data', count=0),
DynamicDependencyFormulaField('c', 'formula', help='Optional wave dependency formula', count=0),
DynamicWaveDataField1('f', 'wData', help='The start of the array of waveform data.', count=0, array=True),
_Field('x', 'padding', help='16 bytes of padding in versions 2 and 3.', count=16, array=True),
DynamicWaveNoteField('c', 'note', help='Optional wave note data', count=0, array=True),
DynamicDependencyFormulaField('c', 'formula', help='Optional wave dependency formula', count=0, array=True),
])
Wave5 = _DynamicStructure(
......@@ -606,13 +611,13 @@ Wave5 = _DynamicStructure(
fields=[
_Field(BinHeader5, 'bin_header', help='Binary wave header'),
_Field(WaveHeader5, 'wave_header', help='Wave header'),
DynamicWaveDataField5('f', 'wData', help='The start of the array of waveform data.', count=0),
DynamicDependencyFormulaField('c', 'formula', help='Optional wave dependency formula.', count=0),
DynamicWaveNoteField('c', 'note', help='Optional wave note data.', count=0),
DynamicDataUnitsField('c', 'data_units', help='Optional extended data units data.', count=0),
DynamicDimensionUnitsField('c', 'dimension_units', help='Optional dimension label data', count=0),
DynamicLabelsField('c', 'labels', help="Optional dimension label data", count=0),
DynamicStringIndicesDataField('P', 'sIndices', help='Dynamic string indices for text waves.', count=0),
DynamicWaveDataField5('f', 'wData', help='The start of the array of waveform data.', count=0, array=True),
DynamicDependencyFormulaField('c', 'formula', help='Optional wave dependency formula.', count=0, array=True),
DynamicWaveNoteField('c', 'note', help='Optional wave note data.', count=0, array=True),
DynamicDataUnitsField('c', 'data_units', help='Optional extended data units data.', count=0, array=True),
DynamicDimensionUnitsField('c', 'dimension_units', help='Optional dimension label data', count=0, array=True),
DynamicLabelsField('c', 'labels', help="Optional dimension label data", count=0, array=True),
DynamicStringIndicesDataField('P', 'sIndices', help='Dynamic string indices for text waves.', count=0, array=True),
])
Wave = _DynamicStructure(
......
......@@ -75,6 +75,11 @@ class ListedDynamicStrDataField (_DynamicStringField, ListedStaticStringField):
class DynamicVarDataField (_DynamicField):
def __init__(self, *args, **kwargs):
if 'array' not in kwargs:
kwargs['array'] = True
super(DynamicVarDataField, self).__init__(*args, **kwargs)
def pre_pack(self, parents, data):
raise NotImplementedError()
......@@ -247,8 +252,8 @@ Variables2 = _DynamicStructure(
DynamicSysVarField('f', 'sysVars', help='System variables', count=0),
DynamicUserVarField(UserNumVarRec, 'userVars', help='User numeric variables', count=0),
DynamicUserStrField(UserStrVarRec2, 'userStrs', help='User string variables', count=0),
_Field(UserDependentVarRec, 'dependentVars', help='Dependent numeric variables.', count=0),
_Field(UserDependentVarRec, 'dependentStrs', help='Dependent string variables.', count=0),
_Field(UserDependentVarRec, 'dependentVars', help='Dependent numeric variables.', count=0, array=True),
_Field(UserDependentVarRec, 'dependentStrs', help='Dependent string variables.', count=0, array=True),
])
......
......@@ -65,7 +65,7 @@ class Field (object):
Example of a multi-dimensional float field:
>>> data = Field(
... 'f', 'data', help='example data', count=(2,3,4))
... 'f', 'data', help='example data', count=(2,3,4), array=True)
>>> data.arg_count
24
>>> list(data.indexes()) # doctest: +ELLIPSIS
......@@ -91,7 +91,7 @@ class Field (object):
Example of a nested structure field:
>>> run = Structure('run', fields=[time, data])
>>> runs = Field(run, 'runs', help='pair of runs', count=2)
>>> runs = Field(run, 'runs', help='pair of runs', count=2, array=True)
>>> runs.arg_count # = 2 * (1 + 24)
50
>>> data1 = numpy.arange(data.arg_count).reshape(data.count)
......@@ -148,12 +148,14 @@ class Field (object):
--------
Structure
"""
def __init__(self, format, name, default=None, help=None, count=1):
def __init__(self, format, name, default=None, help=None, count=1,
array=False):
self.format = format
self.name = name
self.default = default
self.help = help
self.count = count
self.array = array
self.setup()
def setup(self):
......@@ -164,6 +166,10 @@ class Field (object):
"""
_LOG.debug('setup {}'.format(self))
self.item_count = _numpy.prod(self.count) # number of item repeats
if not self.array and self.item_count != 1:
raise ValueError(
'{} must be an array field to have a count of {}'.format(
self, self.count))
if isinstance(self.format, Structure):
self.structure_count = sum(
f.arg_count for f in self.format.fields)
......@@ -182,7 +188,7 @@ class Field (object):
def indexes(self):
"""Iterate through indexes to a possibly multi-dimensional array"""
assert self.item_count > 1, self
assert self.array, self
try:
i = [0] * len(self.count)
except TypeError: # non-iterable count
......@@ -202,7 +208,7 @@ class Field (object):
If the field is repeated (count > 1), the incoming data should
be iterable with each iteration returning a single item.
"""
if self.item_count > 1:
if self.array:
if data is None:
data = []
if hasattr(data, 'flat'): # take advantage of numpy's ndarray.flat
......@@ -230,7 +236,7 @@ class Field (object):
item = None
for arg in self.pack_item(item):
yield arg
elif self.item_count:
else:
for arg in self.pack_item(data):
yield arg
......@@ -272,7 +278,8 @@ class Field (object):
count = self.count
else:
count = 0 # padding bytes, etc.
if count == 1:
if not self.array:
assert count == 1, (self, self.count)
return unpacked[0]
if isinstance(self.format, Structure):
try:
......@@ -377,11 +384,12 @@ class Structure (_struct.Struct):
>>> time = Field('I', 'time', default=0, help='POSIX time')
>>> data = Field(
... 'h', 'data', default=0, help='example data', count=(2,3))
... 'h', 'data', default=0, help='example data', count=(2,3),
... array=True)
>>> run = Structure('run', fields=[time, data])
>>> version = Field(
... 'H', 'version', default=1, help='example version')
>>> runs = Field(run, 'runs', help='pair of runs', count=2)
>>> runs = Field(run, 'runs', help='pair of runs', count=2, array=True)
>>> experiment = Structure('experiment', fields=[version, runs])
The structures automatically calculate the flattened data format:
......@@ -453,7 +461,8 @@ class Structure (_struct.Struct):
If you set ``count=0``, the field is ignored.
>>> experiment2 = Structure('experiment', fields=[
... version, Field('f', 'ignored', count=0), runs], byte_order='>')
... version, Field('f', 'ignored', count=0, array=True), runs],
... byte_order='>')
>>> experiment2.format
'>HIhhhhhhIhhhhhh'
>>> d = experiment2.unpack(b)
......@@ -656,7 +665,7 @@ class DynamicStructure (Structure):
>>> dynamic_length_vector = DynamicStructure('vector',
... fields=[
... DynamicLengthField('I', 'length'),
... Field('h', 'data', count=0),
... Field('h', 'data', count=0, array=True),
... ],
... byte_order='>')
>>> class DynamicDataField (DynamicField):
......@@ -667,7 +676,7 @@ class DynamicStructure (Structure):
>>> dynamic_data_vector = DynamicStructure('vector',
... fields=[
... Field('I', 'length'),
... DynamicDataField('h', 'data', count=0),
... DynamicDataField('h', 'data', count=0, array=True),
... ],
... byte_order='>')
......@@ -746,18 +755,18 @@ class DynamicStructure (Structure):
f.setup()
f.format.setup()
if isinstance(f.format, DynamicStructure):
if f.item_count == 1:
# TODO, fix in case we *want* an array
d[f.name] = {}
f.format.unpack_stream(
stream, parents=parents, data=data, d=d[f.name])
else:
if f.array:
d[f.name] = []
for i in range(f.item_count):
x = {}
d[f.name].append(x)
f.format.unpack_stream(
stream, parents=parents, data=data, d=x)
else:
assert f.item_count == 1, (f, f.count)
d[f.name] = {}
f.format.unpack_stream(
stream, parents=parents, data=data, d=d[f.name])
if hasattr(f, 'post_unpack'):
_LOG.debug('post-unpack {}'.format(f))
repeat = f.post_unpack(parents=parents, data=data)
......@@ -774,7 +783,8 @@ class DynamicStructure (Structure):
f.setup()
f.format.setup()
x = [f.format.unpack_from(b) for b in bs]
if len(x) == 1: # TODO, fix in case we *want* an array
if not f.array:
assert len(x) == 1, (f, f.count, x)
x = x[0]
return x
else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment