struct.py 27.5 KB
Newer Older
1
2
# Copyright

3
4
5
6
7
8
9
"""Structure and Field classes for declaring structures

There are a few formats that can be used to represent the same data, a
binary packed format with all the data in a buffer, a linearized
format with each field in a single Python list, and a nested format
with each field in a hierarchy of Python dictionaries.
"""
10
11

from __future__ import absolute_import
12
13
14
import io as _io
import logging as _logging
import pprint as _pprint
15
16
17
18
import struct as _struct

import numpy as _numpy

19
20
from . import LOG as _LOG

21
22
23
24
25
26
27

_buffer = buffer  # save builtin buffer for clobbered situations


class Field (object):
    """Represent a Structure field.

28
29
30
31
32
33
34
35
36
37
38
39
40
41
    The format argument can be a format character from the ``struct``
    documentation (e.g., ``c`` for ``char``, ``h`` for ``short``, ...)
    or ``Structure`` instance (for building nested structures).

    Examples
    --------

    >>> from pprint import pprint
    >>> import numpy

    Example of an unsigned short integer field:

    >>> time = Field(
    ...     'I', 'time', default=0, help='POSIX time')
42
    >>> time.arg_count
43
44
45
46
47
48
49
50
51
52
53
54
55
56
    1
    >>> list(time.pack_data(1))
    [1]
    >>> list(time.pack_item(2))
    [2]
    >>> time.unpack_data([3])
    3
    >>> time.unpack_item([4])
    4

    Example of a multi-dimensional float field:

    >>> data = Field(
    ...     'f', 'data', help='example data', count=(2,3,4))
57
    >>> data.arg_count
58
59
60
61
62
63
64
65
66
67
    24
    >>> list(data.indexes())  # doctest: +ELLIPSIS
    [[0, 0, 0], [0, 0, 1], [0, 0, 2], [0, 0, 3], [0, 1, 0], ..., [1, 2, 3]]
    >>> list(data.pack_data(
    ...     [[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]],
    ...      [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]]])
    ...     )  # doctest: +ELLIPSIS
    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ..., 19, 20, 21, 22, 23]
    >>> list(data.pack_item(3))
    [3]
68
    >>> data.unpack_data(range(data.arg_count))
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    array([[[ 0,  1,  2,  3],
            [ 4,  5,  6,  7],
            [ 8,  9, 10, 11]],
    <BLANKLINE>
           [[12, 13, 14, 15],
            [16, 17, 18, 19],
            [20, 21, 22, 23]]])
    >>> data.unpack_item([3])
    3

    Example of a nested structure field:

    >>> run = Structure('run', fields=[time, data])
    >>> runs = Field(run, 'runs', help='pair of runs', count=2)
83
    >>> runs.arg_count  # = 2 * (1 + 24)
84
    50
85
86
    >>> data1 = numpy.arange(data.arg_count).reshape(data.count)
    >>> data2 = data1 + data.arg_count
87
88
89
90
91
92
93
94
    >>> list(runs.pack_data(
    ...     [{'time': 100, 'data': data1},
    ...      {'time': 101, 'data': data2}])
    ...     )  # doctest: +ELLIPSIS
    [100, 0, 1, 2, ..., 22, 23, 101, 24, 25, ..., 46, 47]
    >>> list(runs.pack_item({'time': 100, 'data': data1})
    ...     )  # doctest: +ELLIPSIS
    [100, 0, 1, 2, ..., 22, 23]
95
    >>> pprint(runs.unpack_data(range(runs.arg_count)))
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    [{'data': array([[[ 1,  2,  3,  4],
            [ 5,  6,  7,  8],
            [ 9, 10, 11, 12]],
    <BLANKLINE>
           [[13, 14, 15, 16],
            [17, 18, 19, 20],
            [21, 22, 23, 24]]]),
      'time': 0},
     {'data': array([[[26, 27, 28, 29],
            [30, 31, 32, 33],
            [34, 35, 36, 37]],
    <BLANKLINE>
           [[38, 39, 40, 41],
            [42, 43, 44, 45],
            [46, 47, 48, 49]]]),
      'time': 25}]
    >>> pprint(runs.unpack_item(range(runs.structure_count)))
    {'data': array([[[ 1,  2,  3,  4],
            [ 5,  6,  7,  8],
            [ 9, 10, 11, 12]],
    <BLANKLINE>
           [[13, 14, 15, 16],
            [17, 18, 19, 20],
            [21, 22, 23, 24]]]),
     'time': 0}

    If you don't give enough values for an array field, the remaining
    values are filled in with their defaults.

    >>> list(data.pack_data(
    ...     [[[0, 1, 2, 3], [4, 5, 6]], [[10]]]))  # doctest: +ELLIPSIS
    Traceback (most recent call last):
      ...
    ValueError: no default for <Field data ...>
    >>> data.default = 0
    >>> list(data.pack_data(
    ...     [[[0, 1, 2, 3], [4, 5, 6]], [[10]]]))
    [0, 1, 2, 3, 4, 5, 6, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

135
136
137
138
139
    See Also
    --------
    Structure
    """
    def __init__(self, format, name, default=None, help=None, count=1):
140
        self.format = format
141
        self.name = name
142
        self.default = default
143
144
        self.help = help
        self.count = count
145
146
147
148
149
150
151
152
153
154
        self.setup()

    def setup(self):
        """Setup any dynamic properties of a field.

        Use this method to recalculate dynamic properities after
        changing the basic properties set during initialization.
        """
        _LOG.debug('setup {}'.format(self))
        self.item_count = _numpy.prod(self.count)  # number of item repeats
155
        if isinstance(self.format, Structure):
156
157
158
159
160
            self.structure_count = sum(
                f.arg_count for f in self.format.fields)
            self.arg_count = self.item_count * self.structure_count
        elif self.format == 'x':
            self.arg_count = 0  # no data in padding bytes
161
        else:
162
            self.arg_count = self.item_count  # struct.Struct format args
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return '<{} {} {}>'.format(
            self.__class__.__name__, self.name, id(self))

    def indexes(self):
        """Iterate through indexes to a possibly multi-dimensional array"""
        assert self.item_count > 1, self
        try:
            i = [0] * len(self.count)
        except TypeError:  # non-iterable count
            for i in range(self.count):
                yield i
        else:
            for i in range(self.item_count):
                index = []
                for j,c in enumerate(reversed(self.count)):
                    index.insert(0, i % c)
                    i /= c
                yield index

    def pack_data(self, data=None):
        """Linearize a single field's data to a flat list.

        If the field is repeated (count > 1), the incoming data should
        be iterable with each iteration returning a single item.
        """
        if self.item_count > 1:
            if data is None:
                data = []
            if hasattr(data, 'flat'):  # take advantage of numpy's ndarray.flat
                items = 0
                for item in data.flat:
                    items += 1
                    for arg in self.pack_item(item):
                        yield arg
                if items < self.item_count:
                    if f.default is None:
                        raise ValueError(
                            'no default for {}.{}'.format(self, f))
                    for i in range(self.item_count - items):
                        yield f.default
            else:
                for index in self.indexes():
                    try:
                        if isinstance(index, int):
                            item = data[index]
                        else:
                            item = data
                            for i in index:
                                item = item[i]
                    except IndexError:
                        item = None
                    for arg in self.pack_item(item):
                        yield arg
221
        elif self.item_count:
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
            for arg in self.pack_item(data):
                yield arg

    def pack_item(self, item=None):
        """Linearize a single count of the field's data to a flat iterable
        """
        if isinstance(self.format, Structure):
            for i in self.format._pack_item(item):
                yield i
        elif item is None:
            if self.default is None:
                raise ValueError('no default for {}'.format(self))
            yield self.default
        else:
            yield item

    def unpack_data(self, data):
        """Inverse of .pack_data"""
240
        _LOG.debug('unpack {} for {} {}'.format(data, self, self.format))
241
242
        iterator = iter(data)
        try:
243
            items = [iterator.next() for i in range(self.arg_count)]
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
        except StopIteration:
            raise ValueError('not enough data to unpack {}'.format(self))
        try:
            iterator.next()
        except StopIteration:
            pass
        else:
            raise ValueError('too much data to unpack {}'.format(self))
        if isinstance(self.format, Structure):
            # break into per-structure clumps
            s = self.structure_count
            items = zip(*[items[i::s] for i in range(s)])
        else:
            items = [[i] for i in items]
        unpacked = [self.unpack_item(i) for i in items]
259
260
261
262
263
        if self.arg_count:
            count = self.count
        else:
            count = 0  # padding bytes, etc.
        if count == 1:
264
265
266
267
268
269
270
271
272
273
            return unpacked[0]
        if isinstance(self.format, Structure):
            try:
                len(self.count)
            except TypeError:
                pass
            else:
                raise NotImplementedError('reshape Structure field')
        else:
            unpacked = _numpy.array(unpacked)
274
275
276
            _LOG.debug('reshape {} data from {} to {}'.format(
                    self, unpacked.shape, count))
            unpacked = unpacked.reshape(count)
277
278
        return unpacked

279
    def unpack_item(self, item):
280
281
282
283
284
285
        """Inverse of .unpack_item"""
        if isinstance(self.format, Structure):
            return self.format._unpack_item(item)
        else:
            assert len(item) == 1, item
            return item[0]
286
287


288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
class DynamicField (Field):
    """Represent a DynamicStructure field with a dynamic definition.

    Adds the methods ``.pre_pack``, ``pre_unpack``, and
    ``post_unpack``, all of which are called when a ``DynamicField``
    is used by a ``DynamicStructure``.  Each method takes the
    arguments ``(parents, data)``, where ``parents`` is a list of
    ``DynamicStructure``\s that own the field and ``data`` is a dict
    hierarchy of the structure data.

    See the ``DynamicStructure`` docstring for the exact timing of the
    method calls.

    See Also
    --------
    Field, DynamicStructure
    """
    def pre_pack(self, parents, data):
        "Prepare to pack."
        pass

    def pre_unpack(self, parents, data):
        "React to previously unpacked data"
        pass

    def post_unpack(self, parents, data):
        "React to our own data"
        pass

    def _get_structure_data(self, parents, data, structure):
        """Extract the data belonging to a particular ancestor structure.
        """
        d = data
        s = parents[0]
        if s == structure:
            return d
        for p in parents[1:]:
            for f in s.fields:
                if f.format == p:
                    s = p
                    d = d[f.name]
                    break
            assert s == p, (s, p)
            if p == structure:
                break
        return d


336
class Structure (_struct.Struct):
337
    r"""Represent a C structure.
338
339
340
341
342
343
344
345
346
347
348

    A convenient wrapper around struct.Struct that uses Fields and
    adds dict-handling methods for transparent name assignment.

    See Also
    --------
    Field

    Examples
    --------

349
350
351
352
353
354
355
356
357
    >>> import array
    >>> from pprint import pprint

    Represent the C structures::

        struct run {
          unsigned int time;
          short data[2][3];
        }
358

359
360
361
        struct experiment {
          unsigned short version;
          struct run runs[2];
362
363
        }

364
    As:
365

366
367
368
369
370
371
372
373
374
375
376
377
    >>> time = Field('I', 'time', default=0, help='POSIX time')
    >>> data = Field(
    ...     'h', 'data', default=0, help='example data', count=(2,3))
    >>> run = Structure('run', fields=[time, data])
    >>> version = Field(
    ...     'H', 'version', default=1, help='example version')
    >>> runs = Field(run, 'runs', help='pair of runs', count=2)
    >>> experiment = Structure('experiment', fields=[version, runs])

    The structures automatically calculate the flattened data format:

    >>> run.format
378
    '@Ihhhhhh'
379
380
381
    >>> run.size  # 4 + 2*3*2
    16
    >>> experiment.format
382
383
384
385
386
387
388
389
390
391
392
393
394
395
    '@HIhhhhhhIhhhhhh'
    >>> experiment.size  # 2 + 2 + 2*(4 + 2*3*2)
    36

    The first two elements in the above size calculation are 2 (for
    the unsigned short, 'H') and 2 (padding so the unsigned int aligns
    with a 4-byte block).  If you select a byte ordering that doesn't
    mess with alignment and recalculate the format, the padding goes
    away and you get:

    >>> experiment.set_byte_order('>')
    >>> experiment.get_format()
    '>HIhhhhhhIhhhhhh'
    >>> experiment.size
396
397
398
399
400
401
402
    34

    You can read data out of any object supporting the buffer
    interface:

    >>> b = array.array('B', range(experiment.size))
    >>> d = experiment.unpack_from(buffer=b)
403
    >>> pprint(d)
404
405
406
407
408
409
410
411
412
413
414
415
416
    {'runs': [{'data': array([[1543, 2057, 2571],
           [3085, 3599, 4113]]),
               'time': 33752069},
              {'data': array([[5655, 6169, 6683],
           [7197, 7711, 8225]]),
               'time': 303240213}],
     'version': 1}
    >>> [hex(x) for x in d['runs'][0]['data'].flat]
    ['0x607L', '0x809L', '0xa0bL', '0xc0dL', '0xe0fL', '0x1011L']

    You can also read out from strings:

    >>> d = experiment.unpack(b.tostring())
417
    >>> pprint(d)
418
419
420
421
422
423
    {'runs': [{'data': array([[1543, 2057, 2571],
           [3085, 3599, 4113]]),
               'time': 33752069},
              {'data': array([[5655, 6169, 6683],
           [7197, 7711, 8225]]),
               'time': 303240213}],
424
     'version': 1}
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439

    If you don't give enough values for an array field, the remaining
    values are filled in with their defaults.

    >>> experiment.pack_into(buffer=b, data=d)
    >>> b.tostring()[:17]
    '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10'
    >>> b.tostring()[17:]
    '\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !'
    >>> run0 = d['runs'].pop(0)
    >>> b = experiment.pack(data=d)
    >>> b[:17]
    '\x00\x01\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f '
    >>> b[17:]
    '!\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459

    If you set ``count=0``, the field is ignored.

    >>> experiment2 = Structure('experiment', fields=[
    ...     version, Field('f', 'ignored', count=0), runs], byte_order='>')
    >>> experiment2.format
    '>HIhhhhhhIhhhhhh'
    >>> d = experiment2.unpack(b)
    >>> pprint(d)
    {'ignored': array([], dtype=float64),
     'runs': [{'data': array([[5655, 6169, 6683],
           [7197, 7711, 8225]]),
               'time': 303240213},
              {'data': array([[0, 0, 0],
           [0, 0, 0]]), 'time': 0}],
     'version': 1}
    >>> del d['ignored']
    >>> b2 = experiment2.pack(d)
    >>> b2 == b
    True
460
    """
461
462
463
    _byte_order_symbols = '@=<>!'

    def __init__(self, name, fields, byte_order='@'):
464
465
466
467
        # '=' for native byte order, standard size and alignment
        # See http://docs.python.org/library/struct for details
        self.name = name
        self.fields = fields
468
469
        self.byte_order = byte_order
        self.setup()
470
471
472
473

    def __str__(self):
        return self.name

474
475
476
477
    def __repr__(self):
        return '<{} {} {}>'.format(
            self.__class__.__name__, self.name, id(self))

478
479
480
481
482
483
484
485
486
487
    def setup(self):
        """Setup any dynamic properties of a structure.

        Use this method to recalculate dynamic properities after
        changing the basic properties set during initialization.
        """
        _LOG.debug('setup {!r}'.format(self))
        self.set_byte_order(self.byte_order)
        self.get_format()

488
489
490
    def set_byte_order(self, byte_order):
        """Allow changing the format byte_order on the fly.
        """
491
492
        _LOG.debug('set byte order for {!r} to {}'.format(self, byte_order))
        self.byte_order = byte_order
493
        for field in self.fields:
494
            if isinstance(field.format, Structure):
495
496
497
498
499
500
501
502
503
504
505
506
                field.format.set_byte_order(byte_order)

    def get_format(self):
        format = self.byte_order + ''.join(self.sub_format())
        # P format only allowed for native byte ordering
        # Convert P to I for ILP32 compatibility when running on a LP64.
        format = format.replace('P', 'I')
        try:
            super(Structure, self).__init__(format=format)
        except _struct.error as e:
            raise ValueError((e, format))
        return format
507

508
    def sub_format(self):
509
510
511
512
513
514
515
516
517
        _LOG.debug('calculate sub-format for {!r}'.format(self))
        for field in self.fields:
            if isinstance(field.format, Structure):
                field_format = list(
                    field.format.sub_format()) * field.item_count
            else:
                field_format = [field.format]*field.item_count
            for fmt in field_format:
                yield fmt
518

519
520
521
522
523
    def _pack_item(self, item=None):
        """Linearize a single count of the structure's data to a flat iterable
        """
        if item is None:
            item = {}
524
        for f in self.fields:
525
526
            try:
                data = item[f.name]
527
528
            except TypeError:
                raise ValueError((f.name, item))
529
530
531
532
            except KeyError:
                data = None
            for arg in f.pack_data(data):
                yield arg
533

534
535
536
537
    def _unpack_item(self, args):
        """Inverse of ._unpack_item"""
        data = {}
        iterator = iter(args)
538
        for f in self.fields:
539
            try:
540
                items = [iterator.next() for i in range(f.arg_count)]
541
542
543
544
            except StopIteration:
                raise ValueError('not enough data to unpack {}.{}'.format(
                        self, f))
            data[f.name] = f.unpack_data(items)
545
        try:
546
547
548
549
550
551
552
553
554
            iterator.next()
        except StopIteration:
            pass
        else:
            raise ValueError('too much data to unpack {}'.format(self))
        return data

    def pack(self, data):
        args = list(self._pack_item(data))
555
556
557
558
        try:
            return super(Structure, self).pack(*args)
        except:
            raise ValueError(self.format)
559
560
561
562
563
564
565
566
567
568
569

    def pack_into(self, buffer, offset=0, data={}):
        args = list(self._pack_item(data))
        return super(Structure, self).pack_into(
            buffer, offset, *args)

    def unpack(self, *args, **kwargs):
        args = super(Structure, self).unpack(*args, **kwargs)
        return self._unpack_item(args)

    def unpack_from(self, buffer, offset=0, *args, **kwargs):
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
        _LOG.debug(
            'unpack {!r} for {!r} ({}, offset={}) with {} ({})'.format(
                buffer, self, len(buffer), offset, self.format, self.size))
        args = super(Structure, self).unpack_from(
            buffer, offset, *args, **kwargs)
        return self._unpack_item(args)

    def get_field(self, name):
        return [f for f in self.fields if f.name == name][0]


class DebuggingStream (object):
    def __init__(self, stream):
        self.stream = stream

    def read(self, size):
        data = self.stream.read(size)
        _LOG.debug('read {} from {}: ({}) {!r}'.format(
                size, self.stream, len(data), data))
        return data


class DynamicStructure (Structure):
    r"""Represent a C structure field with a dynamic definition.

    Any dynamic fields have their ``.pre_pack`` called before any
    structure packing is done.  ``.pre_unpack`` is called for a
    particular field just before that field's ``.unpack_data`` call.
    ``.post_unpack`` is called for a particular field just after
    ``.unpack_data``.  If ``.post_unpack`` returns ``True``, the same
    field is unpacked again.

    Examples
    --------

    >>> from pprint import pprint

    This allows you to define structures where some portion of the
    global structure depends on earlier data.  For example, in the
    quasi-C structure::

        struct vector {
          unsigned int length;
          short data[length];
        }

    You can generate a Python version of this structure in two ways,
    with a dynamic ``length``, or with a dynamic ``data``.  In both
    cases, the required methods are the same, the only difference is
    where you attach them.

    >>> def packer(self, parents, data):
    ...     vector_structure = parents[-1]
    ...     vector_data = self._get_structure_data(
    ...         parents, data, vector_structure)
    ...     length = len(vector_data['data'])
    ...     vector_data['length'] = length
    ...     data_field = vector_structure.get_field('data')
    ...     data_field.count = length
    ...     data_field.setup()
    >>> def unpacker(self, parents, data):
    ...     vector_structure = parents[-1]
    ...     vector_data = self._get_structure_data(
    ...         parents, data, vector_structure)
    ...     length = vector_data['length']
    ...     data_field = vector_structure.get_field('data')
    ...     data_field.count = length
    ...     data_field.setup()

    >>> class DynamicLengthField (DynamicField):
    ...     def pre_pack(self, parents, data):
    ...         packer(self, parents, data)
    ...     def post_unpack(self, parents, data):
    ...         unpacker(self, parents, data)
    >>> dynamic_length_vector = DynamicStructure('vector',
    ...     fields=[
    ...         DynamicLengthField('I', 'length'),
    ...         Field('h', 'data', count=0),
    ...         ],
    ...     byte_order='>')
    >>> class DynamicDataField (DynamicField):
    ...     def pre_pack(self, parents, data):
    ...         packer(self, parents, data)
    ...     def pre_unpack(self, parents, data):
    ...         unpacker(self, parents, data)
    >>> dynamic_data_vector = DynamicStructure('vector',
    ...     fields=[
    ...         Field('I', 'length'),
    ...         DynamicDataField('h', 'data', count=0),
    ...         ],
    ...     byte_order='>')

    >>> b = '\x00\x00\x00\x02\x01\x02\x03\x04'
    >>> d = dynamic_length_vector.unpack(b)
    >>> pprint(d)
    {'data': array([258, 772]), 'length': 2}
    >>> d = dynamic_data_vector.unpack(b)
    >>> pprint(d)
    {'data': array([258, 772]), 'length': 2}

    >>> d['data'] = [1,2,3,4]
    >>> dynamic_length_vector.pack(d)
    '\x00\x00\x00\x04\x00\x01\x00\x02\x00\x03\x00\x04'
    >>> dynamic_data_vector.pack(d)
    '\x00\x00\x00\x04\x00\x01\x00\x02\x00\x03\x00\x04'

    The implementation is a good deal more complicated than the one
    for ``Structure``, because we must make multiple calls to
    ``struct.Struct.unpack`` to unpack the data.
    """
    #def __init__(self, *args, **kwargs):
    #     pass #self.parent = ..

    def _pre_pack(self, parents=None, data=None):
        if parents is None:
            parents = [self]
        else:
            parents = parents + [self]
        for f in self.fields:
            if hasattr(f, 'pre_pack'):
                _LOG.debug('pre-pack {}'.format(f))
                f.pre_pack(parents=parents, data=data)
            if isinstance(f.format, DynamicStructure):
                _LOG.debug('pre-pack {!r}'.format(f.format))
                f._pre_pack(parents=parents, data=data)

    def pack(self, data):
        self._pre_pack(data=data)
        self.setup()
        return super(DynamicStructure, self).pack(data)

    def pack_into(self, buffer, offset=0, data={}):
        self._pre_pack(data=data)
        self.setup()
        return super(DynamicStructure, self).pack_into(
            buffer=buffer, offset=offset, data=data)

    def unpack_stream(self, stream, parents=None, data=None, d=None):
        # `d` is the working data directory
        if data is None:
            parents = [self]
            data = d = {}
712
            if _LOG.level <= _logging.DEBUG:
713
714
715
716
717
718
719
                stream = DebuggingStream(stream)
        else:
            parents = parents + [self]

        for f in self.fields:
            _LOG.debug('parsing {!r}.{} (count={}, item_count={})'.format(
                    self, f, f.count, f.item_count))
720
721
            if _LOG.level <= _logging.DEBUG:
                _LOG.debug('data:\n{}'.format(_pprint.pformat(data)))
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
            if hasattr(f, 'pre_unpack'):
                _LOG.debug('pre-unpack {}'.format(f))
                f.pre_unpack(parents=parents, data=data)

            if hasattr(f, 'unpack'):  # override default unpacking
                _LOG.debug('override unpack for {}'.format(f))
                d[f.name] = f.unpack(stream)
                continue

            # setup for unpacking loop
            if isinstance(f.format, Structure):
                f.format.set_byte_order(self.byte_order)
                f.setup()
                f.format.setup()
                if isinstance(f.format, DynamicStructure):
                    if f.item_count == 1:
                        # TODO, fix in case we *want* an array
                        d[f.name] = {}
                        f.format.unpack_stream(
                            stream, parents=parents, data=data, d=d[f.name])
                    else:
                        d[f.name] = []
                        for i in range(f.item_count):
                            x = {}
                            d[f.name].append(x)
                            f.format.unpack_stream(
                                stream, parents=parents, data=data, d=x)
                    if hasattr(f, 'post_unpack'):
                        _LOG.debug('post-unpack {}'.format(f))
                        repeat = f.post_unpack(parents=parents, data=data)
                        if repeat:
                            raise NotImplementedError(
                                'cannot repeat unpack for dynamic structures')
                    continue
            if isinstance(f.format, Structure):
                _LOG.debug('parsing {} bytes for {}'.format(
                        f.format.size, f.format.format))
                bs = [stream.read(f.format.size) for i in range(f.item_count)]
                def unpack():
                    f.format.set_byte_order(self.byte_order)
                    f.setup()
                    f.format.setup()
                    x = [f.format.unpack_from(b) for b in bs]
                    if len(x) == 1:  # TODO, fix in case we *want* an array
                        x = x[0]
                    return x
            else:
                field_format = self.byte_order + f.format*f.item_count
                field_format = field_format.replace('P', 'I')
                try:
                    size = _struct.calcsize(field_format)
                except _struct.error as e:
                    _LOG.error(e)
                    _LOG.error('{}.{}: {}'.format(self, f, field_format))
                    raise
                _LOG.debug('parsing {} bytes for preliminary {}'.format(
                        size, field_format))
                raw = stream.read(size)
                if len(raw) < size:
                    raise ValueError(
                        'not enough data to unpack {}.{} ({} < {})'.format(
                            self, f, len(raw), size))
                def unpack():
                    field_format = self.byte_order + f.format*f.item_count
                    field_format = field_format.replace('P', 'I')
                    _LOG.debug('parse previous bytes using {}'.format(
                            field_format))
                    struct = _struct.Struct(field_format)
                    items = struct.unpack(raw)
                    return f.unpack_data(items)

            # unpacking loop
            repeat = True
            while repeat:
                d[f.name] = unpack()
                if hasattr(f, 'post_unpack'):
                    _LOG.debug('post-unpack {}'.format(f))
                    repeat = f.post_unpack(parents=parents, data=data)
                else:
                    repeat = False
                if repeat:
                    _LOG.debug('repeat unpack for {}'.format(f))

        return data

    def unpack(self, string):
        stream = _io.BytesIO(string)
        return self.unpack_stream(stream)

    def unpack_from(self, buffer, offset=0, *args, **kwargs):
        args = super(Structure, self).unpack_from(
            buffer, offset, *args, **kwargs)
814
        return self._unpack_item(args)