Add ArrayConverter for array payload members

A payload member spanning several elements had no expressible type. The
sub-array dtype IdentityConverter needs carries np.void as its scalar
type, so the member resolved as void, and pinning the type parameter
instead resolves it as the scalar. ArrayConverter takes the element type
and a length and resolves as an NDArray of that element. It is
typing-only and builds an IdentityConverter over the same sub-array
dtype, so the runtime object is the passthrough converter every code
path and isinstance already expects. The runtime emitter uses it for a
member longer than one element, so a schema-built module and a generated
package agree.

harp-benchmarks was the one package src tree outside the pyright
include, and no test imports its benchmark module, so three
parse_to_dataframe calls still passed timestamp and raised TypeError.
They now pass time_index, a masked np.int32 member no longer takes a
Python int, and the StartPulseTrain default check supplies the two
members that dataclass_transform makes required.

Author: glopesdev

pyproject.toml

@@ -118,6 +118,7 @@ include = [
     "src/packages/harp-device/src",
     "src/packages/harp-serial/src",
     "src/packages/harp-data/src",
+    "src/packages/harp-benchmarks/src",
     "docs/examples",
     "tests/conformance.py",
 ]

src/packages/harp-benchmarks/src/harp/benchmarks/benchmark.py

@@ -110,13 +110,13 @@ def benchmark_register(reg: BenchmarkedRegister, path: Path, *, runs: int) -> Re
         file_bytes=file_bytes,
     )
     df_pre = _time(
-        lambda: parse_to_dataframe(register, raw, timestamp=reg.timestamped),
+        lambda: parse_to_dataframe(register, raw, time_index=reg.timestamped),
         runs=runs,
         frames=frames,
         file_bytes=file_bytes,
     )
     df_re = _time(
-        lambda: parse_to_dataframe(register, path.read_bytes(), timestamp=reg.timestamped),
+        lambda: parse_to_dataframe(register, path.read_bytes(), time_index=reg.timestamped),
         runs=runs,
         frames=frames,
         file_bytes=file_bytes,
@@ -326,7 +326,7 @@ def main() -> None:
             f"df={_fmt_ms(res.df_preread.mean):>9s}ms"
         )
         if args.head:
-            df = parse_to_dataframe(reg.register, path.read_bytes(), timestamp=True)
+            df = parse_to_dataframe(reg.register, path.read_bytes(), time_index=True)
             print(df.head(5))
             print()
 

src/packages/harp-benchmarks/src/harp/benchmarks/register_models.py

@@ -6,6 +6,7 @@
 
 from harp.protocol import (
     AnonymousPayload,
+    ArrayConverter,
     BitMask,
     BoolConverter,
     Converter,
@@ -109,9 +110,7 @@ class AnalogDataPayload(StructPayload[np.float32], length=6):
     analog0: np.float32 = Field(IdentityConverter(np.float32), offset=0)
     analog1: np.float32 = Field(IdentityConverter(np.float32), offset=1)
     analog2: np.float32 = Field(IdentityConverter(np.float32), offset=2)
-    accelerometer: NDArray[np.float32] = Field(
-        IdentityConverter(np.dtype((np.float32, (3,)))), offset=3
-    )
+    accelerometer: NDArray[np.float32] = Field(ArrayConverter(np.float32, 3), offset=3)
 
 
 class AnalogData(RegisterBase[AnalogDataPayload]):
@@ -149,9 +148,7 @@ class VersionPayload(StructPayload[np.uint8], length=32):
     firmware_version: HarpVersion = Field(HarpVersionConverter(np.uint8), offset=3)
     hardware_version: HarpVersion = Field(HarpVersionConverter(np.uint8), offset=6)
     core_id: str = Field(StringConverter(3), offset=9)
-    interface_hash: NDArray[np.uint8] = Field(
-        IdentityConverter(np.dtype((np.uint8, (20,)))), offset=12
-    )
+    interface_hash: NDArray[np.uint8] = Field(ArrayConverter(np.uint8, 20), offset=12)
 
 
 class Version(RegisterBase[VersionPayload]):
@@ -375,7 +372,7 @@ def main() -> None:  # pragma: no cover - manual exploration entry point
     assert int(p.header) == 7 and int(p.data) == -1234
     print("CustomMemberConverter    OK")
 
-    p = _roundtrip(BitmaskSplitter, BitmaskSplitterPayload(low=0xA, high=0x5))
+    p = _roundtrip(BitmaskSplitter, BitmaskSplitterPayload(low=np.int32(0xA), high=np.int32(0x5)))
     assert int(p.low) == 0xA and int(p.high) == 0x5
     assert p.payload_array.tobytes() == bytes([0x5A])
     print("BitmaskSplitter          OK")
@@ -411,7 +408,10 @@ def main() -> None:  # pragma: no cover - manual exploration entry point
     assert p.digital_output == PwmPort.PWM1 and int(p.pulse_width) == 300
     assert int(p.frequency) == 200 and int(p.pulse_count) == 50
     assert StartPulseTrainPayload.payload_dtype.itemsize == 4
-    assert int(StartPulseTrainPayload(pulse_count=np.uint8(3)).frequency) == 1  # defaultValue
+    partial = StartPulseTrainPayload(
+        digital_output=PwmPort.PWM0, pulse_width=np.uint16(0), pulse_count=np.uint8(3)
+    )
+    assert int(partial.frequency) == 1  # defaultValue
     print("StartPulseTrain          OK  (4 masked members, 2 words, default frequency=1)")
 
     p = _roundtrip(EncoderMode, EncoderModeMask.DISPLACEMENT)

src/packages/harp-device/src/harp/device/schema/_emit.py

@@ -15,6 +15,7 @@
     Field,
     GroupMask,
     HarpVersionConverter,
+    ArrayConverter,
     IdentityConverter,
     RegisterBase,
     RegisterFloat,
@@ -318,7 +319,9 @@ def _resolve_converter(self, ctx: ConverterContext) -> Converter[Any]:
         if ctx.mask is not None:
             return IdentityConverter(ctx.member_dtype)  # bit-field: native slice of the element
         if it is None:
-            return IdentityConverter(ctx.raw_dtype)  # raw passthrough / sub-array
+            if ctx.length > 1:
+                return ArrayConverter(ctx.element, ctx.length)  # raw passthrough, sub-array
+            return IdentityConverter(ctx.element)  # raw passthrough, single element
         # A known primitive that didn't fit is re-interpreted per field (``{Name}Converter``);
         # an unknown interfaceType is a domain type (``{InterfaceType}Converter``).
         symbol = f"{ctx.name}Converter" if entry is not None else f"{it}Converter"

src/packages/harp-protocol/src/harp/protocol/__init__.py

@@ -1,6 +1,7 @@
 from ._message import HarpMessage, HarpParseError
 from ._message_type import MessageType
 from ._payload_converters import (
+    ArrayConverter,
     BoolConverter,
     Converter,
     EnumConverter,
@@ -78,6 +79,7 @@
     # Converters
     "Converter",
     "IdentityConverter",
+    "ArrayConverter",
     "StringConverter",
     "BoolConverter",
     "EnumConverter",

src/packages/harp-protocol/src/harp/protocol/_payload_converters.py

@@ -1,6 +1,6 @@
 import enum as _enum
 from abc import ABC, abstractmethod
-from typing import Any, Generic, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast
 from dataclasses import dataclass
 import numpy as np
 from numpy.typing import NDArray
@@ -121,6 +121,33 @@ def __init__(self) -> None:
         super().__init__(np.float32)
 
 
+if TYPE_CHECKING:
+
+    class ArrayConverter(Converter["NDArray[NpScalarT]"]):
+        """Pass-through converter for a member spanning several elements of one type.
+
+        The array counterpart of :class:`IdentityConverter`, decoding to an ``NDArray``
+        of the element type rather than to a single scalar. It exists to carry that type
+        and builds an :class:`IdentityConverter` over the equivalent sub-array dtype, so
+        every code path and every ``isinstance`` sees the passthrough converter it
+        already knows. Passing that dtype directly instead types the member as a scalar,
+        since a sub-array dtype carries ``np.void`` as its scalar type.
+        """
+
+        def __init__(self, dtype: "np.dtype[NpScalarT] | type[NpScalarT]", length: int) -> None: ...
+
+        def decode_scalar(self, view: np.generic) -> "NDArray[NpScalarT]": ...
+
+        def decode_batch(self, view: "NDArray[np.generic]") -> Any: ...
+
+        def encode_into(self, view: "NDArray[np.generic]", value: "NDArray[NpScalarT]") -> None: ...
+
+else:
+
+    def ArrayConverter(dtype, length):
+        return IdentityConverter(np.dtype((dtype, (length,))))
+
+
 class BoolConverter(Converter[bool]):
     """Whole-element ``interfaceType: bool`` (or a single masked bit via ``Field(BoolConverter(), mask=...)``).
 

tests/conformance.py

@@ -12,8 +12,16 @@
 from harp.device.client import Device, ITransport
 from harp.device.core import OperationControl, OperationControlPayload, WhoAmI
 from harp.device.schema import DeviceModule, DeviceModuleLike, create_device_module
-from harp.protocol import HarpMessage, RegisterBase
+from harp.protocol import (
+    ArrayConverter,
+    Field,
+    HarpMessage,
+    IdentityConverter,
+    RegisterBase,
+    StructPayload,
+)
 from harp.serial import open_device
+from numpy.typing import NDArray
 
 
 def schema_built_registers(yml: str) -> None:
@@ -131,3 +139,20 @@ class Hybrid(Device[None]):
 
     device = open_device(Hybrid, port="COM3")
     assert_type(device, Hybrid)
+
+
+def array_payload_members() -> None:
+    """A member spanning several elements resolves as an array of the element type.
+
+    The equivalent sub-array dtype passed to IdentityConverter produces the same bytes
+    but resolves as a scalar, since a sub-array dtype carries np.void as its scalar
+    type, so ArrayConverter is what carries the element type through the descriptor.
+    """
+
+    class Payload(StructPayload[np.float32], length=6):
+        analog0: np.float32 = Field(IdentityConverter(np.float32), offset=0)
+        accelerometer: NDArray[np.float32] = Field(ArrayConverter(np.float32, 3), offset=3)
+
+    payload = Payload(analog0=np.float32(0), accelerometer=np.zeros(3, dtype=np.float32))
+    assert_type(payload.analog0, np.float32)
+    assert_type(payload.accelerometer, NDArray[np.float32])