From 3187d59ba6436f3d8d3e94e27a1a99ebb67fdd39 Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Tue, 11 Aug 2026 13:32:47 -0400 Subject: [PATCH 1/7] HIT L2: added data transformation into 10-minute chunks to L2 --- .../cdf/config/imap_constant_attrs.yaml | 15 ++++ imap_processing/hit/l2/hit_l2.py | 68 ++++++++++++++++++- 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/imap_processing/cdf/config/imap_constant_attrs.yaml b/imap_processing/cdf/config/imap_constant_attrs.yaml index 5ca7d059f3..8d893def86 100644 --- a/imap_processing/cdf/config/imap_constant_attrs.yaml +++ b/imap_processing/cdf/config/imap_constant_attrs.yaml @@ -21,6 +21,21 @@ epoch: VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) VAR_TYPE: support_data +epoch_delta: + NAME: epoch_delta + DATA_TYPE: CDF_INT8 + CATDESC: Epoch delta + VAR_TYPE: support_data + RECORD_VARYING: RV + DEPEND_0: epoch + FIELDNAM: Epoch Delta + FORMAT: I19 + LABLAXIS: Epoch delta + UNITS: ns + VALIDMIN: 0 + VALIDMAX: 86000000000000 + FILLVAL: -9223372036854775808 + SCALETYP: linear # <=== Data Variables ===> # Default Attrs for all metadata variables unless overridden diff --git a/imap_processing/hit/l2/hit_l2.py b/imap_processing/hit/l2/hit_l2.py index 8c2230b214..007ad8bb18 100644 --- a/imap_processing/hit/l2/hit_l2.py +++ b/imap_processing/hit/l2/hit_l2.py @@ -143,6 +143,10 @@ def add_cdf_attributes( ) dataset.coords[f"{dim}_label"] = label_array + if "macropixel" in logical_source and "epoch" in dataset.coords: + dataset["epoch"].attrs["DELTA_MINUS_VAR"] = "epoch_delta" + dataset["epoch"].attrs["DELTA_PLUS_VAR"] = "epoch_delta" + return dataset @@ -782,4 +786,66 @@ def process_macropixel_intensity( {var: f"{var}_macropixel_intensity"} ) - return macropixel_intensity_dataset + return transform_to_10_minute_chunks(macropixel_intensity_dataset) + + +def transform_to_10_minute_chunks(macropixel_dataset: xr.Dataset) -> xr.Dataset: + """Transform macropixel records into 10-minute integration chunks. + + Parameters + ---------- + macropixel_dataset : xarray.Dataset + Macropixel data containing one species and energy combination per + one-minute epoch. + + Returns + ------- + xarray.Dataset + Macropixel data combined into one record per 10-minute integration. + """ + species_energy = [ + ("h", 3), + ("he4", 2), + ("cno", 2), + ("nemgsi", 2), + ("fe", 1), + ] + + # Note(Leo): need to check if doing in-place is acceptable + transformed_dataset = macropixel_dataset.isel( + epoch=slice(None, None, 10), + ).copy(deep=True) + + species_i = 0 + for species, num_energy_levels in species_energy: + energy_dim = f"{species}_energy_mean" + species_variables = [ + var + for var in macropixel_dataset.data_vars + if macropixel_dataset[var].dims[:2] == ("epoch", energy_dim) + ] + + for energy_i in range(num_energy_levels): + for var in species_variables: + _data = macropixel_dataset[var].values[species_i::10, energy_i] + transformed_dataset[var].values[:, energy_i] = _data + species_i += 1 + + minute_cadence_epochs = macropixel_dataset["epoch"].values + ten_minute_cadence_epochs = minute_cadence_epochs.reshape(-1, 10) + new_epochs = [] + nanoseconds_per_10_min = SECONDS_PER_10_MIN * 1_000_000_000 + nanoseconds_per_5_min = nanoseconds_per_10_min // 2 + for chunk in ten_minute_cadence_epochs: + start_time = chunk[0] + end_time = chunk[-1] + new_epoch = start_time + (end_time - start_time) // 2 - nanoseconds_per_10_min + new_epochs.append(new_epoch) + + transformed_dataset = transformed_dataset.assign_coords(epoch=np.array(new_epochs)) + transformed_dataset["epoch_delta"] = xr.DataArray( + np.full(len(new_epochs), nanoseconds_per_5_min, dtype=np.int64), + dims=["epoch"], + ) + + return transformed_dataset From cf8f6ebb9a2254f8435261ff3fc14fbba8133f4d Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Tue, 11 Aug 2026 14:35:16 -0400 Subject: [PATCH 2/7] imap_processing/cdf/config/imap_constant_attrs.yaml: fixed YAML attribute ordering --- .../cdf/config/imap_constant_attrs.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/imap_processing/cdf/config/imap_constant_attrs.yaml b/imap_processing/cdf/config/imap_constant_attrs.yaml index 8d893def86..a4cf8ab610 100644 --- a/imap_processing/cdf/config/imap_constant_attrs.yaml +++ b/imap_processing/cdf/config/imap_constant_attrs.yaml @@ -22,20 +22,20 @@ epoch: VAR_TYPE: support_data epoch_delta: - NAME: epoch_delta - DATA_TYPE: CDF_INT8 CATDESC: Epoch delta - VAR_TYPE: support_data - RECORD_VARYING: RV + DATA_TYPE: CDF_INT8 DEPEND_0: epoch FIELDNAM: Epoch Delta + FILLVAL: -9223372036854775808 FORMAT: I19 LABLAXIS: Epoch delta + NAME: epoch_delta + RECORD_VARYING: RV + SCALETYP: linear UNITS: ns - VALIDMIN: 0 VALIDMAX: 86000000000000 - FILLVAL: -9223372036854775808 - SCALETYP: linear + VALIDMIN: 0 + VAR_TYPE: support_data # <=== Data Variables ===> # Default Attrs for all metadata variables unless overridden From 758f4911b612ab78a81cb2b7d121e2c55a5d1790 Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Thu, 13 Aug 2026 14:01:31 -0400 Subject: [PATCH 3/7] Update imap_processing/hit/l2/hit_l2.py Co-authored-by: Tim Plummer --- imap_processing/hit/l2/hit_l2.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/imap_processing/hit/l2/hit_l2.py b/imap_processing/hit/l2/hit_l2.py index 007ad8bb18..33c49be4ce 100644 --- a/imap_processing/hit/l2/hit_l2.py +++ b/imap_processing/hit/l2/hit_l2.py @@ -836,11 +836,9 @@ def transform_to_10_minute_chunks(macropixel_dataset: xr.Dataset) -> xr.Dataset: new_epochs = [] nanoseconds_per_10_min = SECONDS_PER_10_MIN * 1_000_000_000 nanoseconds_per_5_min = nanoseconds_per_10_min // 2 - for chunk in ten_minute_cadence_epochs: - start_time = chunk[0] - end_time = chunk[-1] - new_epoch = start_time + (end_time - start_time) // 2 - nanoseconds_per_10_min - new_epochs.append(new_epoch) + start_times = ten_minute_cadence_epochs[:, 0] + end_times = ten_minute_cadence_epochs[:, -1] + new_epochs = start_times + (end_times - start_times) // 2 - nanoseconds_per_10_min transformed_dataset = transformed_dataset.assign_coords(epoch=np.array(new_epochs)) transformed_dataset["epoch_delta"] = xr.DataArray( From 90dea245b806d8a531f0b92ba1a94f6e22ee5d24 Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Thu, 13 Aug 2026 14:21:49 -0400 Subject: [PATCH 4/7] imap_processing/hit/l2/hit_l2.py: added comments and improved variable names around transformation algorithm --- imap_processing/hit/l2/hit_l2.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/imap_processing/hit/l2/hit_l2.py b/imap_processing/hit/l2/hit_l2.py index 33c49be4ce..cd5c836d28 100644 --- a/imap_processing/hit/l2/hit_l2.py +++ b/imap_processing/hit/l2/hit_l2.py @@ -811,14 +811,18 @@ def transform_to_10_minute_chunks(macropixel_dataset: xr.Dataset) -> xr.Dataset: ("fe", 1), ] - # Note(Leo): need to check if doing in-place is acceptable + # Use the first record in each 10-record group as the output template. transformed_dataset = macropixel_dataset.isel( epoch=slice(None, None, 10), ).copy(deep=True) + # Each minute in a 10-record group contains one species/energy combination, + # ordered as described by species_energy. Track that minute's packet offset. species_i = 0 for species, num_energy_levels in species_energy: energy_dim = f"{species}_energy_mean" + # Gather the intensity and uncertainty variables that share this + # species' energy dimension. species_variables = [ var for var in macropixel_dataset.data_vars @@ -827,17 +831,18 @@ def transform_to_10_minute_chunks(macropixel_dataset: xr.Dataset) -> xr.Dataset: for energy_i in range(num_energy_levels): for var in species_variables: - _data = macropixel_dataset[var].values[species_i::10, energy_i] - transformed_dataset[var].values[:, energy_i] = _data + # Select this species/energy packet from every 10-record group + # and place it in the corresponding output energy plane. + data_i = macropixel_dataset[var].values[species_i::10, energy_i] + transformed_dataset[var].values[:, energy_i] = data_i species_i += 1 minute_cadence_epochs = macropixel_dataset["epoch"].values ten_minute_cadence_epochs = minute_cadence_epochs.reshape(-1, 10) - new_epochs = [] nanoseconds_per_10_min = SECONDS_PER_10_MIN * 1_000_000_000 nanoseconds_per_5_min = nanoseconds_per_10_min // 2 - start_times = ten_minute_cadence_epochs[:, 0] - end_times = ten_minute_cadence_epochs[:, -1] + start_times = ten_minute_cadence_epochs[:, 0] + end_times = ten_minute_cadence_epochs[:, -1] new_epochs = start_times + (end_times - start_times) // 2 - nanoseconds_per_10_min transformed_dataset = transformed_dataset.assign_coords(epoch=np.array(new_epochs)) From 53d37c0167e715c1eb56f9072875ef0ed226f318 Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Mon, 17 Aug 2026 15:03:41 -0400 Subject: [PATCH 5/7] Per Tim's suggestion, move HIT macropixel epoch attributes to YAML --- .../cdf/config/imap_hit_l2_variable_attrs.yaml | 4 ++++ imap_processing/hit/l2/hit_l2.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/imap_processing/cdf/config/imap_hit_l2_variable_attrs.yaml b/imap_processing/cdf/config/imap_hit_l2_variable_attrs.yaml index 9fe72e117c..4ab012e52a 100644 --- a/imap_processing/cdf/config/imap_hit_l2_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_hit_l2_variable_attrs.yaml @@ -48,6 +48,10 @@ default_angle_attrs: &default_angle VAR_TYPE: support_data # <=== Coordinates ===> +epoch_macropixel: + DELTA_MINUS_VAR: epoch_delta + DELTA_PLUS_VAR: epoch_delta + zenith: <<: *default_angle CATDESC: Angle from the spin axis (0 deg.) to anti-spin axis (180 deg.) in 8 bins diff --git a/imap_processing/hit/l2/hit_l2.py b/imap_processing/hit/l2/hit_l2.py index cd5c836d28..557a82be3b 100644 --- a/imap_processing/hit/l2/hit_l2.py +++ b/imap_processing/hit/l2/hit_l2.py @@ -142,10 +142,10 @@ def add_cdf_attributes( ), ) dataset.coords[f"{dim}_label"] = label_array - - if "macropixel" in logical_source and "epoch" in dataset.coords: - dataset["epoch"].attrs["DELTA_MINUS_VAR"] = "epoch_delta" - dataset["epoch"].attrs["DELTA_PLUS_VAR"] = "epoch_delta" + elif "macropixel" in logical_source: + dataset["epoch"].attrs.update( + attr_mgr.get_variable_attributes("epoch_macropixel", check_schema=False) + ) return dataset From 06f97b73fb655248a9b4f92d88b37e6f6046092a Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Wed, 26 Aug 2026 14:55:46 -0400 Subject: [PATCH 6/7] Remove non-ISTP DATA_TYPE/NAME/RECORD_VARYING keys from epoch_delta attrs --- imap_processing/cdf/config/imap_constant_attrs.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/imap_processing/cdf/config/imap_constant_attrs.yaml b/imap_processing/cdf/config/imap_constant_attrs.yaml index a4cf8ab610..86dc67edfc 100644 --- a/imap_processing/cdf/config/imap_constant_attrs.yaml +++ b/imap_processing/cdf/config/imap_constant_attrs.yaml @@ -23,14 +23,11 @@ epoch: epoch_delta: CATDESC: Epoch delta - DATA_TYPE: CDF_INT8 DEPEND_0: epoch FIELDNAM: Epoch Delta FILLVAL: -9223372036854775808 FORMAT: I19 LABLAXIS: Epoch delta - NAME: epoch_delta - RECORD_VARYING: RV SCALETYP: linear UNITS: ns VALIDMAX: 86000000000000 From 174439c07736a53fbd597f655f0bd4327f1eeedd Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Wed, 26 Aug 2026 16:43:39 -0400 Subject: [PATCH 7/7] imap_processing/tests/hit/test_hit_l2.py: added 10-minute chunk algorithm test --- imap_processing/tests/hit/test_hit_l2.py | 71 ++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/imap_processing/tests/hit/test_hit_l2.py b/imap_processing/tests/hit/test_hit_l2.py index bbc11768eb..facb82f5cb 100644 --- a/imap_processing/tests/hit/test_hit_l2.py +++ b/imap_processing/tests/hit/test_hit_l2.py @@ -34,6 +34,7 @@ process_standard_intensity, process_summed_intensity, reshape_for_sectored, + transform_to_10_minute_chunks, ) EXPECTED_STANDARD_LABLAXIS = { @@ -793,6 +794,76 @@ def test_process_macropixel_intensity( ) +def test_transform_to_10_minute_chunks(): + """Test that transform_to_10_minute_chunks correctly regroups one-minute + macropixel records into 10-minute chunks and re-centers the epoch.""" + n_minutes = 20 + minute_ns = 60_000_000_000 + epochs = np.arange(n_minutes, dtype=np.int64) * minute_ns + + # Each species/energy combination occupies one fixed minute slot within + # every 10-record group, in this order (mirrors the physical packet + # cadence handled by transform_to_10_minute_chunks). + species_energy = [ + ("h", 3), + ("he4", 2), + ("cno", 2), + ("nemgsi", 2), + ("fe", 1), + ] + + data_vars = {} + slot_for = {} + species_i = 0 + for species, num_energy_levels in species_energy: + energy_dim = f"{species}_energy_mean" + values = np.array( + [[m * 100 + e for e in range(num_energy_levels)] for m in range(n_minutes)], + dtype=np.float32, + ) + data_vars[f"{species}_macropixel_intensity"] = (("epoch", energy_dim), values) + for energy_i in range(num_energy_levels): + slot_for[(species, energy_i)] = species_i + species_i += 1 + + macropixel_dataset = xr.Dataset(data_vars, coords={"epoch": epochs}) + + result = transform_to_10_minute_chunks(macropixel_dataset) + + n_chunks = n_minutes // 10 + assert len(result["epoch"]) == n_chunks + + # epoch_delta is always half of a 10-minute chunk. + expected_epoch_delta = np.full( + n_chunks, SECONDS_PER_10_MIN * 1_000_000_000 // 2, dtype=np.int64 + ) + np.testing.assert_array_equal(result["epoch_delta"].values, expected_epoch_delta) + + # Each new epoch is centered on its 10-minute group, then shifted back by + # a full 10-minute chunk. + for chunk in range(n_chunks): + start = epochs[chunk * 10] + end = epochs[chunk * 10 + 9] + expected_epoch = start + (end - start) // 2 - SECONDS_PER_10_MIN * 1_000_000_000 + assert result["epoch"].values[chunk] == expected_epoch + + # Each species/energy variable should pull its value from the minute slot + # it physically occupies within every 10-record group. + for species, num_energy_levels in species_energy: + var = f"{species}_macropixel_intensity" + for energy_i in range(num_energy_levels): + slot = slot_for[(species, energy_i)] + expected = np.array( + [(chunk * 10 + slot) * 100 + energy_i for chunk in range(n_chunks)], + dtype=np.float32, + ) + np.testing.assert_array_equal( + result[var].values[:, energy_i], + expected, + err_msg=f"Mismatch for {var} energy index {energy_i}", + ) + + def test_process_summed_intensity(l1b_summed_rates_dataset, ancillary_dependencies): """Test the variables in the summed intensity dataset"""