Add min peak frequency floor (CACH v4) and Batch Export View as Images
Two strands of in-progress work, committed together because they overlap in sras_workers.py and main_window.py. Min peak frequency floor: - CACH tail bumped to version 4, adding u32 min_freq_khz provenance in fixed-point kHz (a float32 20.1 reads back as 20.10000038 and would report a spurious mismatch forever). v1-v3 tails read as no floor. - Stored FFT caches are accepted when the reader's floor is at or above the stored one, since a higher floor is re-applicable by masking. - Floor plumbed through compute_rf_image, BatchCacheWorker and the viewer. Batch Export View as Images: - New sras_render.py holds draw_view_image, shared by the Qt canvas and the headless exporter so a PNG cannot drift from what the GUI shows. Deliberately Qt-free so it is importable in a pool subprocess. - BatchExportImagesWorker renders the current view settings across many files, process-pooled with an inline fallback, reporting per-file output names so the caller can flag same-stem collisions. - _axes_extent extracted into sras_format for both render paths. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Thomas Ales [M S E]
+294
-2
@@ -180,12 +180,18 @@ def test_cach_v1_reads_as_natural_resolution(tmp_path):
|
||||
# Rewrite the tail as a genuine CACH v1 block (old header, no pad field).
|
||||
v2 = SrasFile(str(path))
|
||||
freq, entries = v2.precomputed_freq_mhz, list(range(v2.n_angles))
|
||||
tail_offset = v2._cache_tail_offset()
|
||||
payload = struct.pack(fmt.CACH_HDR_FMT, fmt.CACH_MAGIC, 1, fmt.CACH_FLAG_FFT)
|
||||
payload += struct.pack(fmt.SFFT_HDR_FMT_V1, fmt.SFFT_MAGIC,
|
||||
fmt.SFFT_FLAG_BG_SUB, len(entries))
|
||||
for a in entries:
|
||||
payload += struct.pack(">H", a) + freq[a].astype(">f4").tobytes()
|
||||
head = path.read_bytes()[:v2._cache_tail_offset()]
|
||||
# Windows refuses to truncate a file with a live mapping (write_bytes
|
||||
# opens 'wb'), and every SrasFile holds its waveform memmaps for life —
|
||||
# drop the instance first. The parsed freq arrays are plain copies and
|
||||
# stay usable.
|
||||
del v2
|
||||
head = path.read_bytes()[:tail_offset]
|
||||
path.write_bytes(head + payload)
|
||||
|
||||
v1 = SrasFile(str(path))
|
||||
@@ -493,12 +499,16 @@ def test_cach_v1_backward_compat_defaults_row_avg_n_zero(tmp_path):
|
||||
|
||||
v2 = SrasFile(str(path))
|
||||
freq, entries = v2.precomputed_freq_mhz, list(range(v2.n_angles))
|
||||
tail_offset = v2._cache_tail_offset()
|
||||
payload = struct.pack(fmt.CACH_HDR_FMT, fmt.CACH_MAGIC, 1, fmt.CACH_FLAG_FFT)
|
||||
payload += struct.pack(fmt.SFFT_HDR_FMT_V1, fmt.SFFT_MAGIC,
|
||||
fmt.SFFT_FLAG_BG_SUB, len(entries))
|
||||
for a in entries:
|
||||
payload += struct.pack(">H", a) + freq[a].astype(">f4").tobytes()
|
||||
head = path.read_bytes()[:v2._cache_tail_offset()]
|
||||
# See test_cach_v1_reads_as_natural_resolution: release the memmaps
|
||||
# before write_bytes truncates, or Windows raises EINVAL.
|
||||
del v2
|
||||
head = path.read_bytes()[:tail_offset]
|
||||
path.write_bytes(head + payload)
|
||||
|
||||
v1 = SrasFile(str(path))
|
||||
@@ -510,6 +520,288 @@ def test_cach_v1_backward_compat_defaults_row_avg_n_zero(tmp_path):
|
||||
"a v1 tail (predating this feature) can never satisfy a row-averaged request"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Min peak frequency floor: serve-time masking, on-disk provenance, batch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _biting_floor(stored: np.ndarray) -> float:
|
||||
"""A floor that zeroes some-but-not-all of *stored*'s positive peaks:
|
||||
the median distinct positive value, so pixels below it get masked and
|
||||
pixels at/above it survive (the mask is a strict <)."""
|
||||
positive = np.unique(stored[stored > 0])
|
||||
assert len(positive) >= 2, "fixture must have varied peak frequencies"
|
||||
return float(positive[len(positive) // 2])
|
||||
|
||||
|
||||
def test_stored_image_served_with_raised_floor_masked(rig, no_fft):
|
||||
"""The core of the 5 m/s bug fix: a floor above the stored one (here 0)
|
||||
is re-applied when the stored image is served — pixels whose stored
|
||||
peak falls below it come back as the 0.0 invalid sentinel, everything
|
||||
else passes through, and no FFT runs. Both directly through
|
||||
cached_rf_image and through compute_rf_image's fast path."""
|
||||
assert rig.sras.precomputed_min_freq_mhz == 0.0, "batched without a floor"
|
||||
stored = rig.sras.precomputed_freq_mhz[0]
|
||||
floor = _biting_floor(stored)
|
||||
expect = np.where(stored < floor, np.float32(0.0), stored)
|
||||
|
||||
img = cached_rf_image(rig.sras, 0, None, apply_bg_sub=True,
|
||||
min_freq_mhz=floor)
|
||||
assert img is not None, "an equal-or-higher floor is servable"
|
||||
assert np.array_equal(img, expect)
|
||||
assert (img == 0).any() and (img > 0).any(), \
|
||||
"the floor bites some pixels but not all"
|
||||
|
||||
via_compute = compute_rf_image(rig.sras, 0, dc_threshold_mv=None,
|
||||
apply_bg_sub=True, min_freq_mhz=floor)
|
||||
assert np.array_equal(via_compute, expect), \
|
||||
"compute_rf_image's fast path applies the same serve-time mask"
|
||||
assert not no_fft, f"serving + masking must not run an FFT: {no_fft}"
|
||||
|
||||
|
||||
def test_min_freq_floor_round_trips_and_gates_serving(tmp_path):
|
||||
"""cache_file records the floor in the SFFT header and the accept rule
|
||||
is asymmetric: an equal-or-higher request is servable, a lower one is
|
||||
refused (the stored search never looked below its floor). 20.1 pins the
|
||||
fixed-point kHz encoding — a float32 header field would read back as
|
||||
20.10000038…, above the requested 20.1, and mismatch forever."""
|
||||
path = tmp_path / "floor_roundtrip.sras"
|
||||
gen.write(path, n_angles=2, seed=31, samples_per_frame=128)
|
||||
floor = 20.1
|
||||
assert cache_file(str(path), "fft", True, min_freq_mhz=floor) == ""
|
||||
|
||||
sras = SrasFile(str(path))
|
||||
assert sras.precomputed_min_freq_mhz == floor, "exact fixed-point round-trip"
|
||||
assert all(((img == 0) | (img >= floor)).all()
|
||||
for img in sras.precomputed_freq_mhz), \
|
||||
"no stored peak below the floor"
|
||||
|
||||
def reasons(f):
|
||||
return compute.cache_mismatch_reasons(
|
||||
sras, n_fft=None, apply_bg_sub=True, row_avg_n=0, min_freq_mhz=f)
|
||||
|
||||
assert reasons(floor) == []
|
||||
assert reasons(floor + 5.0) == [], "a higher request is servable (masked)"
|
||||
low = reasons(0.0)
|
||||
assert low and "min-peak-freq floor" in low[0], \
|
||||
"a lower request cannot be answered by the stored search"
|
||||
assert cached_rf_image(sras, 0, None, apply_bg_sub=True) is None, \
|
||||
"default floor-0 request refused against a floored store"
|
||||
assert cached_rf_image(sras, 0, None, apply_bg_sub=True,
|
||||
min_freq_mhz=floor) is not None
|
||||
|
||||
|
||||
def test_batch_recompute_resolves_not_masks(tmp_path, no_fft):
|
||||
"""Re-batching an already-cached file at a raised floor must run the
|
||||
real FFT and store re-resolved peaks — never let compute_rf_image's
|
||||
fast path serve the file's own stale cache back to it and bake the
|
||||
masked copy in as if it were a recompute (silent, permanent data
|
||||
loss: a masked pixel's true above-floor peak is unrecoverable)."""
|
||||
path = tmp_path / "rebatch.sras"
|
||||
gen.write(path, n_angles=2, seed=32, samples_per_frame=256)
|
||||
assert cache_file(str(path), "fft", True) == ""
|
||||
|
||||
first = SrasFile(str(path))
|
||||
n_angles = first.n_angles
|
||||
# On the header's kHz grid up front (as the spinbox value would be), so
|
||||
# the recorded floor reads back equal — cache_file quantizes whatever it
|
||||
# is given, and this test wants that to be the identity.
|
||||
floor = round(_biting_floor(first.precomputed_freq_mhz[0]) * 1000) / 1000.0
|
||||
bites = [img < floor for img in first.precomputed_freq_mhz]
|
||||
assert bites[0].any(), "the floor must actually bite this fixture"
|
||||
# What a real floored compute gives, from a view blinded to the cache.
|
||||
first.precomputed_freq_mhz = [None] * n_angles
|
||||
expected = [compute_rf_image(first, a, dc_threshold_mv=None,
|
||||
apply_bg_sub=True, min_freq_mhz=floor)
|
||||
for a in range(n_angles)]
|
||||
del first # release memmaps before cache_file rewrites the tail
|
||||
|
||||
no_fft.clear()
|
||||
assert cache_file(str(path), "fft", True, min_freq_mhz=floor) == ""
|
||||
assert no_fft, "the re-batch ran a real FFT"
|
||||
|
||||
after = SrasFile(str(path))
|
||||
assert after.precomputed_min_freq_mhz == floor
|
||||
for a in range(n_angles):
|
||||
assert np.array_equal(after.precomputed_freq_mhz[a], expected[a]), \
|
||||
f"angle {a}: stored image is a real floored recompute"
|
||||
assert (after.precomputed_freq_mhz[a][bites[a]] >= floor).all(), \
|
||||
f"angle {a}: bitten pixels re-resolved above the floor, not zeroed"
|
||||
|
||||
|
||||
def test_min_freq_carries_forward_through_dc_write(tmp_path):
|
||||
"""A later DC-only write must leave the FFT block's recorded floor
|
||||
untouched, like row_avg_n and pad_factor."""
|
||||
path = tmp_path / "floor_carry.sras"
|
||||
gen.write(path, n_angles=2, seed=33, samples_per_frame=128)
|
||||
assert cache_file(str(path), "fft", True, min_freq_mhz=75.0) == ""
|
||||
assert cache_file(str(path), "dc", True) == ""
|
||||
after = SrasFile(str(path))
|
||||
assert after.precomputed_min_freq_mhz == 75.0, \
|
||||
"floor survives a DC-only write"
|
||||
|
||||
|
||||
def test_cach_v3_backward_compat_defaults_floor_zero(tmp_path):
|
||||
"""A v3 CACH tail predates the min peak frequency floor entirely (no
|
||||
min_freq_khz field) — readers must still parse it in full, treating it
|
||||
as floor 0: servable as-is at floor 0, and serve-maskable at any higher
|
||||
one. This is what protects existing real-world v7 caches from silently
|
||||
becoming unusable after the v4 bump ships."""
|
||||
path = tmp_path / "v3_floor.sras"
|
||||
gen.write(path, n_angles=2, seed=34, samples_per_frame=128)
|
||||
assert cache_file(str(path), "fft", True) == ""
|
||||
|
||||
# Rewrite the tail as a genuine CACH v3 block (no min_freq field).
|
||||
v4 = SrasFile(str(path))
|
||||
freq, entries = v4.precomputed_freq_mhz, list(range(v4.n_angles))
|
||||
tail_offset = v4._cache_tail_offset()
|
||||
payload = struct.pack(fmt.CACH_HDR_FMT, fmt.CACH_MAGIC, 3, fmt.CACH_FLAG_FFT)
|
||||
payload += struct.pack(fmt.SFFT_HDR_FMT_V3, fmt.SFFT_MAGIC,
|
||||
fmt.SFFT_FLAG_BG_SUB, len(entries), 0, 1)
|
||||
for a in entries:
|
||||
payload += struct.pack(">H", a) + freq[a].astype(">f4").tobytes()
|
||||
# See test_cach_v1_reads_as_natural_resolution: release the memmaps
|
||||
# before write_bytes truncates, or Windows raises EINVAL.
|
||||
del v4
|
||||
head = path.read_bytes()[:tail_offset]
|
||||
path.write_bytes(head + payload)
|
||||
|
||||
v3 = SrasFile(str(path))
|
||||
assert v3.precomputed_min_freq_mhz == 0.0
|
||||
assert v3.precomputed_pad_factor == 1 and v3.precomputed_bg_sub is True
|
||||
assert all(np.array_equal(v3.precomputed_freq_mhz[a], freq[a])
|
||||
for a in entries), "v3 images read back unchanged"
|
||||
assert cached_rf_image(v3, 0, None, apply_bg_sub=True) is not None
|
||||
floor = _biting_floor(freq[0])
|
||||
served = cached_rf_image(v3, 0, None, apply_bg_sub=True,
|
||||
min_freq_mhz=floor)
|
||||
assert served is not None
|
||||
assert np.array_equal(served,
|
||||
np.where(freq[0] < floor, np.float32(0.0), freq[0]))
|
||||
|
||||
|
||||
def test_min_freq_validation(tmp_path):
|
||||
path = tmp_path / "floor_bad.sras"
|
||||
gen.write(path, n_angles=1, seed=35, samples_per_frame=64)
|
||||
assert cache_file(str(path), "fft", True, min_freq_mhz=-1.0), \
|
||||
"negative floor must be an error, not a write"
|
||||
assert cache_file(str(path), "fft", True, min_freq_mhz=float("nan")), \
|
||||
"NaN floor must be an error, not a write"
|
||||
sras = SrasFile(str(path))
|
||||
with pytest.raises(ValueError):
|
||||
sras.write_v7_cache(new_min_freq_mhz=-0.5)
|
||||
|
||||
|
||||
def test_viewer_reapplies_floor_to_stored_images_without_computing(
|
||||
rig, no_fft, monkeypatch):
|
||||
"""The session-cache poisoning bug behind the '95 MHz peak but 5 m/s'
|
||||
report: changing 'Min peak freq' against a stored cache used to re-file
|
||||
the identical un-floored image under a key claiming the new floor — the
|
||||
UI looked updated, the pixels weren't. Now the floor really is
|
||||
re-applied on serve (masked, still no compute), and clearing it
|
||||
restores the unmasked image, still without computing."""
|
||||
app = QApplication.instance() or QApplication([]) # noqa: F841
|
||||
win = SrasViewerWindow()
|
||||
win.show()
|
||||
|
||||
dispatched = []
|
||||
original_start = type(win)._start_compute
|
||||
monkeypatch.setattr(type(win), "_start_compute",
|
||||
lambda self: (dispatched.append(self.spin_angle.value()),
|
||||
original_start(self))[1])
|
||||
try:
|
||||
win._load_file(str(rig.path))
|
||||
assert wait_until(lambda: win._sras is not None), "file loaded"
|
||||
assert wait_until(lambda: all((a, CH4_IDX) in win._dc_cache
|
||||
for a in range(rig.n_angles))), \
|
||||
"DC precompute finished"
|
||||
|
||||
win.combo_channel.setCurrentIndex(CH1_IDX)
|
||||
assert wait_until(lambda: win._current_ch == CH1_IDX), "CH1 displayed"
|
||||
assert np.allclose(win._current_image, rig.fresh[0], atol=1e-3)
|
||||
|
||||
# Round to the spinbox's 3-decimal granularity; the chosen bin value
|
||||
# still survives its own (strict-<) floor after rounding down.
|
||||
floor = round(_biting_floor(rig.fresh[0]), 3)
|
||||
expect = np.where(rig.fresh[0] < floor, np.float32(0.0), rig.fresh[0])
|
||||
|
||||
dispatched.clear()
|
||||
no_fft.clear()
|
||||
win.spin_min_freq_mhz.setValue(floor)
|
||||
win._on_min_freq_changed()
|
||||
pump(120)
|
||||
assert np.allclose(win._current_image, expect, atol=1e-3), \
|
||||
"raised floor re-masks the stored image on serve"
|
||||
assert (win._current_image == 0).any() and (win._current_image > 0).any()
|
||||
assert dispatched == [] and not no_fft, \
|
||||
f"re-masked serve needs no compute (jobs={dispatched}, fft={no_fft})"
|
||||
key = (0, win.spin_threshold_mv.value(), win.spin_min_freq_mhz.value())
|
||||
assert key in win._fft_cache
|
||||
assert np.allclose(win._fft_cache[key], expect, atol=1e-3), \
|
||||
"the session cache holds the value its key claims"
|
||||
|
||||
win.spin_min_freq_mhz.setValue(0.0)
|
||||
win._on_min_freq_changed()
|
||||
pump(120)
|
||||
assert np.allclose(win._current_image, rig.fresh[0], atol=1e-3), \
|
||||
"clearing the floor restores the unmasked stored image"
|
||||
assert dispatched == [] and not no_fft
|
||||
finally:
|
||||
win.close()
|
||||
pump(300)
|
||||
|
||||
|
||||
def test_viewer_batch_fft_records_the_floor(tmp_path, no_fft, monkeypatch):
|
||||
"""Convert → Batch Compute FFT with a floor set: the live spinbox value
|
||||
reaches cache_file, lands in the reloaded file's provenance and the
|
||||
info panel, and the viewer then serves the floored cache without
|
||||
recomputing — the tooltip's promised remedy, end to end."""
|
||||
path = tmp_path / "floor_gui.sras"
|
||||
gen.write(path, n_angles=2, seed=36, samples_per_frame=256)
|
||||
|
||||
app = QApplication.instance() or QApplication([]) # noqa: F841
|
||||
win = SrasViewerWindow()
|
||||
win.show()
|
||||
|
||||
dispatched = []
|
||||
original_start = type(win)._start_compute
|
||||
monkeypatch.setattr(type(win), "_start_compute",
|
||||
lambda self: (dispatched.append(self.spin_angle.value()),
|
||||
original_start(self))[1])
|
||||
try:
|
||||
win._load_file(str(path))
|
||||
assert wait_until(lambda: win._sras is not None), "file loaded"
|
||||
assert wait_until(lambda: all((a, CH4_IDX) in win._dc_cache
|
||||
for a in range(win._sras.n_angles))), \
|
||||
"DC precompute finished"
|
||||
|
||||
floor = 100.0
|
||||
win.spin_min_freq_mhz.setValue(floor)
|
||||
with patch("sras_viewer.main_window.QFileDialog.getOpenFileNames",
|
||||
return_value=([str(path)], "")):
|
||||
win._on_batch_compute("fft")
|
||||
assert wait_until(lambda: not win._job_running("batch"), 60000), "batch ran"
|
||||
assert wait_until(lambda: win._sras is not None
|
||||
and win._sras.version == 7), "file reloaded as v7"
|
||||
pump(200)
|
||||
assert win._sras.precomputed_min_freq_mhz == floor, \
|
||||
"the viewer's floor reached the stored provenance"
|
||||
assert "floor ≥ 100 MHz" in win.lbl_frame_warn.text()
|
||||
|
||||
no_fft.clear()
|
||||
dispatched.clear()
|
||||
win.combo_channel.setCurrentIndex(CH1_IDX)
|
||||
assert wait_until(lambda: win._current_ch == CH1_IDX), "CH1 displayed"
|
||||
pump(120)
|
||||
assert dispatched == [] and not no_fft, \
|
||||
f"floored cache serves the view directly (jobs={dispatched}, fft={no_fft})"
|
||||
img = win._current_image
|
||||
assert ((img == 0) | (img >= floor)).all(), \
|
||||
"no displayed peak below the floor"
|
||||
finally:
|
||||
win.close()
|
||||
pump(300)
|
||||
|
||||
|
||||
def test_viewer_batch_row_average_dispatch(tmp_path, monkeypatch, no_fft):
|
||||
"""Driving the new 'Batch Compute Row-Averaged FFT and Store' action
|
||||
end-to-end through the real menu handler: dialog values reach the
|
||||
|
||||
Reference in New Issue
Block a user