Merge the Documents/sras-viewer working copy into this repo
The two clones had diverged from origin/main (191d1b8) and both had
grown uncommitted work. This merges the other clone's three commits and
reconciles four conflicting files.
Both clones independently grew a feature called "batch export", but they
are different sweeps and both are kept:
- BatchExportWorker (this clone) — every angle x channel of the one open
file, fixed colorbar per channel, under a new Export menu.
- BatchExportImagesWorker (other clone) — the current view across many
selected files, process-pooled, under Convert.
Conflict resolutions of note:
- sras_average.py: the other clone's memory-bounded rewrite had silently
reverted this clone's float32 -> int16 accumulator fix, which exists to
keep averaged output from landing 1 ADC count off the original tool.
Kept the rewrite, re-applied the fix, and corrected the two docstrings
that still claimed float32.
- tests/test_compute.py: kept the other clone's meta["waveforms"] key
(meta["data"] no longer exists for this fixture and would KeyError)
and its laser_freq_hz assertions, plus this clone's int16 expectation
and its dropped subprocess returncode assertions.
- ComputeWorker: row_avg_n and min_freq_mhz were added independently on
either side; both are now threaded through.
- compute_rf_image's `exact` parameter is gone, removed by the other
clone's PyFFTW peak-search rewrite. It had no remaining callers.
Verified: 149 passed with a complete dependency set. The failures seen
with this clone's own .venv are a missing scikit-image, which predates
this merge and reproduces identically on the pre-merge commit.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Thomas Ales [M S E]
+171
-54
@@ -12,6 +12,7 @@ from pathlib import Path
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import sras_average
|
||||
import sras_compute as compute
|
||||
from sras_compute import (
|
||||
cache_file, compute_dc_image, compute_rf_image, dc_image_mv,
|
||||
@@ -122,7 +123,7 @@ def test_parallel_identity(tmp_path, monkeypatch):
|
||||
# the block size so every chunk splits into many FFT tasks — the worst
|
||||
# case for boundary bugs.
|
||||
monkeypatch.setattr(compute, "_TOTAL_BYTES_BUDGET", 8 * n_frames * spf * 4)
|
||||
monkeypatch.setattr(compute, "_FFT_BLOCK", 4)
|
||||
monkeypatch.setattr(compute, "_FFT_BLOCK_MAX", 4)
|
||||
fft_rows = compute._plan_fft_rows(n_frames, spf, compute._TOTAL_BYTES_BUDGET)
|
||||
assert fft_rows < n_rows, \
|
||||
f"FFT work actually splits into multiple chunks ({fft_rows} of {n_rows})"
|
||||
@@ -155,43 +156,10 @@ def test_parallel_identity(tmp_path, monkeypatch):
|
||||
"rf image identical (masked, padded/zoom)"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spf,bps", [(64, 2), (37, 1)])
|
||||
def test_zoom_identity(tmp_path, monkeypatch, spf, bps):
|
||||
"""The zoom peak search must reproduce the full padded-rfft argmax
|
||||
bit-for-bit, across pad factors, masking, bg-sub, dtype, and backend."""
|
||||
path = tmp_path / f"zoom_{spf}.sras"
|
||||
gen.write(path, n_angles=2, seed=6, samples_per_frame=spf, bps=bps)
|
||||
sras = SrasFile(str(path))
|
||||
dc4 = dc_image_mv(sras, 0, CH4_IDX)
|
||||
thr = float(np.median(dc4))
|
||||
|
||||
backends = ["scipy"] + (["pyfftw"] if compute.PYFFTW_AVAILABLE else [])
|
||||
for backend in backends:
|
||||
monkeypatch.setattr(compute, "_fft_backend", backend)
|
||||
for pad in (4, 8, 40):
|
||||
n_fft = spf * pad
|
||||
for thr_v in (None, thr):
|
||||
for bg in (False, True):
|
||||
ref = compute_rf_image(sras, 0, dc_threshold_mv=thr_v,
|
||||
apply_bg_sub=bg, n_fft=n_fft,
|
||||
exact=True)
|
||||
zoom = compute_rf_image(sras, 0, dc_threshold_mv=thr_v,
|
||||
apply_bg_sub=bg, n_fft=n_fft)
|
||||
diff = int((ref != zoom).sum())
|
||||
assert diff == 0, \
|
||||
(f"{diff} px differ: backend={backend} pad={pad} "
|
||||
f"thr={thr_v} bg={bg} spf={spf}")
|
||||
|
||||
# A threshold above every pixel masks everything: both paths must agree
|
||||
# on an all-zero image.
|
||||
all_masked = compute_rf_image(sras, 0, dc_threshold_mv=1e9, n_fft=spf * 8)
|
||||
assert not all_masked.any()
|
||||
|
||||
|
||||
def test_zoom_identity_fuzz():
|
||||
"""Hammer _peak_bins_zoom directly with adversarial spectra: noise,
|
||||
def test_peak_bins_fuzz():
|
||||
"""Hammer _peak_bins directly with adversarial spectra: noise,
|
||||
un-subtracted DC offsets, on-bin and off-bin tones, near-tie tone pairs,
|
||||
and all-zero rows."""
|
||||
and all-zero rows — against an independent scipy.fft reference."""
|
||||
import scipy.fft as scipy_fft
|
||||
|
||||
rng = np.random.default_rng(42)
|
||||
@@ -222,14 +190,102 @@ def test_zoom_identity_fuzz():
|
||||
P[:, 0] = 0.0
|
||||
ref = np.argmax(P, axis=1)
|
||||
|
||||
zp = compute._zoom_plan(spf, n_fft)
|
||||
got = compute._peak_bins_zoom(w, zp)
|
||||
got = compute._peak_bins(w, n_fft)
|
||||
bad = np.nonzero(ref != got)[0]
|
||||
assert not len(bad), \
|
||||
(f"spf={spf} pad={pad}: rows {bad.tolist()} picked "
|
||||
f"{got[bad].tolist()} instead of {ref[bad].tolist()}")
|
||||
|
||||
|
||||
def test_peak_bins_fuzz_min_freq():
|
||||
"""Same adversarial-spectra fuzz as test_peak_bins_fuzz, but with a swept
|
||||
min_bin floor: bins below the floor must be excluded from the argmax
|
||||
exactly as the independent scipy.fft reference is, when zeroed the same
|
||||
way before argmax."""
|
||||
import scipy.fft as scipy_fft
|
||||
|
||||
rng = np.random.default_rng(43)
|
||||
for _ in range(25):
|
||||
spf = int(rng.integers(16, 220))
|
||||
pad = int(rng.choice([4, 5, 8, 16, 40]))
|
||||
n_fft = spf * pad
|
||||
n_wf = 24
|
||||
w = rng.normal(scale=20.0, size=(n_wf, spf))
|
||||
t = np.arange(spf)
|
||||
for r in range(6):
|
||||
f = rng.uniform(1.0, spf / 2 - 1)
|
||||
w[r] = 60 * np.sin(2 * np.pi * f * t / spf) + w[r] * (r % 2)
|
||||
f1, f2 = rng.uniform(2.0, spf / 2 - 2, size=2)
|
||||
w[6] = 50 * np.sin(2 * np.pi * f1 * t / spf) \
|
||||
+ 49.9 * np.sin(2 * np.pi * f2 * t / spf)
|
||||
w[7] = 50 * np.sin(2 * np.pi * f1 * t / spf) \
|
||||
+ 50 * np.cos(2 * np.pi * f2 * t / spf)
|
||||
w[8] = 90 + rng.normal(scale=5.0, size=spf)
|
||||
w[9] = 0.0
|
||||
w = w.astype(np.float32)
|
||||
|
||||
S = scipy_fft.rfft(w, n=n_fft, axis=-1, workers=1)
|
||||
P = S.real ** 2
|
||||
P += S.imag ** 2
|
||||
n_bins_fine = n_fft // 2 + 1
|
||||
min_bin = int(rng.integers(1, max(2, n_bins_fine // 3)))
|
||||
P[:, :min_bin] = 0.0
|
||||
ref = np.argmax(P, axis=1)
|
||||
|
||||
got = compute._peak_bins(w, n_fft, min_bin)
|
||||
bad = np.nonzero(ref != got)[0]
|
||||
assert not len(bad), \
|
||||
(f"spf={spf} pad={pad} min_bin={min_bin}: rows {bad.tolist()} picked "
|
||||
f"{got[bad].tolist()} instead of {ref[bad].tolist()}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spf", [64, 500, 2500])
|
||||
def test_fft_block_for(spf):
|
||||
"""Block size == _FFT_BLOCK_MAX at natural resolution, shrinks and stays
|
||||
>= _FFT_BLOCK_MIN as n_len grows, and the implied per-thread byte
|
||||
estimate respects _FFT_PLAN_BYTES_BUDGET except when the floor is
|
||||
engaged."""
|
||||
block_natural = compute._fft_block_for(spf, spf)
|
||||
assert block_natural == compute._FFT_BLOCK_MAX
|
||||
|
||||
prev = compute._FFT_BLOCK_MAX
|
||||
for pad in (2, 4, 8, 40, 500):
|
||||
n_len = spf * pad
|
||||
block = compute._fft_block_for(spf, n_len)
|
||||
assert compute._FFT_BLOCK_MIN <= block <= prev
|
||||
bytes_per_wf = 4 * spf + 8 * (n_len // 2 + 1)
|
||||
if block > compute._FFT_BLOCK_MIN:
|
||||
assert block * bytes_per_wf <= compute._FFT_PLAN_BYTES_BUDGET
|
||||
prev = block
|
||||
|
||||
|
||||
def test_compute_rf_image_min_freq_mhz(tmp_path):
|
||||
"""min_freq_mhz threads through compute_rf_image end-to-end, for both
|
||||
the natural-resolution and padded paths: 0.0 (default) must reproduce
|
||||
the pre-existing image exactly, and a floor above every real peak must
|
||||
collapse the image to bin 0 (0 MHz) — the same fallback the low-level
|
||||
search uses when nothing survives the floor."""
|
||||
path = tmp_path / "floor_e2e.sras"
|
||||
gen.write(path, n_angles=1, seed=12, samples_per_frame=64)
|
||||
sras = SrasFile(str(path))
|
||||
huge_floor = float(sras.freq_axis_mhz(None)[-1]) + 1.0 # above Nyquist
|
||||
|
||||
for n_fft in (None, 64 * 8):
|
||||
unfiltered = compute_rf_image(sras, 0, dc_threshold_mv=None,
|
||||
apply_bg_sub=False, n_fft=n_fft)
|
||||
same = compute_rf_image(sras, 0, dc_threshold_mv=None, apply_bg_sub=False,
|
||||
n_fft=n_fft, min_freq_mhz=0.0)
|
||||
assert np.array_equal(unfiltered, same), \
|
||||
f"n_fft={n_fft}: min_freq_mhz=0.0 changed the output"
|
||||
assert unfiltered.any(), \
|
||||
f"n_fft={n_fft}: fixture should have real signal"
|
||||
|
||||
collapsed = compute_rf_image(sras, 0, dc_threshold_mv=None, apply_bg_sub=False,
|
||||
n_fft=n_fft, min_freq_mhz=huge_floor)
|
||||
assert not collapsed.any(), \
|
||||
f"n_fft={n_fft}: floor above Nyquist should collapse to 0 MHz"
|
||||
|
||||
|
||||
def test_nomask_equals_low_threshold(tmp_path):
|
||||
"""dc_threshold_mv=None must equal a threshold below every pixel, while
|
||||
skipping the CH4 read."""
|
||||
@@ -307,42 +363,50 @@ def test_legacy_parse(tmp_path):
|
||||
|
||||
|
||||
def test_sras_average(tmp_path):
|
||||
"""The sras_average.py CLI: frame averaging with remainder handling."""
|
||||
src = tmp_path / "legacy_v4.sras"
|
||||
meta = gen.write_legacy(src, version=4, n_angles=2, n_rows=4, n_frames=12,
|
||||
samples_per_frame=32, seed=4)
|
||||
dst = tmp_path / "legacy_v4_avg.sras"
|
||||
"""The sras_average.py CLI: v6 frame averaging with remainder handling,
|
||||
per-angle ragged geometry, and the laser_freq_hz X-axis correction."""
|
||||
src = tmp_path / "v6.sras"
|
||||
meta = gen.write(src, n_angles=2, seed=4, samples_per_frame=32,
|
||||
geometry=[(4, 12)])
|
||||
src_sras = SrasFile(str(src))
|
||||
|
||||
dst = tmp_path / "v6_avg.sras"
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(REPO / "sras_average.py"), str(src), str(dst), "--n", "4"],
|
||||
capture_output=True, text=True, cwd=REPO)
|
||||
assert proc.returncode == 0, (proc.stderr or proc.stdout).strip()[-200:]
|
||||
|
||||
avg = SrasFile(str(dst))
|
||||
assert avg.version == 4
|
||||
assert avg.version == 6
|
||||
assert list(avg.n_frames) == [3, 3], f"{list(avg.n_frames)}"
|
||||
assert (avg.n_angles == 2 and list(avg.n_rows) == [4, 4]
|
||||
and avg.n_channels == meta["n_channels"])
|
||||
assert np.allclose(avg.ch_ymult_mv, SrasFile(str(src)).ch_ymult_mv), \
|
||||
and avg.n_channels == src_sras.n_channels == 3)
|
||||
assert np.allclose(avg.ch_ymult_mv, src_sras.ch_ymult_mv), \
|
||||
"calibration preserved"
|
||||
assert np.array_equal(avg.background, SrasFile(str(src)).background), \
|
||||
assert np.array_equal(avg.background, src_sras.background), \
|
||||
"background preserved"
|
||||
src_data = meta["data"]
|
||||
# int16 (not float32) before .mean(): matches average_rows' own
|
||||
assert avg.laser_freq_hz == pytest.approx(src_sras.laser_freq_hz / 4), \
|
||||
"laser_freq_hz divided by N keeps pixel_x_mm correct after binning"
|
||||
assert np.array_equal(avg.x_start_mm, src_sras.x_start_mm), \
|
||||
"per-angle x_start unchanged"
|
||||
|
||||
src_waves = meta["waveforms"]
|
||||
# int16 (not float32) before .mean(): matches _average_block's own
|
||||
# float64-accumulator behavior for integer input, so this doesn't
|
||||
# drift from what average_rows actually guarantees.
|
||||
expect0 = src_data[0][:, :, 0:4, :].astype(np.int16).mean(axis=2).astype(np.int16)
|
||||
# drift from what _average_block actually guarantees.
|
||||
expect0 = src_waves[0][:, :, 0:4, :].astype(np.int16).mean(axis=2).astype(np.int16)
|
||||
assert np.array_equal(np.asarray(avg.data[0])[:, :, 0, :], expect0), \
|
||||
"first averaged group equals the mean of its 4 source frames"
|
||||
|
||||
# Remainder handling: 12 frames / 5 -> 2 full groups + 1 partial.
|
||||
dst2 = tmp_path / "legacy_v4_avg5.sras"
|
||||
dst2 = tmp_path / "v6_avg5.sras"
|
||||
proc2 = subprocess.run([sys.executable, str(REPO / "sras_average.py"),
|
||||
str(src), str(dst2), "--n", "5"],
|
||||
capture_output=True, text=True, cwd=REPO)
|
||||
assert proc2.returncode == 0, (proc2.stderr or proc2.stdout).strip()[-200:]
|
||||
assert list(SrasFile(str(dst2)).n_frames) == [3, 3], \
|
||||
"partial trailing group kept by default"
|
||||
dst3 = tmp_path / "legacy_v4_avg5d.sras"
|
||||
dst3 = tmp_path / "v6_avg5d.sras"
|
||||
proc3 = subprocess.run([sys.executable, str(REPO / "sras_average.py"),
|
||||
str(src), str(dst3), "--n", "5", "--discard-remainder"],
|
||||
capture_output=True, text=True, cwd=REPO)
|
||||
@@ -351,6 +415,59 @@ def test_sras_average(tmp_path):
|
||||
"--discard-remainder drops the partial group"
|
||||
|
||||
|
||||
def test_sras_average_v7_cache_dropped(tmp_path):
|
||||
"""A v7 input's cache tail is indexed by frame count, so it's invalid
|
||||
after averaging changes that count -- the output must always be plain
|
||||
v6, never a v7 carrying a stale cache."""
|
||||
src = tmp_path / "v7.sras"
|
||||
gen.write(src, n_angles=2, seed=1, samples_per_frame=32, geometry=[(3, 8)])
|
||||
src_sras = SrasFile(str(src))
|
||||
src_sras.write_v7_cache(
|
||||
new_dc3_mv=[compute_dc_image(src_sras, a, CH3_IDX) for a in range(src_sras.n_angles)],
|
||||
new_dc4_mv=[compute_dc_image(src_sras, a, CH4_IDX) for a in range(src_sras.n_angles)])
|
||||
assert SrasFile(str(src)).version == 7
|
||||
|
||||
dst = tmp_path / "v7_avg.sras"
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(REPO / "sras_average.py"), str(src), str(dst), "--n", "2"],
|
||||
capture_output=True, text=True, cwd=REPO)
|
||||
assert proc.returncode == 0, (proc.stderr or proc.stdout).strip()[-200:]
|
||||
assert SrasFile(str(dst)).version == 6, "cache-bearing input still writes plain v6"
|
||||
|
||||
|
||||
def test_sras_average_rejects_legacy(tmp_path):
|
||||
"""This tool only speaks v6/v7 now; a legacy file must fail clearly
|
||||
rather than being silently misparsed."""
|
||||
src = tmp_path / "legacy_v4.sras"
|
||||
gen.write_legacy(src, version=4, n_angles=1, n_rows=2, n_frames=8,
|
||||
samples_per_frame=16, seed=0)
|
||||
dst = tmp_path / "legacy_v4_avg.sras"
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(REPO / "sras_average.py"), str(src), str(dst), "--n", "2"],
|
||||
capture_output=True, text=True, cwd=REPO)
|
||||
assert proc.returncode != 0
|
||||
assert "v6" in proc.stderr and "v7" in proc.stderr
|
||||
|
||||
|
||||
def test_sras_average_chunking_matches_unchunked(tmp_path):
|
||||
"""A tiny memory budget (forcing one row per chunk) must produce
|
||||
byte-identical output to a huge budget (everything in one chunk) -- the
|
||||
load-bearing correctness claim of the memory-bounded rewrite: chunk
|
||||
boundaries must never affect the averaged result."""
|
||||
src = tmp_path / "v6.sras"
|
||||
gen.write(src, n_angles=2, seed=7, samples_per_frame=48,
|
||||
geometry=[(6, 10), (5, 13)])
|
||||
sras = SrasFile(str(src))
|
||||
|
||||
dst_tiny = tmp_path / "avg_tiny.sras"
|
||||
dst_big = tmp_path / "avg_big.sras"
|
||||
sras_average.write_v6_averaged(sras, dst_tiny, 3, False, budget=1)
|
||||
sras_average.write_v6_averaged(sras, dst_big, 3, False, budget=1 << 30)
|
||||
|
||||
assert dst_tiny.read_bytes() == dst_big.read_bytes(), \
|
||||
"chunk size must not affect the averaged output"
|
||||
|
||||
|
||||
def test_unsupported_version_reported(tmp_path):
|
||||
"""cache_file must report, not raise, for a file it can't handle."""
|
||||
bogus = tmp_path / "bogus.sras"
|
||||
|
||||
Reference in New Issue
Block a user