Merge the Documents/sras-viewer working copy into this repo
The two clones had diverged from origin/main (191d1b8) and both had
grown uncommitted work. This merges the other clone's three commits and
reconciles four conflicting files.
Both clones independently grew a feature called "batch export", but they
are different sweeps and both are kept:
- BatchExportWorker (this clone) — every angle x channel of the one open
file, fixed colorbar per channel, under a new Export menu.
- BatchExportImagesWorker (other clone) — the current view across many
selected files, process-pooled, under Convert.
Conflict resolutions of note:
- sras_average.py: the other clone's memory-bounded rewrite had silently
reverted this clone's float32 -> int16 accumulator fix, which exists to
keep averaged output from landing 1 ADC count off the original tool.
Kept the rewrite, re-applied the fix, and corrected the two docstrings
that still claimed float32.
- tests/test_compute.py: kept the other clone's meta["waveforms"] key
(meta["data"] no longer exists for this fixture and would KeyError)
and its laser_freq_hz assertions, plus this clone's int16 expectation
and its dropped subprocess returncode assertions.
- ComputeWorker: row_avg_n and min_freq_mhz were added independently on
either side; both are now threaded through.
- compute_rf_image's `exact` parameter is gone, removed by the other
clone's PyFFTW peak-search rewrite. It had no remaining callers.
Verified: 149 passed with a complete dependency set. The failures seen
with this clone's own .venv are a missing scikit-image, which predates
this merge and reproduces identically on the pre-merge commit.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
+122
-8
@@ -22,6 +22,7 @@ import sras_compute as compute
|
||||
from sras_align_export import write_aligned_sras
|
||||
from sras_compute import cache_file, compute_rf_image, dc_image_mv
|
||||
from sras_format import CH1_IDX, CH3_IDX, CH4_IDX, SrasFile
|
||||
from sras_render import export_view_image
|
||||
|
||||
# Concurrency caps. Batch conversion runs one process per file, and each of
|
||||
# those processes threads internally, so the two must be divided rather than
|
||||
@@ -120,7 +121,8 @@ class ComputeWorker(CancellableWorker):
|
||||
dc_threshold_mv: float = 0.0,
|
||||
dc4_mv: np.ndarray | None = None,
|
||||
is_fft_mode: bool = False,
|
||||
row_avg_n: int = 0):
|
||||
row_avg_n: int = 0,
|
||||
min_freq_mhz: float = 0.0):
|
||||
super().__init__()
|
||||
self._sras = sras
|
||||
self._angle = angle_idx
|
||||
@@ -131,6 +133,7 @@ class ComputeWorker(CancellableWorker):
|
||||
self._dc4_mv = dc4_mv
|
||||
self._is_fft_mode = is_fft_mode
|
||||
self._row_avg_n = row_avg_n
|
||||
self._min_freq_mhz = min_freq_mhz
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
@@ -139,7 +142,8 @@ class ComputeWorker(CancellableWorker):
|
||||
self._sras, self._angle, dc_threshold_mv=self._dc_threshold,
|
||||
apply_bg_sub=self._apply_bg_sub, n_fft=self._n_fft,
|
||||
dc4_mv=self._dc4_mv, should_stop=self._stopped,
|
||||
row_avg_n=self._row_avg_n)
|
||||
row_avg_n=self._row_avg_n,
|
||||
min_freq_mhz=self._min_freq_mhz)
|
||||
else:
|
||||
img = dc_image_mv(self._sras, self._angle, self._ch,
|
||||
should_stop=self._stopped)
|
||||
@@ -201,7 +205,9 @@ class BatchCacheWorker(QObject):
|
||||
|
||||
Both FFT modes cache at *pad_factor*, which the caller sets from the
|
||||
viewer's own padding — a cache stored at a pad the user is not viewing
|
||||
at is one the display can never use.
|
||||
at is one the display can never use. *min_freq_mhz* travels the same
|
||||
way: the viewer's live min-peak-freq floor, recorded in the store as
|
||||
provenance so a reader knows which bins the peak search considered.
|
||||
|
||||
Files are processed one per subprocess: they are fully independent, each
|
||||
opens its own memmap and writes only its own bytes, and only path strings
|
||||
@@ -215,7 +221,7 @@ class BatchCacheWorker(QObject):
|
||||
|
||||
def __init__(self, paths: list[str], mode: str, apply_bg_sub: bool,
|
||||
dc_threshold_mv: float | None = None, row_avg_n: int = 0,
|
||||
pad_factor: int = 1):
|
||||
pad_factor: int = 1, min_freq_mhz: float = 0.0):
|
||||
super().__init__()
|
||||
self._paths = paths
|
||||
self._mode = mode
|
||||
@@ -223,6 +229,7 @@ class BatchCacheWorker(QObject):
|
||||
self._dc_threshold = dc_threshold_mv
|
||||
self._row_avg_n = row_avg_n
|
||||
self._pad_factor = pad_factor
|
||||
self._min_freq_mhz = min_freq_mhz
|
||||
|
||||
def _report(self, path: str, err: str, done: int, total: int):
|
||||
self.file_done.emit(path, err)
|
||||
@@ -247,10 +254,11 @@ class BatchCacheWorker(QObject):
|
||||
with ProcessPoolExecutor(max_workers=n_procs) as executor:
|
||||
futures = {
|
||||
executor.submit(cache_file, p, self._mode, self._apply_bg_sub,
|
||||
compute.get_fft_backend(), per_proc_workers,
|
||||
per_proc_workers,
|
||||
pad_factor=self._pad_factor,
|
||||
dc_threshold_mv=self._dc_threshold,
|
||||
row_avg_n=self._row_avg_n): p
|
||||
row_avg_n=self._row_avg_n,
|
||||
min_freq_mhz=self._min_freq_mhz): p
|
||||
for p in paths
|
||||
}
|
||||
for fut in as_completed(futures):
|
||||
@@ -273,11 +281,11 @@ class BatchCacheWorker(QObject):
|
||||
for path in paths:
|
||||
try:
|
||||
err = cache_file(path, self._mode, self._apply_bg_sub,
|
||||
compute.get_fft_backend(),
|
||||
compute.default_max_workers(),
|
||||
pad_factor=self._pad_factor,
|
||||
dc_threshold_mv=self._dc_threshold,
|
||||
row_avg_n=self._row_avg_n)
|
||||
row_avg_n=self._row_avg_n,
|
||||
min_freq_mhz=self._min_freq_mhz)
|
||||
except Exception as exc:
|
||||
err = str(exc)
|
||||
done += 1
|
||||
@@ -449,6 +457,112 @@ class BatchExportWorker(CancellableWorker):
|
||||
self.finished.emit()
|
||||
|
||||
|
||||
class BatchExportImagesWorker(QObject):
|
||||
"""Renders the view settings captured by the caller at trigger time
|
||||
(angle/channel/threshold/etc.) to one PNG per file in *paths*, via
|
||||
sras_render.export_view_image.
|
||||
|
||||
Same process-pool-with-inline-fallback strategy as BatchCacheWorker
|
||||
above (same _BATCH_MAX_PROCS / _BATCH_POOL_MIN_BYTES thresholds): an
|
||||
FFT-derived (CH1/Velocity) view is exactly the same expensive per-file
|
||||
compute Batch Compute FFT already parallelizes this way. Never mutates
|
||||
*paths* — each file is opened read-only — so unlike BatchCacheWorker
|
||||
there is no version gate.
|
||||
|
||||
Emits progress(int) (0-100 by files completed), file_done(str, str, str)
|
||||
(path, error message or "", output filename this file targeted — set
|
||||
even on most failures so the caller can flag same-stem collisions across
|
||||
the batch without any cross-process bookkeeping), and finished().
|
||||
"""
|
||||
progress = pyqtSignal(int)
|
||||
file_done = pyqtSignal(str, str, str)
|
||||
finished = pyqtSignal()
|
||||
|
||||
def __init__(self, paths: list[str], **render_kwargs):
|
||||
"""*render_kwargs* is exactly export_view_image's keyword-only
|
||||
settings (out_dir, angle_idx, ch_idx, is_fft_mode, is_velocity,
|
||||
dc_threshold_mv, apply_bg_sub, pad_factor, min_freq_mhz, grating_um,
|
||||
cmap, auto_scale, vmin, vmax, highlight_masked, mode_str,
|
||||
colorbar_label, mask_color) — bundled rather than repeated as
|
||||
positional params across __init__/_run_pooled/_run_inline."""
|
||||
super().__init__()
|
||||
self._paths = paths
|
||||
self._kw = render_kwargs
|
||||
|
||||
def _report(self, path: str, err: str, out_name: str, done: int, total: int):
|
||||
self.file_done.emit(path, err, out_name)
|
||||
self.progress.emit(int(done / max(1, total) * 100))
|
||||
|
||||
def _run_pooled(self, paths: list[str], n_procs: int) -> list[str]:
|
||||
"""Same contract as BatchCacheWorker._run_pooled: returns the paths
|
||||
that never got a real answer because the pool itself died, so the
|
||||
caller can retry them in-process."""
|
||||
per_proc_workers = max(1, (os.cpu_count() or 4) // n_procs)
|
||||
unresolved: list[str] = []
|
||||
done = 0
|
||||
|
||||
with ProcessPoolExecutor(max_workers=n_procs) as executor:
|
||||
futures = {
|
||||
executor.submit(export_view_image, p,
|
||||
max_workers=per_proc_workers, **self._kw): p
|
||||
for p in paths
|
||||
}
|
||||
for fut in as_completed(futures):
|
||||
path = futures[fut]
|
||||
try:
|
||||
err, out_name = fut.result()
|
||||
except BrokenProcessPool:
|
||||
unresolved.append(path)
|
||||
continue
|
||||
except Exception as exc:
|
||||
err, out_name = str(exc), ""
|
||||
done += 1
|
||||
self._report(path, err, out_name, done, len(paths))
|
||||
|
||||
return unresolved
|
||||
|
||||
def _run_inline(self, paths: list[str], done: int, total: int):
|
||||
for path in paths:
|
||||
try:
|
||||
err, out_name = export_view_image(
|
||||
path, max_workers=compute.default_max_workers(), **self._kw)
|
||||
except Exception as exc:
|
||||
err, out_name = str(exc), ""
|
||||
done += 1
|
||||
self._report(path, err, out_name, done, total)
|
||||
|
||||
def _worth_pooling(self, paths: list[str]) -> bool:
|
||||
if len(paths) < 2:
|
||||
return False
|
||||
total = 0
|
||||
for p in paths:
|
||||
try:
|
||||
total += os.path.getsize(p)
|
||||
except OSError:
|
||||
pass # unreadable files are reported by export_view_image
|
||||
return total >= _BATCH_POOL_MIN_BYTES
|
||||
|
||||
def run(self):
|
||||
paths = self._paths
|
||||
n_procs = max(1, min(_BATCH_MAX_PROCS, len(paths)))
|
||||
|
||||
if not self._worth_pooling(paths):
|
||||
self._run_inline(paths, 0, len(paths))
|
||||
self.finished.emit()
|
||||
return
|
||||
|
||||
try:
|
||||
unresolved = self._run_pooled(paths, n_procs)
|
||||
except Exception:
|
||||
# The pool could not be created or collapsed wholesale.
|
||||
unresolved = list(paths)
|
||||
|
||||
if unresolved:
|
||||
self._run_inline(unresolved, len(paths) - len(unresolved), len(paths))
|
||||
|
||||
self.finished.emit()
|
||||
|
||||
|
||||
class Ch4MaskWorker(_PooledWorker):
|
||||
"""Fetches each requested angle's CH4 (Bias B) DC image in mV, for the
|
||||
alignment wizard's initial threshold-mask stack.
|
||||
|
||||
Reference in New Issue
Block a user