diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index 8a8b393..fac3009 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -48,3 +48,19 @@ rig, check whether the camera apps run without the shim; if they do, delete `lib/ueye_loader.{c,so}`. Either way, record in SETUP.md where `libueye_api64.so.3.82` came from (IDS SDK version) and how the loader is meant to be used. + +## Per-angle background: trigger round trip mid-scan + +Every angle now captures its own background, so the scope switches from the +scan-time logic-AND trigger back to the single-record edge trigger and +returns to it once per angle (`core/scope_sras.py`: +`configure_background_trigger` → `capture_background` → +`configure_scan_trigger`). Before this, that transition happened once per +scan, with the stage idle and nothing depending on how long it took. + +**Bench check:** run a multi-angle scan and watch the first row after each +background. If frames go missing at the start of an angle, the 0.2 s settle +in `configure_scan_trigger` is not enough for FastFrame to re-arm after an +AVERAGE-mode sequence, and the row-packing warning ("N frames acquired, M +expected") will say so in the log. Raise the settle rather than the ramp +buffer — the stage geometry is not what changed. diff --git a/README.md b/README.md index f2738cc..c492f84 100755 --- a/README.md +++ b/README.md @@ -12,10 +12,12 @@ scanengine-3 is a unified platform for scanning acoustic microscopy and precisio - **Laser Systems**: Helios pulsed laser and Genesis CW laser control - **Data Acquisition**: Tektronix oscilloscope integration with fast-frame support - **Scan Planning**: Automated raster scan generation and execution +- **Per-Angle Background**: every angle opens with its own background + capture (Genesis off, Helios on), stored ahead of that angle's data - **Angle Inspection**: Park the rig at random points across a plan's angles to check the SAW response on the scope before committing to a long scan - **SAW Quality Check**: Acquire one row per angle — the row-wise middle of - the ROI — as a v10 `.sras`, then compare every angle's SAW frequency on one + the ROI — as a v11 `.sras`, then compare every angle's SAW frequency on one graph to judge the alignment before a full run - **Real-time Monitoring**: Live status updates and progress tracking @@ -65,7 +67,7 @@ scanengine-3/ │ ├── angle_inspect.py # AngleInspector — park on a point per angle │ ├── saw_check.py # Middle-row SAW check: plan + alignment read-out │ ├── rotation.py # GR rotation axis settings + moves -│ ├── sras_format.py # v6/v10 .sras writer/reader (memory-mapped) +│ ├── sras_format.py # v7/v11 .sras writer, v6/v10 reader (mmap) │ ├── sras_analysis.py # Image reducers + SAW matched filter │ └── config.py # ScanDefaults ⇄ aui_defaults.json │ @@ -98,7 +100,7 @@ scanengine-3/ ├── sc3-aui-*.ui # Qt Designer files loaded at runtime │ ├── tests/ # pytest suite -│ ├── golden/ # v6 .sras + geometry fixtures +│ ├── golden/ # legacy v6 .sras + geometry fixtures │ ├── fakes.py # Recording fake stage/scope/rotator │ └── test_*.py │ @@ -220,7 +222,7 @@ print(f"wrote {result.rows_written} rows to {result.path}") ### Running a SAW quality check Same engine, same hardware sequence — the plan is reduced to one row per -angle and the result is tagged v10 so the viewer knows it is a check rather +angle and the result is tagged v11 so the viewer knows it is a check rather than a scan cut short: ```python diff --git a/core/saw_check.py b/core/saw_check.py index a3e93f5..70f8eb5 100644 --- a/core/saw_check.py +++ b/core/saw_check.py @@ -6,7 +6,7 @@ without the other: *Acquisition* — ``middle_row_plan`` reduces a full ScanPlan to a single row per angle, the row-wise middle of the ROI. ScanEngine runs the result -exactly like any other scan and writes it as a v10 .sras file +exactly like any other scan and writes it as a v11 .sras file (``sras_format.VERSION_SAW_CHECK``), so a check costs one row-time per angle instead of the hours a full multi-angle scan takes. @@ -55,7 +55,7 @@ def middle_row_plan(plan: ScanPlan) -> ScanPlan: An even row count has no exact middle; the upper of the two central rows is taken (``n_rows // 2``), which is also the row the viewer picks when it - reads the middle row out of a full v6 scan. + reads the middle row out of a full scan. """ if plan.n_angles == 0: raise ScanGeometryError("Cannot build a SAW check from a plan with no angles") @@ -181,18 +181,24 @@ class AlignmentSummary: def frequency_traces(sras: SrasFile, *, dc_threshold_mv: float = 0.0, - background: np.ndarray | None = None, + subtract_background: bool = False, gate_start_ns: float | None = None, gate_end_ns: float | None = None, calib: ChannelCalibration | None = None, on_progress=lambda done, total: None) -> list[AngleTrace]: """Peak SAW frequency along the middle row of every angle in ``sras``. - Works on a v10 check (one row per angle, so the middle row is the only - row) and on a full v6 scan alike — the same middle row the check would + Works on a v11 check (one row per angle, so the middle row is the only + row) and on a full scan alike — the same middle row the check would have acquired is pulled out of the scan, which is what lets a finished scan be re-examined with the check's own read-out. + ``subtract_background`` takes each angle's own background out of its + frames (v6/v10 files have only the one, which every angle then shares). + Doing it per angle is the point of the per-angle capture: comparing + angles is exactly what this read-out is for, so they must not be + referenced against one background taken at whichever angle came first. + Angles with nothing on disk (an aborted file) are skipped rather than reported as flat zero. """ @@ -210,6 +216,7 @@ def frequency_traces(sras: SrasFile, *, dc_threshold_mv: float = 0.0, row = middle_row_index(st.n_rows_available) view = sras.load_angle(st.index, n_rows=st.n_rows_available)[row:row + 1] + background = sras.background_array(st.index) if subtract_background else None img = compute_rf_image(view, calib, freq_axis, dc_threshold_mv, background=background, gate_start_ns=gate_start_ns, gate_end_ns=gate_end_ns, diff --git a/core/scan_engine.py b/core/scan_engine.py index 715baf4..7abc64f 100644 --- a/core/scan_engine.py +++ b/core/scan_engine.py @@ -18,7 +18,10 @@ from typing import Callable from core import scope_burst, scope_sras from core.rotation import RotationAxis from core.scan_geometry import ScanPlan, validate_plan -from core.sras_format import SCAN_CHANNELS, VERSION, create_scan_file +from core.sras_format import ( + BG_LEN_SIZE, SCAN_CHANNELS, VERSION, create_scan_file, + write_background_block, +) logger = logging.getLogger(__name__) @@ -42,8 +45,15 @@ class ScanAborted(Exception): @dataclass class ResumeTarget: - """One angle selected for (re)acquisition in an existing file.""" + """One angle selected for (re)acquisition in an existing file. + + ``bg_offset`` is where the angle's background block starts and + ``data_offset`` where its rows do; the gap between them is the room the + file already has for a background, which a re-acquired angle must fill + exactly or every row behind it would shift. + """ angle_idx: int + bg_offset: int data_offset: int n_rows: int angle_deg: float @@ -280,27 +290,14 @@ class ScanEngine: if self._resume is None: self._preambles = scope_sras.read_preambles(self._scope, SCAN_CHANNELS) - self._prompt( - "Background Capture", - "Please ensure the Helios laser is ON and the Genesis laser is OFF,\n" - "then click OK to capture the background waveform." - ) - self._background = scope_sras.capture_background( - self._scope, should_abort=self._abort.is_set, - on_status=self._cb.on_status) - self._prompt( - "Begin Scanning", - "Background captured successfully.\n\n" - "Please ensure the Genesis laser is back ON,\n" - "then click OK to begin scanning." - ) else: - # Resuming: the file's existing background waveform and channel - # preambles are reused as-is (the format has no way to replace - # them without rewriting the whole file), so background capture is - # skipped. Sanity-check that this scope still produces the same - # record length the file was started with — a mismatch would - # silently corrupt the ragged per-row byte layout on append. + # Resuming: the file's channel preambles are reused as-is (the + # format has no way to replace them without rewriting the whole + # file). Each re-acquired angle still captures its own fresh + # background, which is rewritten in place over the old one. + # Sanity-check that this scope still produces the same record + # length the file was started with — a mismatch would silently + # corrupt the ragged per-row byte layout on append. if samples_per_frame != self._resume.samples_per_frame: raise RuntimeError( f"Oscilloscope record length ({samples_per_frame} samples/frame) " @@ -314,8 +311,8 @@ class ScanEngine: f"angle(s) {targets} of {self._plan.n_angles}.\n\n" "Please re-home the GR axis to 0° before continuing — the scan " "will rotate it directly from angle to angle before scanning resumes.\n\n" - "Please ensure the Genesis laser is ON,\n" - "then click OK to continue scanning." + "Each angle begins with its own background capture, so you will " + "be asked to switch the Genesis laser off and on again per angle." ) scope_sras.configure_scan_trigger(self._scope) @@ -341,7 +338,7 @@ class ScanEngine: return open(self._resume.path, "r+b") return create_scan_file( self._out_path, self._plan, samples_per_frame, - scope_sras.SAMPLE_RATE_HZ, self._preambles, self._background, + scope_sras.SAMPLE_RATE_HZ, self._preambles, version=self._file_version, ) @@ -363,11 +360,12 @@ class ScanEngine: continue # not selected for (re)acquisition self._pause_point() - if targets_by_ai is not None: + target = None if targets_by_ai is None else targets_by_ai[ai] + if target is not None: # Interior angles may already have valid data on either side, # so seek to this angle's fixed offset rather than relying on # the file's current position. - scan_file.seek(targets_by_ai[ai].data_offset) + scan_file.seek(target.bg_offset) if self._rotator is not None and self._rotator.is_available: delta = pa.angle_deg - self._rotator.current_deg @@ -376,6 +374,9 @@ class ScanEngine: f"Rotating GR to {pa.angle_deg:.1f}° (Δ{delta:+.1f}°) …") self._rotator.rotate_to(pa.angle_deg) + self._write_angle_background(scan_file, ai, n_angles, + pa.angle_deg, target) + if self._burst_mode: # Burst mode sizes the FastFrame count from the scope's whole # capacity instead (see scope_burst.start_burst), so there is @@ -391,6 +392,49 @@ class ScanEngine: result.angles_acquired.append(ai) + def _write_angle_background(self, scan_file, ai: int, n_angles: int, + angle_deg: float, target: ResumeTarget | None): + """Capture this angle's background and write it ahead of its rows. + + The Genesis laser has to be off for the capture and back on for the + scan, so every angle costs two operator prompts and one averaged + record. That buys a background taken minutes from the data it will + be subtracted from, instead of one taken hours earlier at angle 1. + + On resume the block is overwritten in place, so it has to be exactly + as long as the one already there — anything else would shift every + row behind it. Checked before the write, not after. + """ + scope = self._scope + scope_sras.configure_background_trigger(scope) + self._prompt( + f"Background Capture — Angle {ai + 1}/{n_angles}", + f"Angle {ai + 1} of {n_angles} ({angle_deg:.1f}°) starts with its " + "own background capture.\n\n" + "Please switch the Genesis laser OFF — leave the Helios laser ON —\n" + "then click OK to capture the background waveform." + ) + background = scope_sras.capture_background( + scope, should_abort=self._abort.is_set, on_status=self._cb.on_status) + self._prompt( + f"Begin Angle {ai + 1}/{n_angles}", + "Background captured successfully.\n\n" + "Please switch the Genesis laser back ON,\n" + f"then click OK to scan angle {ai + 1} of {n_angles}." + ) + scope_sras.configure_scan_trigger(scope) + + if target is not None: + room = target.data_offset - target.bg_offset + if BG_LEN_SIZE + len(background) != room: + raise RuntimeError( + f"Angle {ai + 1}: the new background block is " + f"{BG_LEN_SIZE + len(background)} bytes but the file has room " + f"for {room} — writing it would shift every row behind it, " + "so the scan stops here." + ) + write_background_block(scan_file, background) + # ── Per-row acquisition (one FastFrame acquisition per row) ─────────────── def _scan_rows_serial(self, scan_file, pa, ai: int, n_angles: int, diff --git a/core/scan_resume.py b/core/scan_resume.py index 53f2b27..998af0a 100644 --- a/core/scan_resume.py +++ b/core/scan_resume.py @@ -8,7 +8,7 @@ from __future__ import annotations from dataclasses import dataclass, field from core.scan_engine import ResumeState, ResumeTarget -from core.sras_format import SrasFile +from core.sras_format import WRITABLE_VERSIONS, SrasFile @dataclass @@ -44,8 +44,9 @@ def plan_resume(statuses, selected: set[int]) -> ResumePlan: final = set(selected) targets = [ - ResumeTarget(angle_idx=s.index, data_offset=s.data_offset, - n_rows=s.n_rows, angle_deg=s.angle_deg) + ResumeTarget(angle_idx=s.index, bg_offset=s.bg_offset, + data_offset=s.data_offset, n_rows=s.n_rows, + angle_deg=s.angle_deg) for s in statuses if s.index in final ] return ResumePlan(targets=targets, @@ -55,9 +56,15 @@ def plan_resume(statuses, selected: set[int]) -> ResumePlan: def is_compatible(sras: SrasFile, *, velocity: float, laser_freq: float, sample_rate: float, n_channels: int) -> bool: - """Whether appending to this file with the current settings is safe.""" + """Whether appending to this file with the current settings is safe. + + A legacy v6/v10 file is not: it has one background for the whole scan, + and every angle this engine acquires writes a background block of its + own, which the older layout has no room for. + """ h = sras.header - return (h.bytes_per_sample == 1 + return (sras.version in WRITABLE_VERSIONS + and h.bytes_per_sample == 1 and h.n_channels == n_channels and abs(h.velocity - velocity) <= 1e-3 and abs(h.laser_freq - laser_freq) <= 1e-3 diff --git a/core/scope_sras.py b/core/scope_sras.py index 0f7885d..fdcb228 100644 --- a/core/scope_sras.py +++ b/core/scope_sras.py @@ -51,11 +51,13 @@ def configure_channels(scope, profiles=None) -> None: scope.set_channel_bandwidth(ch, p.bandwidth_hz) -def configure_acquisition(scope) -> int: - """Program the edge trigger and timebase; returns samples per frame. +def configure_background_trigger(scope) -> None: + """Program the edge trigger used for a background capture. - Edge trigger on the rising edge of CH2 (laser pulse). FastFrame stays - off here so the background capture runs as a single record. + Edge trigger on the rising edge of CH2 (laser pulse), FastFrame off so + the capture runs as a single record. Every angle starts with a fresh + background, so the scan comes back here between angles from the + logic-AND trigger configure_scan_trigger leaves behind. """ scope.write("TRIGger:A:TYPe EDGE") scope.set_trigger_source(2) @@ -64,6 +66,11 @@ def configure_acquisition(scope) -> int: scope.set_trigger_mode("NORMAL") # wait for trigger (don't auto-sweep) scope.set_acquire_mode("SAMPLE") scope.set_fastframe_state(False) + + +def configure_acquisition(scope) -> int: + """Program the edge trigger and timebase; returns samples per frame.""" + configure_background_trigger(scope) # Pin the transfer format instead of inheriting front-panel state — the # file header hardcodes bytes_per_sample=1, and a scope left on 2 bytes # would corrupt every frame written. diff --git a/core/sras_format.py b/core/sras_format.py index 8977c68..4f54361 100644 --- a/core/sras_format.py +++ b/core/sras_format.py @@ -1,4 +1,5 @@ -"""SRAS binary scan-file format (v6 and v10) — the single implementation. +"""SRAS binary scan-file format (v7/v11, reading v6/v10 too) — the single +implementation. Full byte-level spec: scan_format.md. Summary: @@ -9,20 +10,33 @@ Full byte-level spec: scan_format.md. Summary: geometry table n_angles × >ffIH (x_start x_delta n_frames n_rows) row tables (ragged) per angle: n_rows × >f (y positions) preambles n_channels × (>H length + utf-8 WFMOutpre string) - background block >I length + raw int8 CH1 average - waveform data angle-major, row-minor, channel-inner: - for each angle, for each row, for each channel, + data block per angle: [background][rows] + background = >I length + raw int8 CH1 average + rows = for each row, for each channel, n_frames × samples_per_frame × bytes_per_sample +Every angle carries its own background: the operator switches the Genesis +laser off before each angle and the engine averages a fresh CH1 record, so +the reference a reader subtracts was taken minutes — not hours — from the +data it is subtracted from. That is the whole difference between v7 and v6, +which held a single background for the entire file, ahead of the data block. + Incomplete files are valid: the data block is one contiguous append-only stream, so the readable prefix defines a single frontier past which nothing -has been written yet (see ``SrasFile.angle_status``). +has been written yet (see ``SrasFile.angle_status``). Because the background +blocks are length-prefixed, the per-angle offsets come from a walk of that +stream at parse time rather than from arithmetic on the geometry table. -Version 10 is the SAW quality check (core.saw_check): byte layout identical -to v6, but every angle declares exactly one row — the row-wise middle of the +Version 11 is the SAW quality check (core.saw_check): byte layout identical +to v7, but every angle declares exactly one row — the row-wise middle of the ROI. The version byte is the whole difference, and it exists so a reader can tell a one-row-per-angle check from a full scan that was aborted after its first row. ``create_scan_file`` enforces the one-row rule at write time. + +v6 and v10 are the pre-per-angle-background versions of the same two files. +They are still read (every scan taken before this change is one); nothing +writes them any more, and a v6 file cannot be resumed into, since appending +v7 blocks to it would shift its data. """ from __future__ import annotations @@ -37,14 +51,21 @@ import numpy as np from core.scan_geometry import AngleGeometry, ScanPlan MAGIC = b"SRAS" -VERSION = 6 +VERSION = 7 # One row per angle, taken from the middle of the ROI — see core.saw_check. -VERSION_SAW_CHECK = 10 -SUPPORTED_VERSIONS = (VERSION, VERSION_SAW_CHECK) +VERSION_SAW_CHECK = 11 +# v6/v10: the same two files with one background for the whole scan, written +# ahead of the data block instead of once per angle. Read-only. +LEGACY_VERSIONS = (6, 10) +WRITABLE_VERSIONS = (VERSION, VERSION_SAW_CHECK) +SUPPORTED_VERSIONS = tuple(sorted(WRITABLE_VERSIONS + LEGACY_VERSIONS)) +SAW_CHECK_VERSIONS = (10, VERSION_SAW_CHECK) HDR_FMT = ">4sBHfffffffIdBB" HDR_SIZE = struct.calcsize(HDR_FMT) # 49 bytes GEOM_FMT = ">ffIH" GEOM_SIZE = struct.calcsize(GEOM_FMT) # 14 bytes +BG_LEN_FMT = ">I" +BG_LEN_SIZE = struct.calcsize(BG_LEN_FMT) # 4 bytes # Oscilloscope channels recorded, in on-disk order. SCAN_CHANNELS = [1, 3, 4] @@ -78,7 +99,8 @@ class AngleStatus: angle_deg: float n_rows: int # declared row_bytes: int - data_offset: int + bg_offset: int # start of this angle's background block + data_offset: int # start of its rows, i.e. just past that block n_rows_available: int status: str # STATUS_OK / STATUS_TRUNCATED / STATUS_MISSING @@ -86,33 +108,38 @@ class AngleStatus: def complete(self) -> bool: return self.status == STATUS_OK + @property + def bg_bytes(self) -> int: + """Size of the background block ahead of the rows (0 on v6/v10).""" + return self.data_offset - self.bg_offset + def create_scan_file(path: Path, plan: ScanPlan, samples_per_frame: int, sample_rate: float, preambles: list[str], - background_waveform: bytes, version: int = VERSION) -> BinaryIO: """Create a new .sras file and write the header + tables. ``version`` selects which kind of file this is — VERSION for a full scan, VERSION_SAW_CHECK for a middle-row quality check. The layout is the same - either way; the one-row-per-angle rule that gives v10 its meaning is - checked here, since nothing downstream can recover from a v10 file that + either way; the one-row-per-angle rule that gives v11 its meaning is + checked here, since nothing downstream can recover from a v11 file that breaks it. - Returns an open binary file positioned at the start of the data block; - the caller appends waveform rows and must close it (try/finally). + Returns an open binary file positioned at the start of the data block. + The caller writes each angle as ``write_background_block()`` followed by + that angle's rows, and must close the file (try/finally). """ - if version not in SUPPORTED_VERSIONS: + if version not in WRITABLE_VERSIONS: raise ValueError( f"Cannot write SRAS format version {version} " - f"(supported: {', '.join(str(v) for v in SUPPORTED_VERSIONS)})" + f"(writable: {', '.join(str(v) for v in WRITABLE_VERSIONS)})" ) if version == VERSION_SAW_CHECK: bad = [f"{pa.angle_deg:.1f}° has {pa.n_rows}" for pa in plan.per_angle if pa.n_rows != 1] if bad: raise ValueError( - "A v10 SAW-check file holds exactly one row per angle, but " + "A v11 SAW-check file holds exactly one row per angle, but " + ", ".join(bad) + " — build the plan with " "core.saw_check.middle_row_plan()." ) @@ -140,19 +167,33 @@ def create_scan_file(path: Path, plan: ScanPlan, samples_per_frame: int, enc = p.encode("utf-8") f.write(struct.pack(">H", len(enc))) f.write(enc) - f.write(struct.pack(">I", len(background_waveform))) - f.write(background_waveform) return f +def write_background_block(f: BinaryIO, waveform: bytes) -> int: + """Write one angle's background block; returns the bytes written. + + Every angle's rows are preceded by one of these, so a reader walking the + data block knows where that angle's frames start. + """ + f.write(struct.pack(BG_LEN_FMT, len(waveform))) + f.write(waveform) + return BG_LEN_SIZE + len(waveform) + + @dataclass class SrasFile: - """Parsed .sras file (v6 or v10): header, tables, and lazy (memmap) access. + """Parsed .sras file (v7/v11, or legacy v6/v10): header, tables, and lazy + (memmap) access. - Parsing reads only the header/tables — never the waveform block — so - opening a multi-GB file is cheap. ``load_angle``/``load_row`` return - read-only numpy views backed by a shared mmap; no data is copied until - the caller computes on it. + Parsing reads only the header/tables and the per-angle background blocks + — never the waveform block — so opening a multi-GB file is cheap. + ``load_angle``/``load_row`` return read-only numpy views backed by a + shared mmap; no data is copied until the caller computes on it. + + ``backgrounds[i]`` is angle *i*'s own background (v7/v11) or the file's + single background repeated for every angle (v6/v10), so a reader never + has to branch on the version to subtract the right one. """ path: Path version: int = field(init=False) @@ -160,7 +201,7 @@ class SrasFile: per_angle: list[AngleGeometry] = field(init=False) preambles: list[str] = field(init=False) preambles_raw: list[bytes] = field(init=False) - background: bytes = field(init=False) + backgrounds: list[bytes] = field(init=False) data_start_offset: int = field(init=False) file_size: int = field(init=False) @@ -215,15 +256,65 @@ class SrasFile: self.preambles_raw.append(f.read(plen)) self.preambles = [p.decode("utf-8", errors="replace") for p in self.preambles_raw] - (n_bg,) = struct.unpack(">I", f.read(4)) - self.background = f.read(n_bg) + shared_bg = None + if self.is_legacy_layout: + # v6/v10: one background for the whole file, ahead of the data. + (n_bg,) = struct.unpack(BG_LEN_FMT, f.read(BG_LEN_SIZE)) + shared_bg = f.read(n_bg) self.data_start_offset = f.tell() + self._walk_data_block(f, shared_bg) + + def _walk_data_block(self, f, shared_bg: bytes | None): + """Locate every angle's background block and the rows behind it. + + v7 interleaves a length-prefixed background ahead of each angle's + rows, so the offsets are no longer pure arithmetic over the geometry + table — the walk reads each prefix as it goes. Past the frontier of + a partial file there is nothing to read, so the remaining offsets are + predicted from the block a writer would have produced (a full record), + which is exactly where a resumed scan writes. + """ + expected_bg = BG_LEN_SIZE + self.header.samples_per_frame + self.backgrounds, self._bg_present = [], [] + self._bg_offsets, self._data_offsets = [], [] + cursor = self.data_start_offset + for ai, pa in enumerate(self.per_angle): + if shared_bg is not None: + bg, bg_bytes, present = shared_bg, 0, True + else: + bg, bg_bytes, present = self._read_background(f, cursor, expected_bg) + self.backgrounds.append(bg) + self._bg_present.append(present) + self._bg_offsets.append(cursor) + cursor += bg_bytes + self._data_offsets.append(cursor) + cursor += self.row_bytes(ai) * pa.n_rows + + def _read_background(self, f, offset: int, expected_bytes: int): + """One angle's background block as (waveform, block_bytes, present). + + A block that runs past the end of the file was never written: the + walk keeps going with the size a writer would have used, and the + angle is reported MISSING. + """ + if offset + BG_LEN_SIZE > self.file_size: + return b"", expected_bytes, False + f.seek(offset) + (n_bg,) = struct.unpack(BG_LEN_FMT, f.read(BG_LEN_SIZE)) + if offset + BG_LEN_SIZE + n_bg > self.file_size: + return b"", expected_bytes, False + return f.read(n_bg), BG_LEN_SIZE + n_bg, True @property def is_saw_check(self) -> bool: - """True for a v10 middle-row SAW quality check rather than a scan.""" - return self.version == VERSION_SAW_CHECK + """True for a middle-row SAW quality check rather than a scan.""" + return self.version in SAW_CHECK_VERSIONS + + @property + def is_legacy_layout(self) -> bool: + """True for v6/v10: one background for the file, not one per angle.""" + return self.version in LEGACY_VERSIONS # ── Frontier / truncation analysis ─────────────────────────────────────── @@ -237,40 +328,48 @@ class SrasFile: Because the data is one contiguous append-only stream, once an angle is found short every later angle is necessarily absent too — there is - a single frontier past which nothing has been written yet. + a single frontier past which nothing has been written yet. An angle + whose background block never made it to disk is short by definition, + even though no row of it was due yet. """ statuses = [] - cursor = self.data_start_offset frontier_seen = False for ai, pa in enumerate(self.per_angle): row_bytes = self.row_bytes(ai) - data_offset = cursor - if frontier_seen: + data_offset = self._data_offsets[ai] + if frontier_seen or not self._bg_present[ai]: n_rows_available = 0 status = STATUS_MISSING + frontier_seen = True else: declared_bytes = row_bytes * pa.n_rows - if row_bytes > 0 and cursor + declared_bytes <= self.file_size: + if row_bytes > 0 and data_offset + declared_bytes <= self.file_size: n_rows_available = pa.n_rows status = STATUS_OK - cursor += declared_bytes else: - remaining = max(0, self.file_size - cursor) + remaining = max(0, self.file_size - data_offset) n_rows_available = remaining // row_bytes if row_bytes > 0 else 0 status = STATUS_MISSING if n_rows_available == 0 else STATUS_TRUNCATED frontier_seen = True statuses.append(AngleStatus( index=ai, angle_deg=pa.angle_deg, n_rows=pa.n_rows, - row_bytes=row_bytes, data_offset=data_offset, + row_bytes=row_bytes, bg_offset=self._bg_offsets[ai], + data_offset=data_offset, n_rows_available=n_rows_available, status=status, )) return statuses def angle_data_offset(self, angle_idx: int) -> int: - offset = self.data_start_offset - for ai in range(angle_idx): - offset += self.row_bytes(ai) * self.per_angle[ai].n_rows - return offset + """Where angle ``angle_idx``'s rows start (past its background).""" + return self._data_offsets[angle_idx] + + def angle_block_offset(self, angle_idx: int) -> int: + """Where angle ``angle_idx``'s block starts, background included. + + Equal to ``angle_data_offset`` on v6/v10, which have no per-angle + background block. + """ + return self._bg_offsets[angle_idx] # ── Lazy data access ───────────────────────────────────────────────────── @@ -286,6 +385,11 @@ class SrasFile: def _dtype(self) -> np.dtype: return np.dtype(np.int16 if self.header.bytes_per_sample == 2 else np.int8) + def background_array(self, angle_idx: int) -> np.ndarray | None: + """One angle's background as float32 ADC counts, or None if absent.""" + bg = np.frombuffer(self.backgrounds[angle_idx], dtype=self._dtype()) + return bg.astype(np.float32) if bg.size else None + def load_angle(self, angle_idx: int, n_rows: int | None = None) -> np.ndarray: """Read-only view of one angle's data block, shape (n_rows, n_channels, n_frames, samples_per_frame). diff --git a/gui/scan_bridge.py b/gui/scan_bridge.py index 950bec7..d11f8b0 100644 --- a/gui/scan_bridge.py +++ b/gui/scan_bridge.py @@ -12,6 +12,7 @@ import traceback from PyQt6.QtCore import QObject, pyqtSignal, pyqtSlot from core.scan_engine import ScanAborted, ScanCallbacks, ScanEngine +from core.sras_format import VERSION class QtScanController(QObject): @@ -33,7 +34,7 @@ class QtScanController(QObject): def __init__(self, stage, scope, rotator, plan, out_path, resume=None, on_scan_active=None, burst_mode=False, - strict_rows=False): + strict_rows=False, file_version=VERSION): super().__init__() self._prompt_event = threading.Event() self._on_scan_active = on_scan_active @@ -50,7 +51,8 @@ class QtScanController(QObject): self._engine = ScanEngine(stage, scope, rotator, plan, out_path, resume=resume, callbacks=callbacks, burst_mode=burst_mode, - strict_rows=strict_rows) + strict_rows=strict_rows, + file_version=file_version) # ── Engine control (called from the GUI thread) ─────────────────────────── diff --git a/saw_check_viewer.py b/saw_check_viewer.py index 1994894..bf07b37 100755 --- a/saw_check_viewer.py +++ b/saw_check_viewer.py @@ -2,7 +2,7 @@ """ SAW Check Viewer — every angle's frequency on one graph. -Opens a v10 middle-row SAW check (written by the main app's "SAW Quality +Opens a middle-row SAW check (written by the main app's "SAW Quality Check") and plots the peak SAW frequency along each angle's row, all angles on the same axes. ``core.saw_check`` explains why that answers an alignment question: every angle's middle row crosses the same ROI centre, so the angles @@ -17,7 +17,7 @@ Two readings share the window: * the summary — each angle's median with ±1σ, plotted against angle, plus the same numbers per angle in a table. -A full v6 scan opens too: the same middle row is pulled out of it, so a scan +A full scan opens too: the same middle row is pulled out of it, so a scan can be re-examined with the check's own read-out after the fact. """ @@ -91,14 +91,17 @@ class LoadedCheck: def __init__(self, path: Path): self.sras = SrasFile(path) self.calib = ChannelCalibration.from_preambles(self.sras.preambles) - bg = np.frombuffer(self.sras.background, dtype=np.int8) - self.background = bg.astype(np.float32) if len(bg) else None + # Each angle carries its own background (v7/v11); the read-out pulls + # the right one per angle, so the viewer only needs to know whether + # there is anything to subtract at all. + self.has_background = any(self.sras.background_array(ai) is not None + for ai in range(self.sras.header.n_angles)) self.traces = [] self.summary = None def describe(self) -> str: h = self.sras.header - kind = ("v10 SAW check" if self.sras.is_saw_check + kind = (f"v{self.sras.version} SAW check" if self.sras.is_saw_check else f"v{self.sras.version} scan — middle row of each angle") return (f"{self.sras.path.name}\n{kind}\n" f"{h.n_angles} angle(s) · {h.samples_per_frame} samples/frame · " @@ -468,10 +471,11 @@ class SawCheckWindow(QMainWindow): self._check = check self.setWindowTitle(f"SAW Check Viewer — {path.name}") self.lbl_file.setText(check.describe()) + self.chk_bg_sub.setEnabled(check.has_background) if not check.sras.is_saw_check: self.lbl_status.setText( - "Not a v10 check — reading the middle row of each angle " + "Not a SAW check file — reading the middle row of each angle " "out of this scan instead.") else: self.lbl_status.setText("") @@ -504,7 +508,7 @@ class SawCheckWindow(QMainWindow): gated = self.chk_gate.isChecked() kwargs = dict( dc_threshold_mv=self.spin_threshold_mv.value(), - background=check.background if self.chk_bg_sub.isChecked() else None, + subtract_background=self.chk_bg_sub.isChecked(), gate_start_ns=self.spin_gate_start.value() if gated else None, gate_end_ns=self.spin_gate_end.value() if gated else None, calib=check.calib, diff --git a/sc3_aui_app.py b/sc3_aui_app.py index c96c524..787911c 100755 --- a/sc3_aui_app.py +++ b/sc3_aui_app.py @@ -38,7 +38,8 @@ from core.scan_geometry import ScanPlan, EtaEstimator, build_plan, format_eta from core.scan_resume import is_compatible, plan_resume from core.scope_sras import SAMPLE_RATE_HZ, configure_channels from core.sras_format import ( - SCAN_CHANNELS, VERSION, VERSION_SAW_CHECK, SrasFile, plan_from_header, + SCAN_CHANNELS, VERSION, VERSION_SAW_CHECK, WRITABLE_VERSIONS, SrasFile, + plan_from_header, ) from gui.jog_panel import ( BBD_JOG_ACCEL_MM_S2, BBD_JOG_SPEED_MM_S, BBD_JOG_STEP_MM, @@ -1369,7 +1370,9 @@ class MainWindow(QMainWindow): self, "SAW Quality Check", f"Acquire the middle row of the ROI at {check_plan.n_angles} angle(s)?\n\n" f"{rows}\n\n" - f"Save → {out_path.name}{overwrite}", + "Each angle starts with its own background capture, so you will be " + "asked to switch the Genesis laser off and back on at every angle." + f"\n\nSave → {out_path.name}{overwrite}", QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, ) if reply != QMessageBox.StandardButton.Yes: @@ -1519,6 +1522,18 @@ class MainWindow(QMainWindow): QMessageBox.warning(self, "Cannot Resume Scan", f"Could not read scan file:\n\n{e}") return + if sras.version not in WRITABLE_VERSIONS: + QMessageBox.warning( + self, "Cannot Resume Scan", + f"{path.name} is a v{sras.version} file, written before each " + "angle carried its own background waveform. Every angle this " + "app acquires now writes a background block that the older " + "layout has no room for, so resuming would shift the file's " + "data. Start a new scan instead — the old file still opens in " + "the viewer." + ) + return + if not is_compatible(sras, velocity=SCAN_VELOCITY_MM_S, laser_freq=LASER_FREQ_HZ, sample_rate=SAMPLE_RATE_HZ, n_channels=len(SCAN_CHANNELS)): diff --git a/scan_format.md b/scan_format.md index 8c7e2d2..dae6052 100755 --- a/scan_format.md +++ b/scan_format.md @@ -1,17 +1,24 @@ -# SRAS Scan Binary Format — Versions 6 and 10 +# SRAS Scan Binary Format — Versions 7 and 11 (reading 6 and 10) Each `.sras` file contains **one complete scan**: all GR rotation angles and all Y rows. Files are named `{prefix}.sras`. -Two versions share this layout byte for byte — only the version field differs, -and with it what the file means: +Two versions are written, sharing this layout byte for byte — only the version +field differs, and with it what the file means: | Version | Meaning | Rows per angle | |---------|---------|----------------| -| 6 | A full scan. | Whatever the ROI needs. | -| 10 | A middle-row SAW quality check (`{prefix}-sawcheck.sras`). | Exactly 1. | +| 7 | A full scan. | Whatever the ROI needs. | +| 11 | A middle-row SAW quality check (`{prefix}-sawcheck.sras`). | Exactly 1. | -See [SAW Quality Check (v10)](#saw-quality-check-v10) below. +See [SAW Quality Check (v11)](#saw-quality-check-v11) below. + +**Versions 6 and 10** are the same two files as they were written before each +angle carried its own background: a single background block sat between the +preambles and the data block, and the data block held nothing but rows. They +are still read — see [Legacy layout (v6/v10)](#legacy-layout-v6v10) — but +nothing writes them any more, and a v6 file cannot be resumed into, since the +background block a resumed angle writes has no room in it. Starting in v6, each angle only scans the **bounding box of the nominal ROI rotated by that specific angle** — not the worst case across all angles — so @@ -30,10 +37,14 @@ instead of forcing every angle to the largest bounding box. [Per-Angle Geometry Table— n_angles × 14 bytes (x_start f32, x_delta f32, n_frames u32, n_rows u16)] [Row Table (ragged) — sum(n_rows) × 4 bytes (float32 per row, angle-major)] [Preamble Blocks — n_channels × (uint16 length + UTF-8 WFMOutpre string)] -[Background Block — uint32 n_bg_samples + n_bg_samples × int8 bytes] -[Waveform Data (ragged) — per angle: n_rows[a] × n_channels × n_frames[a] × samples_per_frame × bps bytes] +[Data Block (ragged) — per angle: [Background Block][Waveform Data]] + [Background Block — uint32 n_bg_samples + n_bg_samples × int8 bytes] + [Waveform Data — n_rows[a] × n_channels × n_frames[a] × samples_per_frame × bps bytes] ``` +So the data block reads `[background][scan][background][scan] …`, one pair per +angle, in angle-table order. + All multi-byte integers and floats use **big-endian** byte order (`>` in Python's `struct` module). @@ -44,7 +55,7 @@ All multi-byte integers and floats use **big-endian** byte order | Offset | Size | Type | Field | Description | |--------|------|-----------|--------------------|--------------------------------------------------| | 0 | 4 | `4s` | `magic` | Always `SRAS` (0x53 0x52 0x41 0x53) | -| 4 | 1 | `uint8` | `version` | Format version — `6` (scan) or `10` (SAW check) | +| 4 | 1 | `uint8` | `version` | Format version — `7` (scan) or `11` (SAW check) | | 5 | 2 | `uint16` | `n_angles` | Number of GR rotation angles | | 7 | 4 | `float32` | `x_start_nominal` | Nominal (pre-rotation) X scan start, mm | | 11 | 4 | `float32` | `y_start_nominal` | Nominal (pre-rotation) Y scan start, mm | @@ -127,54 +138,77 @@ to convert raw ADC values to volts. --- -## Background Block +## Data Block (ragged) -Immediately after the preamble blocks: a single CH1 waveform captured with the -**Helios (generation) laser enabled** and the **Genesis (detection) laser -disabled**. This provides a noise/background reference for subtraction during -post-processing. +Immediately after the preamble blocks, and running to the end of the file: +for each angle in angle-table order, that angle's **background block** +followed by that angle's **waveform data**. + +``` +for angle a in 0 … n_angles-1: + uint32 n_bg_samples # background block + int8[] bg_data + for row in 0 … n_rows[a]-1: # waveform data + for channel in [CH1, CH3, CH4]: # 3 channels, fixed order + for frame in 0 … n_frames[a]-1: + samples[0 … samples_per_frame-1] # bps bytes each +``` + +### Background block + +One CH1 waveform captured with the **Helios (generation) laser enabled** and +the **Genesis (detection) laser disabled**, averaged over 1024 shots +(`core/scope_sras.py`, `BACKGROUND_AVERAGES`). It is a noise/background +reference for subtraction during post-processing. ``` uint32 n_bg_samples — number of samples in the background waveform int8[] bg_data — raw ADC samples (same encoding as waveform data) ``` -`n_bg_samples` equals `samples_per_frame` under normal acquisition settings. +`n_bg_samples` equals `samples_per_frame` under normal acquisition settings, +but is **not** assumed to: readers take the per-angle offsets from a walk of +the data block, reading each length prefix as they go, rather than from +arithmetic over the geometry table alone. ---- +Every angle carries its own. The operator is prompted to switch the Genesis +laser off before each angle and back on after the capture, so the reference is +taken minutes from the data it will be subtracted from — a multi-angle scan +runs for hours, and one background captured at the first angle has drifted by +the last. It also makes the angles comparable, which is the entire point of a +multi-angle scan: each is referenced against its own noise floor rather than +against whichever angle happened to be scanned first. -## Waveform Data (ragged) +### Waveform data -Immediately after the background block. Data is stored in **angle-major, -row-minor** order, but unlike earlier versions each angle contributes a +Stored in **angle-major, row-minor** order, and each angle contributes a different number of rows (`n_rows[a]`) and a different number of frames per row (`n_frames[a]`), both taken from that angle's Per-Angle Geometry Table entry. Within each row, channels are interleaved in ascending channel-index order, with each channel's FastFrame data written in frame order. -``` -for angle a in 0 … n_angles-1: - for row in 0 … n_rows[a]-1: - for channel in [CH1, CH3, CH4]: # 3 channels, fixed order - for frame in 0 … n_frames[a]-1: - samples[0 … samples_per_frame-1] # bps bytes each -``` - Each sample is a raw signed ADC value. With `bytes_per_sample = 1` this is **int8** (−128 … +127). With `bytes_per_sample = 2` this is **big-endian int16**. -Total data size: +Total data-block size: ``` -sum over angles a of: n_rows[a] × 3 × n_frames[a] × samples_per_frame × bytes_per_sample +sum over angles a of: 4 + n_bg_samples[a] + + n_rows[a] × 3 × n_frames[a] × samples_per_frame × bytes_per_sample ``` > **Incomplete files:** If a scan is aborted the file is closed immediately and -> the data block will be shorter than the expected size. Readers should -> reconstruct the expected per-angle byte offsets from the Per-Angle Geometry -> Table and check `file_size` against the running total before reshaping — -> a fixed `(n_angles, n_rows, ...)` reshape (as in pre-v6 readers) will not -> work since row/frame counts are no longer uniform across angles. +> the data block will be shorter than the expected size. Readers should walk +> the data block from its start — background length prefix, then that angle's +> declared row bytes from the Per-Angle Geometry Table — checking `file_size` +> against the running total before reshaping. A fixed +> `(n_angles, n_rows, ...)` reshape (as in pre-v6 readers) will not work since +> row/frame counts are no longer uniform across angles. +> +> An angle whose background block is not fully on disk has nothing of itself +> written yet: it is *missing*, not truncated, and the walk continues past it +> assuming the block a writer would have produced +> (`4 + samples_per_frame` bytes), which is where a resumed scan writes. --- @@ -197,8 +231,9 @@ using that angle's `x_start` from the Per-Angle Geometry Table (not | Parameter | Value | |-----------------------|------------------------------------------| -| Setup trigger | CH2, rising edge, 0.500 V (`TRIG_LEVEL_V`) | +| Background trigger | CH2, rising edge, 0.500 V (`TRIG_LEVEL_V`), FastFrame off | | Scan trigger | Logic AND, CH2 HIGH ∧ CH3 HIGH, 0.500 V | +| Background average | 1024 shots (`BACKGROUND_AVERAGES`), once per angle | | Horizontal position | 30 (`HORizontal:POSition`) | | Sample rate | 6.25 GS/s (160 ps/sample) | | Transfer format | `DATa:ENCdg RIBinary`, `DATa:WIDth 1` | @@ -226,6 +261,10 @@ is not recorded in the file. | Curve transfers | one per channel per row | one per channel per burst | | Stage X trigger out | armed for the whole scan | armed per acquiring pass, dropped for the flyback | +Both paths take the same per-angle background: the scope returns to the +single-record edge trigger for the capture and back to the logic-AND trigger +before the angle's rows, so the two paths still produce byte-identical files. + Burst mode runs a single acquisition across several rows, so the return move must not trigger: the trigger output is dropped before each flyback and re-armed for each acquiring pass. Row boundaries inside the burst come from @@ -252,21 +291,22 @@ the file always ends on a whole-row boundary. --- -## SAW Quality Check (v10) +## SAW Quality Check (v11) A full multi-angle scan takes hours, and a rig whose angles disagree produces all of them before anyone finds out. The SAW quality check acquires **one row -per angle — the row-wise middle of the ROI** — and writes it as a v10 file. +per angle — the row-wise middle of the ROI** — and writes it as a v11 file. The cost is one row-time per angle instead of `n_rows` of them. -Nothing about the byte layout changes. A v10 file is a v6 file in which every +Nothing about the byte layout changes. A v11 file is a v7 file in which every angle's Per-Angle Geometry Table entry declares `n_rows = 1`, and its Row Table -holds that angle's single middle Y position. Every v6 reader that works from -the geometry table (rather than assuming a uniform shape) reads a v10 file -unchanged. +holds that angle's single middle Y position. Every v7 reader that works from +the geometry table (rather than assuming a uniform shape) reads a v11 file +unchanged. Each angle still carries its own background, so a check costs the +same two operator prompts per angle a scan does. The version byte earns its keep because the two are otherwise -indistinguishable: **a v6 scan aborted after its first row is not a check**, +indistinguishable: **a v7 scan aborted after its first row is not a check**, even though both hold one row per angle. A reader that guessed from the row count would treat a failed scan as a deliberate measurement. @@ -278,12 +318,33 @@ of the rig — which is what makes it an alignment check. `saw_check_viewer.py` plots every angle's frequency on one graph for exactly that comparison. Writers must honour the one-row rule; `core.sras_format.create_scan_file` -refuses a v10 write for any plan that breaks it. Producing the plan is +refuses a v11 write for any plan that breaks it. Producing the plan is `core.saw_check.middle_row_plan(plan)`, and `n_rows // 2` is the middle-row rule (the upper of the two central rows when the count is even). --- +## Legacy layout (v6/v10) + +A v6 or v10 file differs in one place: the background block sits **once**, +between the preamble blocks and the data block, and the data block is +waveform data alone. + +``` +[Preamble Blocks] +[Background Block — uint32 n_bg_samples + n_bg_samples × int8 bytes] +[Waveform Data (ragged) — per angle: rows, as above, with no background between] +``` + +Everything else — header, tables, row order, spatial mapping — is identical, +which is why `core.sras_format.SrasFile` reads both: it hands the one legacy +background to every angle, so a reader that asks for angle *a*'s background +never has to know which layout it is looking at. Nothing writes v6/v10 any +more, and a resume refuses them, because a re-acquired angle writes a +background block the layout has no room for. + +--- + ## Version History | Version | Change | @@ -294,5 +355,7 @@ rule (the upper of the two central rows when the count is even). | 4 | Added background waveform block (CH1, Helios ON / Genesis OFF) after the preamble blocks; stored as `uint32` sample count followed by raw `int8` ADC bytes. | | 5 | (skipped) | | 6 | Each angle now scans only the bounding box of the nominal ROI rotated by that angle instead of the AABB-expanded worst case across all angles. Header no longer carries a single global `x_start`/`x_delta`/`n_rows` — replaced with `*_nominal` reference fields plus a new Per-Angle Geometry Table (`x_start`, `x_delta`, `n_frames`, `n_rows` per angle) and a ragged Row Table / Waveform Data block sized per angle. **Not compatible with v4 readers** (e.g. `sras_viewer.py`, which has not yet been updated for v6). | -| 7–9 | (skipped) | -| 10 | Middle-row SAW quality check. Byte layout identical to v6, with every angle declaring exactly one row — the row-wise middle of the ROI. A v6 reader that derives its shape from the Per-Angle Geometry Table reads these unchanged; the version byte exists so a check is not confused with a scan aborted after its first row. Written by the main app's *SAW Quality Check*, read by `saw_check_viewer.py`. | +| 7 | Background moved into the data block, one per angle: the block now reads `[background][scan][background][scan] …`. Each angle is preceded by its own `uint32` + `int8[]` background, captured (Genesis off, Helios on) just before that angle is scanned, so the reference is contemporary with the data and the angles are comparable to each other. Per-angle offsets therefore come from a walk of the data block rather than arithmetic over the geometry table. **v6 files still read; v6 files cannot be resumed into.** | +| 8–9 | (skipped) | +| 10 | Middle-row SAW quality check on the v6 layout. Byte layout identical to v6, with every angle declaring exactly one row — the row-wise middle of the ROI. Superseded by v11; still read. | +| 11 | Middle-row SAW quality check on the v7 layout: identical to v7 with every angle declaring exactly one row, per-angle backgrounds included. The version byte exists so a check is not confused with a scan aborted after its first row. Written by the main app's *SAW Quality Check*, read by `saw_check_viewer.py`. | diff --git a/sras_scan_manager.py b/sras_scan_manager.py index 901c947..d75bc5b 100755 --- a/sras_scan_manager.py +++ b/sras_scan_manager.py @@ -1,20 +1,22 @@ #!/opt/srasenv/bin/python3 """ SRAS Scan Manager -Command-line / interactive TUI for inspecting v6 .sras files. +Command-line / interactive TUI for inspecting .sras files. A .sras file (see scan_format.md) holds one acquisition run across several GR rotation angles, each with its own geometry (x_start, x_delta, n_frames, -n_rows) and waveform data block. This tool lists those per-angle sub-scans -and lets you export a subset to a new .sras file, or delete a subset from -the file in place — both operations rewrite the angle/geometry/row tables -and stream-copy only the selected angles' waveform data, producing a file -that is itself a valid .sras readable by sras_viewer.py-style tools -(once updated for v6) or sc3_aui_app.py. +n_rows), background waveform, and waveform data block. This tool lists those +per-angle sub-scans and lets you export a subset to a new .sras file, or +delete a subset from the file in place — both operations rewrite the +angle/geometry/row tables, carry each kept angle's background across, and +stream-copy only the selected angles' waveform data, producing a file that is +itself a valid .sras readable by sras_viewer.py or sc3_aui_app.py. -Format versions 6 (full scan) and 10 (middle-row SAW check) are supported. -A subset keeps the version of the file it came from — a v10 check exports as -a v10 check, since dropping angles from one leaves it one row per angle. +Format versions 7 (full scan) and 11 (middle-row SAW check) are supported, +as are their pre-per-angle-background predecessors 6 and 10. A subset keeps +the version — and therefore the background layout — of the file it came +from: a v11 check exports as a v11 check, since dropping angles from one +leaves it one row per angle. """ import argparse @@ -26,7 +28,9 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from core.sras_format import GEOM_FMT, HDR_FMT, MAGIC, VERSION_SAW_CHECK, SrasFile +from core.sras_format import ( + BG_LEN_FMT, GEOM_FMT, HDR_FMT, MAGIC, SrasFile, +) @dataclass @@ -39,7 +43,8 @@ class AngleEntry: n_rows_declared: int y_positions: list # declared length; may exceed what's actually on disk row_bytes: int - data_offset: int # byte offset into the file where this angle's data starts + background: bytes # this angle's own background (v7/v11) + data_offset: int # byte offset into the file where this angle's rows start n_rows_available: int = 0 data_size_available: int = 0 complete: bool = True @@ -60,6 +65,7 @@ class SrasScanFile: sras = SrasFile(self.path) h = sras.header self.version = sras.version + self.is_saw_check = sras.is_saw_check self.x_start_nominal = h.x_start_nominal self.y_start_nominal = h.y_start_nominal self.x_delta_nominal = h.x_delta_nominal @@ -72,7 +78,10 @@ class SrasScanFile: self.bytes_per_sample = h.bytes_per_sample self.n_channels = h.n_channels self.preambles_raw = sras.preambles_raw - self.background_raw = sras.background + self.legacy_layout = sras.is_legacy_layout + # v6/v10 keep one background ahead of the data block; v7/v11 keep one + # per angle inside it. Either way sras.backgrounds is per angle. + self.shared_background = sras.backgrounds[0] if sras.is_legacy_layout else b"" self.data_start_offset = sras.data_start_offset self.file_size = sras.file_size @@ -82,12 +91,13 @@ class SrasScanFile: x_start=pa.x_start, x_delta=pa.x_delta, n_frames=pa.n_frames, n_rows_declared=pa.n_rows, y_positions=pa.y_positions, row_bytes=st.row_bytes, - data_offset=st.data_offset, + background=bg, data_offset=st.data_offset, n_rows_available=st.n_rows_available, data_size_available=st.n_rows_available * st.row_bytes, complete=st.complete, ) - for pa, st in zip(sras.per_angle, sras.angle_status(), strict=True) + for pa, st, bg in zip(sras.per_angle, sras.angle_status(), + sras.backgrounds, strict=True) ] def get(self, index: int) -> AngleEntry: @@ -137,10 +147,18 @@ def _write_subset(sf: SrasScanFile, indices: list, dst_path: Path) -> list: dst.write(struct.pack(">H", len(praw))) dst.write(praw) - dst.write(struct.pack(">I", len(sf.background_raw))) - dst.write(sf.background_raw) + if sf.legacy_layout: + dst.write(struct.pack(BG_LEN_FMT, len(sf.shared_background))) + dst.write(sf.shared_background) for e in selected: + if not sf.legacy_layout: + if not e.background: + warnings.append( + f"angle[{e.index}] ({e.angle_deg:.2f} deg): no background " + "on disk — exported with an empty background block") + dst.write(struct.pack(BG_LEN_FMT, len(e.background))) + dst.write(e.background) src.seek(e.data_offset) remaining = e.data_size_available chunk_size = 1 << 20 @@ -224,7 +242,7 @@ def parse_index_spec(spec: str, max_index: int) -> list: def print_summary(sf: SrasScanFile, selected: set): print() - kind = " SAW check" if sf.version == VERSION_SAW_CHECK else "" + kind = " SAW check" if sf.is_saw_check else "" print(f"File: {sf.path} (v{sf.version}{kind}, {_human_size(sf.file_size)})") print(f"Nominal ROI: x_start={sf.x_start_nominal:.4f} x_delta={sf.x_delta_nominal:.4f} " f"y_start={sf.y_start_nominal:.4f} y_delta={sf.y_delta_nominal:.4f} mm " @@ -346,7 +364,7 @@ def interactive_loop(path: Path): def main(): ap = argparse.ArgumentParser( - description="Inspect, export, or delete per-angle sub-scans in a v6 .sras file.") + description="Inspect, export, or delete per-angle sub-scans in a .sras file.") ap.add_argument("file", type=Path, help="path to a .sras file") ap.add_argument("--list", action="store_true", help="print the angle table and exit") ap.add_argument("--export", metavar="SPEC", help="angle index spec to export, e.g. '0,2,4-6' or 'all'") diff --git a/sras_viewer.py b/sras_viewer.py index a16fb6c..e07c546 100755 --- a/sras_viewer.py +++ b/sras_viewer.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ SRAS Scan File Viewer -PyQt6 application for visualizing channel data from v6 .sras scan files. +PyQt6 application for visualizing channel data from .sras scan files. Channel semantics (fixed by sc3_aui_app.py acquisition settings): CH1 — RF Acoustic Packet: FFT → peak frequency @@ -76,11 +76,18 @@ class LoadedScan: def __init__(self, path: str): self.sras = SrasFile(Path(path)) self.calib = ChannelCalibration.from_preambles(self.sras.preambles) - bg = np.frombuffer(self.sras.background, dtype=np.int8) - self.background = bg.astype(np.float32) if len(bg) else None + # One background per angle (v7/v11), captured just before that angle + # was scanned. A legacy v6/v10 file has a single one, which SrasFile + # repeats for every angle, so nothing here branches on the version. + self.backgrounds = [self.sras.background_array(ai) + for ai in range(self.sras.header.n_angles)] # Rows actually on disk per angle (aborted/resumed scans) self.rows_available = [s.n_rows_available for s in self.sras.angle_status()] + def background(self, angle_idx: int) -> np.ndarray | None: + """That angle's background waveform, or None if it has none.""" + return self.backgrounds[angle_idx] + def angle_view(self, angle_idx: int) -> np.ndarray: """Available rows of one angle: (rows, n_ch, n_frames, spf) int8 view.""" return self.sras.load_angle(angle_idx, n_rows=self.rows_available[angle_idx]) @@ -123,7 +130,7 @@ def compute_image(scan: LoadedScan, angle_idx: int, ch_idx: int, if view.shape[0] == 0: return np.zeros((0, 0), dtype=np.float32) sras = scan.sras - bg = scan.background if apply_bg_sub else None + bg = scan.background(angle_idx) if apply_bg_sub else None if ch_idx in (CH1_IDX, VELOCITY_MODE_IDX): return compute_rf_image( @@ -232,7 +239,7 @@ class WaveformCanvas(FigureCanvasQTAgg): dc4_val = float(sras.load_row(angle_idx, row_idx, CH4_IDX)[frame_idx] .mean(dtype=np.float32)) - bg = scan.background if apply_bg_sub else None + bg = scan.background(angle_idx) if apply_bg_sub else None waveform_plot = waveform - bg if bg is not None else waveform self.ax_wave.cla() @@ -696,7 +703,7 @@ class SrasViewerWindow(QMainWindow): self.chk_bg_sub.setChecked(True) self.chk_bg_sub.setEnabled(False) self.chk_bg_sub.setToolTip( - "Subtract the stored background waveform from each CH1 frame\n" + "Subtract this angle's stored background waveform from each CH1 frame\n" "before computing the FFT." ) self.chk_bg_sub.toggled.connect(self._on_bg_sub_toggled) @@ -1043,7 +1050,7 @@ class SrasViewerWindow(QMainWindow): self._on_view_changed() def _update_angle_info(self): - """Per-angle info fields (v6 geometry is ragged across angles).""" + """Per-angle info fields (geometry is ragged across angles).""" scan = self._scan if scan is None: return @@ -1060,8 +1067,9 @@ class SrasViewerWindow(QMainWindow): if avail < pa.n_rows: notes.append(f"! Angle {ai}: only {avail}/{pa.n_rows} rows on disk " "(scan was aborted or is still running)") - if scan.background is not None: - notes.append(f"Background waveform: {len(scan.background)} samples") + bg = scan.background(ai) + if bg is not None: + notes.append(f"Background waveform (angle {ai}): {len(bg)} samples") self.lbl_frame_warn.setText("\n".join(notes)) # ------------------------------------------------------------------ @@ -1083,7 +1091,7 @@ class SrasViewerWindow(QMainWindow): is_ch1 = enabled and ch_idx in CH1_DERIVED_MODES is_fft = enabled and ch_idx in (CH1_IDX, VELOCITY_MODE_IDX) self.spin_threshold_mv.setEnabled(is_ch1) - has_bg = has_file and scan.background is not None + has_bg = has_file and scan.background(self.spin_angle.value()) is not None self.chk_bg_sub.setEnabled(has_bg and is_ch1) # Time gate only for FFT modes (the SAW pipeline has its own gating) self.chk_gate.setEnabled(is_fft) @@ -1197,6 +1205,9 @@ class SrasViewerWindow(QMainWindow): idx = self.spin_angle.value() self.lbl_angle_deg.setText(f"({self._scan.sras.per_angle[idx].angle_deg:.1f}°)") self._update_angle_info() + # Backgrounds are per angle, so whether there is one to subtract can + # change with the angle (a truncated file may be missing later ones). + self._update_controls_enabled(True) self._request_compute() # ------------------------------------------------------------------ @@ -1382,7 +1393,8 @@ class SrasViewerWindow(QMainWindow): angle_idx = self.spin_angle.value() n_shots_req = self.spin_saw_n_shots.value() row_sel, frame_sel = self._last_row, self._last_frame - apply_bg = scan.background is not None and self.chk_bg_sub.isChecked() + background = scan.background(angle_idx) + apply_bg = background is not None and self.chk_bg_sub.isChecked() if row_sel is not None and frame_sel is not None: src_desc = f"selected pixel (row={row_sel}, frame={frame_sel})" @@ -1405,7 +1417,7 @@ class SrasViewerWindow(QMainWindow): rows, frames = np.divmod(flat, n_frames) shots = view[rows, CH1_IDX, frames].astype(np.float32) if apply_bg: - shots -= scan.background + shots -= background pipeline.build_template(shots) return None diff --git a/tests/fakes.py b/tests/fakes.py index ffddcbe..62c4512 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -99,6 +99,16 @@ class FakeStage: round(at_speed_mm * LASER_FREQ_HZ / SCAN_VELOCITY_MM_S)) +def background_record(n: int, samples_per_frame: int) -> bytes: + """The waveform FakeScope returns from its n-th background capture. + + Every angle captures its own, so the tests need to tell one from the + next: a background that landed under the wrong angle would otherwise + look exactly like the right one. + """ + return bytes((n * 17 + s) % 256 for s in range(samples_per_frame)) + + class FakeScope: """Stands in for TektronixOscilloscopeBase. @@ -113,6 +123,7 @@ class FakeScope: self._acq_polls = 0 self._running = False self._acquired = 0 + self._backgrounds_taken = 0 # Per-channel running frame index. Frame content is a function of # (channel, index) alone, so the same total frame sequence yields the # same bytes however it is chopped into transfers. @@ -194,7 +205,9 @@ class FakeScope: def transfer_curve(self): self._t.record("transfer_curve") - return bytes(range(self.samples_per_frame)) + n = self._backgrounds_taken + self._backgrounds_taken += 1 + return background_record(n, self.samples_per_frame) def _frames(self, ch, count): spf = self.samples_per_frame diff --git a/tests/golden/sras_expected.json b/tests/golden/sras_expected.json index a8b397e..03d5aba 100644 --- a/tests/golden/sras_expected.json +++ b/tests/golden/sras_expected.json @@ -141,7 +141,7 @@ "angle_deg": -180.0, "n_rows": 3, "row_bytes": 96, - "data_offset": 265, + "data_offset": 553, "n_rows_available": 0, "status": "MISSING" } @@ -161,10 +161,10 @@ "angle_deg": -180.0, "n_rows": 3, "row_bytes": 96, - "data_offset": 265, + "data_offset": 553, "n_rows_available": 0, "status": "MISSING" } ] } -} \ No newline at end of file +} diff --git a/tests/golden_util.py b/tests/golden_util.py index f370b81..79841e0 100644 --- a/tests/golden_util.py +++ b/tests/golden_util.py @@ -1,9 +1,18 @@ -"""Shared constants for the golden .sras fixtures. +"""Shared constants and writers for the .sras test fixtures. -These mirror the values tests/gen_goldens.py used when the fixtures were -generated against the pre-refactor code (commit d185676); they must never -change, or the byte-identical comparisons stop meaning anything. +The constants mirror the values tests/gen_goldens.py used when the committed +golden files were generated against the pre-refactor code (commit d185676); +they must never change, or the comparisons against those files stop meaning +anything. Those goldens are legacy v6 files — the one background per file +layout — and are now read-only fixtures for the parser. + +``write_v7`` builds the current layout (one background per angle, inside the +data block) over the same geometry, for the tests that need a file this +version of the app could actually have written. """ +from core.scan_geometry import build_plan +from core.sras_format import VERSION, create_scan_file, write_background_block + SPF = 8 SAMPLE_RATE = 6.25e9 CHANNELS = [1, 3, 4] @@ -19,3 +28,31 @@ VELOCITY_MM_S = 100.0 def synthetic_frame(ai, ri, ci, fi): return bytes((ai * 7 + ri * 5 + ci * 3 + fi + s) % 256 for s in range(SPF)) + + +def tiny_plan(): + """The fixture geometry: 2 angles × 3 rows × 4 frames.""" + return build_plan(**TINY_PLAN_ARGS, laser_freq_hz=LASER_FREQ_HZ, + velocity_mm_s=VELOCITY_MM_S) + + +def angle_background(ai): + """A background that differs per angle, so tests can tell them apart.""" + return bytes((ai * 11 + s) % 256 for s in range(SPF)) + + +def write_v7(path, plan=None, version=None): + """Write a complete current-format file: [background][rows] per angle.""" + plan = plan if plan is not None else tiny_plan() + f = create_scan_file(path, plan, SPF, SAMPLE_RATE, PREAMBLES, + version=VERSION if version is None else version) + try: + for ai, pa in enumerate(plan.per_angle): + write_background_block(f, angle_background(ai)) + for ri in range(pa.n_rows): + for ci in range(len(CHANNELS)): + for fi in range(pa.n_frames): + f.write(synthetic_frame(ai, ri, ci, fi)) + finally: + f.close() + return plan diff --git a/tests/test_saw_check.py b/tests/test_saw_check.py index b764da5..a30a9e3 100644 --- a/tests/test_saw_check.py +++ b/tests/test_saw_check.py @@ -1,7 +1,7 @@ -"""Middle-row SAW quality check: plan reduction, the v10 file, and the read-out. +"""Middle-row SAW quality check: plan reduction, the v11 file, and the read-out. The acquisition half runs on the same fake rig as the scan tests; the -analysis half runs on a synthetic v10 file whose CH1 is a pure sine at a +analysis half runs on a synthetic v11 file whose CH1 is a pure sine at a known FFT bin, so the frequency a trace reports is a number the test knows in advance rather than one it copies from the implementation. """ @@ -19,6 +19,7 @@ from core.scan_engine import ScanCallbacks, ScanEngine from core.scan_geometry import ScanGeometryError, build_plan from core.sras_format import ( SCAN_CHANNELS, VERSION, VERSION_SAW_CHECK, SrasFile, create_scan_file, + write_background_block, ) from fakes import FakeScope, FakeStage, FakeT3R, Trace @@ -51,13 +52,19 @@ def sine_frame(k: int) -> bytes: return np.round(100 * np.sin(2 * math.pi * k * n / SPF)).astype(np.int8).tobytes() -def write_check(path, bins, n_masked_frames=0, plan=None): - """A synthetic v10 file: angle `i`'s CH1 is a sine at FFT bin `bins[i]`.""" +def write_check(path, bins, n_masked_frames=0, plan=None, backgrounds=None): + """A synthetic v11 file: angle `i`'s CH1 is a sine at FFT bin `bins[i]`. + + ``backgrounds`` supplies each angle's own background block; the default + is a flat zero one per angle, which subtracts to nothing. + """ plan = plan if plan is not None else middle_row_plan(full_plan(len(bins))) - f = create_scan_file(path, plan, SPF, SAMPLE_RATE, PREAMBLES, bytes(SPF), + f = create_scan_file(path, plan, SPF, SAMPLE_RATE, PREAMBLES, version=VERSION_SAW_CHECK) try: for ai, pa in enumerate(plan.per_angle): + write_background_block( + f, bytes(SPF) if backgrounds is None else backgrounds[ai]) wave = sine_frame(bins[ai]) for ch in SCAN_CHANNELS: for fi in range(pa.n_frames): @@ -127,9 +134,9 @@ def test_middle_row_plan_rejects_an_angle_with_no_rows(): middle_row_plan(plan) -# ── The v10 file ───────────────────────────────────────────────────────────── +# ── The v11 file ───────────────────────────────────────────────────────────── -def test_v10_write_read_roundtrip(tmp_path): +def test_saw_check_write_read_roundtrip(tmp_path): out = tmp_path / "check.sras" plan = write_check(out, bins=(8, 8, 8)) @@ -141,24 +148,25 @@ def test_v10_write_read_roundtrip(tmp_path): sras.close() -def test_v10_rejects_a_multi_row_plan(tmp_path): +def test_saw_check_rejects_a_multi_row_plan(tmp_path): plan = full_plan(num_angles=2) assert any(pa.n_rows > 1 for pa in plan.per_angle) with pytest.raises(ValueError, match="exactly one row per angle"): create_scan_file(tmp_path / "bad.sras", plan, SPF, SAMPLE_RATE, - PREAMBLES, bytes(SPF), version=VERSION_SAW_CHECK) + PREAMBLES, version=VERSION_SAW_CHECK) assert not (tmp_path / "bad.sras").exists() def test_unknown_version_rejected_at_write(tmp_path): - with pytest.raises(ValueError, match="version 7"): + with pytest.raises(ValueError, match="version 99"): create_scan_file(tmp_path / "bad.sras", middle_row_plan(full_plan(1)), - SPF, SAMPLE_RATE, PREAMBLES, bytes(SPF), version=7) + SPF, SAMPLE_RATE, PREAMBLES, version=99) -def test_v6_file_is_not_a_saw_check(): - sras = SrasFile("tests/golden/complete.sras") - assert sras.version == VERSION and not sras.is_saw_check +def test_a_scan_is_not_a_saw_check(): + """Legacy and current scans alike: only the check versions say check.""" + assert not SrasFile("tests/golden/complete.sras").is_saw_check + assert VERSION not in (10, VERSION_SAW_CHECK) # ── Acquisition through the engine ─────────────────────────────────────────── @@ -176,7 +184,7 @@ def run_engine(tmp_path, num_angles=3): return engine.run(), plan, check, trace -def test_engine_writes_a_complete_v10_check(tmp_path): +def test_engine_writes_a_complete_saw_check(tmp_path): result, plan, check, _ = run_engine(tmp_path) assert not result.aborted @@ -200,7 +208,7 @@ def test_engine_visits_each_middle_row_once(tmp_path): assert y_moves == [round(pa.y_positions[0], 4) for pa in check.per_angle] -def test_engine_still_writes_v6_by_default(tmp_path): +def test_engine_still_writes_a_full_scan_by_default(tmp_path): trace = Trace() scope = FakeScope(trace, samples_per_frame=SPF) stage = FakeStage(trace, scope=scope) @@ -229,6 +237,30 @@ def test_traces_report_the_injected_frequency(tmp_path): assert trace.drift_mhz_per_mm == pytest.approx(0.0, abs=1e-6) +def test_background_subtraction_uses_each_angles_own(tmp_path): + """Every angle is referenced against its own background, not angle 1's. + + Each angle's background here is a copy of that angle's own CH1 wave, so + subtracting the right one leaves nothing to read at any angle — where + reusing angle 1's everywhere would leave angles 2 and 3 reporting their + sine unchanged. + """ + out = tmp_path / "check.sras" + bins = (8, 9, 10) + backgrounds = [sine_frame(k) for k in bins] + write_check(out, bins=bins, backgrounds=backgrounds) + + with SrasFile(out) as sras: + assert sras.backgrounds == backgrounds + plain = frequency_traces(sras, dc_threshold_mv=DC_THRESHOLD_MV) + subtracted = frequency_traces(sras, dc_threshold_mv=DC_THRESHOLD_MV, + subtract_background=True) + + assert [t.median_mhz for t in plain] == [pytest.approx(bin_mhz(k)) for k in bins] + for trace in subtracted: + assert np.isnan(trace.freq_mhz).all() + + def test_masked_pixels_become_nan_not_zero(tmp_path): out = tmp_path / "check.sras" write_check(out, bins=(8, 8, 8), n_masked_frames=2) diff --git a/tests/test_scan_engine.py b/tests/test_scan_engine.py index b315d1e..4af7f76 100644 --- a/tests/test_scan_engine.py +++ b/tests/test_scan_engine.py @@ -14,8 +14,8 @@ from core.scan_engine import ( ResumeTarget, ) from core.scan_geometry import ScanGeometryError, build_plan -from core.sras_format import SCAN_CHANNELS, SrasFile -from fakes import FakeScope, FakeStage, FakeT3R, Trace +from core.sras_format import BG_LEN_SIZE, SCAN_CHANNELS, SrasFile +from fakes import FakeScope, FakeStage, FakeT3R, Trace, background_record SPF = 8 @@ -57,7 +57,41 @@ def test_single_angle_scan_writes_readable_file(tmp_path): # File is complete: every declared row present on disk assert [s.status for s in sras.angle_status()] == ["OK"] assert len(sras.preambles) == 3 - assert sras.background == bytes(range(SPF)) + assert sras.backgrounds == [background_record(0, SPF)] + + +def test_each_angle_captures_and_stores_its_own_background(tmp_path): + """One background per angle, taken after the rotation, kept ahead of it.""" + prompts = [] + engine, trace, plan = build( + tmp_path, num_angles=3, + callbacks=ScanCallbacks(prompt=lambda title, msg: prompts.append(title))) + engine.run() + + sras = SrasFile(tmp_path / "out.sras") + assert trace.count("transfer_curve") == 3 + assert sras.backgrounds == [background_record(i, SPF) for i in range(3)] + for st in sras.angle_status(): + assert st.bg_bytes == BG_LEN_SIZE + SPF + assert st.status == "OK" + + # Each capture follows the rotation to the angle it belongs to (the last + # rotation is the return to home after the final angle). + assert [c[0] for c in trace.calls + if c[0] in ("t3r_rotate", "transfer_curve")] == [ + "transfer_curve", "t3r_rotate", "transfer_curve", + "t3r_rotate", "transfer_curve", "t3r_rotate"] + + # Two prompts per angle: Genesis off for the capture, back on to scan. + assert prompts == [title for i in range(3) for title in + (f"Background Capture — Angle {i + 1}/3", + f"Begin Angle {i + 1}/3")] + + # The scope goes back to the scan trigger after every capture, not just + # once at the start — the capture needs the single-record edge trigger. + cmds = [c[1] for c in trace.of("write")] + assert cmds.count("TRIGger:A:TYPe EDGE") == 4 # prepare + one per angle + assert cmds.count("TRIGger:A:TYPe LOGIc") == 4 def test_command_sequence_order(tmp_path): @@ -236,8 +270,9 @@ def test_resume_seeks_to_angle_offset_and_skips_others(tmp_path): target = statuses[1] resume = ResumeState( path=path, - targets=[ResumeTarget(target.index, target.data_offset, - target.n_rows, target.angle_deg)], + targets=[ResumeTarget(target.index, target.bg_offset, + target.data_offset, target.n_rows, + target.angle_deg)], samples_per_frame=SPF, ) @@ -249,19 +284,51 @@ def test_resume_seeks_to_angle_offset_and_skips_others(tmp_path): assert result.rows_written == plan.per_angle[1].n_rows rewritten = path.read_bytes() assert len(rewritten) == len(original) - # Angle 0's block is untouched; angle 1's changed (fresh frame data) - a1_start, a1_end = target.data_offset, target.data_offset + target.row_bytes * target.n_rows + # Angle 0's block is untouched; angle 1's changed — background included, + # since a re-acquired angle captures a fresh one over the old. + a1_start = target.bg_offset + a1_end = target.data_offset + target.row_bytes * target.n_rows assert rewritten[:a1_start] == original[:a1_start] assert rewritten[a1_start:a1_end] != original[a1_start:a1_end] assert rewritten[a1_end:] == original[a1_end:] + # The new background is angle 1's own, written in place of its old one. + reread = SrasFile(path) + assert reread.backgrounds[1] == background_record(0, SPF) + assert reread.backgrounds[0] == SrasFile(path).backgrounds[0] + assert [s.status for s in reread.angle_status()] == ["OK"] * 3 + + +def test_resume_rejects_a_background_that_would_shift_the_file(tmp_path): + """A re-acquired angle's background must fit the room the file has. + + Nothing else in the file records where an angle's rows begin, so a longer + or shorter background would push every row behind it out of position. + """ + engine, _, _ = build(tmp_path, num_angles=2) + engine.run() + path = tmp_path / "out.sras" + target = SrasFile(path).angle_status()[0] + resume = ResumeState( + path=path, + targets=[ResumeTarget(0, target.bg_offset, target.data_offset, + target.n_rows, target.angle_deg)], + samples_per_frame=SPF, + ) + engine2, _, _ = build(tmp_path, num_angles=2, resume=resume) + # A scope that hands back a longer record than the file was written with + engine2._scope.transfer_curve = lambda: bytes(SPF + 4) + + with pytest.raises(RuntimeError, match="shift every row"): + engine2.run() + def test_resume_record_length_mismatch_rejected(tmp_path): engine, trace, plan = build(tmp_path) engine.run() path = tmp_path / "out.sras" resume = ResumeState(path=path, - targets=[ResumeTarget(0, 0, 1, 0.0)], + targets=[ResumeTarget(0, 0, 0, 1, 0.0)], samples_per_frame=SPF + 1) # scope changed engine2, _, _ = build(tmp_path, resume=resume) with pytest.raises(RuntimeError, match="record length"): @@ -461,7 +528,7 @@ def test_strict_row_packing_writes_nothing_for_the_failed_row(tmp_path): # Row 1 was written in full; row 2 aborted before writing anything, so # the file ends exactly on a row boundary. sras = SrasFile(tmp_path / "out.sras") - written = (tmp_path / "out.sras").stat().st_size - sras.data_start_offset + written = (tmp_path / "out.sras").stat().st_size - sras.angle_data_offset(0) assert written == sras.row_bytes(0) diff --git a/tests/test_scan_resume.py b/tests/test_scan_resume.py index 417ffb3..36ea4e8 100644 --- a/tests/test_scan_resume.py +++ b/tests/test_scan_resume.py @@ -3,6 +3,7 @@ from pathlib import Path from core.scan_resume import is_compatible, plan_resume from core.sras_format import SrasFile +from golden_util import write_v7 GOLDEN = Path(__file__).parent / "golden" @@ -47,11 +48,17 @@ def test_selecting_only_a_later_angle_pulls_in_the_frontier(): assert plan.auto_added == [0] -def test_targets_carry_offsets_and_rows(): - st = _statuses("complete.sras") +def test_targets_carry_offsets_and_rows(tmp_path): + out = tmp_path / "v7.sras" + write_v7(out) + st = SrasFile(out).angle_status() plan = plan_resume(st, selected={0, 1}) for target, status in zip(plan.targets, st, strict=True): + # A re-acquired angle rewrites its background too, so a target has to + # know where the block starts as well as where the rows do. + assert target.bg_offset == status.bg_offset assert target.data_offset == status.data_offset + assert target.bg_offset < target.data_offset assert target.n_rows == status.n_rows assert target.angle_deg == status.angle_deg assert plan.total_rows == sum(s.n_rows for s in st) @@ -65,9 +72,19 @@ def test_to_state_carries_samples_per_frame(): assert state.target_indices == {0} -def test_is_compatible_checks_acquisition_settings(): +def test_is_compatible_rejects_a_legacy_file(): + """A v6 file has no room for the background block each angle now writes.""" sras = SrasFile(GOLDEN / "complete.sras") h = sras.header + assert not is_compatible(sras, velocity=h.velocity, laser_freq=h.laser_freq, + sample_rate=h.sample_rate, n_channels=h.n_channels) + + +def test_is_compatible_checks_acquisition_settings(tmp_path): + out = tmp_path / "v7.sras" + write_v7(out) + sras = SrasFile(out) + h = sras.header ok = dict(velocity=h.velocity, laser_freq=h.laser_freq, sample_rate=h.sample_rate, n_channels=h.n_channels) assert is_compatible(sras, **ok) diff --git a/tests/test_sras_analysis.py b/tests/test_sras_analysis.py index 17b3e7f..757dc91 100644 --- a/tests/test_sras_analysis.py +++ b/tests/test_sras_analysis.py @@ -22,7 +22,9 @@ def _loaded_scan(name="complete.sras"): def test_loaded_scan_basics(): scan = _loaded_scan() assert scan.rows_available == [3, 3] - assert scan.background is not None and len(scan.background) == 8 + # A legacy v6 fixture: its one background stands in for every angle. + assert all(scan.background(ai) is not None and len(scan.background(ai)) == 8 + for ai in (0, 1)) assert len(scan.calib.ymult_mv) == 3 view = scan.angle_view(0) assert view.shape == (3, 3, 4, 8) diff --git a/tests/test_sras_format.py b/tests/test_sras_format.py index 0ff2d7e..127eda2 100644 --- a/tests/test_sras_format.py +++ b/tests/test_sras_format.py @@ -1,21 +1,27 @@ -"""core.sras_format vs the pre-refactor golden fixtures. +"""core.sras_format: the current writer against the spec, and the parser +against the pre-refactor golden fixtures. -The goldens were produced by the original sc3_aui_app implementation; the -extracted module must reproduce them byte-for-byte (writer) and -field-for-field (parser + frontier walk). +The goldens are legacy v6 files produced by the original sc3_aui_app +implementation — one background for the whole file. Nothing writes that +layout any more, so they lock the parser (field-for-field, including the +frontier walk) rather than the writer. The writer is locked instead against +bytes this test lays out from scan_format.md itself. """ import json +import struct from dataclasses import asdict from pathlib import Path import numpy as np import pytest -from core.scan_geometry import build_plan -from core.sras_format import SrasFile, create_scan_file +from core.sras_format import ( + BG_LEN_FMT, GEOM_FMT, HDR_FMT, MAGIC, VERSION, VERSION_SAW_CHECK, + SrasFile, create_scan_file, +) from golden_util import ( - BACKGROUND, CHANNELS, LASER_FREQ_HZ, PREAMBLES, SAMPLE_RATE, SPF, - TINY_PLAN_ARGS, VELOCITY_MM_S, synthetic_frame, + BACKGROUND, CHANNELS, PREAMBLES, SAMPLE_RATE, SPF, angle_background, + synthetic_frame, tiny_plan, write_v7, ) GOLDEN = Path(__file__).parent / "golden" @@ -27,28 +33,43 @@ def expected(): return json.load(f) -def _tiny_plan(): - return build_plan(**TINY_PLAN_ARGS, laser_freq_hz=LASER_FREQ_HZ, - velocity_mm_s=VELOCITY_MM_S) +def spec_bytes(plan): + """The v7 layout spelled out from scan_format.md, writer not involved.""" + buf = bytearray() + buf += struct.pack(HDR_FMT, MAGIC, VERSION, plan.n_angles, + plan.x_start_nominal, plan.y_start_nominal, + plan.x_delta_nominal, plan.y_delta_nominal, + plan.row_spacing, plan.velocity_mm_s, plan.laser_freq_hz, + SPF, SAMPLE_RATE, 1, len(CHANNELS)) + buf += struct.pack(f">{plan.n_angles}f", *plan.angles) + for pa in plan.per_angle: + buf += struct.pack(GEOM_FMT, pa.x_start, pa.x_delta, pa.n_frames, pa.n_rows) + for pa in plan.per_angle: + buf += struct.pack(f">{pa.n_rows}f", *pa.y_positions) + for pre in PREAMBLES: + buf += struct.pack(">H", len(pre)) + pre.encode("utf-8") + for ai, pa in enumerate(plan.per_angle): + bg = angle_background(ai) + buf += struct.pack(BG_LEN_FMT, len(bg)) + bg + for ri in range(pa.n_rows): + for ci in range(len(CHANNELS)): + for fi in range(pa.n_frames): + buf += synthetic_frame(ai, ri, ci, fi) + return bytes(buf) -def _write_complete(path): - plan = _tiny_plan() - f = create_scan_file(path, plan, SPF, SAMPLE_RATE, PREAMBLES, BACKGROUND) - try: - for ai, pa in enumerate(plan.per_angle): - for ri in range(pa.n_rows): - for ci in range(len(CHANNELS)): - for fi in range(pa.n_frames): - f.write(synthetic_frame(ai, ri, ci, fi)) - finally: - f.close() +def test_writer_matches_the_spec_byte_for_byte(tmp_path): + out = tmp_path / "v7.sras" + plan = write_v7(out) + assert out.read_bytes() == spec_bytes(plan) -def test_writer_byte_identical_to_golden(tmp_path): - out = tmp_path / "rewrite.sras" - _write_complete(out) - assert out.read_bytes() == (GOLDEN / "complete.sras").read_bytes() +def test_writer_refuses_the_legacy_versions(tmp_path): + for version in (6, 10): + with pytest.raises(ValueError, match=f"version {version}"): + create_scan_file(tmp_path / "bad.sras", tiny_plan(), SPF, + SAMPLE_RATE, PREAMBLES, version=version) + assert not (tmp_path / "bad.sras").exists() def test_header_matches_golden(expected): @@ -75,15 +96,107 @@ def test_header_matches_golden(expected): def test_frontier_all_truncation_variants(expected): + """Every field the goldens recorded, plus the one added since. + + The expectations predate AngleStatus.bg_offset, so they are compared key + by key; a v6 angle has no background block of its own, which is exactly + what bg_offset == data_offset says. + + Two of the recorded data_offsets were corrected when the walk moved into + the parser: an angle past the frontier used to report the frontier's own + offset, because the old walk stopped advancing its cursor there, which + handed a resumed scan the same write position for every missing angle. + They are now the declared position each angle will be written at. + """ for name, exp_statuses in expected["statuses"].items(): statuses = SrasFile(GOLDEN / name).angle_status() - assert [asdict(s) for s in statuses] == exp_statuses, f"mismatch for {name}" + assert len(statuses) == len(exp_statuses), f"mismatch for {name}" + for status, exp in zip(statuses, exp_statuses, strict=True): + got = asdict(status) + assert {k: got[k] for k in exp} == exp, f"mismatch for {name}" + assert status.bg_offset == status.data_offset + assert status.bg_bytes == 0 -def test_preambles_and_background_roundtrip(): +def test_legacy_preambles_and_shared_background(): + """A v6 file's one background stands in for every angle's.""" sras = SrasFile(GOLDEN / "complete.sras") assert sras.preambles == PREAMBLES - assert sras.background == BACKGROUND + assert sras.is_legacy_layout + assert sras.backgrounds == [BACKGROUND] * sras.header.n_angles + assert np.array_equal(sras.background_array(1), + np.frombuffer(BACKGROUND, dtype=np.int8)) + + +# ── Per-angle backgrounds (v7/v11) ─────────────────────────────────────────── + +def test_each_angle_keeps_its_own_background(tmp_path): + out = tmp_path / "v7.sras" + plan = write_v7(out) + + sras = SrasFile(out) + assert not sras.is_legacy_layout + assert sras.backgrounds == [angle_background(ai) + for ai in range(plan.n_angles)] + assert [s.status for s in sras.angle_status()] == ["OK"] * plan.n_angles + # Every angle's rows start just past its own background block … + for st in sras.angle_status(): + assert st.bg_bytes == 4 + SPF + assert st.data_offset == st.bg_offset + st.bg_bytes + # … and the data itself still reads back frame for frame. + assert sras.load_row(1, 2, 0).tobytes() == b"".join( + synthetic_frame(1, 2, 0, fi) for fi in range(plan.per_angle[1].n_frames)) + sras.close() + + +def test_saw_check_version_also_carries_per_angle_backgrounds(tmp_path): + from core.saw_check import middle_row_plan + out = tmp_path / "check.sras" + plan = write_v7(out, plan=middle_row_plan(tiny_plan()), + version=VERSION_SAW_CHECK) + + sras = SrasFile(out) + assert sras.is_saw_check and not sras.is_legacy_layout + assert sras.backgrounds == [angle_background(ai) + for ai in range(plan.n_angles)] + sras.close() + + +def test_angle_missing_its_background_is_the_frontier(tmp_path): + """A file cut inside a background block stops at that angle. + + Nothing of that angle is on disk yet — not even the reference its rows + would be read against — so it is MISSING rather than TRUNCATED, and its + predicted offsets are where a resumed scan would write. + """ + out = tmp_path / "v7.sras" + write_v7(out) + whole = out.read_bytes() + bg1 = SrasFile(out).angle_status()[1].bg_offset + + for cut, expected_status in ((bg1, "MISSING"), (bg1 + 4 + SPF // 2, "MISSING")): + out.write_bytes(whole[:cut]) + statuses = SrasFile(out).angle_status() + assert [s.status for s in statuses] == ["OK", expected_status] + assert statuses[1].n_rows_available == 0 + assert statuses[1].bg_offset == bg1 + # The absent block is predicted at a full record's worth of bytes, + # which is what the writer will produce when the scan resumes. + assert statuses[1].data_offset == bg1 + 4 + SPF + + +def test_rows_after_a_background_still_truncate_by_row(tmp_path): + out = tmp_path / "v7.sras" + plan = write_v7(out) + whole = out.read_bytes() + st1 = SrasFile(out).angle_status()[1] + + out.write_bytes(whole[:st1.data_offset + 2 * st1.row_bytes]) + statuses = SrasFile(out).angle_status() + assert [s.status for s in statuses] == ["OK", "TRUNCATED"] + assert statuses[1].n_rows_available == 2 + assert SrasFile(out).load_angle(1, n_rows=2).shape[0] == 2 + assert plan.per_angle[1].n_rows == 3 def test_load_angle_memmap_equals_eager():