From de0e376b74bacd01571d2d9cf269b0948eb36eef Mon Sep 17 00:00:00 2001 From: Alexander Sandercock <39815775+alex-sandercock@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:22:39 -0500 Subject: [PATCH 1/5] infer-design: the fuzzy pass cuts raw reads at the leading anchor _infer_fuzzy_init stored the leading anchor and then cleared the state it had stored it in, so the fuzzy pass scanned raw reads uncut. On a raw delivery its reads did not vote. The anchor is now stored with the rest of the state, after the clear, as the exact pass does. The new test fails without the fix: five raw reads with an exact anchor gave no fuzzy votes. --- docs/DEVELOPMENT.md | 4 ++-- docs/src/development/changelog.md | 2 ++ madcats.py | 3 ++- tests/test_madcats.py | 22 ++++++++++++++++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 6addbc4..70daee8 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -4191,8 +4191,8 @@ By default sample columns use the unique FASTQ filename stem. python3 -m unittest discover -s tests -v ``` -The suite contains 257 unit/end-to-end tests in six files. -`test_madcats.py` (86) covers `generate` and the commands around it, +The suite contains 258 unit/end-to-end tests in six files. +`test_madcats.py` (87) covers `generate` and the commands around it, `test_call.py` (112) the genotype caller, `test_filter_paralogs.py` (49) the paralog filter, `test_tiled_markers.py` (5) tiled markers through all three, `test_example.py` (4) the bundled example and `test_docs.py` (1) the match diff --git a/docs/src/development/changelog.md b/docs/src/development/changelog.md index 390742e..befafe7 100644 --- a/docs/src/development/changelog.md +++ b/docs/src/development/changelog.md @@ -111,6 +111,8 @@ Notable changes by version. Measurements behind each change are in the (now 6,562–103,852 each), and one pair with identical tags had split its reads 12,299 / 80,306 (now about 112,400 each). All other markers' rows are unchanged. - The tile families are part of the per-sample cache key. Panels without tiles keep their caches. +- `infer-design --leading-anchor`: the fuzzy pass cuts raw reads at the anchor, as the exact pass + does. It had dropped the anchor before scanning, so on a raw delivery its reads did not vote. ## 0.13.0 diff --git a/madcats.py b/madcats.py index 09b08f3..f94feb1 100644 --- a/madcats.py +++ b/madcats.py @@ -1345,7 +1345,6 @@ def _infer_fuzzy_init( marker: enumerate_variant_windows(targets[marker], tag_length) for marker in unresolved } - _INFER_STATE["leading_anchor"] = leading_anchor seed_length = max(12, min(18, tag_length // 3)) offsets = sorted({*range(0, tag_length - seed_length + 1, seed_length), tag_length - seed_length}) seeds: Dict[str, set] = {} @@ -1370,6 +1369,8 @@ def _infer_fuzzy_init( "offsets": offsets, "seed_length": seed_length, "sinks": tuple(sink_tags), + # Stored with the rest, after the clear, so the fuzzy pass cuts raw reads as the exact pass does. + "leading_anchor": leading_anchor, } ) diff --git a/tests/test_madcats.py b/tests/test_madcats.py index 4504bdf..2b9b125 100644 --- a/tests/test_madcats.py +++ b/tests/test_madcats.py @@ -1859,6 +1859,28 @@ def test_fastq_records_leading_anchor_cuts_and_skips(self): self.assertEqual([seq for seq, _ in records], ["ACGTACGT", "TTTT"]) self.assertTrue(all(len(seq) == len(qual) for seq, qual in records)) + def test_infer_fuzzy_pass_cuts_raw_reads_at_the_anchor(self): + # The fuzzy pass once cleared the anchor it had just stored and voted + # with uncut raw reads; it must cut them as the exact pass does. + rng = random.Random(5) + flank = lambda n: "".join(rng.choice("ACGT") for _ in range(n)) + target = flank(40) + "[A/G]" + flank(40) + anchor = "GATGGATCCAGTGC" + with tempfile.TemporaryDirectory() as tmp: + design = Path(tmp) / "design.tsv" + design.write_text("MarkerName\tTargetSequence\n" + f"m1\t{target}\n") + windows = mp.enumerate_variant_windows(target, 30) + ref_window = sorted(windows.values())[0][0] + read_tag = ref_window[:25] + ("A" if ref_window[25] != "A" else "C") + ref_window[26:] # one error: fuzzy tier + fastq = Path(tmp) / "raw.fastq" + sequence = "TTCATAAGTTGC" + anchor + read_tag + "CCGATC" + fastq.write_text("".join(f"@r{n}\n{sequence}\n+\n{'I' * len(sequence)}\n" for n in range(5))) + mp._infer_fuzzy_init(str(design), 30, 3, ["m1"], (), anchor) + self.assertEqual(mp._INFER_STATE["leading_anchor"], anchor) + _, votes = mp._infer_fuzzy_scan(str(fastq)) + self.assertEqual(sum(votes.values()), 5) + self.assertTrue(all(marker == "m1" for marker, _key in votes)) + def test_raw_delivery_is_detected_and_named(self): anchor = "GATGGATCCAGTGC" tag = "ACGTACGTACGTACGTACGT" From 75a12819d0b414b28b43f912af690edbb2bbdf66 Mon Sep 17 00:00:00 2001 From: Alexander Sandercock <39815775+alex-sandercock@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:36:45 -0500 Subject: [PATCH 2/5] generate: --leading-anchor-mismatches, one substitution in the leading anchor A read whose common sequence carries a sequencing error has no exact leading anchor and was counted as reads_no_leading_anchor. With --leading-anchor-mismatches 1 (the default), such a read is cut after the one placement within a single substitution in the search window, and the QC counts it as reads_leading_anchor_mismatch. - An exact copy is tried first, so a read that has one is cut as before. - No placement, or more than one, still counts as no anchor. Indels are not matched, because they would move the cut. - An anchor shorter than 12 bases is always matched exactly. - 0 matches every anchor exactly, as 0.13.0 did. - The setting joins the cache key only when an anchor is matched with a substitution, so other caches stay valid. - infer-design still matches the anchor exactly; its reads only vote. Measured on the salmon 3K DArTag plate (1,104 raw FASTQs): with 0 the table is byte for byte the 0.13.0 table after the provenance cell. With 1, reads without an anchor fall from 13,505,332 to 2,048,910, 1.06% more reads are assigned, and no row changes category. The numbers are in docs/DEVELOPMENT.md. --- docs/DEVELOPMENT.md | 36 ++++++++++- docs/src/commands/generate.md | 1 + docs/src/development/changelog.md | 8 +++ docs/src/tutorials/dartag.md | 5 +- madcats.py | 89 +++++++++++++++++++++++--- tests/test_madcats.py | 100 ++++++++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 13 deletions(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 70daee8..1762a96 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -41,7 +41,10 @@ which inputs they require: library's common sequence `GATGGATCCAGTGC` in front of every amplicon. `--leading-anchor GATGGATCCAGTGC` cuts each read where that sequence ends (it must end within 40 bases beyond its own length) and counts - reads without it as `reads_no_leading_anchor`. Preprocessed deliveries + reads without it as `reads_no_leading_anchor`. With no exact copy, the + one placement within a single substitution is used instead + (`--leading-anchor-mismatches 1`, the default since 0.14.0; counted as + `reads_leading_anchor_mismatch`). Preprocessed deliveries are primer-started and need no anchor. The anchor is part of the cache key and the provenance line. `infer-design` takes the same option, so the primer-free route reads raw deliveries too. Both commands check the @@ -1293,6 +1296,33 @@ DArTag panels do. The primer-free route on the raw alfalfa reads, with no primer file, resolves 2,999 of 3,000 loci in the exact tier, all identical to the design tags. +**One substitution in the leading anchor (0.14.0).** A read whose common +sequence carries a sequencing error has no exact anchor, and 0.13.0 +counted it as `reads_no_leading_anchor`. With +`generate --leading-anchor-mismatches 1` (the default), such a read is cut +after the one placement within a single substitution in the search window +and counted as `reads_leading_anchor_mismatch`. An exact copy is tried +first, so a read that has one is cut as before. A read with no such +placement, or more than one, still counts as having no anchor. Indels are +not matched, because they would move the cut. An anchor shorter than 12 +bases is always matched exactly. The setting joins the cache key only when +an anchor is matched with a substitution, so other caches stay valid. +`infer-design` still matches the anchor exactly, since its reads only vote. +Measured on the salmon 3K DArTag plate (DSalm21-6350, 1,104 raw FASTQs, +54-bp tags, `--gapped-rescue off`): with `0`, the table is byte for byte +the 0.13.0 table after the provenance cell. With `1`, reads without an +anchor fall from 13,505,332 to 2,048,910 (2.4% to 0.4% of reads), +2,789,193 more reads are assigned (+1.06%) and 2,304,149 more reach emitted +rows (+0.90%, and 0.65% to 3.20% per sample). 16 new rows pass the evidence +rules, none is lost and no row changes category. The Alt fraction per +marker and sample (depth of 20 or more) changes by 0.0000 at the median and +0.0185 at the 99th percentile, and 379 of 2,501,106 cells move by more than +0.05. Against the vendor MADC, recall is unchanged (99.50%), precision goes +from 77.40% to 77.27% (the 16 rows), and the share of cells with exactly +the vendor's count rises from 64.65% to 66.95%. In one sample, 2,441 of +5,468 recovered on-target reads had the substitution at the anchor's last +base (C read as T in 2,223), spread over 1,106 markers. + **In the design route, a tag carries its variant.** Tag reconstruction from the primer table and probe design now keeps only the primer-started windows that contain the bracketed variant. The pecan design has one marker @@ -4191,8 +4221,8 @@ By default sample columns use the unique FASTQ filename stem. python3 -m unittest discover -s tests -v ``` -The suite contains 258 unit/end-to-end tests in six files. -`test_madcats.py` (87) covers `generate` and the commands around it, +The suite contains 261 unit/end-to-end tests in six files. +`test_madcats.py` (90) covers `generate` and the commands around it, `test_call.py` (112) the genotype caller, `test_filter_paralogs.py` (49) the paralog filter, `test_tiled_markers.py` (5) tiled markers through all three, `test_example.py` (4) the bundled example and `test_docs.py` (1) the match diff --git a/docs/src/commands/generate.md b/docs/src/commands/generate.md index 1f743ec..6e8cbd6 100644 --- a/docs/src/commands/generate.md +++ b/docs/src/commands/generate.md @@ -62,6 +62,7 @@ The [Evidence Model](../reference/evidence-model.md) explains stages 2–6 in de |--------|---------|-------------| | `--min-identity ` | 0.80 | Minimum substitution-only identity to a Ref or Alt tag. A gapped alignment uses the same edit limit | | `--leading-anchor ` | none | Constant sequence before the amplicon in raw deliveries (for DArT, `GATGGATCCAGTGC`). Each read is cut where the anchor ends (it must end within 40 bases beyond its own length). `auto` detects it from the first FASTQ | +| `--leading-anchor-mismatches {0,1}` | 1 | Substitutions allowed in the leading anchor when a read has no exact copy of it. With `1`, the read is cut after the one placement within a single substitution. Reads with no such placement, or more than one, count as `reads_no_leading_anchor`, and the reads it recovers count as `reads_leading_anchor_mismatch`. An anchor shorter than 12 bases is always matched exactly. `0` matches every anchor exactly, as in 0.13.0 | | `--primer-mismatches {0,1}` | 1 | Mismatches allowed while assigning a primer/anchor. With `0`, reads from Match haplotypes whose distinguishing SNP falls inside the anchor are lost | | `--gapped-rescue {on,off}` | `on` | Give reads that fail substitution-only identity one gapped alignment, so the counts include undesigned indels | | `--max-indel-length ` | 10 | Widest gap (alignment band) a gapped alignment may use | diff --git a/docs/src/development/changelog.md b/docs/src/development/changelog.md index befafe7..7988f6d 100644 --- a/docs/src/development/changelog.md +++ b/docs/src/development/changelog.md @@ -113,6 +113,14 @@ Notable changes by version. Measurements behind each change are in the - The tile families are part of the per-sample cache key. Panels without tiles keep their caches. - `infer-design --leading-anchor`: the fuzzy pass cuts raw reads at the anchor, as the exact pass does. It had dropped the anchor before scanning, so on a raw delivery its reads did not vote. +- `generate --leading-anchor-mismatches {0,1}` (default 1). A read with no exact copy of the + leading anchor is cut after the one placement within a single substitution, and the QC counts it + as `reads_leading_anchor_mismatch`. Anchors shorter than 12 bases are always matched exactly, + and `0` matches every anchor exactly, as 0.13.0 did. On the salmon 3K DArTag plate (1,104 raw + FASTQs), `0` reproduces the 0.13.0 table byte for byte. With `1`, reads without an anchor fall + from 13.5 million to 2.0 million, 1.06% more reads are assigned, and no row changes category. + The setting is part of the cache key only when an anchor is matched with a substitution, so + other caches stay valid. `infer-design` still matches the anchor exactly. ## 0.13.0 diff --git a/docs/src/tutorials/dartag.md b/docs/src/tutorials/dartag.md index 839ff71..393f251 100644 --- a/docs/src/tutorials/dartag.md +++ b/docs/src/tutorials/dartag.md @@ -107,4 +107,7 @@ madcats generate ... --leading-anchor auto madcats generate ... --leading-anchor GATGGATCCAGTGC ``` -The QC counts reads without the anchor as `reads_no_leading_anchor`. +A read whose anchor carries one sequencing error is still cut where the anchor ends, as long as +there is only one such placement (`--leading-anchor-mismatches 1`, the default). The QC counts +those reads as `reads_leading_anchor_mismatch`, and reads without the anchor as +`reads_no_leading_anchor`. diff --git a/madcats.py b/madcats.py index f94feb1..9729a0a 100644 --- a/madcats.py +++ b/madcats.py @@ -235,6 +235,8 @@ GAP_VERIFIED_SUFFIX = 12 # bases that must follow the last gap before a gapped reading may re-read the target PARTIAL_TAIL_DEFAULT = 9 # --min-partial-length defaults to --tag-length minus this: at most 9 unobserved tail bases LEADING_ANCHOR_SEARCH = 40 # --leading-anchor: the anchor must end within this many bases beyond its own length +LEADING_ANCHOR_MISMATCHES_DEFAULT = 1 # --leading-anchor-mismatches: substitutions tolerated in the anchor +LEADING_ANCHOR_MISMATCH_MIN_LENGTH = 12 # shortest anchor that may be matched with a substitution (fewer bases match junk by chance) DENOISE_LOOKUP_THRESHOLD = 2 # denoiser: look up single-substitution neighbours once centroids exceed this many per tag base GAP_OPEN_SCALE = 1 << 10 # annotation alignments weigh one edit this much more than one gap opening @@ -331,6 +333,7 @@ class AssignmentSettings: gap_band: int # --max-indel-length, or 0 with --gapped-rescue off leading_anchor: str = "" # --leading-anchor: constant sequence preceding the amplicon in raw deliveries tile_families: Tuple[Tuple[int, ...], ...] = () # --tile-groups: marker indices credited together (tile_families) + leading_anchor_mismatches: int = 0 # --leading-anchor-mismatches: substitutions tolerated when no exact anchor is found def cache_arguments(self) -> Dict[str, object]: arguments: Dict[str, object] = { @@ -347,6 +350,10 @@ def cache_arguments(self) -> Dict[str, object]: # tile family (and any change to the families) gives a new fingerprint. if self.tile_families: arguments["tile_families"] = [list(family) for family in self.tile_families] + # Likewise keyed only when it changes the read pass: exact-anchor and + # anchor-free runs keep the fingerprints (and caches) they had before. + if self.leading_anchor and self.leading_anchor_mismatches: + arguments["leading_anchor_mismatches"] = self.leading_anchor_mismatches return arguments @@ -1966,12 +1973,53 @@ def match(self, sequence: str, max_mismatches: int = 0) -> List[int]: return sorted(candidates) +def locate_leading_anchor(sequence: str, anchor: str, max_mismatches: int = 0) -> Tuple[int, int]: + """Where the amplicon starts in a raw read: ``(cut, mismatches)``, or ``(-1, 0)``. + + The anchor must end within LEADING_ANCHOR_SEARCH bases beyond its own + length. Its first exact occurrence wins. Only when there is none, and + ``max_mismatches`` is 1, may one placement within a single substitution + stand in — and only when it is the one such placement in the window, so a + read never gets a guessed cut. Substitutions only: an indel would move + the cut, and the primer match that follows needs the exact frame. A + sequencing error in the anchor is upstream of the amplicon, so the reads + this recovers carry no allele bias. An anchor shorter than + LEADING_ANCHOR_MISMATCH_MIN_LENGTH is always matched exactly, because + fewer bases match unrelated sequence by chance. Candidates come from + the pigeonhole principle (with one substitution, one half of the anchor + still matches exactly), so the search stays in ``str.find``. + """ + width = len(anchor) + search_end = width + LEADING_ANCHOR_SEARCH + start = sequence.find(anchor, 0, search_end) + if start >= 0: + return start + width, 0 + if max_mismatches <= 0 or width < LEADING_ANCHOR_MISMATCH_MIN_LENGTH: + return -1, 0 + last_start = min(search_end, len(sequence)) - width + if last_start < 0: + return -1, 0 + half = width // 2 + placements = set() + for offset, piece in ((0, anchor[:half]), (half, anchor[half:])): + end = last_start + offset + len(piece) + position = sequence.find(piece, offset, end) + while position >= 0: + candidate = position - offset + if candidate not in placements and tag_mismatches(sequence[candidate:candidate + width], anchor, stop_after=1) <= 1: + placements.add(candidate) + position = sequence.find(piece, position + 1, end) + if len(placements) != 1: + return -1, 0 + return placements.pop() + width, 1 + + def fastq_records(path: Path, leading_anchor: str = "") -> Iterator[Tuple[str, str]]: """``(sequence, quality)`` records. With ``leading_anchor`` (raw deliveries that keep the library's common sequence in front of the amplicon) each - read starts where the anchor ends, and reads without it are skipped; - callers that must count those reads cut for themselves (``process_sample``).""" - search_end = len(leading_anchor) + LEADING_ANCHOR_SEARCH + read starts where an exact copy of the anchor ends, and reads without one + are skipped. ``process_sample`` cuts for itself: it counts the reads it + skips and may match the anchor with a substitution.""" with open_text(path, "rt") as handle: record_number = 0 while True: @@ -1991,10 +2039,9 @@ def fastq_records(path: Path, leading_anchor: str = "") -> Iterator[Tuple[str, s if len(sequence) != len(quality): raise MadcError(f"Sequence/quality length mismatch at FASTQ record {record_number} in {path}") if leading_anchor: - cut = sequence.find(leading_anchor, 0, search_end) + cut, _ = locate_leading_anchor(sequence, leading_anchor) if cut < 0: continue - cut += len(leading_anchor) sequence, quality = sequence[cut:], quality[cut:] yield sequence, quality @@ -2485,11 +2532,14 @@ def process_sample( if settings.leading_anchor: # Raw deliveries that keep the library's common sequence in front # of the amplicon: the read starts where that anchor ends. - cut = sequence.find(settings.leading_anchor, 0, len(settings.leading_anchor) + LEADING_ANCHOR_SEARCH) + cut, anchor_mismatches = locate_leading_anchor( + sequence, settings.leading_anchor, settings.leading_anchor_mismatches + ) if cut < 0: qc["reads_no_leading_anchor"] += 1 continue - cut += len(settings.leading_anchor) + if anchor_mismatches: + qc["reads_leading_anchor_mismatch"] += 1 sequence, quality = sequence[cut:], quality[cut:] if len(sequence) < settings.tag_length: if settings.partial_min_length <= 0 or len(sequence) < settings.partial_min_length: @@ -4629,6 +4679,7 @@ def _assignment_settings(args: argparse.Namespace, families: Tuple[Tuple[int, .. gap_band=args.max_indel_length if args.gapped_rescue == "on" else 0, leading_anchor=(getattr(args, "leading_anchor", None) or "").upper(), tile_families=families, + leading_anchor_mismatches=args.leading_anchor_mismatches, ) @@ -4759,7 +4810,10 @@ def _generate_provenance(args: argparse.Namespace) -> str: f"annotations={args.annotations}", f"partial_read_mode={args.partial_read_mode}", f"min_partial_length={args.min_partial_length}", - *([f"leading_anchor={args.leading_anchor.upper()}"] if getattr(args, "leading_anchor", None) else []), + *( + [f"leading_anchor={args.leading_anchor.upper()}", f"leading_anchor_mismatches={args.leading_anchor_mismatches}"] + if getattr(args, "leading_anchor", None) else [] + ), f"denoise_distance={args.denoise_distance}", f"global_denoise_distance={args.global_denoise_distance}", f"denoise_alpha={args.denoise_alpha}", @@ -4796,6 +4850,9 @@ def _read_accounting( return { "reads_total": total("reads_total"), "reads_no_leading_anchor": total("reads_no_leading_anchor"), + # Of the reads that went on past the anchor, those cut at a placement + # one substitution away (--leading-anchor-mismatches 1). + "reads_leading_anchor_mismatch": total("reads_leading_anchor_mismatch"), "reads_assigned_full_length": total("reads_assigned"), "reads_assigned_gapped": total("reads_assigned_gapped"), # A read credited to several tiles of one family counts once above; @@ -4829,6 +4886,7 @@ def _generate_parameters(args: argparse.Namespace, settings: AssignmentSettings, "minimum_partial_length": args.min_partial_length, "leading_anchor": (getattr(args, "leading_anchor", None) or "").upper() or None, "leading_anchor_detected": bool(getattr(args, "leading_anchor_detected", False)), + "leading_anchor_mismatches": args.leading_anchor_mismatches if getattr(args, "leading_anchor", None) else None, "sample_denoise_distance": args.denoise_distance, "global_denoise_distance": args.global_denoise_distance, "denoise_alpha": args.denoise_alpha, @@ -5694,6 +5752,19 @@ def _add_generate_command(subparsers) -> None: "primer-started, and a run whose reads carry an undeclared constant prefix stops and names it)" ), ) + assignment.add_argument( + "--leading-anchor-mismatches", + type=int, + choices=(0, 1), + default=LEADING_ANCHOR_MISMATCHES_DEFAULT, + help=( + "substitutions allowed in the leading anchor when a read has no exact copy of it: with 1, a " + "read is cut after the one placement within a single substitution (none, or more than one, " + "counts as reads_no_leading_anchor), and such reads are counted as " + f"reads_leading_anchor_mismatch; an anchor shorter than {LEADING_ANCHOR_MISMATCH_MIN_LENGTH} bases " + f"is always matched exactly (default: {LEADING_ANCHOR_MISMATCHES_DEFAULT})" + ), + ) assignment.add_argument( "--primer-mismatches", type=int, @@ -5756,7 +5827,7 @@ def _add_generate_command(subparsers) -> None: "--cache-dir", help=( "folder for a reusable per-sample cache, so re-runs skip the read pass. The cache is rebuilt " - "when the panel or --tag-length, --min-identity, --leading-anchor, --primer-mismatches, " + "when the panel or --tag-length, --min-identity, --leading-anchor (or its mismatches), --primer-mismatches, " "--gapped-rescue, --max-indel-length, --min-partial-length or the tile families " "(--tile-groups) change" ), diff --git a/tests/test_madcats.py b/tests/test_madcats.py index 2b9b125..90d2392 100644 --- a/tests/test_madcats.py +++ b/tests/test_madcats.py @@ -1859,6 +1859,106 @@ def test_fastq_records_leading_anchor_cuts_and_skips(self): self.assertEqual([seq for seq, _ in records], ["ACGTACGT", "TTTT"]) self.assertTrue(all(len(seq) == len(qual) for seq, qual in records)) + def test_locate_leading_anchor_with_one_substitution(self): + anchor = "GATGGATCCAGTGC" + tail = "ACGTTGCAAGCTTGACCGATAGGCTTACGG" + barcode = "TTCATAAGTTGC" + cut = len(barcode) + len(anchor) + # exact anchor: same cut either way, no mismatch spent + self.assertEqual(mp.locate_leading_anchor(barcode + anchor + tail, anchor, 1), (cut, 0)) + self.assertEqual(mp.locate_leading_anchor(barcode + anchor + tail, anchor, 0), (cut, 0)) + # one substitution, at either end or inside, and an N: recovered only with mismatches allowed + for position in (0, 6, len(anchor) - 1): + for base in ("A", "N"): + if anchor[position] == base: + continue + read = barcode + anchor[:position] + base + anchor[position + 1:] + tail + self.assertEqual(mp.locate_leading_anchor(read, anchor, 0), (-1, 0)) + self.assertEqual(mp.locate_leading_anchor(read, anchor, 1), (cut, 1)) + # two substitutions: no anchor + two = barcode + "C" + anchor[1:6] + "A" + anchor[7:] + tail + self.assertEqual(mp.locate_leading_anchor(two, anchor, 1), (-1, 0)) + # an exact copy wins over an earlier one-substitution placement + damaged = anchor[:3] + "C" + anchor[4:] + read = "AA" + damaged + "TTTT" + anchor + tail + self.assertEqual(mp.locate_leading_anchor(read, anchor, 1), (2 + len(anchor) + 4 + len(anchor), 0)) + # two one-substitution placements are ambiguous: no guessed cut + other = anchor[:9] + "T" + anchor[10:] + self.assertEqual(mp.locate_leading_anchor("AA" + damaged + "TT" + other + tail, anchor, 1), (-1, 0)) + # the placement must end inside the search window, as an exact anchor must + late = "C" * (mp.LEADING_ANCHOR_SEARCH + 1) + damaged + tail + self.assertEqual(mp.locate_leading_anchor(late, anchor, 1), (-1, 0)) + self.assertEqual(mp.locate_leading_anchor("C" * mp.LEADING_ANCHOR_SEARCH + damaged + tail, anchor, 1), + (mp.LEADING_ANCHOR_SEARCH + len(anchor), 1)) + # a read shorter than the anchor + self.assertEqual(mp.locate_leading_anchor(damaged[:-2], anchor, 1), (-1, 0)) + # an anchor shorter than the minimum is matched exactly, whatever is allowed + short = anchor[:9] + self.assertLess(len(short), mp.LEADING_ANCHOR_MISMATCH_MIN_LENGTH) + self.assertEqual(mp.locate_leading_anchor(barcode + short + tail, short, 1), (len(barcode) + len(short), 0)) + self.assertEqual(mp.locate_leading_anchor(barcode + "T" + short[1:] + tail, short, 1), (-1, 0)) + + def test_locate_leading_anchor_matches_a_full_scan(self): + # The pigeonhole search must find exactly the placements a scan of + # every window position finds. + anchor = "GATGGATCCAGTGC" + width = len(anchor) + + def scan(read): + end = min(width + mp.LEADING_ANCHOR_SEARCH, len(read)) + exact = read.find(anchor, 0, width + mp.LEADING_ANCHOR_SEARCH) + if exact >= 0: + return exact + width, 0 + hits = [start for start in range(0, end - width + 1) + if sum(a != b for a, b in zip(read[start:start + width], anchor)) <= 1] + return (hits[0] + width, 1) if len(hits) == 1 else (-1, 0) + + rng = random.Random(11) + for _ in range(3000): + prefix = "".join(rng.choice("ACGT") for _ in range(rng.randint(0, 45))) + body = list(anchor) + for _ in range(rng.choice([0, 1, 1, 2])): + body[rng.randrange(width)] = rng.choice("ACGTN") + read = prefix + "".join(body) + "".join(rng.choice("ACGT") for _ in range(rng.randint(0, 30))) + self.assertEqual(mp.locate_leading_anchor(read, anchor, 1), scan(read), read) + + def test_leading_anchor_mismatch_recovers_reads_and_is_keyed_only_when_used(self): + ref = "ACGTACGTACGTACGTACGT" + alt = ref[:6] + "T" + ref[7:] + locus = mp.Locus(0, "m1", ref[:6], "", "G", "T", 6, ref, alt) + anchor = "GATGGATCCAGTGC" + one = anchor[:-1] + "A" # the anchor's last base misread + two = "C" + anchor[1:-1] + "A" # two substitutions + with tempfile.TemporaryDirectory() as tmp: + fastq = Path(tmp) / "s1.fastq" + records = (["TTCATAAGTTGC" + anchor + ref + "CCGATC"] * 20 + + ["TTCATAAGTTGC" + one + ref + "CCGATC"] * 7 + + ["TTCATAAGTTGC" + two + ref + "CCGATC"] * 3 + + [ref + "CCGATCTGGG"] * 5) + with fastq.open("w") as handle: + for number, sequence in enumerate(records): + handle.write(f"@r{number}\n{sequence}\n+\n{'I' * len(sequence)}\n") + sample = mp.Sample("s1", "s1", str(fastq), {}) + base = dict(tag_length=20, max_offtarget_mismatches=4, primer_mismatches=1, + partial_min_length=30, gap_band=10, leading_anchor=anchor) + exact = mp.AssignmentSettings(**base, leading_anchor_mismatches=0) + tolerant = mp.AssignmentSettings(**base, leading_anchor_mismatches=1) + strict = mp.process_sample(sample, [locus], mp.PrimerIndex([locus]), exact) + loose = mp.process_sample(sample, [locus], mp.PrimerIndex([locus]), tolerant) + self.assertEqual(strict.counts[(0, ref)], 20) + self.assertEqual(strict.qc["reads_no_leading_anchor"], 15) + self.assertEqual(strict.qc.get("reads_leading_anchor_mismatch", 0), 0) + self.assertEqual(loose.counts[(0, ref)], 27) + self.assertEqual(loose.qc["reads_no_leading_anchor"], 8) + self.assertEqual(loose.qc["reads_leading_anchor_mismatch"], 7) + # The cache key changes only when the substitution can change the read + # pass, so exact-anchor and anchor-free caches stay valid. + self.assertNotIn("leading_anchor_mismatches", exact.cache_arguments()) + self.assertEqual(tolerant.cache_arguments()["leading_anchor_mismatches"], 1) + no_anchor = mp.AssignmentSettings(**{**base, "leading_anchor": ""}, leading_anchor_mismatches=1) + self.assertNotIn("leading_anchor_mismatches", no_anchor.cache_arguments()) + self.assertEqual(exact.cache_arguments(), mp.AssignmentSettings(**base).cache_arguments()) + def test_infer_fuzzy_pass_cuts_raw_reads_at_the_anchor(self): # The fuzzy pass once cleared the anchor it had just stored and voted # with uncut raw reads; it must cut them as the exact pass does. From ece33ce88f7c6e86b74ee7ed45f8ef6e410d313d Mon Sep 17 00:00:00 2001 From: Alexander Sandercock <39815775+alex-sandercock@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:45:22 -0500 Subject: [PATCH 3/5] Leading-anchor detection: one prefix per file, so barcodes of several lengths do not leak into the anchor detect_leading_anchor pooled the sampled reads and built the anchor from the prefixes at the modal tag offset. DArT inline barcodes have 10 to 14 bases, so of 8 sampled files only two or three sit at that offset, and the anchor grew into the barcode bases those few shared. An anchor one base too long fits about a third of the samples, and the rest lose every read. Each sampled file now gives one prefix, the most common among its shifted reads, and the anchor is the longest suffix that every file's prefix shares. The check samples 24 files instead of 8. A file that mixes barcodes has its prefix shortened until 90% of its shifted reads would be cut by it. Files holding 90% of the shifted reads must end in the same 8 bases, and the rest are set aside, so a failed sample does not hide the anchor. The gates in resolve_leading_anchor are unchanged. With one or two FASTQs the anchor may keep barcode bases. It still cuts those files where the amplicon starts, so auto uses it and warns not to reuse it for other samples, and the stop message for an undeclared prefix says the same. The messages give the tag start as a range when barcodes differ in length. On the salmon plate DSalm21-6350 (common sequence CGATGGATCCAGTGC), random subsets of 1, 2, 3 and 4 files (60 draws each) and of 8 files (150 draws) returned the exact sequence in 0, 9, 19, 39 and 133 draws before, and in 0, 39, 57, 58 and 150 now. Draws of 16 and of 24 files: 150 of 150 each. The help text and the command pages still say auto reads the first FASTQ, and the test counts in docs/DEVELOPMENT.md are not updated: both sit next to lines the anchor-mismatch branch changes. --- docs/DEVELOPMENT.md | 46 ++++++++++++ docs/src/development/changelog.md | 9 +++ madcats.py | 96 +++++++++++++++++------- tests/test_madcats.py | 121 ++++++++++++++++++++++++++++++ 4 files changed, 244 insertions(+), 28 deletions(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 6addbc4..3f3fa2a 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -1343,6 +1343,52 @@ byte-identically from cache). Protection, the final bookkeeping and the table write stay in the main process. The table is unchanged with `--jobs 1`. +**Raw deliveries with barcodes of several lengths (0.14.0).** The 0.12.0 +detection of the leading anchor pooled the +sampled reads and built the anchor from the prefixes at the modal tag +offset. DArT inline barcodes have 10 to 14 bases, about a fifth of the +samples each, so of 8 sampled files only two or three sit at the modal +offset, and the anchor grew into the barcode bases those few shared. On +the salmon validation plate (DSalm21-6350: 1,104 FASTQs, 2,979 loci, +common sequence `CGATGGATCCAGTGC` in the delivery's targets file; three +truncated files left out of the draws), detection from random subsets of +1, 2, 3 and 4 files (60 draws each) and of 8 files (150 draws) returned +the exact 15 bases in 0, 9, 19, 39 and 133 draws. Of the 17 misses with 8 +files, 13 were one base too long and 4 two. A barcode on this plate ends +in A, G or T, 30–36% each, so an anchor one base too long fits about a +third of the samples and the others lose every read. + +Detection now works per file. A barcode is constant within a file, so +each file gives one prefix, the most common among its shifted reads +(seen at least twice). That also absorbs a tag found a base early, which +the modal offset used to. The anchor is the longest suffix that every +file's prefix shares, and the check samples 24 files instead of 8. On the +same draws the exact sequence came back in 0, 39, 57, 58 and 150, and in +150 of 150 draws of 16 and of 24 files. In 3,000 further draws of 8 files +two anchors were one base too long, about what the barcode endings +predict (the chance that 8 barcodes end alike is 1 in 1,900). In 1,000 +further draws of 24 files none was. Sampling 24 files of the plate takes +about 0.02 s. + +Two rules cover files that are not one clean sample. A file that mixes +barcodes has its prefix shortened until 90% of its shifted reads would be +cut by it. The plate's three truncated files are such files (variants of +one barcode, the commonest in 18–27% of the shifted reads), and alone each gives +the common sequence plus the 2 or 3 bases its variants share, which cuts +95–96% of its tag-carrying reads. Files holding 90% of the shifted reads +must end in the same 8 bases, and the rest are set aside, so a failed +sample does not hide the anchor. + +With up to 24 files the sample is the whole run, and the anchor is in +every file's prefix by construction. It then cuts every file where the +amplicon starts, even when it is longer than the common sequence: from +one file it is the whole prefix, barcode included, and from two files it +kept one to five barcode bases in 21 of 60 draws. In 40 draws each of 1, 2 +and 3 files, the detected anchor was found in at least 97% of the +tag-carrying reads of every file in the run. So `auto` uses an anchor from +fewer than three files and warns not to reuse it for other samples, and +the stop message for an undeclared prefix says the same. + ## Truncated reads: partial-read salvage with a locus guard (0.11.0) A DArTag order whose delivered files were trimmed to 109–113 bp per file, diff --git a/docs/src/development/changelog.md b/docs/src/development/changelog.md index 390742e..ad15d59 100644 --- a/docs/src/development/changelog.md +++ b/docs/src/development/changelog.md @@ -99,6 +99,15 @@ Notable changes by version. Measurements behind each change are in the overflow and record it in their QC reports. Neither command corrects the counts, since the wrapped ones cannot be recovered. The lettuce 3K vendor MADC has 90,913 such cells across 1,108 markers. +- `--leading-anchor auto`, and the raw-delivery check that names the anchor, find the common + sequence when inline barcodes differ in length. Each sampled FASTQ gives its own prefix, and the + anchor is the suffix that all of them share. 24 files are sampled instead of 8. Before, the + anchor came only from reads at the most common tag offset. DArT barcodes have 10 to 14 bases, so + few files share that offset, and the anchor could keep one or two barcode bases and drop every + read of the samples whose barcode ended otherwise. On a salmon DArTag plate (1,101 samples), + detection from 8 random files returned the exact common sequence in 133 of 150 draws before and + in 150 of 150 now. With one or two FASTQs the anchor may still include barcode bases. It cuts + those files correctly, and `auto` warns not to reuse it for other samples. - For tiled markers, `generate` credits a read that fits several tiles of one amplicon to each of them, instead of dropping it as a tie. Tiles are markers that read the same tag window, with a shared or nested left primer and Ref tags that agree except at the markers' variant positions. diff --git a/madcats.py b/madcats.py index 09b08f3..6f403c6 100644 --- a/madcats.py +++ b/madcats.py @@ -1999,22 +1999,26 @@ def fastq_records(path: Path, leading_anchor: str = "") -> Iterator[Tuple[str, s LEADING_ANCHOR_PREFIX = 15 # bases of a panel tag that identify where the amplicon starts in a read -LEADING_ANCHOR_SAMPLE = 2000 # reads of the first FASTQ inspected by the raw-delivery check +LEADING_ANCHOR_SAMPLE = 2000 # reads inspected by the raw-delivery check, split evenly over the sampled FASTQs LEADING_ANCHOR_MIN_LENGTH = 8 # a detected anchor must be at least this long -LEADING_ANCHOR_FILES = 8 # FASTQs the check samples, so per-sample barcodes vary and only the shared part survives +LEADING_ANCHOR_FILES = 24 # FASTQs the check samples: each has its own barcode, and the anchor is what they all share +LEADING_ANCHOR_MIN_FILES = 3 # with fewer, barcode bases the files happen to share cannot be told from the anchor def detect_leading_anchor(fastqs: Sequence[Path], tag_prefixes: Set[str], sample_reads: int = LEADING_ANCHOR_SAMPLE) -> Dict[str, object]: """Do these FASTQs look like a raw delivery, with a constant sequence in front of every amplicon? A read is *anchored* when a panel tag begins at its first base and *shifted* when a panel tag begins further in; the - bases before a shifted tag are barcode plus common sequence, and the - anchor is the suffix that at least 90% of those prefixes share. Reads - are sampled from several files because an inline barcode is constant - within a sample and must not be mistaken for the common sequence.""" + bases before a shifted tag are an inline barcode plus the common + sequence. The barcode is constant within a file and differs between + files, in length too, so each file gives one prefix, the most common + among its shifted reads, and the anchor is the longest suffix those + prefixes share. With a single file that is the whole prefix, barcode + included (``anchor_files`` says how many files defined the anchor).""" anchored = shifted = sampled = 0 - prefixes: List[str] = [] offsets: Counter = Counter() + file_prefixes: List[Tuple[str, int]] = [] # per file: its prefix, and the shifted reads behind it + file_offsets: List[int] = [] search = LEADING_ANCHOR_PREFIX + LEADING_ANCHOR_SEARCH + 20 # Spread the sample across the run: neighbouring files often share a # plate and part of their barcode, which must not be read as anchor. @@ -2025,6 +2029,8 @@ def detect_leading_anchor(fastqs: Sequence[Path], tag_prefixes: Set[str], sample per_file = max(1, sample_reads // max(1, len(files))) for fastq in files: taken = 0 + prefixes: Counter = Counter() + reads: List[str] = [] for sequence, _quality in fastq_records(Path(fastq)): sampled += 1 taken += 1 @@ -2034,42 +2040,59 @@ def detect_leading_anchor(fastqs: Sequence[Path], tag_prefixes: Set[str], sample for offset in range(1, min(len(sequence) - LEADING_ANCHOR_PREFIX, search)): if sequence[offset:offset + LEADING_ANCHOR_PREFIX] in tag_prefixes: shifted += 1 - prefixes.append(sequence[:offset]) + prefixes[sequence[:offset]] += 1 + reads.append(sequence) offsets[offset] += 1 break if taken >= per_file: break + # The file's prefix is its most common one, seen at least twice. + # That sets aside sequencing errors, and a tag found a base or two + # early: a candidate window that happens to begin with the last + # bases of the anchor (chance 1 in 4 per base). + prefix, count = prefixes.most_common(1)[0] if prefixes else ("", 0) + if count < 2: + continue + file_offsets.append(len(prefix)) + # A file that mixes barcodes has no prefix of its own: shorten the + # most common one until 90% of its shifted reads would be cut by it, + # which leaves what those barcodes share. + while prefix and sum(read.find(prefix, 0, len(prefix) + LEADING_ANCHOR_SEARCH) >= 0 for read in reads) < 0.9 * len(reads): + prefix = prefix[1:] + if len(prefix) >= LEADING_ANCHOR_MIN_LENGTH: + file_prefixes.append((prefix, len(reads))) anchor = "" + agreeing: List[str] = [] + if file_prefixes: + # Files holding 90% of the shifted reads must end in the same bases, + # so one failed sample does not hide the anchor. Among the files that + # do, every one must agree on each further base: a base that even one + # of them lacks is barcode, and an anchor that kept it would lose + # every read of the samples without it. + endings: Counter = Counter() + for prefix, reads_behind in file_prefixes: + endings[prefix[-LEADING_ANCHOR_MIN_LENGTH:]] += reads_behind + ending, support = endings.most_common(1)[0] + if support >= 0.9 * sum(endings.values()): + agreeing = [prefix for prefix, _reads_behind in file_prefixes if prefix.endswith(ending)] + anchor = os.path.commonprefix([prefix[::-1] for prefix in agreeing])[::-1] modal_offset = offsets.most_common(1)[0][0] if offsets else None - if prefixes: - # The amplicon starts at the modal offset; a tag found a base or two - # earlier is a candidate window that happens to begin with the same - # base as the prefix (chance 1 in 4 per base), so only prefixes of - # the modal length define the anchor. - modal_prefixes = [prefix for prefix in prefixes if len(prefix) == modal_offset] - length = 1 - while modal_prefixes and length <= modal_offset: - suffix, count = Counter(prefix[-length:] for prefix in modal_prefixes).most_common(1)[0] - if count < 0.9 * len(modal_prefixes): - break - anchor = suffix - length += 1 - if len(anchor) < LEADING_ANCHOR_MIN_LENGTH: - anchor = "" return { "fastqs": [str(fastq) for fastq in files], "reads_sampled": sampled, "anchored_rate": anchored / max(1, sampled), "shifted_rate": shifted / max(1, sampled), "offset": modal_offset, + "offsets": (min(file_offsets), max(file_offsets)) if file_offsets else None, "anchor": anchor, + "anchor_files": len(agreeing), } def resolve_leading_anchor(args: argparse.Namespace, fastqs: Sequence[Path], tag_prefixes: Set[str], command: str) -> str: """Settle ``--leading-anchor`` before any read pass: an explicit sequence - is used as given; ``auto`` detects it from the first FASTQ; with no - option the first FASTQ is still checked, and a run that would assign + is used as given; ``auto`` detects it from a sample of the FASTQs; with + no option the same sample is still checked, and a run that would assign almost nothing because the reads carry a constant prefix stops and names the anchor to use, rather than producing an empty table.""" value = getattr(args, "leading_anchor", None) @@ -2080,6 +2103,13 @@ def resolve_leading_anchor(args: argparse.Namespace, fastqs: Sequence[Path], tag report = detect_leading_anchor([Path(fastq) for fastq in fastqs], tag_prefixes) where = Path(fastqs[0]).name + (f" and {len(report['fastqs']) - 1} more" if len(report["fastqs"]) > 1 else "") looks_raw = report["shifted_rate"] >= 0.5 and report["anchored_rate"] < 0.05 and report["anchor"] + # Barcodes differ in length, so the tag starts at a range of offsets. + low, high = report["offsets"] or (report["offset"], report["offset"]) + tag_start = f"tag start at offset {low}" + (f" to {high}" if high != low else "") + # From one or two files the anchor may keep barcode bases they share. + # It still cuts these files correctly, so it is used, and flagged. + few = report["anchor"] and report["anchor_files"] < LEADING_ANCHOR_MIN_FILES + files_behind = f"{report['anchor_files']} FASTQ{'s' if report['anchor_files'] != 1 else ''}" if value and value.lower() == "auto": if not report["anchor"] or report["shifted_rate"] < 0.5: raise MadcError( @@ -2091,16 +2121,26 @@ def resolve_leading_anchor(args: argparse.Namespace, fastqs: Sequence[Path], tag args.leading_anchor_detected = True print( f"{command}: leading anchor {report['anchor']} detected in {where} " - f"(tag start at offset {report['offset']} in {report['shifted_rate']:.0%} of {report['reads_sampled']} reads)", + f"({tag_start}, in {report['shifted_rate']:.0%} of {report['reads_sampled']} reads)", file=sys.stderr, ) + if few: + print( + f"warning: the leading anchor was detected from {files_behind}, too few to tell the common sequence " + "from the sample barcode in front of it; it fits these reads, but do not reuse it for other samples", + file=sys.stderr, + ) return report["anchor"] if looks_raw: + caution = ( + f"; with {files_behind} that prefix may include the sample barcode, so it fits these files only" + if few else "" + ) raise MadcError( f"{command}: {where} looks like a raw delivery: {report['shifted_rate']:.0%} of the first " f"{report['reads_sampled']} reads carry a panel tag after a constant prefix ending in {report['anchor']} " - f"(tag start at offset {report['offset']}), and {report['anchored_rate']:.0%} start with one; rerun with " - f"--leading-anchor {report['anchor']} (or --leading-anchor auto)" + f"({tag_start}), and {report['anchored_rate']:.0%} start with one; rerun with " + f"--leading-anchor {report['anchor']} (or --leading-anchor auto){caution}" ) return "" diff --git a/tests/test_madcats.py b/tests/test_madcats.py index 4504bdf..2c78cbe 100644 --- a/tests/test_madcats.py +++ b/tests/test_madcats.py @@ -1,7 +1,9 @@ import argparse +import contextlib import csv import dataclasses import gzip +import io import json import random import tempfile @@ -1898,6 +1900,125 @@ def test_raw_delivery_is_detected_and_named(self): # an explicit value is used as given, without reading the FASTQ self.assertEqual(mp.resolve_leading_anchor(argparse.Namespace(leading_anchor="acgt"), raw, prefixes, "generate"), "ACGT") + @staticmethod + def _write_raw_reads(path, reads): + with Path(path).open("w") as handle: + for number, sequence in enumerate(reads): + handle.write(f"@r{number}\n{sequence}\n+\n{'I' * len(sequence)}\n") + return Path(path) + + def test_leading_anchor_detection_with_barcodes_of_several_lengths(self): + # DArT inline barcodes have 10 to 14 bases, so the tag starts at five + # offsets and only a few files share the commonest one. Here the three + # files with 12-base barcodes all end in GT: read at the commonest + # offset alone, the anchor grows by those two bases and every sample + # whose barcode ends otherwise loses its reads. Each file gives its + # own prefix instead, and the anchor is what all eight share. + anchor = "CGATGGATCCAGTGC" + tag, other = "ACGTACGTACGTACGTACGT", "TTGCAAGCTTGACCGATAGG" + barcodes = ("TTCATAAGCAGT", "GGTAAGCATTGT", "ACCTGAGTACGT", "CAGGTCATGA", + "TGACCAGTTAG", "GATTACAGGCATA", "CCATAGGATTGACT", "AGTTCCGGAG") + # A candidate window that begins one base early (the last anchor base, + # then the tag) is found first in a quarter of the reads; the file's + # commonest prefix is still the whole one. + prefixes = {tag[:mp.LEADING_ANCHOR_PREFIX], other[:mp.LEADING_ANCHOR_PREFIX], anchor[-1] + other[:mp.LEADING_ANCHOR_PREFIX - 1]} + with tempfile.TemporaryDirectory() as tmp: + raws = [ + self._write_raw_reads( + Path(tmp) / f"raw_{number}.fastq", + [barcode + anchor + tag + "CCGATC"] * 150 + [barcode + anchor + other + "CCGATC"] * 50, + ) + for number, barcode in enumerate(barcodes) + ] + report = mp.detect_leading_anchor(raws, prefixes) + self.assertEqual(report["anchor"], anchor) + self.assertEqual(report["anchor_files"], len(barcodes)) + self.assertEqual(report["offsets"], (10 + len(anchor), 14 + len(anchor))) + self.assertEqual(report["shifted_rate"], 1.0) + with self.assertRaises(mp.MadcError) as context: + mp.resolve_leading_anchor(argparse.Namespace(leading_anchor=None), raws, prefixes, "generate") + message = str(context.exception) + self.assertIn(f"--leading-anchor {anchor} (or", message) + self.assertIn("tag start at offset 25 to 29", message) + self.assertNotIn("these files only", message) + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + self.assertEqual(mp.resolve_leading_anchor(argparse.Namespace(leading_anchor="auto"), raws, prefixes, "generate"), anchor) + self.assertNotIn("warning", stderr.getvalue()) + + def test_leading_anchor_from_one_or_two_files_is_used_and_flagged(self): + # One file cannot separate its barcode from the common sequence, and + # two may share barcode bases. The anchor still cuts those files where + # the amplicon starts, so it is used, with a warning not to reuse it. + anchor = "CGATGGATCCAGTGC" + tag = "ACGTACGTACGTACGTACGT" + prefixes = {tag[:mp.LEADING_ANCHOR_PREFIX]} + barcodes = ("TTCATAAGTTGA", "GGTAAGCATGCA", "ACCTGAGTACGT") # the first two end in the same base + with tempfile.TemporaryDirectory() as tmp: + raws = [ + self._write_raw_reads(Path(tmp) / f"raw_{number}.fastq", [barcode + anchor + tag + "CCGATC"] * 200) + for number, barcode in enumerate(barcodes) + ] + for files, expected, count in ((raws[:1], barcodes[0] + anchor, "1 FASTQ,"), (raws[:2], "A" + anchor, "2 FASTQs,")): + report = mp.detect_leading_anchor(files, prefixes) + self.assertEqual(report["anchor"], expected) + self.assertEqual(report["anchor_files"], len(files)) + stderr = io.StringIO() + args = argparse.Namespace(leading_anchor="auto") + with contextlib.redirect_stderr(stderr): + self.assertEqual(mp.resolve_leading_anchor(args, files, prefixes, "generate"), expected) + self.assertIn(f"warning: the leading anchor was detected from {count}", stderr.getvalue()) + # the anchor keeps every read of the files it came from + for fastq in files: + self.assertEqual(len(list(mp.fastq_records(fastq, expected))), 200) + with self.assertRaises(mp.MadcError) as context: + mp.resolve_leading_anchor(argparse.Namespace(leading_anchor=None), files, prefixes, "generate") + self.assertIn(f"--leading-anchor {expected} (or", str(context.exception)) + self.assertIn("these files only", str(context.exception)) + # three files are enough: no warning + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + self.assertEqual(mp.resolve_leading_anchor(argparse.Namespace(leading_anchor="auto"), raws, prefixes, "generate"), anchor) + self.assertNotIn("warning", stderr.getvalue()) + + def test_leading_anchor_detection_with_mixed_and_failed_files(self): + anchor = "CGATGGATCCAGTGC" + tag = "ACGTACGTACGTACGTACGT" + prefixes = {tag[:mp.LEADING_ANCHOR_PREFIX]} + barcodes = ("TTCATAAGTTGC", "GGTAAGCATGCA", "ACCTGAGTACGT", "CAGGTCATGATG", "TGACCAGTTAG") + with tempfile.TemporaryDirectory() as tmp: + # A file that mixes barcodes: its commonest prefix covers 40% of + # the reads, and an anchor that kept that barcode would drop the + # rest. The file gives what its barcodes share. + mixed = self._write_raw_reads( + Path(tmp) / "mixed.fastq", + [barcode + anchor + tag + "CCGATC" for barcode, copies in zip(barcodes, (80, 30, 30, 30, 30)) for _ in range(copies)], + ) + report = mp.detect_leading_anchor([mixed], prefixes) + self.assertEqual(report["anchor"], anchor) + self.assertEqual(len(list(mp.fastq_records(mixed, report["anchor"]))), 200) + # A failed sample, with a handful of reads that carry a tag after + # something else, does not hide the anchor of the others. + raws = [ + self._write_raw_reads(Path(tmp) / f"raw_{number}.fastq", [barcode + anchor + tag + "CCGATC"] * 200) + for number, barcode in enumerate(barcodes) + ] + failed = self._write_raw_reads( + Path(tmp) / "failed.fastq", ["TTTTTTTTTTTTTTTTTTTTTT" + tag + "CCGATC"] * 6 + ["GGGGGGGGGGGGGGGGGGGGGGGGGGGGGG"] * 194, + ) + report = mp.detect_leading_anchor(raws + [failed], prefixes) + self.assertEqual(report["anchor"], anchor) + self.assertEqual(report["anchor_files"], len(raws)) + # Files that end in two different sequences share no anchor, and auto says so. + others = [ + self._write_raw_reads(Path(tmp) / f"other_{number}.fastq", [barcode + "TTAGGCATCAAGCTA" + tag + "CCGATC"] * 200) + for number, barcode in enumerate(barcodes[:3]) + ] + self.assertEqual(mp.detect_leading_anchor(raws[:3] + others, prefixes)["anchor"], "") + with self.assertRaises(mp.MadcError) as context: + mp.resolve_leading_anchor(argparse.Namespace(leading_anchor="auto"), raws[:3] + others, prefixes, "generate") + self.assertIn("found no constant prefix", str(context.exception)) + def test_denoise_neighbour_lookup_matches_the_scan(self): # The distance-1 path enumerates neighbours instead of scanning # every centroid; it must pick exactly the parents the scan picks. From b7ac068b98a6dd91625a1734575b2544ccf12980 Mon Sep 17 00:00:00 2001 From: Alexander Sandercock <39815775+alex-sandercock@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:52:49 -0500 Subject: [PATCH 4/5] generate: leading-anchor help and docs after the merge The help, the stop message and the docs still said that anchor detection reads the first FASTQ. Since the per-file detection, it samples up to 24 FASTQs spread across the run. - --leading-anchor help: says auto samples up to 24 FASTQs. It shrinks from 99 words, the longest of the 37 generate options, to 44, in the active voice. The search-window detail stays on the docs page. - --leading-anchor-mismatches help: 56 words to 35, active voice. - The raw-delivery stop message counts "sampled reads", not "the first" reads. - generate.md, the DArTag tutorial and DEVELOPMENT.md match, with the warning for runs of one or two FASTQs. The test count is 264. --- docs/DEVELOPMENT.md | 15 ++++++++------- docs/src/commands/generate.md | 4 ++-- docs/src/tutorials/dartag.md | 9 ++++++--- madcats.py | 23 ++++++++++------------- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index ee45894..17e8b00 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -46,11 +46,12 @@ which inputs they require: (`--leading-anchor-mismatches 1`, the default since 0.14.0; counted as `reads_leading_anchor_mismatch`). Preprocessed deliveries are primer-started and need no anchor. The anchor is part of the cache - key and the provenance line. `infer-design` takes the same option, so - the primer-free route reads raw deliveries too. Both commands check the - first FASTQ before any read pass. When almost no read starts with a - panel tag but most carry one after a constant prefix, the run stops and - names that prefix (the suffix shared by the bases in front of the tag), + key and the provenance line. `infer-design` takes `--leading-anchor` + as well, so the primer-free route reads raw deliveries too. Both commands + check a sample of the FASTQs (up to 24, spread across the run) before + any read pass. When almost no read starts with a panel tag but most + carry one after a constant prefix, the run stops and names that prefix + (the suffix that the sampled files' prefixes share), and `--leading-anchor auto` applies the detected sequence instead, recording it in the QC as detected. 2. **Vendor-MADC bootstrap** (`scripts/madc_to_reference_tags.py`). It @@ -4267,8 +4268,8 @@ By default sample columns use the unique FASTQ filename stem. python3 -m unittest discover -s tests -v ``` -The suite contains 261 unit/end-to-end tests in six files. -`test_madcats.py` (90) covers `generate` and the commands around it, +The suite contains 264 unit/end-to-end tests in six files. +`test_madcats.py` (93) covers `generate` and the commands around it, `test_call.py` (112) the genotype caller, `test_filter_paralogs.py` (49) the paralog filter, `test_tiled_markers.py` (5) tiled markers through all three, `test_example.py` (4) the bundled example and `test_docs.py` (1) the match diff --git a/docs/src/commands/generate.md b/docs/src/commands/generate.md index 6e8cbd6..130e5e1 100644 --- a/docs/src/commands/generate.md +++ b/docs/src/commands/generate.md @@ -61,8 +61,8 @@ The [Evidence Model](../reference/evidence-model.md) explains stages 2–6 in de | Option | Default | Description | |--------|---------|-------------| | `--min-identity ` | 0.80 | Minimum substitution-only identity to a Ref or Alt tag. A gapped alignment uses the same edit limit | -| `--leading-anchor ` | none | Constant sequence before the amplicon in raw deliveries (for DArT, `GATGGATCCAGTGC`). Each read is cut where the anchor ends (it must end within 40 bases beyond its own length). `auto` detects it from the first FASTQ | -| `--leading-anchor-mismatches {0,1}` | 1 | Substitutions allowed in the leading anchor when a read has no exact copy of it. With `1`, the read is cut after the one placement within a single substitution. Reads with no such placement, or more than one, count as `reads_no_leading_anchor`, and the reads it recovers count as `reads_leading_anchor_mismatch`. An anchor shorter than 12 bases is always matched exactly. `0` matches every anchor exactly, as in 0.13.0 | +| `--leading-anchor ` | none | Constant sequence before the amplicon in raw deliveries (for DArT, `GATGGATCCAGTGC`). MADCATS cuts each read where the anchor ends, and the anchor must end within 40 bases beyond its own length. `auto` detects it from a sample of up to 24 FASTQs. With one or two FASTQs the detected anchor may include the sample barcode, and the run warns about it | +| `--leading-anchor-mismatches {0,1}` | 1 | Substitutions allowed in the leading anchor. With `1`, MADCATS cuts a read that has no exact copy after its only one-substitution match, and counts it as `reads_leading_anchor_mismatch`. A read with no such match, or more than one, counts as `reads_no_leading_anchor`. Anchors under 12 bases must match exactly. `0` requires an exact match for every anchor, as in 0.13.0 | | `--primer-mismatches {0,1}` | 1 | Mismatches allowed while assigning a primer/anchor. With `0`, reads from Match haplotypes whose distinguishing SNP falls inside the anchor are lost | | `--gapped-rescue {on,off}` | `on` | Give reads that fail substitution-only identity one gapped alignment, so the counts include undesigned indels | | `--max-indel-length ` | 10 | Widest gap (alignment band) a gapped alignment may use | diff --git a/docs/src/tutorials/dartag.md b/docs/src/tutorials/dartag.md index 393f251..37cdce8 100644 --- a/docs/src/tutorials/dartag.md +++ b/docs/src/tutorials/dartag.md @@ -97,9 +97,9 @@ The defaults are the recommendation for diploid and polyploid DArTag projects al ## Raw DArT Deliveries Older DArT orders deliver reads with an inline barcode and the library's common sequence -`GATGGATCCAGTGC` in front of the amplicon, so nothing is primer-started. MADCATS checks the first -FASTQ before any read pass. When almost no read starts with a panel tag but most carry one after a -constant prefix, the run stops and names that prefix. Re-run with: +`GATGGATCCAGTGC` in front of the amplicon, so nothing is primer-started. MADCATS checks a sample +of the FASTQs (up to 24) before any read pass. When almost no read starts with a panel tag but most +carry one after a constant prefix, the run stops and names that prefix. Re-run with: ```bash madcats generate ... --leading-anchor auto @@ -107,6 +107,9 @@ madcats generate ... --leading-anchor auto madcats generate ... --leading-anchor GATGGATCCAGTGC ``` +With one or two FASTQs, the anchor that `auto` finds may include the sample barcode. It fits those +files, and the run warns not to reuse it for other samples. + A read whose anchor carries one sequencing error is still cut where the anchor ends, as long as there is only one such placement (`--leading-anchor-mismatches 1`, the default). The QC counts those reads as `reads_leading_anchor_mismatch`, and reads without the anchor as diff --git a/madcats.py b/madcats.py index 6a2ce13..a994b3f 100644 --- a/madcats.py +++ b/madcats.py @@ -2185,8 +2185,8 @@ def resolve_leading_anchor(args: argparse.Namespace, fastqs: Sequence[Path], tag if few else "" ) raise MadcError( - f"{command}: {where} looks like a raw delivery: {report['shifted_rate']:.0%} of the first " - f"{report['reads_sampled']} reads carry a panel tag after a constant prefix ending in {report['anchor']} " + f"{command}: {where} looks like a raw delivery: {report['shifted_rate']:.0%} of " + f"{report['reads_sampled']} sampled reads carry a panel tag after a constant prefix ending in {report['anchor']} " f"({tag_start}), and {report['anchored_rate']:.0%} start with one; rerun with " f"--leading-anchor {report['anchor']} (or --leading-anchor auto){caution}" ) @@ -5784,12 +5784,10 @@ def _add_generate_command(subparsers) -> None: "--leading-anchor", default=None, help=( - "constant sequence that precedes the amplicon in every read of a raw delivery (DArT raw " - "FASTQs carry an inline barcode and the common sequence GATGGATCCAGTGC); each read is cut " - f"to start right after its first occurrence within the first {LEADING_ANCHOR_SEARCH} bases " - "beyond the anchor's length, and reads without it are counted as reads_no_leading_anchor; " - "'auto' detects the anchor from the first FASTQ (default: none; preprocessed deliveries are " - "primer-started, and a run whose reads carry an undeclared constant prefix stops and names it)" + "constant sequence before the amplicon in raw deliveries (DArT: GATGGATCCAGTGC): cut each " + "read where it ends, and count reads without it as reads_no_leading_anchor; 'auto' detects it " + f"from up to {LEADING_ANCHOR_FILES} FASTQs (default: none; on a raw delivery, the run stops " + "and names the anchor)" ), ) assignment.add_argument( @@ -5798,11 +5796,10 @@ def _add_generate_command(subparsers) -> None: choices=(0, 1), default=LEADING_ANCHOR_MISMATCHES_DEFAULT, help=( - "substitutions allowed in the leading anchor when a read has no exact copy of it: with 1, a " - "read is cut after the one placement within a single substitution (none, or more than one, " - "counts as reads_no_leading_anchor), and such reads are counted as " - f"reads_leading_anchor_mismatch; an anchor shorter than {LEADING_ANCHOR_MISMATCH_MIN_LENGTH} bases " - f"is always matched exactly (default: {LEADING_ANCHOR_MISMATCHES_DEFAULT})" + "substitutions allowed in the leading anchor: with 1, cut a read that has no exact copy after " + "its only one-substitution match, and count it as reads_leading_anchor_mismatch; anchors under " + f"{LEADING_ANCHOR_MISMATCH_MIN_LENGTH} bases must match exactly " + f"(default: {LEADING_ANCHOR_MISMATCHES_DEFAULT})" ), ) assignment.add_argument( From a55418e584058190a1649b0cdbbfd835ea531b0f Mon Sep 17 00:00:00 2001 From: Alexander Sandercock <39815775+alex-sandercock@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:07:26 -0500 Subject: [PATCH 5/5] generate: short anchors keep their cache key with --leading-anchor-mismatches 1 An anchor shorter than 12 bases is always matched exactly, so the mismatch setting cannot change its read pass. It still entered the cache key, so a default run rebuilt short-anchor caches for nothing. The key now needs the anchor to be long enough for a substitution, as the matcher does. Found by Copilot review on PR #6. --- madcats.py | 9 +++++++-- tests/test_madcats.py | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/madcats.py b/madcats.py index a994b3f..65bc09c 100644 --- a/madcats.py +++ b/madcats.py @@ -351,8 +351,13 @@ def cache_arguments(self) -> Dict[str, object]: if self.tile_families: arguments["tile_families"] = [list(family) for family in self.tile_families] # Likewise keyed only when it changes the read pass: exact-anchor and - # anchor-free runs keep the fingerprints (and caches) they had before. - if self.leading_anchor and self.leading_anchor_mismatches: + # anchor-free runs keep the fingerprints (and caches) they had before, + # and so do anchors too short to be matched with a substitution. + if ( + self.leading_anchor + and self.leading_anchor_mismatches + and len(self.leading_anchor) >= LEADING_ANCHOR_MISMATCH_MIN_LENGTH + ): arguments["leading_anchor_mismatches"] = self.leading_anchor_mismatches return arguments diff --git a/tests/test_madcats.py b/tests/test_madcats.py index 6287a39..5c62e07 100644 --- a/tests/test_madcats.py +++ b/tests/test_madcats.py @@ -1960,6 +1960,10 @@ def test_leading_anchor_mismatch_recovers_reads_and_is_keyed_only_when_used(self no_anchor = mp.AssignmentSettings(**{**base, "leading_anchor": ""}, leading_anchor_mismatches=1) self.assertNotIn("leading_anchor_mismatches", no_anchor.cache_arguments()) self.assertEqual(exact.cache_arguments(), mp.AssignmentSettings(**base).cache_arguments()) + # An anchor too short for a substitution is matched exactly, so 1 keeps the key 0 gives. + short = {**base, "leading_anchor": anchor[:mp.LEADING_ANCHOR_MISMATCH_MIN_LENGTH - 1]} + self.assertEqual(mp.AssignmentSettings(**short, leading_anchor_mismatches=1).cache_arguments(), + mp.AssignmentSettings(**short, leading_anchor_mismatches=0).cache_arguments()) def test_infer_fuzzy_pass_cuts_raw_reads_at_the_anchor(self): # The fuzzy pass once cleared the anchor it had just stored and voted