diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0c691a9..0344786 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,12 @@ jobs: matrix: os: [ubuntu-latest] python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + exclude: + - os: ubuntu-latest + python: "3.9" include: + - os: ubuntu-24.04 + python: "3.9" - os: macos-latest python: "3.13" runs-on: ${{ matrix.os }} @@ -25,6 +30,13 @@ jobs: with: python-version: ${{ matrix.python }} + - name: Install the optional genome aligner (filter-paralogs tests) + run: | + # mappy has no wheel for this macOS Python, and its source does not compile as the + # universal (arm64 + x86_64) binary that setup-python's interpreter asks for. + if [ "$RUNNER_OS" = "macOS" ]; then export ARCHFLAGS="-arch $(uname -m)"; fi + python -m pip install mappy + - name: Run tests run: python -m unittest discover -s tests -v diff --git a/.gitignore b/.gitignore index 9c780c5..1c71d17 100644 --- a/.gitignore +++ b/.gitignore @@ -185,3 +185,6 @@ docs/book/ # macOS .DS_Store examples/madcats-demo/results/ + +# minimap2 indexes built by filter-paralogs +*.mmi diff --git a/README.md b/README.md index 343d86e..d1198b2 100644 --- a/README.md +++ b/README.md @@ -9,18 +9,19 @@ `madcats` is a command-line tool that turns targeted-sequencing FASTQs into MADC microhaplotype count tables. It counts the alleles a panel was designed for and finds additional ones in the reads. -The output is a raw MADC table, which goes through HapApp before dosage calling and VCF conversion. +The output is a raw MADC table, which goes through HapApp before dosage calling and VCF conversion, +or straight to `madcats call` for genotypes and VCFs. The same code handles DArTag, AgriSeq (Ion Torrent), GT-seq, rhAmpSeq and Flex-Seq capture data. It needs only the Python 3 standard library. Every analysis threshold is a command-line option, and each run writes a JSON file recording its parameters and read counts. -> **A prior MADC is not an input.** Generation uses only the FASTQs plus the assay design: the +> **A prior MADC is not an input.** Generation uses only the FASTQs plus the assay design, either the > DArTag primer/probe files, a probe design alone, or a platform-neutral Ref/Alt tag manifest. ## Highlights -- Closely matches vendor output: read counts correlate at r = 0.9997 with the vendor MADC on a +- Closely matches vendor output. Read counts correlate at r = 0.9997 with the vendor MADC on a sweetpotato DArTag panel, and dosage calls agree 95–99% across five DArTag panels. - Designed to be platform independent, with testing performed on DArTag, GT-Seq, rhAmpSeq, Flex-Seq, and AgriSeq. - Builds the panel from vendor design files (`madcats design`), from the probe design and the reads @@ -34,8 +35,13 @@ each run writes a JSON file recording its parameters and read counts. paired-end reads (`madcats merge-pairs`). - Writes raw MADC tables for HapApp, which prepares them for BIGr and other tools. Each row also lists its variants and marker type. -- Pure Python with no dependencies. Runs samples in parallel and caches per-sample results, so - reruns are fast. +- Calls diploid and polyploid genotypes from the MADC (`madcats call`) and writes VCFs of the + designed targets, of all tag variants, or of whole microhaplotypes. +- Optional genome-based paralog filter (`madcats filter-paralogs`, needs + `pip install 'madcats[genome]'`) for panels where co-amplified paralogous copies inflate + heterozygosity. It works on vendor and HapApp fixed-allele-ID MADCs too. +- Pure Python with no dependencies for everything else. Runs samples in parallel and caches + per-sample results, so reruns are fast. ## Installation @@ -53,11 +59,11 @@ To keep it isolated from other Python packages, use [pipx](https://pipx.pypa.io/ ### Pre-built Binaries -Standalone executables that need no Python installation are attached to each -[release](https://github.com/Breeding-Insight/MADCATS/releases): -- `madcats-linux-x86_64.tar.gz`: Linux -- `madcats-macos-x86_64.tar.gz`: macOS (Intel) -- `madcats-macos-aarch64.tar.gz`: macOS (Apple Silicon) +Each [release](https://github.com/Breeding-Insight/MADCATS/releases) includes standalone +executables that need no Python installation: +- `madcats-linux-x86_64.tar.gz` for Linux +- `madcats-macos-x86_64.tar.gz` for macOS (Intel) +- `madcats-macos-aarch64.tar.gz` for macOS (Apple Silicon) ```bash # Download and extract (example: Linux) @@ -103,6 +109,12 @@ madcats validate \ --generated results/project_MADC.csv \ --expected vendor_MADC.csv \ --report results/validation.json + +# 4) Optional: call genotypes and write a VCF +madcats call \ + --madc results/project_MADC.csv \ + --ploidy 4 \ + --output-prefix results/project ``` No primer file? Infer the panel from the probe design and the reads: @@ -114,7 +126,7 @@ madcats infer-design \ --output reference_tags.csv ``` -Reads that are not primer-started (for example AgriSeq on Ion Torrent) are normalized first: +Normalize reads that are not primer-started (for example AgriSeq on Ion Torrent) first: ```bash madcats normalize-reads --probe-design markers.tsv --fastq-dir raw_fastqs/ --out-dir normalized/ --jobs 8 @@ -127,13 +139,13 @@ madcats generate --fastq-dir normalized/ --fastq-glob "*.fastq" \ Full documentation is available at [madcats.readthedocs.io](https://madcats.readthedocs.io), including: -- [Getting Started](https://madcats.readthedocs.io/en/latest/getting-started/installation.html): installation and first steps -- [Tutorials](https://madcats.readthedocs.io/en/latest/tutorials/dartag.html): walkthroughs for DArTag, primer-free, AgriSeq and paired-end data -- [Command Reference](https://madcats.readthedocs.io/en/latest/commands/overview.html): every subcommand and option -- [Evidence Model](https://madcats.readthedocs.io/en/latest/reference/evidence-model.html): how alleles are counted, discovered and denoised +- [Getting Started](https://madcats.readthedocs.io/en/latest/getting-started/installation.html) for installation and first steps +- [Tutorials](https://madcats.readthedocs.io/en/latest/tutorials/dartag.html) with step-by-step examples for DArTag, primer-free, AgriSeq and paired-end data, the paralog filter and genotype calling +- [Command Reference](https://madcats.readthedocs.io/en/latest/commands/overview.html) for every subcommand and option +- [Evidence Model](https://madcats.readthedocs.io/en/latest/reference/evidence-model.html) on how MADCATS counts, discovers and denoises alleles -The design rationale and version-by-version measurements behind every default are recorded in the -[development record](https://github.com/Breeding-Insight/MADCATS/blob/main/docs/DEVELOPMENT.md). +The [development record](https://github.com/Breeding-Insight/MADCATS/blob/main/docs/DEVELOPMENT.md) +gives the design rationale and version-by-version measurements behind every default. ## Commands @@ -146,6 +158,8 @@ The design rationale and version-by-version measurements behind every default ar | `madcats window-scan` | Sweep candidate tag lengths and recommend one from the project's own reads | | `madcats generate` | Generate the MADC count table | | `madcats validate` | Compare a generated MADC against a reference MADC | +| `madcats filter-paralogs` | Drop or flag MADC rows that a reference genome places at another locus (needs `mappy`) | +| `madcats call` | Call genotypes from a MADC and write target, all-variant or haplotype VCFs | Run `madcats --help` for full options. @@ -162,18 +176,18 @@ Chr01_000123,ACGTTGCAGGTCAAGT,ACGTTGCAGGTCAAGTTCA...,ACGTTGCAGGTCAAGTTCG... `LocusGroup` are optional. ### Probe design (TSV/CSV) -`MarkerName` and `TargetSequence` with one bracketed variant: a SNP such as `[C/T]` or an +`MarkerName` and `TargetSequence` with one bracketed variant, either a SNP such as `[C/T]` or an anchored indel such as `[A/ACTTG]`. ### FASTQs -One FASTQ (optionally gzipped) per sample in a folder. Sample columns are named by the FASTQ stem, -or by a `--sample-lookup` table. +One FASTQ (optionally gzipped) per sample in a folder. MADCATS names sample columns by the FASTQ +stem, or by a `--sample-lookup` table. ## Output | File | Contents | |------|----------| -| `project_MADC.csv` | MADC table: `AlleleID`, `CloneID`, `AlleleSequence`, summary columns, one read-count column per sample | +| `project_MADC.csv` | MADC table with `AlleleID`, `CloneID`, `AlleleSequence`, summary columns and one read-count column per sample | | `project_MADC.csv.qc.json` | Parameters, provenance, and per-sample read accounting | | `project_MADC.csv.variants.csv` | One line per variant event per row (with `--annotations on`, the default) | @@ -185,13 +199,13 @@ compared with the reference genotypes. | Platform | Accuracy | Validated against | |----------|----------|-------------------| -| DArTag (sweetpotato, red clover, blackberry, alfalfa, pecan) | count r = 0.9997; dosage agreement 95.4–98.8% | vendor MADC, vendor/updog dosage, families, skim-seq VCF | -| AgriSeq, Ion Torrent (honeybee, oyster) | 94.9% vs Torrent VariantCaller; 98.0% replicate concordance | vendor caller + replicates | +| DArTag (sweetpotato, red clover, blackberry, alfalfa, pecan) | count r = 0.9997, dosage agreement 95.4–98.8% | vendor MADC, vendor/updog dosage, families, skim-seq VCF | +| AgriSeq, Ion Torrent (honeybee, oyster) | 94.9% vs Torrent VariantCaller, 98.0% replicate concordance | vendor caller + replicates | | GT-seq (delta smelt) | 99.49% | independent BWA/bcftools truth VCF | | Flex-Seq capture (cranberry) | 95.13% genotype agreement | published population VCF + 20 trios | | rhAmpSeq (8 grape families) | 99.92–99.99% family transmission | Mendelian constraints + same-reads head-to-head | -Details: [Validation & Accuracy](https://madcats.readthedocs.io/en/latest/reference/validation.html). +See [Validation & Accuracy](https://madcats.readthedocs.io/en/latest/reference/validation.html) for details. ## Development @@ -209,12 +223,12 @@ If MADCATS is useful in your work, please cite: > Sandercock, A.M. (2026). MADCATS: Marker-Anchored Discovery and Counting of Alleles from Targeted > Sequencing. GitHub repository: https://github.com/Breeding-Insight/MADCATS -*(A formal article/DOI is planned; this section will be updated when available.)* +*(A formal article/DOI is planned, and this section will be updated when available.)* For the MADC format, see Breeding Insight's [MADC file format overview](https://breeding-insight.github.io/BIGr_documentation/posts/madc-file-format/). -**AI assistance:** MADCATS was developed in collaboration with Claude Opus 5.5 and Fable 5.1 +**AI assistance.** MADCATS was developed in collaboration with Claude Opus 5.5 and Fable 5.1 (Anthropic), which helped write its code and documentation. The authors directed the work, reviewed and tested the results, and are ultimately responsible for the application. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 9519c80..6addbc4 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -2,11 +2,11 @@ **Marker-Anchored Discovery and Counting of Alleles from Targeted Sequencing** -> This is the long-form development record: design rationale, version-by-version +> This long-form development record holds design rationale, version-by-version > changes, and the measurements behind every default. It was the project README > through 0.12.1. For installation and everyday usage, start with the > [README](../README.md) or the [documentation site](https://madcats.readthedocs.io). -> Command examples below use `python3 madcats.py`; after `pip install madcats` +> Command examples below use `python3 madcats.py`. After `pip install madcats` > the same commands run as `madcats `. A platform-independent tool for recovering design-specified alleles and @@ -18,97 +18,99 @@ origins as a FASTQ-to-MADC workflow, where MADC denotes the Missing Allele Discovery Counts format.) **A prior MADC is not an input to the workflow.** Generation uses the FASTQs -plus the assay design: either the DArTag primer/probe files or a +plus the assay design, either the DArTag primer/probe files or a platform-neutral Ref/Alt tag manifest. -Every generation run consumes one panel definition: a platform-neutral +Every generation run consumes one panel definition, a platform-neutral reference-tag CSV holding each marker's primer-started, fixed-length Ref and Alt tag sequences. Three routes produce that definition, differing only in which inputs they require: -1. **Design route** (`design` command): the vendor's primer CSV plus the - bracket-notation probe-design file. Deterministic reconstruction from - design files alone — no sequencing data involved. Use when both files - are available; it is the reference implementation the other routes are - validated against. A tag is the primer-started window that carries the - bracketed variant, so a primer occurrence in a duplicated flank that - never reaches the variant is not a candidate (the pecan 3K panel has one - such marker; before this rule it was reported as ambiguous). Any DArT +1. **Design route** (`design` command). It takes the vendor's primer CSV + plus the bracket-notation probe-design file and reconstructs the tags + deterministically from these design files alone, with no sequencing + data involved. Use when both files are available. It is the reference + implementation for validating the other routes. A tag is the + primer-started window that carries the bracketed variant, so a primer + occurrence in a duplicated flank that never reaches the variant is not + a candidate (the pecan 3K panel has one such marker, which this route + reported as ambiguous before the rule). Any DArT marker table with `MarkerName` and a bracket-notation `TargetSequence` serves as the probe design, extra markers and columns included. Raw DArT deliveries (older orders) keep an inline barcode and the - library's common sequence `GATGGATCCAGTGC` in front of every amplicon; + library's common sequence `GATGGATCCAGTGC` in front of every amplicon. `--leading-anchor GATGGATCCAGTGC` cuts each read where that sequence ends (it must end within 40 bases beyond its own length) and counts reads without it as `reads_no_leading_anchor`. Preprocessed deliveries are primer-started and need no anchor. The anchor is part of the cache - key and the provenance line; `infer-design` takes the same option, so + key and the provenance line. `infer-design` takes the same option, so the primer-free route reads raw deliveries too. Both commands check the - first FASTQ before any read pass: when almost no read starts with a + first FASTQ before any read pass. When almost no read starts with a panel tag but most carry one after a constant prefix, the run stops and names that prefix (the suffix shared by the bases in front of the tag), and `--leading-anchor auto` applies the detected sequence instead, recording it in the QC as detected. -2. **Vendor-MADC bootstrap** (`scripts/madc_to_reference_tags.py`): any - delivered MADC from the same panel. The vendor's canonical Ref/Alt rows - carry the exact tags, their first difference gives the variant position, - and a tag prefix substitutes for the primer as the assignment anchor. - Use when a prior vendor report exists but design files are private; the - result reproduces the vendor's tags exactly, including their IUPAC - curation. Markers that cannot become loci — above all Ref and Alt rows +2. **Vendor-MADC bootstrap** (`scripts/madc_to_reference_tags.py`). It + takes any delivered MADC from the same panel. The vendor's canonical + Ref/Alt rows carry the exact tags, their first difference gives the + variant position, and a tag prefix substitutes for the primer as the + assignment anchor. Use when a prior vendor report exists but design + files are private. The result reproduces the vendor's tags exactly, + including their IUPAC curation. The script announces markers that + cannot become loci with a `WARNING:` per reason and lists them with + their reason in `OUTPUT.skipped.tsv`. The main case is Ref and Alt rows that carry the same sequence, which means the target lies one base past - the window (or the design is presence/absence) — are announced with a - `WARNING:` per reason and listed with their reason in - `OUTPUT.skipped.tsv`; on one DArTag project that was 46 of 3,086 - markers, invisible until the sidecar existed. -3. **Primer-free inference** (`infer-design` command): the probe-design - file plus the project's own FASTQs, with the primer file never consulted. - Candidate tag windows are enumerated from the design targets and - selected by exact read-prefix votes (an identity-envelope fuzzy pass - covers loci with no exact support). Use when the primer file is withheld - and no vendor MADC exists; accuracy is measured per project in the - validation sections below. + the window (or the design is presence/absence). On one DArTag project + that was 46 of 3,086 markers, invisible until the sidecar existed. +3. **Primer-free inference** (`infer-design` command). It takes the + probe-design file plus the project's own FASTQs and never consults the + primer file. It enumerates candidate tag windows from the design + targets and selects among them by exact read-prefix votes (an + identity-envelope fuzzy pass covers loci with no exact support). Use + when the primer file is withheld and no vendor MADC exists. The + validation sections below measure its accuracy per project. A hand-curated CSV from any other source (for example a public panel -database) is equally valid input — the reference-tags CSV is the contract, -not its producer. +database) is equally valid input, because the reference-tags CSV is the +contract, not its producer. -Platforms whose reads are not primer-started take one extra stage first: +Platforms whose reads are not primer-started take one extra stage first. `normalize-reads` locates each raw read against the probe-design targets by k-mer, orients it forward, and excises a fixed tag-length window containing the variant, emitting pipeline-ready per-sample FASTQs together with the matching window reference-tags CSV and a QC ledger. The window is the -platform-neutral substitute for a primer-defined tag; after normalization, +platform-neutral substitute for a primer-defined tag. After normalization, generation runs unchanged. Because the window is an analytical choice rather than a physical one, every later run of the same panel should pass -`--reuse-tags` with the first run's window CSV — freezing placement is -what keeps microhaplotype sequences comparable across sequencing runs. +`--reuse-tags` with the first run's window CSV. Freezing placement keeps +microhaplotype sequences comparable across sequencing runs. See the AgriSeq pathway section below. -Paired-end platforms add one geometry decision before normalization — -merge overlapping pairs (`merge-pairs`) or pool non-overlapping mates — -and panels may tile several windows onto one locus (`LocusGroup`); both -are 0.9.0 features documented in their own section below. Adapter -trimming is not required on any route: reads are located middle-out, and -adapter content beyond the fragment cannot enter a fully covered window. +Paired-end platforms add one geometry decision before normalization, +which is to merge overlapping pairs (`merge-pairs`) or pool +non-overlapping mates. Panels may also tile several windows onto one +locus (`LocusGroup`). Both are 0.9.0 features with their own section +below. No route requires adapter trimming, because reads are located +middle-out, and adapter content beyond the fragment cannot enter a fully +covered window. ## Performance at a glance -Five platforms validated through one engine, each against the strongest +One engine covers five platforms, each validated against the strongest comparator its data permits. "Usable windows" is the fraction of raw -reads that survive to clean, counted evidence at a panel window (see -the read-accounting guardrails below — some loss is correct rejection -of off-target and dimer reads, and external pipelines' percentages use -different denominators). +reads that survive filtering to become counted evidence at a panel +window (see the read-accounting guardrails below). Some loss is correct +rejection of off-target and dimer reads, and external pipelines' +percentages use different denominators. | platform (dataset) | reads -> usable windows | accuracy | validated against | |---|---|---|---| -| DArTag (hexaploid sweetpotato; diploid red clover; mixed-ploidy blackberry) | 56.5% of delivered records — vs the vendor's own 50.9% on the same FASTQs (19.6% of records are shorter than the 81-bp tag and unusable by any pipeline) | red clover 3-bin dosage agreement **96.98%** with the vendor table (DP>=20, 682,226 cells, 0.11.0; updog on both tables 99.73% at DP>=10, 0.9.x); sweetpotato count correlation r=0.9997 on 1.20M shared-row cells (0.11.0); blackberry dosage agreement 95.39% (967,299 cells); blackberry design tags 3,015/3,015 match the vendor canonicals (2,881 identical, 134 differ only by the vendor's IUPAC codes); alfalfa panel-paper validation set (two 2021 orders, 54-bp design, raw deliveries cut at the common sequence, 0.12.0): category agreement 99.60% and 99.83%, dosage agreement 98.40% and 97.40%, F1 and BC1 progeny haplotypes within the parental sets at 98.6–98.7% (vendor table 98.9%); pecan 3K (2023 order, 81-bp design, 0.12.0): category agreement 98.89%, dosage agreement **98.78%** on 1.02M cells, F1 progeny haplotypes within the parental set at 99.82% (vendor 99.86%), 93.3% agreement with skim-sequencing genotypes on 11 shared cultivars (vendor 93.3%) | vendor MADC + vendor/updog dosage calls + families + skim-sequencing VCF | -| AgriSeq, Ion Torrent (2 honeybee chips; 372-sample oyster) | 88.4% | **94.9%** vs Torrent VariantCaller (152,698 cells; core 97.8%; 94.5% before 0.10.0's indel recovery); honeybee replicate concordance **98.0%**, cross-chip 98.2% | vendor caller + replicates | -| GT-seq (96-sample delta smelt) | 84.5% merged | **99.49%** R1-only (98.80% before 0.10.0 — the gain is one real insertion allele the fixed-offset reading misclassified); 98.67% with tiles on +54% cells; end-of-chain (HapApp->BIGr->updog) **98.42%** targets + **98.79%** on 38K off-target SNPs; `infer-design` 368/368 primer-free | independent BWA/bcftools truth VCF | -| Flex-Seq capture (192-sample cranberry) | 81.9% (110-bp windows) | end-of-chain **95.13%** targets (295,068 cells); trio incompatibility **3.6%** vs 6.2% for the comparison VCF's own source; locus recovery 80.7%/99.84% vs vendor 83.1%/99.8% | published population VCF + 20 pedigree trios | -| rhAmpSeq (608 runs, 8 biparental grape families) | 86.6% (cohort-discovery panel; 79.3% on the original 16-sample-discovery panel) | family transmission **99.92-99.99%** (~14.9K locus-family tests); head-to-head vs the platform's own `analyze_amplicon` pipeline on identical reads: **93.0%** exact haplotype-genotype agreement, +19%/+79% more callable cells at 10x/30x, and MADCATS is the more Mendelian-consistent of the pair (99.947% vs 99.850% within the inferred parental four). Cohort-scale discovery (per-family union across all 8 families) designs 1,993/2,000 panel amplicons and delivers +43% genotype calls over the subset-discovery panel at held three-nines transmission, with every added locus 100.000% family-consistent (84,757 calls, zero violations) | Mendelian family constraints + same-reads tool head-to-head (no public truth matrix) | -| synthetic truth suite | — | dosage **100%** on clean/linked/paralog/repeat classes; 50.8% on engineered null alleles (the correct negative); paired-end merge 100% with 98.6% molecular recovery | exact simulated truth | +| DArTag (hexaploid sweetpotato, diploid red clover, mixed-ploidy blackberry) | 56.5% of delivered records, against the vendor's own 50.9% on the same FASTQs (19.6% of records are shorter than the 81-bp tag and unusable by any pipeline) | Red clover 3-bin dosage agreement with the vendor table is 96.98% (DP>=20, 682,226 cells, 0.11.0), and updog on both tables agrees at 99.73% (DP>=10, 0.9.x). Sweetpotato count correlation is r=0.9997 on 1.20M shared-row cells (0.11.0). Blackberry dosage agreement is 95.39% (967,299 cells), and blackberry design tags match the vendor canonicals at 3,015/3,015 (2,881 identical, 134 differ only by the vendor's IUPAC codes). The alfalfa panel-paper validation set (two 2021 orders, 54-bp design, raw deliveries cut at the common sequence, 0.12.0) has category agreement 99.60% and 99.83%, dosage agreement 98.40% and 97.40%, and F1 and BC1 progeny haplotypes within the parental sets at 98.6–98.7% (vendor table 98.9%). Pecan 3K (2023 order, 81-bp design, 0.12.0) has category agreement 98.89%, dosage agreement 98.78% on 1.02M cells, F1 progeny haplotypes within the parental set at 99.82% (vendor 99.86%), and 93.3% agreement with skim-sequencing genotypes on 11 shared cultivars (vendor 93.3%) | vendor MADC + vendor/updog dosage calls + families + skim-sequencing VCF | +| AgriSeq, Ion Torrent (2 honeybee chips, 372-sample oyster) | 88.4% | 94.9% vs Torrent VariantCaller (152,698 cells, core 97.8%, 94.5% before 0.10.0's indel recovery), and honeybee replicate concordance 98.0%, cross-chip 98.2% | vendor caller + replicates | +| GT-seq (96-sample delta smelt) | 84.5% merged | 99.49% R1-only (98.80% before 0.10.0, a gain from one real insertion allele the fixed-offset reading misclassified), 98.67% with tiles on +54% cells, end-of-chain (HapApp->BIGr->updog) 98.42% targets + 98.79% on 38K off-target SNPs, and `infer-design` 368/368 primer-free | independent BWA/bcftools truth VCF | +| Flex-Seq capture (192-sample cranberry) | 81.9% (110-bp windows) | end-of-chain 95.13% targets (295,068 cells), trio incompatibility 3.6% vs 6.2% for the comparison VCF's own source, and locus recovery 80.7%/99.84% vs vendor 83.1%/99.8% | published population VCF + 20 pedigree trios | +| rhAmpSeq (608 runs, 8 biparental grape families) | 86.6% (cohort-discovery panel, 79.3% on the original 16-sample-discovery panel) | family transmission 99.92-99.99% (~14.9K locus-family tests). Head-to-head against the platform's own `analyze_amplicon` pipeline on identical reads, MADCATS reaches 93.0% exact haplotype-genotype agreement, +19%/+79% more callable cells at 10x/30x, and the more Mendelian-consistent calls of the pair (99.947% vs 99.850% within the inferred parental four). Cohort-scale discovery (per-family union across all 8 families) designs 1,993/2,000 panel amplicons and delivers +43% genotype calls over the subset-discovery panel at held three-nines transmission, with every added locus 100.000% family-consistent (84,757 calls, zero violations) | Mendelian family constraints + same-reads tool head-to-head (no public truth matrix) | +| synthetic truth suite | — | dosage 100% on clean/linked/paralog/repeat classes, 50.8% on engineered null alleles (the correct negative), and paired-end merge 100% with 98.6% molecular recovery | exact simulated truth | Version 0.10.0 adds recovery of *undesigned* indel haplotypes through a general gapped reading (see its section below). On a private in-house @@ -117,49 +119,52 @@ that the vendor MADC lacked, 78% became MADC rows at 97% read recall (0.4% under 0.9.x), and the SNP calls on every truth-scored set above held or improved. -Accuracy tracks comparator quality more than platform: where truth is -clean (GT-seq, DArTag) MADCATS scores 98-99%+; against noisier -comparators (TVC, raw mpileup) the *mutual* agreement is 94-95% while +Accuracy tracks comparator quality more than platform. Where the truth +set is reliable (GT-seq, DArTag), MADCATS scores 98-99%+. Against noisier +comparators (TVC, raw mpileup) the *mutual* agreement is 94-95%, while internal instruments (replicates, trios) show MADCATS' own calls at the -cleaner end of each pair — on cranberry, measurably more +more accurate end of each pair. On cranberry they are measurably more Mendelian-consistent than the raw calls behind the comparison VCF. ## Important scope -DArT describes its analytical pipeline as proprietary. Exact vendor -equivalence therefore cannot be guaranteed because its adapter/quality -filtering, partial-read handling, consensus construction, and error-correction -rules are not documented. This implementation is an open and deterministic -replacement, not a claim to be DArT's undisclosed implementation. - -The generated file preserves the MADC fields needed for microhaplotype work: -`AlleleID`, `CloneID`, `AlleleSequence`, `ClusterConsensusSequence`, and one -integer read-count column per sample, following the vendor conventions +DArT describes its analytical pipeline as proprietary. MADCATS therefore +cannot guarantee exact vendor equivalence, because DArT does not document +its adapter/quality filtering, partial-read handling, consensus +construction, and error-correction rules. This implementation is an open +and deterministic replacement, not a claim to be DArT's undisclosed +implementation. + +The generated file preserves the MADC fields needed for microhaplotype work, +which are `AlleleID`, `CloneID`, `AlleleSequence`, +`ClusterConsensusSequence`, and one integer read-count column per sample, +following the vendor conventions observed in the supplied report (canonical rows carry their own sequence as -the consensus; Match/Other rows carry the Ref consensus). `AvgCountRef` and -`AvgCountSnp` are nonzero means, and `CallRate` is filled with the fraction of +the consensus, and Match/Other rows carry the Ref consensus). `AvgCountRef` and +`AvgCountSnp` are nonzero means, and `CallRate` holds the fraction of samples having canonical Ref or Alt reads at the locus. The remaining vendor dosage/genotype summary fields are downstream calls rather than FASTQ -evidence and are emitted as `.` (missing value; see the output-format -section). +evidence, so the pipeline emits them as `.` (a missing value, described in +the output-format section). -Matching is substitution-only first: reads are compared to same-length -tags at each position. **Designed indel markers are fully supported** — +Matching is substitution-only first, comparing reads to same-length +tags at each position. Designed indel markers are fully supported, because vendors report indel alleles as fixed-length windows whose suffix shifts at the indel, and both tags are ordinary same-length sequences to the counting -layer. Since 0.10.0, **undesigned indels are recovered too**: a read that -frameshifts against every canonical tag is offered one gapped reading under +layer. Since 0.10.0, undesigned indels are recovered too. A read that +frameshifts against every canonical tag gets one gapped reading under the same edit budget (gaps of up to `--max-indel-length` bases, 10 by -default, counted as edits), and is kept as an ordinary fixed-length -shifted-suffix row when that reading is composed like an indel haplotype -(few substitutions, few gap events). The target base is read through the -alignment. Out of scope remain indels longer than the band and gapped -readings that need many substitutions — those are co-amplified paralogs, -not alleles, and stay off-target. +default, count as edits), and the pipeline keeps it as an ordinary +fixed-length shifted-suffix row when that reading has the composition of +an indel haplotype (few substitutions, few gap events). The pipeline reads +the target base through the alignment. Indels longer than the band and +gapped readings that need many substitutions remain out of scope. Those +are co-amplified paralogs, not alleles, and stay off-target. ## Quick start for the supplied sweetpotato project -The defaults are the recommended configuration; no evidence flags are needed: +The defaults are the recommended configuration, so the command needs no +evidence flags: ```bash python3 madcats.py generate \ @@ -172,7 +177,7 @@ python3 madcats.py generate \ ``` A stricter, higher-precision view for high-depth projects raises the -reproducibility depth: add `--reproducible-min-reads 20`. +reproducibility depth with `--reproducible-min-reads 20`. To inspect or reuse the reconstructed 3,120 Ref/Alt tags, run the separate design command: @@ -185,8 +190,8 @@ python3 madcats.py design \ ``` Other targeted platforms can pass this same reference-tag format with -`--reference-tags`. The provided DArT report is used only after generation, -as an external validation target: +`--reference-tags`. The validate command uses the provided DArT report only +after generation, as an external validation target: ```bash python3 madcats.py validate \ @@ -199,104 +204,108 @@ python3 madcats.py validate \ For each marker, the pipeline locates the left primer in the supplied target sequence in either orientation, extracts the primer-started tag, verifies the -target-SNP position, and constructs the Alt tag by substitution. Each FASTQ -read is assigned by its left-primer prefix, retained as a raw full tag of the -configured length (81 bp by default), and accepted when it has at least 80% -substitution-only identity to Ref or Alt — or, when every candidate rejects -it, under one gapped reading that spends the same 20% edit budget -(`--gapped-rescue on`, the default; see the 0.10.0 section). - -Canonical Ref and Alt rows are always emitted, including at zero counts. A -**non-canonical sequence** becomes an output row when it reaches +target-SNP position, and constructs the Alt tag by substitution. It assigns +each FASTQ read by its left-primer prefix, retains it as a raw full tag of +the configured length (81 bp by default), and accepts it when it has at +least 80% substitution-only identity to Ref or Alt, or, when every +candidate rejects it, under one gapped reading that spends the same 20% +edit budget (`--gapped-rescue on`, the default, described in the 0.10.0 +section). + +The pipeline always emits canonical Ref and Alt rows, including at zero +counts. A non-canonical sequence becomes an output row when it reaches `--min-global-count` project-wide reads and passes at least one enabled evidence route: -- **Reproducibility route** (primary; default on): at least +- **Reproducibility route** (primary, default on). It requires at least `--reproducible-min-reads` reads (default 10) in each of at least `--reproducible-min-samples` samples (default 2). Identical 81-mers recurring independently across samples are strong evidence even when a base position is systematically low-quality. -- **Depth route** (default on): at least `--depth-min-reads` reads (default - 30) within a single sample. This keeps private alleles discoverable in - unreplicated designs such as diversity panels of singleton accessions; on - the replicated project below it adds a single row, because deep sequences - there always recur across the paired extracts. -- **Quality route** (default off): at least `--quality-min-clean-reads` reads - whose every tag base is at or above `--discovery-min-quality` (default - Q20). On the supplied project this route's marginal precision was 3% - (2,159 extra rows for 73 additional vendor-matching rows), and whole-tag - cleanliness vetoes real alleles with one systematically low-quality cycle, - so it is no longer a default gate. +- **Depth route** (default on). It requires at least `--depth-min-reads` + reads (default 30) within a single sample. This keeps private alleles + discoverable in unreplicated designs such as diversity panels of + singleton accessions. On the replicated project below it adds a single + row, because deep sequences there always recur across the paired extracts. +- **Quality route** (default off). It requires at least + `--quality-min-clean-reads` reads whose every tag base is at or above + `--discovery-min-quality` (default Q20). On the supplied project this + route's marginal precision was 3% (2,159 extra rows for 73 additional + vendor-matching rows), and whole-tag cleanliness vetoes real alleles + with one systematically low-quality cycle, so it is no longer a default + gate. An optional HaploCharmer-style dosage-aware floor, `--min-supporting-freq` -(default 0 = off), additionally requires a sample's reads for the sequence to +(default 0 = off), also requires a sample's reads for the sequence to reach that share of the sample's locus total before the sample can support the reproducibility or depth route. Measured on the supplied project, a 0.01 floor raises precision 2.2 points and costs 0.35 recall points (17,319 rows, 86.2%/77.2%), but it removes 33 of the vendor-confirmed rows the evidence -redesign recovered: amplicon pools inflated by co-amplified paralogs leave -real minor haplotypes at sub-percent shares, so the floor stays off by -default. It is most useful as an artifact guard on the depth route in -unreplicated designs, where around 0.01 is a reasonable starting point. +redesign recovered, because amplicon pools inflated by co-amplified +paralogs leave real minor haplotypes at sub-percent shares. The floor +therefore stays off by default. It is most useful as an artifact guard on +the depth route in unreplicated designs, where around 0.01 is a +reasonable starting point. Evidence thresholds are absolute read counts and therefore do not scale with sequencing depth. At the honeybee AgriSeq run's ~750x per sample per marker, 10 reads sits below the Ion error noise floor, and the defaults admitted 1,752 novel rows of which roughly 92% were error haplotypes -(median carrier share near zero). Sweep-backed guidance by depth regime: -at low depth (~60x) and at standard DArTag depth the defaults stand; at -very high depth (above roughly 500x per sample per marker) enable -`--min-supporting-freq 0.10` — on the honeybee run it collapsed the novel -rows to 147 while raising caller concordance from 89.8% to 90.6%, and -locus-share floors dominate rescaled absolute thresholds because -per-marker depth varies tenfold within a single run. +(median carrier share near zero). Sweeps back the following guidance by +depth regime. At low depth (~60x) and at standard DArTag depth the +defaults stand. At very high depth (above roughly 500x per sample per +marker), enable `--min-supporting-freq 0.10`. On the honeybee run it +collapsed the novel rows to 147 while raising caller concordance from +89.8% to 90.6%, and locus-share floors dominate rescaled absolute +thresholds because per-marker depth varies tenfold within a single run. Setting every route's threshold to 0 emits every sequence at or above `--min-global-count` (exhaustive discovery). The same rule protects qualifying sequences from abundance-skew denoising and gates partial-read extension, so discovery, protection, and extension stay consistent. Abundance-skew denoising (UNOISE-style, `2^(alpha*d+1)`, closest parent -first) then absorbs low-abundance neighbors within each sample; a pooled +first) then absorbs low-abundance neighbors within each sample. A pooled pass (`--global-denoise-distance`, default 1) afterwards folds an evidence-failing sequence into the protected parent the project-wide skew -names — but only when both carry the same target allele, so no move can +names, but only when both carry the same target allele, so no move can change a sample's Ref/Alt dosage (0.9.x moved reads across alleles, which -could turn a private 25/25 heterozygote into Ref=50). Evidence-passing and -canonical sequences are never absorbed. Reads whose sequences pass no route -remain visible in the QC read accounting rather than silently vanishing. At the default distance of 1 the eligible parents of a sequence are its single-substitution neighbours, which the denoiser enumerates and looks up rather than finding by scanning every centroid; the selection is identical, and a paralogous locus with thousands of unique sequences per sample (the pecan 3K panel has one) no longer makes the pass quadratic. +could turn a private 25/25 heterozygote into Ref=50). The denoiser never +absorbs evidence-passing and canonical sequences. Reads whose sequences +pass no route remain visible in the QC read accounting rather than silently vanishing. At the default distance of 1 the eligible parents of a sequence are its single-substitution neighbours, which the denoiser enumerates and looks up rather than finding by scanning every centroid. The selection is identical, and a paralogous locus with thousands of unique sequences per sample (the pecan 3K panel has one) no longer makes the pass quadratic. -Classification against the design is IUPAC-aware on the design side: with an +Classification against the design is IUPAC-aware on the design side. With an ambiguity-coded design allele, a compatible concrete observed base classifies as RefMatch/AltMatch. An ambiguous observed base such as N is missing data and classifies as Other, matching the vendor's convention. Ref/Alt category assignment and per-row counts always use exact sequence equality, so an -ambiguity-coded canonical tag reports zero exact counts; the pipeline does +ambiguity-coded canonical tag reports zero exact counts. The pipeline does not guess which concrete variants a curated IUPAC consensus was meant to absorb (DArT uses such codes to encode real secondary SNPs, so collapsing -them would merge genuine microhaplotypes — measured on the blackberry -panel). A read whose tag contains an N is never counted as a haplotype of -its own: it is credited to the one N-free sequence at its locus it is -compatible with, or dropped (`reads_n_mapped` / `reads_n_dropped`). - -Reads shorter than the tag length are salvaged by default -(`--partial-read-mode assign`): a trimmed prefix is credited to the one -learned full tag it matches, and only when it is longer than every prefix -shared between two learned alleles of its locus (see the 0.11.0 section +them would merge genuine microhaplotypes, as measured on the blackberry +panel). The pipeline never counts a read whose tag contains an N as a +haplotype of its own. It credits the read to the one N-free sequence at +its locus that the read is compatible with, or drops it +(`reads_n_mapped` / `reads_n_dropped`). + +By default (`--partial-read-mode assign`) the pipeline salvages reads +shorter than the tag length. It credits a trimmed prefix to the one +learned full tag it matches, and only when the prefix is longer than every +prefix shared between two learned alleles of its locus (see the 0.11.0 section "Truncated reads: partial-read salvage with a locus guard"). `off` excludes such -reads. A partial read never establishes an allele: a prefix of nothing -learned is left unmapped. Note that a partial read's quality is judged over +reads. A partial read never establishes an allele, so a prefix of nothing +learned stays unmapped. The pipeline judges a partial read's quality over its observed bases only. ## Recovering a panel without the primer file (`infer-design`) Some providers treat the primer file as proprietary. Its only irreplaceable -content is where each tag starts inside the probe-design target: the tag must +content is where each tag starts inside the probe-design target. The tag must be a tag-length window of the bracket-expanded target containing the SNP, so inference selects among a closed candidate set (~4x tag-length windows per locus) rather than reconstructing sequences freely. Because reads are primer-started, exact read prefixes identify the true window almost digitally, and Ref/Alt identity comes from bracket order (the first allele is -the reference-genome base — verified at all 3,120 loci of this panel with +the reference-genome base, verified at all 3,120 loci of this panel with zero swaps). ```bash @@ -307,59 +316,62 @@ python3 madcats.py infer-design \ --output inferred_reference_tags.csv ``` -Two evidence tiers, strictly ordered: exact read-prefix votes decide first -and are authoritative; an identity-envelope fuzzy pass (seeded matching -within the same 80% identity rule as generation, with already-resolved tags -acting as competition sinks) runs only for loci with zero exact support and -can never overrule exact evidence. Each locus needs a dominant window -(default 10x the runner-up); the written anchor (`LeftPrimerSequence`) is the -shortest panel-unique tag prefix, since the true primer boundary is -unobservable in seamless reads and unnecessary for assignment. `--tag-length` -sets the panel's reported tag length (54, 81, 109, ...); reads must reach it -and the design flanks must contain a full window, both validated per locus. - -Measured blinded on this project (primer file untouched, ~11 minutes): -3,068/3,120 loci inferred — 2,742 by exact evidence (100.0% correct against -the primer-derived truth) and 326 by the fuzzy tier (319 correct, 97.9%); -zero Ref/Alt swaps or variant-position errors; 52 loci excluded with stated -reasons (no reads within identity, support below threshold, or ambiguous -windows). Running `generate` on the inferred panel reproduces 99.1% of the -full-input MADC's rows (novel Match/Other rows: 99.65%) with 100% -classification agreement; count totals run 0.6% high because the short -anchors reject fewer error-carrying reads than the truer, longer primers. +Inference uses two evidence tiers in strict order. Exact read-prefix votes +decide first and are authoritative. An identity-envelope fuzzy pass +(seeded matching within the same 80% identity rule as generation, with +already-resolved tags acting as competition sinks) runs only for loci +with zero exact support and can never overrule exact evidence. Each locus +needs a dominant window (default 10x the runner-up). The written anchor +(`LeftPrimerSequence`) is the shortest panel-unique tag prefix, since +the reads do not show the true primer boundary and assignment does not +need it. `--tag-length` +sets the panel's reported tag length (54, 81, 109, ...). Reads must reach it +and the design flanks must contain a full window, and inference checks +both per locus. + +In a blinded run on this project (primer file untouched, ~11 minutes), +inference resolved 3,068/3,120 loci, 2,742 by exact evidence (100.0% +correct against the primer-derived truth) and 326 by the fuzzy tier (319 +correct, 97.9%). It made zero Ref/Alt swaps or variant-position errors +and excluded 52 loci with stated reasons (no reads within identity, +support below threshold, or ambiguous windows). Running `generate` on +the inferred panel reproduces 99.1% of the full-input MADC's rows (novel +Match/Other rows 99.65%) with 100% classification agreement. Count totals +run 0.6% high because the short anchors reject fewer error-carrying reads +than the truer, longer primers. The `*.qc.json` report records each locus's route, support, margin, anchor -length, and flags. The fuzzy tier is the review tier: its errors win by +length, and flags. The fuzzy tier is the review tier. Its errors win by large margins (paralog haplotype pools), so no internal statistic can catch -them — cross-check `fuzzy_anchor` loci against any delivered MADC's +them. Cross-check `fuzzy_anchor` loci against any delivered MADC's canonical rows or a public panel asset, and prefer those sources outright when available (a single delivered MADC reconstructs the panel exactly). -Inference is a one-time operation per panel; freeze its CSV and reuse it. +Inference is a one-time operation per panel, so freeze its CSV and reuse it. `scripts/validate_inferred_design.py` compares an inferred panel against a ground-truth reference-tag CSV, with per-route accuracy breakdowns. ## Migration from 0.4.x -Version 0.5.0 changed the discovery semantics, so the evidence flags were -renamed to make old commands fail loudly rather than silently change meaning: +Version 0.5.0 changed the discovery semantics and renamed the evidence +flags so that old commands fail loudly rather than silently change meaning: - `--min-quality-support` became `--quality-min-clean-reads` (and the route - is now optional and off by default); + is now optional and off by default). - `--novel-min-reads-per-sample`/`--novel-min-samples` became `--reproducible-min-reads`/`--reproducible-min-samples`, and the reproducibility test moved from a post-hoc AND filter into the core - evidence rule (routes combine by OR); -- `--depth-min-reads` is new; -- `--allele-catalogue`/`--known-only` were removed: fixed allele-ID + evidence rule (routes combine by OR). +- `--depth-min-reads` is new. +- `--allele-catalogue`/`--known-only` are gone, because fixed allele-ID assignment and the cross-project allele database are downstream HapApp responsibilities, and reprocessing against a frozen catalogue is no longer - this pipeline's job; -- the per-sample cache format version rose to 0.4.0 (fuzzy primer matches now + this pipeline's job. +- The per-sample cache format version rose to 0.4.0 (fuzzy primer matches now resolve across all matched depths by tag identity instead of preferring the - deepest primer), so caches are rebuilt on first use. Rebuilding is cheap: - assignment decisions are memoized per unique sequence, which made - per-sample processing 7.1x faster; the full 80-sample, 154.6M-read project - processes from scratch in about 4.5 minutes with 8 workers and regenerates - from a warm cache in about a minute. + deepest primer), so caches rebuild on first use. Rebuilding is cheap + because the pipeline memoizes assignment decisions per unique sequence, + which made per-sample processing 7.1x faster. The full 80-sample, + 154.6M-read project processes from scratch in about 4.5 minutes with 8 + workers and regenerates from a warm cache in about a minute. ## Validation on the supplied project @@ -379,80 +391,81 @@ are exactly equal, and the shared count total is 2.0% above DArT's. ### Why the evidence model changed -The 0.4.x discovery gate required project-wide reads whose **every** tag base +The 0.4.x discovery gate required project-wide reads whose every tag base reaches Q20. Auditing the raw caches showed this vetoed 286 vendor rows that -are present as exact 81-bp FASTQ prefixes with strong support — up to 6,626 -reads spread across as many as 70 samples — because one base position is -systematically low-quality (for many, below Q10, so no clean-read threshold -can rescue them). Symmetrically, the old `10x4` post-filter removed 598 -vendor rows that had clean-read support but narrower sample spread. The -reproducibility-primary rule recovers both classes; every vetoed-but- -reproducible vendor row is present in the 0.5.0 default output. +are present as exact 81-bp FASTQ prefixes with strong support, up to 6,626 +reads spread across as many as 70 samples, because one base position is +systematically low-quality (for many, below Q10, so no clean-read +threshold can rescue them). Symmetrically, the old `10x4` post-filter +removed 598 vendor rows that had clean-read support but narrower sample +spread. The reproducibility-primary rule recovers both classes, and every +vetoed-but-reproducible vendor row is present in the 0.5.0 default output. The vendor's own file shows DArT applies no within-project support threshold -of this kind: 27.3% of its positive-count rows have fewer than 10 reads in +of this kind. Of its positive-count rows, 27.3% have fewer than 10 reads in fewer than 2 samples of its own counts, 4,081 rows have under 10 total reads, and 1,417 rows are all-zero. Its row list reflects proprietary clustering -plus a cross-project haplotype database — reproducibility evidence at the -database level, which a single-project de novo run cannot see. The +plus a cross-project haplotype database, which supplies reproducibility +evidence at the database level that a single-project de novo run cannot see. The within-project evidence rule is therefore this pipeline's explicit, configurable substitute, not an imitation of a vendor threshold. ### Independent validation: red clover (diploid, 109-bp tags, low depth) -The DCl25-10856 red clover project (376 samples, 3,000 loci, 109-bp tags, -mean read length ~88 bp so only ~65% of reads reach the tag length, ~60x mean -locus depth) was processed with no primer file: the probe-design targets plus -`infer-design` recovered 2,968 loci — 2,902 exactly matching the vendor's -canonical tags, 56 differing only by the vendor's IUPAC curation, and the -remaining handful confined to loci where the vendor's own tags have zero -reads. `scripts/madc_to_reference_tags.py` converted the delivered MADC into -the equivalent panel (3,000/3,000) as the cross-check route. +MADCATS processed the DCl25-10856 red clover project (376 samples, 3,000 +loci, 109-bp tags, mean read length ~88 bp so only ~65% of reads reach the +tag length, ~60x mean locus depth) with no primer file. The probe-design +targets plus `infer-design` recovered 2,968 loci. Of these, 2,902 match the +vendor's canonical tags exactly, 56 differ only by the vendor's IUPAC +curation, and the remaining handful sit at loci where the vendor's own +tags have zero reads. `scripts/madc_to_reference_tags.py` converted the +delivered MADC into the equivalent panel (3,000/3,000) as the +cross-check route. Generation at the defaults gave 85.4% precision and 62.9% raw recall with -100% classification agreement and a 0.9975 shared-count total ratio. The raw -recall is dominated by a structural ceiling: 36.7% of the vendor's -positive-count rows fail every within-project evidence route on the vendor's -own counts (their cross-project catalogue at low depth); against +100% classification agreement and a 0.9975 shared-count total ratio. A +structural ceiling dominates the raw recall. Of the vendor's +positive-count rows, 36.7% fail every within-project evidence route on +the vendor's own counts (their cross-project catalogue at low depth). Against evidence-bearing vendor rows, recall is 94.1%. Softening to `--reproducible-min-reads 5 --depth-min-reads 15` captured nearly the whole recoverable band (67.2% recall against a 67.0% ceiling) at a 10-point -precision cost — but carrier-frequency profiling shows why the bare soft -view is not recommended: its vendor-shared additions look like real alleles +precision cost. Carrier-frequency profiling shows why the bare soft +view is not recommended. Its vendor-shared additions look like real alleles (median 44% of their carrier's locus reads, matching the 48% of established -rows), while its generated-only additions sit at a median 7.5% — not a -diploid genotype. The defaults remain the recommendation for diploid and +rows), while its generated-only additions sit at a median 7.5%, which is +not a diploid genotype. The defaults remain the recommendation for diploid and polyploid projects alike. When a low-depth diploid project needs more -sensitivity, add the diploid-justified frequency floor: +sensitivity, add the diploid-justified frequency floor. With it, `--reproducible-min-reads 5 --depth-min-reads 15 --min-supporting-freq 0.10` recovers about 40% of the available recall band while excluding the -artifact-profile rows, and its extra rows should still be confirmed by -segregation or replicates downstream. On this project the pre-0.10 partial +artifact-profile rows, and segregation or replicates should still confirm +its extra rows downstream. On this project the pre-0.10 partial salvage (minimum length 30, no locus guard) added the ~24M primer-bearing short reads to counts without changing the row set, and lowered count concordance with the vendor (Pearson 0.976 to 0.941), showing the vendor -does not count short fragments; the 0.10.0 default (minimum length +does not count short fragments. The 0.10.0 default (minimum length `--tag-length` minus 9) excludes those fragments, and `--partial-read-mode off` restores the exact pre-salvage counts when vendor comparability matters. Segregation-based validation closed the loop on this project. The run contains a 152-progeny bi-parental mapping family with both parents, and -both MADCs were pushed through the same updog (diploid) + BIGr workflow. +both MADCs went through the same updog (diploid) + BIGr workflow. Updog imputes dosages at low or zero depth (about a third of calls sit at DP=0), so all comparisons are depth-filtered. Dosage concordance between the two pipelines' calls is 99.73% at DP>=10 in both (99.77% at DP>=20). -Mendelian-violation rates in the family are statistically identical — +Mendelian-violation rates in the family are statistically identical, at 3.38% for this pipeline versus 3.35% for the vendor MADC (parents DP>=20, -progeny DP>=10, MPP>=0.9) — so the shared violation floor reflects the +progeny DP>=10, MPP>=0.9), so the shared violation floor reflects the assay and family, not either counting implementation. Family transmission of microhaplotype rows (present at >=5 reads in >=2 progeny must appear in -a parent) adjudicates the discovery tiers directly: vendor-shared novel -rows are parent-confirmed at 97.4% (the assay background), the default -configuration's generated-only rows at 72.0% — majority genuinely -transmitted alleles absent from the vendor's catalogue — and the soft -band's generated-only rows at only 28.4%, independently confirming the -carrier-frequency verdict that the bare soft view admits mostly artifacts. -The symmetric test on the vendor's own counts completes the picture: only +a parent) adjudicates the discovery tiers directly. Parents confirm +vendor-shared novel rows at 97.4% (the assay background), the default +configuration's generated-only rows at 72.0% (mostly transmitted alleles +absent from the vendor's catalogue), and the soft band's generated-only +rows at only 28.4%. That independently confirms the carrier-frequency +verdict that the bare soft view admits mostly artifacts. +The symmetric test on the vendor's own counts completes the picture. Only 313 of their 10,175 exclusive novel rows segregate in the family at all (the rest are read-starved catalogue content this project can neither validate nor falsify), and those confirm at 67.7% against a 97.8% @@ -460,12 +473,12 @@ consensus background on the same count matrix. Each pipeline's exclusive rows therefore carry the same real-but-imperfect profile (72.0% versus 67.7%, indistinguishable at n=313), while this pipeline contributes about three times as many transmission-validated alleles (~633) as it forgoes -(~212) — and the forgone ones sit in known, tunable territory just below -the evidence thresholds. +(~212), and the forgone ones sit in known, tunable territory immediately +below the evidence thresholds. -The canonical Ref/Alt rows — the input to downstream dosage calling — -compare to the vendor's as follows. Of our 5,936 canonical rows, 5,827 -(98.2%) are sequence-identical to the vendor's; the 109 differences are 89 +The canonical Ref/Alt rows, which are the input to downstream dosage +calling, compare to the vendor's as follows. Of our 5,936 canonical rows, 5,827 +(98.2%) are sequence-identical to the vendor's. The 109 differences are 89 vendor-IUPAC-curated sequences plus the Ref/Alt pairs of the 10 loci whose vendor tags carry zero reads in this project. Across the 2.19 million matched count cells, Pearson correlation is 0.975 with a mean absolute @@ -474,86 +487,87 @@ Ref/(Ref+Alt) ratio, evaluated over the 499,414 locus-by-sample cells where both files have at least 20 reads of canonical depth, the median ratio difference is 0.0000 (95th percentile 0.015) and 99.68% of cells imply the identical naive diploid genotype class. The zero-count canonical structure -also mirrors the vendor's (644 all-zero rows versus their 604) — the -germplasm-divergent loci both pipelines observe. +also mirrors the vendor's (644 all-zero rows versus their 604), reflecting +the germplasm-divergent loci both pipelines observe. ### Independent validation: blackberry (mixed ploidy, 109-bp tags, indels, degenerate primers) The DBla25-10927 blackberry project (376 samples of mixed *Rubus* species and ploidies, 3,015 loci, 109-bp tags, ~59-70% of reads reaching tag length) is the panel that motivated version 0.7.0's indel support and is -the first project validated through **all three panel-definition routes**. -Its design contains 10 indel markers — encoded with anchored VCF-style +the first project validated through all three panel-definition routes. +Its design contains 10 indel markers, encoded with anchored VCF-style brackets in the probe-design file but unanchored dash notation in the -primer CSV — and, as the data revealed, primers synthesized with +primer CSV. As the data revealed, it also has primers synthesized with degenerate IUPAC bases over flanking SNPs, whose oligo sequence the -vendor's tags carry verbatim. DArT's delivered sample names were -misassigned for this project (a correction crosswalk exists); this -pipeline's targetid-named columns are immune to that class of error. +vendor's tags carry verbatim. DArT misassigned the delivered sample names +for this project (a correction crosswalk exists), but this pipeline's +targetid-named columns are immune to that class of error. Version 0.7.0 replaced the SNP-only window construction with one general -rule — the divergence point of a tag pair is the shared prefix of the two -full allele expansions — which handles SNPs, insertions, deletions, +rule, that the divergence point of a tag pair is the shared prefix of the +two full allele expansions. The rule handles SNPs, insertions, deletions, anchored brackets, and homopolymer shifts identically, and reconstructs tags as primer-sequence overlay plus genomic suffix so degenerate oligo bases persist. SNP-panel outputs are byte-identical through the change. Route-by-route results against the vendor's canonical tags: -- **Design route**: 2,881/3,015 exact; all 134 differences are vendor-IUPAC - loci; zero unexplained. All 10 indel loci reconstruct exactly, including - divergence positions shifted past homopolymer runs. -- **Bootstrap route**: exact by construction (3,015/3,015 converted, none - skipped, indels included). Generation from this panel: 92.9% precision, - 73.7% raw recall against a 75.7% within-project evidence ceiling (23.7% - of the vendor's positive rows carry no such evidence on their own - counts), **94.7% fair recall** — the third project inside the 94-95% - band — with 100% canonical row recovery, Pearson 0.981, and 99.2% - classification agreement (557 of the 639 disagreements sit at - IUPAC-canonical loci and 68 at indel loci, where the positional variant - heuristic and vendor curation legitimately differ). Family transmission - in the reciprocal Columbia Star x Kotata families (both parents - sequenced, 92 progeny) reproduces the red clover profile: - vendor-shared novel rows 98.5% parent-confirmed (n=12,064), - generated-only rows 74.0% (n=2,027); the test asserts only that progeny - alleles exist in a parent, so unknown hexaploid inheritance and - segmental allopolyploidy do not affect it. -- **Primer-free inference**: 2,943/3,015 loci inferred (72 excluded with - stated reasons); 2,796 exact plus 126 concrete realizations of - vendor-IUPAC tags — identical under IUPAC compatibility, the +- **Design route.** 2,881/3,015 tags are exact, all 134 differences are + vendor-IUPAC loci, and none is unexplained. All 10 indel loci + reconstruct exactly, including divergence positions shifted past + homopolymer runs. +- **Bootstrap route.** The tags are exact by construction (3,015/3,015 + converted, none skipped, indels included). Generation from this panel + gives 92.9% precision, 73.7% raw recall against a 75.7% within-project + evidence ceiling (23.7% of the vendor's positive rows carry no such + evidence on their own counts), and 94.7% fair recall, the third project + inside the 94-95% band. It also gives 100% canonical row recovery, + Pearson 0.981, and 99.2% classification agreement (557 of the 639 + disagreements sit at IUPAC-canonical loci and 68 at indel loci, where + the positional variant heuristic and vendor curation legitimately + differ). Family transmission in the reciprocal Columbia Star x Kotata + families (both parents sequenced, 92 progeny) reproduces the red clover + profile, with vendor-shared novel rows 98.5% parent-confirmed + (n=12,064) and generated-only rows 74.0% (n=2,027). The test asserts + only that progeny alleles exist in a parent, so unknown hexaploid + inheritance and segmental allopolyploidy do not affect it. +- **Primer-free inference.** Inference resolved 2,943/3,015 loci and + excluded 72 with stated reasons. It found 2,796 exact tags, 126 concrete + realizations of vendor-IUPAC tags, and 18 wrong windows (0.6%). The + realizations are identical under IUPAC compatibility, the information-theoretic limit when the oligo's degeneracy is unknowable - without the primer file — and 18 wrong windows (0.6%). Of the 10 indel - loci, 9 resolve exactly by read votes (one via the fuzzy tier at a locus - whose canonicals carry no reads) and 1 is excluded as ambiguous rather - than guessed. - -The 18 wrong windows decompose into three classes, none an inference -defect in read-bearing data: 8 are read-starved loci anchored on scraps; -2 are short-amplicon loci (72-bp and 105-bp amplicons, real reads that -never reach tag length); and ~8 are loci where the vendor counted + without the primer file. Of the 10 indel loci, 9 resolve exactly by + read votes (one via the fuzzy tier at a locus whose canonicals carry no + reads), and inference excludes 1 as ambiguous rather than guessing it. + +The 18 wrong windows fall into three classes, none an inference +defect in read-bearing data. Of these, 8 are read-starved loci anchored +on scraps, 2 are short-amplicon loci (72-bp and 105-bp amplicons, real +reads that never reach tag length), and ~8 are loci where the vendor counted thousands of reads for tags occurring zero times in the delivered FASTQs in any form. The leading explanation for the last class is the vendor's pre-delivery overrepresented-read filtering (quantified for sweetpotato -above): dominant amplicons are precisely what such a filter removes, the +above). Dominant amplicons are precisely what such a filter removes, the vendor counts predate it, and the affected loci are then unknowable to any FASTQ-based pipeline. -A germplasm note from the same run: the project's two "Marion" samples -are genetically different individuals (canonical-count correlation 0.47 -where technical replicates approach 1); one is near-clonal with the -"Newberry" sample (r=0.958) and the other matches nothing in the panel — -a plate-provenance question for curation, caught by targetid-level -fingerprinting. +The same run raised a germplasm question. The project's two "Marion" +samples are genetically different individuals (canonical-count +correlation 0.47 where technical replicates approach 1). One is +near-clonal with the "Newberry" sample (r=0.958) and the other matches +nothing in the panel. Targetid-level fingerprinting caught this +plate-provenance question for curation. ### AgriSeq (Ion Torrent) pathway: honeybee validation -AgriSeq reads violate the primer-started model three ways: amplicons are -sequenced from both ends (~50/50 orientation split), primer digestion and -primers placed outside the submitted targets disperse read starts, and Ion -Torrent's dominant error is the homopolymer indel (a median 24% of reads -per marker carry one across ~100 bp on the honeybee run, up to 77% at -run-rich loci — the rate climbs monotonically with the longest run in the -window). The pathway is two commands: +AgriSeq reads violate the primer-started model in three ways. The platform +sequences amplicons from both ends (~50/50 orientation split), primer +digestion and primers placed outside the submitted targets disperse read +starts, and Ion Torrent's dominant error is the homopolymer indel (a +median 24% of reads per marker carry one across ~100 bp on the honeybee +run, up to 77% at run-rich loci, and the rate climbs monotonically with +the longest run in the window). The pathway is two commands: ```bash # first run of a panel: learn window placement from observed coverage @@ -571,153 +585,156 @@ madcats.py generate --fastq-dir normalized/ --fastq-glob "*.fastq" \ --output project_MADC.csv ``` -The 41-bp default window is measured, not assumed: sweeping 41/61/81 bp on +The 41-bp default window comes from a measurement. Sweeping 41/61/81 bp on the 81-marker honeybee validation (24.2M reads, 81 samples of mixed drone -and worker material) showed longer windows lose coverage and indel-free -reads faster than they add context — end-to-end read yield 64%/59%/44% and -genotype call rate 96.6%/93.9%/85.6% — with the 81-bp concordance uptick a -survivor artifact of scoring only the easiest cells. Data-driven window +and worker material) showed that longer windows lose coverage and +indel-free reads faster than they add context, with end-to-end read yield +64%/59%/44% and genotype call rate 96.6%/93.9%/85.6%. The 81-bp +concordance uptick is a survivor artifact of scoring only the easiest +cells. Data-driven window placement (per marker, from observed read coverage, keeping the variant out of the anchor) raised adapter yield from 66% to 85.5% of raw reads. Since 0.8.1, auto placement scores coverage minus a soft tandem-repeat penalty (`--str-penalty`, default 0.002 per repeat bp beyond 4, unit -sizes 1-3): the original homopolymer tiebreak provably never fired, +sizes 1-3). The original homopolymer tiebreak provably never fired, because real coverage fractions are never exactly tied, and it was blind to dinucleotide repeats entirely. Genotypes called from target-base dosage agree with the vendor's Torrent caller at 90.6%, and both pipelines agree with an independent pre-existing -reference genotype set at 94.4% versus 94.5% — a dead heat, with disputed -cells splitting 48:48 and the residual disagreement concentrated at a -handful of three-way-inconsistent markers — most of which the second-chip +reference genotype set at 94.4% versus 94.5%, a dead heat. Disputed +cells split 48:48, and the residual disagreement concentrates at a +handful of three-way-inconsistent markers, most of which the second-chip study below traced to window placement rather than to the loci -themselves. Row accuracy against the Amel_HAv3.1 genome: all 81 +themselves. For row accuracy against the Amel_HAv3.1 genome, all 81 real markers' Ref tags occur verbatim at their named coordinates and every -Alt tag is structurally exact; the only four failures are the vendor's +Alt tag is structurally exact. The only four failures are the vendor's four design-failed markers, whose submitted target sequences do not match -the genome at their named positions — a pre-submission target-versus- -genome check would have caught them before synthesis. Caveats: a single +the genome at their named positions. A pre-submission target-versus-genome +check would have caught them before synthesis. The caveats are a single project, two imperfect arbiters, and mixed-ploidy material, which the row-level checks tolerate by construction. ### Second chip, replicates, and window freezing (0.8.1) -A second honeybee chip on the same panel (96 wells: 65 unique samples, 30 -within-chip technical replicate pairs, one no-template control; 40.8M -reads at 2.5x the first chip's depth) ran the recipe unchanged: 84.8% of -raw reads normalized, the control clean at 119 reads against a -160,000-read median sample, vendor-caller concordance 92.4%, and — the -first truth-free accuracy instrument — 98.2% genotype concordance across -the 30 replicate pairs. The depth-scaling law reproduced exactly: -default evidence thresholds admitted 4,410 novel rows at this depth -(1,752 on chip 1), while `--min-supporting-freq 0.10` pinned the count -near 145 on both chips and improved every concordance metric, settling -the frequency floor as the AgriSeq default. +A second honeybee chip on the same panel (96 wells holding 65 unique +samples, 30 within-chip technical replicate pairs and one no-template +control, with 40.8M reads at 2.5x the first chip's depth) ran the recipe +unchanged. It normalized 84.8% of raw reads, its control held only 119 +reads against a 160,000-read median sample, and vendor-caller +concordance was 92.4%. The first truth-free accuracy instrument, genotype +concordance across the 30 replicate pairs, reached 98.2%. The +depth-scaling law reproduced. Default evidence thresholds admitted 4,410 +novel rows at this depth (1,752 on chip 1), while +`--min-supporting-freq 0.10` pinned the count near 145 on both chips and +improved every concordance metric, settling the frequency floor as the +AgriSeq default. The chips also exposed a stability failure with consequences for -microhaplotype databases: re-learning window placement per run moved 42 +microhaplotype databases. Re-learning window placement per run moved 42 of 85 windows, which made 58% of the second chip's haplotype sequences -spuriously "novel" against the first chip — technical artifacts +spuriously "novel" against the first chip. Those technical artifacts are indistinguishable from biology in any downstream database. Freezing the panel with `--reuse-tags` (the emitted window CSV is byte-identical to the source panel, every row validated against the design, partial mismatches refused) raised cross-chip haplotype sharing from 42% to 88%, -with the remainder consistent with genuinely new germplasm on the second -chip. +with the remainder consistent with new germplasm on the second chip. Window placement also turned out to be the dominant cause of apparently -bad markers, through allele-biased read dropout: where a variant allele +bad markers, through allele-biased read dropout. Where a variant allele extends an adjacent tandem repeat, its reads carry longer runs and die preferentially inside any window spanning the repeat, faking monomorphism. Surgical single-marker re-windowing rescued four of the five worst markers on both chips (9%->100%/96%, 27%->99%/96%, 48%->100%/99% vendor concordance), leaving only two markers plus one -sub-amplicon-spaced marker cluster genuinely bad on every window tried — -the true redesign list. The settled production workflow is: first chip -with penalized auto placement, patch the few coverage-forced windows the -cross-run comparison exposes, freeze the curated panel, and run every -subsequent chip with `--reuse-tags`. The curated frozen honeybee panel +sub-amplicon-spaced marker cluster bad on every window tried. Those form +the true redesign list. The settled production workflow is to run the +first chip with penalized auto placement, patch the few coverage-forced +windows the cross-run comparison exposes, freeze the curated panel, and +run every subsequent chip with `--reuse-tags`. The curated frozen honeybee panel scores 93.4%/93.9% vendor concordance on the two chips with 98.0% replicate concordance. Re-sweeping window sizes under the new scoring (31/41/61 bp, frequency floor on) confirmed 41 bp at the knee of the concordance-versus- -information curve: 31 bp genotypes marginally better (92.7% vs 91.8%) -but carries 20% fewer distinct discovered microhaplotypes (110 vs 137), -while 61 bp adds essentially none (138) and loses two concordance points -— on this chemistry, indel exposure destroys the extra flanking -variation as fast as a longer window adds it. Curation matters more than -size: the curated 41-bp panel outscores every uncurated size. +information curve. At 31 bp, genotyping is marginally better (92.7% vs +91.8%) but the panel carries 20% fewer distinct discovered +microhaplotypes (110 vs 137), while 61 bp adds almost none (138) and +loses two concordance points. On this chemistry, indel exposure destroys +the extra flanking variation as fast as a longer window adds it. +Curation matters more than size. The curated 41-bp panel outscores every +uncurated size. ### External benchmarks on public datasets (GT-seq, rhAmpSeq) Public datasets extend the validation record beyond in-house data, with -intake converters archived beside each dataset (the Flex-Seq capture -benchmark is documented in the 0.9.0 section below). +intake converters archived beside each dataset (the 0.9.0 section below +documents the Flex-Seq capture benchmark). -**Pacific oyster AgriSeq** (PRJNA1118744; 372 Ion S5 samples, 591 +**Pacific oyster AgriSeq** (PRJNA1118744, 372 Ion S5 samples, 591 amplicons of median 170 bp, design published as Ion AmpliSeq Region + Hotspot BEDs converted against the oyster_v9 genome with zero REF -mismatches): with the honeybee-derived defaults unmodified, 88.4% of +mismatches). With the honeybee-derived defaults unmodified, 88.4% of 144.6M raw reads normalized and genotypes agreed with the vendor's -Torrent VariantCaller at **94.52%** (143,383/151,693 cells across 371 +Torrent VariantCaller at 94.52% (143,383/151,693 cells across 371 name-matched samples, depth >= 20), 97.85% on the 460/542 markers above -90% — within half a point of the curated in-house honeybee panel, on a -foreign lab, species, chip generation, and target length. One panel-size -effect appeared on schedule: 12-bp assignment anchors collide at 591 -markers (the birthday bound predicts it), resolved at the time with -`--anchor-length 13`. Since 0.8.3, normalize-reads assigns anchors per -locus instead (the same allocator infer-design uses): every marker keeps +90%. This result is within half a point of the curated in-house honeybee panel, +on a foreign lab, species, chip generation, and target length. One +panel-size effect appeared on schedule. Assignment anchors of 12 bp +collide at 591 markers (the birthday bound predicts it), which +`--anchor-length 13` resolved at the time. Since 0.8.3, normalize-reads +assigns anchors per locus instead (the same allocator infer-design uses). +Every marker keeps the `--anchor-length` minimum, only colliding prefix groups grow, and windows identical all the way to the variant share an anchor with a surfaced warning (their reads resolve by full-tag identity downstream). -A 17,500-marker genome-wide stress panel measured the necessity — 111 -colliding 12-mer groups, far above the random-sequence bound — and runs +A 17,500-marker genome-wide stress panel measured the necessity, with 111 +colliding 12-mer groups, far above the random-sequence bound. It runs in ~820 MB with 17,291 markers at the minimum and 209 grown to 13-32 bp. -**Delta smelt GT-seq** (Zenodo 10.5281/zenodo.19069550; 96 PE100 -samples, 410 loci, and — uniquely — the panel's real primer file): +**Delta smelt GT-seq** (Zenodo 10.5281/zenodo.19069550, 96 PE100 +samples, 410 loci, and, uniquely, the panel's real primer file). GT-seq R1 reads are primer-started, so this platform needs no normalize-reads stage and exercised the primer-informed `design` route on non-DArT data for the first time, unchanged. Genotypes from the R1-only MADC agreed with the archive's independent BWA/bcftools -reference VCF at **98.80%** (33,928/34,340 cells; 99.72% on the -well-behaved core; per-sample median 98.9% across all 96 samples) — the -pipeline's best platform result, on indel-free Illumina chemistry. The -same dataset settled the primer-free question: `infer-design`, run with -the primer file withheld, recovered **368/368 loci's Ref and Alt tags -exactly** as the physical primers define them — a 100% inference score +reference VCF at 98.80% (33,928/34,340 cells, 99.72% on the +well-behaved core, per-sample median 98.9% across all 96 samples). That +was the pipeline's best platform result, on indel-free Illumina +chemistry. The same dataset settled the primer-free question. With the +primer file withheld, `infer-design` recovered 368/368 loci's Ref and Alt +tags exactly as the physical primers define them, a 100% inference score against ground truth rather than a vendor report. -**Vitis rhAmpSeq** (PRJNA281110, Cornell; the published 2,000-marker -collinear-core panel, NextSeq PE150, 200-300-bp amplicons): the panel +**Vitis rhAmpSeq** (PRJNA281110, Cornell, the published 2,000-marker +collinear-core panel, NextSeq PE150, 200-300-bp amplicons). The panel was designed on the collinear core of multiple *Vitis* genomes, so many -markers have no exact home on any single reference — intake is -**reference-free**: each marker's target frame is the consensus of its +markers have no exact home on any single reference. Intake is therefore +reference-free, and each marker's target frame is the consensus of its own primer-anchored read pileup. An R1-only first pass recovered 1,405 -markers at 63% read assignment; the paired-end route then measured the -geometry (94.8% of pairs overlap and merge — the platform's own +markers at 63% read assignment. The paired-end route then measured the +geometry (94.8% of pairs overlap and merge, and the platform's own published pipeline merges with FLASH, independently confirming -merge-first) and rebuilt consensus from full merged amplicons: **1,759 -markers (+25%), frames of median 228 bp, up to three 110-bp tiles per -amplicon** (2,951 markers, 1,030 multi-SNP amplicons), 79.3% read +merge-first) and rebuilt consensus from full merged amplicons. That gave +1,759 markers (+25%), frames of median 228 bp, up to three 110-bp tiles +per amplicon (2,951 markers, 1,030 multi-SNP amplicons), and 79.3% read extraction with `covered` routing. The tiled windows capture ~90% of all segregating sites (production's full-insert haplotypes span ~174 bp -and capture all sites plus cross-tile phase; the trade buys fixed-length, -frozen, cross-run-comparable MADC haplotypes). The merger itself was -validated against FLASH 1.2.11, vsearch, and fastp on a four-regime -exact-truth pair set — equal to the best (vsearch) in every regime, -zero false merges, and on real NextSeq data the only inter-tool +and capture all sites plus cross-tile phase, and the trade buys +fixed-length, frozen, cross-run-comparable MADC haplotypes). Tests +against FLASH 1.2.11, vsearch, and fastp on a four-regime exact-truth +pair set put the merger equal to the best (vsearch) in every regime, +with zero false merges. On real NextSeq data the only inter-tool differences are equal-quality overlap ties, where a per-locus pileup jury rules our R1-preferring pick correct 62.6% of the time. No public genotype matrix exists for these accessions, but family -structure is its own truth: progeny of a biparental cross can carry at -most four haplotypes per locus (two per parent), each individual at -most two. All **eight biparental families in the submission (763 runs, -~17.5 GB) were processed family-by-family under a 56 GB disk budget** -against one frozen 2,951-window panel (`--reuse-tags`), so haplotypes -are identical objects across families: +structure is its own truth. Progeny of a biparental cross can carry at +most four haplotypes per locus (two per parent), and each individual at +most two. All eight biparental families in the submission (763 runs, +~17.5 GB) went through the pipeline family-by-family under a 56 GB disk +budget against one frozen 2,951-window panel (`--reuse-tags`), so +haplotypes are identical objects across families: | family | progeny | informative loci | <=4 family haps | <=2 per progeny | calls within inferred parental 4 | |---|---|---|---|---|---| @@ -729,54 +746,56 @@ are identical objects across families: (Three additional tiny families, n = 2-7, are consistent at 99.5-100%.) Across ~14,900 locus-family tests spanning eight independent crosses -and three sequencing batches, **99.92-99.99% of all haplotype calls in +and three sequencing batches, 99.92-99.99% of all haplotype calls in every family are alleles of that locus's four inferred parental -haplotypes** — at three nines, the microhaplotypes the MADC reports in +haplotypes. At three nines, the microhaplotypes the MADC reports in 763 offspring are the parental alleles Mendelian transmission requires. ### Read accounting -The QC JSON now reports where every read goes. For the default run: -154,584,628 FASTQ records; 88,126,464 assigned to a locus at full length; -87,373,009 (99.1% of assigned) retained in emitted rows; 753,455 dropped with -their 558,194 low-evidence sequences (mean 1.3 reads each); 3,294,541 moved -between sequences by within-sample denoising and 103,628 by cross-sample -denoising (moved reads stay in the table under their centroid). Per-sample +The QC JSON now reports where every read goes. The default run had +154,584,628 FASTQ records, of which the pipeline assigned 88,126,464 to a +locus at full length and retained 87,373,009 (99.1% of assigned) in +emitted rows. It dropped 753,455 with their 558,194 low-evidence +sequences (mean 1.3 reads each). Within-sample denoising moved 3,294,541 +between sequences and cross-sample denoising moved 103,628 (moved reads +stay in the table under their centroid). Per-sample `reads_in_emitted_rows` joins the existing per-sample QC. Twelve samples have -fewer than 100,000 assignable 81-bp tags (lowest: `4404075` with 23); these -FASTQs are dominated by short or primer-less reads and downstream -completeness should be reviewed for them. (Those figures are the 0.5.0 run; -since 0.10.0 the pooled pass folds same-allele sequences only — -`reads_moved_by_global_denoise` keeps its name — and the ledger gains +fewer than 100,000 assignable 81-bp tags (the lowest, `4404075`, has 23). +Short or primer-less reads dominate these FASTQs, so review their +downstream completeness. (Those figures are from the 0.5.0 run. Since +0.10.0 the pooled pass folds same-allele sequences only, though +`reads_moved_by_global_denoise` keeps its name, and the ledger gains `reads_assigned_gapped`, `reads_n_mapped` and `reads_n_dropped`.) ### Historical exploratory analyses (0.3.x) -The `results/v03_*` files document the audits that motivated the redesign: -characterization of generated-only rows, right-primer support, cross-extract -reproducibility of paired leaf/root samples, and a diagnostic BLAST against -the public NSP306 *I. trifida* reference following the open `dartag_alleles` -pattern (best hit at the designed locus for most high-support extra rows; -off-target best hits flagged for paralog review, without automatic exclusion -because cultivated sweetpotato is hexaploid and the reference is diploid). -Those analyses used the 0.3.x quality-gated outputs; their row lists are -superseded by the evidence model above, but the biological conclusions — -filter the low-support tail, keep reproducible on-target microhaplotypes, -flag multimappers — are what the current defaults implement. +The `results/v03_*` files document the audits that motivated the redesign, +covering characterization of generated-only rows, right-primer support, +cross-extract reproducibility of paired leaf/root samples, and a +diagnostic BLAST against the public NSP306 *I. trifida* reference +following the open `dartag_alleles` pattern (best hit at the designed +locus for most high-support extra rows, and off-target best hits flagged +for paralog review, without automatic exclusion because cultivated +sweetpotato is hexaploid and the reference is diploid). +Those analyses used the 0.3.x quality-gated outputs. The evidence model +above supersedes their row lists, but the current defaults implement +their biological conclusions, which are to filter the low-support tail, +keep reproducible on-target microhaplotypes, and flag multimappers. ### Evidence about prior FASTQ filtering and tag length -Every vendor `AlleleSequence` and `ClusterConsensusSequence` is exactly 81 -bp; the panel design reconstructs a 97-bp primer-to-primer amplicon at all +Every vendor `AlleleSequence` and `ClusterConsensusSequence` is 81 +bp long. The panel design reconstructs a 97-bp primer-to-primer amplicon at all 3,120 loci, so an 81-bp tag is an observed prefix of a longer amplicon. The two target manifests report 175,642,223 `tagcounttotal` observations versus -154,584,628 delivered FASTQ records; applying each manifest's `overreppct` +154,584,628 delivered FASTQ records. Applying each manifest's `overreppct` predicts 18,989,753 removed overrepresented reads, and per-sample predicted -and observed deficits are strongly correlated. The delivered FASTQs were -therefore already filtered before release, which supports the preprocessing +and observed deficits are strongly correlated. The vendor therefore +filtered the delivered FASTQs before release, which supports the preprocessing hypothesis without identifying the exact vendor commands. A raw-data audit finds 17,334 of the vendor's 18,677 pure-ACGT rows as exact 81-bp FASTQ -prefixes; the vendor also reports 569 Ref/Alt rows with curated IUPAC +prefixes. The vendor also reports 569 Ref/Alt rows with curated IUPAC ambiguity (their concrete groupings are not derivable from the A/C/G/T probe design) and 73 Match/Other rows containing `N`. All 6,238 81-bp prefixes of the public Breeding Insight PanelHub 109-bp design sequences agree exactly @@ -787,112 +806,112 @@ with this pipeline's independently reconstructed tags. Statistics columns the pipeline does not compute (`OneRatioRef` through `AvgPIC`, and `RatioAvgCountRefAvgCountSnp`) emit `.` rather than a blank cell in every data row (with `--annotations on`, the default, `PICSnp` and -`AvgPIC` instead carry the `Variants` and `MarkerType` annotations — see -the 0.10.0 section). This is a 0.8.2 format change: byte comparisons -against MADCs generated by earlier versions will differ in exactly those -nine columns and nothing else. +`AvgPIC` instead carry the `Variants` and `MarkerType` annotations +described in the 0.10.0 section). This is a 0.8.2 format change, so byte +comparisons against MADCs from earlier versions will differ in those +nine columns only. Three `generate` flags reproduce the raw-MADC framing that preamble-expecting consumers such as HapApp and HapApp_utils rely on: - `--dart-preamble` prepends the vendor-style seven starred rows above the - header: service/project label, plate barcode, plate number, plate row, - plate column, plate name, and the targetid row, which carries the - FASTQ-derived sample ids directly above the header row's sample names — - the same fastq-id-over-sample-name adjacency as vendor files. Every - preamble cell without a value is `*`, exactly as in vendor output, with - one exception: the file's very first cell holds a full provenance string + header. They are the service/project label, plate barcode, plate number, + plate row, plate column, plate name, and the targetid row, which carries + the FASTQ-derived sample ids directly above the header row's sample + names, the same fastq-id-over-sample-name adjacency as vendor files. + Every preamble cell without a value is `*`, as in vendor output, with + one exception. The file's very first cell holds a full provenance string (pipeline, version, and every generation parameter as `key=value`, prefixed with `* ` so consumers that skip preamble lines by their leading star still skip it), so the file records how it was made even - when separated from its QC sidecar. No timestamp is included — output - stays byte-reproducible. + when separated from its QC sidecar. The preamble includes no timestamp, + so output stays byte-reproducible. - `--preamble-project` sets the label repeated across the first preamble - row's sample cells (default: the output filename before its first dot). + row's sample cells (by default the output filename before its first dot). - `--sample-lookup` takes a two-column CSV/TSV (FASTQ-derived sample id, - desired sample name; a header line is detected and skipped) and replaces + desired sample name, with any header line detected and skipped) and replaces the output column names while the targetid row keeps the FASTQ ids. Validation is deliberately strict, because a rename table with holes - silently mislabels count columns: every run sample must be mapped, ids - matching no sample are errors, and duplicate names are errors. + silently mislabels count columns. The table must map every run sample, + ids matching no sample are errors, and duplicate names are errors. -`read_madc` round-trips preambled output: the header is found by its -`AlleleID` cell, and a numeric, unique targetid row is adopted for sample -identity exactly as when reading vendor files. +`read_madc` round-trips preambled output. It finds the header by its +`AlleleID` cell and adopts a numeric, unique targetid row for sample +identity, as when reading vendor files. ## Paired-end reads and multi-window tiling (0.9.0) The single-window model reads one excised window per locus, which caps -usable data whenever the sequenced fragment is larger than one read: on +usable data whenever the sequenced fragment is larger than one read. On a dual-probe capture platform (Flex-Seq, ~300-bp probe-to-probe fragments, NovaSeq 2x150) only ~52% of reads could ever overlap the one window, by geometry rather than by quality. 0.9.0 adds the two missing -pieces — paired-end intake and multiple windows per locus — as explicit, +pieces, paired-end intake and multiple windows per locus, as explicit, separately validated mechanisms. **Paired-end intake is one explicit decision, never auto-detected.** Measured read geometry decides it: - **Overlapping pairs** (insert shorter than R1+R2, e.g. GT-seq PE100 - over 130-200-bp amplicons): merge first with the `merge-pairs` + over 130-200-bp amplicons). Merge first with the `merge-pairs` subcommand, then feed merged fragments to `normalize-reads` or `generate`. The merger reverse-complements R2, scans overlaps longest - first, and handles both geometries — the merged read is always R1's - head plus the overlap plus R2's tail, so in the extended geometry that - is the whole fragment, and in engulfed read-through R1's adapter tail - past the fragment end falls outside the overlap and is discarded by - construction (no adapter trimming step) while R2 bases past R1's end - are kept when the reads differ in length; disagreeing overlap bases - take the higher-quality call, `--unmerged r1|pool|drop` states what - happens to the remainder, mates are paired by read ID (out-of-step - names or unequal counts are a hard error), and samples are independent - so `--jobs` merges them in parallel processes. On a synthetic paired-end truth set with modeled - adapter read-through, 100% of 513,715 pairs merged and molecular count - recovery was 98.6% — one fragment, one count. -- **Non-overlapping dual-anchored designs** (Flex-Seq: R1s anchor the - left probe, R2s the right, mates cover opposite halves of a fragment - neither read spans): pool both mates as independent reads. Merging is - geometrically impossible there and pooling cannot double-count a - window, because no window is covered by both mates. - -**Multi-window tiling** puts several markers on one physical locus: an + first, and handles both geometries. The merged read is always R1's + head plus the overlap plus R2's tail. In the extended geometry this + covers the whole fragment. In engulfed read-through, R1's adapter tail past + the fragment end falls outside the overlap and drops out by + construction (no adapter trimming step), while the merger keeps R2 + bases past R1's end when the reads differ in length. Disagreeing + overlap bases take the higher-quality call, `--unmerged r1|pool|drop` + states what happens to the remainder, the merger pairs mates by read + ID (out-of-step names or unequal counts are a hard error), and samples + are independent, so `--jobs` merges them in parallel processes. On a + synthetic paired-end truth set with modeled adapter read-through, 100% + of 513,715 pairs merged and molecular count recovery was 98.6%, so each + fragment gave one count. +- **Non-overlapping dual-anchored designs** (Flex-Seq, where R1s anchor + the left probe, R2s the right, and mates cover opposite halves of a + fragment neither read spans). Pool both mates as independent reads. + Merging is geometrically impossible there and pooling cannot + double-count a window, because the two mates never cover the same + window. + +**Multi-window tiling** puts several markers on one physical locus. An optional `LocusGroup` column (probe-design TSV and reference-tags CSV) declares markers as tiles of one target region. Tiles must share an identical Ref expansion frame (validated at load), share one k-mer territory, and reads locate to the group before routing to member -windows. Panels without the column behave exactly as before, to the -byte. +windows. Panels without the column behave as before, to the byte. **Routing between a group's windows** is `--group-routing`: -- `nearest` (default): each read feeds exactly one tile — the fully +- `nearest` (default) feeds each read to a single tile, the fully covered window whose center is nearest the read's center. -- `covered`: the read's excision is emitted at every member window it +- `covered` emits the read's excision at every member window it fully covers. Each window still sees each fragment at most once in - both modes; the QC separates the extra per-window emissions + both modes, and the QC separates the extra per-window emissions (`tile_extra_emissions`) from the read-fate accounting. The distinction matters because merged full-span fragments all share -the amplicon's center, so `nearest` degenerates to winner-take-all: on -the GT-seq benchmark, one tile took effectively all depth at 163 of 164 +the amplicon's center, so `nearest` degenerates to winner-take-all. On +the GT-seq benchmark, one tile took almost all depth at 163 of 164 tiled loci and the other went dark. Use `covered` whenever reads span more than one tile (merged fragments, or long reads over closely spaced -tiles); a fragment that covers two windows genuinely observes both -SNPs. +tiles), because a fragment that covers two windows observes both SNPs. **Distinct-window guard.** Tiles share a coverage profile, so fresh -window placement can collapse two members onto the same start — and two +window placement can collapse two members onto the same start, and two markers with identical Ref tags make every ref-carrying read an exact -tie, which the assignment layer drops as ambiguous. Measured on GT-seq, +tie, which the assignment layer drops as ambiguous. On GT-seq, that silently zeroed a locus's ref depth and flipped its calls from 100% to 3% concordance. Fresh placement therefore forces member windows to excise distinct sequences (a colliding member takes its best-ranked -distinct start; the nudge is listed in QC `locus_groups.window_dedup_nudged`; -no distinct start at all is an error), and `--reuse-tags` refuses a +distinct start, QC lists the nudge in `locus_groups.window_dedup_nudged`, +and no distinct start at all is an error), and `--reuse-tags` refuses a frozen panel containing twin windows instead of replaying the defect. -**Window freezing is load-bearing for paired-end modes.** A four-way -GT-seq experiment on identical reads and truth (96 samples, 409 +**Paired-end modes depend on window freezing.** A four-way GT-seq +experiment ran on identical reads and truth (96 samples, 409 original + 206 added tile/rescued markers, DP >= 20 against the archive's independent BWA/bcftools VCF): @@ -901,52 +920,52 @@ archive's independent BWA/bcftools VCF): | R1-only | 98.80% | — | 34,340 | | pooled mates + fresh windows | 95.18% | 87.92% | — | | merged + fresh windows | 92.99% | 90.06% | — | -| merged + frozen tags + tiles, `covered` | **98.80%** | **98.43%** | **52,971** | +| merged + frozen tags + tiles, `covered` | 98.80% | 98.43% | 52,971 | Every fresh-placement regression was window drift (all 368 windows -moved; the worst landed on twin-window ambiguity), and the recipe that -holds is: `merge-pairs`, then `normalize-reads --reuse-tags` with the -proven panel plus tile rows, then `generate` — original markers +moved, and the worst landed on twin-window ambiguity). The recipe that +holds is `merge-pairs`, then `normalize-reads --reuse-tags` with the +proven panel plus tile rows, then `generate`. With it, original markers reproduce the R1-only result on the same cells while the tiles add 54% more truth-validated genotype calls at parity. On the dual-probe capture benchmark (cranberry Flex-Seq, 24 samples, 17,242 reconstructed clusters scored against the population truth VCF), -the same mechanisms lift the geometry ceiling: R1-only single-window -92.39% on 32,675 cells with median 7,733 loci recovered per sample; -pooling both mates raises it to 93.24% on 57,577 cells and 13,421 loci -(the published vendor pipeline recovered 14,550 of 17,502); a -three-tile 41,033-marker panel with `covered` routing reaches 82.9% -read extraction and **94.59% on 120,261 cells** — 3.7x the callable +the same mechanisms lift the geometry ceiling. R1-only single-window +calling scores 92.39% on 32,675 cells with median 7,733 loci recovered +per sample. Pooling both mates raises it to 93.24% on 57,577 cells and +13,421 loci (the published vendor pipeline recovered 14,550 of 17,502). +A three-tile 41,033-marker panel with `covered` routing reaches 82.9% +read extraction and 94.59% on 120,261 cells, 3.7x the callable genotypes of the single-window R1 run, at higher concordance. A subsequent full-cohort campaign (192 samples, K=4 tiles at 45-bp spacing with 110-bp windows, one cohort-level `generate`) raised capture of the population's segregating-site catalogue from 64.5% to -78.4% while *improving* trio incompatibility from 4.5% to 3.6% — -window geometry solved the coverage gap (outside-window misses fell -from 18.6% to 1.9% of sites) and cohort-scale evidence produced a -cleaner haplotype catalogue, not just a larger one. +78.4% while *improving* trio incompatibility from 4.5% to 3.6%. +Window geometry solved the coverage gap (outside-window misses fell +from 18.6% to 1.9% of sites), and cohort-scale evidence produced a more +accurate haplotype catalogue as well as a larger one. **Adapter trimming is not required, and adapter cannot reach the -emitted haplotypes.** Three structural guarantees, one per read mode: -windows are defined in frame coordinates and bounded by the frame -(`window start + length <= frame length`), so on amplicon platforms — -where the frame is the amplicon — a window cannot reach past the -biology into adapter by construction; merge mode removes read-through -adapter geometrically (the engulfed merge drops R1's tail past the overlap; -proven on the synthetic paired-end set with modeled read-through: 100% -merged, zero length errors); and pooled mode exists precisely for +emitted haplotypes.** There are three structural guarantees, one per +read mode. Windows live in frame coordinates and the frame bounds them +(`window start + length <= frame length`), so on amplicon platforms, +where the frame is the amplicon, a window cannot reach past the +biology into adapter by construction. Merge mode removes read-through +adapter geometrically (the engulfed merge drops R1's tail past the +overlap, and on the synthetic paired-end set with modeled read-through +it gave 100% merged and zero length errors). Pooled mode exists precisely for long-fragment designs whose reads end before the fragment does, so there is no read-through to leak. The residual theoretical path (capture platforms whose genome-slice frames outrun a short fragment, on the unmerged tail) funnels into identity-gate rejection, because variable shear junctions never reproduce as one sequence. The census -agrees: across 127,667 emitted rows in six MADCs spanning five +agrees. Across 127,667 emitted rows in six MADCs spanning five platforms (GT-seq at two window lengths, rhAmpSeq pooled and family-scale, the 41K-tile capture panel, Ion AgriSeq), zero rows contain TruSeq or Ion P1 adapter at a strict 13-mer probe. (A loose 10-mer probe flags one capture locus whose genome happens to contain -`AGATCGGAAG` diverging from TruSeq at base 11 — a chance genomic +`AGATCGGAAG` diverging from TruSeq at base 11. It is a chance genomic k-mer inside the reference tag itself, not contamination.) Measured on the capture benchmark, trimming adapters before the run changed dosage concordance by 0.05 points (92.24% untrimmed vs 92.19% trimmed) while @@ -954,350 +973,365 @@ costing 2 points of extraction from the shortened reads. ## Choosing the window length (`window-scan`) -Tag/window length was historically chosen per platform against external +Earlier versions chose tag/window length per platform against external truth (the honeybee 41-bp knee came from a concordance-versus-length sweep against vendor calls). `window-scan` makes the same decision from -the project's own reads: one coverage pass predicts **retention** +the project's own reads. One coverage pass predicts retention (fraction of located reads fully covering the best window) for a dense grid of lengths, and each candidate length then runs the real normalize-and-generate machinery on a sample subset to measure -**cleanliness** (assigned reads not absorbed by denoising or dropped as -failed evidence — per-base error and indel exposure accumulate with -length) and **information** (evidence-passing novel haplotype rows per -100 markers — rises with length until the local linked variation is -exhausted). The recommendation is **survival-first**: survival = -retention x assignment x cleanliness; candidates within `--info-band` +cleanliness (assigned reads not absorbed by denoising or dropped as +failed evidence, which falls as per-base error and indel exposure +accumulate with length) and information (evidence-passing novel +haplotype rows per 100 markers, which rises with length until the local +linked variation runs out). The recommendation is survival-first, where +survival = retention x assignment x cleanliness. Candidates within `--info-band` (default 10%) of the best survival are eligible, and the most informative eligible length wins, ties breaking short. Information -deliberately cannot lead the decision, because it can be inflated by -exactly the rows one fears: systematic errors (Ion homopolymer indel +deliberately cannot lead the decision, because the rows one fears can +inflate it. Systematic errors (Ion homopolymer indel families recur across samples) pass the reproducibility route and masquerade as haplotype diversity. An earlier score that weighted -information directly followed that inflation long — on Ion oyster data -it recommended 81 bp against the externally validated 41-bp knee — and -the survival-first rule was the correction. Output: a TSV of the -curves, a QC JSON, and per-length run directories; a candidate the -distinct-window guard refuses (tiles inseparable at that length) is -reported failed rather than aborting the sweep. +information directly followed that inflation long. On Ion oyster data +it recommended 81 bp against the externally validated 41-bp knee, and +the survival-first rule was the correction. The output is a TSV of the +curves, a QC JSON, and per-length run directories. The scan reports a +candidate the distinct-window guard refuses (tiles inseparable at that +length) as failed rather than aborting the sweep. Under the survival-first rule the scan reproduces every externally -arbitrated choice: Ion oyster 41 bp (assignment falls 15 points from -31 to 81 bp as indel-bearing reads fail the substitution-only identity -gate — the Ion damage shows in assignment, not cleanliness), GT-seq -81 bp (the frozen-validated length; the 96-sample truth test measured -110 bp at only -0.08 points, so the band's conservatism is cheap), and -capture 61 bp. On merged rhAmpSeq amplicons — clean chemistry, -fragments dwarfing every candidate, information unsaturated through -150 bp — the rule recommends 81 while the family instrument cleared -110 and the platform's own pipeline uses full inserts: where richer -haplotypes are the goal and an external or genetic arbiter is -available, widening `--info-band` expresses that choice explicitly. +arbitrated choice. It picks 41 bp for Ion oyster (assignment falls 15 +points from 31 to 81 bp as indel-bearing reads fail the substitution-only +identity gate, so the Ion damage shows in assignment rather than +cleanliness), 81 bp for GT-seq (the frozen-validated length, and the +96-sample truth test measured 110 bp at only -0.08 points, so the band's +conservatism is cheap), and 61 bp for capture. On merged rhAmpSeq +amplicons, with low-error chemistry, fragments dwarfing every candidate, +and information unsaturated through 150 bp, the rule recommends 81 while +the family instrument cleared 110 and the platform's own pipeline uses +full inserts. Where richer haplotypes are the goal and an external or +genetic arbiter is available, widening `--info-band` expresses that +choice explicitly. **Seed sensitivity (0.11.0).** The scan runs on a sample subset (six files by default), and a recommendation can be a property of that subset rather than of the panel. `--scan-draws N` repeats the whole sweep on N independent seeded draws (seeds `--scan-seed`, +1, +2, ...) into `draw_1/`, `draw_2/`, ... and reports each draw's recommendation, the -votes per length, and their agreement (`recommendation_stability`); -the consensus is the most-voted length, ties breaking short. A +votes per length, and their agreement (`recommendation_stability`). +The consensus is the most-voted length, ties breaking short. A recommendation that flips between draws is the signal to scan more -samples or to arbitrate externally — the one check a user without truth -can make on the length decision itself. +samples or to arbitrate externally. It is the one check a user without +truth can make on the length decision itself. -Falling cleanliness deserves a precise reading: a churned read is one +Falling cleanliness deserves a precise reading. A churned read is one whose exact sequence the denoiser recognized as an error neighbor and -folded back into its parent — the count survives, no error row is -emitted, so churn measures error-handling load, not leakage. The +folded back into its parent. The count survives and no error row +appears, so churn measures error-handling load, not leakage. The narrow leak path is a systematic error (same position, same substitution, recurring across samples) clearing the reproducibility -route and emitting as a false Match row — and the scan's information +route and emitting as a false Match row. The scan's information metric cannot distinguish that from real linked variation, so the scan is a screening instrument whose recommendation still wants an external -or genetic arbiter. On GT-seq the arbitration is done: the scan +or genetic arbiter. On GT-seq the arbitration is done. The scan recommended 110 bp over the historical 81, and against the archive's independent truth VCF on all 96 samples (identical merged reads, -identical fresh placement), 110 bp scores 98.62% versus 81 bp's 98.70% -— at most 0.08 points of dosage cost while emitting 27% more Match -haplotypes (1,568 vs 1,231 rows). The information the scan saw is not -corrupting target genotypes; whether each added Match row is linked -biology versus a systematic error family is exactly what the family -instrument settles (on rhAmpSeq at 110 bp: under 1% family-bound -violations across 763 progeny). The same experiment refined the -window-drift picture: fresh 81-bp placement WITH the distinct-window -guard scores 98.77/98.57 (old/new markers) versus the pre-guard 92.99/ -90.06 — the drift "catastrophe" was entirely the twin-window defect, -and honest residual drift costs only 0.03-0.14 points against frozen -windows. Freezing remains mandatory for cross-run haplotype identity; -it is no longer load-bearing for accuracy. +identical fresh placement), 110 bp scores 98.62% versus 81 bp's 98.70%. +The longer window costs at most 0.08 points of dosage while emitting 27% +more Match haplotypes (1,568 vs 1,231 rows). The information the scan saw is not +corrupting target genotypes. The family instrument settles whether each +added Match row is linked biology or a systematic error family (on +rhAmpSeq at 110 bp it found under 1% family-bound violations across 763 +progeny). The same experiment refined the window-drift picture. Fresh +81-bp placement WITH the distinct-window guard scores 98.77/98.57 +(old/new markers) versus the pre-guard 92.99/90.06, so the drift +"catastrophe" was entirely the twin-window defect, and the remaining +drift costs only 0.03-0.14 points against frozen windows. Freezing +remains mandatory for cross-run haplotype identity, but accuracy no +longer depends on it. ## General indel recovery and review fixes (0.10.0) Two independent code reviews and a colleague's finding on a DArTag project -— genome assemblies predicted indel-bearing microhaplotypes (1–13 bp, -often several events per tag) that the vendor MADC never reported, and -neither did 0.9.x — motivated this release. The 0.9.x identity gate was -substitution-only, so an undesigned indel frameshifts the tag against -every canonical sequence and the read was dropped as off-target; on that -project only 6 of 447 read-supported indel microhaplotypes fell inside the -envelope, while 423 lay within the same 20% budget once gaps count as -edits. +motivated this release. In that project, genome assemblies predicted +indel-bearing microhaplotypes (1–13 bp, often several events per tag) +that the vendor MADC never reported, and neither did 0.9.x. The 0.9.x +identity gate was substitution-only, so an undesigned indel frameshifts +the tag against every canonical sequence and the gate dropped the read as +off-target. On that project only 6 of 447 read-supported indel +microhaplotypes fell inside the envelope, while 423 lay within the same +20% budget once gaps count as edits. **One general mechanism, not a catalogue of cases.** When every primer -candidate rejects a read, `select_marker` offers one gapped reading: a +candidate rejects a read, `select_marker` offers one gapped reading. It is a start-anchored banded alignment (band = `--max-indel-length`, default 10) whose end may overhang by up to a band on either side (a read tail past the reference is a deletion's signature, a reference tail past the read an insertion's), scored under the same edit budget as substitutions. The -reading is believed only when it is composed like an indel haplotype — -at most 6 substitutions and 6 gap events alongside the gaps — because a +pipeline accepts the reading only when its composition is that of an +indel haplotype, with at most 6 substitutions and 6 gap events alongside +the gaps, because a co-amplified paralog reaches the budget through many substitutions (the captured reads on the motivating project carried a median of 13 substitutions per read against 2 for true indel haplotypes, and one locus -would otherwise have swollen from 351 to 38,653 reads). Accepted reads are -counted as ordinary fixed-length rows in the vendor's shifted-suffix -convention; the counting layer, evidence rules and denoiser are unchanged. +would otherwise have swollen from 351 to 38,653 reads). Accepted reads +count as ordinary fixed-length rows in the vendor's shifted-suffix +convention, and the counting layer, evidence rules and denoiser are unchanged. `classify_sequence` reads the target base at the position the *cheaper* -explanation puts it: the fixed offset unless a gapped reading is strictly -cheaper, well composed, and verified by at least 12 bases after its last -gap (clustered errors at a read end must not masquerade as a shift); a -target inside a deletion or pushed past the read is `Other`. This also -corrects in-envelope indel reads that 0.9.x classified at a shifted offset -— the delta smelt locus `NC_061083`, whose real 1-bp insertion scored 26% -against the truth VCF, is now concordant. Cost is bounded by an exact -lower bound: a Myers/Hyyrö bit-vector edit distance (C-speed integer -operations) proves most hopeless tags cannot meet the budget before any +explanation puts it. This position is the fixed offset unless a gapped +reading is strictly cheaper, well composed, and verified by at least 12 +bases after its last gap (clustered errors at a read end must not +masquerade as a shift). A target inside a deletion or pushed past the +read is `Other`. This also corrects in-envelope indel reads that 0.9.x +classified at a shifted offset. +The delta smelt locus `NC_061083`, whose real 1-bp insertion scored 26% +against the truth VCF, is now concordant. An exact lower bound limits the +cost. A Myers/Hyyrö bit-vector edit distance (C-speed integer +operations) proves that most hopeless tags cannot meet the budget before any alignment runs, so a 15-sample DArTag run takes 75 s against 41 s (a prototype without the prescreen took 150–290 s). `--gapped-rescue off` -restores the 0.9 assignment path; QC reports `reads_assigned_gapped`. +restores the 0.9 assignment path, and QC reports `reads_assigned_gapped`. **Review fixes adopted.** The pooled denoising pass now folds an evidence-failing sequence only into a protected parent carrying the same -target allele (0.9.x moved reads across alleles); parent ranking is -closest-first, then deepest; an observed N counts as a mismatch and -N-bearing tags are credited to their unique compatible sequence or dropped, -never emitted; a k-mer repeated at two positions inside one target frame no -longer anchors reads (it kept only its last position); `merge-pairs` keeps +target allele (0.9.x moved reads across alleles). Parent ranking is +closest-first, then deepest. An observed N counts as a mismatch, and the +pipeline credits N-bearing tags to their unique compatible sequence or +drops them, never emitting them. A k-mer repeated at two positions +inside one target frame no longer anchors reads (it kept only its last +position). `merge-pairs` keeps R2's extension when the reads differ in length and errors on mates out of -step; `window-scan` takes `--scan-seed` for a random sample subset and +step. `window-scan` takes `--scan-seed` for a random sample subset and `--generate-args` to forward evidence flags to the candidate runs. The per-sample cache format is 0.5.0, so existing caches rebuild once. **Considered and not adopted, with the measurement.** Collapsing reads onto -IUPAC-coded canonical tags: DArT uses those codes to encode real secondary -SNPs, and on the blackberry panel the collapse lost 272 vendor-matching -rows and moved count agreement from 0.982 to 0.976 — so rows stay exact -and ambiguity-coded canonicals keep reporting zero exact counts. A -synthesized-primer mask (worse still on blackberry), an unconditional -merge of indel families in long homopolymer runs (it absorbed 66 real -indel alleles on the DArTag project at its Ion-safe default), a -hard-coded ubiquity filter, and a single-gap-only indel tier with its -downstream-context columns were all left out in favour of the one +IUPAC-coded canonical tags failed because DArT uses those codes to encode +real secondary SNPs. On the blackberry panel the collapse lost 272 +vendor-matching rows and moved count agreement from 0.982 to 0.976, so +rows stay exact and ambiguity-coded canonicals keep reporting zero exact +counts. The release left out a synthesized-primer mask (worse still on +blackberry), an unconditional merge of indel families in long homopolymer +runs (it absorbed 66 real indel alleles on the DArTag project at its +Ion-safe default), a hard-coded ubiquity filter, and a single-gap-only +indel tier with its downstream-context columns, in favour of the one mechanism above. -**Validation, identical inputs, 0.9.1 → 0.10.0 — every platform in the -corpus.** Delta smelt GT-seq vs the BWA/bcftools VCF 98.80% → **99.49%** -R1-only and 98.80% → **99.49%** on the merged tiled panel (core 99.80%; -358/361 markers ≥90%); oyster AgriSeq vs Torrent VariantCaller 94.51% → -**94.91%** (markers ≥90%: 460 → 472); honeybee AgriSeq chip 2 vs the -vendor caller 91.60% → **93.84%**, replicate concordance across 30 pairs -97.64% → **98.49%**, depth per sample-marker +28%; cranberry Flex-Seq -(24-sample subset, pooled mates, frozen K4 panel) vs the truth VCF 94.50% -→ **94.78%** on 3.8% more callable cells; vitis rhAmpSeq MM531, 165 -progeny, 193,737 → **202,811** calls (+4.7%) at within-parental-four -transmission 99.890% → **99.894%**, and the 16-sample head-to-head against -`analyze_amplicon` 93.0% → **94.9%** exact agreement at 99.953% Mendelian -consistency; the three in-house DArTag projects `validate`d against their -vendor MADCs kept every substitution-side number (sweetpotato count -Pearson 0.9992 → 0.9993, red clover 0.9733 → 0.9733, blackberry 376 -samples 0.9814 → 0.9814) while adding 1,544 / 2,393 / 8,313 indel-type -rows respectively — rows the vendor pipeline structurally cannot report — -and the private DArTag project (15 checked samples): 2 → **348 of 447** -assembly-predicted indel microhaplotypes recovered at 97% read recall, -including 72–82% of those with two or more indel events, while the new -heterozygotes those rows create have allele fractions indistinguishable -from vendor-agreeing heterozygotes and a third are confirmed by the same -sample's genome assembly. Indel-rich loci gain reads (about 60 markers +**Validation on identical inputs, 0.9.1 → 0.10.0, for every platform in +the corpus.** Delta smelt GT-seq vs the BWA/bcftools VCF rose 98.80% → +99.49% R1-only and 98.80% → 99.49% on the merged tiled panel (core +99.80%, 358/361 markers ≥90%). Oyster AgriSeq vs Torrent VariantCaller +rose 94.51% → 94.91% (460 → 472 markers ≥90%). Honeybee AgriSeq chip 2 +vs the vendor caller rose 91.60% → 93.84%, with replicate concordance +across 30 pairs 97.64% → 98.49% and depth per sample-marker +28%. +Cranberry Flex-Seq (24-sample subset, pooled mates, frozen K4 panel) vs +the truth VCF rose 94.50% → 94.78% on 3.8% more callable cells. Vitis +rhAmpSeq MM531, 165 progeny, rose 193,737 → 202,811 calls (+4.7%) at +within-parental-four transmission 99.890% → 99.894%, and the 16-sample +head-to-head against `analyze_amplicon` rose 93.0% → 94.9% exact +agreement at 99.953% Mendelian consistency. The three in-house DArTag +projects `validate`d against their vendor MADCs kept every +substitution-side number (sweetpotato count Pearson 0.9992 → 0.9993, red +clover 0.9733 → 0.9733, blackberry 376 samples 0.9814 → 0.9814) while +adding 1,544 / 2,393 / 8,313 indel-type rows respectively, rows the +vendor pipeline structurally cannot report. In the private DArTag +project (15 checked samples), recovered assembly-predicted indel +microhaplotypes rose 2 → 348 of 447 at 97% read recall, including 72–82% +of those with two or more indel events. The new heterozygotes those rows +create have allele fractions indistinguishable from vendor-agreeing +heterozygotes, and the same sample's genome assembly confirms a third of +them. Indel-rich loci gain reads (about 60 markers more than doubled on that project, the same set two independent indel implementations found), and Ion row counts rise about 9% (5,890 → 6,443 -on oyster) as slippage haplotypes are now assigned rather than dropped; -their target base is still read correctly, which is why SNP accuracy -rose. Two vendor-agreement effects are expected and explained: DArT emits +on oyster) as the pipeline now assigns slippage haplotypes rather than +dropping them. It still reads their target base correctly, which is why +SNP accuracy rose. Two vendor-agreement effects are expected and +explained. DArT emits N-bearing rows that 0.10.0 never does (29 of the 30 sweetpotato rows that "disappeared"), and at indel-shifted rows inside the substitution envelope the vendor classifies at the fixed offset while 0.10.0 reads through the gap (153 sweetpotato rows, mostly vendor `Other` becoming Ref/AltMatch or -Ref/AltMatch swapping) — the smelt truth VCF settled that question in +Ref/AltMatch swapping). The smelt truth VCF settled that question in favour of the gapped reading. Where an indel allele carries the alternate base, the recovered rows create heterozygotes the vendor calls homozygous -(red clover: updog dosage agreement with the vendor MADC 97.86% → 95.95% at -DP≥10, 13,797 new indel-driven heterozygotes). The 152-progeny red clover -mapping family with both parents adjudicates them: with every MADC pushed +(in red clover, updog dosage agreement with the vendor MADC fell 97.86% → +95.95% at DP≥10, with 13,797 new indel-driven heterozygotes). The +152-progeny red clover mapping family with both parents adjudicates +them. With every MADC pushed through the same updog (diploid, norm) fit, the Mendelian-violation rate is -1.59% under the vendor MADC, 1.61% under 0.9.1 and **1.30% under 0.10.0** -on 2% more tests (parents DP≥20, progeny DP≥10, MPP≥0.9) — the new +1.59% under the vendor MADC, 1.61% under 0.9.1 and 1.30% under 0.10.0 +on 2% more tests (parents DP≥20, progeny DP≥10, MPP≥0.9). The new heterozygotes are inherited, and the vendor's disagreement is the vendor missing real alleles. **Band width.** `--max-indel-length` 10 is the validated default. On the private DArTag project's full 558-sample order, band 10 recovered 374 of -the 447 read-supported assembly-predicted indel microhaplotypes (84%; the -extra samples let 26 more clear the reproducibility route than in the -15-sample run) and band 20 recovered 393 (88%) by adding exactly 76 rows — -all 11–20 bp indel haplotypes, median ~5,600 reads each — with the same -row-plausibility profile, vendor SNP concordance within 0.04 pp -(97.87% vs 97.83% over 1.5M cells) and equal runtime. That order also +the 447 read-supported assembly-predicted indel microhaplotypes (84%, as +the extra samples let 26 more clear the reproducibility route than in the +15-sample run), and band 20 recovered 393 (88%) by adding 76 rows, all +11–20 bp indel haplotypes with a median of ~5,600 reads each. The wider +band kept the same row-plausibility profile, vendor SNP concordance within +0.04 pp (97.87% vs 97.83% over 1.5M cells) and equal runtime. That order also carries a 264-progeny F2 family and three genotypes in technical -quadruplicate, which arbitrate the rows directly: indel-haplotype calls +quadruplicate, which arbitrate the rows directly. Indel-haplotype calls fall inside the founders' four haplotypes at 99.86–99.96%, the same rate -as substitution haplotypes (99.91–99.94%; the vendor's own rows 99.95%), -and an indel haplotype called in one replicate is re-called in its twins -98.4% of the time against 99.3% for substitution haplotypes — inherited -alleles, with the replicate gap consistent with indel alleles amplifying a -little less efficiently. Widen the band on -panels where long indels are expected; the composition guard, not the -band, is what keeps paralogs out. - -**Annotations: `MarkerType`, `Variants`, and the variants sidecar -(`--annotations on`, the default).** Two vendor statistics this pipeline -never computes are repurposed, header included, and a sidecar is written -next to the table; `--annotations off` restores the exact vendor header +as substitution haplotypes (99.91–99.94%, and the vendor's own rows +99.95%). An indel haplotype called in one replicate is re-called in its +twins 98.4% of the time against 99.3% for substitution haplotypes. These +are inherited alleles, and the replicate gap fits indel alleles +amplifying a little less well. Widen the band on panels where long +indels are expected. The composition guard, not the band, keeps +paralogs out. + +**The `MarkerType` and `Variants` annotations and the variants sidecar +(`--annotations on`, the default).** The pipeline repurposes two vendor +statistics it never computes, header included, and writes a sidecar +next to the table. `--annotations off` restores the exact vendor header and writes no sidecar. The column positions are unchanged, so positional consumers see the same 16-column layout. -- `AvgPIC` becomes **`MarkerType`**: what the row carries *beyond its - canonical*, derived from its Variants. The Ref row is `.`; the Alt row - is labelled by the design itself (`snp` for a designed SNP, `indel` for - a designed indel); every other row by its events once the marker's - designed events are removed: `snp` (substitutions only), `indel` (gaps - only), `both`, or `.` (no concrete event beyond the canonical — a row - that merely realises an ambiguity code). A downstream user who wants +- `AvgPIC` becomes `MarkerType`, which records what the row carries + *beyond its canonical*, derived from its Variants. The Ref row is `.`. + The design itself labels the Alt row (`snp` for a designed SNP, `indel` + for a designed indel). Every other row takes its label from its events + after removing the marker's designed events, so it is `snp` + (substitutions only), `indel` (gaps only), `both`, or `.` (no concrete + event beyond the canonical, as in a row that merely realises an + ambiguity code). A downstream user who wants SNP-only microhaplotypes drops `indel`/`both` rows by this label alone and keeps designed-indel markers intact. -- `PICSnp` becomes **`Variants`**: the row's events against the Ref tag, - one token per event, `;`-separated, in 1-based coordinates on the tag as - printed (design strand; a consumer that knows the strand flips them, as - it already does for substitutions). Substitutions are `91:A>T`; - insertions and deletions are VCF-anchored on the preceding reference - base and left-normalised (`55:C>CAAAA`, `70:TAG>T`); then comes the - window geometry a fixed-length row cannot show: `clip=k` when an - insertion pushed the last *k* Ref bases out of the window, `overhang=k` - when a deletion pulled *k* downstream bases in. The Ref row is `.`, the - Alt row of a designed SNP is exactly the design (`46:C>G`), and an - AltMatch row lists the designed substitution among its events. A - position where the Ref tag carries an ambiguity code is never an event - (a concrete base realises the code), matching the IUPAC handling - elsewhere. -- **`OUTPUT.variants.csv`** lists the same events one per line — +- `PICSnp` becomes `Variants`, which lists the row's events against the + Ref tag, one token per event, `;`-separated, in 1-based coordinates on + the tag as printed (design strand, and a consumer that knows the strand + flips them, as it already does for substitutions). Substitutions are + `91:A>T`. Insertions and deletions are VCF-anchored on the preceding + reference base and left-normalised (`55:C>CAAAA`, `70:TAG>T`). Then + comes the window geometry a fixed-length row cannot show, with `clip=k` + when an insertion pushed the last *k* Ref bases out of the window and + `overhang=k` when a deletion pulled *k* downstream bases in. The Ref + row is `.`, the Alt row of a designed SNP is exactly the design + (`46:C>G`), and an AltMatch row lists the designed substitution among + its events. A position where the Ref tag carries an ambiguity code is + never an event (a concrete base realises the code), matching the IUPAC + handling elsewhere. +- **`OUTPUT.variants.csv`** lists the same events one per line, as `CloneID, AlleleID, AlleleSequence, Position, Ref, Alt, Type` with Type `snp`, `ins`, `del`, `clip` (Ref holds the unobserved bases) or - `overhang` (Alt holds the pulled-in bases) — keyed by CloneID and + `overhang` (Alt holds the pulled-in bases). It is keyed by CloneID and sequence so it survives allele-ID renumbering downstream. -Every row is described by its cheapest reading against Ref — a gapped -reading only when strictly cheaper than substitutions — through the -alignment with the fewest gap openings among equal-edit ones (a -three-state banded table), so an inserted block is one event even where -its bases happen to match the reference, as an affine-gap aligner would -describe it. Rows of a marker whose own Alt differs from Ref by a gap -are always read with gaps, so an AltMatch of a designed insertion that -adds one SNP lists the inherited insertion plus the SNP (and is labelled -`snp`) rather than a run of shifted substitutions; because MarkerType is -derived from the same events, the two annotations cannot disagree. -Canonical tags that share an ambiguity code are scored as matching at -that column by the alignment, its prescreen and the substitution count -alike — the relation `tag_mismatches` uses. Rerunning an output with +The pipeline describes every row by its cheapest reading against Ref (a +gapped reading only when strictly cheaper than substitutions), through +the alignment with the fewest gap openings among equal-edit ones (a +three-state banded table). An inserted block is therefore one event even +where its bases happen to match the reference, as an affine-gap aligner +would describe it. Rows of a marker whose own Alt differs from Ref by a +gap always get a gapped reading, so an AltMatch of a designed insertion +that adds one SNP lists the inherited insertion plus the SNP (and carries +the label `snp`) rather than a run of shifted substitutions. MarkerType +comes from the same events, so the two annotations cannot disagree. +The alignment, its prescreen and the substitution count alike score +canonical tags that share an ambiguity code as matching at that column, +which is the relation `tag_mismatches` uses. Rerunning an output with `--annotations off` removes a sidecar left by an earlier run. A -microhaplotype with several SNPs and indels is simply a longer -list: phase lives in the row, and a per-site consumer emits one record +microhaplotype with several SNPs and indels gives a longer list. Phase +lives in the row, and a per-site consumer emits one record per event, which is what `BIGr::madc2vcf_all` does today for -substitutions by realigning each row to Ref. Checks: the Alt-row token +substitutions by realigning each row to Ref. In the checks, the Alt-row token reproduces the design's own `position:Ref>Alt` for 3,120/3,120 sweetpotato, 3,005/3,015 blackberry-design, 2,982/3,000 red clover and 3,030/3,040 private-DArTag-panel loci (every remainder is an Alt tag that -is not a single SNP — ambiguity codes, or a vendor Alt tag that carries -an 8-bp insertion, reported as such); replaying each annotated row's -events onto its Ref tag returns the row's own sequence for all 9,430 -annotated rows of the public sweetpotato plate (16,954 rows; 43,766 +is not a single SNP, such as ambiguity codes, or a vendor Alt tag that +carries an 8-bp insertion, reported as such). Replaying each annotated +row's events onto its Ref tag returns the row's own sequence for all 9,430 +annotated rows of the public sweetpotato plate (16,954 rows with 43,766 substitutions, 1,700 insertions, 1,830 deletions), the delta smelt GT-seq -run (1,775 rows) and the private project's 15-sample run (9,430 rows; a +run (1,775 rows) and the private project's 15-sample run (9,430 rows, a median of two events per annotated row, 41 on the most heavily -substituted one); the annotations add no measurable runtime (the 15-sample -run: 78.7 s off, 76.8 s on). Row counts by +substituted one). The annotations add no measurable runtime (the +15-sample run took 78.7 s off and 76.8 s on). Row counts by label (`marker_types`) and by event kind, plus the sidecar path (`variants`), are in the QC summary. -Consumers: HapApp's fixed-allele-ID step keeps columns 1–3 plus the +Among consumers, HapApp's fixed-allele-ID step keeps columns 1–3 plus the samples, so these columns do not travel through it to -`BIGr::madc2vcf_all` (the sidecar does not need to); BIGr's raw-MADC +`BIGr::madc2vcf_all` (the sidecar does not need to). BIGr's raw-MADC readers (`get_countsMADC`, `filterMADC`) drop statistics columns by -name, so until `MarkerType` and `Variants` are on their list, files -headed there should be generated with `--annotations off` or the two -names added; polyRAD's `readDArTag` and HapApp locate samples -positionally and are unaffected. +name, so until `MarkerType` and `Variants` are on their list, generate +files headed there with `--annotations off` or add the two names. +polyRAD's `readDArTag` and HapApp locate samples positionally and are +unaffected. ## Raw deliveries, tag reconstruction and denoiser scaling (0.12.0) -Three changes made while setting up the alfalfa (DAl21-5779, DAl21-6024) -and pecan (DPec23-8107) panel-paper datasets, none of which alters a table -produced from a preprocessed delivery: +Three changes came out of setting up the alfalfa (DAl21-5779, DAl21-6024) +and pecan (DPec23-8107) panel-paper datasets, and none of them alters a +table produced from a preprocessed delivery. **Raw DArT deliveries.** Older DArT FASTQs keep the inline barcode and the library's common sequence `GATGGATCCAGTGC` in front of every amplicon -(alfalfa 2021: 12 + 14 bases, then the 54-bp tag, on 83-bp reads; pecan -2023: the same 26 bases, the 81-bp tag, then adapter and poly-G on 138-bp -reads), so nothing is primer-started and a 0.11.0 run assigned no read. +(alfalfa 2021 has 12 + 14 bases, then the 54-bp tag, on 83-bp reads, and +pecan 2023 has the same 26 bases, the 81-bp tag, then adapter and poly-G +on 138-bp reads), so nothing is primer-started and a 0.11.0 run assigned no read. `generate --leading-anchor SEQ` and `infer-design --leading-anchor SEQ` cut each read where the anchor ends (it must end within 40 bases beyond -its own length) and count reads without it as `reads_no_leading_anchor`; -the anchor is part of the cache key, the provenance line and the QC +its own length) and count reads without it as `reads_no_leading_anchor`. +The anchor is part of the cache key, the provenance line and the QC parameters (so assignment caches written by 0.11.0 rebuild once, as they -did when the partial-read length joined the key). Both commands check the first files before any read pass: -when almost no read starts with a panel tag but most carry one after a +did when the partial-read length joined the key). Both commands check +the first files before any read pass. +When almost no read starts with a panel tag but most carry one after a constant prefix, the run stops and names that prefix (the suffix shared by the bases in front of the tag at the modal offset, sampled across files so per-sample barcodes do not survive), and `--leading-anchor auto` applies the detected sequence instead (`leading_anchor_detected` in the QC). A -preprocessed delivery passes the check silently (red clover: 87% of reads -start with a panel tag, 0.3% carry one further in), and `auto` refuses on -it rather than cutting anything. Measured: alfalfa assigns 5–6% more reads -per sample than the vendor counted, as the other DArTag panels do; the -primer-free route on the raw alfalfa reads, with no primer file, resolves -2,999 of 3,000 loci in the exact tier, all identical to the design tags. - -**Design route: a tag carries its variant.** Tag reconstruction from the +preprocessed delivery passes the check silently (in red clover, 87% of +reads start with a panel tag and 0.3% carry one further in), and `auto` +refuses on it rather than cutting anything. In the measurements, alfalfa +assigns 5–6% more reads per sample than the vendor counted, as the other +DArTag panels do. The primer-free route on the raw alfalfa reads, with +no primer file, resolves 2,999 of 3,000 loci in the exact tier, all +identical to the design tags. + +**In the design route, a tag carries its variant.** Tag reconstruction from the primer table and probe design now keeps only the primer-started windows that contain the bracketed variant. The pecan design has one marker -(Chr04_45131234) whose target carries a duplicated primer-bearing flank; -both placements showed the Ref base at the variant offset and the marker -was reported as ambiguous. With the rule, the pecan design tags match the -vendor's Ref/Alt rows at 3,100 of 3,100 markers (3,084 identical, 16 under -vendor IUPAC codes). Any DArT marker table with `MarkerName` and a bracket -`TargetSequence` serves as the probe design, extra markers included. +(Chr04_45131234) whose target carries a duplicated primer-bearing flank. +Both placements showed the Ref base at the variant offset, so the design +route reported the marker as ambiguous. With the rule, the pecan design +tags match the vendor's Ref/Alt rows at 3,100 of 3,100 markers (3,084 +identical, 16 under vendor IUPAC codes). Any DArT marker table with +`MarkerName` and a bracket `TargetSequence` serves as the probe design, +extra markers included. **Denoiser scaling.** At the default distance of 1 the eligible parents of a sequence are its single-substitution neighbours, which `denoise_counts` now enumerates and looks up once the centroids outnumber those neighbours -(the scan stays cheaper at an ordinary locus with a few dozen centroids; -the choice is made per sequence); the selection is identical (a randomized test checks it against the scan, and -the sweetpotato table regenerates byte-identically from cache, and the alfalfa tables re-stamped with 0.12.0 are byte-identical in body to the ones made with the 0.11.0 denoiser). -The pecan panel has one paralogous locus (Chr04_23494647: reads 10–16 +(the scan stays cheaper at an ordinary locus with a few dozen centroids, +and the function chooses per sequence). The selection is identical (a +randomized test checks it against the scan, the sweetpotato table +regenerates byte-identically from cache, and the alfalfa tables +re-stamped with 0.12.0 are byte-identical in body to the ones made with +the 0.11.0 denoiser). +The pecan panel has one paralogous locus (Chr04_23494647, with reads 10–16 mismatches from the design tags around a GGGGGG homopolymer, about 6,000 -unique sequences per sample, 409 rows in the vendor table) that made the -old pass quadratic: its finalisation ran for more than 90 minutes before -being stopped, and completes in minutes now. The locus is not flagged in -the generate QC yet; a per-locus diversity and paralog warning is the -natural next addition. +unique sequences per sample, and 409 rows in the vendor table) that made +the old pass quadratic. Its finalisation ran for more than 90 minutes +before being stopped, and now completes in minutes. The generate QC does +not flag the locus yet, and a per-locus diversity and paralog warning is +the natural next addition. -80 unit tests. +The test suite has 80 unit tests. -**0.12.1: the finalisation runs across the workers.** `generate`'s +**In 0.12.1 the finalisation runs across the workers.** `generate`'s finalisation (protection, per-sample denoising, the pooled pass, the -bookkeeping) ran in one process; on a family of 169 rhAmpSeq runs with +bookkeeping) ran in one process. On a family of 169 rhAmpSeq runs with 150-bp windows and 637K reads per sample (311K distinct sequences per sample, 67 per locus, so the lookup rule above never triggers) that stage took 107 minutes against 32 for the eight-worker assignment. Stage timers -(`finalize: ... in N s` on stderr) showed where it went on a 44-run family: -protection 30 s, per-sample denoise 154 s, pooled pass 1,123 s. Both +(`finalize: ... in N s` on stderr) showed where it went on a 44-run +family, with protection at 30 s, per-sample denoise at 154 s, and the +pooled pass at 1,123 s. Both denoising passes are independent per unit of work, the per-sample pass per sample and the pooled pass per locus (its aggregate, the protected set and the gapped classifier that checks a move keeps the target allele), so @@ -1313,145 +1347,2771 @@ table write stay in the main process. The table is unchanged with A DArTag order whose delivered files were trimmed to 109–113 bp per file, with several amplicons 1–9 bases shorter than the 109-bp window, showed -what the 0.10 rule "reads shorter than the tag are dropped" costs: two loci +what the 0.10 rule "reads shorter than the tag are dropped" costs. Two loci with 101- and 102-bp amplicons had no reads at all in the table (the vendor -counted 104,507 and 69,381), a Y-specific locus lost 46 of 211 carrier -samples because one plate's reads were 106 bp, and in cells where one -allele's amplicon is shorter than the other's every read of that allele was -discarded, so the cell was called homozygous for the other allele (546 +counted 104,507 and 69,381), and a Y-specific locus lost 46 of 211 carrier +samples because one plate's reads were 106 bp. In cells where one +allele's amplicon is shorter than the other's, the rule discarded every +read of that allele, so the pipeline called the cell homozygous for the +other allele (546 Ref reads of 109 bp against 478 Alt reads of 108 bp in one sample). **Truncated reads support alleles but never invent them (partial-read -salvage, on by default).** Reads shorter than the window — adapter- or -quality-trimmed reads at amplicons a few bases short of the tag, or files -trimmed to fewer bases than the tag — are primer-assigned and then credited -to the one full-length allele that has already earned a row and of which the -read is an exact prefix. Two guards -keep this from moving a call: the read must be longer than the longest +salvage, on by default).** The pipeline primer-assigns reads shorter than +the window (adapter- or quality-trimmed reads at amplicons a few bases +short of the tag, or files trimmed to fewer bases than the tag) and then +credits them to the one full-length allele that has already earned a row +and of which the read is an exact prefix. Two guards keep this from +moving a call. The read must be longer than the longest prefix shared between any two learned alleles of its locus (a locus credits truncated reads for both target alleles or for neither, so a ratio can never shift, and a read that does not reach the target base is always ambiguous), and `--min-partial-length` defaults to `--tag-length` minus 9, -so at most nine tail bases are unobserved and primer-dimer fragments never -qualify. Defaults: `--partial-read-mode assign`; `off` restores the drop -behaviour. An `extend` mode that borrowed the design Ref suffix to turn an -unmatched prefix into a provisional allele existed through 0.11.0 and was -removed: on the project above it manufactured a RefMatch row from a 30-bp -primer-dimer. A tag table that carries identical Ref +so at most nine tail bases go unobserved and primer-dimer fragments never +qualify. The default is `--partial-read-mode assign`, and `off` restores +the drop behaviour. An `extend` mode that borrowed the design Ref suffix +to turn an unmatched prefix into a provisional allele existed through +0.11.0 and is now gone, because on the project above it manufactured a +RefMatch row from a 30-bp primer-dimer. A tag table that carries identical Ref and Alt tags (presence/absence designs) shares its whole tag between the two -alleles, so no partial is ever credited at such a locus. The QC reports `partial_reads_assigned`, -`partial_reads_ambiguous` and `partial_reads_unmapped` per sample. The -minimum partial length is part of the cache fingerprint, so runs that relied -on the old default of 30 rebuild their cache once. +alleles, so the pipeline never credits a partial at such a locus. The QC +reports `partial_reads_assigned`, `partial_reads_ambiguous` and +`partial_reads_unmapped` per sample. The minimum partial length is part +of the cache fingerprint, so runs that relied on the old default of 30 +rebuild their cache once. **Measured on that project (564 samples, 3,040 markers, band 20).** The default salvage credits 3.04M reads (+0.98%) to 37,359 cells at 979 loci, makes 15,987 more cells callable for dosage, and raises count agreement with the vendor on the 24,755 shared rows (Pearson r over nonzero cells) from -0.961 to 0.971, because the vendor counts these near-full reads too; dosage +0.961 to 0.971, because the vendor counts these near-full reads too. Dosage agreement with the vendor moves from 97.83% to 97.85% on 1.53M cells, and BIGr `madc_summary` missingness (cells under 10 reads) falls from 7.55% to -6.62% overall against the vendor's 6.74% (per-sample median 7.20% → 6.22% -against 6.38%). The locus guard, run against the same cache without it, removes 504,677 of the -salvaged reads from 7,914 cells at 269 loci — loci with a learned Match row -that differs from Ref or Alt only beyond the truncated length — and reverts +6.62% across all cells against the vendor's 6.74% (per-sample median +7.20% → 6.22% against 6.38%). The locus guard, run against the same cache +without it, removes 504,677 of the salvaged reads from 7,914 cells at 269 +loci, which are loci with a learned Match row that differs from Ref or +Alt only beyond the truncated length. It also reverts 469 of the 472 dosage-class changes that unguarded crediting introduces. The 2,316 cells (0.15%) whose dosage class differs from the no-salvage table are the allele-specific amplicon-length cases above, which the no-salvage -table calls homozygous; two independent instruments say the new calls are -right: their haplotypes fall within the locus's four founder haplotypes in -the F2 families at 99.8% (GEN, n=1,067) and 100% (DV, n=558), the F2 rate, -and re-call in technical replicates at 98.6% against 98.5% for the calls -they replace, while the vendor's own call agrees with the new call in 1,276 -of 2,164 cells and with the old one in 737. Reads that stop before the -target (a 35-bp junction product at one Y-specific locus, 90-bp amplicons -whose target sits beyond them, primer-dimers) stay out: they are below the -length minimum, and would be ambiguous between Ref and Alt even above it. +table calls homozygous. Two independent instruments show that the new +calls are right. Their haplotypes fall within the locus's four founder +haplotypes in the F2 families at 99.8% (GEN, n=1,067) and 100% (DV, +n=558), the F2 rate, and re-call in technical replicates at 98.6% +against 98.5% for the calls they replace. The vendor's own call agrees +with the new call in 1,276 of 2,164 cells and with the old one in 737. +Reads that stop before the target (a 35-bp junction product at one +Y-specific locus, 90-bp amplicons whose target sits beyond them, +primer-dimers) stay out, because they are below the length minimum and +would be ambiguous between Ref and Alt even above it. The assign stage adds under a minute to a 50-minute run. The preamble now records `min_partial_length=` beside `partial_read_mode=`. -**Measured on three other DArTag panels (salvage on against off; same code, -scripts and band 10).** Sweetpotato (80 samples, 81-bp tags): +40,438 reads -(0.04%) at 304 loci; 5 of 196,494 called dosage cells change class and the -vendor's own call agrees with the salvaged class in all 5; leaf/root -replicate re-call 90.11% either way; count correlation with the vendor table -0.9993 → 0.9997. Red clover (376 samples, 109-bp tags, primer-free panel): -+941,538 reads (1.3%) at 823 loci, 8,733 more callable cells, 786 of 717,768 -called cells change class (the vendor sides with the salvaged class in 714, -with the unsalvaged in 14); dosage agreement 96.87% → 96.98%, count -correlation 0.9765 → 0.9845. Blackberry (376 samples, 109-bp tags): -+1,552,191 reads (0.76%) at 634 loci, 6,160 more callable cells, 2,112 of -1,006,967 change class (1,994 against 50); dosage agreement 95.18% → 95.39%, -count correlation 0.9823 → 0.9867. Rows are identical in every pair: a -partial read never establishes one. The credited reads sit at loci whose +**Measured on three other DArTag panels (salvage on against off, with the +same code, scripts and band 10).** Sweetpotato (80 samples, 81-bp tags) +gains +40,438 reads (0.04%) at 304 loci. Of 196,494 called dosage cells, +5 change class, and the vendor's own call agrees with the salvaged class +in all 5. Leaf/root replicate re-call is 90.11% either way, and count +correlation with the vendor table goes 0.9993 → 0.9997. Red clover (376 +samples, 109-bp tags, primer-free panel) gains +941,538 reads (1.3%) at +823 loci and 8,733 more callable cells, and 786 of 717,768 called cells +change class (the vendor sides with the salvaged class in 714, with the +unsalvaged in 14). Dosage agreement goes 96.87% → 96.98% and count +correlation 0.9765 → 0.9845. Blackberry (376 samples, 109-bp tags) gains ++1,552,191 reads (0.76%) at 634 loci and 6,160 more callable cells, and +2,112 of 1,006,967 change class (1,994 against 50). Dosage agreement goes +95.18% → 95.39% and count correlation 0.9823 → 0.9867. Rows are +identical in every pair, because a partial read never establishes one. +The credited reads sit at loci whose amplicons run short of the tag, where the counts land on the vendor's totals -(one red clover locus: 2,606 → 83,324 reads against the vendor's 83,303); -loci whose amplicons are shorter than the partial minimum (78–84 bp at the -largest red clover examples) or whose reads end before the distinguishing -base stay uncredited, by design. +(one red clover locus went 2,606 → 83,324 reads against the vendor's +83,303). Loci whose amplicons are shorter than the partial minimum (78–84 +bp at the largest red clover examples) or whose reads end before the +distinguishing base stay uncredited, by design. -**One reading per row (0.11.0).** 0.10.0 read a row twice: classification +**One reading per row (0.11.0).** 0.10.0 read a row twice. Classification confirmed a gapped reading on a unit-cost traceback with the composition guard and the 12-base verified suffix, while `Variants`/`MarkerType` took the fewest-openings alignment unconfirmed, so a row could list an indel its category ignored, or be classified through a gap its annotation placed -elsewhere. Both now use `row_reading`: the cheaper of the two +elsewhere. Both now use `row_reading`. The cheaper of the two substitution-only readings is the bound, the fewest-openings alignment is -the reading, and it is accepted only when verified by GAP_VERIFIED_SUFFIX -bases after its last gap and well composed — composition being judged on -the equal-edit explanation that needs the fewest substitutions -(`fewest_substitution_alignment`), because the fewest-openings explanation -trades a gap opening for a substitution wherever it can, and against the -canonical the row is read from, so a designed substitution never counts -against an AltMatch row's indel reading. A reading that fails is not a -reading of the row at all: the row is classified at the fixed offset and -described as substitutions, by both consumers. On the project above this +the reading, and the function accepts it only when GAP_VERIFIED_SUFFIX +bases after its last gap verify it and it is well composed. It judges +composition on the equal-edit explanation that needs the fewest +substitutions (`fewest_substitution_alignment`), because the +fewest-openings explanation trades a gap opening for a substitution +wherever it can, and against the canonical the row is read from, so a +designed substitution never counts against an AltMatch row's indel +reading. A reading that fails is not a reading of the row at all. Both +consumers then classify the row at the fixed offset and describe it as +substitutions. On the project above this re-labelled 76 of 28,793 rows (53 `Other` rows became RefMatch/AltMatch, 16 swapped between RefMatch and AltMatch, 7 became `Other`) and rewrote the annotation of 554 rows whose listed indel sat inside the last 12 bases or -whose composition failed the guard (`both`/`indel` → `snp`); counts moved by +whose composition failed the guard (`both`/`indel` → `snp`). Counts moved by 97 reads in 66 rows through the category-aware pooled denoise. -**Freeze health (0.11.0).** Freezing a window panel (`--reuse-tags`) is -what keeps haplotype identity comparable across runs, but read geometry +**Freeze health (0.11.0).** Freezing a window panel (`--reuse-tags`) +keeps haplotype identity comparable across runs, but read geometry drifts run to run and a frozen window can slide off the reads a new run delivers. `normalize-reads --reuse-tags` therefore still runs the coverage -pass fresh placement would use and scores every frozen window against it: -the QC's `markers` table carries each window's predicted full coverage on +pass fresh placement would use and scores every frozen window against it. +The QC's `markers` table carries each window's predicted full coverage on this run beside the window fresh placement would choose today and its coverage, and `freeze_health` summarises the read-weighted coverage of the frozen panel against the fresh one, the markers losing more than ten -points, and the ten worst. One line on stderr says what freezing costs -this run; `--window-placement center` skips the pass. The excised FASTQs -and the tag table are unchanged by the check. +points, and the ten worst. One line on stderr reports what freezing costs +this run, and `--window-placement center` skips the pass. The check +leaves the excised FASTQs and the tag table unchanged. **Vendor-only markers, re-derived.** Of the 53 markers in the tag table that had no reads in the 0.10 table, 46 carry vendor counts (40,101 called -cells): 33 markers (29,861 cells) have only primer-bearing fragments that -end before the target (the vendor counts them as Ref), 10 (8,478 cells) -have 44–77-bp amplicons that do contain the target — recoverable only by a -per-locus window, not by salvage — or, for two of them, reads that are -largely absent from the delivered files, 2 (1,762 cells) have 101/102-bp -amplicons and are counted since 0.11.0, and 7 have no reads anywhere and no -vendor counts. The 46 markers whose vendor Ref and Alt rows are identical -are a separate class: their target sits at position 110, one base past the +cells). By class, 33 markers (29,861 cells) have only primer-bearing +fragments that end before the target (the vendor counts them as Ref), and +10 (8,478 cells) have 44–77-bp amplicons that do contain the target, +recoverable only by a per-locus window and not by salvage, or, for two of +them, reads that are largely absent from the delivered files. Another 2 +(1,762 cells) have 101/102-bp amplicons, which the pipeline has counted +since 0.11.0, and 7 have no reads anywhere and no vendor counts. The 46 +markers whose vendor Ref and Alt rows are identical are a separate class. +Their target sits at position 110, one base past the 109-bp window (the vendor's SNP report lists them without a position, and the 110th base segregates in the plates whose reads are longer than 109 bp). +## Genome paralog filter (`filter-paralogs`, 0.14.0) + +On the public lettuce 3K DArTag data about two thirds of the reads come from +paralogous copies. A paralog read that carries the target base becomes a +RefMatch/AltMatch row, and `BIGr::madc2vcf_all` sums it into the target SNP, +so inbred lines look heterozygous. Raising `generate --min-identity` removes +the diverged copies but not the close ones. `filter-paralogs` is an opt-in +filter for such panels. It reads any MADC, from MADCATS or the vendor, with +or without the preamble, so it also filters vendor deliveries, and HapApp's +fixed-allele-ID output (columns `Code_version`, `AlleleID`, `CloneID`, +`AlleleSequence` and samples, IDs such as `|Ref_0001`, and merged counts +written as `12.0`). The command lives in `madcats_paralogs.py`, which imports +the MADC helpers it shares with `validate` from `madcats`, and `build_parser` +imports it when it builds the parser. Run as `python3 madcats.py`, the file +is `__main__`, so it registers itself as `madcats` before the filter module +imports it. Otherwise the module would load a second copy whose `MadcError` +`main` does not catch. + +Design: + +- **Aligner.** The command uses minimap2 through `mappy`, an optional extra + imported only by this command, so MADCATS still has no required + dependency. The index is a standard `-x sr` index (k=21, w=11). At map + time the command allows single-seed chains (`min_cnt=1`, + `min_chain_score=20`) and 50 hits (`best_n`). mappy reports neither AS + nor XS, so the command recomputes each hit's DP score from CIGAR, NM, + matching bases and block length with the pinned short-read scoring + (+2/−8, gaps 12+2 or 24+1, ambiguous −1). Ties are exact. +- **Marker placement.** The command aligns each marker's Ref tag and keeps + every equal-best site the search returns. It reads the CloneID + `CHROM_POS` as a position on the genome given. The position picks among + equal-best Ref-tag sites and stands in (±tag length) when the Ref tag does + not align. When no equal-best site covers it, the command realigns the Ref + tag to a k=11 index of the window around the position (±tag length + + 20 bp), and an alignment covering the position becomes the marker's site + (`clone_id_site`), even when the tag scores better elsewhere. Without one + the marker is `unplaced`. `--clone-positions ignore` turns these uses off, + for a genome other than the one the CloneIDs refer to. On the lettuce + vendor MADC against v8 it changes 11 markers. The 9 fallback windows + become `unplaced`, keeping 1,032 rows labelled `paralog`, and 1 tie stays + multi-copy. `Chr2_001081815` goes back to Chr4, dropping 11 rows that + `use` keeps. The id95 labels do not change. +- **Labels.** Each haplotype gets its best score at its marker's site and + its best score elsewhere. A haplotype that scores higher at the marker is + `on_target`, one that scores higher elsewhere is `paralog`, and a tie is + `ambiguous`. The command also realigns every haplotype to a k=11 index of + its own marker's sites (±20 bp), and the site score is the higher of that + and the genome-wide hit there. Without that realignment, 18% of id95 + haplotypes had no hit at their own marker, and 326 that bwa places on + target were unmapped. Each marker's sites get an index of their own, so + markers sharing a site never compete for the preset's hit limit. Hits are + one locus when they share at least half of the shorter interval, both + when the command merges a sequence's hits and when it matches a hit to its + marker's site. +- **Actions.** The command drops paralog and ambiguous rows by default, or + flags them in the `PICRef` column, renamed `GenomeLabel`. It never drops + canonical rows. Every input row, dropped or kept, gets a line in the + labels sidecar. + +Measurements (lettuce v8 genome, 2.4 Gb, 12-core laptop): + +| | id95 MADCATS MADC | DArT vendor MADC | +|---|---|---| +| Rows / distinct sequences | 17,847 / 17,846 | 732,534 / 732,154 | +| Run time, prebuilt index | 12 s (8 threads) | 2 min 15 s (8 threads), 9.4 GB | +| Rows kept | 12,695 | 113,667 | +| Reads in paralog rows | 16.0% | 63.3% | +| Keep/drop agreement with bwa-mem labels | 98.2% | 94.9% | +| Keep/drop agreement with the bwa prototype's output, rows / reads | 98.2% / 99.7% | 94.9% / 99.4% | +| Het rate (proxy), unfiltered → prototype → `filter-paralogs` | 12.6% → 6.44% → 6.40% | 19.9% → 6.79% → 7.02% | + +Indexing took 50 s and 10.6 GB at peak, and the `.mmi` is 5.7 GB. The het +proxy sums Ref+RefMatch and Alt+AltMatch reads per marker and sample, calls +cells with ≥10 reads, and counts het where the minor allele has ≥10%. The +bwa-mem labels came from aligning the same haplotypes with `bwa mem -T 20` +(AS > XS unique). Variants tried and rejected: + +- With default sr chaining, keep/drop agreement on id95 was 97.3%, against + 98.2% with single-seed chains and `best_n` 50. +- A k=15, w=10 index gave a DArT agreement of 94.4%, against 94.9% with + k=21, and slower mapping. +- Also dropping site-realigned DArT haplotypes with ≥6 mismatches gave a + het proxy of 6.93%, against 7.02%. The rest of the gap to bwa is minimap2 + finding fewer distant copies of highly diverged vendor haplotypes. + +Count overflow. The lettuce vendor MADC stores counts in a signed byte, so +90,913 cells across 1,108 markers are negative (minimum −128) and the +largest count is 127. Counts of 128–255 appear as count − 256 +(adding 256 matches MADCATS's counts for the same cells), and counts of 256 +or more wrap silently. `read_madc` (and so `validate`) and `filter-paralogs` +now warn and record `count_overflow` in their reports. They never correct +the counts, since the wrapped ones are indistinguishable from real small +counts. The figures in the table above use the counts as stored. + +HapApp layout. The id95 MADC rewritten in HapApp's layout (fixed IDs, +leading `Code_version`, every tenth row's counts as floats) gets the same +label, action and scores for all 17,847 rows as the raw MADC, and the same +QC totals. Moving the command into its own module left the id95 output, both +sidecars included, byte-identical. + +Review fixes. An external review of the first version (cd2d7cb) gave a +counterexample for each of the defects below, and each is now a test +(`ReviewCaseTests`, and `FilterTests` with the fake aligner). Effects on +the lettuce MADCs, default `use` mode: + +| Fix | Effect | +|---|---| +| Ref tag realigned at its CloneID position on disagreement | For `Chr2_001081815` the genome-wide search returned 51 hits, best on Chr4 (96), none at its Chr2 design site (156). Placed on Chr2, 11 of its 14 dropped DArT rows become `on_target` | +| Every haplotype scored at its own site, and the higher of the two scores kept | 530 DArT rows that the shared site index had crowded out get a site score, and 1 row (`Chr5_323370531`, genome-wide 86 at the site, 88 elsewhere, 94 realigned) becomes `on_target` | +| One locus rule for merging hits and for target membership | No label changes, but 850 DArT sequences had adjacent hits merged before | +| Every equal-best Ref-tag site kept for placement | No lettuce Ref tag has more than 8 | +| Multi-part `.mmi` refused | The lettuce index is one part, and reading its layout takes 0.03 s | + +Together, 12 DArT rows (994 reads) go from dropped to kept and none the +other way. Keep/drop agreement with bwa-mem on the 726,693 DArT Match/Other +rows with bwa labels rises from 94.8128% to 94.8136% (+6 rows), and id95 +labels do not change (97.3484% on 11,842 rows). The DArT run took 76 s, +against 78 s for cd2d7cb in the same session. +For `Chr2_001081815`, both the hit limit and the ambiguity codes matter. The +vendor tag, or the tag with R and M read as N, misses the design site at +`best_n` 50, 200 and 1,000 and finds it at 5,000. With the codes resolved +to bases the search finds it at `best_n` 50. bwa agreement is concordance, +not truth, and source-known fixtures for both errors (target evidence +removed, paralog evidence kept) are still missing. + +Review of the module. Two readers went through `madcats_paralogs.py`, one for +structure and one for names, comments, sources and tests, as for +`madcats_call.py`. They found three defects, now fixed: + +| Defect | Effect before | Now | +|---|---|---| +| A minimap2 index in several parts was refused only as `--genome-index` | Given as `--genome` it loaded, mappy searched its first part, and the command judged markers on later contigs without their own site | Refused whichever option names it | +| `validate` and `filter-paralogs` took a processed MADC's `readCountSum` column for a sample (only `call` skipped it) | QC read totals of `filter-paralogs` doubled, and `validate` listed it among the samples | Skipped by the helper all three use | +| A run that failed while writing (a non-numeric count in a late row) | A truncated MADC and labels file stayed on disk | Removed | + +The rest changed no output. `filter_paralogs`, 211 lines doing five jobs, is +now five functions, and both passes read their rows through one generator. +The command checks the MADC's header before it loads the index. Names and +comments now agree with the code. They say that a marker's CloneID position +places it as well as its Ref tag, that the score is that of the reported +alignment and not minimap2's `AS`, and that the genome-wide search returns +up to 51 hits and not 50. The module says what is minimap2's (Li 2018 and +Li 2021) and what is MADCATS's own. +The outputs of eight runs on the data below were the same before and after, +byte for byte, and the same on one thread and on four. + +Both passes skip rows with an empty `AlleleID`, or too short to hold the +identity columns, and the QC totals do not count them. That is now +documented and tested. Counting them would add a key to the QC JSON and was +left for a decision. + +What the kept rows of the vendor MADC are. The filter keeps 113,667 of the +lettuce DArT MADC's 732,534 rows. Read counts in this paragraph and the next +add 256 to the negative cells. + +| Kept rows | Rows | Share of the kept reads | +|---|---|---| +| Canonical Ref and Alt, `on_target` | 5,687 | 87.5% | +| RefMatch, AltMatch and Other, `on_target` | 105,913 | 12.4% | +| `unmapped`, and 9 canonical rows kept despite their label | 2,067 | 0.1% | + +Most of the 105,913 are not haplotypes that recur in the germplasm. In +89,070 of them (84%) no sample has 5 reads. Another 8,779 reach 5 reads only +in 25 samples that stand apart from the other 351. The 25 are the samples in +which the designed alleles hold less than three quarters of the kept reads +(4 to 71%, against a median of 90% in the others), and the filter drops +76.1% of their reads at the median, against 57.5%. The sample table names +them (PI accessions, SAL lines, IVT 280, UC96US23, two other lines and two +mixes of Salinas DNA with one of them) but not their species. Of the +remaining rows, 6,367 reach 5 reads in one to four of the 351 samples and +1,697 in five or more. These 1,697, at 1,037 markers, hold 95% of the reads +that all 105,913 rows have in the 351 samples, and 1,290 of them are within +3 differences of the reference at their site. bwa mem keeps 71% of the +105,913 rows and 92% of their reads. + +Lettuce against blastn. blastn 2.17.0 (`-task blastn -word_size 11`, no +masking, as for alfalfa below) aligned a sample of 18,763 rows of the vendor +MADC to the v8 genome. Each row got a label from the bit scores by the +filter's rule, with the marker's site widened by 20 bases as its home. The +sample holds every kept `on_target` RefMatch, AltMatch and Other row that +has 5 reads in a sample and reads in 5 samples ("well supported"), and +random draws of the other rows. The whole table would have taken about 11 +hours on 8 threads. "Same decision" is to keep or to drop, as the default +does. + +| Rows | In the sample | Filter and blastn, same decision | Filter and bwa mem | bwa mem and blastn | +|---|---|---|---|---| +| Dropped as `paralog`, well supported | 3,000 of 65,119 | 99.6% | 99.8% | 99.5% | +| Dropped as `paralog`, others | 2,000 of 528,208 | 99.4% | 99.2% | 99.0% | +| Dropped as `ambiguous` | 1,500 of 25,540 | 91.5% | 89.3% | 87.2% | +| Kept as `on_target`, well supported | 8,763 of 8,763 | 68.9% (88.9% of reads) | 80.4% | 85.0% | +| Kept as `on_target`, others | 3,000 of 97,150 | 50.8% (71.1% of reads) | 71.9% | 75.3% | +| `unmapped`, which the filter keeps | 500 of 2,056 | 7.2% | 31.2% | 74.4% | + +The three agree on what the filter drops. They differ on what it keeps, and +the difference grows with the row's distance from the reference at its own +site (the score lost there, 10 for a substitution). Of the well-supported +kept rows: + +| Distance from the site | Rows | Share of their reads | Kept by bwa mem and by blastn | Dropped by both | +|---|---|---|---|---| +| 1 to 3 | 3,726 | 76.9% | 97.4% | 0.8% | +| 4 or 5 | 2,001 | 12.0% | 71.2% | 12.3% | +| 6 or more | 3,036 | 11.1% | 27.4% | 42.5% | + +blastn aligns 86% of the rows 6 or more from the site over the whole tag +there, at 95% identity or less, so they are diverged sequences and not +partial reads. The genome-wide search seeds with 21 bases and misses copies +of such sequences that bwa mem and blastn find, while the realignment at the +site seeds with 11. Of the 1,697 rows that recur in the 351 samples, both +aligners keep 1,480 (87%, with 91% of the reads) and both would drop 122 +(7%, with 5%). Rows the filter keeps and bwa mem would drop hold 0.97% of +all kept reads, well-supported rows that blastn would drop hold 1.26%, and +well-supported rows that both would drop hold 0.71%. These reads are +concentrated. Of 2,986 markers, 61 have a tenth or more of their kept reads +in rows bwa mem would drop, 61 in rows blastn would, 33 in rows both would, +and 10 have a quarter or more in rows both would. blastn places 90% of the +sampled `unmapped` rows elsewhere, but all 2,056 of them hold 26,000 reads. +So both aligners confirm the rows the filter drops, and it keeps about 1% of +reads that they would not, in rows far from the reference. The two aligners +share the reference with the filter, and their agreement is still not truth. + +Delta smelt GT-seq, placement against the primers. The public smelt data +(see "Delta smelt GT-seq against alignment-based genotypes" under genotype +calling) give a second species and platform, and an independent answer for +where each marker lies. That answer is the 81 bases from its forward primer, +found by exact search of both primers in the genome (GCA_021917145.1, +437 Mb). The MADC has 368 markers, 1,527 rows and 10.2 million reads. + +| Run | Markers at the expected site | Unplaced | Rows labelled | +|---|---|---|---| +| CloneIDs renamed to CHROM_POS of the target | 368 | 0 | 1,527 on_target | +| The same, `--clone-positions ignore` | 368 | 0 | 1,527 on_target | +| 29 markers' positions moved 5,000 bases and 15 put on another chromosome | 324 | 44, the moved ones | 1,350 on_target, 177 unplaced | +| The same, `--clone-positions ignore` | 368 | 0 | 1,527 on_target | +| CloneIDs as generated (locus names such as `NC_061060_1_U618`) | 354 | 14 | 1,494 on_target, 33 unplaced | + +The filter labelled no row `paralog` or `ambiguous` and dropped none. The +panel has no paralogs in this genome, so this checks placement and false +positives. A marker with a wrong position is left unplaced, not misplaced. +In the last run 100 locus names end in an underscore and digits, and the +filter reads them as CHROM_POS. For 14 the number is not within the tag and +the marker stays unplaced. The filter takes a CloneID that looks like a +position for one, which `--clone-positions ignore` avoids. Eight runs took +38 s on four threads, building the index once. + +Planted paralogs. The record above says the agreement with bwa mem is +concordance and that cases with a known answer were still missing. They +were built in the smelt genome. For each of 105 markers drawn at random, a +changed copy of the genome around its tag (the tag and 100 bases either +side) went into the genome as a contig of its own, and the MADC got the rows +a co-amplified copy would give (the copy's tag, and that tag with one more +substitution, with half the Ref row's reads each). The runs used fifteen +markers per class, three random draws, CloneIDs in CHROM_POS form and the +default options: + +| Planted copy | Rows from the copy labelled `paralog` | Rows of the marker itself labelled `on_target` | +|---|---|---| +| None (263 markers per draw) | | 3,348 of 3,348 | +| 1 substitution from the Ref tag | 90 of 90 | 173 of 179 (4 `ambiguous`, 2 `paralog`) | +| 2 substitutions | 90 of 90 | 168 of 170 (2 `ambiguous`) | +| 4 substitutions | 90 of 90 | 165 of 165 | +| 8 substitutions | 90 of 90 | 183 of 183 | +| 1 substitution and a 2-base deletion | 90 of 90 | 167 of 167 | +| Identical copy | | 1 of 192 and 191 `ambiguous`, of which the filter kept 90 canonical rows | +| 4 substitutions, copy not added to the genome | 0 of 90 because all are `on_target` | 177 of 177 | + +- The filter labelled every row from a copy that is in the genome `paralog` + and dropped it, 450 of 450, and mislabelled no row of an unplanted marker. +- Of the 864 rows of markers with a copy in the genome, the filter lost 8 + rows, all at copies one or two bases from the Ref tag. An allele of the + marker that is as near the copy as its own site is `ambiguous`, and one + that carries the copy's base is the copy's sequence. +- An identical copy makes the marker's rows `ambiguous`, and the default + drops them and keeps the canonical Ref and Alt rows. +- The filter cannot see a copy that is not in the assembly, so it keeps that + copy's rows as alleles of the marker. + +The same design, twenty markers on a random genome, is in the test suite +(`PlantedCopyTests`). + +Alfalfa, two aligners and two families. The alfalfa 3K panel is a second real +panel with paralogs, in an autotetraploid, and its public data allow three +checks that the lettuce record lacks. They place each marker from more +sequence than its tag, label each row with two other aligners, and map each +row genetically. + +- The MADCs are the two F1 families of the G3 article jkag131, with fixed + allele IDs. I195 x J432 has 186 samples, 2,999 markers and 7,517 rows of + 54-base tags, and E115 x F628 has 374 samples, 2,989 markers and 8,731 + rows of 81-base tags. The filter also ran on the panel's haplotype + database (23,469 haplotypes, Zenodo doi:10.5281/zenodo.10681524) as a + third table. +- The first genome is the monoploid assembly of cultivar Bolivia + (GCA_051527215.1, eight gapless chromosomes, 776 Mb), a genome of another + cultivar than the panel's. The panel's 361-base design sequences differ + from it at a median of 3 bases, and at 17 or more for a tenth of the + markers. The second is the genome the panel was designed on, chromosome + set 1 of XinJiangDaYe (see "The design genome" below). The first shows + what a user gets without the design genome, the second what the filter + does in its intended use. +- A marker's home on this genome is where its design sequence (180 bases + either side of the target, from the panel's probe table) aligns. bwa mem + placed 2,745 of the 3,000 uniquely and minimap2 2,780. The 2,735 that both + place uniquely are at the same base in both, and are the markers judged + below. Another 22 have a unique place on a chromosome with another number + than the panel names, and the rest have no unique place. +- Each table had two runs, one with `--clone-positions ignore`, as a user + must run it on a genome the CloneIDs do not refer to, and one with the + default after renaming the CloneIDs to the chromosome and target position + on this genome, as it runs on a design genome. The first run took 47 s and + 5.2 GB with the index built, the others 7 to 8 s. + +Placement, for the markers with a home: + +| | E115 x F628, 81-base tags (2,730 markers) | I195 x J432, 54-base tags (2,734) | +|---|---|---| +| `ignore`, at home | 2,669 | 2,487 | +| `ignore`, several equal-best sites with one at home | 17 | 28 | +| `ignore`, somewhere else | 15 | 25 | +| `ignore`, not placed | 29 | 194 | +| Positions on this genome, at home | 2,730 | 2,727 (7 not placed) | + +Without positions, the filter places a Ref tag that differs from this genome +where it matches best, or nowhere. It puts 0.5 to 0.9% of the markers in +another copy and leaves 1% (81 bases) to 7% (54 bases) unplaced, keeping all +their rows. With positions, it aligns the tag again at its own site and +places every marker but seven. + +Rows and reads by label, with positions. In E115 x F628 7,928 rows are +`on_target` (90.9% of reads), 357 `paralog` (4.3%), 297 `ambiguous` (3.8%), +9 `unmapped` and 140 `unplaced` (0.9%). The filter drops 437 rows with 4.1% +of the reads and keeps 217 canonical rows with 4.0% despite their label. In +I195 x J432 the shares are 88.7%, 4.7%, 4.5%, 0.3% and 1.8%, and the filter +drops 4.2% of the reads. Of the database's 23,469 haplotypes 1,145 are +`paralog` and 930 `ambiguous`. Lettuce lost 63% of its reads, while alfalfa +has little to remove. + +Labels against bwa mem and blastn. bwa mem (`-a -k 12 -T 20`) and blastn +(word size 11, no masking) each aligned every row to the same genome, and +each row got a label from the aligner's own score by the rule the filter +uses. The best score is at home, is equalled elsewhere, or is elsewhere. The +table counts rows of the markers placed at home. "Same decision" joins the +labels by what the default does with the row, which keeps `on_target` and +`unmapped` and drops `paralog` and `ambiguous`: + +| Table, run | Rows | Same label and same decision as bwa mem | Same label and same decision as blastn | +|---|---|---|---| +| E115 x F628, positions | 7,916 | 99.42%, 99.81% | 99.42%, 99.68% | +| E115 x F628, `ignore` | 7,716 | 99.49%, 99.86% | 99.49%, 99.74% | +| I195 x J432, positions | 6,768 | 98.63%, 99.44% | 98.73%, 99.56% | +| I195 x J432, `ignore` | 6,145 | 99.27%, 99.56% | 99.35%, 99.63% | +| Haplotype database, positions | 21,155 | 99.14%, 99.53% | 98.90%, 99.28% | +| Haplotype database, `ignore` | 20,551 | 99.17%, 99.54% | 98.99%, 99.35% | + +bwa mem and blastn give each other the same decision for 99.82% and 99.65% of +the two families' rows and 99.37% of the database's, so the filter is as near +to each of them as they are to one another. For RefMatch and AltMatch rows alone the filter and bwa mem +agree on 99.5% (81 bases) and 98.8% (54 bases). All 297 rows the filter calls +`paralog` in E115 x F628 are `paralog` for bwa mem. + +Where the rows map genetically. In a full-sib family a haplotype is inherited +with the haplotypes near it on the same chromosome copy, so across the +progeny its share of the marker's reads rises and falls with theirs, and a +row from a paralogous copy follows the haplotypes around that copy instead. +For each row whose share varies in the family, the check finds the ten rows +of other markers that correlate with it most strongly (0.4 or more in +absolute value, three or more needed) and looks up their markers' homes. The +row is linked at home when 60% or more of them lie within 15 Mb of its +marker, and linked elsewhere when 60% or more lie on one chromosome and +fewer than 20% near the marker. No alignment of the row enters this. + +| Family, run | Rows judged | Kept rows linked at home | Rows labelled `paralog` that are linked elsewhere, at home with the other site far away, and at home with it near | +|---|---|---|---| +| E115 x F628, positions | 2,877 | 2,701 of 2,768 (97.6%) | 38, 7, 29 | +| E115 x F628, `ignore` | 2,816 | 2,671 of 2,736 (97.6%) | 36, 5, 21 | +| I195 x J432, positions | 2,585 | 2,425 of 2,485 (97.6%) | 22, 4, 31 | +| I195 x J432, `ignore` | 2,408 | 2,286 of 2,340 (97.7%) | 20, 1, 21 | + +- Of the rows the filter keeps, 97.6% map to their marker in both families. + More than half of the others are at some 30 markers where every kept row + maps elsewhere. There the marker itself is somewhere else in the family + than its design sequence is on this genome, which no row label can mend. + The rest, 22 to 28 rows or about 1% of the kept rows judged, are rows from + another place that the filter did not catch, because it cannot see a copy + that this genome lacks or holds in another form. +- Of the `paralog` rows that linkage can decide, 84 to 95% map elsewhere, and + for 25 of 38 and 18 of 22 the region they map to holds the site the filter + named. +- For 21 to 31 `paralog` rows the better site is on the marker's own + chromosome within 15 Mb, for most of them 1 kb to 1 Mb away. Such a copy is + inherited with the marker, so linkage says nothing about them. +- RefMatch and AltMatch rows that map elsewhere are 9.2% of those judged + before the filter and 5.5% after in the larger family, and 8.3% and 5.2% in + the smaller. +- A RefMatch or AltMatch row with reads in every sample of the family is two + to three times as common among `paralog` rows as among `on_target` rows, + at 23% against 8%, and 43% against 13%. + +Genotypes. `call --prior norm` ran on each family before and after the +filter, and the comparison is with the article's published dosages, which +come from the same reads. Over the 986 and 974 published targets the share +of equal dosages moved from 89.80% to 89.87% and from 90.11% to 90.33%. Of +the 211 markers that lost a row in the larger family only 29 are among the +published ones (2.9%, against 7.1% of the panel), and there the share rose +from 86.3% to 86.9% with 11% of the genotypes changed. The estimated +overdispersion rose from 40 to 50 and from 30 to 40 because with those rows +gone the counts scatter less. + +A count of the progeny genotypes that the two parents' genotypes cannot give +was tried as a test and is not one. Reads of a paralog make a parent look +heterozygous, and a heterozygous parent allows every progeny genotype, so the +count rewards the rows the filter removes. It was 1.86% before and 2.20% +after at the markers that lost a row. At chr4.1_040870396 it went from 0 to +189 of 372 when the filter dropped one row. That row has a quarter to three +quarters of the marker's reads in every one of the progeny, which no allele +of a tetraploid cross has (two copies in each parent leave 1 progeny in 36 +without it). The row differs from the Ref tag at five bases and matches a +copy 10 kb away better. + +The design genome. The panel's probe table names +`M_sativa_genome_XinJiangDaYe_set1_v1.fasta`, and every CloneID is on a +chromosome named `chrN.1`, so the panel was designed on one of the four +chromosome sets of the XinJiangDaYe assembly (figshare, +doi:10.6084/m9.figshare.12327602, CC BY 4.0, with `chr1.1` to `chr8.4` and +9,789 unplaced sequences). Set 1, `chr1.1` to `chr8.1`, served as the +genome, and the MADCs ran as they are, in the default mode, so that their +own CloneIDs are the positions. bwa mem places 2,984 of the 3,000 design +sequences uniquely and minimap2 2,992, every one at the position its CloneID +names and without a single difference, so here a marker's home is its CloneID +position. + +| | E115 x F628, 81-base tags | I195 x J432, 54-base tags | Haplotype database | +|---|---|---|---| +| Markers placed at their CloneID position | 2,989 of 2,989 | 2,999 of 2,999 | 3,000 of 3,000 | +| Of them, Ref tags with several equal-best sites, settled by the position | 17 | 34 | 17 | +| Rows `on_target`, `paralog`, `ambiguous` | 8,388, 207, 136 | 7,211, 145, 159 | 22,309, 680, 480 | +| Share of reads on those rows | 96.0%, 2.2%, 1.8% | 94.6%, 2.6%, 2.8% | | +| Rows dropped, and their reads | 296, 2.6% | 202, 3.0% | 1,113 | +| Same label as bwa mem, same decision | 99.61%, 99.85% | 99.48%, 99.76% | 99.32%, 99.69% | +| Same label as blastn, same decision | 99.59%, 99.72% | 99.49%, 99.68% | 99.10%, 99.43% | +| Kept rows linked at home | 3,053 of 3,184 (95.9%) | 2,740 of 2,845 (96.3%) | | +| `paralog` rows linked elsewhere, at home with the other site far away, and at home with it near | 29, 3, 27 | 17, 1, 33 | | + +- On the design genome no marker is unplaced or put in another copy, with + CloneID positions or without them (`ignore` leaves the 17 and 34 Ref tags + with several sites unsettled and makes no other change), and the filter + places the 54-base tags as well as the 81-base ones. On the other + cultivar's genome 1 to 7% were unplaced. +- Fewer rows are labelled, with 207 `paralog` and 136 `ambiguous` in the + larger family against 357 and 297 on the other genome, and no row + `unplaced` against 140. A genome of another cultivar makes some alleles + look like copies. +- The filter keeps or drops the same rows as bwa mem for 99.7 to 99.9% and as + blastn for 99.4 to 99.7%, as on the other genome. +- Of the `paralog` rows that linkage can decide, 91% (29 of 32) and 94% (17 + of 18) map elsewhere. Of the kept rows 96% map to their marker, and the + others are mostly at some 60 markers where every kept row maps elsewhere + (102 of 131 rows at 63 markers, and 86 of 105 at 54), where the marker is + somewhere else in the family than in this assembly. Apart from those, 29 + and 19 kept rows, under 1% of those judged, map away from their marker. +- Genotypes do not gain. Against the published dosages the larger family + stays at 89.80% and the smaller goes from 90.11% to 89.93%. In alfalfa the + filter removes 2.6 to 3.0% of the reads, its labels hold up against two + aligners and against inheritance, and it has no measurable effect on the + target genotypes. Alfalfa is a panel that does not need the filter. + +A haplotype-resolved reference. The assembly the alfalfa panel was designed +on has four copies of every chromosome, and the panel used one set of them. +Another tetraploid cultivar whose haplotypes are in the INSDC archives +(RegenSY, with haplotype 3, GCA_054855475, and haplotype 4, +GCA_054883285) shows what the filter does when it gets more than one set. +The genome was chromosomes 1 to 3 of one haplotype, then of both, and the +table was the 3,378 rows of the larger family at the 1,193 markers designed +on those chromosomes. + +| Genome, run | Markers placed at one site | Ref tag with several equal-best sites | Rows `on_target` | `ambiguous` | `paralog` | RefMatch and AltMatch rows dropped | +|---|---|---|---|---|---|---| +| One haplotype, `ignore` | 1,077 | 27 | 2,921 | 115 | 71 | 114 of 992 | +| One haplotype, positions | 1,103 | 16 | 2,954 | 103 | 94 | 115 of 992 | +| Two haplotypes, `ignore` | 648 | 518 | 1,372 | 1,565 | 361 | 655 of 992 | +| Two haplotypes, positions | 1,148 | 29 | 1,171 | 1,490 | 670 | 637 of 992 | + +With the second haplotype in the genome the filter drops two thirds of the +RefMatch and AltMatch rows where it dropped one in nine, and keeps 1,271 to +1,523 Ref and Alt rows only because it always keeps canonical rows. A row +that is the other haplotype's allele matches that haplotype better and is +labelled `paralog`, and one that both haplotypes carry is `ambiguous`. +CloneID positions place the markers (1,148 at one site against 648) and do +not help the rows. + +The design assembly itself says the same. With sets 1 and 2 of XinJiangDaYe +as the genome and the larger family's whole MADC, 1,199 of the 2,989 Ref +tags have an identical copy on set 2 (all settled by the CloneID position), +3,757 rows are `ambiguous` and 1,229 `paralog` where 136 and 207 were, and +the filter drops 1,969 of the 2,753 RefMatch and AltMatch rows where it +dropped 296. It keeps 3,017 Ref and Alt rows only as canonical rows. +Indexing two sets (1.6 Gb) took 8.3 GB of memory, and three sets and all +four (2.7 Gb) did not fit in the 13 GB available here. A polyploid or +haplotype-resolved assembly has to be cut down to one set of chromosomes +before it goes to the filter, and for this panel that is set 1. + +This record does not cover FASTQs of this panel, and so a MADC from +`generate` (a search found no public alfalfa DArTag reads), or planted +copies in these genomes. + +Tiled markers with the paralog filter and `call`. `generate` credits a read +that fits several tiles of one amplicon to each of them (`--tile-groups`). +That work and the two new commands were written on separate branches and +first met at the merge. A synthetic panel checks them together +(`tests/test_tiled_markers.py`). It has two tiles with variants at tag +positions 31 and 56 of one 81-base window, a third marker elsewhere, ten +diploid samples with every combination of the two variants, and the tags +written both ways a tile's tag can carry the other tile's variant, as an +ambiguity code (the DArTag way) and as the reference base. + +- `generate` gives both tiles all 60 reads of every sample. With ambiguity + codes in the tags no read equals a Ref or Alt tag, so those two rows are + empty and every read is on a RefMatch or AltMatch row. A genotype taken + from the Ref and Alt rows alone would have no reads. +- `filter-paralogs` places both tiles at the one site and labels every row + `on_target`, so the filtered table is the same as its input. +- `call` gives each tile the true dosage of its own variant in every sample. + With the reference base in the tag, each tile's `all` VCF also has the other + tile's variant as an off-target record, with that tile's target genotypes. + With an ambiguity code there the position gets no off-target record, and + the VCF reports the variant once. + +## Genotype calling (`call`, 0.14.0) + +`madcats call` (in `madcats_call.py`) calls genotypes from a MADC and writes +VCFs. The model takes from polyRAD 2.0.1 a beta-binomial likelihood of each +allele's read count against the rest of the record, a uniform or +Hardy-Weinberg prior, and (for `hwe`) iteration of the allele frequencies. +It adapts only the parts needed for this and leaves out polyRAD's +population-structure, linkage and mapping-population priors, its ploidy +tests and its import and export code. + +Three parts of updog 2.1.7's model followed later. They are the `norm` +genotype prior, the allelic bias, and a prior with a free weight for every +dosage (updog's `flex`), which `call` uses only while estimating the read +model. Four parts are MADCATS's own and have no published counterpart. They +are the maximum-likelihood estimate of the overdispersion, the table-wide +background-read rate estimated with it, the search that fits the bias, and +the `structure` prior. `madcats_call.py` names the polyRAD or updog function +each of its functions follows and says where it differs ("Audit against both +packages' sources" below). The paragraphs below follow the order of the +work, so the earlier tables come from before the later parts existed. The +figures for the finished model are under "Finished model on the alfalfa +populations", "Tables from updog's and polyRAD's simulators" and "Time and +memory". + +**Differences from polyRAD.** + +- MADCATS keeps likelihoods as logarithms. polyRAD exponentiates them and + sets a sample's likelihoods to 1 when they all underflow to 0. +- Each record iterates until its own allele frequencies settle. polyRAD stops + all loci together on their mean change. +- There is an iteration limit (`--max-iterations`). +- Ties between dosages, and corrections with no possible step, give a missing + genotype. + +**Agreement with polyRAD.** `scripts/make_polyrad_fixture.R` runs polyRAD on +three simulated loci (2, 3 and 4 alleles, and 16 samples of ploidy 2, 3, 4 +and 6) and stores the values in `tests/fixtures/polyrad_parity.json`: + +| Compared | Agreement | +|---|---| +| Likelihoods, priors and posteriors from the same starting frequencies | 9 or more decimal places | +| Allele frequencies and posteriors after `IterateHWE(tol = 1e-10)` | 7 decimal places | +| Genotypes after the correction to the ploidy | identical (144 sample-by-allele values) | + +**Correction to the ploidy.** With more than two alleles the model finds the +most probable dosage of each allele separately, and the dosages need not add +up to the ploidy. polyRAD adds or removes one copy at a time. An exact search +for the dosages with the highest product of posteriors +(`best_dosages_summing_to_ploidy`) gave the same genotype in all 4,104 +corrected genotypes of a simulation (hexaploid, five alleles, 40 markers, 200 +samples, 20–200 reads), where 51% of genotypes needed correcting. `call` uses +polyRAD's method and counts in the QC JSON how often the exact search differs. + +**Overdispersion.** polyRAD's default of 9 allows more scatter in read +fractions than the gap between neighbouring polyploid dosages. On a simulated +mixed-ploidy table without extra scatter (500 markers, 300 samples, target +records, other settings at their defaults): + +| `--overdispersion` | 2x called / correct | 4x called / correct | 6x called / correct | +|---|---|---|---| +| 9 | 93.1% / 99.83% | 11.4% / 83.71% | 2.5% / 65.46% | +| 50 | 97.6% / 99.98% | 76.7% / 99.63% | 11.1% / 99.25% | +| 200 | 97.7% / 99.98% | 86.2% / 99.39% | 51.5% / 98.70% | +| `auto` (chose 300) | 97.7% / 99.98% | 86.9% / 99.33% | 55.2% / 98.34% | + +`--overdispersion auto`, the default since the real-data comparison below, is +a maximum-likelihood estimate over a grid +(`estimate_overdispersion`). A port of polyRAD's `TestOverdispersion` came +first and reproduced polyRAD's scores to four decimal places on three +simulated tables, but both chose 10 for a tetraploid table simulated with 50 +and 4 for a hexaploid table simulated with 20 (and 25 for a diploid table +simulated with 20). It tests the read counts of samples that a rough estimate +assigns one allele copy, and in polyploids that estimate is wrong often +enough to make the counts look more scattered than they are. MADCATS dropped +the port. The table below gives the maximum-likelihood estimate on 200 +simulated markers and 150 samples per case (20–150 reads unless stated, no +stray reads), with the simulated value in the first column. It shows the +estimate as it now stands, with free weights for the dosages and the +background-read rate estimated alongside (both described further down): + +| Simulated | 2x | 4x | 6x | mixed 2x/4x/6x | 4x, 8–40 reads | 4x F1 | 6x F1 | 2x inbred | +|---|---|---|---|---|---|---|---|---| +| 5 | 5 | 6 | 8 | 6 | 6 | 5 | 7 | 2 | +| 10 | 10 | 10 | 10 | 10 | 10 | 10 | 10 | 2 | +| 20 | 20 | 20 | 20 | 20 | 20 | 20 | 20 | 2 | +| 50 | 50 | 50 | 50 | 50 | 50 | 50 | 50 | 2 | +| none (binomial) | 500 | 500 | 500 | 500 | 500 | 500 | 500 | 2 | + +From 10 upward the estimate recovers every value. The first version of the +estimate, which assumed Hardy-Weinberg proportions, read 12–30 for the F1 +columns at 10 and 20, and 75 for the hexaploid F1 at 50. At a simulated 5, +scatter wider than anything seen in the real tables, polyploid estimates are +6 to 8, because the estimate takes part of the scatter for background reads. +The background-read rate came out at the lowest value tried, 0.0002, in 36 +of the 40 cases, and at 0.001 to 0.01 in the other four. Those four are +random-mating tetraploids and hexaploids at 5, and the low-depth tetraploids +at 5 and 10. + +The inbred diploids here have no heterozygotes and no stray reads, so there is +no scatter to measure and the estimate ends at the lowest value. `call` then +warns and uses 9. On the two real alfalfa F1 populations below the first +version chose 40 and 50, and the finished one 30 and 40 with a background-read +rate of 0.003. + +**Real alfalfa DArTag data (tetraploid, 3K panel).** The `call` model and +polyRAD 2.0.1 (`IterateHWE`, overdispersion 9, then `GetProbableGenotypes` +with `multiallelic = "correct"`) called three processed MADCs as tetraploids +on the same counts. They are the two F1 populations of the G3 article +jkag131 (I195 × J432 with 186 samples, and E115 × F628 with 374 samples) and +project DAl22-7249 (615 samples from 20 crosses and their parents). + +| MADC | Target genotype values identical | Haplotype genotype values identical | +|---|---|---| +| I195 × J432 (tolerance 1e-7) | 1,103,352 of 1,103,352 | 1,385,886 of 1,385,886 | +| E115 × F628 (tolerance 1e-6) | 2,222,352 of 2,223,056 | 3,251,974 of 3,252,678 | +| DAl22-7249 (tolerance 1e-6) | 3,485,814 of 3,485,820 | 5,306,214 of 5,306,220 | + +Every difference is at one marker (E115 × F628) or two (DAl22-7249) where +polyRAD had stopped while that marker's allele frequency was still moving. Its +stopping rule is the mean change over all loci, and those frequencies were +0.01–0.03 from where they settle. Elsewhere the two tools' allele frequencies +differ by 1e-7 at the median. + +The article publishes the dosages used for its linkage maps. Of its markers, +974 and 986 are panel targets, and the rest are off-target SNPs. The table +compares target calls with the published dosages over 166 and 338 progeny +(the published tables count Ref copies, and count the other allele at 38 and +43 markers, which were re-oriented). These rows use a contamination rate of +0.001 and no background-read rate, which is how `call` ran at that point: + +| Settings | I195 × J432, called / same dosage | E115 × F628, called / same dosage | +|---|---|---| +| Overdispersion 9, no filters (what polyRAD calls) | 100% / 88.81% | 100% / 86.33% | +| `--overdispersion auto`, no filters | 100% / 90.80% | 100% / 89.93% | +| Overdispersion 9, default filters | 40.9% / 93.17% | 41.3% / 91.78% | +| `--overdispersion auto`, default filters | 89.0% / 93.15% | 91.8% / 92.19% | +| `--overdispersion auto --prior uniform`, default filters | 86.6% / 93.01% | 90.2% / 91.98% | + +In every row 98.5–99.4% of calls are within one copy of the published dosage. +The published dosages come from another caller, so a difference is not +necessarily an error of either. Calling from the Ref and Alt rows alone, +without the Match rows, lowered agreement to 80%, so the published calls also +counted Match rows. + +To see what a port would gain on these families, polyRAD's other priors ran +on the same target counts at the published markers, with the overdispersion +`call` estimated (40 and 50): + +| polyRAD prior | I195 × J432, called / same dosage | E115 × F628, called / same dosage | +|---|---|---| +| `IterateHWE` | 100% / 90.80% | 100% / 89.93% | +| `IteratePopStruct` | 100% / 90.74% | 100% / 89.91% | +| `PipelineMapping2Parents` (F1, no linkage) | 93.9% / 91.08% | 95.0% / 90.42% | + +A single F1 family has no population structure, so the population-structure +prior changes nothing here. At these depths (medians of about 180 and 430 +reads per sample and marker) the prior matters little. + +**Against updog's norm model.** updog 2.1.7 (`flexdog`, `model = "norm"`) ran +on the same target counts at the published markers. Unfiltered agreement +with the published dosages was 92.98% and 92.68%, against 90.80% and 89.93% +for `call` with the HWE prior, and updog took 235 s and 453 s on four cores. +The two callers give the same dosage for 96.99% and 96.54% of genotypes, and +where they differ the published dosage sides with updog about nine times in +ten, so the published dosages probably come from updog or a similar caller. +Split by the allelic bias updog fitted for each marker: + +| Fitted bias of the marker | Share of genotypes | updog = `call` | updog = published | `call` = published | +|---|---|---|---|---| +| 0.8–1.25 | 73%, 69% | 99.19%, 99.28% | 93.59%, 92.91% | 93.09%, 92.50% | +| outside that, within 0.67–1.5 | 17%, 21% | 94.96%, 95.36% | 93.63%, 92.85% | 89.31%, 89.05% | +| outside 0.67–1.5 | 10%, 11% | 83.77%, 81.16% | 87.09%, 90.83% | 75.54%, 75.11% | + +updog's per-marker fits had a median bias of 0.99 and 0.98 (10th–90th +percentile 0.72–1.24 and 0.72–1.30), a median sequencing-error rate of 0.09% +and 0.24%, and a median overdispersion of 395 and 183 on polyRAD's scale. The +single values `call` estimated for these tables (40 and 50) are lower because +they absorb the bias, which `call` did not model at that point. + +On simulated tetraploid tables with known genotypes (100 markers, 150 samples, +20–60 reads, overdispersion 40, 0.5% read errors, one table per case), +accuracy of the most probable dosage: + +| Population | updog norm | `call` defaults | `call`, contamination 0.01 and overdispersion 40 | +|---|---|---|---| +| Random mating | 87.5% | 84.8% | 89.2% | +| F1 | 88.7% | 88.2% | 89.4% | +| Two subpopulations | 88.8% | 80.3% | 89.5% | +| Inbred lines | 83.4% | 86.6% | 96.8% | +| Random mating, biased markers | 86.0% | 79.7% | 82.3% | +| F1, biased markers | 86.9% | 82.4% | 84.9% | +| Two subpopulations, biased markers | 89.0% | 77.9% | 85.3% | +| Inbred lines, biased markers | 84.9% | 86.2% | 96.6% | + +Biased markers had a bias of 2^N(0, 0.5). Three things separate the columns. +(1) Read errors above the contamination rate pull the overdispersion estimate +down. With a true value of 40 it was 30, 16, 12 and 8 at error rates of 0.1%, +0.3%, 0.5% and 1%, and 40–50 throughout once the contamination rate was 0.01. +(2) The estimate assumed Hardy-Weinberg proportions and failed on the +two-subpopulation table (7) and on inbred lines (fell back to 9). (3) With +the right overdispersion and stray-read rate, `call` matches or beats updog on +unbiased markers and trails it by 2–4 points on biased ones. A prototype of a +normal-shaped prior in the `call` engine (not in the repository) changed these +figures by at most 1.5 points. The background-read rate, the free-weights +estimate and the allelic bias, all described below, answer these three points +in turn, and the same eight tables come back, called with the finished model, +under "Tables from updog's and polyRAD's simulators". + +**The norm prior (`--prior norm`).** updog's `norm` genotype distribution +became a prior. It makes dosage probabilities proportional to a normal +density, with a mean and standard deviation fitted to the summed posteriors +in every iteration. The fit is exact, since the prior depends on the samples +only through those sums. It uses Newton's method on the two natural +parameters, where the function has a single peak, and searches the edges of +updog's parameter limits when the peak lies outside them, and it takes +0.2 ms on average. On 130 sets of dosage weights no point of a fine grid +fitted better. + +The fit shortens a Newton step that does not improve it by adding to the +diagonal of the curvature (Levenberg-Marquardt). The first version halved the +step instead, and stopped at its starting values when they put almost all of +the prior on one dosage, because the curvature there is next to nothing and +the step is too long by tens of orders of magnitude. The audit found it in 1 +of 40 single fits (weights 0.0003, 0.27 and 14.0 on the dosages of a diploid, +start at mean 1.98 and standard deviation 0.137), 0.39 below updog's optimum +in mean log prior. On 2,954 fits with many near-zero weights and arbitrary +starts the damped fit was never worse than the halved one and better in 337. +Calls changed little. The potato comparison below is the same to the last digit +(92.10% with `norm`, 96.57% with `norm` and the bias), and of twelve stored +runs the three that use `norm` changed in 8 of 257,400 target and haplotype +genotypes and in the off-target records of three markers, each with a higher +penalised likelihood than before. + +`scripts/make_updog_fixture.R` stores updog 2.1.7 values in +`tests/fixtures/updog_norm.json`. For 14 sets of weights (ploidies 2 to 8 +and the shapes one peak, rare and common alleles, two peaks, flat and a +single class) the fitted priors agree to 2e-5. For four simulated markers +(random-mating tetraploid, one hexaploid family, inbred diploid, rare-allele +tetraploid), fitted by updog with its bias, sequencing error and +overdispersion held fixed, the priors agree to 1e-5 and every genotype is +the same. Posteriors agree to 1e-6 except for samples with a few reads of an +allele, where updog's minimum sequencing-error rate of 1e-6 leaves dosage 0 +a probability of up to 5e-4. + +Each ploidy among the samples gets its own fit, and a ploidy with fewer +than 10 samples uses Hardy-Weinberg proportions. On the two alfalfa F1 +populations the norm prior agreed with the published dosages as often as HWE +did (90.38% against 90.80%, and 89.78% against 89.93%, unfiltered) and took +about 35% longer (29 s against 22 s, and 37 s against 27 s, for all 3,000 +markers on four workers). + +The processed layout of these files has a `readCountSum` column before the +samples, which `call` skips. + +**Background-read rate (`--background-rate`).** In polyRAD a read of an allele +the sample does not carry can only come from contamination, at the allele's +frequency among the other samples. That cannot account for stray reads of an +allele that is rare in the table, and read errors beyond the contamination +rate counted as scatter and pulled the overdispersion estimate down (point +1 above). `call` therefore has updog's sequencing-error term, under the name +background-read rate. A read shows the other allele with probability *e*, so +an expected fraction *f* becomes *f*(1 − *e*) + (1 − *f*)*e*. It is one value +for the table, where updog fits one per marker, and the default contamination +rate became 0. With polyRAD's settings a diploid with 10 Ref reads and 1 Alt +read comes out heterozygous (posterior 0.95 in a table of 21 samples), while +with a background rate of 0.002 and an overdispersion of 40 the same sample +is homozygous with a posterior of 0.69. + +`estimate_read_model` estimates the rate together with the overdispersion. +For a given rate the overdispersion is the maximum-likelihood value on its +grid, and the rate kept is the one at which the counts are most probable, +found by starting at 0.002 and moving to a neighbouring rate (eight values +from 0.0002 to 0.02) for as long as that raises the likelihood. The estimate +can separate the two because they show in different samples. Stray reads +show in samples that do not carry an allele, and scatter in samples that +carry both. + +The estimate ran on eighty simulated tables of 50 markers and 120 samples. +They cover random mating, one F1 family, two subpopulations and inbred +lines, each with a background rate of 0.001, 0.003 and 0.01 at an +overdispersion of 40, and with 0.003 at 150. Estimates of the +overdispersion, and the number of tables whose background rate was the +simulated one: + +| Samples | Reads per sample | Simulated 40 (12 tables) | Simulated 150 (4 tables) | Background rate | +|---|---|---|---|---| +| 4x | 20–120 | 40 in all 12 | 150 in all 4 | 16 of 16 | +| 2x | 20–120 | 40 in all 12 | 150 in 3, 200 in 1 | 16 of 16 | +| 6x | 30–150 | 40 in 8, all within 14–50 | 150 in all 4 | 15 of 16 | +| 4x | 6–25 | 40 in 6, all within 30–75 | 150 in 2, all within 100–500 | 15 of 16 | +| 2x | 6–25 | 40 in 10, all within 30–50 | 100–300 | 16 of 16 | + +The background rate is the simulated one in 78 of the 80 tables and one grid +step away in the other two. The overdispersion is the simulated one in 31 of +the 32 diploid and tetraploid tables with 20 or more reads. The two +hexaploid estimates furthest out (14 and 25) are at a background rate of 1%. +With 6 to 25 reads the estimate cannot measure an overdispersion of 150. At +that depth the sampling of reads hides scatter that small, and for the same +reason it changes the calls little. Inbred lines are estimable here because +their homozygotes carry stray reads. + +On the alfalfa populations the estimate is a background rate of 0.003 with an +overdispersion of 30 (I195 × J432) and 40 (E115 × F628). With the bias fitted +it is 0.0005 with 150, and 0.001 with 150, nearer updog's per-marker medians +for the same tables (0.09% and 0.24%, and 395 and 183). + +**Free weights for the estimate.** The estimate sums the genotypes out, so it +needs genotype proportions for each marker. With Hardy-Weinberg proportions it +failed on two subpopulations and on inbred lines (point 2 above). With the +`norm` shape, which cannot describe an excess of both homozygotes, it still +failed on inbred and partly inbred tables. The estimate now gives every dosage +its own probability at each marker (`PRIOR_FREE`, the summed posteriors +normalised), under any `--prior` the run uses. This is the form of updog's +`flex` model. updog's default is `norm`, and it suggests `norm` in place of +`flex` when there are fewer than 10 × (ploidy + 1) samples, since `flex` has +one free weight per dosage at every marker. Here only the estimate, which +pools the markers, uses it, and the calls never do. +Estimates without bias fitting, +as overdispersion and background rate, on eight of the simulated tables +described under "Tables from updog's and polyRAD's simulators": + +| Table | Simulated | `norm` shape | Free weights | +|---|---|---|---| +| polyRAD 4x, inbreeding 0.5 | 20 with 0.001 contamination and 0.001 errors | 2 (fails), 0.0002 | 20, 0.001 | +| polyRAD 4x outbred | same | 20, 0.001 | 20, 0.001 | +| polyRAD 6x outbred | same | 20, 0.001 | 20, 0.001 | +| updog 4x random mating, biased | 49–499, 0.005 | 25, 0.01 | 25, 0.01 | +| updog 4x F1, biased | 49–499, 0.005 | 30, 0.01 | 30, 0.01 | +| Two subpopulations | 40, 0.005 | 40, 0.005 | 40, 0.005 | +| Inbred lines | 40, 0.005 | 2 (fails), 0.0002 | 40, 0.005 | +| F1, biased | 40, 0.005 | 30, 0.005 | 30, 0.005 | + +Free weights give the same values as the `norm` shape where that worked, and +the simulated values on both tables where it failed. On biased tables both +read low unless the run fits the bias, and then both gave 100 and 0.005 on +the two updog tables. The inbred table is estimable because its homozygotes +carry stray reads. A table with neither heterozygotes nor stray reads still +ends at the lowest overdispersion tried, and `call` then warns and uses 9. + +**Allelic bias (`--allele-bias fit`).** The comparison with updog put the gap +at markers where a copy of one allele yields more reads than a copy of the +other (point 3 above). updog's bias now applies to records with two alleles. +With bias *h*, an expected reference fraction *f* becomes +*f* / (*h*(1 − *f*) + *f*). As in updog, the log of the bias has a normal +prior (mean 0, standard deviation 0.7), the search starts from five values +(exp(−1) to exp(1)), and it keeps the fit with the highest log-likelihood +after that prior. Two things differ from updog: + +- MADCATS applies the bias to the sample's own allele copies and adds + background reads afterwards. updog applies its sequencing error first. In + updog's order, samples that do not carry an allele inform its bias through + their stray reads, and with one background rate for the whole table that + would turn markers with more or fewer stray reads than average into biased + ones. +- The search is over the bias alone. Each trial value is a complete fit of the + record at that bias, scored by its log-likelihood with the genotypes summed + out. The search fits the five starting values from scratch and refines + every one that scores at least as high as its neighbours by a + golden-section search within half a log unit of it, to 0.01 in the log of + the bias. updog optimises bias, sequencing error and overdispersion + together from each start. The range searched is exp(−2) to exp(2), 0.14 to + 7.4. + +`tests/fixtures/updog_norm.json` holds ten updog fits with the bias estimated +and the other parameters fixed. They cover five simulated markers (a diploid +simulated with a bias of 0.5, a hexaploid family with 1.5, and tetraploids +with 0.6, 1.9 and 1), each under updog's `norm` and `hw` models. The fitted +biases agree to 0.07%, the priors to 1e-4 and the posteriors to 1e-3, and +all 1,140 genotypes are the same. For the hexaploid family both tools fit a +bias of 2.1 (`norm`) and 2.7 (`hw`) where the simulation used 1.5, because +with few dosage classes a bias and a shift of every dosage explain the +counts about equally well. + +The first version of the search refined only the best starting value. On +updog-simulated F1 markers with one or two dosage classes it then kept the +wrong one (90.7% of genotypes correct against updog's 94.0%). Refining every +start that scores at least as high as its neighbours gave 93.3%. + +Haplotype records and sites with three or more bases get calls without a +bias, because updog defines its bias for two alleles, and a bias per allele +of a multi-allele record has not been worked out or tested. + +**Finished model on the alfalfa populations.** Target calls against the +published dosages, without filters, with the estimates above: + +| Settings | I195 × J432, same dosage | E115 × F628, same dosage | +|---|---|---| +| polyRAD's settings (overdispersion 9, contamination 0.001) | 88.81% | 86.33% | +| `--prior hwe` (the defaults) | 90.87% | 90.04% | +| `--prior norm` | 90.11% | 89.80% | +| `--prior hwe --allele-bias fit` | 92.17% | 91.79% | +| `--prior norm --allele-bias fit` | 92.26% | 91.94% | +| updog 2.1.7, `norm` | 92.98% | 92.68% | + +In the four rows of the finished model 98.3–98.7% of calls are within one copy +of the published dosage. The published dosages probably come from updog +(above), so the last row is partly a comparison of updog with itself. With +`--prior norm`, of the 2,966 and 2,972 target records fitted, 38% and 40% +have a bias outside 0.8–1.25 and 26% and 27% outside 0.67–1.5 (median 1.0), +and 3% end at a limit of the search. + +**Potato DArTag against SNP-array genotypes.** The alfalfa dosages above +probably come from updog, so they cannot say whether updog or `call` is nearer +the truth. The potato data of Endelman et al. (2024, The Plant Genome, +doi:10.1002/tpg2.20484) have genotypes from a second technology. The +polyBreedR repository (`inst/vignette_data`) carries the part of that dataset +from a five-parent half-diallel. It has 85 tetraploid clones in ten F1 +families, with DArTag read counts (`diallel_DArTag.vcf.gz`, the `AD` field) +and Infinium array genotypes called with fitPoly (`diallel_array.vcf.gz`). +The two files share 1,544 markers with the same REF and ALT, and the median +depth is 304 reads per sample and marker. The VCF has no tag sequences, so +`call` got the reference and alternate counts as a two-row-per-marker MADC +with stand-in sequences. Most probable dosage against the array dosage, +without filters, over the 131,224 genotypes with reads: + +| Caller | Same dosage | RMSE | Markers with under 10% wrong | Bias 0.8–1.25 (1,135 markers) | Bias outside (409) | +|---|---|---|---|---|---| +| `call` with polyRAD's settings | 87.92% | 0.353 | 905 | 91.9% | 76.9% | +| polyRAD `IterateHWE`, overdispersion 9 | 87.92% | 0.353 | 905 | 91.9% | 76.9% | +| polyRAD `IteratePopStruct`, overdispersion 9 | 90.53% | 0.314 | 1,095 | 94.5% | 79.5% | +| `call --prior hwe` (the defaults) | 92.26% | 0.290 | 1,199 | 96.7% | 79.9% | +| `call --prior norm` | 92.10% | 0.294 | 1,191 | 96.7% | 79.4% | +| `call --prior hwe --allele-bias fit` | 96.60% | 0.192 | 1,396 | 97.1% | 95.2% | +| `call --prior norm --allele-bias fit` | 96.57% | 0.192 | 1,394 | 97.1% | 95.2% | +| updog `norm` on the same 85 samples | 96.69% | 0.188 | 1,403 | 97.2% | 95.4% | +| updog `norm` as stored in the VCF (fitted on 703 samples) | 96.86% | 0.181 | 1,399 | 97.3% | 95.7% | + +The last two columns split the markers by the bias updog stored in the VCF. +Every row has 99.8–99.9% of calls within one copy. + +- `call` with polyRAD's settings and polyRAD give the same share to two + decimals, as on the alfalfa data. +- The estimated read model (overdispersion 75, background rate 0.003) adds + 4.3 points over polyRAD's defaults. +- Fitting the bias adds another 4.3 points, 15 at the 409 markers updog + found biased. The estimate with the bias fitted is 300 and 0.001, and 31% of + the fitted biases are outside 0.8–1.25. +- With the bias fitted, `call` is 0.1 points below updog on the same samples + and took 52–56 s on four workers against updog's 150 s (CPU 197–214 s + against 394 s, memory 150 MB against 693 MB). Without the bias it took 6–7 s. +- polyRAD's population-structure prior adds 2.6 points at an overdispersion + of 9, where the prior carries more weight, and still ends below `call` with + the HWE prior and estimated values. With an overdispersion of 75 polyRAD's + HWE prior gives 92.41%, leaving 210 genotypes uncalled, and its structure + prior stops with an error (a regression with no usable cases). + +**The whole potato deposit, and the paper's own error table.** The deposit +of Endelman et al. (2024) is on Dryad (doi:10.5061/dryad.8pk0p2nw4, CC0) with +a copy on Zenodo (record 11286799). File S1 has the DArTag read counts of 703 +tetraploid samples at the 2,503 markers of the first panel version, File S5 +the array genotypes of 298 of them, and File S12 the paper's error of the +DArTag genotypes (updog's `norm`, stored in File S1) against the array for +each of the 1,862 markers that are in both files with the same alleles. Each +caller called all 703 samples together, without filters, and the scores use +the 298 with array genotypes. The median depth is 201 reads per sample and +marker. + +The first three columns are File S12's measures, per marker over the genotypes +with an array dosage, as mean and median over the 1,862 markers. They are the +root mean squared error, the share with another dosage, and the share in +another class of absent, mixed and fixed. The last column is the share of all +550,052 genotypes with reads that have the array's dosage. + +| Caller | RMSE, mean and median | Another dosage, mean and median | Another class, mean and median | Same dosage, all genotypes | +|---|---|---|---|---| +| File S12 as published | 0.363, 0.224 | 13.4%, 5.0% | 6.5%, 0.7% | | +| updog `norm` as stored in File S1 | 0.363, 0.224 | 13.4%, 5.0% | 6.5%, 0.7% | 87.15% | +| `call --prior hwe --allele-bias fit` | 0.347, 0.224 | 12.9%, 5.0% | 5.8%, 0.7% | 87.46% | +| `call --prior norm --allele-bias fit` | 0.349, 0.232 | 12.9%, 5.0% | 5.9%, 0.7% | 87.45% | +| `call --prior hwe` (the defaults) | 0.415, 0.278 | 17.5%, 7.7% | 7.1%, 0.7% | 82.81% | +| `call --prior norm` | 0.418, 0.278 | 17.6%, 7.7% | 7.2%, 0.7% | 82.74% | +| `call --prior structure` | 0.412, 0.278 | 17.3%, 7.4% | 7.1%, 0.7% | 83.02% | +| polyRAD `IterateHWE`, overdispersion 75 | 0.414, 0.284 | 17.4%, 7.7% | 6.7%, 1.0% | 83.02% | +| polyRAD `IteratePopStruct`, overdispersion 75 | 0.413, 0.278 | 17.2%, 7.7% | 6.7%, 1.0% | 83.23% | +| `call` with polyRAD's settings | 0.498, 0.393 | 22.6%, 15.4% | 7.9%, 4.0% | 77.73% | +| polyRAD `IterateHWE`, overdispersion 9 | 0.502, 0.393 | 22.6%, 15.4% | 7.9%, 4.0% | 77.72% | +| polyRAD `IteratePopStruct`, overdispersion 9 | 0.474, 0.362 | 20.4%, 12.8% | 7.4%, 3.4% | 80.03% | + +- The stored genotypes scored here give File S12's figures to the last + decimal, so these columns measure what the paper measured. +- With the bias fitted `call` has the paper's median error and a lower mean, + 12.9% of genotypes with another dosage against 13.4%. The estimate was an + overdispersion of 150 and a background rate of 0.003. +- Without the bias (estimate 75 and 0.005) `call` is 4.6 points behind, level + with polyRAD given the same overdispersion and 5 points ahead of polyRAD's + defaults. Fitting the bias gains as much here as on the 85 clones. +- The structure priors add 0.2 points in both tools. polyRAD's finished at an + overdispersion of 75 with 703 samples, but with 85 it stopped with an error. +- Agreement is lower than on the 85 clones for every caller because the + markers had no selection. At 331 of the 1,862 markers fewer than 80% of the + stored genotypes have the array's dosage, and at 153 fewer than half. No + sample is under 60%, and the median marker is at 95%. The disagreement is + between the two assays at particular markers, which is why the medians say + more than the means. +- On four workers that other jobs were also using, `call` took 19 s with the + HWE prior, 38 s with the structure prior, and 233 s (HWE) and 432 s (norm) + with the bias fitted, while polyRAD's structure model took 90 s. + +**A vendor MADC with diploid and tetraploid samples (potato, second panel +version).** File S4 of the same deposit is the MADC as DArT delivered it for +the second panel version, with 375 samples, 52 of them diploid, and 3,915 +markers. File S3 has the paper's genotypes of those samples, and File S13 +its error against the array per marker, as the share of genotypes in another +class of absent, mixed and fixed (the measure that suits both ploidies). +`call` read File S4 as it is, with `--ploidy 4` and a `--ploidy-file` naming +the diploids (each sample's ploidy came from its genotypes in File S3). It +wrote 3,906 target records, 5,879 records in the `all` VCF and 3,915 +haplotype records. Nine markers have no target record because their Ref and +Alt tags differ at more than one position. The median depth is 273 reads. +78 samples are on the array (38 diploid, 40 tetraploid), and 2,603 markers +are in both files with the same alleles and have a target record, which +gives 202,143 genotypes with reads. + +| Caller | Same dosage, all | Diploids (98,339) | Tetraploids (103,804) | Another class per marker, mean and median | +|---|---|---|---|---| +| File S13 as published | | | | 5.3%, 3% | +| Genotypes stored in File S3 | 91.83% | 92.61% | 91.09% | 5.3%, 2.6% | +| `call --prior hwe --allele-bias fit` | 91.91% | 92.61% | 91.25% | 5.1%, 2.6% | +| `call --prior norm --allele-bias fit` | 91.89% | 92.58% | 91.23% | 5.1%, 2.6% | +| `call --prior hwe` (the defaults, no filters) | 89.14% | 92.02% | 86.41% | 5.7%, 2.6% | +| `call --prior norm` | 88.80% | 92.04% | 85.74% | 5.7%, 2.6% | +| `call --prior structure` | 89.31% | 92.03% | 86.74% | 5.7%, 2.6% | +| polyRAD `IterateHWE`, overdispersion 30 | 89.38% | 92.28% | 86.62% | 5.4%, 2.6% | +| polyRAD `IteratePopStruct`, overdispersion 30 | 89.60% | 92.31% | 87.03% | 5.4%, 2.6% | +| `call` with polyRAD's settings | 86.25% | 92.62% | 80.21% | 6.6%, 3.8% | +| polyRAD `IterateHWE`, overdispersion 9 | 86.23% | 92.60% | 80.20% | 6.6%, 3.8% | +| polyRAD `IteratePopStruct`, overdispersion 9 | 87.80% | 92.64% | 83.22% | 6.2%, 3.8% | + +File S13 rounds to two decimals, and the stored genotypes scored here give a +mean of 5.30% against its 5.28%. + +- One run calls a vendor MADC of mixed ploidy, and with the bias fitted the + result is level with the paper's genotypes for the diploids and 0.2 points + above them for the tetraploids. The estimate was an + overdispersion of 100 and a background rate of 0.003, and 1,507 of the + 3,904 fitted biases are outside 0.8 to 1.25. +- Without the bias (estimate 30 and 0.005) the tetraploids are 4.8 points + behind, level with polyRAD at the same overdispersion and 6.2 points ahead + of polyRAD's defaults. +- For the diploids the defaults are 0.6 points behind polyRAD's settings. The + two differ in 1,512 diploid genotypes at 528 markers. In 1,034 the array has + a heterozygote that the defaults call as fixed and polyRAD's settings as + mixed, and in 445 the array has a fixed genotype that the defaults get + right. These are markers where one allele gives few reads. For example, a + diploid heterozygote with 210 reference and 47 alternate reads, at a marker + with a fitted bias of 0.27, gets a call without the alternate allele under + an overdispersion of 30, because with a background rate the homozygote's + distribution has a long tail. Under an overdispersion of 9, which lets a + heterozygote stray further from half, it comes out mixed. Fitting the bias + settles it and gives 92.61%. +- The default filters keep 87.5% of the genotypes (98.4% of the diploid, + 77.1% of the tetraploid), and 92.2% of those have the array's dosage. +- On shared workers the run that wrote all three VCFs took 88 s, the norm + prior 34 s, the structure prior 63 s, and the runs with the bias 159 to + 169 s, while polyRAD's structure model took 113 to 200 s. + +**Which rows a target genotype counts.** `call` counts every row of a marker +by the base it has at the target, so a RefMatch or AltMatch haplotype adds +its reads to the allele it carries. A workflow that takes the exact Ref and +Alt rows only, as the sweetpotato study below did, leaves those reads out. +The potato vendor MADC and the array decide between the two. +File S4 was cut down to its 7,830 Ref and Alt rows (of 13,187) and called +again. + +| Counts | `--prior hwe`, for all, diploids and tetraploids | With the bias fitted | +|---|---|---| +| Every row, by its base at the target | 89.14%, 92.02%, 86.41% | 91.91%, 92.61%, 91.25% | +| Ref and Alt rows only | 82.06%, 85.25%, 79.04% | 84.36%, 85.77%, 83.03% | + +With the Ref and Alt rows alone 7 points fewer genotypes have the array's +dosage, and 2.1% of the genotypes have no reads at all. The reads on the +other rows are reads of the target's alleles. + +**Which thresholds on the VCF remove the worst markers.** The two potato +panels have array dosages for every marker and no selection for good +behaviour, so they show which statistics of a `call` VCF pick out the markers +that disagree with the array. Both runs used `--prior hwe --allele-bias +fit` and no filters, so that every genotype has its GQ. A marker is +"bad" when fewer than 80% of its genotypes have the array's dosage. That +holds for 324 of 1,862 on the first panel and 229 of 2,602 on the second. + +For a genotype, on the first panel (550,052 genotypes) and the second +(202,143), the table gives the share with the array's dosage among those +kept, and the share kept. + +| Genotype kept when | First panel | Second panel | +|---|---|---| +| Any GQ | 87.5%, all | 91.9%, all | +| GQ 10 or more | 90.8% of 90.3% | 93.2% of 96.2% | +| GQ 20 or more | 92.4% of 80.4% | 93.9% of 91.1% | +| GQ 30 or more | 93.1% of 70.3% | 94.2% of 84.6% | +| GQ 40 or more | 93.4% of 59.4% | 94.3% of 76.4% | +| GQ 10 or more and depth 10 or more (the defaults) | 91.3% of 89.4% | 93.3% of 96% | + +For a marker, how well each statistic separates the bad markers from the +others (the probability that a bad marker ranks worse than a good one, where +0.5 is chance), with every sample of the VCF used and no truth: + +| Statistic | First panel | Second panel | +|---|---|---| +| Share of samples that pass the default filters (call rate) | 0.86 | 0.83 | +| Mean depth | 0.78 | 0.66 | +| Estimated share miscalled (1 minus the mean posterior of the called genotype) | 0.77 | 0.77 | +| Fewer heterozygous calls than random mating gives at the marker's allele frequency | 0.76 | 0.59 | +| Mean GQ | 0.73 | 0.69 | +| `BIAS`, as distance from 1 | 0.69 | 0.67 | +| Minor allele frequency | 0.56 | 0.46 | + +Filters, as markers removed, share of the removed that are bad, share of the +bad markers removed, and the agreement of the genotypes that pass the default +filters at the markers left (91.3% and 93.3% with no marker filter): + +| Marker removed when | First panel | Second panel | +|---|---|---| +| Call rate under 0.8 | 284 / 64% / 56% / 92.8% | 205 / 36% / 32% / 94.0% | +| Call rate under 0.9 | 555 / 45% / 77% / 94.1% | 530 / 28% / 65% / 95.0% | +| `BIAS` outside 0.5 to 2 | 162 / 62% / 31% / 92.8% | 179 / 32% / 25% / 93.9% | +| `BIAS` outside 0.33 to 3 | 94 / 73% / 21% / 92.5% | 69 / 42% / 13% / 93.6% | +| Estimated share miscalled over 0.15 | 90 / 88% / 24% / 91.5% | 33 / 70% / 10% / 93.4% | +| Heterozygous calls 0.2 under expectation | 25 / 92% / 7% / 91.8% | 20 / 95% / 8% / 93.6% | +| Mean depth under 30 | 120 / 79% / 29% / 91.7% | 63 / 51% / 14% / 93.4% | +| Call rate under 0.8, or `BIAS` outside 0.5 to 2 | 369 / 58% / 66% / 93.6% | 333 / 31% / 45% / 94.4% | + +- GQ sorts genotypes as far as 20, and beyond that the genotypes lost are + mostly right ones. A genotype with a GQ of 30 or more is wrong 6 to 7 times + in 100 here, where its posterior says once in 1,000. The errors left are at + markers where the model is confident and the array differs, which a + genotype threshold cannot reach. +- The call rate under the default filters is the best single marker + statistic on both panels. At an equal number of markers removed, the lowest + call rates take out more bad markers than the most extreme biases, the + highest miscall estimates or the lowest depths (of 200 removed on the first + panel, 146, 111, 133 and 125 are bad). +- `BIAS` helps less as a filter than its use in updog suggests. A marker + with a strong bias gets the right calls once the run fits the bias, so the + range 0.5 to 2 removes 62 good markers of 162 on the first panel and 121 of + 179 on the second. Added to a call-rate filter it gains 0.2 to 0.3 points + on the first panel over removing the same number by call rate alone, and + nothing on the second. +- Two statistics remove few markers and almost only bad ones. They are a + deficit of heterozygous calls of 0.2, which is what a null allele gives, + and a miscall estimate over 0.15. +- Without the bias fitted the GQs of a biased panel are lower and a call-rate + threshold removes far more. Under 0.8 it removes 454 markers on the first + panel and 1,125 on the second. The thresholds above belong to a run with + the bias. +- On the delta smelt panel, which has no such markers, the same rules remove + little. Of 983 records 16 or 17 have a call rate under 0.8, and 37 of 946 a + bias outside 0.5 to 2 (16 outside 0.33 to 3). +- No filter finds every bad marker, and the worst are not a calling problem. + Of the 132 and 84 markers with under half their genotypes right, the + called dosage follows the array's closely (correlation 0.9 or more) at 5 + and 2, and at the others the two assays measure something different. The + strictest filter in the table, a call rate under 0.9, still leaves 30% and + 38% of them. + +**Replicates of eight hexaploid sweetpotato clones.** The deposit of a study +of DNA from sweetpotato leaves and roots (Zenodo record 22288064, CC-BY) has +a vendor MADC of 80 samples, ten of each of eight clones, and the study's +genotypes from updog's `norm` model (`multidog`, ploidy 6, written by BIGr's +`updog2vcf`). There is no second assay, so the measure is how often two +samples of one clone get the same dosage. The MADC as delivered names each +sample twice, once per extraction method, and `call` refuses repeated sample +names, so the second of each pair got a suffix. + +The study's genotypes use the exact Ref and Alt rows, so `call` first ran +on those rows alone (6,240 of 19,319), which gives both tools the same +counts. 2,636 markers are in every output and 622,513 pairs of samples of one +clone both have reads: + +| Caller | Pairs with the same dosage | Different dosages among the eight clones, per marker | Same genotype as the study's | +|---|---|---|---| +| updog `norm`, as stored in the deposit | 87.31% | 2.16 | | +| `call --prior norm --allele-bias fit` | 86.68% | 2.18 | 90.86% | +| `call --prior norm` | 86.14% | 2.12 | 80.76% | +| `call --prior hwe --allele-bias fit` | 85.66% | 2.19 | 87.48% | +| `call --prior hwe` | 84.79% | 2.17 | 80.24% | +| `call` with polyRAD's settings | 80.02% | 2.00 | 74.57% | +| `call --prior hwe`, default filters | 99.02% of the 53.0% of pairs with both called | 1.77 | 93.70% | +| `call --prior norm --allele-bias fit`, default filters | 96.90% of the 65.9% | 2.01 | 93.69% | + +- On the same counts `call` with the norm prior and the bias fitted is 0.6 + points behind updog in agreement between replicates and gives the study's + genotype in 91% of cases. The eight clones are ten samples each, which is + not a population in Hardy-Weinberg proportions, and the norm prior suits it + better than the HWE prior by 1 to 1.4 points. +- The estimate was an overdispersion of 50 and a background rate of 0.002, + and 150 and 0.001 with the bias fitted. The default filters leave two + thirds or less of the pairs called, and 97 to 99% of those agree. +- With every row counted, as `call` does by default, the picture changes. + Then 72.8% (HWE) to 75.9% (norm) of pairs agree, the clones show 3.2 + different dosages per marker instead of 2.2, and only 36% of the genotypes + are the study's. At Chr01_000084128 the Alt row has 7 reads in all 80 + samples and the study's genotypes have no alternate allele. AltMatch rows + with the alternate base carry 7 to 29% of each sample's reads, alike within + a clone and different between clones, and `call` gives dosages of 1 and 2. +- Agreement between replicates cannot rank the two count sets, because a + marker that looks monomorphic agrees with itself. The potato comparison + above can, because there the rows left out are reads of the target's + alleles. The lower agreement with every row counted is what more + heterozygous genotypes cost in a hexaploid at this overdispersion, and no + truth for these clones is at hand to say how many of them are right. + +**Tables from updog's and polyRAD's simulators.** To have truth that does not +come from MADCATS code, the simulators the two packages ship (`rgeno` and +`rflexdog` in updog, and `SimGenotypes` and `SimAlleleDepth` in polyRAD) also +drew read counts, 150 markers by 150 samples per table, with +negative-binomial depths of mean 150, or 25 where the table is marked low +depth. updog's tables have a sequencing-error rate of 0.5%, an overdispersion +of 49–499 on polyRAD's scale drawn per marker and, where marked biased, a bias +of exp(N(0, 0.5)) per marker. polyRAD's have an overdispersion of 20, a +contamination rate of 0.001 and an error rate of 0.001. The first eight rows +are the tetraploid tables of the earlier comparison (100 markers, 150 samples, +20–60 reads, overdispersion 40, 0.5% read errors, bias 2^N(0, 0.5) where +marked). The table gives the share of genotypes whose most probable dosage is +the true one, without filters, and polyRAD is `IterateHWE` with its defaults: + +| Table | polyRAD | updog `norm` | `call` hwe | `call` norm | hwe + bias | norm + bias | +|---|---|---|---|---|---|---| +| 4x random mating | – | 87.5% | 89.5% | 89.5% | 89.1% | 88.9% | +| 4x F1 | – | 88.7% | 89.5% | 90.7% | 85.6% | 89.9% | +| 4x two subpopulations | – | 88.8% | 89.8% | 90.2% | 87.2% | 88.7% | +| 4x inbred lines | – | 83.4% | 97.4% | 98.6% | 71.7% | 90.3% | +| 4x random mating, biased | – | 86.0% | 81.8% | 81.3% | 87.4% | 87.4% | +| 4x F1, biased | – | 86.9% | 85.5% | 85.2% | 85.9% | 88.9% | +| 4x two subpopulations, biased | – | 89.0% | 85.4% | 85.4% | 88.7% | 89.6% | +| 4x inbred lines, biased | – | 84.9% | 97.3% | 98.4% | 72.8% | 89.8% | +| updog 4x random mating | 84.3% | 96.7% | 96.8% | 96.7% | 96.7% | 96.7% | +| updog 4x random mating, biased | 72.3% | 96.3% | 82.3% | 81.5% | 96.2% | 96.2% | +| updog 4x F1, biased | 76.1% | 94.0% | 81.6% | 81.1% | 87.6% | 93.9% | +| updog 4x `norm` genotypes, biased | 74.6% | 95.9% | 84.2% | 83.6% | 95.8% | 95.9% | +| updog 2x random mating, biased | 99.5% | 100.0% | 99.6% | 99.6% | 100.0% | 100.0% | +| updog 6x random mating, biased | 58.3% | 86.4% | 67.3% | 66.6% | 86.5% | 86.3% | +| updog 4x random mating, biased, low depth | 74.5% | 84.1% | 76.3% | 75.6% | 84.6% | 84.4% | +| polyRAD 4x outbred | 87.0% | 87.3% | 89.0% | 88.8% | 88.5% | 88.5% | +| polyRAD 4x, inbreeding 0.5 | 88.7% | 93.6% | 93.2% | 93.9% | 90.7% | 93.1% | +| polyRAD 2x outbred | 99.7% | 99.9% | 99.9% | 99.9% | 99.9% | 99.9% | +| polyRAD 6x outbred | 73.1% | 65.7% | 75.3% | 75.2% | 71.7% | 65.3% | +| polyRAD 4x outbred, low depth | 81.5% | 79.2% | 82.4% | 82.4% | 81.8% | 81.7% | + +`call` leaves a sample without reads uncalled, and the other two call it from +the prior. Such samples are 0.1% of the low-depth tables, and the scores for +`call` leave them out. What the table shows: + +- Without bias in the data, `call` without bias fitting is within half a point + of updog or ahead of it on every table. It is ahead by 0 to 3 points on + outbred tetraploids, 10 on the polyRAD hexaploid and 14 to 15 on inbred + lines. On polyRAD's partly inbred table `hwe` is 0.4 behind and `norm` 0.3 + ahead. +- With bias in the data, `--prior norm --allele-bias fit` is within 0.7 points + of updog on all seven of updog's tables and ahead of it on the four biased + tables of the first group. Without bias fitting `call` is 8 to 20 points + behind updog on updog's biased polyploid tables. +- Fitting a bias that is not there costs 0 to 1.5 points on outbred + tetraploids with `--prior norm`. Fitting one costs 8 points on inbred lines, + whether or not the data are biased, and 10 on the polyRAD hexaploid. updog, + which always fits a bias, is at 83–85% and 66% on those tables. Without + heterozygotes, or with seven dosage classes of scattered counts, a bias and + a shift of the dosages are hard to tell apart. +- `--prior hwe --allele-bias fit` fails where the samples are not in + Hardy-Weinberg proportions, with 72–73% on inbred lines, and 86–88% on the + three F1 tables against 89–93% with `norm`. The bias needs a prior that can + follow the genotype proportions. +- polyRAD with its default overdispersion of 9 is 2 to 12.5 points below + `call` hwe, the same model with estimated values, on updog's polyploid + tables, whose scatter is far smaller than 9 allows. On polyRAD's own tables, + simulated with 20, it is 0 to 4.5 points below. +- With no bias fitted, the mean probability `call` gives its called dosage is + within 1.5 points of its accuracy on every table without bias in the data. + With `--prior norm --allele-bias fit` it is within 3 points on the outbred + tetraploid tables, apart from updog's F1 table (4 above). updog's is 4.5 to + 8 points above its accuracy on the outbred tetraploid tables that are not + its own and 0 to 4 above on its own tables, 21 above on the polyRAD + hexaploid and 22 below on inbred lines. +- With the bias fitted, the overdispersion estimate was one grid step high on + several unbiased tables (50 for a simulated 40, 25 for 20), and wrong on the + polyRAD hexaploid, at 10 with a background rate of 0.0002, where the + simulation used 20 and 0.001 and the estimate without bias gave 20 and + 0.001. + +**Other seeds, and samples of mixed ploidy.** The tables above are one draw +each. Fifteen scenarios of the same two simulators were drawn again with +other seeds (150 samples, 150 markers), and `call`, polyRAD and, for the +first seed, updog called them. Two seeds finished all fifteen tables and two +more finished part of them before the runs reached their time limit, so each +scenario has two to four draws. + +Three of the scenarios have diploid, tetraploid and hexaploid samples in one +table, which the tables above did not test (polyRAD's simulator, +overdispersion 20). Share of dosages right, in all and by ploidy, as means +over the draws: + +| Scenario | `call` hwe for all, 2x, 4x and 6x | `call` structure | polyRAD `IterateHWE`, overdispersion 20 | polyRAD `IteratePopStruct`, overdispersion 20 | +|---|---|---|---|---| +| One population, 25 reads | 85.6%, 99.1%, 82.1%, 65.5% | 85.6%, 99.1%, 82.1%, 65.5% | 85.5%, 99.0%, 82.1%, 65.5% | 84.3%, 99.0%, 80.5%, 62.5% | +| Three groups, 25 reads | 86.8%, 99.2%, 83.9%, 67.7% | 87.9%, 99.3%, 85.2%, 70.8% | 86.7%, 99.1%, 83.8%, 67.7% | 87.5%, 99.1%, 84.6%, 70.3% | +| Three groups, 8 reads | 79.2%, 95.6%, 73.7%, 57.6% | 81.1%, 95.9%, 76.1%, 61.3% | 79.0%, 95.5%, 73.4%, 57.3% | 80.5%, 95.5%, 75.4%, 60.4% | + +- With samples of three ploidies in one table `call` with the HWE prior has + polyRAD's shares at the same overdispersion, to 0.3 points for every + ploidy, and `structure` is 0.4 to 0.6 points above polyRAD's + population-structure model where there are groups. With one population + `structure` equals `hwe` and polyRAD's structure model loses 1.2 points, + 3 for the hexaploids. +- The other twelve scenarios repeat the comparison above, and the draws agree + with it. Without bias in the data `call` is level with updog on updog's + tetraploids (97.0% against 96.9%) and ahead on polyRAD's low-depth + tetraploids (82.4% against 78.7%) and hexaploids (73.7% against 63.1%). + With bias in the data, `--prior norm --allele-bias fit` has 96.5% against + updog's 96.7% on biased tetraploids, 84.9% against 84.0% on hexaploids and + 92.1% against 90.6% on the F1 table, and is 1.1 points behind at low depth + (84.7% against 85.8%). +- Draws of one scenario differ by 1.5 points at most for `call` on the + tables without bias, and by up to 3 where a bias in the data stays + unfitted. The F1 table stands apart. Each draw picks new parental genotypes, + and `--prior hwe` ranges from 75.2% to 82.7% where `--prior norm --allele-bias + fit` ranges from 91.0% to 93.6%. +- The overdispersion estimate was the simulated 20 on polyRAD's tables in + most draws (18 or 25 in the others), and 30 or 40 on the mixed-ploidy table + at 8 reads. + +**Time and memory.** All three tools called one table, the first 1,000 +markers of I195 × J432 (186 tetraploid samples, 2,486 MADC rows), with target +records and default filters, on a four-core Intel Xeon at 2.1 GHz with 15 GB +of memory. Each command ran in its own session, and every process in that +session was sampled ten times a second for the wall time, the CPU time of all +its processes together, and the peak of their summed proportional memory, +which counts a page shared by several processes once. polyRAD is +`IterateHWE` with an overdispersion of 9 followed by `GetProbableGenotypes`, +and updog is `multidog` with `model = "norm"`. Both read the reference and +alternate counts `call` derives from the MADC rows. `IterateHWE` has no +option to use more than one core. + +| Tool and settings | Cores | Wall | CPU | Peak memory | +|---|---|---|---|---| +| polyRAD 2.0.1, `IterateHWE` | 1 | 5 s | 5 s | 284 MB | +| `call` with polyRAD's settings (nothing estimated) | 1 | 6.5 s | 6 s | 56 MB | +| `call` with polyRAD's settings | 4 | 2 s | 7 s | 109 MB | +| `call --prior hwe` (the defaults) | 1 | 23 s | 22 s | 59 MB | +| `call --prior hwe` | 4 | 6.5 s | 23 s | 159 MB | +| `call --prior norm` | 1 | 24 s | 24 s | 58 MB | +| `call --prior norm` | 4 | 7 s | 25 s | 159 MB | +| `call --prior structure` | 1 | 30 s | 29 s | 67 MB | +| `call --prior structure` | 4 | 9 s | 32 s | 204 MB | +| `call --prior hwe --allele-bias fit` | 4 | 43 s | 160 s | 182 MB | +| `call --prior norm --allele-bias fit` | 1 | 170 s | 168 s | 64 MB | +| `call --prior norm --allele-bias fit` | 4 | 47 s | 170 s | 181 MB | +| updog 2.1.7, `norm` | 1 | 1,115 s | 1,108 s | 281 MB | +| updog 2.1.7, `norm` | 4 | 325 s | 1,128 s | 765 MB | + +The estimate of the overdispersion and the background-read rate is a fixed +cost, because it works on at most 20,000 sample-by-marker cells however large +the table is (107 of the 1,000 markers here). The same runs on one core with +the two values given as numbers, so that the run estimates nothing: + +| Settings, values given | Wall | Share of the full run | +|---|---|---| +| `--prior hwe` | 4.5 s | 4.5 of 23 s | +| `--prior norm` | 7 s | 7 of 24 s | +| `--prior norm --allele-bias fit` | 57 s | 57 of 170 s | + +- With polyRAD's settings `call` takes 6.5 s on one core where polyRAD takes + 5 s, in a fifth of the memory, and 2 s on four cores. +- A default run spends 18 of its 23 s on the estimate. Calling alone takes + 4.5 s with `hwe` and 7 s with `norm`. +- With the bias fitted the estimate takes about 115 s, since it fits the bias + of every sampled marker again for each pair of values tried. Calling with + the bias takes 57 s, eight times as long as without. +- The whole run with the bias is 6.5 times as fast as updog on one core and 7 + times on four, with the accuracy shown above. Without the bias, + `call --prior norm` is 46 times as fast as updog on one core and 45 times on + four. +- Four workers give 3.1 to 3.6 times the speed of one. Memory with four + workers is that of five Python processes, 110–205 MB. + +The `call` rows come from a second measurement, after a review of the code +for work done more than once. polyRAD's and updog's rows are from the first +measurement. Before the review the default run took 33 s on one core and 10 s +on four, `norm` 38 s and 10 s, and `norm` with the bias 301 s and 80 s. +Calling alone took 6 s, 10 s and 97 s. `call` with polyRAD's settings, whose +code changed least, took 7 s then and 6.5 s now, so the two measurements are +of comparable machines. Three changes account for the gain, and none changes +a result (51 output files of twelve runs were the same byte for byte before +and after): + +- The code computes the likelihoods of a record's read counts once and not + in every pass. They depend on the allele frequencies only through + polyRAD's contamination term, so it recomputes them in every pass only + when `--contamination-rate` is above 0. +- The estimate chose its markers again for every background-read rate it + tried, and now chooses them once. +- The code computes the binomial coefficient of a sample's counts once per + sample and the terms that depend only on the overdispersion tried once per + value, not once per dosage. + +**Shortening the estimate when the bias is fitted was tried and not done.** +With the bias fitted, two thirds of a run is the read-model estimate (113 of +170 s above). On the benchmark table, in one process, the estimate takes 108 s +against 17 s without the bias. It works on 107 markers, tries five background +rates and passes over the markers nine times, and each pass fits every +marker's bias again, 963 fits of 0.1 s. A fit calls the record at about 21 +bias values (17 to 43), and each call takes 3 passes with `hwe`, 4 with `norm` +and 6 with the free weights the estimate uses. A profile of the fit found no +step done twice. The time is in the passes themselves, not in the read-count +likelihoods, which the fit already computes once per bias value. + +That leaves fitting the bias less often, tried in three ways: + +| Change | Estimate on the benchmark table | What it cost | +|---|---|---| +| None | 108 s, 150 and 0.0005 | | +| Fit each marker's bias once, under the read model the search starts from | not run | The bias depends on the overdispersion the fit uses. Against the fit under the final read model, the log bias differed by a median of 0.14 and by more than 0.05 at 64 of the 107 markers (9 of 107 when fitted under an overdispersion of 100 where 150 is final) | +| Keep each marker's bias for an overdispersion when the search moves to another background rate | 86 s, 150 and 0.001 | A different background rate on both real tables (0.003 on potato where it is 0.001), for a whole run that is 19% shorter on the benchmark table and 5% longer on potato. Potato accuracy did not change (96.59% against 96.57%) | +| Start the search from an estimate made on a quarter of the markers | 53 s, 150 and 0.0005 | No gain without the bias (17.3 s against 16.9 s). With an eighth of the markers, 91 s with the bias and 24 s without. The gain depends on the pilot landing near the answer | + +None was kept. The first two change what the estimate finds, because two +neighbouring background rates often fit almost equally well and any change in +the biases decides between them. The third leaves the estimate alone but does +not reliably save time. A run with the bias gets shorter with more workers, +with `--overdispersion` and `--background-rate` given as numbers once a first +run has estimated them (57 s where the full run takes 170 s), or with a +compiled kernel, which is not planned. + +**Keeping a fitted bias only where it earns its place (`--allele-bias auto`).** +Fitting the bias raises agreement with the array by 4 points on the potato +data and lowers accuracy on tables without bias, by up to 8 points for +hexaploids pooled from real diploid reads (below). A rule was tried in a +scratch copy of the module, and then built. Under it, a record keeps its +fitted bias only when the fit raises the log-likelihood, after the prior on +the bias, by a set margin over calling without one. Each run used +`--prior hwe` and the read model that the data set's own run with the bias +had estimated, so that only the rule differs. Share of genotypes right, +without filters: + +| Data set | No bias (its own read model) | Bias always | Margin 2 | Margin 5 | Margin 10 | Margin 20 | +|---|---|---|---|---|---|---| +| Potato, 85 clones, against the array | 92.26% | 96.60% | 96.59% | 96.56% | 96.47% | 96.23% | +| Potato vendor MADC, diploids | 92.02% | 92.61% | 92.60% | 92.60% | 92.60% | 92.58% | +| Potato vendor MADC, tetraploids | 86.41% | 91.25% | 91.25% | 91.23% | 91.19% | 91.15% | +| Delta smelt targets (diploid) | 99.41% | 99.28% | 99.27% | 99.27% | 99.27% | 99.27% | +| Delta smelt off-target | 99.70% | 99.39% | 99.39% | 99.39% | 99.39% | 99.39% | +| Pooled tetraploids, 50 reads, targets | 90.56% | 89.65% | 90.17% | 90.46% | 90.53% | 90.64% | +| Pooled hexaploids, 50 reads, targets | 77.60% | 69.83% | 75.53% | 77.17% | 77.60% | 77.60% | +| Pooled hexaploids, off-target | 89.03% | 85.92% | 88.03% | 88.56% | 88.99% | 89.03% | + +Potato keeps a bias at 1,513, 1,153, 965, 789 and 590 of 1,544 records at +margins 0, 2, 5, 10 and 20. The vendor MADC keeps one at 3,851, 3,266, 2,949, +2,591 and 2,155 of 3,904. Smelt keeps one at 905, 330, 195, 137 and 84 of 946. +Pooled tetraploids keep one at 920, 123, 53, 29 and 19 of 946. Pooled +hexaploids keep one at 926, 89, 40, 26 and 18 of 944. + +- A margin of 10 keeps the gain where there is bias (96.47% of 96.60%, + 91.19% of 91.25%) and removes the loss where there is none (90.53% against + 90.56%, 77.60% against 77.60%). On real unbiased reads nine records in ten + give the bias up, while on potato half to two thirds keep it. +- Smelt stays 0.14 points (targets) and 0.31 (off-target) under the run + without bias at every margin. There the loss does not come with the many + small biases. It comes with the read model the bias run estimated + (overdispersion 200 where the run without bias has 150) or with the few + records that keep a large bias. +- The rule is now `--allele-bias auto`, with a margin of 10 + (`BIAS_AUTO_MIN_GAIN`). A record that gives its bias up gets the same call + as without one, since the call at a bias of 1 is among those the search + makes. The output still holds its fitted bias. `INFO/BIAS` and the bias + table hold the fit for every two-allele record, the same values a run with + `fit` writes, and a record called without it carries `INFO/BIASUNUSED` + (`Used` 0 in the table). On the potato data 755 of 1,544 records are called + without their bias, and 26 of those have a fitted bias outside 0.8 to 1.25, + down to 0.37, which a filter on the bias would not see if the field held + the bias used. With the read model the run itself estimates, `auto` gives + the figures of the margin-10 column for all five data sets, and on the + potato data the same records and bias table as the scratch copy. +- The estimate of the overdispersion and background rate fits every record's + bias and keeps it, as under `fit`. Applying the rule in the estimate too + gave estimates that were the same or one step of the background rate apart + (potato 300 and 0.002 against 0.001, pooled tetraploids 500 and 0.002 + against 0.003), and calls that were the same or 0.02 to 0.07 points worse + (potato 96.45% against 96.47%, pooled tetraploids 90.46% against 90.53%). +- `off` stays the default. `auto` takes as long as `fit`, several times a run + without a bias, and on smelt it is 0.14 points under `off`. + +**`auto` on other kinds of population, and inbred lines.** The rule was chosen +on five data sets of outbred samples. The twenty tables of the simulator +comparison above (the same draws) were called again with each prior and each +setting of `--allele-bias`, every run with the read model it estimates. Share +of dosages right, without filters, and the records whose bias `auto` used: + +| Table | hwe, `off` | `fit` | `auto` | norm, `off` | `fit` | `auto` | Biases `auto` used with hwe and norm | +|---|---|---|---|---|---|---|---| +| 4x random mating | 89.5% | 89.1% | 89.5% | 89.5% | 89.0% | 89.4% | 0 of 100, 0 of 100 | +| 4x F1 | 89.5% | 85.6% | 89.5% | 90.7% | 90.0% | 90.7% | 0 of 100, 0 of 100 | +| 4x two subpopulations | 89.8% | 87.2% | 88.6% | 90.2% | 88.7% | 90.0% | 9 of 100, 1 of 100 | +| 4x inbred lines | 97.4% | 71.7% | 71.7% | 98.6% | 90.3% | 90.3% | 100 of 100, 100 of 100 | +| 4x random mating, biased | 81.8% | 87.4% | 85.4% | 81.3% | 87.4% | 85.2% | 22 of 100, 20 of 100 | +| 4x F1, biased | 85.5% | 85.9% | 85.2% | 85.2% | 88.9% | 86.5% | 21 of 100, 13 of 100 | +| 4x two subpopulations, biased | 85.4% | 88.7% | 87.9% | 85.4% | 89.6% | 87.7% | 18 of 100, 12 of 100 | +| 4x inbred lines, biased | 97.3% | 72.8% | 72.8% | 98.4% | 89.8% | 89.8% | 100 of 100, 100 of 100 | +| updog 4x random mating | 96.8% | 96.7% | 96.8% | 96.7% | 96.7% | 96.7% | 0 of 150, 0 of 150 | +| updog 4x random mating, biased | 82.3% | 96.2% | 96.1% | 81.5% | 96.2% | 96.1% | 115 of 150, 115 of 150 | +| updog 4x F1, biased | 81.6% | 87.6% | 90.5% | 81.1% | 93.9% | 93.8% | 109 of 150, 107 of 150 | +| updog 4x `norm` genotypes, biased | 84.3% | 95.8% | 95.7% | 83.6% | 95.9% | 95.8% | 114 of 150, 114 of 150 | +| updog 2x random mating, biased | 99.6% | 100.0% | 100.0% | 99.6% | 100.0% | 100.0% | 115 of 150, 115 of 150 | +| updog 6x random mating, biased | 67.3% | 86.5% | 83.8% | 66.6% | 86.3% | 82.6% | 76 of 150, 69 of 150 | +| updog 4x random mating, biased, low depth | 76.3% | 84.6% | 83.0% | 75.6% | 84.4% | 82.2% | 66 of 150, 56 of 150 | +| polyRAD 4x outbred | 89.0% | 88.5% | 89.0% | 88.9% | 88.5% | 88.9% | 0 of 150, 0 of 150 | +| polyRAD 4x, inbreeding 0.5 | 93.2% | 90.7% | 92.9% | 93.9% | 93.1% | 94.1% | 14 of 150, 2 of 150 | +| polyRAD 2x outbred | 99.9% | 99.9% | 99.9% | 99.9% | 99.9% | 99.9% | 0 of 150, 0 of 150 | +| polyRAD 6x outbred | 75.3% | 71.8% | 73.4% | 75.2% | 65.3% | 72.0% | 0 of 150, 0 of 150 | +| polyRAD 4x outbred, low depth | 82.4% | 81.8% | 82.4% | 82.4% | 81.7% | 82.4% | 0 of 150, 0 of 150 | + +Eighty of these figures repeat the earlier table. Seventy-nine came out the +same. `--prior norm --allele-bias fit` on updog's F1 table gives 93.9% where +the earlier table recorded 93.3%, and that table now has 93.9%. + +- Without bias in the data, `auto` uses no record's bias in one population or + an F1 family (six tables) and is within 0.1 points of `off`, where `fit` + loses up to 3.8 points. In two subpopulations it uses 9 and 1 of 100 and + loses 1.2 points with `hwe` and 0.2 with `norm`. With half-inbred samples + it uses 14 and 2 of 150 and is within 0.3 points of `off`. +- With bias in the data, how much of the gain `auto` keeps goes with the depth + and the size of the biases. On updog's tetraploid tables at 150 reads it + uses three quarters of the biases and is within 0.2 points of `fit` (ahead + of it by 3.0 on the F1 table with `hwe`). At 25 reads it uses 4 in 10 and + is 1.6 to 2.1 points behind, and on the hexaploids it uses half and is 2.7 + to 3.7 behind. On the first group's biased tables (20 to 60 reads, smaller + biases) it uses an eighth to a fifth and is 0.8 to 2.4 points behind + `fit`, and 0.3 below `off` on the F1 table with `hwe`. +- On the polyRAD hexaploid table `auto` uses no record's bias and is still + 1.9 points (`hwe`) and 3.2 (`norm`) under `off`. The loss comes with the + read model a run with a bias estimates. It has an overdispersion of 10 and + a background rate of 0.0002, where the simulation used 20 and 0.001, the + values the run without a bias estimates. +- On inbred lines `auto` fails as `fit` does. Every record's bias ends at a + limit of the search (0.135 or 7.39) with a gain of 19 to 115, over the + margin of 10, so the rule uses every one. A bias changes the expected read + fraction of the heterozygous dosages only. Pushed to the limit under a + prior that expects heterozygotes, it lets samples with all their reads on + one allele pass for one to three copies. With `hwe`, the run calls 2,222 + of 6,965 samples without the allele with one to three copies, where the + run without a bias miscalls 208. + +For inbred lines the setting that matters is the selfing rate. The two inbred +tables with `--prior hwe`, how many of the 100 records ended at a limit, and +how many `auto` used: + +| Table, selfing rate | `off` | `fit` | `auto` | At a limit | `auto` used | +|---|---|---|---|---|---| +| 4x inbred lines, none | 97.4% | 71.7% | 71.7% | 100 | 100 | +| 4x inbred lines, 0.95 | 99.6% | 98.5% | 99.4% | 85 | 7 | +| 4x inbred lines, 0.99 | 99.9% | 99.6% | 99.9% | 0 | 0 | +| 4x inbred lines, biased, none | 97.3% | 72.8% | 72.8% | 100 | 100 | +| 4x inbred lines, biased, 0.95 | 99.6% | 98.3% | 99.0% | 79 | 18 | +| 4x inbred lines, biased, 0.99 | 99.8% | 99.7% | 99.8% | 0 | 0 | + +With the selfing rate given, the prior no longer wants heterozygotes and the +bias has little to gain (at 0.99 no gain reaches 3). The bias in the data +makes no difference to these samples. Without a bias fitted, the biased and +unbiased tables are within 0.1 points of each other. + +A scratch copy tried a change to the rule, so that under `auto` a bias that +ends at a limit of the search goes unused. Each data set under the read model +of its run with `auto`, with `--prior hwe`: + +| Data set | `auto` | `auto`, biases at a limit not used | At a limit and used | +|---|---|---|---| +| 4x inbred lines | 71.7% | 97.6% | 100 of 100 | +| 4x inbred lines, biased | 72.8% | 97.6% | 100 of 100 | +| The other six tables of the first group | unchanged | unchanged | 0 of 100 | +| Potato, 85 clones, against the array | 96.47% | 96.36% | 4 of 1,544 | +| Potato vendor MADC | 91.88% | 91.80% | 25 of 3,904 | +| Delta smelt targets | 99.27% | 99.27% | 10 of 946 | +| Delta smelt off-target | 99.39% | 99.43% | | +| Pooled tetraploids, 50 reads, targets | 90.53% | 90.53% | 15 of 946 | +| Pooled hexaploids, 50 reads, targets | 77.60% | 77.60% | 23 of 944 | + +It restores the inbred tables (98.7% and 98.5% with `--prior norm`) and costs +0.1 points on both potato sets, where the few markers at a limit are strongly +biased in truth. The share of records at a limit separates the cases without +changing a call. It is 0.3 to 2.5% on the five real data sets and all of them +on inbred lines. + +The rule was then built (`ModelSettings.reject_bias_at_limit`), because the +gain on inbred lines was judged worth the 0.1 points lost on potato. Under +`auto`, `call` does not use a bias at a limit of the search and warns, the +bias table gives the reason in a `Rejected` column and the QC JSON lists the +records. `fit` still uses such a bias, and warns when a fifth or more of the +fits end at a limit. + +**What `auto` changed on the unbiased hexaploid table, and the read model of +a record called without its bias.** On the polyRAD hexaploid table `auto` +used no record's bias and was still 1.85 points under `off`. Every record got +the same call as without a bias, so the difference had to be in the two +values every record shares, and it was. The run had an overdispersion of 10 +where `off` estimates 20, and a background rate of 0.0002 where it estimates +0.001. Run without a bias at each pair: + +| Overdispersion, background rate | Share of dosages right | +|---|---| +| 20, 0.001 (what `off` estimates) | 75.28% | +| 10, 0.0002 (what the run with a bias estimated) | 73.43%, the figure of `auto` | +| 10, 0.001 | 73.55% | +| 20, 0.0002 | 75.15% | + +The overdispersion accounts for 1.7 of the 1.85 points. It was not a better +fit of a different model but a search that stopped early. Log-likelihood of +the markers the estimate uses, at the better background rate for each value: + +| Overdispersion | Without a bias | With every record's bias fitted | +|---|---|---| +| 10 | −77,432 | −77,294 | +| 20 | −77,297 | −77,189 | + +With the bias fitted too, 20 is more probable than 10 by 105. The estimate +calls every marker at the current overdispersion, holds each marker's allele +frequency, dosage weights and bias, scores every overdispersion with them, +and repeats from the best until the choice stops changing. With a bias and +seven dosage weights fitted at 10 and held, 10 scored best, so started from +polyRAD's 9 the search moved to 10 and stayed. Two changes followed: + +- The estimate with a bias now starts from the estimate without one. On this + table it then finds 20, and `fit` goes from 71.75% to 73.78% (73.88% under + the read model of `off`, so 1.4 of its 3.5 points under `off` are the + biases themselves). +- Under `auto`, a record called without its bias uses the estimate made + without a bias, the same way `off` does. Only the records that keep their + bias use the estimate made with the bias fitted. On this table `auto` then + has the genotypes of `off`. + +The table compares four ways of choosing the read model under `auto`, each +with the limit rule and `--prior hwe`. It gives the share of dosages right, +and in brackets how far the mean probability `call` gives its called dosage +lies above that share (positive means over-confident): + +| Table | Every record under the estimate with every bias fitted (as before) | Records without their bias under the estimate without a bias (built) | Every record under the estimate without a bias | One estimate that applies the rule itself | +|---|---|---|---|---| +| 4x random mating, biased | 85.35% (+2.3) | 85.00% (−1.2) | 84.02% (−1.1) | 84.02% (−1.1) | +| updog 4x random mating, biased | 96.14% (+0.6) | 95.94% (−0.6) | 93.14% (−3.0) | | +| updog 4x random mating, biased, low depth | 83.02% (+2.8) | 82.82% (0.0) | 79.94% (+0.8) | | +| updog 4x F1, biased | 90.53% (+5.8) | 90.65% (+4.6) | 87.99% (+2.6) | | +| updog 6x random mating, biased | 83.84% (+4.2) | 83.04% (−0.2) | | | +| 4x two subpopulations | 88.57% (+2.8) | 88.45% (+2.2) | | | +| polyRAD 4x, inbreeding 0.5 | 92.94% (+2.4) | 92.53% (+2.0) | | | +| 4x random mating | 89.49% (+1.0) | 89.49% (+0.1), as `off` | | | +| Potato, 85 clones, against the array | 96.34% | 96.37% | 96.41% | 96.34% | +| Potato vendor MADC | 91.80% | 91.76% | 91.47% | 91.80% | +| Delta smelt targets | 99.27% | 99.27% | 99.28% | 99.27% | +| Pooled tetraploids, 50 reads | 90.46% | 90.46% | 90.46% | 90.46% | +| Pooled hexaploids, 50 reads | 77.60% | 77.60% | 77.60% | 77.60% | + +- Refitting the read model pays off for the records that keep their bias. + With every record under the estimate without a bias, the biased simulated + tables lose 1 to 3 points and the vendor MADC 0.3. +- One estimate that applies the rule itself did not move from where it + started. Started from the estimate without a bias, few biases gain the + margin under its wide scatter, so the scatter stays wide (first row). +- For the records called without their bias, the estimate with every bias + fitted is between 0.1 points worse and 0.8 better on the simulated tables + and no better on the real data. It is over-confident on every simulated + table, by 0.6 to 5.8 points. With the estimate without a bias the four + random-mating tables with bias are within 1.2 points and the table + without bias within 0.1, as `off` is. These records now get the estimate + without a bias, so a record `auto` leaves without a bias is then the + record `off` writes, on every table. + +**Replicates and calibration of `off`, `fit` and `auto`.** The comparisons +above are one draw per scenario, and several came before the rules were +final. With the rules as built, three sets of simulated tables were called +with each setting of `--allele-bias`, without filters: + +- Ten scenarios drawn with stated seeds, five draws each, from the model of + the first group of tables (100 markers, 150 samples, 20 to 60 reads, an + overdispersion of 40, 0.5% of reads showing the wrong allele, and where + marked a bias of 2^N(0, 0.5) per marker), tetraploid and hexaploid. +- Seven scenarios of updog's simulator, four draws each (150 markers, 150 + samples, 150 reads or 25 at low depth, a bias of exp(N(0, 0.5))). +- Eight scenarios of polyRAD's simulator, four draws each, three of them + with diploid, tetraploid and hexaploid samples in one table. + +Share of dosages right with `--prior hwe`, as the mean over the draws, and +for `auto` against `off` the difference within each draw: + +| Scenario | Draws | `off` | `auto` | `fit` | `auto` minus `off`, mean (lowest to highest draw) | Biases `auto` used | +|---|---|---|---|---|---|---| +| 4x random mating | 5 | 89.69% | 89.69% | 89.23% | +0.00 (+0.00 to +0.00) | 0% | +| 4x random mating, biased | 5 | 83.90% | 86.98% | 88.68% | +3.08 (+2.71 to +3.61) | 24% | +| 4x F1 | 5 | 90.05% | 89.37% | 85.29% | -0.68 (-1.17 to -0.09) | 2% | +| 4x F1, biased | 5 | 85.56% | 86.58% | 87.05% | +1.02 (-0.07 to +2.21) | 27% | +| 4x two subpopulations | 5 | 89.65% | 89.02% | 87.63% | -0.63 (-0.79 to -0.37) | 4% | +| 4x two subpopulations, biased | 5 | 83.63% | 85.52% | 87.12% | +1.89 (+0.70 to +2.39) | 25% | +| 4x inbred lines | 5 | 97.34% | 97.34% | 73.71% | +0.00 (+0.00 to +0.00) | 0% | +| 4x inbred lines, biased | 5 | 97.16% | 97.16% | 71.60% | +0.00 (+0.00 to +0.00) | 0% | +| 6x random mating | 5 | 75.02% | 75.02% | 73.26% | +0.00 (+0.00 to +0.00) | 0% | +| 6x random mating, biased | 5 | 66.69% | 67.83% | 72.56% | +1.14 (+0.45 to +1.83) | 7% | +| updog 4x random mating | 4 | 96.99% | 96.99% | 96.91% | +0.00 (+0.00 to +0.00) | 0% | +| updog 4x random mating, biased | 4 | 84.80% | 96.25% | 96.46% | +11.45 (+9.80 to +13.47) | 72% | +| updog 4x random mating, biased, low depth | 4 | 76.10% | 82.82% | 84.70% | +6.72 (+5.63 to +8.03) | 35% | +| updog 4x `norm` genotypes, biased | 4 | 82.47% | 95.35% | 95.74% | +12.88 (+11.46 to +14.55) | 70% | +| updog 4x F1, biased | 4 | 79.28% | 83.95% | 81.44% | +4.67 (-0.99 to +8.21) | 70% | +| updog 2x random mating, biased | 4 | 99.63% | 99.98% | 99.98% | +0.35 (+0.24 to +0.63) | 75% | +| updog 6x random mating, biased | 4 | 66.73% | 82.56% | 85.83% | +15.82 (+14.68 to +19.05) | 52% | +| polyRAD 4x outbred | 4 | 88.94% | 88.94% | 88.53% | +0.00 (+0.00 to +0.00) | 0% | +| polyRAD 4x outbred, low depth | 4 | 82.39% | 82.39% | 81.91% | +0.00 (+0.00 to +0.00) | 0% | +| polyRAD 4x, inbreeding 0.5 | 4 | 93.47% | 92.68% | 90.61% | -0.79 (-1.11 to -0.60) | 12% | +| polyRAD 2x outbred | 4 | 99.94% | 99.94% | 99.94% | +0.00 (+0.00 to +0.00) | 0% | +| polyRAD 6x outbred | 4 | 73.68% | 73.68% | 72.38% | +0.00 (+0.00 to +0.00) | 0% | +| polyRAD 2x, 4x and 6x, one population, 25 reads | 4 | 85.49% | 85.49% | 85.28% | +0.00 (+0.00 to +0.00) | 0% | +| polyRAD 2x, 4x and 6x, three groups, 25 reads | 4 | 86.81% | 86.80% | 86.54% | -0.01 (-0.03 to +0.00) | 0% | +| polyRAD 2x, 4x and 6x, three groups, 8 reads | 4 | 79.33% | 79.31% | 79.04% | -0.02 (-0.10 to +0.00) | 0% | + +Calibration. The probability `call` gives a called genotype comes from its +GQ. The figure is the mean of that probability over the called genotypes +minus the share of them that are right, in points, as the mean over the +draws, and is positive when the calls are over-confident. + +| Scenario | `off` | `auto` | `fit` | +|---|---|---|---| +| 4x random mating | -0.1 | -0.1 | +0.6 | +| 4x random mating, biased | +1.9 | -0.3 | +1.1 | +| 4x F1 | -0.3 | +0.2 | +3.0 | +| 4x F1, biased | +2.4 | +1.1 | +2.3 | +| 4x two subpopulations | +0.5 | +1.2 | +2.9 | +| 4x two subpopulations, biased | +2.1 | +1.3 | +2.7 | +| 4x inbred lines | +0.4 | +0.4 | +14.4 | +| 4x inbred lines, biased | +0.5 | +0.5 | +14.0 | +| 6x random mating | -0.5 | -0.5 | +4.0 | +| 6x random mating, biased | +4.6 | +3.7 | +5.0 | +| updog 4x random mating | +0.3 | +0.3 | +0.4 | +| updog 4x random mating, biased | +5.4 | -0.5 | +0.5 | +| updog 4x random mating, biased, low depth | +4.8 | -0.2 | +1.1 | +| updog 4x `norm` genotypes, biased | +6.4 | -0.6 | +0.8 | +| updog 4x F1, biased | +8.6 | +11.1 | +14.7 | +| updog 2x random mating, biased | 0.0 | 0.0 | 0.0 | +| updog 6x random mating, biased | +10.8 | 0.0 | +2.2 | +| polyRAD 4x outbred | -0.2 | -0.2 | +0.4 | +| polyRAD 4x outbred, low depth | 0.0 | 0.0 | +0.5 | +| polyRAD 4x, inbreeding 0.5 | +1.4 | +2.3 | +4.7 | +| polyRAD 2x outbred | 0.0 | 0.0 | 0.0 | +| polyRAD 6x outbred | -0.1 | -0.1 | +1.4 | +| polyRAD 2x, 4x and 6x, one population, 25 reads | +0.2 | +0.2 | +0.7 | +| polyRAD 2x, 4x and 6x, three groups, 25 reads | +0.3 | +0.4 | +1.6 | +| polyRAD 2x, 4x and 6x, three groups, 8 reads | +0.7 | +0.8 | +2.1 | + +- Where the data have no bias and the samples are one population or inbred + lines, `auto` is `off` in every draw. No record keeps a bias, and the + genotypes are the same. That holds for diploids, tetraploids and + hexaploids from all three simulators, and for diploid, tetraploid and + hexaploid samples in one table, where `fit` loses up to 1.8 points, and + for inbred lines, where `fit` loses 24 to 26. The polyRAD hexaploid + scenario is the one on which `auto` was 1.9 points under `off` before a + record without its bias got the read model of `off`. With three groups of + samples of mixed ploidy `auto` is within 0.1 points of `off`. +- Where the data have a bias, what `auto` keeps of the gain of `fit` goes + with the depth and the size of the biases. It keeps 97 to 100% at 150 + reads in diploids and tetraploids and 83% in hexaploids, 78% at 25 reads, + and at 20 to 60 reads with the smaller biases 54 to 68% in tetraploids and + 19% in hexaploids, where it uses 7 records in 100. +- `auto` is the best calibrated of the three where the samples mate at + random and the data have a bias. It is within 0.6 points on the five such + tetraploid and diploid scenarios and exact on updog's hexaploids, where + `off` is over-confident by up to 10.8 points and `fit` by up to 2.2. The + exception is the hexaploids at 20 to 60 reads, where it uses few biases + and stays near `off` (+3.7 against +4.6). +- In F1 families, two subpopulations and half-inbred samples without bias, + `auto` with `--prior hwe` uses a bias at 2, 4 and 12 records in 100 and + loses 0.7, 0.6 and 0.8 points (`fit` loses 4.8, 2.0 and 2.9). The prior + does not fit such samples, and a bias can make up for it. None of these + biases ends at a limit of the search, so the limit rule does not see them. + On updog's F1 tables with bias every setting is over-confident under + `hwe`, by 9 to 15 points. + +The same with `--prior norm`, on the scenarios it ran on: + +| Scenario | Draws | `off` | `auto` | `fit` | `auto` minus `off`, mean (lowest to highest draw) | Biases `auto` used | +|---|---|---|---|---|---|---| +| 4x random mating | 5 | 89.66% | 89.66% | 89.15% | +0.00 (+0.00 to +0.00) | 0% | +| 4x random mating, biased | 5 | 83.37% | 86.26% | 88.56% | +2.89 (+2.37 to +3.77) | 20% | +| 4x F1 | 5 | 91.02% | 91.02% | 90.51% | +0.00 (+0.00 to +0.00) | 0% | +| 4x F1, biased | 5 | 85.52% | 89.09% | 91.42% | +3.57 (+2.01 to +4.93) | 24% | +| 4x two subpopulations | 5 | 89.94% | 89.85% | 88.79% | -0.09 (-0.18 to +0.00) | 1% | +| 4x two subpopulations, biased | 5 | 83.84% | 86.22% | 88.34% | +2.38 (+1.23 to +2.86) | 19% | +| 4x inbred lines | 5 | 98.43% | 98.43% | 90.49% | +0.00 (+0.00 to +0.00) | 0% | +| updog 4x F1, biased | 4 | 78.68% | 91.51% | 91.64% | +12.83 (+9.77 to +16.21) | 71% | +| polyRAD 4x, inbreeding 0.5 | 4 | 94.07% | 93.99% | 93.05% | -0.09 (-0.16 to +0.00) | 2% | + +| Scenario | `off` | `auto` | `fit` | +|---|---|---|---| +| 4x random mating | 0.0 | 0.0 | +0.7 | +| 4x random mating, biased | +2.8 | +0.5 | +1.3 | +| 4x F1 | +0.4 | +0.4 | +1.1 | +| 4x F1, biased | +4.6 | +1.9 | +1.2 | +| 4x two subpopulations | +0.1 | +0.2 | +1.5 | +| 4x two subpopulations, biased | +1.9 | +0.3 | +1.2 | +| 4x inbred lines | +0.1 | +0.1 | -7.4 | +| updog 4x F1, biased | +12.4 | +4.9 | +5.6 | +| polyRAD 4x, inbreeding 0.5 | +0.8 | +0.9 | +2.2 | + +- With `norm`, `auto` on F1 families without bias uses no record's bias and + is `off` in every draw, and in two subpopulations it loses 0.1 points. +- `norm` is ahead of `hwe` in F1 families, by 1.0 point without a bias (0.6 + to 1.5 over the draws), by 2.5 with `auto` and 4.4 with `fit` where the + data have a bias, and on updog's F1 tables by 7.6 with `auto` and 10.2 + with `fit` (91.5% and 91.6% against 84.0% and 81.4%). In two + subpopulations it is 0.3 ahead without a bias and 0.7 with `auto`. On + inbred lines it is 1.1 ahead without a selfing rate. +- In one randomly mating population `norm` is level with `hwe` where the + data have no bias (0.03 points below) and 0.5 to 0.7 below it where they + have one and the run leaves it unfitted or uses `auto`. With `fit` it is + 0.1 below. +- On the half-inbred samples, where `auto` with `hwe` loses 0.8 points, a + prior that fits them removes the loss. With `norm`, `auto` uses 2 records + in 100 and is 0.1 points under its `off`, which is itself 0.6 above + `hwe`. With `--prior hwe --selfing-rate 0.5` the share is 93.91% without + a bias (93.47% for `hwe` alone) and 93.70% with `auto`, which uses 2 to 7 + records of 150. +- `--prior structure`, which takes no bias, is 0.6 points ahead of `hwe` in + two subpopulations without bias and 1.1 with (five draws each, 40 + neighbours chosen in each), and equals `hwe` in one F1 family, where it + chose no neighbours in any draw. + +The real data sets with the rules as built, `--prior hwe`: + +| Data set | `off` | `auto` | `fit` | Biases `auto` used | Calibration of `off`, `auto`, `fit` | +|---|---|---|---|---|---| +| Potato, 85 clones, against the array | 92.26% | 96.37% | 96.59% | 785 of 1,544 | +4.8, +2.3, +2.5 | +| Potato vendor MADC | 89.14% | 91.76% | 91.91% | 2,566 of 3,904 | | +| Potato vendor MADC, diploids | 92.02% | 92.51% | 92.61% | | | +| Potato vendor MADC, tetraploids | 86.41% | 91.06% | 91.25% | | | +| Delta smelt targets | 99.41% | 99.27% | 99.28% | 127 of 946 | +0.5, +0.6, +0.6 | +| Delta smelt off-target | 99.70% | 99.43% | 99.39% | | | +| Pooled tetraploids, 50 reads, targets, three draws | 90.44% | 90.35% | 89.47% | 15, 16 and 29 of 946 | +1.1, +1.2, +2.3 | +| Pooled hexaploids, 50 reads, targets, three draws | 77.79% | 77.79% | 71.03% | 3, 10 and 12 of 944 | +0.8, +0.8, +8.3 | + +- On the pooled polyploids, whose reads have no bias, `auto` is 0.1 points + under `off` in each tetraploid draw and equal to it at the targets in each + hexaploid draw, where `fit` loses 0.9 to 1.1 and 6.2 to 7.8. +- On the potato clones `auto` halves the over-confidence of `off` against + the array (the array and the reads also differ for reasons a caller cannot + see, so the gap is not all the caller's). On the pooled reads its + calibration is that of `off`, and `fit` is over-confident by 2 and 8 + points. +- The limit rule under `auto` rejected 4 records on the potato clones, 25 + on the vendor MADC, 10 on smelt and 14 to 23 on the pooled tables. + +**`MPP` and `PMC`, updog's two summaries of the posterior.** BIGr's +`updog2vcf` writes updog's `maxpostprob` for each sample as `MPP` and its +`prop_mis` for each marker as `PMC`, and its `filterVCF` filters on both. +`call` writes them under those names on two-allele records. The source of +updog 2.1.7 shows what the two values are: + +- `flexdog_full` (`R/ashdog.R`) takes each individual's posterior over the + dosages from `get_wik_mat` (`src/flexdog.cpp`). That posterior is the + prior times the beta-binomial likelihood at the fitted bias, sequencing + error and overdispersion, normalised. `geno` is the first dosage with the + largest posterior, `maxpostprob` is that posterior, and `prop_mis` is + `1 - mean(maxpostprob)`. +- updog removes individuals with a missing count before the fit, and they + take no part in the mean. Individuals with a depth of 0 stay. Their + posterior is the prior and their `maxpostprob` its largest probability. + `multidog` drops a marker only when every individual has a depth of 0 or + is missing. +- The documentation calls `prop_mis` the posterior proportion of individuals + mis-genotyped, and gives as an example discarding markers over 0.05 and + individuals with a `maxpostprob` under 0.95. The vignettes use 0.05 + together with a bias between exp(-1) and exp(1), and call 0.2 a loose cut + to start from. Gerard et al. + (2018) define the two values and report the estimated minus the true + proportion misgenotyped, and Gerard and Ferrão (2020) report it by prior. +- BIGr 0.7.2 writes `maxpostprob` and `prop_mis` unchanged. `filterVCF` sets + a genotype to missing where `MPP` is under `filter.MPP` and keeps markers + with `PMC` under `filter.PMC`. It reads `PMC` with the pattern + `PMC=([0-9.]+)`, which a number written with an exponent does not match, + and takes the INFO and FORMAT keys of a file from its first record. + +In `call` the posterior of a two-allele record is the same object, the prior +times the beta-binomial likelihood normalised over the dosages +(`dosage_posterior`), for whichever prior, bias and read model the call used. +`MPP` is the largest value of `GP`, `GQ` is `MPP` on the Phred scale, and +`PMC` is one minus the mean of `MPP`. Both have four decimals and no +exponent. Three points needed a decision: + +- `PMC` leaves out samples without reads, and they have no `MPP`. `call` + writes no genotype for them, and counting them would mix missing data into + a statement about genotypes. This is what updog gives when it gets such + samples as missing. Given a depth of 0 they raise its value. +- `--min-dp` and `--min-gq` do not enter `PMC`. It describes the model's fit + at the marker, as in updog, which filters nothing before computing it. +- Records with more than two alleles carry neither. They have one posterior + per allele, and `MPC` is an upper bound, not the genotype's probability. + +*Against updog's own numbers.* `scripts/make_updog_fixture.R` now stores +`maxpostprob` and `prop_mis` of its 14 fits (regenerating it left every +earlier value unchanged), and a fit of 100 tetraploids of which 8 have no +reads. With updog's prior, bias and overdispersion, the record `call` writes: + +| Fit | Prior | updog `prop_mis` | `PMC` | Largest difference of `MPP` from `maxpostprob` | +|---|---|---|---|---| +| Random-mating tetraploids | norm | 0.11309 | 0.1131 | 0.0003 | +| One hexaploid family | norm | 0.17282 | 0.1728 | 0.0001 | +| Inbred diploids | norm | 0.00001 | 0 | 0.0001 | +| Rare allele, tetraploids | norm | 0.00040 | 0.0004 | 0.0005 | +| Tetraploids, bias 0.6 | norm, hw | 0.14372, 0.14516 | 0.1437, 0.1451 | 0.0006, 0.0004 | +| Tetraploids, bias 1.9 | norm, hw | 0.16657, 0.16682 | 0.1666, 0.1668 | 0.0001, 0.0009 | +| Tetraploids, no bias, bias fitted | norm, hw | 0.10257, 0.09483 | 0.1025, 0.0948 | 0.0010, 0.0010 | +| Diploids, bias 0.5 | norm, hw | 0.00001, 0.00001 | 0, 0 | 0.0001, 0.0001 | +| Hexaploid family, bias 1.5 | norm, hw | 0.36025, 0.40599 | 0.3602, 0.4059 | 0.0003, 0.0007 | + +In the fit with 8 samples without reads, updog's `prop_mis` is 0.15484 with +them given a depth of 0 (each at 0.4046, the largest probability of the +prior) and 0.11653 with them given as missing, and `call` writes 0.1165. + +*Against known genotypes.* On the replicate tables above, called without +filters, `PMC` and the share of samples whose most probable dosage is wrong +were compared marker by marker. Averaged over a table, `PMC` minus that +share is the calibration gap of the table above with its sign turned. +Different scripts computed the two, one from `GQ` and one from `GP`, and +they agree. Mean over the draws, in points, `--prior hwe`, where negative is +too low: + +| Table | `off` | `auto` | `fit` | Mean difference at one marker, `auto` | +|---|---|---|---|---| +| 4x random mating | +0.1 | +0.1 | −0.6 | 1.7 | +| 4x random mating, biased | −2.0 | +0.2 | −1.2 | 3.2 | +| 4x F1 | +0.3 | −0.3 | −3.0 | 2.3 | +| 4x two subpopulations | −0.5 | −1.2 | −3.0 | 2.5 | +| 4x inbred lines | −0.4 | −0.4 | −14.4 | 1.0 | +| 6x random mating | +0.3 | +0.3 | −4.2 | 2.7 | +| 6x random mating, biased | −4.7 | −3.8 | −5.1 | 7.0 | +| updog 4x random mating | −0.3 | −0.3 | −0.4 | 1.1 | +| updog 4x random mating, biased | −5.4 | +0.5 | −0.5 | 2.0 | +| updog 6x random mating, biased | −10.9 | −0.1 | −2.3 | 6.5 | +| updog 4x F1, biased | −8.7 | −11.1 | −14.7 | 13.7 | +| polyRAD 4x outbred | +0.2 | +0.2 | −0.5 | 1.8 | +| polyRAD 6x outbred | 0.0 | 0.0 | −1.5 | 2.7 | +| polyRAD 4x outbred, low depth | −0.1 | −0.1 | −0.5 | 2.2 | +| polyRAD 2x, 4x and 6x, three groups, 8 reads | −0.8 | −0.8 | −2.2 | 2.5 | + +updog's `prop_mis` on the tables it ran on (one draw each), beside +`call` with `auto` on the same draw: + +| Table | `call`, hwe | `call`, norm | updog `norm` | +|---|---|---|---| +| updog 4x random mating | −0.3 | | −0.6 | +| updog 4x random mating, biased | +0.1 | | −0.7 | +| updog 4x random mating, biased, low depth | +0.3 | | −2.1 | +| updog 6x random mating, biased | −1.5 | | −6.2 | +| updog 4x F1, biased | −13.4 | −4.7 | −6.9 | +| polyRAD 4x outbred | +0.9 | | −5.6 | +| polyRAD 4x outbred, low depth | −0.6 | | −8.5 | +| polyRAD 6x outbred | −0.4 | | −22.8 | +| polyRAD 4x, inbreeding 0.5 | −1.9 | −0.7 | −3.8 | +| Potato, 85 clones, against the array | −2.3 | | −2.3 | + +- Where the model fits the samples, `PMC` is the share of wrong genotypes to + within 0.4 points with `off` and `auto`. For one marker of 150 samples it + is off by 1.7 to 2.7 points on average. +- Where the model does not fit, `PMC` is too low, as with a bias in the data + and none fitted (2 to 11 points), a bias fitted on inbred lines (14) and a + biased F1 family under `hwe` (9 to 15). +- updog's value is further from the truth than `call`'s on every polyploid + table but the F1 one under `hwe`. On polyRAD's tables its overdispersion, + fitted at every marker, read less scatter than the simulation put in. It + had a median of 41, 95 and 248 on `call`'s scale in the three tables, for + a simulated 20, and in the hexaploid table 23% of markers got a bias + beyond 1.5-fold where the simulation had none. +- `PMC` depends on ploidy and depth as much as on the marker. Of the 4x + random-mating markers at 20 to 60 reads, 17% have a `PMC` under 0.05, and + of the 6x ones under 1%. Nothing is wrong with the others. + +*As a marker filter.* The test uses the two potato panels of the filter +study above, called again with the code as committed (`--allele-bias fit`, +which reproduces the figures of that study). A marker is bad when fewer than +80% of its genotypes have the array's dosage. Markers removed, share of them +bad, share of the bad removed, and agreement of the genotypes that pass the +default filters at the markers left (91.3% and 93.3% with no marker filter): + +| Marker removed when | First panel | Second panel | +|---|---|---| +| `PMC` of 0.05 or more | 356, 52%, 57%, 92.7% | 290, 32%, 40%, 94.2% | +| `PMC` of 0.1 or more | 164, 71%, 36%, 91.8% | 86, 51%, 19%, 93.6% | +| `PMC` of 0.2 or more | 53, 94%, 15%, 91.3% | 17, 94%, 7%, 93.4% | +| Call rate under 0.9 | 555, 45%, 77%, 94.1% | 530, 28%, 65%, 95.0% | +| As many markers as the row above, by highest `PMC` | 555, 40%, 69%, 93.2% | 530, 25%, 58%, 94.7% | + +- The call rate under the default filters finds more of the bad markers than + `PMC` at an equal number removed. On both panels every marker with a + `PMC` of 0.05 or more also has a call rate under 0.9. +- With `auto` the figures are close. On the first panel a `PMC` of 0.05 or + more removes 383 markers and leaves 92.2% agreement (90.9% with no marker + filter), and a call rate under 0.9 removes 590 and leaves 93.8%. On the + second they give 555 and 94.1% (from 93.3%) against 811 and 94.8%. +- On the 85 potato clones, with `auto`, a `PMC` under 0.05 keeps 1,438 of + 1,544 markers and raises agreement of all genotypes from 96.4% to 97.0%, + while updog's `prop_mis` under 0.05 keeps 1,466 of its markers and raises + its agreement from 96.7% to 97.4%. + +*Read by BIGr.* BIGr failed to install here (its dependency vcfR is not +available), so `filterVCF` itself never ran. Its two functions that read +INFO, copied from its source and run on the potato VCF of `call`, find +`PMC` and `BIAS` among the keys of the first record and read all 1,544 +values of each as written. `filter.PMC = 0.05` keeps 1,438 records. `MPP` +taken by its place in FORMAT, as vcfR's `extract.gt` takes it, is a number in +131,224 cells and missing in the 16 without reads, and `filter.MPP = 0.95` +would set 2,687 of 126,058 genotypes to missing. In an `all` VCF whose first +record has more than two alleles, `filterVCF` would not find the two keys. + +**The structure prior (`--prior structure`).** polyRAD's `IteratePopStruct` +gives each sample its own allele frequencies. It runs a principal component +analysis of the genotype estimates of all markers, regresses every allele on +the components, and repeats both with the calling until they settle. A port +would need a matrix library and would replace the marker-by-marker design +with an iteration over the whole table. `call` instead finds, once, the +samples most like each sample (`find_neighbours`) and then calls each record +separately with Hardy-Weinberg proportions at the allele frequency among a +sample's neighbours. This is a second run of the record's iteration +(`_iterate_record`), started from the result of the first. Likeness is the +sum of squared +differences between two samples' expected allele shares over the markers, +from a first pass with the `hwe` prior, which ranks pairs as a genomic +relationship matrix would. The number of neighbours (0, 5, 10, 20, 40 or 80) +is the one whose neighbours, found on one half of the markers, best predict +the samples' shares at the other half. A choice of 0 is the `hwe` prior. + +The test tables are tetraploid, with 300 samples and 500 markers, read +counts from polyRAD's `SimAlleleDepth` (overdispersion 20, contamination and +error rates 0.001) and negative-binomial depths of mean 8, 25 or 150. The +populations are three groups of 100 with Balding-Nichols allele frequencies +at three values of FST, a cline in which every sample is a different mixture +of two sources (FST 0.3 between them), fifteen full-sib families of twenty, +and one random-mating population. The last two rows are the 85 potato clones +above with every read kept with probability 10/366 or 25/366, scored against +the array genotypes. polyRAD got the simulated overdispersion (75 for the +potato rows), and `call` got that value and a background rate of 0.001 +(0.003). Share of genotypes with reads whose most probable dosage is the true +one: + +| Population | Mean depth | polyRAD `IterateHWE` | polyRAD `IteratePopStruct` | `call` hwe | `call` structure | Neighbours chosen | +|---|---|---|---|---|---|---| +| Three groups, FST 0.05 | 8 | 71.2% | 71.3% | 71.1% | 71.9% | 80 | +| Three groups, FST 0.15 | 8 | 72.3% | 74.7% | 72.3% | 75.1% | 80 | +| Three groups, FST 0.15 | 25 | 83.2% | 84.6% | 83.4% | 84.8% | 80 | +| Three groups, FST 0.15 | 150 | 89.5% | 90.3% | 89.7% | 90.5% | 80 | +| Three groups, FST 0.30 | 8 | 73.9% | 79.0% | 73.9% | 79.4% | 80 | +| Three groups, FST 0.30 | 25 | 84.7% | 87.0% | 84.8% | 87.3% | 80 | +| Cline | 8 | 75.4% | 76.7% | 75.4% | 76.7% | 80 | +| Cline | 25 | 85.1% | 85.7% | 85.3% | 85.8% | 80 | +| Fifteen families | 8 | 70.8% | 73.1% | 70.8% | 73.8% | 20 | +| Fifteen families | 25 | 81.9% | 83.4% | 82.0% | 83.9% | 20 | +| One population | 8 | 70.5% | 67.1% | 70.5% | 70.5% | 0 | +| Potato families, thinned | 10 | 71.0% | 71.1% | 70.9% | 71.8% | 10 | +| Potato families, thinned | 25 | 80.8% | 81.4% | 80.7% | 81.4% | 10 | + +- The structure prior calls as many genotypes correctly as `IteratePopStruct` + on every table or more, by up to 0.7 points. The cline is the case + principal components describe best, and the two are level there. +- On the six tables with three groups it equals what calling each group + separately with the `hwe` prior gives (72.0%, 75.1%, 84.8%, 90.5%, 79.4% + and 87.3%), which is the most a prior can gain from knowing the groups. +- With one population `IteratePopStruct` kept 51 components and lost 3.4 + points. The structure prior chose 0 neighbours. +- The gain over `hwe` falls with depth. It is 0.8 to 5.5 points at 8 reads, + 0.5 to 2.5 at 25, and 0.8 at 150. At full depth the potato clones go from + 92.3% to 92.3% (10 neighbours), and the two alfalfa populations from + 90.87% and 90.04% agreement with the published dosages to 90.89% and + 89.98% (20 neighbours each). +- With the read model estimated instead of given, three of the tables, run + again, gave the same shares (79.4%, 73.8% and 71.8%). + +A test bench with one likelihood and one first pass (not in the repository, +built on numpy) compared three ways of using the same relationships. They +were the nearest samples as above, principal components with their number +chosen on held-out markers, and a ridge predictor from the whole relationship +matrix with its shrinkage chosen the same way. On the group and cline tables +the three were within 0.3 points of each other. On the families the nearest +samples gave 73.8% and 83.9%, the components 73.1% and 83.8%, and the ridge +predictor 72.7% and 83.4%. On the thinned potato counts the ridge predictor +failed (33.9% and 56.0%, with the least shrinkage offered), and the other two +were level (71.9% and 81.3% against 71.7% and 81.4%). All three left a +sample's own value out of its own prediction. + +The number of neighbours, components or the shrinkage has to be chosen on +markers that took no part in measuring the relationships. Chosen on the same +markers, the bench took the least shrinkage and the most components on offer +(44.1% and 72.9% on the first table, against 72.3% for `hwe`), because each +marker is then one of the features its own prediction uses. + +On the benchmark table of "Time and memory" the structure prior took 30 s on +one worker against 23 s for `hwe`, and 9 s against 6.5 s on four, with 204 MB +against 159 MB. The read-model estimate is 18 s of both. With the read model +given it took 10.6 s against 4.0 s on one worker. That time covers the first +pass over the markers that the samples are compared on, the comparison, and +the second run of every record. (An earlier version of this paragraph gave +34.6 s against 33.4 s for the whole run, which understated the difference. +That code, measured again with the read model given, took 12.5 s against +5.1 s.) On the simulated tables, 500 markers and 300 samples, it took 4 to +5 s against 2 s on four workers with the read model given. + +No accuracy test covers ploidies other than four, samples of mixed ploidy, +or read counts from other simulators. A run with diploid and tetraploid +samples and all three VCF types completes, and `bcftools view` reads its +output. The prior does not combine with the allelic bias or the selfing +rate, and `call` refuses both combinations. + +**Family priors, measured and not built.** updog's `f1` model and polyRAD's +`PipelineMapping2Parents` give the offspring of a cross the dosages their +parents' genotypes can segregate into. A test bench (not in the repository) +measured whether `call` needs such a prior. It uses the `call` likelihood +and, per family and marker, the pair of parental dosages that makes the +reads most probable, under polysomic inheritance without double reduction +and with 1% of the prior spread evenly over the dosages for off-types. + +One biparental population with both parents among the samples is the common +case. The two alfalfa populations have that. Their reads were also thinned, +parents included, and the calls of the 974 and 986 published markers compared +with the published dosages, which come from the full depth: + +| Population | Mean depth | hwe | norm | structure | F1, both parents' reads | F1, one parent's reads | F1, offspring only | +|---|---|---|---|---|---|---|---| +| I195 × J432 | 174 | 90.87% | 90.11% | 90.85% | 91.01% | 90.82% | 90.64% | +| I195 × J432 | 25 | 84.53% | 84.08% | 84.60% | 85.14% | 84.95% | 84.79% | +| I195 × J432 | 10 | 75.72% | 76.04% | 75.74% | 76.73% | 76.63% | 76.58% | +| E115 × F628 | 424 | 90.04% | 89.80% | 89.98% | 90.33% | 90.26% | 90.15% | +| E115 × F628 | 25 | 82.23% | 82.25% | 82.25% | 82.84% | 82.78% | 82.71% | +| E115 × F628 | 10 | 74.15% | 74.68% | 74.33% | 75.05% | 74.99% | 74.94% | + +With both parents' reads the F1 prior is 0.1 to 0.3 points above the best of +the three priors `call` has at full depth, 0.5 to 0.6 at 25 reads and 0.4 to +0.7 at 10. The parents' reads account for 0.1 to 0.4 of that. The rest comes +from the segregation shape, which a family of this size shows by itself. + +Many small families are the other case. On the potato half-diallel (ten +families of 1 to 20) and on fifteen simulated full-sib families of 20: + +| Data | Mean depth | hwe | structure | F1, parents estimated from offspring | F1, parental genotypes given | +|---|---|---|---|---|---| +| Simulated families | 8 | 70.8% | 73.8% | 75.0% | | +| Simulated families | 25 | 82.0% | 83.9% | 84.9% | | +| Potato, thinned | 10 | 70.9% | 71.8% | 71.8% | 76.1% | +| Potato, thinned | 25 | 80.7% | 81.4% | 81.1% | 84.1% | +| Potato, full depth | about 300 | 92.3% | 92.3% | 92.3% | 93.4% | + +Only parental genotypes known from elsewhere help much here. Those given were +the article's phased parents, from array data and linkage analysis, which are +better than a parent's own reads would give. That needs a pedigree, which is +often not available, and is not built. A single biparental population gains +under a point from an F1 prior, so that prior is not built either. + +A simulation of the biparental case also tried the parent models of the +other two tools, polyRAD's `PipelineMapping2Parents` (F1, no linkage, given +the overdispersion `call` estimated) and updog's `f1` model with both +parents named, against updog's `norm`. The tables are tetraploid, with 200 +progeny and both parents and 300 markers, and their genotypes and reads come +from polyRAD's simulators (`SimGenotypesMapping` and `SimAlleleDepth`, +overdispersion 20, no bias) or updog's (`rgeno` and `rflexdog`, sequencing +error 0.5%, overdispersion 49–499 and a bias of exp(N(0, 0.5)) per marker). +The parents were sequenced like the progeny. +Share of the progeny's genotypes called correctly: + +| Reads from | Mean depth | `call` hwe | `call` norm | `call` norm + bias | F1 prior on `call`'s likelihood | polyRAD mapping | updog `norm` | updog `f1` | +|---|---|---|---|---|---|---|---|---| +| polyRAD | 8 | 74.9% | 77.1% | | 77.5% | 77.3% | 74.0% | 75.4% | +| polyRAD | 25 | 84.5% | 85.7% | | 86.0% | 85.8% | 80.6% | 82.3% | +| polyRAD | 150 | 90.3% | 91.2% | | 91.3% | 91.2% | 88.9% | 89.5% | +| updog | 8 | 69.3% | 70.1% | 75.3% | 70.3% | 70.6% | 75.5% | 74.4% | +| updog | 25 | 79.4% | 79.1% | 87.8% | 80.1% | 80.0% | 87.6% | 87.6% | +| updog | 150 | 85.2% | 84.5% | 97.1% | 85.5% | 84.9% | 97.1% | 97.3% | + +On unbiased reads the `norm` prior is within 0.4 points of the F1 prior and +of polyRAD's mapping prior, because it fits the narrow spread of one family's +dosages. On biased reads `call` with `norm` and the bias fitted is within 0.3 +points of updog's `norm` and `f1` at 25 and 150 reads. The structure prior +gave the same shares as `hwe` on all six tables. polyRAD's mapping prior left +13–16% of the genotypes of the biased tables uncalled. + +With 10% of the progeny replaced by unrelated samples (25 reads), true +progeny and off-types apart: + +| Reads from | Caller | True progeny | Off-types | +|---|---|---|---| +| polyRAD | `call` hwe | 84.6% | 81.3% | +| polyRAD | `call` norm | 85.8% | 79.4% | +| polyRAD | F1 prior on `call`'s likelihood | 86.2% | 76.2% | +| polyRAD | polyRAD `IterateHWE` | 84.4% | 81.4% | +| polyRAD | polyRAD mapping | 86.1% | 70.8% | +| polyRAD | updog `norm` | 79.9% | 78.9% | +| polyRAD | updog `f1` | 82.4% | 75.6% | +| updog | `call` norm + bias | 87.6% | 82.0% | +| updog | updog `norm` | 87.6% | 82.6% | +| updog | updog `f1` | 87.3% | 79.0% | +| updog | polyRAD mapping | 78.1% | 65.6% | + +A family prior costs 3 to 11 points on the samples that are not of the +family, because it leaves them only the family's dosage classes, and gains +at most 2.5 points on those that are. + +The same parent models ran on the alfalfa populations, with the parents +named and polyRAD given the overdispersion `call` estimated. Agreement with +the published dosages, which updog itself called from the full depth. For +polyRAD's mapping prior the brackets give the share of genotypes it called, +and its agreement is among those: + +| Population | Mean depth | `call` hwe | `call` norm | `call` norm + bias | polyRAD `IterateHWE` | polyRAD mapping | updog `norm` | updog `f1` | +|---|---|---|---|---|---|---|---|---| +| I195 × J432 | 174 | 90.87% | 90.11% | 92.27% | 90.78% | 90.83% (92.7%) | 92.98% | 93.29% | +| I195 × J432 | 25 | 84.53% | 84.08% | 85.77% | 84.44% | 85.12% (90.0%) | 86.26% | 86.61% | +| I195 × J432 | 10 | 75.72% | 76.04% | 76.03% | 75.52% | 77.02% (87.5%) | 75.87% | 75.56% | +| E115 × F628 | 424 | 90.04% | 89.80% | 92.00% | 89.93% | 90.42% (93.7%) | 92.68% | 93.06% | +| E115 × F628 | 25 | 82.23% | 82.25% | 84.36% | 81.81% | 83.23% (90.9%) | 84.53% | 84.39% | + +E115 × F628 at 10 reads is missing because the run reached its time limit +twice before that row. updog's `f1` model is 0.3 to 0.4 points above its +`norm` at full depth and, in the first family, at 25 reads. In the second it +is 0.1 below at 25 reads, and in the first 0.3 below at 10. polyRAD's mapping +prior is 0.05 to 1.5 points above its `IterateHWE` on the genotypes it calls +and leaves 6 to 13% uncalled. `call` with `norm` and the bias is 0.2 to 0.7 +points below updog's `norm` at full depth and at 25 reads and level with it +at 10. The truth here favours updog, whose calls it is. + +**Priors on haplotype records.** Neither polyRAD's paper nor its package +gives accuracy figures for multi-allele (haplotype) calls. The paper scores +two-allele SNPs with array truth, and the package has import examples and +small demonstration datasets without expected genotypes. On polyRAD's own +DArTag example (`inst/extdata/DArTag_example.csv`, two markers with three and +five haplotypes, four tetraploid samples) `call --vcf haplotype` with +polyRAD's settings gives polyRAD's genotype for all eight sample-by-marker +genotypes. + +The accuracy test used microhaplotype tables from polyRAD's simulators +(`SimGenotypes`, `SimGenotypesMapping` and `SimAlleleDepth`, with +overdispersion 20 and contamination and error rates 0.001). They have 300 +markers of 3 to 6 alleles and 300 samples, as one population, three groups +(FST 0.15) or fifteen full-sib families of 20. polyRAD got the +overdispersion, and `call` that and a background rate of 0.001. Share of +genotypes with reads in which the dosage of every allele is right, without +filters: + +| Ploidy | Population | Mean depth | polyRAD `IterateHWE` | polyRAD `IteratePopStruct` | `call` hwe | `call` norm | `call` structure | `call` uniform | +|---|---|---|---|---|---|---|---|---| +| 4 | One population | 25 | 75.0% | stopped with an error | 74.6% | 74.6% | 74.6% (0 neighbours) | 70.9% | +| 4 | Three groups | 8 | 55.2% | 58.9% | 55.1% | 56.8% | 59.3% | 53.6% | +| 4 | Three groups | 25 | 75.1% | 77.0% | 75.0% | 75.5% | 77.0% | 73.5% | +| 4 | Three groups | 150 | 85.8% | 86.8% | 85.7% | 85.8% | 86.7% | 84.5% | +| 4 | Families | 8 | 54.7% | 56.8% | 54.7% | 54.8% | 58.6% | 48.5% | +| 4 | Families | 25 | 75.0% | 76.8% | 74.6% | 74.9% | 77.1% | 71.0% | +| 2 | Three groups | 8 | 92.5% | 93.0% | 92.6% | 92.7% | 93.0% | 92.2% | +| 2 | Families | 8 | 91.8% | 92.3% | 91.8% | 92.1% | 92.6% | 90.9% | + +- `hwe` is within 0.4 points of polyRAD's HWE prior, as it should be. +- `structure` is never below `hwe` and gains 1 to 4 points in tetraploids + with groups or families. It is within 0.1 points of `IteratePopStruct` or + above it on every table, by 1.8 points on the low-depth families. With one + population it chooses 0 neighbours. +- `norm` is never below `hwe` and gains at most 1.7 points. It takes two to + five times as long, and 2 and 11 of the 300 records of the two low-depth + tetraploid tables reached the iteration limit. +- `uniform` is the lowest on every table, by up to 6 points. + +With the default filter (`--min-marginal-posterior 0.9`), 94 to 99% of the +genotypes kept are right under each of `hwe`, `norm` and `structure`. The +share kept at this overdispersion is 4 to 37% in tetraploids and 74 to 83% in +diploids, and is 3 to 8 points higher with `structure` than with `hwe` where +there are groups or families. + +The tests do not cover read counts from other simulators, real haplotype +data with known genotypes, or hexaploids. + +**Off-target records (`--vcf all`).** The same eight tables give truth for +the off-target records, which no earlier comparison scored. Every haplotype +other than Ref and Alt carries one substitution of its own, so the true +dosage of a substitution is that of its haplotype. `call --vcf all` ran with +its defaults, so it estimated the overdispersion (18 to 20 where 20 was +simulated, and 30 on the tetraploid families at 8 reads) and the background +rate (0.0002 to 0.001). Share of genotypes with reads called correctly, for +target and off-target records, without filters, and for `hwe` with the +default filters (`--min-dp 10`, `--min-gq 10`), the share of the off-target +genotypes kept that are right and the share kept: + +| Ploidy | Population | Mean depth | Off-target records | hwe, target and off-target | structure, target and off-target | norm, target and off-target | hwe with default filters, right and kept | +|---|---|---|---|---|---|---|---| +| 4 | One population | 25 | 760 | 86.8%, 88.2% | 86.8%, 88.2% | 86.7%, 88.2% | 97.8%, 61% | +| 4 | Three groups | 8 | 758 | 78.4%, 80.5% | 80.6%, 82.2% | 79.2%, 81.0% | 96.8%, 22% | +| 4 | Three groups | 25 | 759 | 87.0%, 88.6% | 88.2%, 89.6% | 87.4%, 88.9% | 98.1%, 62% | +| 4 | Three groups | 150 | 760 | 92.2%, 93.1% | 92.9%, 93.7% | 92.4%, 93.2% | 98.8%, 79% | +| 4 | Families | 8 | 758 | 77.8%, 79.7% | 79.7%, 81.3% | 77.8%, 79.8% | 96.2%, 22% | +| 4 | Families | 25 | 760 | 86.9%, 88.2% | 88.0%, 89.2% | 86.9%, 88.3% | 97.8%, 61% | +| 2 | Three groups | 8 | 747 | 96.5%, 96.9% | 96.7%, 97.1% | 96.6%, 97.0% | 99.8%, 33% | +| 2 | Families | 8 | 750 | 96.4%, 96.6% | 96.8%, 96.9% | 96.5%, 96.7% | 99.8%, 33% | + +- Off-target records are called as well as target records, with 0.2 to 2.1 + points more of their genotypes right. Their alternate alleles are a little + rarer in these tables (mean frequency 0.14 to 0.23, against 0.25 for the + target's). +- The priors rank as they do on target records. `structure` gains up to 2.2 + points on targets and 1.7 on off-target records where there are groups or + families and nothing with one population, and `norm` is within 0.1 points + below and 0.8 above `hwe`. +- With the default filters 96 to 99.8% of the off-target genotypes kept are + right. + +**Off-target substitutions shared by haplotypes.** In the tables above a +substitution is on one haplotype only and never on the Alt tag. In real tags +several haplotypes can carry the same one, and its record then adds their +reads together. The same reads and true haplotype dosages got other stand-in +tags. One substitution is on two of the haplotypes with the reference base +at the target, a second is on two as well, and a third is on a +haplotype that has the target's alternate base, so that the target's record +adds two haplotypes too. The true dosage of a substitution is the sum over the +haplotypes that carry it. Of the 760 off-target records 311 add two +haplotypes, and 149 of the 300 target records do. Share of genotypes with +reads called correctly, as in the table above: + +| Ploidy | Population | Mean depth | hwe, target and off-target | structure, target and off-target | norm, target and off-target | hwe with default filters, target right and kept | hwe with default filters, off-target right and kept | +|---|---|---|---|---|---|---|---| +| 4 | One population | 25 | 82.8%, 85.4% | 82.8%, 85.4% | 82.8%, 85.4% | 97.0%, 47% | 97.4%, 53% | +| 4 | Three groups | 8 | 73.4%, 76.5% | 75.9%, 78.8% | 74.2%, 77.3% | 96.2%, 16% | 96.6%, 19% | +| 4 | Three groups | 25 | 83.9%, 86.1% | 85.2%, 87.3% | 84.3%, 86.5% | 97.6%, 50% | 97.8%, 56% | +| 4 | Three groups | 150 | 90.2%, 91.6% | 91.0%, 92.2% | 90.3%, 91.7% | 98.3%, 69% | 98.6%, 73% | +| 4 | Families | 8 | 71.9%, 75.7% | 74.9%, 77.9% | 72.1%, 75.8% | 95.9%, 15% | 96.0%, 18% | +| 4 | Families | 25 | 83.0%, 85.5% | 84.6%, 86.8% | 83.0%, 85.6% | 97.0%, 47% | 97.4%, 54% | +| 2 | Three groups | 8 | 95.8%, 96.3% | 96.0%, 96.5% | 95.8%, 96.3% | 99.8%, 33% | 99.8%, 33% | +| 2 | Families | 8 | 95.4%, 96.0% | 96.0%, 96.4% | 95.5%, 96.0% | 99.7%, 33% | 99.7%, 33% | + +- A record that adds the reads of several haplotypes is the same call as a + two-allele marker with those reads. The reads of the carriers and of the + other haplotypes were added up outside `call` and called as two-row markers + under the same overdispersion and background rate. All 2,538,900 genotypes + of the eight tables are the same as in the `all` VCF. +- Fewer genotypes are right than in the tables with one haplotype per + substitution (by 1 to 6 points for tetraploids, under 1 for diploids) + because the added alleles are more frequent, with a mean of 0.33 against + 0.25 at targets and 0.29 against 0.21 at off-target records. More + genotypes then have a dosage in the middle, where neighbouring dosages are + harder to tell apart. +- Off-target records are still called as well as target records, the priors + rank as before, and with the default filters 96 to 99.8% of the genotypes + kept are right. + +**Hexaploid microhaplotype tables.** The haplotype and off-target tables +above are diploid and tetraploid. The same simulator drew five hexaploid +tables (300 samples, 300 markers of three to six haplotypes, overdispersion +20), scored the same three ways. A haplotype genotype is right when the +dosage of every allele is right: + +| Population, mean depth | polyRAD `IterateHWE` | polyRAD `IteratePopStruct` | `call` hwe | `call` norm | `call` structure | +|---|---|---|---|---|---| +| One population, 25 | 47.4% | 44.7% | 47.3% | 47.1% | 47.3% | +| Three groups, 8 | 31.0% | 37.5% | 30.9% | 34.5% | 37.8% | +| Three groups, 25 | 49.2% | 53.5% | 49.1% | 51.0% | 53.6% | +| Three groups, 150 | 62.5% | 65.6% | 62.6% | 63.6% | 65.7% | +| Families, 25 | 47.3% | 49.1% | 47.2% | 47.4% | 51.2% | + +Target and off-target records of `--vcf all`, share of genotypes right +without filters, with one haplotype per substitution and with substitutions +shared by haplotypes, and for `hwe` with the default filters, the share of +the target and off-target genotypes kept that are right and the share kept: + +| Population, mean depth | hwe, target and off-target | structure | norm | Shared, hwe | Shared, structure | Shared, norm | hwe with default filters, right and kept | Shared, default filters | +|---|---|---|---|---|---|---|---|---| +| One population, 25 | 74.2%, 76.0% | 74.2%, 76.0% | 74.1%, 75.9% | 67.4%, 71.3% | 67.4%, 71.3% | 67.3%, 71.2% | 96.3%, 36% and 96.0%, 39% | 95.8%, 22% and 95.8%, 29% | +| Three groups, 8 | 65.7%, 68.4% | 69.1%, 71.3% | 66.8%, 69.3% | 57.7%, 62.9% | 62.5%, 66.6% | 59.4%, 64.2% | 95.0%, 14% and 94.7%, 16% | 94.2%, 8% and 94.4%, 12% | +| Three groups, 25 | 74.8%, 77.2% | 77.3%, 79.4% | 75.8%, 78.0% | 68.8%, 72.9% | 71.9%, 75.6% | 70.0%, 73.9% | 97.1%, 40% and 97.1%, 44% | 96.5%, 28% and 96.7%, 36% | +| Three groups, 150 | 81.2%, 83.3% | 83.0%, 84.8% | 81.8%, 83.8% | 76.7%, 80.0% | 78.9%, 81.8% | 77.5%, 80.6% | 98.5%, 55% and 98.6%, 60% | 98.2%, 44% and 98.4%, 52% | +| Families, 25 | 74.2%, 76.1% | 76.0%, 77.9% | 74.2%, 76.1% | 67.3%, 71.4% | 69.9%, 73.6% | 67.4%, 71.4% | 96.0%, 37% and 96.0%, 40% | 95.2%, 24% and 95.5%, 31% | + +- In hexaploids the priors rank as in tetraploids. `call` with the HWE prior + has polyRAD's share to 0.1 points on every table, and `structure` equals or + passes polyRAD's population-structure model (by 2.1 points for families). + Where there is one population polyRAD's structure model loses 2.7 points to + its own HWE model and `structure` loses nothing, because it found no + useful neighbours. +- Fewer genotypes are right than in tetraploids at the same depth (74% of + target genotypes at 25 reads against 87%), as seven dosage classes leave + less room between neighbours than five. With the default filters 94 to + 98.6% of the genotypes kept are right, and the filters keep fewer, 36 to + 44% at 25 reads against 61%. +- At 8 reads the estimate put the overdispersion at 100 and 50 where the + simulation used 20, while from 25 reads on it is 20. At that depth + hexaploid counts hold too little to tell scatter from dosage. + +The delta smelt comparison below tests real off-target genotypes. + +**The polyRAD paper's own benchmark.** Clark et al. (2019) tested polyRAD on +read depths simulated from the real SNP-array and GoldenGate genotypes of +diversity panels and mapping populations, with reads drawn binomially at a +mean depth of about 10 (15 to 21% of the cells have no reads). The data, the +scripts and the results are in the Illinois Data Bank +(doi:10.13012/B2IDB-9729830_V2), with the error of every marker under each +model of polyRAD 0.4-0 and under EBG and GATK-style calls. `call` ran on the +same read depths and was scored as the paper did. The score is the root mean +squared error between true and estimated copy number at each marker, over +the cells with reads, for the posterior mean ("continuous") and for the most +probable genotype ("discrete"), averaged over the markers. Lower is better. +`call` estimated an overdispersion of 500, the highest it tries, as it should +for binomial reads (100 on the soybean set). Continuous and discrete error: + +| Data set | Samples × markers | polyRAD HWE (archive) | `call` hwe | polyRAD PopStruct (archive) | `call` structure | `call` norm | +|---|---|---|---|---|---|---| +| *Miscanthus sinensis* diversity panel, 2x | 565 × 395 | 0.154, 0.175 | 0.154, 0.175 | 0.140, 0.161 | 0.140, 0.162 | 0.150, 0.170 | +| The same, simulated tetraploids | 565 × 395 | 0.420, 0.464 | 0.421, 0.464 | 0.338, 0.384 | 0.326, 0.372 | 0.398, 0.457 | +| Apple diversity panel, 2x | 96 × 3,704 | 0.173, 0.197 | 0.173, 0.197 | 0.176, 0.200 | 0.172, 0.196 | 0.173, 0.196 | +| Potato diversity panel, 4x | 221 × 3,763 | 0.439, 0.499 | 0.439, 0.499 | 0.427, 0.491 | 0.428, 0.488 | 0.438, 0.502 | +| *Glycine soja*, 2x, selfing rate 0.95 | 1,179 × 2,957 | 0.037, 0.034 | 0.037, 0.034 | 0.038, 0.037 | not with a selfing rate | | + +Two sets are F1 populations, which polyRAD called with its mapping prior and +the parents named, while `call` has no family prior: + +| Data set | Samples × markers scored | polyRAD mapping (archive) | `call` hwe | `call` norm | +|---|---|---|---|---| +| *Miscanthus* F1, 2x | 85 × 185 | 0.186, 0.205 | 0.200, 0.226 | 0.186, 0.213 | +| Potato F1, 4x | 238 × 2,306 | 0.391, 0.447 | 0.409, 0.478 | 0.385, 0.446 | + +- `call` with the `hwe` prior has polyRAD's published HWE error to the third + decimal on all five diversity sets, soybean with `--selfing-rate 0.95`. +- `structure` equals polyRAD's population-structure model on *Miscanthus* and + potato and is better on the simulated tetraploids (0.326 against 0.338) and + on apple, where polyRAD's model was worse than its HWE. +- `norm`, told nothing about the cross, matches polyRAD's mapping prior on + the *Miscanthus* F1 and is 0.006 below it on the potato F1 in continuous + error. In discrete error it is within 0.008. +- polyRAD's models that use linkage between markers are better than `call` on + five of the seven sets, by 0.003 to 0.044 in continuous error. They reach + 0.137 on *Miscanthus* (`call` at its best 0.140), 0.412 on potato (0.428), + 0.164 on apple (0.172), and 0.170 and 0.340 on the two F1 populations + (0.186 and 0.385). `call` treats markers independently. On the simulated + tetraploids `structure` is below polyRAD's structure-and-linkage model + (0.326 against 0.335), and on soybean linkage made polyRAD worse. +- Without the selfing rate the soybean errors are 0.157 with `hwe`, 0.124 with + `norm` and 0.110 with `structure`, against 0.037 with it, so for inbred + samples the selfing rate has to be given. +- EBG's HWE model, the other caller in the archive with a population prior, + has the discrete error of `call` with `hwe` (0.464 against 0.464 on the + simulated tetraploids, 0.500 against 0.499 on potato), as two HWE callers + should on binomial reads. The GATK-style calls, which have no population + prior, are far above both on the polyploid sets (0.523 and 0.696). +- With polyRAD's present defaults (overdispersion 9) `call` is a little + worse on these binomial reads, 0.458 against 0.421 on the simulated + tetraploids. + +The archive's eighth set is a simulated allohexaploid (soybean genotypes on +one of three subgenomes, so 0 to 2 copies of the allele out of 6, at a mean +depth of 30). polyRAD modelled it as three diploid genomes and has 0.194 and +0.219 with HWE and 0.178 and 0.199 with population structure. `call` has no +such model and called it as an autohexaploid. It has 0.360 and 0.429 with +`hwe` (82.1% of genotypes right), 0.312 and 0.365 with `norm` (87.3%), and +0.336 and 0.402 with `structure`. +Markers that sit on one subgenome of an allopolyploid are outside what `call` +models, because it spreads the prior over all 0 to 6 copies. + +**Delta smelt GT-seq against alignment-based genotypes (diploid).** The +example data of gtseq_microhap (Zenodo, doi:10.5281/zenodo.19069550, +CC-BY-4.0) are 96 delta smelt, 410 loci, 150,000 read pairs per fish, with the +panel's primer file and a VCF of 2,787 biallelic SNPs called from the same +reads by BWA and bcftools on the reference genome. `generate` has been +measured on them (see "External benchmarks on public datasets"), and here +they test `call` on real diploid reads, at targets and at off-target SNPs, +against genotypes from an alignment-based caller. + +Its two primers placed every locus on the genome (GCA_021917145.1), all 410 +once. For the 368 loci with a polymorphic SNP of the comparison VCF in the +first 81 bases, the tag is the genome sequence from the forward primer and +the target is the SNP with the highest minor allele frequency. `generate` on +the R1 reads gave 1,527 allele rows and assigned about 100,000 reads per +fish, some 280 per marker. `call --vcf all --ploidy 2` wrote 368 target +records and 615 off-target records. Of these off-target records, 467 are a +SNP of the comparison VCF with the same two bases (111 are not in it, 36 +have more than two alleles and 1 has other bases). Agreement with the +comparison genotype: + +| Caller | Targets (35,073 genotypes) | Off-target SNPs (44,777) | +|---|---|---| +| `call --prior hwe` (the defaults, with overdispersion 150 and background 0.003) | 99.57% of the 98.8% called | 99.80% of the 99.1% called | +| `call --prior norm` | 99.58% of 98.8% | 99.81% of 99.1% | +| `call --prior structure` (5 neighbours) | 99.58% of 98.8% | 99.80% of 99.1% | +| `call --prior hwe --allele-bias fit` | 99.42% of 98.8% | 99.48% of 99.1% | +| `call` with polyRAD's settings | 99.20% of 98.1% | 99.43% of 98.7% | +| `call --prior hwe`, no filters | 99.41% | 99.70% | +| polyRAD `IterateHWE`, overdispersion 9 | 98.71% | 99.08% | +| polyRAD `IteratePopStruct`, overdispersion 9 | 98.85% | 99.13% | +| polyRAD `IterateHWE`, overdispersion 150 | 99.38% of the 99.7% called | 99.66% | +| updog `norm` | 99.27% | 99.38% | + +Every row uses the same genotypes, those where the sample has reads at the +record and the comparison genotype rests on 10 or more reads. polyRAD and +updog also give a genotype where a sample has no reads, and the scores leave +those out. + +polyRAD and updog got the reference and alternate counts of the 947 +two-allele records and call every genotype, so the row to compare them with is +`call` without filters. + +- On real diploid reads `call` with its defaults agrees with the + alignment-based genotypes more often than polyRAD or updog do on the same + counts, by 0.14 points on targets and 0.32 on off-target SNPs over updog, + and by 0.70 and 0.62 over polyRAD's defaults. +- Off-target SNPs are called as well as targets, here on real reads, and the + simulated tables under "Off-target records" said the same. +- Fitting the bias lowers the agreement, by 0.15 points on targets and 0.3 on + off-target SNPs. updog, which always fits one (median 0.98 here, with 176 + of 946 outside 0.8 to 1.25), is also below `call` without it. On the + potato data fitting the bias raised agreement with the array by 4.3 + points. Whether it helps depends on the data, which is a reason to leave + it off by default. +- polyRAD's population-structure model stopped with an error at an + overdispersion of 150, as it did on the potato data at 75. +- The comparison genotypes come from the same reads, so this measures + agreement between callers and not error against an independent assay. + +**Polyploids pooled from real diploid reads.** No public panel has both a +vendor-style read table and independent dosages for a polyploid at low depth, +so polyploids with known dosages were made from the smelt reads, as Cooke et +al. (2022, Genome Research 32:766) made synthetic polyploids from human +samples. The 96 diploids were divided at random into groups of two +(tetraploids, 48 samples) or three (hexaploids, 32). At each marker every +member gave the same number of reads, drawn without replacement, and the +group's true dosage at a SNP is the sum of its members' alignment-based +genotypes, known where every member's genotype rests on 10 or more reads. +The pooled reads were also thinned to a mean of 50, 25 and 10 per sample and +marker (524 for tetraploids and 744 for hexaploids before thinning). The +reads are real, with their real scatter and without allelic bias beyond what +diploid reads have, but the genotype frequencies are those of pooled +individuals, not of a polyploid population. Share of genotypes with reads +called correctly, targets and off-target SNPs, without filters, as means +over three random groupings (two for the hexaploids at 25 and 10 reads, +where the run reached its time limit), which differ from one another by 0.9 +points at most for `call` without the bias and 1.9 for the other columns +(updog ran on the first grouping only): + +| Ploidy, mean depth | `call --prior hwe` | `call --prior norm` | `call --prior hwe --allele-bias fit` | `call` with polyRAD's settings | polyRAD `IterateHWE`, overdispersion 9 | polyRAD at the overdispersion `call` estimated | updog `norm` | +|---|---|---|---|---|---|---|---| +| 4x, all reads | 98.3%, 98.9% | 98.3%, 98.9% | 97.9%, 98.3% | 93.1%, 94.6% | 93.1%, 94.6% | did not finish | 98.2%, 98.5% | +| 4x, 50 | 90.4%, 95.3% | 90.4%, 95.2% | 89.5%, 94.9% | 85.2%, 90.7% | 85.0%, 90.7% | 90.1%, 95.2% | 89.1%, 94.9% | +| 4x, 25 | 83.2%, 92.0% | 82.9%, 91.9% | 81.7%, 91.4% | 79.8%, 89.2% | 79.4%, 89.1% | 82.8%, 91.8% | 80.9%, 91.0% | +| 4x, 10 | 70.4%, 85.1% | 69.9%, 84.8% | 68.3%, 84.5% | 69.0%, 84.3% | 68.6%, 84.2% | 69.9%, 85.0% | 66.4%, 83.9% | +| 6x, all reads | 96.9%, 98.1% | 96.9%, 98.0% | 96.1%, 96.8% | 71.5%, 80.1% | 71.5%, 80.1% | did not finish | 96.9%, 97.5% | +| 6x, 50 | 77.8%, 88.9% | 77.4%, 88.7% | 70.9%, 86.4% | 65.6%, 82.2% | 65.4%, 82.2% | 77.4%, 88.7% | 68.0%, 84.9% | +| 6x, 25 | 67.1%, 83.1% | 66.6%, 82.8% | 59.6%, 80.5% | 59.8%, 79.5% | 59.3%, 79.3% | 66.6%, 82.7% | 57.1%, 79.1% | +| 6x, 10 | 53.7%, 74.1% | 52.0%, 73.6% | 48.6%, 72.2% | 50.4%, 72.9% | 49.5%, 72.7% | 52.8%, 73.9% | 47.2%, 70.8% | + +- `call` estimated an overdispersion of 500, the highest value it tries, in + all but two tables (150 for tetraploids at 10 reads in two groupings), + because pooled reads of well-behaved diploid markers scatter little. With + that value in place of polyRAD's 9, 5 points more tetraploid and 12 to 25 + points more hexaploid target genotypes are right at 50 reads and above. +- polyRAD given the same overdispersion is 0.3 to 0.9 points below `call` + with the HWE prior, which also has a background rate. At full depth its + iteration did not end in 300 s at an overdispersion of 500, in any + grouping. +- updog, which fits a bias and an overdispersion for every SNP, equals `call` + at full depth and is 1 to 4 points behind it for tetraploids and 6 to 10 + for hexaploids once the reads are thinned. `call` with the bias fitted + loses the same way, up to 7.6 points for hexaploids at 25 reads, because + on reads without bias a parameter per SNP fits noise. +- With the default filters 95.7 to 98.9% of the tetraploid target genotypes + kept are right, with 98%, 74%, 52% and 20% of them kept from full depth + down to 10 reads. For hexaploids the figures are 93.3 to 98.4%, with 96%, + 40%, 17% and 4% kept. +- Off-target SNPs score better than targets in every row. The targets are + the most polymorphic SNP of each tag, so more of their genotypes are + heterozygous. + +**Audit against both packages' sources.** The audit read every function of +`madcats_call.py` against polyRAD 2.0.1's and updog 2.1.7's source and ran +it against the packages on the same inputs, to settle what is adopted, what +is adapted and what is MADCATS's own. The module's opening comment and the +functions' comments now record the origin of each part, with the full +references. + +| Compared | Result | +|---|---| +| Six simulated records (ploidy 2, 4, 6 and mixed, with 2 to 4 alleles and one with selfing) against `IterateHWE` | likelihoods, posteriors, frequencies and corrected genotypes agree to 1e-13 | +| 366 random corrections of dosages to the ploidy against `CorrectGenos` | 366 the same | +| The same where no step has any probability | `call` gives no call, while polyRAD returned all-zero dosages in 33 cases and did not return in one | +| Ties between dosages | `BestGenos` compares in single precision and takes the first, so posteriors 0.4, 0.4, 0.2 give dosage 0 in polyRAD and no call in `call` | +| Prior on the bias, bias formula at error rate 0, limits of `norm` | identical to updog's `pen_bias`, `xi_fun` and bounds | +| 40 single `norm` fits against updog's | 39 agree to 4e-5, and the fortieth is the stalled fit described under "The norm prior", since fixed | + +Three statements were wrong and are corrected. This record listed the +free-weights prior as MADCATS's own (it is updog's `flex`, and only its use +for the estimate is new), the command's description said "The model is +polyRAD's", and a comment described the frequency floor of 1 / (2 × samples) +as polyRAD's for a diploid population (polyRAD divides by the largest locus +ploidy it receives, which is 2 in the stored comparisons, regardless of the +samples' ploidy). + +**Other checks.** On the bundled example all 96 target and 96 haplotype +genotypes equal the simulated truth. `bcftools view` reads the three VCF types +without warnings, including mixed-ploidy output. Output is identical for any +`--jobs`. The simulated table above took 63 s on one core and 17 s on four for +all three VCF types. + ## Downstream fixed allele IDs This pipeline's responsibility ends with generation of the raw MADC-like -microhaplotype count table — and that boundary is certified end-to-end: -MADCATS output has been carried through the full production stack +microhaplotype count table, and end-to-end tests certify that boundary. +They carried MADCATS output through the full production stack (HapApp_utils fixed allele IDs -> `BIGr::madc2vcf_all()` -> updog -dosage) on two platforms and scored against external truth at the far +dosage) on two platforms and scored it against external truth at the far end. On GT-seq the whole downstream stack costs 0.25 points (98.42% at the end of the chain vs 98.67% at the MADC), and the off-target SNPs recovered from Match haplotypes score *higher* than the targets (98.79% on 38,244 additional cells). On the 192-sample capture cohort the chain delivers 95.13% on 295,068 target cells and reproduces the source -publication's population-genetic findings from MADCATS calls alone. Fixed allele-ID assignment and maintenance of the -cross-project panel allele database are downstream HapApp responsibilities; -the former `--allele-catalogue` reprocessing mode was removed accordingly. +publication's population-genetic findings from MADCATS calls alone. Fixed +allele-ID assignment and maintenance of the cross-project panel allele +database are downstream HapApp responsibilities, so MADCATS dropped the +former `--allele-catalogue` reprocessing mode. ## Input contracts @@ -1459,45 +4119,45 @@ The DArT primer CSV must contain: `MarkerName, LeftPrimerSequence, RightPrimerSequence, VariantRef, VariantAlt, VariantPositionInTagSequence`. Indel markers use dash notation (`-` on the empty side, the inserted or deleted bases on the other) with the position -naming where the affected span begins; the pipeline's stored variant +naming where the affected span begins. The pipeline's stored variant position becomes the first base at which the two tags actually differ, which can sit beyond the declared position when the flank repeats the indel content. Primers may contain IUPAC codes (degenerate oligos over flanking -SNPs); tags are reconstructed as the primer sequence verbatim plus the -genomic continuation, matching the vendor's representation. +SNPs). The pipeline reconstructs tags as the primer sequence verbatim plus +the genomic continuation, matching the vendor's representation. The probe-design TSV/CSV must contain `MarkerName` and `TargetSequence`, -with a single bracketed variant — a SNP such as `[C/T]` or an +with a single bracketed variant, either a SNP such as `[C/T]` or an anchored-base indel such as `[A/ACTTG]` or `[TTTTTTG/T]`. Panel construction rejects designs whose variant lies inside the left primer (Alt reads could never carry the primer), identical alleles, and SNP designs whose constructed Alt tag does not appear among the bracket expansions (a primer-table/probe-design disagreement). An optional `LocusGroup` column declares markers as tiles of one physical locus -(see the 0.9.0 section); members of a group must present the same -target region — identical Ref expansions, differing only in which -variant is bracketed — and violations are load-time errors. The +(see the 0.9.0 section). Members of a group must present the same +target region, with identical Ref expansions that differ only in which +variant carries the brackets, and violations are load-time errors. The reference-tags CSV accepts and emits the same optional column, and it only appears in output when some locus actually uses it, keeping ungrouped panels byte-identical to earlier versions. The platform-neutral reference-tag CSV must contain: `MarkerName, LeftPrimerSequence, RefSequence, AltSequence`. Add -`VariantPositionInTagSequence` when Ref and Alt differ at more than one base; -it is validated as an in-range number pointing at an actual Ref/Alt -difference. Identical Ref/Alt sequences or alleles are rejected. +`VariantPositionInTagSequence` when Ref and Alt differ at more than one base. +The pipeline checks that it is an in-range number pointing at an actual +Ref/Alt difference, and rejects identical Ref/Alt sequences or alleles. `VariantRef`, `VariantAlt` (validated for IUPAC compatibility with the tag bases), and `RightPrimerSequence` are optional. Curated tags may contain -IUPAC codes; see the classification note above for what that means for +IUPAC codes. The classification note above explains what that means for canonical counts. -Left primers that are strict prefixes of other panel primers are supported, -and under the default `--primer-mismatches 1` every primer within the -mismatch budget is considered regardless of depth or mismatch count — an -exact short primer cannot suppress a longer one-mismatch primer. Reads -matching several primers are resolved by full-tag identity, and -unresolvable ties are dropped as ambiguous. The default changed from 0 to -1 in 0.9.0: every validated benchmark on every platform ran with 1, and +The pipeline supports left primers that are strict prefixes of other panel +primers, and under the default `--primer-mismatches 1` it considers every +primer within the mismatch budget regardless of depth or mismatch count, so +an exact short primer cannot suppress a longer one-mismatch primer. +Full-tag identity resolves reads matching several primers, and the pipeline +drops unresolvable ties as ambiguous. The default changed from 0 to 1 in +0.9.0, because every validated benchmark on every platform ran with 1, and exact-only matching silently drops Match-haplotype reads whose distinguishing SNP falls inside the assignment anchor, which biases depth by haplotype. Pass `--primer-mismatches 0` to restore exact-only @@ -1507,24 +4167,23 @@ Process a cohort in ONE `generate` invocation whenever novel-allele discovery matters. The evidence model's reproducibility route counts supporting samples within the invocation, so splitting a cohort into several smaller `generate` runs weakens discovery in proportion to the -split: a real haplotype spread thinly across sample subsets can fail -the `--reproducible-min-samples` gate in every subset while easily -passing it cohort-wide (measured on a 192-sample capture cohort -processed as 8 batches: a materially thinner Match-row catalogue than -one full-cohort pass). Batch the expensive `normalize-reads` stage -freely — its outputs are per-sample and windows stay frozen — but feed -all samples to a single `generate`. The per-sample cache -(`--cache-dir`) makes re-running one large invocation cheap after the -first pass. +split. A real haplotype spread thinly across sample subsets can fail +the `--reproducible-min-samples` gate in every subset while passing it +cohort-wide (a 192-sample capture cohort processed as 8 batches gave a +materially thinner Match-row catalogue than one full-cohort pass). +Batch the expensive `normalize-reads` stage freely, since its outputs +are per-sample and windows stay frozen, but feed all samples to a +single `generate`. The per-sample cache (`--cache-dir`) makes +re-running one large invocation cheap after the first pass. FASTQ input to `generate` is one primer-started single-end file per -sample; `normalize-reads` accepts any read that contains a panel window. -Paired-end data enters through one of the two 0.9.0 routes — `merge-pairs` -for overlapping pairs, or pooling both mates into one per-sample FASTQ for -non-overlapping dual-anchored designs — chosen explicitly from the assay's -measured geometry. Target manifests (`targets_*.csv`) are optional and used -only for sample names/metadata; by default sample columns use the unique -FASTQ filename stem. +sample, while `normalize-reads` accepts any read that contains a panel +window. Paired-end data enters through one of the two 0.9.0 routes, +`merge-pairs` for overlapping pairs or pooling both mates into one +per-sample FASTQ for non-overlapping dual-anchored designs. Choose the +route explicitly from the assay's measured geometry. Target manifests +(`targets_*.csv`) are optional and serve only for sample names/metadata. +By default sample columns use the unique FASTQ filename stem. ## Tests @@ -1532,7 +4191,14 @@ FASTQ filename stem. python3 -m unittest discover -s tests -v ``` -The suite contains 59 unit/end-to-end tests, including evidence-route +The suite contains 257 unit/end-to-end tests in six files. +`test_madcats.py` (86) covers `generate` and the commands around it, +`test_call.py` (112) the genotype caller, `test_filter_paralogs.py` (49) the +paralog filter, `test_tiled_markers.py` (5) tiled markers through all three, +`test_example.py` (4) the bundled example and `test_docs.py` (1) the match +between the command pages and the parser. The 14 tests that run the real +aligner are skipped when the optional `mappy` is not installed. The tests in +`test_madcats.py` include evidence-route protection and emission, single-sample depth discovery, the supporting- frequency floor, exact and fuzzy prefix-nested primer resolution, design-guard rejections, vendor-preamble sample-ID detection, @@ -1574,16 +4240,17 @@ reads counted as window-scan churn, and length validation in the named FASTQ parser). The diagnostic scripts in `scripts/` refuse cache directories holding multiple fingerprints for one sample (pass `--cache-fingerprint` to select one) and are runnable directly without -`PYTHONPATH` tricks. The 0.5.0 refactor was additionally gated on -byte-for-byte regressions: fresh per-sample processing reproduces every -0.4.0 cache payload exactly on all 80 samples, and the new quality-only +`PYTHONPATH` tricks. Byte-for-byte regressions also gated the 0.5.0 +refactor. Fresh per-sample processing reproduces every 0.4.0 cache +payload exactly on all 80 samples, and the new quality-only configuration reproduces the 0.3.x `full_s5` output file byte-for-byte. ## Interpretation and references Breeding Insight describes MADC rows as 81-bp amplicon microhaplotypes with -per-sample counts. Ref and Alt contain only the designed target difference; -RefMatch and AltMatch have the respective target base plus other differences. +per-sample counts. Ref and Alt contain only the designed target difference, +while RefMatch and AltMatch have the respective target base plus other +differences. The official format overview is at . @@ -1602,3 +4269,19 @@ PanelHub: The sweetpotato panel publication gives the same classification definitions and states that DArT supplies the MADC from its pipeline: . + +The genotype model of `call` is adapted from polyRAD and updog (see "Genotype +calling" for which part comes from where): + +- Clark LV, Lipka AE, Sacks EJ (2019) polyRAD: genotype calling with + uncertainty from sequencing data in polyploids and diploids. G3 + 9(3):663-673. +- Gerard D, Ferrão LFV, Garcia AAF, Stephens M (2018) Genotyping polyploids + from messy sequencing data. Genetics 210(3):789-807. + +- Gerard D, Ferrão LFV (2020) Priors for genotyping polyploids. + Bioinformatics 36(6):1795-1800. + +- de Silva HN, Hall AJ, Rikkerink E, McNeilage MA, Fraser LG (2005) + Estimation of allele frequencies in polyploids under certain patterns of + inheritance. Heredity 95:327-334. diff --git a/docs/src/README.md b/docs/src/README.md index 8417450..9ead48b 100644 --- a/docs/src/README.md +++ b/docs/src/README.md @@ -11,7 +11,7 @@ platform-independent command-line tool that turns targeted-sequencing FASTQs int microhaplotype count tables. It recovers the alleles a panel was designed to detect, discovers the additional microhaplotypes present in the reads, and writes a raw MADC table. That table goes through HapApp (or HapApp_utils) before dosage calling and VCF conversion with BIGr and related -tools. +tools, or straight to [`madcats call`](commands/call.md) for genotypes and VCFs. The name also reflects the tool's origins as a FASTQ-to-MADC workflow, where MADC is the *Missing Allele Discovery Counts* format. @@ -27,26 +27,34 @@ The name also reflects the tool's origins as a FASTQ-to-MADC workflow, where MAD | [`madcats window-scan`](commands/window-scan.md) | Choose the tag/window length from the project's own reads | | [`madcats generate`](commands/generate.md) | Generate the MADC count table | | [`madcats validate`](commands/validate.md) | Compare a generated MADC against a reference MADC | +| [`madcats filter-paralogs`](commands/filter-paralogs.md) | Drop or flag MADC rows that a reference genome places at another locus | +| [`madcats call`](commands/call.md) | Call genotypes from a MADC and write target, all-variant or haplotype VCFs | ## Key Features -- **Open and deterministic**: every threshold is on the command line, and a QC/provenance JSON is - written beside every result -- **Platform-independent**: validated on DArTag, AgriSeq (Ion Torrent), GT-seq, rhAmpSeq and - Flex-Seq capture data with the same code -- **Evidence-based discovery**: a new microhaplotype is reported only when the reads support it +- **Open and deterministic.** Every threshold is on the command line, and MADCATS writes a + QC/provenance JSON beside every result. +- **Platform-independent.** The same code has been validated on DArTag, AgriSeq (Ion Torrent), + GT-seq, rhAmpSeq and Flex-Seq capture data. +- **Evidence-based discovery.** MADCATS reports a new microhaplotype only when the reads support it (see the [evidence model](reference/evidence-model.md)), and denoising never changes a sample's - Ref/Alt dosage -- **Indel-aware**: designed indel markers are fully supported, and undesigned indels are recovered - with a gapped alignment that caps the number of substitutions and gaps -- **No dependencies**: pure Python 3 standard library, with parallel workers and a reusable - per-sample cache + Ref/Alt dosage. +- **Indel-aware.** MADCATS fully supports designed indel markers and recovers undesigned indels + with a gapped alignment that caps the number of substitutions and gaps. +- **No dependencies.** MADCATS uses only the Python 3 standard library, with parallel workers and a + reusable per-sample cache. The optional genome paralog filter needs `mappy`. +- **Optional paralog filter.** For panels where co-amplified paralogs inflate heterozygosity, + [`filter-paralogs`](commands/filter-paralogs.md) drops or flags haplotypes a reference genome + places at another locus. +- **Genotype calling.** [`call`](commands/call.md) estimates diploid and polyploid genotypes from + the MADC and writes VCFs of the designed targets, of all tag variants, or of whole + microhaplotypes. ## A prior MADC is not an input -Generation uses only the FASTQs plus the assay design: the DArTag primer and probe files, the probe -design alone, or a platform-neutral Ref/Alt tag manifest. A vendor MADC, when you have one, is used -only *after* generation, as an external check with [`madcats validate`](commands/validate.md). +Generation uses only the FASTQs plus the assay design, either the DArTag primer and probe files, the +probe design alone, or a platform-neutral Ref/Alt tag manifest. You use a vendor MADC, when you have +one, only *after* generation, as an external check with [`madcats validate`](commands/validate.md). ## Quick Example @@ -67,18 +75,20 @@ madcats generate \ ## Getting Started -New to MADCATS? Start here: +New to MADCATS? Begin with these pages: -1. [Installation](getting-started/installation.md): install with pip or download a binary -2. [Quick Start](getting-started/quickstart.md): your first MADC in two commands -3. [Try the Example](getting-started/example.md): run a small simulated project and check it against the truth -4. [Choosing a Workflow](getting-started/workflows.md): which commands your platform and inputs need +1. [Installation](getting-started/installation.md) with pip or a downloaded binary +2. [Quick Start](getting-started/quickstart.md) for your first MADC in two commands +3. [Try the Example](getting-started/example.md) to run a small simulated project and check it + against the truth +4. [Choosing a Workflow](getting-started/workflows.md) to find which commands your platform and + inputs need ## Scope MADCATS is an independent implementation of the MADC format and does not aim to reproduce any -vendor's table exactly. How closely it agrees, and against which comparators, is recorded in -[Validation & Accuracy](reference/validation.md). +vendor's table exactly. [Validation & Accuracy](reference/validation.md) records how closely it +agrees, and against which comparators. ## Getting Help diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 49674f2..271521f 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -16,6 +16,8 @@ - [Panels Without a Primer File](tutorials/primer-free.md) - [AgriSeq and Other Non-Primer-Started Reads](tutorials/agriseq.md) - [Paired-End Reads and Tiling](tutorials/paired-end.md) +- [Filtering Paralogs with a Reference Genome](tutorials/paralog-filter.md) +- [Calling Genotypes from a MADC](tutorials/genotype-calling.md) # Command Reference @@ -27,6 +29,8 @@ - [madcats window-scan](commands/window-scan.md) - [madcats generate](commands/generate.md) - [madcats validate](commands/validate.md) +- [madcats filter-paralogs](commands/filter-paralogs.md) +- [madcats call](commands/call.md) # Reference diff --git a/docs/src/citation.md b/docs/src/citation.md index b06f7ea..9961db2 100644 --- a/docs/src/citation.md +++ b/docs/src/citation.md @@ -19,21 +19,21 @@ BibTeX: } ``` -> **Note:** A formal publication with DOI is planned. This section will be updated when available. +> The authors plan a formal publication with DOI and will update this section when it is available. ## AI Assistance MADCATS was developed in collaboration with Claude Opus 5.5 and Fable 5.1 (Anthropic), which helped write its code and documentation. The authors directed the work, reviewed and tested the -results, and are ultimately responsible for the application. +results, and are responsible for the application. ## Related Resources - Breeding Insight, [MADC file format](https://breeding-insight.github.io/BIGr_documentation/posts/madc-file-format/) - Breeding Insight, [MADC data processing](https://breeding-insight.github.io/BIGr_documentation/posts/madc-data-processing/) -- Breeding Insight, [BIGapp-PanelHub](https://github.com/Breeding-Insight/BIGapp-PanelHub): public panel metadata and design Ref/Alt tags -- Breeding Insight, [dartag_alleles](https://github.com/Breeding-Insight/dartag_alleles): the reference-BLAST / paralog audit pattern -- The sweetpotato DArTag panel publication: [PMC11354173](https://pmc.ncbi.nlm.nih.gov/articles/PMC11354173/) +- Breeding Insight, [BIGapp-PanelHub](https://github.com/Breeding-Insight/BIGapp-PanelHub), with public panel metadata and design Ref/Alt tags +- Breeding Insight, [dartag_alleles](https://github.com/Breeding-Insight/dartag_alleles), for the reference-BLAST / paralog audit pattern +- The sweetpotato DArTag panel publication, [PMC11354173](https://pmc.ncbi.nlm.nih.gov/articles/PMC11354173/) If you call dosages from MADCATS output with updog, please also cite: @@ -42,10 +42,38 @@ Gerard, D., Ferrão, L.F.V., Garcia, A.A.F., & Stephens, M. (2018). Genotyping polyploids from messy sequencing data. Genetics 210(3):789-807. ``` +If you filter a MADC with `madcats filter-paralogs`, please also cite minimap2, which does its +alignments: + +``` +Li, H. (2018). Minimap2: pairwise alignment for nucleotide sequences. +Bioinformatics 34(18):3094-3100. https://doi.org/10.1093/bioinformatics/bty191 + +Li, H. (2021). New strategies to improve minimap2 alignment accuracy. +Bioinformatics 37(23):4572-4574. https://doi.org/10.1093/bioinformatics/btab705 +``` + +If you call genotypes with `madcats call`, please also cite the methods its model adapts. +The likelihood and the Hardy-Weinberg prior come from polyRAD, and the allelic bias +(`--allele-bias fit`) and the `norm` prior (`--prior norm`) come from updog: + +``` +Clark, L.V., Lipka, A.E., & Sacks, E.J. (2019). polyRAD: genotype calling with +uncertainty from sequencing data in polyploids and diploids. G3 9(3):663-673. +https://doi.org/10.1534/g3.118.200913 + +Gerard, D., Ferrão, L.F.V., Garcia, A.A.F., & Stephens, M. (2018). Genotyping +polyploids from messy sequencing data. Genetics 210(3):789-807. +https://doi.org/10.1534/genetics.118.301468 + +Gerard, D., & Ferrão, L.F.V. (2020). Priors for genotyping polyploids. +Bioinformatics 36(6):1795-1800. https://doi.org/10.1093/bioinformatics/btz852 +``` + ## Example Acknowledgment In your methods section: > Microhaplotype read counts were generated from the demultiplexed FASTQ files with MADCATS -> v0.13.0 (Sandercock, 2026), using panel tags reconstructed from the vendor design files and +> v0.14.0 (Sandercock, 2026), using panel tags reconstructed from the vendor design files and > default evidence and denoising settings. diff --git a/docs/src/commands/call.md b/docs/src/commands/call.md new file mode 100644 index 0000000..a75dd3d --- /dev/null +++ b/docs/src/commands/call.md @@ -0,0 +1,467 @@ +# madcats call + +Call genotypes from a MADC and write them as VCFs. + +## Synopsis + +```bash +madcats call --madc --output-prefix [--vcf {target,all,haplotype} ...] [OPTIONS] +``` + +## Description + +`call` estimates each sample's genotype from the read counts in a MADC. It needs no other tool. +The MADC can come straight from `generate` or `filter-paralogs`, from a vendor delivery (with or +without the starred preamble), or from HapApp (the fixed-allele-ID layout). Ploidy can differ +between samples. + +`--vcf` chooses what a record is. `call` writes each type to its own file, and one run can write +several: + +| `--vcf` | File | One record per | Alleles | +|---------|------|----------------|---------| +| `target` (default) | `PREFIX.targets.vcf` | marker | The designed variant. Ref and RefMatch reads count for the reference allele, and Alt and AltMatch reads for the alternate allele. Reads of Other rows do not count | +| `all` | `PREFIX.all.vcf` | variant | The target records, plus one record for every other tag position where a row with reads shows a base substitution | +| `haplotype` | `PREFIX.haplotypes.vcf` | marker | Whole tag sequences (microhaplotypes), first the Ref tag and then every other sequence with reads | + +The RefMatch and AltMatch rows are part of a target genotype's evidence. On a potato vendor MADC +of diploid and tetraploid samples, target genotypes from every row had the dosage of a SNP array +for 89.1% of 202,143 genotypes. From the exact Ref and Alt rows alone, 82.1% did, with 2% of the +genotypes left without reads. + +Coordinates are marker-local. `CHROM` is the marker (its CloneID) and `POS` is a 1-based +position on its Ref tag, so these files are not placed on a genome. The header declares every +marker as a contig, and a `##madcatsCoordinates` line records that the coordinates are +marker-local. + +### Which settings to use + +The defaults suit one outbred population or a diversity panel, diploid or autopolyploid. They use +the Hardy-Weinberg prior, estimate the overdispersion and background-read rate from the table, and +fit no allelic bias. Two kinds of samples do better with a change: + +| Your samples | Use | What it changed | +|--------------|-----|-----------------| +| Polyploids on an amplicon panel such as DArTag | `--allele-bias auto` | In 85 tetraploid potato clones, the share of dosages matching a SNP array rose from 92.3% to 96.4% | +| Inbred lines, doubled haploids, selfing species | `--selfing-rate 0.95` to `0.99` | In simulated inbred tetraploids, the share of correct dosages rose from 97.4% to 99.6–99.9% | + +Two other priors helped under particular conditions and not otherwise: + +| Your samples | Consider | Where it helped | Where it did not | +|--------------|----------|-----------------|------------------| +| Families (F1, half-sibs) | `--prior norm` | In simulated tetraploid F1 families at 20 to 60 reads, over five draws, it gained 1.0 point (0.6 to 1.5), and 2.5 with `--allele-bias auto` on a biased panel. With it, `auto` kept no bias that was not there. In two F1 populations at about 10 reads, the error was 0.186 against 0.200, and 0.385 against 0.409 | In alfalfa and potato families at high read depth, it ranged from 0.8 points below `hwe` to 0.2 above. In one population it was up to 0.7 below | +| Subpopulations at 60 reads or fewer, on a panel without bias | `--prior structure` | In simulated tetraploids it gained 0.8 to 5.5 points at 8 reads, 0.5 to 2.5 at 25, and 0.6 at 20 to 60. In potato clones of ten families thinned to 10 and 25 reads, it gained 0.9 and 0.7 points | The same potato clones at 300 reads showed no gain. In one F1 family it matched `hwe`. `call` refuses to combine it with a fitted bias or a selfing rate, and on a biased panel `norm` with `auto` was 1.5 points ahead of it | + +Three things to know before changing them: + +- **A bias matters only where there are heterozygotes.** It changes only the read fraction that + the model expects a heterozygous genotype to show. Diploids, with one heterozygous class, moved + by −0.1 to +0.6 points with it, and tetraploid potato gained 4 to 5. Inbred lines gain nothing. + `fit` miscalls a quarter of their genotypes, and `auto` leaves them as without a bias and warns + (see [Allelic bias](#allelic-bias)). +- **The other priors and the bias cost time.** `norm` and `structure` take two to five times as + long as `hwe`, and a fitted bias seven to eight times. +- **The QC JSON counts the records called with the bias and those at a search limit.** + `allele_bias.records_called_with_the_bias` was half to two thirds of the records on the potato + data, 13 in 100 on diploid smelt and 3 in 100 or fewer on polyploids pooled from reads without + bias. `allele_bias.at_search_limit` was 3 in 100 or fewer on every real data set tested, and + every record on simulated inbred lines. + +The measurements behind this section are in +[Validation & Accuracy](../reference/validation.md#genotype-calling-call). + +### The genotype model + +`call` adapts the model of [polyRAD](https://github.com/lvclark/polyRAD) (Clark, Lipka and Sacks +2019), with the `norm` prior of [updog](https://github.com/dcgerard/updog) (Gerard and Ferrão +2020) and a prior for structured populations as options. Within a record the model considers each +allele separately, against all the other alleles together: + +1. **Likelihood.** In a sample of ploidy *P* an allele can be present in 0 to *P* copies (its + dosage). The probability of the allele's read count, out of the sample's reads in the record, + is beta-binomial. The expected read fraction of a dosage is dosage / *P*, changed by the + allelic bias if one is fitted (`--allele-bias`) and then by the share of reads that show the + wrong allele (`--background-rate`). `--overdispersion` sets how far read fractions may stray + from the expected fraction. +2. **Prior.** `--prior uniform` makes every dosage equally probable. `--prior hwe` (the default) + uses Hardy-Weinberg proportions for the allele's frequency, estimated over all the samples + together. `--prior norm` gives the dosages a normal shape, with a mean and a spread fitted to + the samples. `--prior structure` uses Hardy-Weinberg proportions at each sample's own allele + frequency, taken from the samples most like it. +3. **Posterior.** The posterior is the prior times the likelihood, normalised over the dosages. +4. **Iteration** (`hwe`, `norm` and `structure`). `call` re-estimates the allele frequencies, and + with `norm` the prior's mean and spread, from the posteriors and repeats steps 1–3 until they + change by no more than `--tol` or it reaches `--max-iterations`. Each record converges + separately. + +All four priors apply to every VCF type. + +**`hwe`** assumes random mating and that every chromosome copy is drawn independently, which holds +for diploids and autopolyploids. It is not the expectation under disomic inheritance in an +allopolyploid, in a biparental family, or when samples of different ploidy come from different +populations. + +For inbred lines and partly selfing species, `--selfing-rate` adjusts it. A share of each +generation comes from self-fertilisation, which moves prior probability from the intermediate +dosages to the two homozygous ones (the equilibrium proportions of de Silva et al. 2005, as in +polyRAD). At 1 only the two homozygous dosages have prior probability, so `call` cannot call a +heterozygote. Use a value slightly below 1, such as 0.99, to allow residual heterozygosity. A rate +below 1 applies to samples of even ploidy, while samples of odd ploidy keep random-mating +proportions. A rate of 1 applies to every ploidy, so samples of odd ploidy also get only the two +homozygous dosages. + +**`norm`** makes no assumption about mating. Its dosage probabilities are proportional to a normal +density at the dosages 0 to *P*, so it can be narrower than Hardy-Weinberg proportions (one +family) or wider (subpopulations), and when the dosages have two peaks (inbred lines) it flattens +out. It is updog's `norm` genotype distribution, fitted the way updog fits it. With the same +likelihood, `call` and updog reach the same prior and the same genotypes. `call` fits it separately +for each ploidy among the samples, since a shape over the dosages of one ploidy says nothing about +another. A ploidy with fewer than 10 samples uses Hardy-Weinberg proportions instead, and `call` +prints a note and lists those ploidies in the QC JSON. In a haplotype record every allele gets its +own fit. + +**`structure`** is for samples that are not one randomly mating population, such as +subpopulations, a cline, or a set of families. It keeps Hardy-Weinberg proportions but gives each +sample its own allele frequency in every record. That frequency is the mean over the samples most +like it (its neighbours), with the record's frequency over all samples counted as one more of +them. A sample's own reads never enter its own prior. `call` finds the neighbours once for the +whole table, before it calls any record: + +1. `call` calls the target counts of the markers (at most 1,000, at even spacing) with the `hwe` + prior, which gives every sample an expected share of alternate copies at each marker. +2. Two samples are alike when those shares differ little over the markers. This ranks pairs of + samples as a genomic relationship matrix would. +3. `call` chooses the number of neighbours from 0, 5, 10, 20, 40 and 80 by prediction. The + neighbours found on one half of the markers predict each sample's shares at the other half, and + `call` uses the number with the smallest error. If none predicts better than the table's own + frequencies, the number is 0 and the prior is the `hwe` prior. `call` prints a note. + +`call` then calls records one at a time as with the other priors, each first with the `hwe` prior +and then with the neighbours. The number of neighbours used and the error of each number tried are +in the QC JSON. The method is MADCATS's own, not a port of polyRAD's. + +- It needs at least 20 samples and 100 markers with reads to compare samples on. Below that it is + the `hwe` prior, with a note. +- `call` refuses to combine it with `--allele-bias fit` or `auto`, or with `--selfing-rate`. These + combinations have not been tested. +- It takes two to three times as long as `--prior hwe`. + +### What the model can and cannot say + +With two alleles, the dosage of the alternate allele fixes the genotype, so its posterior is +the genotype posterior. These records carry `GQ`, `MPP`, `GP` and `PL`, and `PMC` in INFO. `call` +computes `PL` from the likelihoods and `GP` from the posteriors, so they differ by the prior. + +With more than two alleles, there is one posterior per allele, not one over whole genotypes. +The most probable dosages of the alleles need not add up to the ploidy. When they do not, `call` +adds or removes copies one at a time where that costs the least probability (polyRAD's method). +These records carry `MPC`, the lowest posterior among the called dosages. It is an upper bound on +the probability that the whole genotype is right, not an estimate of it, which is why it is not +called `GQ`. `call` writes every haplotype record this way, and every `all` record with three or +more bases. + +The model assumes that every copy of a sample's chromosomes can carry any allele (autopolyploid +inheritance), so a hexaploid's dosage runs from 0 to 6. In an allopolyploid whose marker +amplifies all subgenomes but varies on one, the dosage can only be 0 to 2 of 6, and `call` has no +model for that. On the simulated allohexaploid of the polyRAD paper it got 82% of genotypes right +with `hwe` and 87% with `norm`, where polyRAD, told the three subgenomes, did better. A marker that +amplifies one subgenome only is a diploid marker, and `call` calls it correctly with `--ploidy 2`. + +### Missing genotypes + +`call` writes a genotype as missing (`./.`, with one dot per chromosome copy) when: + +- the sample has fewer than `--min-dp` reads in the record, +- two dosages are equally probable, +- its `GQ` is below `--min-gq` (two-allele records) or its `MPC` is below + `--min-marginal-posterior` (other records), or +- more than `--max-other-fraction` of its reads were left out of the call (when that option is + given). + +`call` always keeps `AD` and `DP`. A sample with reads but below `--min-dp` keeps its probabilities +too. A sample with no reads has `.` for them, since they would only repeat the prior. + +### Finding the target + +The target record needs the designed variant's position and alleles. `call` takes them from the +marker's Ref and Alt tags (the Ref and Alt rows of the MADC, or `--reference-tags` when given): + +- If the two tags differ by a single substitution, insertion or deletion, that is the target. + `call` writes a designed indel the VCF way, from the base before it. +- If they differ in more places, the target is the substitution at the reference tags' + `VariantPositionInTagSequence`. Without reference tags the marker gets no target record. +- A marker without an Alt row gets no target record. + +`call` never guesses the target. The QC JSON counts skipped markers by reason, and they still get +haplotype records and off-target records. + +## Required Arguments + +| Argument | Description | +|----------|-------------| +| `--madc ` | MADC to call from, written by `generate` or `filter-paralogs`, or a vendor DArT MADC, or a HapApp fixed-allele-ID MADC. `call` reads gzipped input directly | +| `--output-prefix ` | Path prefix of the outputs | + +## Output Selection + +| Option | Default | Description | +|--------|---------|-------------| +| `--vcf {target,all,haplotype}` | `target` | VCF types to write, one file each. Give one or more, for example `--vcf target haplotype` | +| `--qc-output ` | `PREFIX.qc.json` | QC JSON | + +## Samples and Panel + +| Option | Default | Description | +|--------|---------|-------------| +| `--ploidy ` | 2 | Ploidy of every sample not listed in `--ploidy-file` | +| `--ploidy-file ` | none | Two columns, sample name and ploidy, separated by a tab or a comma, with or without a header line. Names are the MADC's sample column names | +| `--reference-tags ` | none | The panel's reference-tag CSV (from `design` or `infer-design`). Its tags and `VariantPositionInTagSequence` identify each marker's target | +| `--max-indel-length ` | 10 | Widest insertion or deletion that `call` considers when it compares a row with the Ref tag. Use the value given to `generate` | + +## Model + +| Option | Default | Description | +|--------|---------|-------------| +| `--prior {uniform,hwe,norm,structure}` | `hwe` | Genotype prior (see [the genotype model](#the-genotype-model)) | +| `--selfing-rate ` | 0 | With `--prior hwe`, the share of individuals that come from self-fertilisation, where 0 is random mating and 1 is fully inbred lines | +| `--allele-bias {off,fit,auto}` | `off` | Allelic bias, where a copy of one allele yields more reads than a copy of the other. `fit` estimates it for each record with two alleles. `auto` estimates it likewise and calls with it only the records that pass two rules (see below). Not with `--prior structure` | +| `--overdispersion ` | `auto` | Beta-binomial overdispersion parameter, as in polyRAD. Lower values allow read fractions to stray further from the expected fraction. `auto` estimates it from the read counts (see below), and a number sets it | +| `--background-rate ` | `auto` | Probability that a read shows one allele although it came from another. `auto` estimates it from the read counts (see below), and a number sets it | +| `--contamination-rate ` | 0 | polyRAD's contamination, the share of a sample's reads that come from other samples and show an allele at its frequency among them. `--background-rate` covers the same reads and more, so this is off unless set. polyRAD uses 0.001 | +| `--tol ` | 1e-5 | With `--prior hwe`, `norm` or `structure`, stop iterating a record once its allele frequencies change by no more than this (with `norm`, also its prior probabilities, and with `structure`, also the samples' own allele frequencies) | +| `--max-iterations ` | 100 | With `--prior hwe`, `norm` or `structure`, the most iterations a record may take, in each of its two stages with `structure` | + +### Overdispersion and background reads + +Two numbers describe how read counts stray from what a genotype predicts, and `call` estimates +both from the table unless you set them. + +- **Overdispersion** is how far read fractions scatter around the expected fraction. It matters + most in polyploids, where neighbouring dosages differ by a small read fraction. polyRAD chose + its default of 9 for RAD-seq. On two tetraploid alfalfa DArTag populations it left 59% of target + genotypes below the confidence thresholds, against 8–11% with estimated values. +- **The background-read rate** is the share of reads that show an allele they did not come from. + It covers sequencing and amplification errors, index hopping and contamination together, as they + remain after the MADC was built. If it is set too low, the model can explain such reads only as + scatter, and the overdispersion estimate drops with it. + +`call` chooses the pair that makes the read counts most probable. It tries background rates from +0.0002 to 0.02 and overdispersions from 2 to 500, on each marker's reference and alternate read +counts with the genotypes summed out, using up to 20,000 sample-by-marker cells from markers at +even spacing through the table. `call` can tell the two apart because they show in different +samples. Stray reads show in samples that do not carry an allele, and scatter in samples that +carry both. The values used, and the log-likelihood of every value tried, are in the QC JSON, and +the VCF header states them. + +- The estimate does not use `--prior`. It gives every dosage its own free probability at each + marker, because the estimate is only as good as the genotype proportions it assumes. +- A table with no heterozygous samples and no stray reads has no scatter to measure. The + overdispersion estimate then ends at the lowest value tried, and `call` warns and uses 9. Inbred + lines whose reads do include stray reads of the other allele can be estimated from those. +- An overdispersion of 500, the highest value tried, means the counts scatter no more than random + sampling of reads would. +- A table with fewer than 100 sample-by-marker cells with reads is too small to estimate from. + `call` prints a note and uses an overdispersion of 9 and a background rate of 0.002. +- With `--allele-bias fit` or `auto` there is a second estimate, which fits every record's bias. + Without the bias, biased markers look like scatter and the overdispersion comes out lower (30 + and 40 against 150 on the two alfalfa populations). `fit` calls every record with the second + estimate. `auto` calls with it the records that keep their bias, and the others with the first. + The QC JSON has the first under `without_bias` in the `overdispersion` and `background_rate` + blocks. + +To run polyRAD's model as polyRAD runs it, use +`--prior hwe --overdispersion 9 --contamination-rate 0.001 --background-rate 0`. + +### Allelic bias + +In amplicon data a copy of one allele often yields more reads than a copy of the other, so a +sample with two copies of each allele in four does not show half its reads for each. Called +without regard to this, such a marker gets shifted dosages. `--allele-bias fit` estimates the bias +of every record with two alleles (targets, and off-target sites with two bases) and calls with it. +It calls haplotype records and sites with three or more bases without a bias. + +`--allele-bias auto` fits the bias in the same way and then calls with it only the records that +pass two rules. It calls the other records without a bias: + +- **The bias must gain enough.** It has to raise the log-likelihood of the read counts, after the + prior on the bias, by 10 or more over no bias. A bias fitted to a marker that has none follows + the noise in its counts and costs accuracy, most of all in hexaploids. +- **The bias must not end at a limit of the search** (0.14 or 7.4). Samples with few + heterozygotes, such as inbred lines, drive the fit to a limit, where homozygous samples pass for + heterozygous ones. `call` warns when it rejects a bias for this, marks the record in the bias + table and lists it in the QC JSON. `--allele-bias fit` uses such a bias. + +`auto` calls a record without a bias the same way `--allele-bias off` calls it, with the +overdispersion and background rate estimated without a bias. Only the records that keep their bias +use the read model estimated with the bias fitted, which is the one `fit` uses for every record. +The VCF header and the QC JSON give both. Both rules are MADCATS's own, while updog always uses the +bias it fits. + +Every two-allele record carries its fitted bias in `INFO/BIAS` with `fit` and with `auto` alike, +so you can filter the records on it afterwards whichever was used. With `auto`, a record whose +genotypes were called without its bias also carries the flag `INFO/BIASUNUSED`. + +The bias is updog's (Gerard et al. 2018). With bias *h*, an expected read fraction *f* of the +reference allele becomes *f* / (*h*(1 − *f*) + *f*), so below 1 the reference allele is +over-represented and above 1 under-represented. Like updog, `call` pulls the log of the bias +towards 0 with a normal prior of standard deviation 0.7, starts from updog's five starting values +(0.37 to 2.7) and keeps the fit with the highest log-likelihood after that prior. With the other +settings equal, the fitted bias and the genotypes match updog's. + +- `call` finds the bias by calling the record at a series of bias values, 17 or more, so a run with + a bias takes seven to eight times as long. The estimate of the read model does not grow with the + table, and giving `--overdispersion` and `--background-rate` as numbers skips it. +- `call` applies it to the sample's own allele copies and adds background reads afterwards. + updog applies its sequencing error first. In this order, samples that do not carry an allele say + nothing about its bias, which matters because the background rate is one value for the table. +- The search covers biases from 0.14 to 7.4. The QC JSON counts a record whose fit ends at either + limit, and the bias table marks it. Such a marker is usually not behaving as one locus. +- The VCF carries the fitted bias as `INFO/BIAS`, and `PREFIX.bias.csv` lists every fitted record + with what its bias gained over no bias (`Gain`), whether its genotypes were called with it + (`Used`) and if not why (`Rejected`). The QC JSON counts the records called with their bias and + those called without it, by reason. + +When to use it: + +- **Use `auto` for polyploids on an amplicon panel.** It needs no knowledge of the panel. On + potato it raised agreement with a SNP array from 92.3% to 96.4%, where `fit` reached 96.6%. On + polyploids pooled from reads without bias it stayed within 0.2 points of `off`, where `fit` lost + 1 point in tetraploids and 7 in hexaploids. +- **Use `fit` for a panel known to be biased, in hexaploids or at fewer than about 50 reads.** + `auto` keeps a bias only where it gains enough, which takes depth. Of what `fit` gained on + simulated biased tetraploids it kept 98% at 150 reads, 78% at 25, and 64% at 20 to 60 reads with + smaller biases. In hexaploids it kept 83% at 150 reads and 19% at 20 to 60. +- **Use neither for diploids.** A bias moved accuracy by −0.1 to +0.6 points. +- **Use neither for inbred lines.** A bias has nothing to correct in them. `fit` miscalled a + quarter of their genotypes on simulated tables. `auto` rejects their biases, warns, and has the + genotypes of `off`. Give `--selfing-rate`. +- **Use it with `--selfing-rate` or `--prior norm` for partly inbred samples.** On simulated + half-inbred tetraploids without bias, `auto` with the `hwe` prior alone kept a bias at 12 markers + in 100 and lost 0.8 points. With `--selfing-rate 0.5` it lost 0.2, and with `norm` 0.1. +- **Use it with `--prior norm` for families.** On simulated F1 families without bias, `auto` with + `hwe` kept a bias at 2 markers in 100 and lost 0.7 points. With `norm` it kept none. With bias, + `norm` was 2.5 points ahead of `hwe` under `auto` and 4.4 under `fit`. On real alfalfa and + potato families at high depth the two priors were within 0.2 points with a bias fitted. + +`auto` is not the default because it takes as long as `fit`, and because on diploid smelt it +stayed 0.1 points under `off`. + +## Filters + +| Option | Default | Description | +|--------|---------|-------------| +| `--min-dp ` | 10 | Write a genotype as missing when the sample has fewer reads than this in the record | +| `--min-gq ` | 10 | For two-allele records, write a genotype as missing when its `GQ` is below this | +| `--min-marginal-posterior ` | 0.9 | For haplotype records and other records with more than two alleles, write a genotype as missing when its `MPC` is below this | +| `--max-other-fraction ` | none | For `target` and `all` records, write a genotype as missing when more than this share of the sample's reads was left out of the call | + +## Performance + +| Option | Default | Description | +|--------|---------|-------------| +| `--jobs ` | up to 4 | Worker processes. `call` calls each record independently, and the output does not depend on this | + +## Examples + +```bash +# Target genotypes of a diploid project +madcats call --madc results/project_MADC.csv --output-prefix results/project + +# Target and haplotype VCFs of a mixed-ploidy project +madcats call --madc results/project_MADC.csv --output-prefix results/project \ + --vcf target haplotype --ploidy 4 --ploidy-file ploidy.tsv + +# A tetraploid diversity panel at low read depth: each sample's prior follows the samples most like it +madcats call --madc results/panel_MADC.csv --output-prefix results/panel \ + --ploidy 4 --prior structure +``` + +## Outputs + +| File | Contents | +|------|----------| +| `PREFIX.targets.vcf`, `PREFIX.all.vcf`, `PREFIX.haplotypes.vcf` | The VCFs chosen with `--vcf` (VCF 4.3, uncompressed) | +| `PREFIX.haplotypes.alleles.csv` | With `--vcf haplotype`, for each marker and allele number, the MADC AlleleIDs it was counted from, its sequence and its total reads | +| `PREFIX.bias.csv` | With `--allele-bias fit` or `auto`, for each fitted record, the fitted bias of the reference allele, the log-likelihood at that bias (after the prior on the bias), what the bias gained over no bias (`Gain`), whether the genotypes were called with it (`Used`, always with `fit`) and if not why (`Rejected`, either `gain_below_margin` or `at_search_limit`), the starting value the search continued from, the number of bias values tried, and whether the fit converged or ended at a search limit | +| `PREFIX.qc.json` | Parameters, the model in words, the overdispersion and background-rate estimates (with a bias fitted, also the estimates made without one), the number of neighbours chosen for the structure prior, a summary of the fitted biases (how many records were called with their bias, how many without and why, and which were rejected at a search limit), samples per ploidy, and for each VCF the records, genotypes called and missing by reason, reads used and excluded, records that did not converge, and targets skipped by reason | + +### VCF fields + +| Field | In | Meaning | +|-------|----|---------| +| `GT` | all | Genotype. Allele copies are unphased, so `/` joins them | +| `AD` | all | Reads of each allele | +| `DP` | all | Reads used for the call, equal to the sum of `AD` | +| `XD` | `target`, `all` | Reads left out of `AD` and `DP`. At a target these are Other rows, and elsewhere rows that do not show the position (a deletion or an insertion removed it from the tag) | +| `GQ` | two-allele records | Phred-scaled posterior probability that `GT` is wrong, capped at 99 | +| `MPP` | two-allele records | Posterior probability of the most probable genotype. It is the largest value of `GP`, and what `GQ` gives on the Phred scale. updog's `maxpostprob` | +| `DS` | all | Posterior mean number of copies of each alternate allele | +| `GP` | two-allele records | Posterior probability of each genotype, from 0 to *P* alternate copies | +| `PL` | two-allele records | Phred-scaled likelihood of each genotype, before the prior | +| `MPC` | records with more than two alleles, and every haplotype record | Lowest marginal posterior among the called dosages | +| `INFO/TARGET` | `target`, `all` | The record is a marker's designed variant | +| `INFO/BIAS` | two-allele records, with `--allele-bias fit` or `auto` | Fitted allelic bias of the reference allele, as updog reports it for reference read counts | +| `INFO/BIASUNUSED` | two-allele records, with `--allele-bias auto` | The record's genotypes were called without its fitted bias, which gained less than the margin or ended at a limit of the search | +| `INFO/NS` | all | Samples with a called genotype | +| `INFO/AF` | all | Estimated frequency of each alternate allele | +| `INFO/PMC` | two-allele records | Estimated proportion of samples with a wrong genotype, which is one minus the mean of `MPP` over the samples with reads. updog's `prop_mis` | + +In the haplotype VCF, `call` adds together rows with the same sequence, and an alternate sequence +with no reads in any sample is not an allele. It writes an indel haplotype as the tag that was +read, which ends a few bases away from where the Ref tag ends. + +In the `all` VCF, the three record kinds come from the same reads, so agreement between them is a +check on consistency, not independent evidence. `call` does not yet write off-target insertions +and deletions as records. A substitution carried by several haplotypes gets one record with their +reads added together, and gets the same calls as a two-allele marker with those reads. + +A position where the Ref tag holds an ambiguity code gets no off-target record. DArTag panels +write a second designed variant inside a tag that way, and it has a marker of its own. + +Tiled markers (`generate --tile-groups`) share the reads of their amplicon, and `call` calls each +from all of them. Each tile's target is its own variant. Where a tile's tag holds the reference +base at another tile's variant, not an ambiguity code, `call` also writes that variant as an +off-target record of this tile, with the same genotypes as the other tile's target record. The +tiles' haplotype records describe the same sequences. Count such a variant once downstream. + +### MPP and PMC + +`MPP` and `PMC` are updog's `maxpostprob` and `prop_mis`, under the names BIGr's `updog2vcf` gives +them. BIGr's `filterVCF` filters on these names (`filter.MPP`, `filter.PMC`) and on `INFO/BIAS`. + +- `MPP` is the probability the model gives a sample's most probable genotype. `GQ` is the same + number on the Phred scale, so an `MPP` of 0.9 is a `GQ` of 10, and 0.99 is 20. +- `PMC` is one minus the mean of `MPP` over a record's samples, which is the share of them + expected to have a wrong genotype. + +`call` takes `PMC` over the samples with reads, and `--min-dp` and `--min-gq` do not change it. +updog averages over every sample it is given a depth for, and a sample with a depth of 0 enters at +the largest probability of the prior, which raises the value. `call` writes what updog gives when +you pass those samples to it as missing. + +Both say how sure the model is, and the model is only as right as it fits the samples: + +- Where it fits, `PMC` averaged within 0.4 points of the share of wrong genotypes. This held for + simulated random mating in diploids, tetraploids and hexaploids, with `--allele-bias off` or + `auto`. +- Where it does not, `PMC` is too low, by 2 to 11 points on biased polyploid panels called + without a bias, and by 9 to 15 on a biased F1 family under `--prior hwe`. updog's own value + was 7 points too low on that family. +- `PMC` rises with ploidy and falls with depth. Of simulated tetraploid markers at 20 to 60 + reads, one in six had a `PMC` under 0.05, the cut updog's documentation gives as an example, + and nothing was wrong with the others. In hexaploids at that depth almost none did. +- Against the potato SNP array `PMC` averaged 1.4% where 3.6% of the genotypes disagreed (1.1% and + 3.3% for updog). The reads and the array also differ for reasons outside the model. + +[Validation & Accuracy](../reference/validation.md#mpp-and-pmc) has the comparison with updog and +the measurements. + +## See Also + +- [Calling Genotypes from a MADC](../tutorials/genotype-calling.md) for a tutorial on the example + project +- [Validation & Accuracy](../reference/validation.md#genotype-calling-call) for what `call` was + checked against +- [madcats generate](generate.md) +- [madcats filter-paralogs](filter-paralogs.md) +- [Output Formats](../reference/output-formats.md) diff --git a/docs/src/commands/design.md b/docs/src/commands/design.md index 68f7141..f26991c 100644 --- a/docs/src/commands/design.md +++ b/docs/src/commands/design.md @@ -16,17 +16,17 @@ time. The other panel routes were validated against it. For each marker it: -1. locates the left primer inside the probe-design target, in either orientation; +1. locates the left primer inside the probe-design target, in either orientation 2. keeps only primer-started tag windows that contain the bracketed variant (a primer occurrence - in a duplicated flank that never reaches the variant is not a candidate); + in a duplicated flank that never reaches the variant is not a candidate) 3. builds the Ref and Alt tags as the primer sequence verbatim plus the genomic continuation, so - degenerate (IUPAC) primer bases persist exactly as the vendor reports them; -4. checks the result and stops with an error if the files disagree. + degenerate (IUPAC) primer bases persist exactly as the vendor reports them +4. checks the result and stops with an error if the files disagree -SNPs, insertions, deletions and homopolymer-shifted indels are handled by one rule: the variant -position is where the two full allele expansions first differ. +One rule handles SNPs, insertions, deletions and homopolymer-shifted indels. The variant position +is where the two full allele expansions first differ. -> **When to use:** whenever you have both design files. `generate` can also read them directly +> **When to use.** Use it whenever you have both design files. `generate` can also read them directly > (`--primers` + `--probe-design`), but running `design` first gives you a reviewable panel you > can reuse without the vendor files. @@ -67,9 +67,9 @@ madcats design \ ## Output Format -A reference-tag CSV with one row per marker; see -[Input Formats](../getting-started/input-formats.md#reference-tags-csv). The number of definitions -written is printed to stderr. +A reference-tag CSV with one row per marker. See +[Input Formats](../getting-started/input-formats.md#reference-tags-csv). The command prints the +number of definitions it wrote to stderr. ## Errors @@ -83,11 +83,11 @@ The run stops, naming the marker, when: | Panel | Design tags vs vendor canonical rows | |-------|--------------------------------------| -| Blackberry (3,015 markers, 10 indels) | 2,881 identical; all 134 others differ only by vendor IUPAC codes | -| Pecan 3K | 3,084 identical of 3,100; 16 differ only by vendor IUPAC codes | +| Blackberry (3,015 markers, 10 indels) | 2,881 identical, and all 134 others differ only by vendor IUPAC codes | +| Pecan 3K | 3,084 of 3,100 identical, and 16 differ only by vendor IUPAC codes | | Sweetpotato | All 6,238 81-bp prefixes of the public PanelHub 109-bp design sequences agree | ## See Also -- [madcats infer-design](infer-design.md): the same panel without the primer file -- [madcats generate](generate.md): use the panel +- [madcats infer-design](infer-design.md) for the same panel without the primer file +- [madcats generate](generate.md) to use the panel diff --git a/docs/src/commands/filter-paralogs.md b/docs/src/commands/filter-paralogs.md new file mode 100644 index 0000000..01c17ca --- /dev/null +++ b/docs/src/commands/filter-paralogs.md @@ -0,0 +1,337 @@ +# madcats filter-paralogs + +Drop or flag MADC haplotypes that a reference genome places at another locus. It is optional and +needs the `mappy` aligner. + +## Synopsis + +```bash +madcats filter-paralogs --madc --genome [--genome-index ] --output [OPTIONS] +madcats filter-paralogs --madc --genome-index --output [OPTIONS] +``` + +## Description + +In some panels, paralogous copies of a target co-amplify with it, and their reads land in the +marker's rows. A paralog that differs from the Ref tag only at non-target bases becomes a RefMatch +or AltMatch row. When a tool sums counts per target SNP (as `BIGr::madc2vcf_all` does), those rows +push homozygous samples towards false heterozygous calls. `filter-paralogs` uses a reference genome +to decide where each haplotype comes from: + +1. **Place each marker.** The filter aligns its Ref tag to the genome. It reads a CloneID of the + form `CHROM_POS` (zero padding allowed, e.g. `Chr1_000333995`) as the marker's position on that + genome. When the Ref tag has several equal-best sites, the CloneID position picks one. If it + cannot, the filter reports the marker as multi-copy. When the Ref tag does not align, the filter + uses a window around the CloneID position. When the Ref tag's best sites are all away from the + CloneID position, it aligns the tag again within that window (see + [CloneID Positions](#cloneid-positions)). All of these uses assume the CloneID refers to the + genome given. See [Using Another Assembly](#using-another-assembly). +2. **Align every distinct haplotype.** Each haplotype gets two scores, its best alignment at its + marker's site and its best alignment anywhere else. The filter also realigns every haplotype to + its own marker's site alone with short seeds, and the better of that score and the genome-wide + one at the site counts. A real allele carrying several SNPs, which the genome-wide search misses + or aligns only partly, still gets a score at home. +3. **Label it**: + + | Label | Meaning | Default action | + |-------|---------|----------------| + | `on_target` | The marker's site scores strictly best | kept | + | `paralog` | Another site scores strictly better, or the haplotype aligns only elsewhere | dropped (`--paralogs`) | + | `ambiguous` | The marker's site ties with another site, or the marker's own Ref tag is multi-copy and unresolved | dropped (`--ambiguous`) | + | `unmapped` | No alignment anywhere | kept | + | `unplaced` | The marker itself could not be placed | kept | + +4. **Write the filtered MADC.** The filter never drops canonical Ref and Alt rows, because + downstream dosage tools need both designed alleles. A canonical row whose label would have + dropped it counts as `protected`. The filter copies kept rows unchanged. The output keeps the + input's layout, so the vendor preamble, the statistics columns, HapApp's `Code_version` column + and the column order stay as they were. + +The filter is opt-in, meant for panels known to co-amplify paralogs. Check the share of reads in +`paralog` rows in the QC JSON. In the lettuce 3K panel it was 63% of all reads in the vendor MADC. +On panels without the problem the filter removes little and is not needed. + +It reads three layouts. These are a MADC from `generate`, a vendor DArT MADC, and the +fixed-allele-ID MADC that HapApp writes (`Code_version`, `AlleleID`, `CloneID`, `AlleleSequence`, +then the samples, with IDs such as `|Ref_0001`). It can therefore run on the raw MADC before +HapApp, or on HapApp's output. + +Scores use minimap2's short-read scoring (match +2, mismatch −8, gap open −12 and extension −2, +ambiguous base −1), recomputed from each alignment, so ties are exact. + +## Required Arguments + +| Argument | Description | +|----------|-------------| +| `--madc ` | MADC to filter, written by `generate`, or a vendor DArT MADC (with or without its starred preamble), or a HapApp fixed-allele-ID MADC. `filter-paralogs` reads gzipped input directly | +| `--output ` | Filtered MADC to write (must differ from `--madc`) | + +Give `--genome`, `--genome-index`, or both. + +## Genome + +| Option | Default | Description | +|--------|---------|-------------| +| `--genome ` | none | Reference genome FASTA, optionally gzipped | +| `--genome-index ` | none | minimap2 index. The command loads it when it exists, and otherwise builds it from `--genome` and saves it here for the next run | +| `--clone-positions {use,ignore}` | `use` | For `CHROM_POS` CloneIDs, `use` them as positions on the genome given, to choose between equal-best Ref-tag sites, stand in for Ref tags that do not align and place markers whose Ref tag aligns better elsewhere (see [CloneID Positions](#cloneid-positions)), or `ignore` them when they refer to another assembly | + +Indexing a large genome is slow and memory-heavy, so build the index once and reuse it. The first +run with both options writes it: + +```bash +madcats filter-paralogs --madc MADC.csv --genome genome.fa --genome-index genome.sr.mmi --output MADC_filtered.csv +``` + +Later runs need only `--genome-index`. An index from the minimap2 command line works too. Build it +with the short-read preset in one part, with `-I` above the genome size, for example +`minimap2 -x sr -I 20G -d genome.sr.mmi genome.fa`. minimap2 splits the index of a genome larger +than `-I` (8 Gb by default) into parts, and mappy searches only the first part, so the command +refuses a multi-part index rather than miss the rest of the genome. An index the command builds +itself is always one part. On the 2.4 Gb lettuce genome indexing took 50 s and 10.6 GB of memory, +and the index file is 5.7 GB. On a 0.8 Gb alfalfa genome it took about 40 s and 5.2 GB, with a +2.0 GB file. + +The genome should hold each locus once. If the assembly is haplotype-resolved, or carries every +copy of each chromosome of an autopolyploid, index one set of chromosomes only (see +[Limitations](#limitations)). + +## Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--paralogs {drop,flag}` | `drop` | Drop `paralog` rows, or keep them labelled in the `GenomeLabel` column | +| `--ambiguous {drop,flag,keep}` | `drop` | Drop, flag, or keep `ambiguous` rows. A kept row still carries its label when `--paralogs flag` adds the `GenomeLabel` column | +| `--qc-output ` | `OUTPUT.qc.json` | QC JSON | +| `--jobs ` | up to 4 | Alignment threads | + +Dropping ambiguous rows is the default because, on lettuce, keeping them left false heterozygous +calls at 3.4% of inbred genotypes, against 0.87% when the filter dropped them. + +## Flag Mode + +With either option set to `flag`, the filter renames the `PICRef` column to `GenomeLabel`, and +every row carries its label there, which is `on_target`, `paralog`, `ambiguous`, `unmapped` or +`unplaced`. The filter keeps flagged rows. Column positions do not change, so positional consumers +still see the 16-column layout. In a vendor MADC this overwrites the vendor's `PICRef` statistic. +Flag mode needs the `PICRef` (or an earlier `GenomeLabel`) column. HapApp's fixed-allele-ID MADC +has none, so on that layout use drop mode and read the labels sidecar. + +A flagged file is for review. Tools that check the vendor's column names, such as HapApp-online's +upload check, reject `GenomeLabel` in place of `PICRef`, so give them the drop-mode output. + +## Outputs + +| File | Contents | +|------|----------| +| `OUTPUT` | The filtered MADC | +| `OUTPUT.genome_labels.csv` | One line per haplotype row, dropped rows included. A row with an empty `AlleleID` is not a haplotype row. The filter passes over it and does not write it to `OUTPUT` either | +| `OUTPUT.genome_markers.csv` | One line per marker, recording where and how the filter placed it | +| `OUTPUT.qc.json` | Parameters, aligner settings, marker placement summary, the number of distinct haplotypes and sequences, haplotypes per label and how many were found again in the realignment at their marker's site, alignment time, rows and reads per label, per outcome and per allele category, and `count_overflow` (see below) | + +The filter sums read totals as stored. If the input has negative counts, the command warns and +`count_overflow` records how many cells and markers carry them. A negative count with a maximum of +127 points to signed 8-bit overflow in a vendor file. See the +[FAQ](../reference/faq.md#my-vendor-madc-has-negative-read-counts). + +When the input carries a MADCATS provenance cell (`generate --dart-preamble`), the filter appends +its settings to it. + +### Labels sidecar + +| Column | Contents | +|--------|----------| +| `CloneID`, `AlleleID`, `AlleleSequence` | Row identity | +| `Label` | `on_target`, `paralog`, `ambiguous`, `unmapped` or `unplaced` | +| `Action` | `kept`, `flagged`, `dropped`, or `protected` (a canonical row kept despite its label) | +| `TargetScore` | Best alignment score at the marker's site (`.` if none) | +| `TargetFrom` | `genome` (the genome-wide search's hit at the site), `site` (the realignment to the marker's site, when it scored higher or the genome-wide search found nothing there) or `.` | +| `OffTargetScore` | Best alignment score anywhere else (`.` if none) | +| `OffTargetSites` | Distinct sites elsewhere at that score, among the 8 sites kept for a haplotype | +| `OffTargetHit` | The best site elsewhere, 1-based, such as `Chr4:61708465-61708545(+)` | + +A perfect 81-base match scores 162 and each mismatch costs 10, so `TargetScore` and +`OffTargetScore` show how many mismatches separate a haplotype from each copy. + +### Markers sidecar + +| Column | Contents | +|--------|----------| +| `CloneID` | Marker | +| `Placement` | `ref_tag`, `ref_tag+clone_id` (multi-copy Ref tag resolved by the CloneID position), `ref_tag_multicopy` (unresolved), `clone_id_site` (the Ref tag's alignment at the CloneID position, though it aligns better elsewhere), `clone_id` (Ref tag did not align) or `unplaced` | +| `RefTagSites` | Equal-best genome sites of the Ref tag (0 when it did not align). The genome-wide search returns at most 51 hits, so this counts the sites found, not necessarily every copy | +| `CloneIDCheck` | `agree`, `disagree`, `no_position` (CloneID is not `CHROM_POS`), `contig_absent`, `fallback` or `ignored` (`--clone-positions ignore`) | +| `Sites` | The marker's site(s), 1-based, `;`-separated | + +## CloneID Positions + +With `--clone-positions use` (the default), the filter reads a `CHROM_POS` CloneID as the marker's +position on the genome given, where the panel was designed. When none of the Ref tag's equal-best +sites is at that position (`CloneIDCheck` `disagree`), the filter aligns the Ref tag again, alone, +within a window around the position. If it aligns there, that alignment is the marker's site +(`clone_id_site`), even when the tag scores better elsewhere, because the genome-wide search can +miss the design site, and the assembly may carry the Alt allele at the target while a paralog +carries Ref. If the tag does not align at the position, the marker is `unplaced` and keeps every +row. + +The command warns when more than 10% of markers disagree. Either the genome is not the one you +meant to use, or the CloneIDs refer to another assembly, in which case rerun with +`--clone-positions ignore`. + +## Using Another Assembly + +You can use any genome that contains the panel's target loci, not only the one the panel was +designed on, because the filter places markers by aligning their Ref tags. Labels against a second +assembly, such as another cultivar's or a newer version, show where the two genomes support a +haplotype differently. A haplotype that is `on_target` against one genome and `paralog` against +another aligns better to some other locus in the second genome. That locus may be a copy the first +genome lacks, but a copy present in both with a different sequence changes the label in the same +way, so confirm a copy difference in the assemblies themselves. + +CloneID positions refer to the design assembly, so against another assembly use +`--clone-positions ignore`. Otherwise, where the chromosome names match, the filter reads a CloneID +position in the new genome's coordinates. It then judges a marker whose Ref tag does not align at +the wrong place and drops its non-canonical rows as paralogs. A marker whose Ref tag aligns away +from its CloneID position is left `unplaced`, unless the tag also aligns at that position. With +`ignore`, a marker whose Ref tag does not align is `unplaced` and keeps every row, and a Ref tag +with several equal-best sites stays multi-copy, so its rows are `ambiguous`. Where the chromosome +names differ, CloneID positions go unused anyway (`contig_absent`). + +Keep the default on the design assembly. On the lettuce vendor MADC, CloneID positions decide the +site of 11 of 2,987 markers. These are 9 whose Ref tag does not align, 1 tie, and `Chr2_001081815`, +whose Ref tag carries two IUPAC ambiguity codes. The genome-wide search placed that tag on Chr4 +(score 96), but it aligns at its CloneID position on Chr2 with 156. With `ignore`, the 9 markers +are `unplaced` and the filter keeps 1,032 rows that the design assembly labels `paralog`, while it +judges `Chr2_001081815` on Chr4, which drops 11 of its rows that the default keeps. + +Each genome needs its own index: + +```bash +madcats filter-paralogs --madc MADC.csv --genome-index v8.sr.mmi --output MADC_v8.csv +madcats filter-paralogs --madc MADC.csv --genome other.fa --genome-index other.sr.mmi \ + --clone-positions ignore --output MADC_other.csv +``` + +To compare the two runs, join their labels sidecars on `CloneID` and `AlleleSequence`: + +```python +import pandas as pd + +key = ["CloneID", "AlleleSequence"] +design = pd.read_csv("MADC_v8.csv.genome_labels.csv") +other = pd.read_csv("MADC_other.csv.genome_labels.csv") +both = design.merge(other, on=key, suffixes=("_v8", "_other")) +print(pd.crosstab(both.Label_v8, both.Label_other)) +``` + +Rows that are `on_target` against the design assembly and `paralog` against the other align +better to another locus in the other assembly. They point to loci to check for a copy that +only the other assembly has, not to such a copy by themselves. + +## Example + +```bash +madcats filter-paralogs \ + --madc DLet22-7611_MADC.csv \ + --genome-index ref/Lsativa_467_v8_Chr.sr.mmi \ + --output DLet22-7611_MADC_filtered.csv \ + --jobs 8 +``` + +On the lettuce 3K DArT MADC (732,534 rows, 376 samples) this took 2 min 15 s and 9.4 GB of memory. +It kept 113,667 rows, with 35.7% of the reads. See the +[paralog filter tutorial](../tutorials/paralog-filter.md) for the whole workflow. + +## Limitations + +- The genome must contain the panel's target loci, so use the assembly the panel was designed on + or one close to it. The filter cannot judge markers whose region is missing or collapsed in the + assembly. +- The filter cannot see a copy that exists in your germplasm but not in the reference assembly. + Its haplotypes look like diverged alleles of the target. A second assembly that has the copy can + show it (see [Using Another Assembly](#using-another-assembly)). +- minimap2 finds fewer distant copies than bwa mem, so the filter judges a haplotype far from every + genome copy on the hits it has. The rows kept or dropped agreed with a bwa-mem classification for + 98.2% of rows of a MADCATS MADC and 94.9% of a vendor MADC, whose rows are more diverged. Those + figures count the canonical Ref and Alt rows, which the filter always keeps. Among the rows the + filter decides, they are 97.3% and 94.8%. The rows that differ are ones the filter keeps far + from the reference. Of the well-supported rows it kept from the vendor MADC, bwa mem and blastn + both keep 97% of those within 3 differences of their site and 27% of those 6 or more away, and + both would drop 43% of the latter. Kept rows that bwa mem would drop hold 1.0% of the kept + reads, and 61 of 2,986 markers have a tenth or more of their kept reads in such rows. +- The filter aligns only the tag (81 bases by default), not the full read. +- An allele of the marker that is as near a copy as it is to its own site is `ambiguous`. With + copies planted one or two bases from the Ref tag, 8 of 349 rows of the markers themselves were + lost this way. With copies that differ more, none of 515 were. +- Give the filter one set of chromosomes. In a reference that carries every haplotype (or, in an + autopolyploid, every copy of each chromosome), a tag has a copy on each, and the filter treats + those copies as any other. It labels an allele that sits on another haplotype `paralog`, and one + that two haplotypes share makes the rows `ambiguous`. The alfalfa panel was designed on + chromosome set 1 of an assembly that has four sets. With set 1 as the genome the filter dropped + 296 of 2,753 RefMatch and AltMatch rows of a family's MADC. With set 2 added it dropped 1,969, + and 46% of the reads were on `ambiguous` rows. CloneID positions do not prevent this. Cut such + an assembly down to one set (for a panel designed on one, that one) before indexing it. All four + sets (2.7 Gb) also needed more than 13 GB of memory to index. +- On a genome of another cultivar than the panel's, some Ref tags do not match their own site + best. With `--clone-positions ignore` on such a genome for alfalfa, the filter placed 0.5 to + 0.9% of the markers in another copy and left 1% (81-base tags) to 7% (54-base tags) unplaced, + with all their rows kept. + +## How It Was Checked + +- **Planted copies.** In the delta smelt genome (437 Mb) copies of 105 markers' tags were added + as new contigs, with the rows a co-amplified copy would give added to a real MADC. Every row + from a copy 1 to 8 bases, or a base and a 2-base deletion, from the Ref tag got the label + `paralog` (450 of 450 over three draws), and the filter mislabelled no row of the 263 untouched + markers (3,348 rows). The filter did not see a copy left out of the genome. +- **Placement.** On the same data the filter placed all 368 markers where their two primers lie in + the genome, with CloneID positions or without. It left markers given a wrong position unplaced, + not placed elsewhere. +- **Lettuce, against bwa mem and blastn.** See the figures above for bwa mem. blastn aligned a + sample of 18,763 rows of the vendor MADC. It made the filter's decision for 99.4 to 99.6% of + the rows dropped as `paralog`, for 91.5% of those dropped as `ambiguous`, and for 69% of the + well-supported rows the filter kept (89% of their reads). A well-supported row has 5 reads in a + sample and reads in 5 samples. The kept rows blastn would drop lie far from the reference (see + [Limitations](#limitations)). +- **Lettuce, what the kept rows are.** Of the 113,667 rows kept from the vendor MADC, 5,687 are + the designed Ref and Alt alleles, with 87.5% of the kept reads. Of the 105,913 other `on_target` + rows, 84% never reach 5 reads in a sample, and 8% reach 5 reads only in 25 samples that differ + strongly from the other 351. In those 351 samples, 1,697 rows at 1,037 markers recur in five or + more samples and hold 95% of these rows' reads. The filter keeps a row when the genome has no + equal or better place for it. Whether the row is an allele depends on its read support. +- **Alfalfa, on its design genome.** The filter ran on two tetraploid F1 families of the alfalfa + 3K panel (7,517 rows of 54-base tags and 8,731 rows of 81-base tags) and the panel's haplotype + database (23,469 haplotypes), on the genome the panel was designed on, chromosome set 1 of + XinJiangDaYe, with their own CloneIDs. It placed every marker at its CloneID position (2,989, + 2,999 and 3,000), with the positions or with `--clone-positions ignore`. The filter labelled 2 + to 3% of the reads `paralog` and as many `ambiguous`. +- **Alfalfa, against bwa mem and blastn.** Every row was also aligned to that genome with bwa mem + and with blastn and labelled from each aligner's own score. The filter kept or dropped the same + rows as bwa mem for 99.7 to 99.9% of rows and as blastn for 99.4 to 99.7%. On a monoploid genome + of another alfalfa cultivar the figures were 99.4 to 99.9% and 99.3 to 99.7%, and the two + aligners agreed with each other on 99.4 to 99.8%. +- **Alfalfa, by inheritance.** In those families a row can be mapped without aligning it, by the + rows of other markers it is inherited with. Of the rows the filter kept, 96% map to their own + marker, and most of the others are at some 60 markers where every row maps elsewhere than the + assembly puts the marker. Of the rows labelled `paralog` whose better site is far from the + marker, 91 and 94% map away from it. A row whose better site is a nearby copy on the same + chromosome is inherited with the marker, and this test cannot judge it. +- **Alfalfa, genotypes.** Target genotypes called before and after the filter agree with the + families' published dosages equally often (89.8% and 89.8%, and 90.1% and 89.9%). This panel + has few paralog reads, and filtering it changes little. +- **Alfalfa, without the design genome.** On the other cultivar's genome with + `--clone-positions ignore`, the filter placed 97.8% of the markers with 81-base tags and 91.0% + of those with 54-base tags where their 361-base design sequence lies, 0.5 to 0.9% in another + copy, and left the rest unplaced. It also labelled more rows `paralog` or `ambiguous` (7.5% of + the larger family's rows against 3.9% on the design genome). +- **Tiled markers.** The filter places two markers that read the same tag window + (`generate --tile-groups`) at the same site, and takes none of their rows for a copy of the + other. On a synthetic panel of two tiles every row was `on_target`, whether the tags hold the + other tile's variant as an ambiguity code or as the reference base. +- Output does not depend on `--jobs`. + +## See Also + +- [Paralog filter tutorial](../tutorials/paralog-filter.md) +- [madcats generate](generate.md) +- [Output Formats](../reference/output-formats.md) diff --git a/docs/src/commands/generate.md b/docs/src/commands/generate.md index 7aef3b1..1f743ec 100644 --- a/docs/src/commands/generate.md +++ b/docs/src/commands/generate.md @@ -11,20 +11,21 @@ madcats generate --fastq-dir --primers --probe-design --outp ## Description -Every workflow ends with `generate`. It runs in seven stages, and the option tables below are -grouped by stage: +Every workflow ends with `generate`. It runs in seven stages, and the option tables below follow +them: 1. Load the panel from a reference-tag CSV (or build it from DArT design files) and find one FASTQ per sample. 2. Assign each read to a marker by its left-primer/anchor prefix, keep the fixed-length tag, and accept it if it reaches `--min-identity` to the marker's Ref or Alt tag. Reads that fail for - every marker get one more try with a gapped alignment under the same edit limit, so undesigned - indels are counted. Reads shorter than the tag are added to a full-length allele whose start - they match. Results are cached per sample. + every marker get one more try with a gapped alignment under the same edit limit, so the counts + include undesigned indels. Credit a read that fits several tiled markers of one amplicon equally + to each of them, and drop other ties. Reads shorter than the tag count toward a full-length + allele whose start they match. Cache results per sample. 3. Count exact tag sequences per sample. -4. Apply the evidence rules. Canonical Ref and Alt rows are always written; any other sequence is - reported only if it passes an evidence route. -5. Denoise. Within each sample, rare sequences one edit away from a much more common one are merged +4. Apply the evidence rules. Always write canonical Ref and Alt rows, and report any other sequence + only if it passes an evidence route. +5. Denoise. Within each sample, merge rare sequences one edit away from a much more common one into it. A pooled pass then merges sequences that failed the evidence rules into a parent that carries the same target allele. 6. Classify each row as Ref, Alt, RefMatch, AltMatch or Other, and annotate it. @@ -42,7 +43,7 @@ The [Evidence Model](../reference/evidence-model.md) explains stages 2–6 in de |----------|-------------| | `--fastq-dir ` | Folder containing one FASTQ(.gz) per sample | | `--output ` | MADC CSV to write | -| `--reference-tags ` | Reference-tag CSV, **or** | +| `--reference-tags ` | Reference-tag CSV, or | | `--primers ` + `--probe-design ` | DArT design files | ## Panel and Samples (stage 1) @@ -52,18 +53,19 @@ The [Evidence Model](../reference/evidence-model.md) explains stages 2–6 in de | `--fastq-glob ` | `*` | FASTQ filename glob within `--fastq-dir` | | `--target-metadata ...` | `targets_*.csv` in the FASTQ folder | Target-manifest CSV(s) for sample metadata | | `--sample-name-field ` | targetid / FASTQ stem | Unique metadata field used for sample columns | -| `--sample-lookup ` | none | Two-column CSV/TSV mapping FASTQ-derived ids to sample names. Every run sample must be mapped | +| `--sample-lookup ` | none | Two-column CSV/TSV mapping FASTQ-derived ids to sample names. Every run sample must have an entry | | `--tag-length ` | 81 | Tag length | ## Read Assignment (stage 2) | Option | Default | Description | |--------|---------|-------------| -| `--min-identity ` | 0.80 | Minimum substitution-only identity to a Ref or Alt tag; a gapped alignment uses the same edit limit | -| `--leading-anchor ` | none | Constant sequence before the amplicon in raw deliveries (DArT: `GATGGATCCAGTGC`). Each read is cut where the anchor ends (it must end within 40 bases beyond its own length). `auto` detects it from the first FASTQ | +| `--min-identity ` | 0.80 | Minimum substitution-only identity to a Ref or Alt tag. A gapped alignment uses the same edit limit | +| `--leading-anchor ` | none | Constant sequence before the amplicon in raw deliveries (for DArT, `GATGGATCCAGTGC`). Each read is cut where the anchor ends (it must end within 40 bases beyond its own length). `auto` detects it from the first FASTQ | | `--primer-mismatches {0,1}` | 1 | Mismatches allowed while assigning a primer/anchor. With `0`, reads from Match haplotypes whose distinguishing SNP falls inside the anchor are lost | -| `--gapped-rescue {on,off}` | `on` | Give reads that fail substitution-only identity one gapped alignment, so undesigned indels are counted | +| `--gapped-rescue {on,off}` | `on` | Give reads that fail substitution-only identity one gapped alignment, so the counts include undesigned indels | | `--max-indel-length ` | 10 | Widest gap (alignment band) a gapped alignment may use | +| `--tile-groups {auto,declared,off}` | `auto` | Link tiled markers (markers that read the same tag window of one amplicon), so a read is credited to every tile it fits instead of being dropped as a tie. `auto` links tiles declared with `LocusGroup` and detects undeclared ones, `declared` links only the declared ones, and `off` links none. See [Tiled Markers](../getting-started/input-formats.md#tiled-markers) | | `--partial-read-mode {on,off}` | `on` | Add a read shorter than the tag to the one full-length allele (already passing the evidence rules) whose start it matches, only when it is longer than every prefix shared between that locus's alleles. `off` drops such reads | | `--min-partial-length ` | tag length − 9 | Minimum length of a credited partial read | | `--cache-dir ` | none | Reusable per-sample count cache | @@ -74,10 +76,10 @@ The [Evidence Model](../reference/evidence-model.md) explains stages 2–6 in de | Option | Default | Description | |--------|---------|-------------| | `--min-global-count ` | 1 | Minimum project-wide reads for a de novo allele | -| `--reproducible-min-reads ` | 10 | Reproducibility route: reads required per supporting sample | -| `--reproducible-min-samples ` | 2 | Reproducibility route: supporting samples required; `0` disables the route | -| `--depth-min-reads ` | 30 | Depth route: reads within one sample that establish an allele on their own; `0` disables | -| `--quality-min-clean-reads ` | 0 | Quality route: project-wide clean reads that establish an allele; `0` (default) disables | +| `--reproducible-min-reads ` | 10 | Reads required per supporting sample in the reproducibility route | +| `--reproducible-min-samples ` | 2 | Supporting samples required by the reproducibility route. `0` disables the route | +| `--depth-min-reads ` | 30 | Reads within one sample that alone establish an allele through the depth route. `0` disables | +| `--quality-min-clean-reads ` | 0 | Project-wide clean reads that establish an allele through the quality route. `0` (default) disables | | `--discovery-min-quality {10,20,30,40}` | 20 | Phred score every tag base must reach for a read to count as clean | | `--min-supporting-freq ` | 0 | Minimum share of a sample's locus reads before that sample can support the reproducibility or depth route | @@ -85,8 +87,8 @@ The [Evidence Model](../reference/evidence-model.md) explains stages 2–6 in de | Option | Default | Description | |--------|---------|-------------| -| `--denoise-distance {0,1,2}` | 1 | Per-sample denoising: the most differing bases allowed between a rare sequence and its parent; `0` disables | -| `--global-denoise-distance {0,1,2}` | 1 | Most differing bases allowed in the pooled same-allele merge; `0` disables | +| `--denoise-distance {0,1,2}` | 1 | Most differing bases allowed between a rare sequence and its parent in per-sample denoising. `0` disables | +| `--global-denoise-distance {0,1,2}` | 1 | Most differing bases allowed in the pooled same-allele merge. `0` disables | | `--denoise-alpha ` | 2.0 | UNOISE-style abundance-skew exponent | ## Output (stages 6–7) @@ -154,7 +156,7 @@ madcats generate ... --reproducible-min-reads 0 --depth-min-reads 0 --quality-mi | File | Contents | |------|----------| -| `OUTPUT` | MADC table; see [Output Formats](../reference/output-formats.md) | +| `OUTPUT` | MADC table, described in [Output Formats](../reference/output-formats.md) | | `OUTPUT.qc.json` | Parameters, provenance, per-sample read accounting | | `OUTPUT.variants.csv` | One variant event per line (`--annotations on`) | diff --git a/docs/src/commands/infer-design.md b/docs/src/commands/infer-design.md index 2e3eb67..7f59803 100644 --- a/docs/src/commands/infer-design.md +++ b/docs/src/commands/infer-design.md @@ -13,20 +13,21 @@ madcats infer-design --probe-design --fastq-dir --output [OP The tag must be a tag-length window of the bracket-expanded target that contains the variant. So inference only has to choose among a fixed set of candidate windows (every such window, on both -strands); it never assembles new sequences. Reads vote for windows in two tiers, in order: +strands), and it never assembles new sequences. Reads vote for windows in two tiers, in order: 1. The exact tier counts reads whose prefix exactly matches a candidate window. When a locus has any exact votes, they decide. 2. The fuzzy tier runs only for loci with no exact votes. It accepts reads that pass the `--min-identity` rule, and tags already resolved for other loci compete for those reads too, so - one locus's reads are not counted for another. + one locus's reads do not count for another. -A locus is resolved when its winning window has at least `--dominance-ratio` times the runner-up's -votes. The `LeftPrimerSequence` written to the CSV is the shortest tag prefix that is unique in the -panel (between `--anchor-min-length` and `--anchor-max-length`, and always ending before the -variant). Loci that cannot be resolved are left out, and the QC report gives the reason for each. +Inference resolves a locus when its winning window has at least `--dominance-ratio` times the +runner-up's votes. The `LeftPrimerSequence` written to the CSV is the shortest tag prefix that is +unique in the panel (between `--anchor-min-length` and `--anchor-max-length`, and always ending +before the variant). Inference leaves out loci it cannot resolve, and the QC report gives the reason +for each. -See [Panels Without a Primer File](../tutorials/primer-free.md) for a walkthrough and accuracy +See [Panels Without a Primer File](../tutorials/primer-free.md) for a tutorial and accuracy figures. ## Required Arguments @@ -46,10 +47,10 @@ figures. | `--tag-length ` | 81 | Reported tag length of the panel (e.g. 54, 81, 109) | | `--min-identity ` | 0.80 | Minimum identity for fuzzy-tier votes, as in `generate` | | `--dominance-ratio ` | 10 | A winning window needs at least this many times the runner-up's votes | -| `--fuzzy-min-reads ` | 30 | Fuzzy-tier votes needed to resolve a locus with no exact votes; `0` disables the fuzzy tier | +| `--fuzzy-min-reads ` | 30 | Fuzzy-tier votes needed to resolve a locus with no exact votes. `0` disables the fuzzy tier | | `--fuzzy-min-samples ` | 2 | Distinct samples that must contribute fuzzy votes | | `--anchor-min-length ` | 15 | Shortest pseudo-primer anchor to try | -| `--anchor-max-length ` | 30 | Longest pseudo-primer anchor; also capped below each variant | +| `--anchor-max-length ` | 30 | Longest pseudo-primer anchor, also capped below each variant | | `--jobs ` | up to 4 | Samples processed in parallel | | `--report ` | `OUTPUT.qc.json` | Inference QC JSON | @@ -88,8 +89,8 @@ madcats infer-design \ ## Output -- `OUTPUT`: the reference-tag CSV, with one row per resolved locus. -- `OUTPUT.qc.json` (for example `inferred_reference_tags.csv.qc.json`): each locus's route +- `OUTPUT` is the reference-tag CSV, with one row per resolved locus. +- `OUTPUT.qc.json` (for example `inferred_reference_tags.csv.qc.json`) records each locus's route (exact or fuzzy), support, margin, anchor length and flags, plus every excluded marker and the reason (no bracketed variant, flanks too short for the tag length, no reads within identity, support below threshold, ambiguous windows). Markers sharing a sequence-identical candidate @@ -104,5 +105,5 @@ madcats infer-design \ ## See Also -- [madcats design](design.md): use this instead when you have the primer file +- [madcats design](design.md) instead, when you have the primer file - [Panels Without a Primer File](../tutorials/primer-free.md) diff --git a/docs/src/commands/merge-pairs.md b/docs/src/commands/merge-pairs.md index c66c298..1981cd0 100644 --- a/docs/src/commands/merge-pairs.md +++ b/docs/src/commands/merge-pairs.md @@ -13,18 +13,18 @@ madcats merge-pairs --fastq-dir --out-dir [OPTIONS] For assays whose inserts are shorter than R1 + R2, `merge-pairs` collapses each pair into one fragment before `normalize-reads` or `generate`: -- R2 is reverse-complemented and overlaps are scanned longest first. +- It reverse-complements R2 and scans overlaps longest first. - The merged read is R1's head, plus the overlap, plus R2's tail. In extended geometry that is the whole fragment. In engulfed (read-through) geometry, R1's adapter tail past the fragment end lies - outside the overlap and never reaches the merged read, so no adapter trimming is needed. R2 - bases past R1's end are kept when the reads differ in length. + outside the overlap and never reaches the merged read, so the reads need no adapter trimming. + When the reads differ in length, the merged read keeps R2 bases past R1's end. - Where overlap bases disagree, the higher-quality call wins. -- Mates are paired by read ID. Mismatched names or unequal read counts stop the run with an error. +- It pairs mates by read ID. Mismatched names or unequal read counts stop the run with an error. -Samples are independent and merge in parallel with `--jobs`. Each sample is written to +Samples are independent and merge in parallel with `--jobs`. The command writes each sample to `OUT_DIR/.fastq.gz`, where `` is the filename with the mate suffix removed. -> **When to use:** only when mates overlap. For dual-anchored capture designs whose mates cover +> **When to use.** Use it only when mates overlap. For dual-anchored capture designs whose mates cover > opposite halves of the fragment, pool both mates instead (see > [Paired-End Reads and Tiling](../tutorials/paired-end.md)). MADCATS does not decide this > for you. @@ -44,7 +44,7 @@ Samples are independent and merge in parallel with `--jobs`. Each sample is writ | `--r2-suffix ` | `_R2.fastq.gz` | Filename suffix of the R2 mate | | `--min-overlap ` | 15 | Shortest overlap, in bases, that lets a pair merge | | `--max-mismatch-rate ` | 0.1 | Mismatch fraction tolerated inside the overlap | -| `--unmerged ` | `r1` | Unmergeable pairs: `r1` keeps the R1, `pool` keeps both mates, `drop` discards the pair | +| `--unmerged ` | `r1` | For unmergeable pairs, `r1` keeps the R1, `pool` keeps both mates, and `drop` discards the pair | | `--jobs ` | up to 4 | Samples processed in parallel | ## Examples @@ -70,12 +70,12 @@ madcats merge-pairs \ ## Accuracy -- Synthetic truth set with modeled adapter read-through: 100% of 513,715 pairs merged, 98.6% - molecular count recovery, zero length errors. -- Compared with FLASH 1.2.11, vsearch and fastp on a four-regime exact-truth set: equal to the best +- On a synthetic truth set with modeled adapter read-through, 100% of 513,715 pairs merged, with + 98.6% molecular count recovery and zero length errors. +- Compared with FLASH 1.2.11, vsearch and fastp on a four-regime exact-truth set, it matched the best (vsearch) in every regime, with zero false merges. ## See Also -- [madcats normalize-reads](normalize-reads.md): the usual next step +- [madcats normalize-reads](normalize-reads.md), the usual next step - [Paired-End Reads and Tiling](../tutorials/paired-end.md) diff --git a/docs/src/commands/normalize-reads.md b/docs/src/commands/normalize-reads.md index 6e10f31..1fb712a 100644 --- a/docs/src/commands/normalize-reads.md +++ b/docs/src/commands/normalize-reads.md @@ -14,21 +14,21 @@ madcats normalize-reads --probe-design --fastq-dir --out-dir `generate` expects every read to start at a fixed anchor. Platforms such as AgriSeq (Ion Torrent) sequence amplicons from both ends with scattered start positions. `normalize-reads` adapts them: -1. Each read is located against the probe-design targets by k-mer (`--kmer-size`) and oriented +1. It locates each read against the probe-design targets by k-mer (`--kmer-size`) and orients it forward. -2. A fixed-length window (`--tag-length`) containing the variant is excised. -3. Per-sample FASTQs of excised windows are written to `--out-dir`, together with the matching +2. It excises a fixed-length window (`--tag-length`) containing the variant. +3. It writes per-sample FASTQs of excised windows to `--out-dir`, together with the matching window reference-tags CSV and a QC report. `generate` then runs unchanged on the output. -Windows are placed from the reads, so a new run can place them differently. Every later run of the -same panel should therefore pass `--reuse-tags` with the first run's window CSV. That keeps -microhaplotype sequences comparable across runs. See -[AgriSeq and Other Non-Primer-Started Reads](../tutorials/agriseq.md). +The command places windows from the reads, so a new run can place them differently. Every later run +of the same panel should therefore pass `--reuse-tags` with the first run's window CSV. That keeps +microhaplotype sequences comparable across runs. See [AgriSeq and Other Non-Primer-Started +Reads](../tutorials/agriseq.md). Markers that share a physical locus (`LocusGroup` in the probe design) are tiles. Reads are located -to the group as a whole, then routed among member windows by `--group-routing`, and fresh placement +to the group as a whole, then `--group-routing` routes them among member windows, and fresh placement forces member windows to excise distinct sequences. ## Required Arguments @@ -45,11 +45,11 @@ forces member windows to excise distinct sequences. |--------|---------|-------------| | `--fastq-glob ` | `*` | FASTQ filename glob within `--fastq-dir` | | `--tag-length ` | 41 | Excised window length. Shorter windows lose fewer reads to indels on indel-prone chemistries | -| `--window-placement ` | `auto` | `auto` learns each marker's window from observed read coverage (an extra pass); `center` needs no read data. With `--reuse-tags`, `auto` scores the frozen windows instead and `center` skips that | -| `--anchor-length ` | 12 | Minimum assignment-anchor prefix length. Anchors grow per locus only where window prefixes collide; the variant is always kept out of the anchor | +| `--window-placement ` | `auto` | `auto` learns each marker's window from observed read coverage (an extra pass), while `center` needs no read data. With `--reuse-tags`, `auto` scores the frozen windows instead and `center` skips that | +| `--anchor-length ` | 12 | Minimum assignment-anchor prefix length. Anchors grow per locus only where window prefixes collide, and the anchor never includes the variant | | `--reuse-tags ` | none | Window reference-tags CSV from a previous run to reuse verbatim, freezing placement | -| `--group-routing ` | `nearest` | For reads covering several tiles of one `LocusGroup`: `nearest` feeds one tile, `covered` feeds every tile the read fully covers | -| `--str-penalty ` | 0.002 | Auto placement: coverage penalty per tandem-repeat bp (unit sizes 1–3) beyond 4 in a window; `0` disables | +| `--group-routing ` | `nearest` | For reads covering several tiles of one `LocusGroup`, `nearest` feeds one tile and `covered` feeds every tile the read fully covers | +| `--str-penalty ` | 0.002 | Auto-placement coverage penalty per tandem-repeat bp (unit sizes 1–3) beyond 4 in a window. `0` disables | | `--kmer-size ` | 21 | k-mer length for locating reads against the design targets | | `--jobs ` | up to 4 | Samples processed in parallel | | `--output-tags ` | `OUT_DIR/window_reference_tags.csv` | Window reference-tags CSV | @@ -100,14 +100,14 @@ madcats normalize-reads \ ## Tips 1. Pass the same `--tag-length` to `generate`. It must match the excised window length. -2. Use `--fastq-glob "*.fastq"` in `generate` so only normalized files are read. -3. With `--reuse-tags`, every row of the frozen panel is checked against the design. Rows that only - partly match, and panels that contain two identical windows, are rejected. +2. Use `--fastq-glob "*.fastq"` in `generate` so it reads only the normalized files. +3. With `--reuse-tags`, the command checks every row of the frozen panel against the design. It + rejects rows that only partly match, and panels that contain two identical windows. 4. With `--reuse-tags`, the QC (`freeze_health`) compares each frozen window's coverage on this run with the window fresh placement would pick, and lists markers that lose more than ten points. ## See Also -- [madcats window-scan](window-scan.md): choose `--tag-length` -- [madcats merge-pairs](merge-pairs.md): paired-end intake +- [madcats window-scan](window-scan.md) to choose `--tag-length` +- [madcats merge-pairs](merge-pairs.md) for paired-end intake - [AgriSeq tutorial](../tutorials/agriseq.md) diff --git a/docs/src/commands/overview.md b/docs/src/commands/overview.md index 6325513..e1c3c01 100644 --- a/docs/src/commands/overview.md +++ b/docs/src/commands/overview.md @@ -1,7 +1,7 @@ # Command Overview -MADCATS is a single executable with seven subcommands. Each handles one stage: defining the panel, -adapting reads, counting, or checking the result. +MADCATS is a single executable with nine subcommands. Each handles one stage of defining the panel, +adapting reads, counting, filtering, checking the result, or calling genotypes. ## Command Structure @@ -37,7 +37,7 @@ Each produces the reference-tag CSV that `generate` consumes. | Command | Description | Primary Use | |---------|-------------|-------------| | [`merge-pairs`](merge-pairs.md) | Merge overlapping paired-end reads | GT-seq PE, rhAmpSeq | -| [`normalize-reads`](normalize-reads.md) | Orient and window non-primer-started reads; emits a window panel | AgriSeq, capture, merged amplicons | +| [`normalize-reads`](normalize-reads.md) | Orient and window non-primer-started reads, and emit a window panel | AgriSeq, capture, merged amplicons | | [`window-scan`](window-scan.md) | Recommend a window length from the project's reads | Before the first `normalize-reads` of a new panel | ### Counting and Checking Commands @@ -47,6 +47,18 @@ Each produces the reference-tag CSV that `generate` consumes. | [`generate`](generate.md) | Build the MADC count table | Every workflow ends here | | [`validate`](validate.md) | Compare a generated MADC with a reference MADC | Checking against a vendor delivery | +### Cleaning Command (Optional) + +| Command | Description | Primary Use | +|---------|-------------|-------------| +| [`filter-paralogs`](filter-paralogs.md) | Drop or flag haplotypes a reference genome places at another locus. It needs `pip install 'madcats[genome]'` | Panels whose co-amplified paralogs inflate heterozygosity, in MADCATS, vendor or HapApp fixed-ID MADCs | + +### Genotype Calling Command + +| Command | Description | Primary Use | +|---------|-------------|-------------| +| [`call`](call.md) | Call genotypes from a MADC and write target, all-variant or haplotype VCFs | Diploid and polyploid genotypes without leaving MADCATS, from MADCATS, vendor or HapApp fixed-ID MADCs | + ## Typical Workflows ### DArTag @@ -55,6 +67,14 @@ Each produces the reference-tag CSV that `generate` consumes. madcats design --primers primers.csv --probe-design probes.txt --output tags.csv madcats generate --fastq-dir fastqs/ --reference-tags tags.csv --jobs 8 --output MADC.csv madcats validate --generated MADC.csv --expected vendor_MADC.csv --report validation.json +madcats call --madc MADC.csv --output-prefix genotypes --vcf target haplotype +``` + +### Paralog-heavy panel (optional genome filter) + +```bash +madcats generate --fastq-dir fastqs/ --reference-tags tags.csv --min-identity 0.95 --output MADC.csv +madcats filter-paralogs --madc MADC.csv --genome genome.fa --genome-index genome.sr.mmi --output MADC_filtered.csv ``` ### AgriSeq @@ -81,19 +101,19 @@ These options appear on several commands: | `--fastq-glob` | Filename glob within `--fastq-dir` (default `*`) | | `--probe-design` | Probe/marker design TSV or CSV with bracketed variants | | `--tag-length` | Tag or window length in bases | -| `--jobs` | Samples processed in parallel (default: up to 4) | +| `--jobs` | Number of samples to process in parallel, up to 4 by default | | `--output` / `--out-dir` | Output file or folder | | `--report` / `--qc-output` | QC/provenance JSON path (each command has a default beside its output) | ## QC and Provenance Every command that processes reads writes a JSON report recording its parameters, version, inputs -and read accounting, beside its main output. Keep it with the result, since it records exactly how -the table was made. +and read accounting, beside its main output. Keep it with the result, since it records how MADCATS +made the table. ## Exit Codes | Code | Meaning | |------|---------| | `0` | Success | -| `2` | Error: invalid arguments, missing or malformed input, or a design inconsistency (the message names the cause) | +| `2` | Error from invalid arguments, missing or malformed input, or a design inconsistency (the message names the cause) | diff --git a/docs/src/commands/validate.md b/docs/src/commands/validate.md index 5416a4b..704ccdb 100644 --- a/docs/src/commands/validate.md +++ b/docs/src/commands/validate.md @@ -14,10 +14,11 @@ madcats validate --generated --expected [--report ] for the same FASTQs. It reports row precision and recall by allele category, classification agreement on shared rows, and count agreement across shared row × sample cells. -The reference MADC is used only for this comparison. It has no effect on `generate`. +The reference MADC serves only this comparison. It has no effect on `generate`. -Either file may carry the vendor's starred preamble rows; the header is located by its `AlleleID` -cell. +Either file may carry the vendor's starred preamble rows, because the command finds the header by +its `AlleleID` cell, which need not be the first (HapApp's fixed-allele-ID output starts with +`Code_version`). ## Required Arguments @@ -32,7 +33,7 @@ cell. |--------|---------|-------------| | `--report ` | none | Also write the JSON report to this file | -The report is always printed to stdout as JSON. +The command always prints the report to stdout as JSON. ## Example @@ -48,8 +49,9 @@ madcats validate \ | Section | Contents | |---------|----------| | `sample_overlap` | Samples in each file and in common | +| `count_overflow` | For each file, the negative count cells, the markers they fall in, and the minimum and maximum count. Negative counts with a maximum of 127 point to signed 8-bit overflow in a vendor file, and the command prints a warning | | `allele_rows` | Rows in common, precision, recall, generated-only and expected-only counts, `classification_agreement` on shared rows, category counts, and `by_category` precision/recall | -| `common_row_count_cells` | Over shared rows × shared samples: exact-match fraction, mean absolute error, Pearson r, and count totals with their ratio | +| `common_row_count_cells` | Exact-match fraction, mean absolute error, Pearson r, and count totals with their ratio over shared rows × shared samples | | `nonzero_union_count_cells` | The same agreement statistics over cells nonzero in either file | ## Interpreting the Results diff --git a/docs/src/commands/window-scan.md b/docs/src/commands/window-scan.md index f7ba73e..0237ca2 100644 --- a/docs/src/commands/window-scan.md +++ b/docs/src/commands/window-scan.md @@ -16,9 +16,9 @@ three curves without external truth: | Curve | Measures | |-------|----------| -| **Retention** | Fraction of located reads that fully cover the best window. Predicted for a dense grid of lengths from one coverage pass | -| **Cleanliness** | Assigned reads not merged away by denoising or dropped for failing the evidence rules | -| **Information** | Evidence-passing novel haplotype rows per 100 markers | +| Retention | Fraction of located reads that fully cover the best window. Predicted for a dense grid of lengths from one coverage pass | +| Cleanliness | Assigned reads not merged away by denoising or dropped for failing the evidence rules | +| Information | Evidence-passing novel haplotype rows per 100 markers | Each candidate in `--lengths` runs the actual `normalize-reads` and `generate` code on a sample subset. @@ -29,14 +29,14 @@ subset. the best survival are eligible. Among them, the most informative length wins, with ties going to the shorter length. -Information is not used on its own, because systematic errors (such as Ion homopolymer indels that +The scan does not use information alone, because systematic errors (such as Ion homopolymer indels that recur across samples) pass the reproducibility route and look like extra haplotypes. With this -rule, the scan picked the length that independent validation favored on every dataset tested: -41 bp on Ion oyster, 81 bp on GT-seq, and 61 bp on capture. +rule, the scan picked the length that independent validation favored on every dataset tested. It +picked 41 bp on Ion oyster, 81 bp on GT-seq, and 61 bp on capture. ### Seed sensitivity -The recommendation can depend on which samples are scanned. `--scan-draws N` repeats the sweep on N +The recommendation can depend on which samples the scan uses. `--scan-draws N` repeats the sweep on N seeded draws, reports each draw's recommendation, and takes the most-voted length. If the recommendation changes between draws, scan more samples or check the choice against external data. @@ -54,7 +54,7 @@ recommendation changes between draws, scan more samples or check the choice agai | Option | Default | Description | |--------|---------|-------------| | `--fastq-glob ` | `*` | FASTQ filename glob within `--fastq-dir` | -| `--scan-samples ` | 6 | Samples per candidate run: the first in sorted order, or a seeded draw | +| `--scan-samples ` | 6 | Samples per candidate run, taken as the first in sorted order or as a seeded draw | | `--scan-seed ` | none | Draw the subset at random with this seed | | `--scan-draws ` | 1 | Independent draws to sweep (seeds `--scan-seed`, +1, +2, ...) | | `--grid-step ` | 10 | Step for the dense predicted-retention grid | @@ -93,12 +93,12 @@ madcats window-scan \ | File | Contents | |------|----------| -| `OUT_DIR/window_scan.tsv` | One row per length: predicted and measured retention, assignment, cleanliness, information, survival, eligibility; then the dense retention grid | +| `OUT_DIR/window_scan.tsv` | One row per length with predicted and measured retention, assignment, cleanliness, information, survival and eligibility, followed by the dense retention grid | | `OUT_DIR/window_scan.qc.json` | Parameters, per-length results and `recommended_tag_length`. With several draws, also `recommendation_votes` and `recommendation_stability` | | `OUT_DIR/len_/` | Per-candidate run directories (under `OUT_DIR/draw_N/` when `--scan-draws` > 1) | -If tiles cannot be given distinct windows at some length, that length is reported as failed and the -scan continues. +If tiles cannot get distinct windows at some length, the scan reports that length as failed and +continues. ## Tips diff --git a/docs/src/development/architecture.md b/docs/src/development/architecture.md index 8fb2faf..6567727 100644 --- a/docs/src/development/architecture.md +++ b/docs/src/development/architecture.md @@ -1,26 +1,34 @@ # Architecture -MADCATS is a single module, `madcats.py`, that uses only the Python standard library. There is -nothing to compile and no dependency to pin, and the same file runs as a script (`python3 -madcats.py`), as the installed `madcats` command, or frozen into a standalone binary. +MADCATS is three Python modules. `madcats.py` holds every command but two and uses only the Python +standard library. `madcats_paralogs.py` holds `filter-paralogs`, the one command with a dependency +outside it. That dependency is the `mappy` aligner, imported when the command runs +(`pip install 'madcats[genome]'`), so no other command depends on it. `madcats_call.py` holds +`call`, the genotype caller, which also uses only the standard library and has its own module +because it is a separate subject. It is a statistical model on a finished table, not read +processing. There is nothing to compile, and the same files run as a script +(`python3 madcats.py`), as the installed `madcats` command, or frozen into a standalone binary. ## Repository Layout | Path | Contents | |------|----------| -| `madcats.py` | The whole tool: library functions and the command line | +| `madcats.py` | Library functions and the command line for every command but `filter-paralogs` and `call` | +| `madcats_paralogs.py` | `filter-paralogs`, with genome alignment, marker placement, labels and the table filter | +| `madcats_call.py` | `call`, with the genotype model, what each VCF type counts, and VCF writing | +| `scripts/make_polyrad_fixture.R`, `scripts/make_updog_fixture.R` | Regenerate `tests/fixtures/polyrad_parity.json` and `tests/fixtures/updog_norm.json`, the polyRAD and updog values the `call` tests compare against (they need R with polyRAD or updog, but the tests do not) | | `tests/` | Unit and end-to-end tests (`unittest`) | | `scripts/` | Diagnostic and conversion scripts (e.g. `madc_to_reference_tags.py`, `validate_inferred_design.py`) | | `docs/` | This documentation (mdBook), plus `DEVELOPMENT.md`, the long-form development record | | `examples/madcats-demo/` | Simulated example project and the script that generates it | -| `assets/` | Full-size logo (2028 × 2028 PNG); the docs use a 600 px copy in `docs/src/images/` | +| `assets/` | Full-size logo (2028 × 2028 PNG). The docs use a 600 px copy in `docs/src/images/` | | `packaging/` | PyInstaller entry point for the standalone binaries | | `pyproject.toml` | Package metadata for PyPI | -| `.github/workflows/` | Tests on every push; PyPI and binary releases on version tags | +| `.github/workflows/` | Tests on every push, and PyPI and binary releases on version tags | ## Reading Map of `madcats.py` -The module is laid out in the order a read travels. Each section opens with a banner comment: +The module follows the order a read travels. Each section opens with a banner comment: | Section | Contents | |---------|----------| @@ -35,6 +43,19 @@ The module is laid out in the order a read travels. Each section opens with a ba | 8. MADC input/output and QC | `read_madc`, `write_madc`, `validate_madc` | | 9. Commands and command line | One `run_*` per subcommand, `build_parser`, `main` | +`madcats_paralogs.py` follows the same order on a smaller scale, with scoring and alignment +(`GenomeAligner`, `WindowIndex`), marker placement and labels (`place_marker`, `score_at_sites`, +`genome_label`), the two-pass table filter (`filter_paralogs`), then its command line. It imports +the MADC helpers it shares with `validate` from `madcats`, so `build_parser` imports it while +building the parser rather than at the top of `madcats.py`. + +The parser imports `madcats_call.py` the same way. The module opens with the genotype model +(`call_record` and the functions it is built from, adapted from polyRAD), then the step from +per-allele posteriors to one genotype (`correct_dosages_to_ploidy`), input (`read_count_table`, +`read_ploidy_table`), what each VCF type counts (`resolve_target`, `RowReading`, `process_marker`), +VCF text (`format_record`, `vcf_header`) and its command line. It reads rows against the Ref tag +with `variant_events`, the same reading that fills the MADC's `Variants` column. + ## The `generate` Pipeline ``` @@ -55,16 +76,17 @@ FASTQs ──► 5. assign reads ──► (per-sample cache) Rows hold the exact sequences observed in reads, with no consensus correction, so the output can be checked against the raw FASTQs. -Decisions that several stages need are made in one place. The same `EvidenceRule` decides -discovery, which sequences denoising protects, and partial-read crediting, and the same -`row_reading` is used for classification and annotation, so those stages always agree. +Decisions that several stages need live in one place. The same `EvidenceRule` decides +discovery, which sequences denoising protects, and partial-read crediting, and classification and +annotation use the same `row_reading`, so those stages always agree. ## Parallelism and Caching - Per-sample read assignment, pair merging, normalization and inference run across a `ProcessPoolExecutor` sized by `--jobs`. -- Since 0.12.1, `generate`'s finalisation also runs across the workers: per-sample denoising per - sample, and the pooled pass per locus. The result is identical to the sequential passes. +- Since 0.12.1, `generate`'s finalisation also runs across the workers, per sample for + per-sample denoising and per locus for the pooled pass. The result is identical to the + sequential passes. - The per-sample cache (`--cache-dir`) is keyed by the panel and every read-assignment parameter, so downstream thresholds never invalidate it. `CACHE_FORMAT_VERSION` is bumped whenever the serialized cache or raw-read processing changes. @@ -78,4 +100,4 @@ discovery, which sequences denoising protects, and partial-read crediting, and t | *marker* / *locus* | The same panel entry (`Locus` is the record) | | *canonical* | The design's Ref and Alt tags | | *category* | Ref, Alt, RefMatch, AltMatch or Other, read at the target base | -| *anchor* | The prefix a read is assigned by: the left primer or a pseudo-primer | +| *anchor* | The prefix a read is assigned by, either the left primer or a pseudo-primer | diff --git a/docs/src/development/changelog.md b/docs/src/development/changelog.md index 3393bb2..390742e 100644 --- a/docs/src/development/changelog.md +++ b/docs/src/development/changelog.md @@ -3,25 +3,134 @@ Notable changes by version. Measurements behind each change are in the [development record](https://github.com/Breeding-Insight/MADCATS/blob/main/docs/DEVELOPMENT.md) on GitHub. +## 0.14.0 + +- New command `call`, in its own module, `madcats_call.py`. It calls genotypes from a MADC (from + `generate` or `filter-paralogs`, a vendor MADC, or HapApp's fixed-allele-ID output) and writes + one VCF per type. The types are `--vcf target` (the designed variant of each marker, and the + default), `all` (targets plus other base substitutions in the tags) and `haplotype` (whole tag + sequences as alleles). It sets ploidy per sample (`--ploidy`, `--ploidy-file`). The model, + adapted from polyRAD, is a beta-binomial likelihood with a uniform or Hardy-Weinberg prior, + iterated per record. Two-allele records carry `GQ`, `GP` and `PL`, and records with more alleles + carry `MPC`, the lowest + marginal posterior among the called dosages. Coordinates are marker-local. It uses only the + standard library. The tests compare its likelihoods, posteriors, converged allele frequencies + and genotypes with polyRAD 2.0.1 on stored reference values. +- `call --prior structure` is a prior for samples from several subpopulations, a cline or families. + Each sample gets Hardy-Weinberg proportions at its own allele frequency, taken from the samples + most like it across the table's markers. The prior chooses how many of them to use by how well + they predict the sample at markers they were not chosen on, and picks 0 (the `hwe` prior) when + there is no structure. On thirteen tables (simulated groups, clines and families at 8 to 150 + reads, and potato DArTag counts thinned to 10 and 25 reads) it called as many genotypes correctly + as polyRAD's `IteratePopStruct` or up to 0.7 points more, and gained 0.5 to 5.5 points over `hwe` + at 8 to 25 reads. It needs no extra dependency. It cannot yet be combined with + `--allele-bias fit` or `--selfing-rate`. +- `call --allele-bias fit` estimates updog's allelic bias for every record with two alleles and + calls with it. With the other settings equal, the fitted bias is within 0.1% of updog 2.1.7's and + the genotypes are the same, on stored reference values. On two tetraploid alfalfa populations it + raised agreement with published dosages from 90.9% and 90.0% to 92.2% and 91.8% (against 93.0% and + 92.7% for updog). Against SNP-array genotypes of 85 tetraploid potato clones it raised the share + of matching dosages from 92.3% to 96.6% (against 96.7% for updog). The VCF gains `INFO/BIAS`, + and `PREFIX.bias.csv` lists each fit. +- `call --allele-bias auto` fits the bias as `fit` does and calls with it only the records where + it raises the log-likelihood of the read counts by 10 or more over no bias, and calls the + others without one. Against `fit`, on potato it keeps the gain (96.4% against 96.6% of array + dosages matched) and on polyploids pooled from real diploid reads, which have no bias, it + removes the loss (77.6% right for hexaploids at 50 reads, as without a bias, against 69.8%). + Every two-allele record still carries its fitted bias in `INFO/BIAS`, and one called without it + carries the flag `INFO/BIASUNUSED`. The bias table gains `Gain`, `Used` and `Rejected` columns. +- `call --allele-bias auto` does not use a bias that ends at a limit of the search. Samples with + few heterozygotes, such as inbred lines, drive every fit to a limit, where homozygous samples + pass for heterozygous ones. On simulated inbred tetraploids 71.7% of dosages were right with + such biases used and 97.4% without. `call` warns, marks the records in the bias table and lists + them in the QC JSON. `--allele-bias fit` still uses such a bias, and warns when a fifth or more + of the fits end at a limit. +- With `auto`, a record called without its bias gets the same calls as with `--allele-bias off`, + under the overdispersion and background rate estimated without a bias. Before, it was called + under the read model estimated with every bias fitted, which cost 1.9 points on a simulated + hexaploid table where no record kept its bias. +- The read-model estimate with a bias fitted starts from the estimate made without one. Started + from a fixed value, its search could stop early. On that hexaploid table it chose an + overdispersion of 10 where 20 is far more probable. +- `call` writes `MPP` (per sample) and `PMC` (per record) on two-allele records. These are + updog's `maxpostprob` and `prop_mis`, under the names BIGr's `updog2vcf` gives them. `MPP` is the + posterior probability of the most probable genotype, and `PMC` is one minus its mean over the + samples with reads, regardless of the genotype filters. With updog's prior, bias and + overdispersion, `MPP` is within 0.001 of updog 2.1.7's value and `PMC` equal to four decimals, on + stored reference fits. updog counts a sample given a depth of 0 at the largest probability of the + prior, while `call` leaves samples without reads out, as updog does when they are passed as + missing. +- `call` estimates a background-read rate (`--background-rate auto`, the default) together with + the overdispersion. This rate is the share of reads that show an allele they did not come from. + It replaces + polyRAD's contamination rate, which is now 0 unless set, because contamination cannot account for + stray reads of an allele no other sample carries. The estimate gives every dosage its own + probability at each marker, so it does not depend on `--prior` and also works for inbred and + partly inbred samples, and it uses the worker processes. +- Two-allele records are computed for one allele and mirrored for the other, and terms shared by + the samples of a ploidy are computed once. Results are unchanged. +- `call --prior norm` adds updog's `norm` genotype distribution. It gives dosage probabilities a + normal shape, fitted per record and per ploidy, and does not assume random mating. With the same + likelihood, its fitted priors and genotypes match updog 2.1.7 on stored reference values. A + ploidy with fewer than 10 samples uses Hardy-Weinberg proportions. +- `call --selfing-rate` adjusts the Hardy-Weinberg prior for self-fertilisation, for inbred lines + and partly selfing species. Its priors and iterated results match polyRAD's `selfing.rate`. +- `call` estimates the beta-binomial overdispersion from the read counts by maximum likelihood + (`--overdispersion auto`, the default, or a number to set it). On simulated diploid, tetraploid, + hexaploid and mixed-ploidy populations it recovered the simulated value. polyRAD's default of 9 + left 59% of target genotypes uncalled on two real tetraploid alfalfa populations, against 8–11% + with the estimated values. +- New optional command `filter-paralogs`, in its own module, `madcats_paralogs.py`. It filters a + MADC (from `generate`, a vendor DArT MADC with or without its preamble, or HapApp's + fixed-allele-ID output) against a reference genome. It places each marker by its Ref tag + and labels every haplotype `on_target`, `paralog` or `ambiguous`. Paralog and ambiguous rows + are dropped (default) or flagged in a `GenomeLabel` column, while canonical rows are always kept. + It writes a labels sidecar, a markers sidecar and a QC JSON. It needs `mappy` + (`pip install 'madcats[genome]'`), imported only by this command, so MADCATS still has no + required dependencies. On the lettuce 3K DArTag panel the filtered vendor MADC agreed with a + bwa-mem-based prototype for 99.4% of reads. +- `filter-paralogs --clone-positions ignore` keeps CloneID coordinates out of marker placement, for + a genome other than the one the CloneIDs refer to. +- The MADC readers find a header row whose `AlleleID` is not the first cell, as in HapApp's + fixed-allele-ID output, which starts with `Code_version`. +- `validate` and `filter-paralogs` warn when a MADC has negative read counts. Some vendor MADCs + store counts in a signed byte, so counts of 128–255 appear as count − 256, and counts of 256 or + more wrap silently. Both commands report a negative count with a maximum of exactly 127 as this + overflow and record it in their QC reports. Neither command corrects the counts, since the + wrapped ones cannot be recovered. + The lettuce 3K vendor MADC has 90,913 such cells across 1,108 markers. +- For tiled markers, `generate` credits a read that fits several tiles of one amplicon to each of + them, instead of dropping it as a tie. Tiles are markers that read the same tag window, with a + shared or nested left primer and Ref tags that agree except at the markers' variant positions. + `--tile-groups auto|declared|off` (default `auto`) links `LocusGroup` members and detects + undeclared tiles. The design route now also reads `LocusGroup` from the probe design. + Ties across families are still dropped. The QC read accounting counts extra credits as + `tile_extra_credits_*`, so `reads_assigned_full_length` still counts each read once. On the + Lettuce 3K DArTag panel (376 samples), 13 markers in 6 amplicons now get, in one full-panel + run, the same table cells as running each tile in a panel of its own. 11 had zero reads before + (now 6,562–103,852 each), and one pair with identical tags had split its reads 12,299 / 80,306 + (now about 112,400 each). All other markers' rows are unchanged. +- The tile families are part of the per-sample cache key. Panels without tiles keep their caches. + ## 0.13.0 -- Removed the `madc_pipeline` compatibility module. Import `madcats` instead; the bundled +- Removed the `madc_pipeline` compatibility module. Import `madcats` instead, as the bundled scripts already do. -- `--partial-read-mode` takes `on`/`off` like the other switches; the old value `assign` still +- `--partial-read-mode` takes `on`/`off` like the other switches, and the old value `assign` still works as `on`, with a warning. -- Clearer command-line help: every command has a description, every option has help text, and - the program is named `madcats` in usage lines. -- The development record (`docs/DEVELOPMENT.md`) is no longer published on the docs site; it - stays in the repository. +- Command-line help is clearer. Every command has a description, every option has help text, and + usage lines name the program `madcats`. +- The docs site no longer publishes the development record (`docs/DEVELOPMENT.md`), which stays in + the repository. - A small simulated example project, with its true haplotypes and genotypes, is included in `examples/madcats-demo` and attached to each release as `madcats-demo.tar.gz`. - The test suite checks that the command reference pages match each command's `--help`. ## 0.12.1 -- Packaged for PyPI: `pip install madcats` installs a `madcats` command. -- Standalone binaries for Linux and macOS are built for each release. -- New documentation site; the former README is now the development record, `docs/DEVELOPMENT.md`. +- Packaged for PyPI, so `pip install madcats` installs a `madcats` command. +- Each release comes with standalone binaries for Linux and macOS. +- New documentation site. The former README is now the development record, `docs/DEVELOPMENT.md`. - Licensed under GPL-3.0-or-later. - `generate`'s finalisation (per-sample denoising and the pooled pass) runs across the `--jobs` workers. The selection is identical to the sequential passes. On a 44-run rhAmpSeq family the @@ -29,42 +138,44 @@ Notable changes by version. Measurements behind each change are in the ## 0.12.0 -- Raw DArT deliveries: `generate` and `infer-design` take `--leading-anchor SEQ|auto`. Reads are - cut where the common sequence ends; runs on an undeclared constant prefix stop and name it. -- Design route: a reconstructed tag must contain the bracketed variant, so primer occurrences +- For raw DArT deliveries, `generate` and `infer-design` take `--leading-anchor SEQ|auto`. They + cut reads where the common sequence ends, and runs on an undeclared constant prefix stop and name + it. +- On the design route, a reconstructed tag must contain the bracketed variant, so primer occurrences in duplicated flanks are no longer candidates. -- Denoiser scaling: at distance 1, parents are found by enumerating single-substitution +- The denoiser scales better. At distance 1, it finds parents by enumerating single-substitution neighbours once centroids outnumber them, fixing quadratic time at highly paralogous loci. Output is identical. ## 0.11.0 -- Partial reads (`--partial-read-mode assign`, now `on`; default): reads shorter than the tag are - added to an allele that already passed the evidence rules and whose start they match, only when - this cannot change an allele ratio. +- With partial reads (`--partial-read-mode assign`, now `on`, the default), reads shorter than the + tag count toward an allele that already passed the evidence rules and whose start they match, + only when this cannot change an allele ratio. `--min-partial-length` defaults to tag length − 9. The experimental `extend` mode was removed. -- One reading per row: classification and annotation share `row_reading`, so they cannot +- Classification and annotation share one reading per row (`row_reading`), so they cannot disagree. -- Freeze health: `normalize-reads --reuse-tags` scores frozen windows against the new run's reads. +- For freeze health, `normalize-reads --reuse-tags` scores frozen windows against the new run's + reads. - `window-scan --scan-draws` reports recommendation stability across seeded sample subsets. ## 0.10.0 -- Undesigned indel recovery: reads that fail substitution-only identity get one gapped +- For undesigned indel recovery, reads that fail substitution-only identity get one gapped alignment under the same edit limit (`--gapped-rescue`, `--max-indel-length`), with substitution and gap limits that keep paralogs out and a bit-vector prescreen for speed. - Target base read through the gapped alignment when that reading is cheaper and verified. -- Annotations (`--annotations on`, default): `Variants` and `MarkerType` columns and the +- Annotations (`--annotations on`, default) add `Variants` and `MarkerType` columns and the `OUTPUT.variants.csv` sidecar. -- The pooled denoising pass merges only into a same-allele parent; N-bearing tags are credited or - dropped, never emitted. +- The pooled denoising pass merges only into a same-allele parent, and credits or drops N-bearing + tags, never emitting them. - `window-scan` gains `--scan-seed` and `--generate-args`. - Cache format 0.5.0 (caches rebuild once). ## 0.9.0 -- Paired-end intake: the `merge-pairs` command, and pooled mates for dual-anchored designs. -- Multi-window tiling: the `LocusGroup` column, `--group-routing nearest|covered`, and the +- Paired-end intake, with the `merge-pairs` command and pooled mates for dual-anchored designs. +- Multi-window tiling, with the `LocusGroup` column, `--group-routing nearest|covered`, and the check that tiles excise distinct windows. - The `window-scan` command. - `--primer-mismatches` default changed from 0 to 1. @@ -90,12 +201,12 @@ Notable changes by version. Measurements behind each change are in the ## 0.7.0 -- Indel markers and degenerate (IUPAC) primers: one general window rule for SNPs, insertions, - deletions and homopolymer shifts; tags built as primer overlay plus genomic suffix. +- Indel markers and degenerate (IUPAC) primers, with one general window rule for SNPs, insertions, + deletions and homopolymer shifts, and tags built as primer overlay plus genomic suffix. ## 0.5.0 -- Evidence model redesign: reproducibility (primary), depth and optional quality routes, +- Evidence model redesign, with reproducibility (primary), depth and optional quality routes, combined by OR. Flags renamed so old commands stop with an error: - `--min-quality-support` → `--quality-min-clean-reads` (route now off by default) - `--novel-min-reads-per-sample` / `--novel-min-samples` → `--reproducible-min-reads` / @@ -107,6 +218,6 @@ Notable changes by version. Measurements behind each change are in the ## Earlier - `infer-design` for panels without a primer file. -- 0.3.x–0.4.x: quality-gated discovery and the exploratory audits that motivated 0.5.0. +- 0.3.x–0.4.x brought quality-gated discovery and the exploratory audits that motivated 0.5.0. - The tool was renamed from `madc_pipeline` to MADCATS, with `madc_pipeline.py` kept as a compatibility shim until 0.13.0. diff --git a/docs/src/development/contributing.md b/docs/src/development/contributing.md index 6ae3ea3..3eff0ca 100644 --- a/docs/src/development/contributing.md +++ b/docs/src/development/contributing.md @@ -1,7 +1,7 @@ # Contributing -Contributions are welcome: bug reports, documentation fixes, validation results on new platforms, -and code. +Contributions are welcome, such as bug reports, documentation fixes, validation results on new +platforms, and code. ## Development Setup @@ -15,7 +15,9 @@ source .venv/bin/activate pip install -e . ``` -No other dependencies are needed to develop or test. +Developing and testing need no other dependencies. The `filter-paralogs` tests that align to a +synthetic genome run only when `mappy` is installed (`pip install -e '.[genome]'`), and the suite +skips them without it. ## Running Tests @@ -36,14 +38,15 @@ its simulated truth. If a change alters the example's numbers, update [Try the Example](../getting-started/example.md) to match. To change the example data itself, edit and rerun `examples/madcats-demo/make_demo_data.py`. -Changes that touch counting should also be checked against real data. The standard is byte-for-byte: -a refactor regenerates existing tables identically from cache, and a behaviour change is measured +Also check changes that touch counting against real data. The standard is byte-for-byte. A +refactor regenerates existing tables identically from cache, and a behaviour change is measured against a comparator before it becomes a default. Record those measurements in [`docs/DEVELOPMENT.md`](https://github.com/Breeding-Insight/MADCATS/blob/main/docs/DEVELOPMENT.md). ## Code Style -- Standard library only. A new dependency needs a strong reason. +- Standard library only. A new dependency needs a strong reason. Only that command's module, + `madcats_paralogs.py`, imports the optional `mappy` extra for `filter-paralogs`. - Keep the section layout of `madcats.py` (see [Architecture](architecture.md)). - Every analysis threshold is a command-line option with a documented default. - Stop with a clear error on inconsistent input instead of writing a table that may be wrong. @@ -61,15 +64,15 @@ mdbook build docs # static site in docs/book/ ## Releasing -Releases are driven by a version tag. The version lives in one place, `__version__` in -`madcats.py`; `pyproject.toml` reads it from there. +A version tag drives each release. The version lives in one place, `__version__` in +`madcats.py`, and `pyproject.toml` reads it from there. 1. Bump `__version__` in `madcats.py` and add an entry to `docs/src/development/changelog.md`. 2. Commit, then tag and push: ```bash - git tag v0.13.0 - git push origin v0.13.0 + git tag v0.14.0 + git push origin v0.14.0 ``` 3. `.github/workflows/release.yml` then: @@ -95,10 +98,10 @@ pyinstaller --onefile --name madcats --paths . packaging/pyinstaller_entry.py ### One-time setup -- PyPI: on [pypi.org](https://pypi.org), add a *pending trusted publisher* for project +- For PyPI, add a *pending trusted publisher* on [pypi.org](https://pypi.org) for project `madcats` with owner `Breeding-Insight`, repository `MADCATS`, workflow `release.yml` and - environment `pypi`. No API token is stored in GitHub. -- Read the Docs: import the GitHub repository at + environment `pypi`. GitHub stores no API token. +- For Read the Docs, import the GitHub repository at [readthedocs.org](https://readthedocs.org/dashboard/import/) under the project slug `madcats`. Builds then run on every push. diff --git a/docs/src/getting-started/example.md b/docs/src/getting-started/example.md index 596b903..c1ee29b 100644 --- a/docs/src/getting-started/example.md +++ b/docs/src/getting-started/example.md @@ -1,8 +1,8 @@ # Try the Example -MADCATS comes with a small simulated DArTag-style project that runs in seconds. The true -haplotypes and genotypes behind its reads are included, so you can check every row of the output -against what was simulated. +MADCATS comes with a small simulated DArTag-style project that runs in seconds. It includes +the true haplotypes and genotypes behind its reads, so you can check every row of the output +against the simulation. ## Get the Data @@ -24,17 +24,17 @@ The same files are in the repository under `examples/madcats-demo`. | `sample_names.csv` | Renames the samples from `S01` to `Line_01`, and so on | | `truth_haplotypes.csv` | Every simulated haplotype's 81-bp tag sequence | | `truth_genotypes.csv` | Each sample's simulated diploid genotype at each marker | -| `make_demo_data.py` | The script that made all of the above; rerunning it reproduces the files exactly | +| `make_demo_data.py` | The script that made all of the above. Rerunning it reproduces the files exactly | ## What Was Simulated -Twelve diploid samples, eight markers with 81-bp tags, and about 550 reads per sample. Beyond each -marker's designed Ref and Alt alleles: +The project has twelve diploid samples, eight markers with 81-bp tags, and about 550 reads per +sample. Beyond each marker's designed Ref and Alt alleles: -- **Chr02** and **Chr04** carry an extra microhaplotype: a second SNP on the Ref or Alt background. -- **Chr06** carries a haplotype with an undesigned 1-bp insertion. -- **Chr08** has no Alt allele in any sample. -- **Chr03** and **Chr07** have their primer on the reverse strand of the probe-design target. +- Chr02 and Chr04 carry an extra microhaplotype with a second SNP on the Ref or Alt background. +- Chr06 carries a haplotype with an undesigned 1-bp insertion. +- Chr08 has no Alt allele in any sample. +- Chr03 and Chr07 have their primer on the reverse strand of the probe-design target. - Every read has a 0.2% chance per base of a sequencing error, about 8% of reads are shorter than the tag, and each sample has a few off-target reads. @@ -60,8 +60,8 @@ madcats generate \ ## Check the Results -The table has 19 rows: a Ref and an Alt row for each of the 8 markers, plus the 3 microhaplotypes -MADCATS discovered. With MADCATS 0.13.0 the discovered rows are: +The table has 19 rows. There is a Ref and an Alt row for each of the 8 markers, plus the 3 +microhaplotypes MADCATS discovered. With MADCATS 0.13.0 the discovered rows are: | AlleleID | Variants | MarkerType | Reads (all samples) | |----------|----------|------------|--------------------:| @@ -71,13 +71,13 @@ MADCATS discovered. With MADCATS 0.13.0 the discovered rows are: Things to look for: -- **Every simulated haplotype is a row, and nothing else is.** Compare the `AlleleSequence` column - with `truth_haplotypes.csv`. The sequencing errors were merged back into their true haplotypes by - denoising, so none of them became rows. -- **The insertion was recovered.** Its reads failed the substitution-only identity check and were - placed by gapped alignment. `Variants` reports the insertion left-aligned (`29:G>GC`), and +- **The rows match the simulated haplotypes one to one.** Compare the `AlleleSequence` column + with `truth_haplotypes.csv`. Denoising merged the sequencing errors back into their true + haplotypes, so none of them became rows. +- **The insertion was recovered.** Its reads failed the substitution-only identity check, and + gapped alignment placed them. `Variants` reports the insertion left-aligned (`29:G>GC`), and `clip=1` notes that it pushed the last Ref base out of the 81-bp window. -- **Chr08 still has an Alt row.** Canonical Ref and Alt rows are always written. Its single read +- **Chr08 still has an Alt row.** MADCATS always writes canonical Ref and Alt rows. Its single read comes from a sequencing error that happened to hit the SNP base. - **Counts follow the genotypes.** In each `Line_` column, a heterozygous sample splits its reads between two haplotypes and a homozygous sample puts nearly all of them on one. Compare with @@ -93,8 +93,8 @@ Each of these reruns `generate` with one setting changed: | Change | What happens | |--------|--------------| -| `--reproducible-min-samples 5 --depth-min-reads 0` | The 3 discovered rows disappear: each extra haplotype is carried by only 4 samples, and the depth route is off | -| `--gapped-rescue off` | The Chr06 insertion row disappears; its reads are no longer assigned | +| `--reproducible-min-samples 5 --depth-min-reads 0` | The 3 discovered rows disappear, because only 4 samples carry each extra haplotype and the depth route is off | +| `--gapped-rescue off` | The Chr06 insertion row disappears, because MADCATS no longer assigns its reads | | `--partial-read-mode off` | Same 19 rows, but reads in rows fall from 6,661 to 6,195 | To compare two runs, use `validate`: @@ -105,6 +105,8 @@ madcats validate --generated results/other_MADC.csv --expected results/demo_MADC ## Next Steps -- [Choosing a Workflow](workflows.md): which commands your own data needs -- [Evidence Model](../reference/evidence-model.md): how reads are assigned, discovered and denoised -- [Output Formats](../reference/output-formats.md): every column of the MADC table +- [Choosing a Workflow](workflows.md) to find which commands your own data needs +- [Calling Genotypes from a MADC](../tutorials/genotype-calling.md) for genotypes and VCFs from this + example's MADC +- [Evidence Model](../reference/evidence-model.md) on how MADCATS assigns, discovers and denoises reads +- [Output Formats](../reference/output-formats.md) for every column of the MADC table diff --git a/docs/src/getting-started/input-formats.md b/docs/src/getting-started/input-formats.md index a3ed33f..7544220 100644 --- a/docs/src/getting-started/input-formats.md +++ b/docs/src/getting-started/input-formats.md @@ -1,23 +1,23 @@ # Input Formats -MADCATS reads plain CSV/TSV design tables and FASTQ files. Column names are matched by header, so -extra columns are ignored. +MADCATS reads CSV/TSV design tables and FASTQ files. It finds columns by their header names, so it +ignores extra columns. ## FASTQs One FASTQ per sample, optionally gzipped, in a single folder (`--fastq-dir`). Restrict which files -are read with `--fastq-glob` (default `*`). +MADCATS reads with `--fastq-glob` (default `*`). -- `generate` expects **primer-started single-end reads**: every read begins at the assignment - anchor. Raw DArT deliveries with a barcode and common sequence in front are handled with - `--leading-anchor`. +- `generate` expects primer-started single-end reads, so every read must begin at the + assignment anchor. `--leading-anchor` handles raw DArT deliveries with a barcode and common + sequence in front. - `normalize-reads` accepts any read that contains a panel window, in either orientation. - `merge-pairs` expects mate files named by a shared stem plus `--r1-suffix` / `--r2-suffix` (defaults `_R1.fastq.gz` / `_R2.fastq.gz`), with mates in the same order. -Sample columns are named by the unique FASTQ filename stem unless a `--sample-lookup` table renames -them. Target manifests (`targets_*.csv`) in the FASTQ folder are picked up automatically for sample -metadata; they are optional. +MADCATS names sample columns by the unique FASTQ filename stem unless a `--sample-lookup` table +renames them. It picks up target manifests (`targets_*.csv`) in the FASTQ folder automatically for +sample metadata. They are optional. ## Primer CSV (DArT) @@ -26,13 +26,13 @@ Used by `design` and by `generate --primers`. | Column | Description | |--------|-------------| | `MarkerName` | Marker identifier, matching the probe design | -| `LeftPrimerSequence` | The primer every read starts with; may contain IUPAC codes for degenerate oligos | +| `LeftPrimerSequence` | The primer every read starts with, which may contain IUPAC codes for degenerate oligos | | `RightPrimerSequence` | Reverse primer | -| `VariantRef`, `VariantAlt` | Alleles. Indels use dash notation: `-` on the empty side | +| `VariantRef`, `VariantAlt` | Alleles. Indels use dash notation, with `-` on the empty side | | `VariantPositionInTagSequence` | Position of the variant (or the start of the indel span) in the tag | -Tags are reconstructed as the primer sequence verbatim plus the genomic continuation, matching the -vendor's representation of degenerate primers. +MADCATS reconstructs each tag as the primer sequence verbatim plus the genomic continuation, +matching the vendor's representation of degenerate primers. ## Probe Design (TSV/CSV) @@ -42,7 +42,7 @@ Used by `design`, `infer-design`, `normalize-reads` and `window-scan`. |--------|-------------| | `MarkerName` | Marker identifier | | `TargetSequence` | Target region with exactly one bracketed variant | -| `LocusGroup` | *Optional.* Declares markers as tiles of one physical locus | +| `LocusGroup` | *Optional.* Declares markers as tiles of one physical locus. See [Tiled Markers](#tiled-markers) | The bracketed variant is a SNP such as `[C/T]` or an anchored-base indel such as `[A/ACTTG]` or `[TTTTTTG/T]`. The first allele is the reference. Any DArT marker table with `MarkerName` and a @@ -59,33 +59,73 @@ Panel construction rejects: - SNP designs whose constructed Alt tag is not among the bracket expansions (a primer-table / probe-design disagreement) -Members of a `LocusGroup` must present the same target region (identical Ref expansions differing -only in which variant is bracketed). Violations are load-time errors. +For `normalize-reads` and `window-scan`, members of a `LocusGroup` must present the same target +region (identical Ref expansions differing only in which variant is bracketed). Violations are +load-time errors. `design` and `generate --primers --probe-design` copy `LocusGroup` onto the panel +without that requirement. ## Reference Tags (CSV) -The platform-neutral panel definition consumed by `generate --reference-tags`. Written by `design`, -`infer-design` and `normalize-reads`, or curated by hand from any source. +This is the platform-neutral panel definition that `generate --reference-tags` consumes. `design`, +`infer-design` and `normalize-reads` write it, or you can curate it by hand from any source. | Column | Required | Description | |--------|----------|-------------| | `MarkerName` | yes | Marker identifier | -| `LeftPrimerSequence` | yes | Assignment anchor: the prefix reads are assigned by | +| `LeftPrimerSequence` | yes | Assignment anchor, the prefix MADCATS assigns reads by | | `RefSequence` | yes | Full-length Ref tag | | `AltSequence` | yes | Full-length Alt tag | | `VariantPositionInTagSequence` | when Ref and Alt differ at more than one base | Validated to point at a real Ref/Alt difference | | `VariantRef`, `VariantAlt` | no | Validated for IUPAC compatibility with the tag bases | | `RightPrimerSequence` | no | | -| `LocusGroup` | no | Tile grouping; written only when some locus uses it | +| `LocusGroup` | no | Tile grouping, written only when some locus uses it. See [Tiled Markers](#tiled-markers) | -Identical Ref/Alt sequences are rejected. Curated tags may contain IUPAC codes; see the -[evidence model](../reference/evidence-model.md#iupac-codes-and-n-bases) for what that means for -counts. +MADCATS rejects identical Ref/Alt sequences. Curated tags may contain IUPAC codes, and the +[evidence model](../reference/evidence-model.md#iupac-codes-and-n-bases) explains what that means +for counts. + +## Tiled Markers + +DArTag panels often tile several target SNPs inside one amplicon. The tiles share a left primer +(or one primer is the other plus a base or two), so every read from the amplicon starts with both, +and each tile's tag carries its neighbours' SNPs as IUPAC codes: + +``` +MarkerName,LeftPrimerSequence,RefSequence,...,VariantPositionInTagSequence,LocusGroup +Chr1_121673941,GGAAGACATGCTGAGGGYAC,GGAAGACATGCTGAGGGYACGTGTCATCRATTTCGG...,23,amp_Chr1_121673 +Chr1_121673947,GGAAGACATGCTGAGGGYAC,GGAAGACATGCTGAGGGYACGTKTCATCGATTTCGG...,29,amp_Chr1_121673 +``` + +A read from this amplicon fits both tiles' tags equally well. `generate` drops such ties as +ambiguous, so without tile handling both markers would get zero reads. Instead, it credits the read +to every tile it fits, as DArT's own pipeline does (see the +[evidence model](../reference/evidence-model.md#tiled-markers)). + +Two markers are tiles of one amplicon when they read the same tag window. Their tags have the +same length, one marker's left primer matches the start of the other's tag, and their Ref tags +agree, up to IUPAC codes, at every position except the two markers' own variant positions. A tile +family is a set of markers linked this way. `generate --tile-groups` chooses which links it uses: + +| `--tile-groups` | Linked markers | +|-----------------|----------------| +| `auto` (default) | Markers sharing a `LocusGroup`, plus undeclared markers that pass the same-window test. Markers declared in different groups are never linked | +| `declared` | Only markers sharing a `LocusGroup`, and only when they pass the same-window test | +| `off` | None, so every tie is dropped as ambiguous | + +Declare `LocusGroup` in the reference-tag CSV, or in the probe design when you build the panel with +`--primers`/`--probe-design`. `design` copies it into the reference tags it writes. Detection uses +only the primers and tags. A reference-tag CSV has no chromosome coordinates, and two markers that +read the same tag window tie wherever their designs place them. `generate` names every family +on stderr and in the QC JSON (`tile_families`). With `declared` or `off` it warns about markers +that pass the same-window test but stay unlinked. + +`normalize-reads` excises tiles of one `LocusGroup` as different windows of the locus. They +never read the same tag window, so `generate` keeps giving each of their reads to one marker. ## Sample Lookup (CSV/TSV) -`generate --sample-lookup` takes a two-column table: FASTQ-derived sample id, then the desired -sample name. A header line is detected and skipped. +`generate --sample-lookup` takes a two-column table with the FASTQ-derived sample id, then the +desired sample name. It detects and skips a header line. ``` fastq_id,sample_name @@ -93,11 +133,11 @@ fastq_id,sample_name 4404076,Covington_rep1 ``` -The table is checked strictly, because a missing or wrong entry would put the wrong name on a count -column: every run sample must be mapped, ids that match no sample are errors, and duplicate names +MADCATS checks the table strictly, because a missing or wrong entry would put the wrong name on a +count column. Every run sample must be mapped, ids that match no sample are errors, and duplicate names are errors. ## Expected MADC -`validate --expected` reads any MADC, with or without the vendor's starred preamble rows: the header -is found by its `AlleleID` cell. +`validate --expected` reads any MADC, with or without the vendor's starred preamble rows, because it +finds the header by its `AlleleID` cell. diff --git a/docs/src/getting-started/installation.md b/docs/src/getting-started/installation.md index ba0b267..ec1568e 100644 --- a/docs/src/getting-started/installation.md +++ b/docs/src/getting-started/installation.md @@ -1,7 +1,9 @@ # Installation MADCATS is a pure-Python tool with no third-party dependencies. Install it from PyPI (recommended), -download a standalone binary, or run it straight from a clone of the repository. +download a standalone binary, or run it straight from a clone of the repository. The one optional +command that aligns to a reference genome, `filter-paralogs`, needs an extra +([below](#optional-genome-aligner)). ## From PyPI (Recommended) @@ -27,6 +29,29 @@ pipx install madcats Upgrade later with `pip install --upgrade madcats` (or `pipx upgrade madcats`). +### Optional: Genome Aligner + +[`madcats filter-paralogs`](../commands/filter-paralogs.md) aligns haplotypes to a reference genome +with minimap2's Python binding, `mappy`. Install it with the `genome` extra: + +```bash +pip install 'madcats[genome]' +``` + +`mappy` compiles minimap2 when no wheel exists for your platform, which needs a C compiler and zlib +headers. Every other command runs without it, and only `filter-paralogs` imports it. The standalone +binaries do not include it. + +On a Mac, the Python that Apple ships and the one from python.org are universal builds, for Apple +Silicon and Intel at once, and `mappy` does not compile for both. If the install fails while +compiling, build `mappy` for your machine alone: + +```bash +ARCHFLAGS="-arch $(uname -m)" pip install 'madcats[genome]' +``` + +A Python built for one architecture, such as conda's, does not need this. + ## Pre-built Binaries Standalone executables bundle their own Python interpreter, so they run on machines without Python. @@ -74,19 +99,19 @@ tar -xzf madcats-macos-x86_64.tar.gz ./madcats --help ``` -> **Note for macOS users**: the binaries are not notarized. If macOS reports that the developer +> **For macOS users.** The binaries are not notarized. If macOS reports that the developer > cannot be verified, remove the quarantine flag: > > ```bash > xattr -d com.apple.quarantine madcats > ``` -The binary takes a couple of seconds to start (it unpacks its bundled interpreter on each run); -processing speed is the same as the pip-installed command. +The binary takes a couple of seconds to start (it unpacks its bundled interpreter on each run), +but processing speed is the same as the pip-installed command. ### Windows -No Windows binary is published yet. Install with `pip install madcats` under Python 3.9+. +There is no Windows binary yet. Install with `pip install madcats` under Python 3.9+. ## From Source @@ -111,7 +136,7 @@ madcats --version ``` ``` -madcats 0.13.0 +madcats 0.14.0 ``` To run the test suite from a source checkout: @@ -122,6 +147,6 @@ python3 -m unittest discover -s tests -v ## Next Steps -- [Quick Start](quickstart.md): generate your first MADC -- [Try the Example](example.md): run a small simulated project in seconds -- [Choosing a Workflow](workflows.md): which commands your data needs +- [Quick Start](quickstart.md) to generate your first MADC +- [Try the Example](example.md) to run a small simulated project in seconds +- [Choosing a Workflow](workflows.md) to find which commands your data needs diff --git a/docs/src/getting-started/quickstart.md b/docs/src/getting-started/quickstart.md index 893a9f7..687d95a 100644 --- a/docs/src/getting-started/quickstart.md +++ b/docs/src/getting-started/quickstart.md @@ -10,8 +10,8 @@ before `generate` (see [Choosing a Workflow](workflows.md)), and all of them fin ## Step 1: Build the Panel -`madcats design` reconstructs each marker's canonical Ref and Alt tags from the design files. No -reads are involved; the result is a platform-neutral reference-tag CSV you can inspect and reuse. +`madcats design` reconstructs each marker's canonical Ref and Alt tags from the design files. It +uses no reads, and the result is a platform-neutral reference-tag CSV you can inspect and reuse. ```bash madcats design \ @@ -38,13 +38,13 @@ madcats generate \ ``` `generate` can also read the design files directly (`--primers` + `--probe-design` in place of -`--reference-tags`). The defaults are the recommended configuration; no evidence flags are needed. +`--reference-tags`). The defaults are the recommended configuration, so you need no evidence flags. This writes: | File | Contents | |------|----------| -| `results/project_MADC.csv` | The MADC table: one row per allele, one read-count column per sample | +| `results/project_MADC.csv` | The MADC table, with one row per allele and one read-count column per sample | | `results/project_MADC.csv.qc.json` | Parameters, provenance and per-sample read accounting | | `results/project_MADC.csv.variants.csv` | One line per variant event per row | @@ -55,7 +55,7 @@ warm cache in about a minute. > **Raw DArT deliveries.** Older DArT orders keep an inline barcode and the common sequence > `GATGGATCCAGTGC` in front of every amplicon. MADCATS detects this and stops with a message -> naming the prefix; re-run with `--leading-anchor auto` (or pass the sequence explicitly). +> naming the prefix. Re-run with `--leading-anchor auto` (or pass the sequence explicitly). ## Step 3 (Optional): Compare Against a Vendor MADC @@ -70,10 +70,10 @@ madcats validate \ ## Process the Whole Cohort at Once -Run **one** `generate` over every sample whenever novel-allele discovery matters. The +Run one `generate` over every sample whenever novel-allele discovery matters. The reproducibility evidence route counts supporting samples within the run, so splitting a cohort -into batches weakens discovery in proportion to the split. Expensive upstream stages such as -`normalize-reads` can be batched freely; feed all of their outputs to a single `generate`. +into batches weakens discovery in proportion to the split. You can batch expensive upstream stages +such as `normalize-reads` freely, but feed all of their outputs to a single `generate`. ## Next: HapApp @@ -89,7 +89,7 @@ See [Output Formats](../reference/output-formats.md) for details. ## Next Steps -- [Try the Example](example.md): run these steps on a small simulated project and check the output -- [Choosing a Workflow](workflows.md): routes for other platforms and inputs -- [madcats generate](../commands/generate.md): every option, grouped by pipeline stage -- [Evidence Model](../reference/evidence-model.md): when a new sequence is reported as a row +- [Try the Example](example.md) to run these steps on a small simulated project and check the output +- [Choosing a Workflow](workflows.md) with routes for other platforms and inputs +- [madcats generate](../commands/generate.md) for every option, grouped by pipeline stage +- [Evidence Model](../reference/evidence-model.md) on when MADCATS reports a new sequence as a row diff --git a/docs/src/getting-started/workflows.md b/docs/src/getting-started/workflows.md index 4fa1949..34e7a3d 100644 --- a/docs/src/getting-started/workflows.md +++ b/docs/src/getting-started/workflows.md @@ -1,8 +1,8 @@ # Choosing a Workflow -Every `generate` run consumes one **panel definition**: a reference-tag CSV holding each marker's -fixed-length Ref and Alt tag sequences plus an assignment anchor. Which commands you run before -`generate` depends on two questions: *how will you get the panel?* and *are your reads +Every `generate` run consumes one panel definition, which is a reference-tag CSV holding each +marker's fixed-length Ref and Alt tag sequences plus an assignment anchor. Which commands you run +before `generate` depends on two questions, *how will you get the panel?* and *are your reads primer-started?* ## Step 1: Define the Panel @@ -12,7 +12,7 @@ primer-started?* | Vendor primer CSV + probe-design file | [`madcats design`](../commands/design.md) | Uses no reads and gives the same result every time. The other routes were validated against it. | | A vendor-delivered MADC from the same panel | `scripts/madc_to_reference_tags.py` | Reproduces the vendor's tags exactly, including its IUPAC curation. | | Probe-design file only | [`madcats infer-design`](../commands/infer-design.md) | Selects each tag window from read evidence. Freeze the result and reuse it. | -| A curated Ref/Alt tag table from any source | Use it directly | `generate` accepts any reference-tag CSV in the documented format, whatever made it. See [Input Formats](input-formats.md). | +| A curated Ref/Alt tag table from any source | Use it directly | `generate` accepts any reference-tag CSV in the documented format. See [Input Formats](input-formats.md). | | Reads that are not primer-started | [`madcats normalize-reads`](../commands/normalize-reads.md) | Produces the panel *and* pipeline-ready FASTQs together. | ## Step 2: Match the Read Geometry @@ -25,8 +25,8 @@ primer-started?* | Paired-end, mates overlap | GT-seq PE, rhAmpSeq | `merge-pairs`, then `normalize-reads` or `generate` | | Paired-end, mates cannot overlap | Flex-Seq (dual-anchored capture) | Pool both mates per sample, then `normalize-reads` | -MADCATS does not detect paired-end data automatically: check how your reads are laid out and choose -the route yourself. Adapter trimming is not required on any route. +MADCATS does not detect paired-end data automatically, so check how your reads are laid out and +choose the route yourself. No route requires adapter trimming. ## Common Pipelines @@ -71,9 +71,31 @@ madcats generate --fastq-dir norm/ --fastq-glob "*.fastq" \ See [Paired-End Reads and Tiling](../tutorials/paired-end.md). +## Optional: Clean Paralogs With a Genome + +If co-amplified paralogous copies inflate heterozygosity in your panel, filter the finished MADC +(from `generate`, from the vendor, or from HapApp) against a reference genome: + +```bash +madcats filter-paralogs --madc MADC.csv --genome genome.fa --genome-index genome.sr.mmi --output MADC_filtered.csv +``` + +See [Filtering Paralogs with a Reference Genome](../tutorials/paralog-filter.md). + +## Optional: Call Genotypes + +`call` estimates each sample's genotype from the MADC (from `generate`, `filter-paralogs`, the +vendor, or HapApp) and writes VCFs. Ploidy can differ between samples: + +```bash +madcats call --madc MADC.csv --ploidy 4 --output-prefix genotypes +``` + +See [Calling Genotypes from a MADC](../tutorials/genotype-calling.md). + ## Freeze Windows Across Runs -A primer-defined tag is fixed by the chemistry, but `normalize-reads` places each window based on +The chemistry fixes a primer-defined tag, but `normalize-reads` places each window based on the reads, so the same panel re-run on new data can place windows differently and make identical biology look like new haplotypes. After the first run of a panel, pass its window CSV to every later run with `--reuse-tags`. On two honeybee chips this raised cross-chip haplotype sharing from 42% to diff --git a/docs/src/reference/evidence-model.md b/docs/src/reference/evidence-model.md index c5c8eed..59e5d18 100644 --- a/docs/src/reference/evidence-model.md +++ b/docs/src/reference/evidence-model.md @@ -1,7 +1,7 @@ # Evidence Model This page explains how `madcats generate` decides which reads count, which sequences become rows, -and how sequencing errors are removed. The design rationale and the measurements behind each default +and how it removes sequencing errors. The design rationale and the measurements behind each default are in the [development record](https://github.com/Breeding-Insight/MADCATS/blob/main/docs/DEVELOPMENT.md) on GitHub. ## Read Assignment @@ -9,35 +9,57 @@ are in the [development record](https://github.com/Breeding-Insight/MADCATS/blob For each marker, the panel holds a fixed-length Ref tag and Alt tag that both start with an assignment anchor (the left primer, or a pseudo-primer prefix). -1. A read is assigned to a marker by its prefix (the anchor). One mismatch is allowed by default +1. `generate` assigns a read to a marker by its prefix (the anchor). It allows one mismatch by default (`--primer-mismatches 1`). Anchors that are prefixes of longer anchors are supported, and every anchor within the mismatch limit is considered. A read that matches several markers goes to the - one whose full tag it matches best; if that is a tie, the read is dropped as ambiguous. -2. The read's first tag-length bases are kept if they have at least `--min-identity` (80%) identity - to the marker's Ref or Alt tag, counting substitutions only. + one whose full tag it matches best. If that is a tie, the read is dropped as ambiguous. The one + exception is a tie within a family of [tiled markers](#tiled-markers). +2. `generate` keeps the read's first tag-length bases if they have at least `--min-identity` (80%) + identity to the marker's Ref or Alt tag, counting substitutions only. 3. If the read fails for every candidate marker, it gets one gapped alignment under the same 20% - edit limit, with gaps of up to `--max-indel-length` (10) bases. The alignment is kept only if it - looks like an indel allele: at most 6 substitutions and 6 gap events. Reads from co-amplified - paralogs need many substitutions to fit, so they are rejected. A fast bit-vector edit-distance + edit limit, with gaps of up to `--max-indel-length` (10) bases. It keeps the alignment only if it + looks like an indel allele, with at most 6 substitutions and 6 gap events. Reads from co-amplified + paralogs need many substitutions to fit, so this test rejects them. A fast bit-vector edit-distance bound skips tags that cannot match before any alignment runs. -4. A read shorter than the tag is added to the one full-length allele whose start it matches +4. `generate` adds a read shorter than the tag to the one full-length allele whose start it matches exactly, but only if that allele has already passed the evidence rules and the read is longer than every prefix shared between two alleles of its locus. A partial read can therefore add to an allele's count, but it cannot create an allele or change an allele ratio. Reads shorter than `--min-partial-length` (tag length − 9) are never counted. +### Tiled markers + +Tiles are markers that read the same tag window of one amplicon. DArTag panels place several +target SNPs in one amplicon, under one shared (or nested) primer, and write each tile's tag with +its neighbours' SNPs as IUPAC codes. A read from the amplicon fits every tile equally well, so the +tie rule above would drop all of them and leave the tiles with zero reads. `--tile-groups` (default +`auto`) links tiles into families, either declared with `LocusGroup` or detected. See +[Tiled Markers](../getting-started/input-formats.md#tiled-markers) for the same-window test. + +When every marker tied for the best fit belongs to one family, the read came from that family's +amplicon. `generate` credits it to every member it fits within the identity limit, by substitution-only +identity or through one gapped alignment when gapped rescue is on. Each member +records the read's exact tag as its own observation. This holds whether or not the read tied. A +family member it fits less well still gets it. So each tile gets the reads it would get in a panel +of its own, as in DArT's pipeline. Partial reads follow the same rule. A tie between markers of different families, or between a family member and an unlinked +marker, is still dropped as ambiguous. + +Tiles share their reads, so their counts are not independent. The same molecules support every +member's row. Use the tiles' calls as one locus, or keep one tile per family, when an analysis +assumes independent markers. + ## Which Sequences Become Rows -Canonical Ref and Alt rows are always written, even with zero counts. +`generate` always writes the canonical Ref and Alt rows, even with zero counts. -Any other sequence becomes a row when it reaches `--min-global-count` reads project-wide **and** +Any other sequence becomes a row when it reaches `--min-global-count` reads project-wide and passes at least one enabled evidence route: | Route | Default | Rule | |-------|---------|------| -| **Reproducibility** (primary) | on | At least `--reproducible-min-reads` (10) reads in each of at least `--reproducible-min-samples` (2) samples | -| **Depth** | on | At least `--depth-min-reads` (30) reads within a single sample. Keeps private alleles discoverable in unreplicated designs | -| **Quality** | off | At least `--quality-min-clean-reads` reads whose every tag base reaches `--discovery-min-quality` (Q20) | +| Reproducibility (primary) | on | At least `--reproducible-min-reads` (10) reads in each of at least `--reproducible-min-samples` (2) samples | +| Depth | on | At least `--depth-min-reads` (30) reads within a single sample. Keeps private alleles discoverable in unreplicated designs | +| Quality | off | At least `--quality-min-clean-reads` reads whose every tag base reaches `--discovery-min-quality` (Q20) | Routes combine by OR. Identical tag sequences that recur independently across samples are strong evidence, even when one base position is systematically low quality, which is why reproducibility @@ -45,7 +67,7 @@ is primary and the whole-tag quality route is off by default. ### Supporting-frequency floor -`--min-supporting-freq` (default 0, off) additionally requires a sample's reads for the sequence to +`--min-supporting-freq` (default 0, off) also requires a sample's reads for the sequence to reach that share of the sample's locus total before the sample can support the reproducibility or depth route. @@ -58,19 +80,19 @@ depth route. ## Denoising -Canonical sequences and sequences that pass the evidence rules are protected: denoising never merges +Canonical sequences and sequences that pass the evidence rules are protected. Denoising never merges them into anything else. 1. Per-sample denoising (UNOISE-style abundance skew). Within each sample, a rare sequence that differs at no more than `--denoise-distance` (1) bases from a more abundant parent is merged into the parent when the - parent is at least `2^(alpha·d + 1)` times more abundant (`--denoise-alpha` 2.0). The closest - parent is chosen first, then the most abundant. + parent is at least `2^(alpha·d + 1)` times more abundant (`--denoise-alpha` 2.0). Denoising picks + the closest parent first, then the most abundant. 2. Pooled same-allele merge (`--global-denoise-distance`, 1). A sequence that failed the evidence rules is merged into the protected parent chosen from project-wide abundances, but only when both carry the same target allele, so no merge can change a sample's Ref/Alt dosage. -Reads in sequences that pass no route and have no parent are dropped, and the QC read accounting -reports them. Setting every route threshold to 0 writes every sequence with at least +`generate` drops reads in sequences that pass no route and have no parent, and the QC read +accounting reports them. Setting every route threshold to 0 writes every sequence with at least `--min-global-count` reads (exhaustive discovery). The same evidence rule decides discovery, denoising protection and partial-read crediting, so the @@ -78,7 +100,7 @@ three stay consistent. ## Classification -Each row is classified against its marker's canonical tags: +`generate` classifies each row against its marker's canonical tags: | Category | Meaning | |----------|---------| @@ -86,16 +108,16 @@ Each row is classified against its marker's canonical tags: | `Alt` | The canonical Alt tag | | `RefMatch` | Carries the Ref target allele plus other differences | | `AltMatch` | Carries the Alt target allele plus other differences | -| `Other` | Neither target allele: the target base is a third base, missing, or deleted | +| `Other` | Neither target allele. The target base is a third base, missing, or deleted | -The target base is normally read at its fixed offset in the tag. When a gapped alignment explains -the row with fewer edits, stays within the substitution and gap limits, and is confirmed by at least -12 bases after its last gap, the target base is read from that alignment instead. A target inside a -deletion, or pushed past the end of the read, is `Other`. +`generate` normally reads the target base at its fixed offset in the tag. When a gapped alignment +explains the row with fewer edits, stays within the substitution and gap limits, and is confirmed by +at least 12 bases after its last gap, the target base comes from that alignment instead. A target +inside a deletion, or pushed past the end of the read, is `Other`. ## IUPAC Codes and N Bases -Classification respects ambiguity codes in the design: if a design allele carries an ambiguity +Classification respects ambiguity codes in the design. If a design allele carries an ambiguity code, an observed base compatible with it classifies as RefMatch/AltMatch. Row identity and counts use exact sequence equality, so an ambiguity-coded canonical tag reports @@ -103,22 +125,35 @@ zero exact counts. MADCATS does not guess which concrete variants a curated IUPA meant to cover, because such codes can stand for real secondary SNPs, and collapsing them would merge distinct microhaplotypes. -A read containing N never becomes a row of its own. It is added to the one N-free sequence at its -locus that it is compatible with, or dropped (`reads_n_mapped` / `reads_n_dropped` in the QC). +A read containing N never becomes a row of its own. `generate` adds it to the one N-free sequence +at its locus that it is compatible with, or drops it (`reads_n_mapped` / `reads_n_dropped` in the QC). ## Indels Designed indel markers are fully supported. Vendors report indel alleles as fixed-length windows whose suffix shifts at the indel, so both tags are ordinary same-length sequences. -Undesigned indels are recovered by the gapped alignment step above and written as fixed-length -rows in the same shifted-suffix convention. Indels longer than `--max-indel-length` are not +The gapped alignment step above recovers undesigned indels, and `generate` writes them as +fixed-length rows in the same shifted-suffix convention. Indels longer than `--max-indel-length` are not recovered, and neither are alignments that need many substitutions, which usually come from paralogs. ## Read Accounting -The QC JSON accounts for every read: FASTQ records, reads assigned at full length, reads assigned -through a gapped alignment (`reads_assigned_gapped`), partial reads credited, ambiguous or unmapped, -reads moved by each denoising pass, reads dropped with low-evidence sequences, and per-sample -`reads_in_emitted_rows`. +The QC JSON accounts for every read. It counts FASTQ records, reads assigned at full length, reads +assigned through a gapped alignment (`reads_assigned_gapped`), partial reads credited, ambiguous or +unmapped, reads moved by each denoising pass, reads dropped with low-evidence sequences, and +per-sample `reads_in_emitted_rows`. + +A read credited to several tiles counts once in `reads_assigned_full_length` (`reads_assigned` +per sample). The QC JSON counts the further credits separately, as `tile_extra_credits_full_length` +and `tile_extra_credits_partial` (`tile_extra_credits` and `partial_tile_extra_credits` per sample). +From assignment on, the ledger counts (marker, read) credits in `reads_n_mapped`, `reads_n_dropped`, +`partial_reads_added_to_counts`, `reads_in_final_count_table`, the denoising moves and +`reads_in_emitted_rows`. For a panel without tiles, credits and reads are the same. The ledger +balances as + +``` +reads_in_final_count_table = reads_assigned_full_length + tile_extra_credits_full_length + − reads_n_dropped + partial_reads_added_to_counts +``` diff --git a/docs/src/reference/faq.md b/docs/src/reference/faq.md index 53284db..69141f1 100644 --- a/docs/src/reference/faq.md +++ b/docs/src/reference/faq.md @@ -4,53 +4,65 @@ ### Do I need a vendor MADC to run MADCATS? -No. Generation uses only the FASTQs plus the assay design. A vendor MADC can be used afterwards +No. Generation uses only the FASTQs plus the assay design. You can use a vendor MADC afterwards with `madcats validate`, or to bootstrap a panel when the design files are private (`scripts/madc_to_reference_tags.py`). ### Does MADCATS reproduce the vendor's MADC exactly? No, and it does not try to. MADCATS is an independent implementation. Canonical-row counts agree -closely (count r = 0.9997 on sweetpotato), while the novel-row lists differ: many vendor rows have -too little support in the project's own reads to pass any within-project evidence route. See -[Validation & Accuracy](validation.md). +closely (count r = 0.9997 on sweetpotato), while the novel-row lists differ, because many vendor +rows have too little support in the project's own reads to pass any within-project evidence route. +See [Validation & Accuracy](validation.md). ### What dependencies does it need? -None beyond Python 3.9+. The standalone binaries need nothing at all. +None beyond Python 3.9+. The standalone binaries need nothing at all. The optional +[`filter-paralogs`](../commands/filter-paralogs.md) command also needs the `mappy` aligner, +which `pip install 'madcats[genome]'` installs. + +### My vendor MADC has negative read counts + +Some vendor MADCs store counts in a signed 8-bit field. Counts of 128–255 appear as the count minus +256 (so −128 to −1), and counts of 256 or more wrap silently to small positive numbers. The lettuce +3K delivery has 90,913 negative cells across 1,108 markers, and its largest count is 127. +Adding 256 to the negatives recovers the 128–255 band, but the wrapped larger counts cannot be told +apart from real small ones. `validate` and `filter-paralogs` warn and record this in their reports +but use the counts as stored. Regenerate the table from the FASTQs with `madcats generate`, or ask +the vendor for a corrected file. ## Running ### My run stopped saying reads carry a constant prefix. What do I do? -You have a raw DArT delivery: an inline barcode and the common sequence `GATGGATCCAGTGC` precede -every amplicon. Re-run with `--leading-anchor auto`, or pass the sequence the message names. +You have a raw DArT delivery, in which an inline barcode and the common sequence `GATGGATCCAGTGC` +precede every amplicon. Re-run with `--leading-anchor auto`, or pass the sequence the message names. ### Should I trim adapters first? -No. Reads are located from the middle out, windows stay inside the amplicon, and `merge-pairs` drops +No. MADCATS locates reads from the middle out, windows stay inside the amplicon, and `merge-pairs` drops adapter read-through as part of merging. Trimming changed concordance by 0.05 points on the capture benchmark while costing 2 points of extraction. ### Can I split a large cohort into batches? -Batch `normalize-reads` freely, but run **one** `generate` over every sample. The reproducibility +Batch `normalize-reads` freely, but run one `generate` over every sample. The reproducibility route counts supporting samples within the run, so splitting a cohort into batches finds fewer new alleles. Use `--cache-dir` to make re-runs cheap. ### How many workers should I use? -`--jobs` sets the number of samples processed in parallel (default: up to 4). Each worker processes +`--jobs` sets the number of samples processed in parallel (by default up to 4). Each worker processes one sample at a time, so memory scales with `--jobs`. The finalisation (denoising) stage also runs across the workers. ### Re-running is slow. Can I reuse earlier work? -Pass `--cache-dir`. Per-sample assignment results are cached and keyed by the read-assignment -settings, so changing evidence, denoising or output options regenerates from cache in a fraction of -the time. Changing an assignment setting (identity, anchor, gapped alignment, partial-read length, tag -length) rebuilds the affected caches. +Pass `--cache-dir`. MADCATS caches per-sample assignment results and keys them by the +read-assignment settings, so changing evidence, denoising or output options regenerates from cache +in a fraction of the time. Changing an assignment setting (identity, anchor, gapped alignment, +partial-read length, tag length) rebuilds the affected caches. ## Panels @@ -62,16 +74,16 @@ primer-free inference only when neither is available. See ### Why do some canonical rows have zero counts? -Two common reasons: the locus differs from the design in your germplasm, or the vendor's curated -tag contains an IUPAC ambiguity code. Rows and counts use exact sequence equality, so an -ambiguity-coded canonical never matches a concrete read; the reads appear in RefMatch/AltMatch rows -instead. +Two common reasons are that the locus differs from the design in your germplasm, or that the +vendor's curated tag contains an IUPAC ambiguity code. Rows and counts use exact sequence equality, +so an ambiguity-coded canonical never matches a concrete read, and the reads appear in +RefMatch/AltMatch rows instead. ### What is `--reuse-tags` for, and when do I need it? -For `normalize-reads` panels only. Window placement is learned from coverage, so it can shift -between runs and make identical biology look like new haplotypes. Pass the first run's window CSV -to every later run of the same panel. +For `normalize-reads` panels only. `normalize-reads` learns window placement from coverage, so +placement can shift between runs and make identical biology look like new haplotypes. Pass the first +run's window CSV to every later run of the same panel. ## Discovery @@ -82,7 +94,7 @@ sample per marker even sequencing errors can pass them. Add `--min-supporting-fr ### How do I get only SNP microhaplotypes? -Filter on the `MarkerType` column: drop `indel` and `both` rows. Designed indel markers stay +Filter on the `MarkerType` column and drop `indel` and `both` rows. Designed indel markers stay intact. ### How do I get the exact vendor header? @@ -97,5 +109,6 @@ Generate with `--dart-preamble`. HapApp expects the vendor's starred preamble ro ### Where did a sample's reads go? -`OUTPUT.qc.json` accounts for every read per sample: assigned, ambiguous, without the leading -anchor, too short, moved by denoising, dropped with low-evidence sequences, and in emitted rows. +`OUTPUT.qc.json` accounts for every read per sample. It counts the reads assigned, ambiguous, +without the leading anchor, too short, moved by denoising, dropped with low-evidence sequences, +and in emitted rows. diff --git a/docs/src/reference/output-formats.md b/docs/src/reference/output-formats.md index 8d861a5..4ea845a 100644 --- a/docs/src/reference/output-formats.md +++ b/docs/src/reference/output-formats.md @@ -3,7 +3,7 @@ ## MADC Table `madcats generate --output` writes a CSV with one row per allele and one integer read-count column -per sample. Rows are grouped by marker and ordered Ref, Alt, RefMatch, AltMatch, Other. +per sample. It groups rows by marker and orders them Ref, Alt, RefMatch, AltMatch, Other. ``` AlleleID,CloneID,AlleleSequence,ClusterConsensusSequence,CallRate,...,RatioAvgCountRefAvgCountSnp,4404001,4404002,... @@ -16,20 +16,20 @@ Chr01_000123|RefMatch,Chr01_000123,ACGT...,ACGT...,0,...,.,57,0,... | Column | Contents | |--------|----------| -| `AlleleID` | Marker name and category joined by a pipe, e.g. `Chr01_000123\|AltMatch`. With `--number-match-ids`, Match/Other rows get a stable serial: `Chr01_000123\|RefMatch_0001` | +| `AlleleID` | Marker name and category joined by a pipe, e.g. `Chr01_000123\|AltMatch`. With `--number-match-ids`, Match/Other rows get a stable serial, such as `Chr01_000123\|RefMatch_0001` | | `CloneID` | Marker name | | `AlleleSequence` | The row's tag sequence | -| `ClusterConsensusSequence` | Canonical rows carry their own sequence; Match/Other rows carry the Ref tag, following vendor convention | -| `CallRate` | Canonical rows: fraction of samples with canonical Ref or Alt reads at the locus. Other rows: `0` | -| `OneRatioRef` … `PICRef` | Vendor downstream statistics, not FASTQ evidence: emitted as `.` | -| `PICSnp` → **`Variants`** | With `--annotations on` (default): the row's events against the Ref tag. Otherwise `.` | -| `AvgPIC` → **`MarkerType`** | With `--annotations on` (default): what the row carries beyond its canonical. Otherwise `.` | -| `AvgCountRef`, `AvgCountSnp` | Canonical rows: nonzero means of the Ref and Alt counts. Other rows: the row's own nonzero mean, in both columns | +| `ClusterConsensusSequence` | Canonical rows carry their own sequence. Match/Other rows carry the Ref tag, following vendor convention | +| `CallRate` | For canonical rows, the fraction of samples with canonical Ref or Alt reads at the locus. For other rows, `0` | +| `OneRatioRef` … `PICRef` | Vendor downstream statistics, not FASTQ evidence, so emitted as `.` | +| `PICSnp` → `Variants` | With `--annotations on` (default), the row's events against the Ref tag. Otherwise `.` | +| `AvgPIC` → `MarkerType` | With `--annotations on` (default), what the row carries beyond its canonical. Otherwise `.` | +| `AvgCountRef`, `AvgCountSnp` | For canonical rows, nonzero means of the Ref and Alt counts. For other rows, the row's own nonzero mean, in both columns | | `RatioAvgCountRefAvgCountSnp` | `.` | | one column per sample | Integer read counts | -Sample columns are named by the FASTQ stem (or a unique `--sample-name-field` from target -manifests), or renamed with `--sample-lookup`. +MADCATS names sample columns by the FASTQ stem (or a unique `--sample-name-field` from target +manifests), or `--sample-lookup` renames them. ### Annotations @@ -37,7 +37,7 @@ manifests), or renamed with `--sample-lookup`. column *positions* are unchanged, so positional consumers see the same 16-column layout. `--annotations off` restores the exact vendor header and writes no sidecar. -**`MarkerType`** says what the row carries beyond its canonical: +**`MarkerType`** records what the row carries beyond its canonical: | Value | Meaning | |-------|---------| @@ -59,8 +59,8 @@ the tag as printed: | `clip=4` | An insertion pushed the last 4 Ref bases out of the window | | `overhang=3` | A deletion pulled 3 downstream bases into the window | -Example: `55:C>CAAAA;91:A>T;clip=4`. The Ref row is `.`; the Alt row of a designed SNP is exactly the -design (e.g. `46:C>G`). +An example value is `55:C>CAAAA;91:A>T;clip=4`. The Ref row is `.`, and the Alt row of a designed SNP +is exactly the design (e.g. `46:C>G`). ## Variants Sidecar @@ -79,29 +79,55 @@ design (e.g. `46:C>G`). `OUTPUT.qc.json` records: - the pipeline version and every parameter - input paths -- per-sample read accounting: records read, reads assigned, gapped and partial assignments, reads +- per-sample read accounting of records read, reads assigned, gapped and partial assignments, reads in emitted rows, reads moved by denoising, reads dropped and why +- the tile families (`tile_families`) and the extra credits they add (`tile_extra_credits_*`, + described in [Read Accounting](evidence-model.md#read-accounting)) - row counts by category, by evidence route, and by `marker_types` - the variants sidecar path -`normalize-reads`, `infer-design` and `window-scan` write their own QC JSONs; see each command's page. +`normalize-reads`, `infer-design`, `window-scan` and `filter-paralogs` write their own QC JSONs. See +each command's page. + +## Genome Labels (filter-paralogs) + +[`madcats filter-paralogs`](../commands/filter-paralogs.md) writes a filtered copy of any MADC. It +copies kept rows unchanged. In flag mode (`--paralogs flag` or `--ambiguous flag`) it repurposes a +third statistics column, keeping the 16-column layout: + +| Column | Contents | +|--------|----------| +| `PICRef` → `GenomeLabel` | `on_target`, `paralog`, `ambiguous`, `unmapped` or `unplaced`, depending on where the reference genome places the row's haplotype | + +It also writes `OUTPUT.genome_labels.csv` (one line per input row, dropped rows included, with +the row's score at its marker and its best score elsewhere) and `OUTPUT.genome_markers.csv` (where +it placed each marker). The command page describes both. + +## Genotype VCFs + +`madcats call` writes up to three VCFs from a MADC, chosen with `--vcf`. They are `PREFIX.targets.vcf` +(the designed variant of each marker), `PREFIX.all.vcf` (targets plus other base substitutions in +the tags) and `PREFIX.haplotypes.vcf` (whole tag sequences as alleles). It also writes a QC JSON and, +for the haplotype VCF, a table tying each allele number to its MADC rows. Coordinates are +marker-local, so `CHROM` is the marker and `POS` a position on its Ref tag. The fields and the +model behind them are on the [command page](../commands/call.md#outputs). ## Vendor-Style Preamble (HapApp) `--dart-preamble` prepends the seven starred rows that vendor raw MADCs carry above the header, so preamble-expecting consumers such as HapApp and HapApp_utils accept the file: -1. service/project label (`--preamble-project`, default: output filename before its first dot) +1. service/project label (`--preamble-project`, by default the output filename before its first dot) 2. plate barcode 3. plate number 4. plate row 5. plate column 6. plate name -7. `targetid`: the FASTQ-derived sample ids, directly above the header's sample names +7. `targetid`, the FASTQ-derived sample ids, directly above the header's sample names Every empty preamble cell is `*`, as in vendor output. The file's first cell holds a full provenance string (pipeline, version, and every generation parameter as `key=value`), prefixed with -`* ` so consumers that skip starred lines still skip it. No timestamp is included, so output stays +`* ` so consumers that skip starred lines still skip it. The file includes no timestamp, so output stays byte-reproducible. Combine with `--sample-lookup` to rename sample columns while the targetid row keeps the FASTQ ids, @@ -112,11 +138,12 @@ matching the raw vendor layout. A MADCATS table is a raw MADC. Its allele IDs are per-project (`|RefMatch`, `|AltMatch`, ...) and are not yet matched to the panel's fixed allele IDs. Run it through HapApp or HapApp_utils (generate with `--dart-preamble`) before using it in BIGr, updog or other downstream tools. HapApp -assigns the fixed allele IDs and maintains the panel's cross-project allele database; MADCATS does -neither. +assigns the fixed allele IDs and maintains the panel's cross-project allele database, and MADCATS does +neither. [`madcats call`](../commands/call.md) is the exception. It calls genotypes from the raw +MADC directly, identifying haplotypes by their sequences. polyRAD's `readDArTag` is one example. In polyRAD 2.0.1 it rejects raw MADCs, from MADCATS or the -vendor: Match rows share allele IDs, and the statistics columns are read as samples. It does read +vendor. Match rows share allele IDs, and it reads the statistics columns as samples. It does read the processed layout (unique allele IDs, then `AlleleID`, `CloneID`, `AlleleSequence` and the sample columns). @@ -130,5 +157,5 @@ them later. ## Format References -- Breeding Insight: [MADC file format](https://breeding-insight.github.io/BIGr_documentation/posts/madc-file-format/) -- Breeding Insight: [MADC data processing](https://breeding-insight.github.io/BIGr_documentation/posts/madc-data-processing/) +- [MADC file format](https://breeding-insight.github.io/BIGr_documentation/posts/madc-file-format/) from Breeding Insight +- [MADC data processing](https://breeding-insight.github.io/BIGr_documentation/posts/madc-data-processing/) from Breeding Insight diff --git a/docs/src/reference/validation.md b/docs/src/reference/validation.md index f69b891..3ac87c3 100644 --- a/docs/src/reference/validation.md +++ b/docs/src/reference/validation.md @@ -10,17 +10,17 @@ version-to-version comparison, is in the [development record](https://github.com loss is off-target and primer-dimer reads that should be rejected, and other pipelines calculate their percentages differently. -"Genotype agreement" means the MADCATS table was run through HapApp, BIGr and updog to call dosages, +"Genotype agreement" means the MADCATS table went through HapApp, BIGr and updog to call dosages, and those calls were compared with the reference genotypes. | Platform (dataset) | Reads → usable windows | Accuracy | Validated against | |---|---|---|---| -| DArTag (sweetpotato, red clover, blackberry, alfalfa, pecan) | 56.5% of delivered records, vs the vendor's 50.9% on the same FASTQs | Sweetpotato count r = 0.9997 (1.20M cells); red clover dosage agreement 96.98%; blackberry 95.39%; alfalfa 98.40% / 97.40%; pecan 98.78% | Vendor MADC, vendor/updog dosage, families, skim-seq VCF | -| AgriSeq, Ion Torrent (2 honeybee chips; 372-sample oyster) | 88.4% | 94.9% vs Torrent VariantCaller (152,698 cells); honeybee replicate concordance 98.0% | Vendor caller + replicates | -| GT-seq (96-sample delta smelt) | 84.5% merged | 99.49% R1-only; 98.42% genotype agreement; `infer-design` 368/368 | Independent BWA/bcftools truth VCF | -| Flex-Seq capture (192-sample cranberry) | 81.9% | 95.13% genotype agreement (295,068 cells); trio incompatibility 3.6% vs 6.2% for the comparison VCF's source | Published population VCF + 20 pedigree trios | -| rhAmpSeq (608 runs, 8 grape families) | 86.6% | Family transmission 99.92–99.99%; 93.0% exact agreement with the platform's own pipeline on identical reads, with +19% / +79% more callable cells at 10x / 30x | Mendelian constraints + same-reads head-to-head | -| Synthetic truth suite | — | Dosage 100% on clean, linked, paralog and repeat classes; paired-end merge 100% with 98.6% molecular recovery | Exact simulated truth | +| DArTag (sweetpotato, red clover, blackberry, alfalfa, pecan) | 56.5% of delivered records, vs the vendor's 50.9% on the same FASTQs | Sweetpotato count r = 0.9997 (1.20M cells), red clover dosage agreement 96.98%, blackberry 95.39%, alfalfa 98.40% / 97.40%, pecan 98.78% | Vendor MADC, vendor/updog dosage, families, skim-seq VCF | +| AgriSeq, Ion Torrent (2 honeybee chips, 372-sample oyster) | 88.4% | 94.9% vs Torrent VariantCaller (152,698 cells) and honeybee replicate concordance 98.0% | Vendor caller + replicates | +| GT-seq (96-sample delta smelt) | 84.5% merged | 99.49% R1-only, 98.42% genotype agreement, `infer-design` 368/368 | Independent BWA/bcftools truth VCF | +| Flex-Seq capture (192-sample cranberry) | 81.9% | 95.13% genotype agreement (295,068 cells) and trio incompatibility 3.6% vs 6.2% for the comparison VCF's source | Published population VCF + 20 pedigree trios | +| rhAmpSeq (608 runs, 8 grape families) | 86.6% | Family transmission 99.92–99.99%, and 93.0% exact agreement with the platform's own pipeline on identical reads, with +19% / +79% more callable cells at 10x / 30x | Mendelian constraints + same-reads head-to-head | +| Synthetic truth suite | — | Dosage 100% on clean, linked, paralog and repeat classes, and paired-end merge 100% with 98.6% molecular recovery | Exact simulated truth | Accuracy depends more on the quality of the comparison data than on the platform. Where the reference genotypes are reliable (GT-seq, DArTag), MADCATS agrees at 98–99% or better. Against @@ -31,8 +31,8 @@ checks show the MADCATS calls to be the more consistent of the two. ### Sweetpotato (hexaploid, 81-bp tags) -The development project: 80 samples, 3,120 markers, 154.6M reads. Against the vendor MADC (19,319 -rows), every shared row had identical classification. +The development project has 80 samples, 3,120 markers and 154.6M reads. Against the vendor MADC +(19,319 rows), every shared row had identical classification. | Configuration | Rows | Precision | Recall | |---|---|---|---| @@ -40,63 +40,65 @@ rows), every shared row had identical classification. | `--reproducible-min-reads 20` | 16,517 | 89.8% | 76.8% | | `--min-supporting-freq 0.01` | 17,319 | 86.2% | 77.2% | -Row recall against a vendor table has a ceiling: 27.3% of the vendor's positive-count rows have +Row recall against a vendor table has a ceiling, because 27.3% of the vendor's positive-count rows have fewer than 10 reads in fewer than 2 samples of the vendor's own counts, below any within-project evidence threshold. ### Red clover (diploid, 109-bp tags, ~60x depth, no primer file) -- `infer-design` recovered 2,968 loci; 2,902 exactly match the vendor canonicals and 56 differ only - by vendor IUPAC curation. -- Recall against vendor rows with enough read support to pass an evidence route: 94.1%. -- Updog dosage concordance with the vendor-derived calls: 99.73% at DP ≥ 10. +- `infer-design` recovered 2,968 loci, of which 2,902 exactly match the vendor canonicals and 56 + differ only by vendor IUPAC curation. +- Recall against vendor rows with enough read support to pass an evidence route was 94.1%. +- Updog dosage concordance with the vendor-derived calls was 99.73% at DP ≥ 10. - In a 152-progeny mapping family, the Mendelian-violation rate was 1.30% under MADCATS against - 1.59% under the vendor MADC. The heterozygotes created by recovered indel alleles are inherited. + 1.59% under the vendor MADC. The progeny inherit the heterozygotes created by recovered indel + alleles. ### Blackberry (mixed ploidy, 109-bp tags, indels, degenerate primers) The first project validated through all three panel routes: -- Design files: 2,881 / 3,015 exact; all 134 others differ only by vendor IUPAC codes; all 10 - indel loci exact. -- Vendor-MADC bootstrap: 92.9% precision, and 94.7% recall against vendor rows with enough read - support to pass an evidence route. -- Primer-free inference: 2,943 inferred; 18 wrong windows (0.6%), all at read-starved or +- With design files, 2,881 / 3,015 were exact, all 134 others differ only by vendor IUPAC codes, + and all 10 indel loci were exact. +- The vendor-MADC bootstrap gave 92.9% precision, and 94.7% recall against vendor rows with enough + read support to pass an evidence route. +- Primer-free inference returned 2,943 with 18 wrong windows (0.6%), all at read-starved or short-amplicon loci. ### Alfalfa and pecan (raw deliveries) -- Alfalfa (two 2021 orders, 54-bp design): category agreement 99.60% / 99.83%, dosage agreement - 98.40% / 97.40%; F1 and BC1 progeny haplotypes within the parental sets at 98.6–98.7% (vendor - 98.9%). -- Pecan 3K (81-bp design): category agreement 98.89%, dosage agreement 98.78% on 1.02M cells; F1 - progeny within the parental set at 99.82% (vendor 99.86%); 93.3% agreement with skim-sequencing - genotypes on 11 shared cultivars (vendor 93.3%). +- Alfalfa (two 2021 orders, 54-bp design) had category agreement of 99.60% / 99.83% and dosage + agreement of 98.40% / 97.40%. F1 and BC1 progeny haplotypes fell within the parental sets at + 98.6–98.7% (vendor 98.9%). +- Pecan 3K (81-bp design) had category agreement of 98.89% and dosage agreement of 98.78% on 1.02M + cells. F1 progeny fell within the parental set at 99.82% (vendor 99.86%), and agreement with + skim-sequencing genotypes on 11 shared cultivars was 93.3% (vendor 93.3%). ## AgriSeq (Ion Torrent) -- Honeybee: two chips on an 81-marker panel. The curated frozen 41-bp panel scored 93.4% / +- Honeybee used two chips on an 81-marker panel. The curated frozen 41-bp panel scored 93.4% / 93.9% vendor-caller concordance and 98.0% concordance across 30 technical replicate pairs. Freezing windows raised cross-chip haplotype sharing from 42% to 88%. -- Pacific oyster (PRJNA1118744; 372 samples, 591 amplicons): with the honeybee defaults +- In Pacific oyster (PRJNA1118744, 372 samples, 591 amplicons), with the honeybee defaults unmodified, 88.4% of reads normalized and genotypes agreed with Torrent VariantCaller at 94.9%, on a different lab, species, chip generation and target length. ## GT-seq (delta smelt, Zenodo 10.5281/zenodo.19069550) -96 samples, 410 loci, and the panel's real primer file. Genotypes agreed with the archive's +The run used 96 samples, 410 loci, and the panel's real primer file. Genotypes agreed with the archive's independent BWA/bcftools VCF at 99.49%. Merged reads with frozen tags plus tiles kept that accuracy on the original markers while adding 54% more truth-validated calls. `infer-design` with the primer file withheld recovered 368/368 loci exactly. ## Flex-Seq Capture (cranberry) -192 samples with a K=4 tiled 110-bp panel and one cohort-level `generate`. Genotype agreement with +The benchmark used 192 samples with a K=4 tiled 110-bp panel and one cohort-level `generate`. +Genotype agreement with the published population VCF was 95.13%. Trio incompatibility was 3.6%, against 6.2% for the raw calls behind the comparison VCF. ## rhAmpSeq (grape, PRJNA281110) -Eight biparental families (763 runs) processed against one frozen 2,951-window panel. Across +MADCATS processed eight biparental families (763 runs) against one frozen 2,951-window panel. Across ~14,900 locus-family tests, 99.92–99.99% of haplotype calls in every family are alleles of that locus's four inferred parental haplotypes. Against the platform's own `analyze_amplicon` pipeline on identical reads, MADCATS was the more Mendelian-consistent of the pair (99.947% vs 99.850%). @@ -106,3 +108,271 @@ on identical reads, MADCATS was the more Mendelian-consistent of the pair (99.94 Across 127,667 emitted rows in six MADCs spanning five platforms, zero rows contain TruSeq or Ion P1 adapter at a strict 13-mer probe. Trimming adapters before a run changed dosage concordance by 0.05 points on the capture benchmark while costing 2 points of extraction. + +## Genotype Calling (`call`) + +`call` adapts polyRAD's model and parts of updog's, so it was checked against both packages and +against data with known or independent genotypes. The tables are in the development record. + +- **Same answers as polyRAD.** With polyRAD's settings `call` gives polyRAD's genotypes. On three + alfalfa MADCs all but 0.01% of 6.8 million target genotype values are identical, and on the polyRAD + paper's own benchmark (five diversity panels) the error of `call --prior hwe` equals the + published error of polyRAD's HWE model to the third decimal. +- **Potato against a SNP array.** On the deposit of Endelman et al. (2024), the study's stored + genotypes scored here reproduce the paper's own per-marker error table. With the bias fitted, + `call` has the array's dosage for 87.5% of 550,052 genotypes of 298 tetraploid clones, where the + study's updog genotypes have 87.2%. Without it the share is 82.8%, level with polyRAD at the same + overdispersion and 5 points above polyRAD's defaults. +- **A vendor MADC of mixed ploidy.** `call` genotyped the same study's DArT MADC (375 samples, 52 of + them diploid) as delivered, and 91.9% of 202,143 genotypes have the array's dosage with the bias + fitted (91.8% for the study's genotypes) and 89.1% without. +- **Real diploid reads.** On 96 delta smelt (GT-seq), 99.6% of target genotypes and 99.8% of + off-target genotypes that pass the default filters agree with genotypes called from alignments + of the same reads, while polyRAD and updog on the same counts agree less often. +- **Polyploids with known dosages from real reads.** On tetraploids and hexaploids pooled from the + smelt diploids, at full depth and thinned to 50, 25 and 10 reads, `call` is level with updog at + full depth (98.3% of tetraploid and 96.9% of hexaploid target genotypes right) and ahead of it + once reads are thinned, by 1 to 4 points for tetraploids and 6 to 10 for hexaploids. +- **Hexaploid replicates.** On ten samples of each of eight sweetpotato clones, with the read + counts the study's updog run used, two samples of a clone get the same dosage in 86.7% of + pairs with `call` (norm prior, bias fitted) and 87.3% with updog. +- **Simulated tables with known dosages**, from updog's and polyRAD's own simulators, covering diploid, + tetraploid and hexaploid, populations, groups and families, 8 to 150 reads, with and without + allelic bias. + +The checks also found limits. Markers on one subgenome of an allopolyploid are outside the model. +The model does not use linkage between markers, and polyRAD's models that use it are better on +mapping populations. Fitting the allelic bias helps where one allele gives fewer reads (potato) +and costs accuracy where it does not, so it is off by default. + +See [`call`](../commands/call.md). + +### Priors + +**`norm` in families.** It helped in simulations and at low depth, and not on real families at +high depth: + +- In five draws of simulated tetraploid F1 families at 20 to 60 reads, 91.0% of dosages were right + against 90.1% with `hwe`, and it was ahead in every draw (0.6 to 1.5 points). On a biased panel it + was 2.5 points ahead with `--allele-bias auto` and 4.4 with `fit`, and on updog's simulator at + 150 reads, 7.6 and 10.2 (91.5% and 91.6% against 84.0% and 81.4%). +- With `norm`, `auto` kept no bias on F1 families without one, while with `hwe` it kept a bias at 2 + markers in 100 and lost 0.7 points. +- On the two F1 populations of the polyRAD paper's benchmark, at about 10 reads, the error was + 0.186 against 0.200 (*Miscanthus*, diploid) and 0.385 against 0.409 (potato, tetraploid), level + with polyRAD's mapping prior, which uses the known parents. +- Two tetraploid alfalfa F1 populations against published dosages gave 90.1% and 89.8% with + `norm`, and 90.9% and 90.0% with `hwe`. With the bias fitted, 92.3% and 91.9% against 92.2% + and 91.8%. +- In 85 potato clones of ten families, the share of dosages matching a SNP array was 92.1% against + 92.3%, and 96.57% against 96.60% with the bias fitted. +- In one randomly mating population (five draws) it was level with `hwe` where the data have no + bias, and 0.5 to 0.7 points below where they have one and it is not fitted or `auto` is used. +- On simulated inbred lines without a selfing rate it was 1.1 points ahead of `hwe`. + +**`structure`.** It matters at low read depth: + +- On simulated tetraploids with 8 reads per sample and marker it called 0.8 to 5.5 points more + genotypes correctly than `hwe`, depending on how strong the structure was. With 25 reads it gained + 0.5 to 2.5 points, and with 150 reads 0.8. +- On 85 potato clones from ten families, thinned to 10 and 25 reads, the gain against SNP-array + genotypes was 0.9 and 0.7 points, but at their full depth of about 300 reads there was none. +- Compared with polyRAD's `IteratePopStruct` on the same thirteen tables it called the same share + of genotypes correctly or more, by up to 0.7 points. On a table without structure polyRAD's + prior lost 3.4 points and `call` chose 0 neighbours. +- polyRAD models structure with principal components. That approach and a relationship-matrix + predictor did no better in place of the neighbours (see the development record). + +**The selfing rate.** On the inbred soybean set of the polyRAD benchmark the error was 0.037 with +`--selfing-rate 0.95`, as polyRAD has, and 0.157 without. On simulated inbred tetraploids 99.6% +and 99.9% of dosages were right at 0.95 and 0.99, against 97.4% without. On simulated half-inbred +tetraploids (four draws) 93.9% were right with `--selfing-rate 0.5`, against 93.5% without and +94.1% with `--prior norm`. + +**Haplotype records.** On simulated microhaplotype tables (3 to 6 alleles per marker, diploid and +tetraploid, with one population, groups and families): + +- `hwe` gave the same share of correct genotypes as polyRAD's HWE prior, to within 0.4 points. +- `structure` was never below `hwe` and gained 1 to 4 points in tetraploids with groups or + families. +- `norm` was never below `hwe`, gained at most 1.7 points, and took two to five times as long. +- `uniform` was the lowest throughout, by up to 6 points. +- With the default `--min-marginal-posterior`, 94 to 99% of the genotypes kept were right under + `hwe`, `norm` and `structure` alike. + +These are simulations with one simulator. There is no published accuracy baseline for haplotype +calls to compare with, and no test has used real haplotype data with known genotypes. + +**Off-target records.** On simulated microhaplotype tables (diploid and tetraploid, 8 to 150 +reads) `call` genotyped off-target records as accurately as target records, whether a substitution was +on one haplotype or shared by two, and 96 to 99.8% of the off-target genotypes that passed the +default filters were right. On real reads of 96 diploid delta smelt, 99.8% of the off-target +genotypes that passed the default filters agreed with genotypes called from alignments of the +same reads, against 99.6% at the targets. + +### Allelic Bias + +Share of dosages right with `--prior hwe`, with the rules of `auto` as released. Real data: + +| Data | `off` | `fit` | `auto` | Biases `auto` used | +|------|-------|-------|--------|--------------------| +| Potato, 85 tetraploid clones, against a SNP array | 92.3% | 96.6% | 96.4% | 785 of 1,544 | +| Potato vendor MADC, tetraploids | 86.4% | 91.3% | 91.1% | 2,566 of 3,904, with the diploids | +| Potato vendor MADC, diploids | 92.0% | 92.6% | 92.5% | | +| Delta smelt (diploid), against alignment-based genotypes | 99.4% | 99.3% | 99.3% | 127 of 946 | +| Tetraploids pooled from real diploid reads, 50 reads, three draws | 90.4% | 89.5% | 90.4% | 15 to 29 of 946 | +| Hexaploids pooled from real diploid reads, 50 reads, three draws | 77.8% | 71.0% | 77.8% | 3 to 12 of 944 | + +Simulated tables, as the mean over five draws (four for updog's simulator): + +| Scenario | `off` | `fit` | `auto` | Biases `auto` used | +|----------|-------|-------|--------|--------------------| +| Tetraploids, random mating, no bias, 20 to 60 reads | 89.7% | 89.2% | 89.7% | none | +| Tetraploids, random mating, biased, 20 to 60 reads | 83.9% | 88.7% | 87.0% | 24% | +| Hexaploids, random mating, no bias, 20 to 60 reads | 75.0% | 73.3% | 75.0% | none | +| Hexaploids, random mating, biased, 20 to 60 reads | 66.7% | 72.6% | 67.8% | 7% | +| Tetraploid inbred lines, no bias | 97.3% | 73.7% | 97.3% | none | +| updog's simulator, tetraploids, biased, 150 reads | 84.8% | 96.5% | 96.3% | 72% | +| updog's simulator, tetraploids, biased, 25 reads | 76.1% | 84.7% | 82.8% | 35% | +| updog's simulator, hexaploids, biased, 150 reads | 66.7% | 85.8% | 82.6% | 52% | +| updog's simulator, diploids, biased, 150 reads | 99.6% | 100.0% | 100.0% | 75% | + +- **Where a bias is there, fitting it gains 4 to 5 points in tetraploid potato** and up to 19 + in simulated hexaploids. It brought `call` level with updog on biased polyploid data (within 0.7 + points on seven tables from updog's simulator), where calling without it was 8 to 20 points + behind. Against the potato array, at the markers updog found biased the share rose from 80% to + 95%. On the whole deposit of that study (703 clones, 1,862 markers, none selected) `fit` raised + the share from 82.8% to 87.5%, where the study's own updog genotypes have 87.2%. +- **Where no bias is there, `auto` leaves the calls alone.** In one population and in inbred + lines it kept no bias in any draw, and its genotypes are those of `off`. `fit` lost 0.1 to 1.8 + points in one population, 7 on hexaploids pooled from real reads and 24 on inbred lines. +- **`auto` needs depth.** Of what `fit` gains it kept 97 to 100% at 150 reads in diploids and + tetraploids and 83% in hexaploids, 78% at 25 reads, and at 20 to 60 reads with smaller biases + 54 to 68% in tetraploids and 19% in hexaploids. +- **Inbred lines.** A bias changes only what a heterozygote shows. Under a prior that expects + heterozygotes, every fit ran to a limit of the search and homozygous samples passed for + heterozygous ones. `auto` rejects a bias at a limit, but `fit` does not. +- **Families, subpopulations and partly inbred samples without bias.** The `hwe` prior + does not fit them, and a bias can make up for it. `auto` kept a bias at 2, 4 and 12 markers in + 100 and lost 0.7, 0.6 and 0.8 points. With `--prior norm` it kept none in F1 families and lost + 0.1 points in subpopulations and in half-inbred samples, and with `--selfing-rate 0.5` it lost 0.2 + in half-inbred samples. None of these biases ends at a search limit. +- **Mixed ploidy.** With diploids, tetraploids and hexaploids in one table (polyRAD's simulator, + no bias), `auto` was within 0.1 points of `off` in every draw. +- **Real diploid reads.** On 96 delta smelt `auto` and `fit` were 0.1 points under `off` at + targets and 0.3 at off-target SNPs. + +### Calibration + +How far the probability `call` gives its called genotypes lies above the share of them that are +right, in points. Positive is over-confident. The probability comes from `GQ`. Simulated +figures are means over the draws. + +| Data | `off` | `auto` | `fit` | +|------|-------|--------|-------| +| Simulated tetraploids, random mating, no bias | −0.1 | −0.1 | +0.6 | +| Simulated tetraploids, random mating, biased | +1.9 | −0.3 | +1.1 | +| Simulated hexaploids, random mating, no bias | −0.5 | −0.5 | +4.0 | +| Simulated hexaploids, random mating, biased | +4.6 | +3.7 | +5.0 | +| Simulated tetraploid inbred lines | +0.4 | +0.4 | +14.4 | +| updog's simulator, tetraploids, biased, 150 reads | +5.4 | −0.5 | +0.5 | +| updog's simulator, hexaploids, biased, 150 reads | +10.8 | 0.0 | +2.2 | +| updog's simulator, tetraploid F1, biased, `--prior hwe` | +8.6 | +11.1 | +14.7 | +| Tetraploids pooled from real reads, three draws | +1.1 | +1.2 | +2.3 | +| Hexaploids pooled from real reads, three draws | +0.8 | +0.8 | +8.3 | +| Potato clones against the array | +4.8 | +2.3 | +2.5 | + +- Without a bias in the data, `off` and `auto` are within 1.2 points. `fit` is over-confident + by 4 to 8 points in hexaploids and 14 on inbred lines. +- With a bias in the data, `off` is over-confident by up to 11 points, because a biased marker is + called wrongly and confidently. `auto` and `fit` are within 2.2 points at 150 reads. +- A prior that does not fit the samples costs calibration with any bias setting. F1 families + under `--prior hwe` are over-confident by 9 to 15 points. +- Against the potato array the gap is not all the caller's, because the array and the reads also differ + for reasons a caller cannot see. A genotype with a `GQ` of 30 or more was wrong 6 to 7 times in + 100 there. + +### MPP and PMC + +`MPP` and `PMC` are updog's `maxpostprob` and `prop_mis`. They give the probability of a sample's +most probable genotype, and one minus its mean over a marker's samples. + +- **The same numbers as updog.** With updog's prior, bias and overdispersion, `MPP` was within + 0.001 of `maxpostprob` for every sample and `PMC` equal to `prop_mis` to four decimals, on 14 + stored updog fits covering diploids, tetraploids and hexaploids, `norm` and Hardy-Weinberg + priors, with and without a bias. The tests repeat the comparison. +- **Samples without reads.** updog keeps a sample given a depth of 0 in the mean, at the largest + probability of the prior. In a stored fit with 8 such samples among 100 its `prop_mis` was + 0.155, and 0.117 with them passed as missing. `call` writes 0.117. +- **Against known genotypes.** Averaged over a table, `PMC` minus the share of wrong genotypes is + the calibration gap above with its sign turned. For one marker of 150 samples it was off by 1.7 + to 2.7 points on average where the model fits. +- **As a marker filter.** On two potato panels with array genotypes for every marker, removing + the markers with a `PMC` of 0.05 or more (356 of 1,862 and 290 of 2,602) raised agreement with + the array from 91.3% to 92.7% and from 93.3% to 94.2%. Removing those with a call rate under + 0.9 raised it to 94.1% and 95.0%, and took out every marker the `PMC` cut did. + +`PMC` minus the share of wrong genotypes, in points, beside updog's `prop_mis` on the same draw +of each table. Negative is too low. `call` is `--prior hwe --allele-bias auto`: + +| Data | `call` | updog `norm` | +|------|--------|--------------| +| updog's simulator, tetraploids, 150 reads | −0.3 | −0.6 | +| updog's simulator, tetraploids, biased, 150 reads | +0.1 | −0.7 | +| updog's simulator, tetraploids, biased, 25 reads | +0.3 | −2.1 | +| updog's simulator, hexaploids, biased, 150 reads | −1.5 | −6.2 | +| updog's simulator, tetraploid F1, biased | −13.4 | −6.9 | +| polyRAD's simulator, tetraploids, 150 reads | +0.9 | −5.6 | +| polyRAD's simulator, tetraploids, 25 reads | −0.6 | −8.5 | +| polyRAD's simulator, hexaploids, 150 reads | −0.4 | −22.8 | +| polyRAD's simulator, tetraploids, inbreeding 0.5 | −1.9 | −3.8 | +| Potato clones against the array | −2.3 | −2.3 | + +- With `--prior norm`, `call` was 4.7 too low on the F1 table and 0.7 on the partly inbred one. +- On polyRAD's tables updog's overdispersion, fitted at every marker, read less scatter than was + simulated, with a median of 41 to 248 on `call`'s scale against a simulated 20. That makes it surer + than the counts warrant. `call` takes one overdispersion from the whole table. + +### Overdispersion and Background Reads + +- In simulated diploids and tetraploids with 20 or more reads per sample the estimate recovered + both values (31 of 32 tables). In hexaploids it recovered the overdispersion in 12 of 16 tables + and read it as between 14 and 50 for a simulated 40 in the rest. With 6 to 25 reads per sample + it still recovered the background rate, read an overdispersion of 40 as 30 to 75, and one of + 150 as anything from 100 to 500. Scatter that small cannot be measured at that depth. +- The estimate gives every dosage its own free probability at each marker. With Hardy-Weinberg + proportions it was badly off for samples from two subpopulations, and with those or the `norm` + shape, neither of which can describe an excess of homozygotes, it failed for inbred samples. +- polyRAD's `TestOverdispersion` chose values well below the true one on simulated tetraploid and + hexaploid counts. On 85 tetraploid potato clones it chose 14 where `call` estimates 75, and + polyRAD's HWE model with 14 matched a SNP array for 91.2% of dosages, against 92.3% for `call` + with its defaults and 87.9% for polyRAD's own default of 9. + +## Genome Paralog Filter (`filter-paralogs`) + +- **Lettuce 3K DArTag** (the panel the filter was built on). The rows kept or dropped agreed with a + bwa-mem classification for 98.2% of the rows of a MADCATS MADC and 94.9% of a vendor MADC, and + among the rows the filter decides, 97.3% and 94.8%. blastn, on a sample of 18,763 vendor rows, + made the same decision for 99.4 to 99.6% of the rows the filter dropped as `paralog` and for 69% + of the well-supported rows it kept (89% of their reads). The kept rows it would drop lie far + from the reference and hold about 1% of the kept reads. +- **Delta smelt GT-seq, placement.** The filter placed all 368 markers where their two primers lie + in the 437 Mb genome, and dropped no row of this paralog-free panel. +- **Delta smelt GT-seq, planted copies.** With copies of 105 markers' tags added to the genome and + the matching rows added to the MADC, every row from a copy 1 to 8 bases from the Ref tag, or + with a 2-base deletion, was labelled `paralog` (450 of 450), and no row of an untouched marker + was mislabelled (3,348 rows). The filter does not see a copy that is not in the genome. + +- **Alfalfa 3K DArTag, design genome.** On two tetraploid F1 families and the panel's haplotype + database, with chromosome set 1 of XinJiangDaYe as the genome, the filter placed every marker at + its CloneID position and kept or dropped the same rows as bwa mem for 99.7 to 99.9% and + as blastn for 99.4 to 99.7%. +- **Alfalfa 3K DArTag, inheritance.** Mapped genetically in the two families, 96% of the rows the + filter kept lie at their own marker, and 91 to 94% of the `paralog` rows that the test can + judge lie somewhere else. The filter removes 3% of the reads of this panel, against 63% in + lettuce, and the target genotypes are the same with and without it. +- **Alfalfa 3K DArTag, more than one chromosome set.** With a second set of the design assembly + in the genome the filter dropped 1,969 of 2,753 RefMatch and AltMatch rows where it dropped 296 + with one. A haplotype-resolved or polyploid assembly has to be cut down to one set first. + +See [`filter-paralogs`](../commands/filter-paralogs.md#how-it-was-checked). diff --git a/docs/src/tutorials/agriseq.md b/docs/src/tutorials/agriseq.md index e810692..f5284a0 100644 --- a/docs/src/tutorials/agriseq.md +++ b/docs/src/tutorials/agriseq.md @@ -34,16 +34,16 @@ madcats generate \ context. Use [`window-scan`](../commands/window-scan.md) to choose a length for a new panel. - `--min-supporting-freq 0.10` is the recommended setting for AgriSeq. At AgriSeq depths (hundreds of reads per sample per marker), even sequencing errors pass the absolute read thresholds. On - honeybee, default evidence admitted 1,752 novel rows, about 92% of them error haplotypes; the - frequency floor brought that down to 147 while improving caller concordance. + honeybee, default evidence admitted 1,752 novel rows, about 92% of them error haplotypes, and + the frequency floor brought that down to 147 while improving caller concordance. ### Window placement -With `--window-placement auto` (the default), each marker's window is placed from observed read -coverage, keeping the variant out of the assignment anchor and steering off tandem repeats -(`--str-penalty`). Repeats matter: when a variant allele lengthens an adjacent repeat, reads -carrying that allele are lost more often in a window that spans the repeat, and the marker can -look monomorphic. +With `--window-placement auto` (the default), MADCATS places each marker's window from observed +read coverage, keeping the variant out of the assignment anchor and steering off tandem repeats +(`--str-penalty`). Repeats matter. When a variant allele lengthens an adjacent repeat, a window +that spans the repeat loses reads carrying that allele more often, and the marker can look +monomorphic. ## Every Later Run: Freeze the Windows @@ -56,15 +56,15 @@ madcats normalize-reads \ --jobs 8 ``` -Windows are placed from each run's own reads. Re-learning placement per run moved 42 of 85 +MADCATS places windows from each run's own reads. Re-learning placement per run moved 42 of 85 windows between two honeybee chips, which made 58% of the second chip's haplotypes falsely appear new. Freezing with `--reuse-tags` raised cross-chip haplotype sharing from 42% to 88%. With `--reuse-tags`, `normalize-reads` still scores the frozen windows against the new run's reads -and reports `freeze_health` in the QC: the read-weighted coverage of the frozen panel against what -fresh placement would choose today, and the markers losing the most. One line on stderr summarizes -what freezing costs this run. +and reports `freeze_health` in the QC. It records the read-weighted coverage of the frozen panel +against what fresh placement would choose today, and the markers losing the most. One line on +stderr summarizes what freezing costs this run. ## Recommended Production Workflow @@ -85,5 +85,5 @@ at the minimum and 209 grown to 13–32 bp. ## Other Platforms The same route handles any amplicon or capture platform whose reads contain a panel window but do -not start at a fixed primer. For paired-end data, decide the geometry first; see +not start at a fixed primer. For paired-end data, decide the geometry first. See [Paired-End Reads and Tiling](paired-end.md). diff --git a/docs/src/tutorials/dartag.md b/docs/src/tutorials/dartag.md index 33d3ca6..839ff71 100644 --- a/docs/src/tutorials/dartag.md +++ b/docs/src/tutorials/dartag.md @@ -29,9 +29,9 @@ For each marker, `design` locates the left primer in the target sequence (either extracts the primer-started tag that carries the bracketed variant, verifies the variant position, and builds the Alt tag. If the design files disagree with each other, the run stops with an error. -Open the CSV and check it: one row per marker with `RefSequence` and `AltSequence` of the tag -length. This file is now the panel. You can hand it to collaborators or use it on later orders -without the vendor files. +Open the CSV and check that it has one row per marker, with `RefSequence` and `AltSequence` of +the tag length. This file is now the panel. You can hand it to collaborators or use it on later +orders without the vendor files. ## 2. Generate the MADC @@ -46,22 +46,23 @@ madcats generate \ What happens, in order: -1. Each read is assigned to a marker by its left-primer prefix (one mismatch allowed by default). -2. The fixed-length tag is kept and accepted if it is within 80% identity of the marker's Ref or - Alt tag. Reads that fail get one more try with a gapped alignment, so undesigned indels are +1. MADCATS assigns each read to a marker by its left-primer prefix (one mismatch allowed by + default). +2. It keeps the fixed-length tag and accepts it if it is within 80% identity of the marker's Ref + or Alt tag. Reads that fail get one more try with a gapped alignment, so undesigned indels are still counted. -3. Reads shorter than the tag are added to the count of a full-length allele whose start they - match exactly. They cannot create a new allele. -4. Canonical Ref and Alt rows are always written. Any other sequence is reported only if it passes - one of the [evidence routes](../reference/evidence-model.md). +3. Reads shorter than the tag add to the count of a full-length allele whose start they match + exactly. They cannot create a new allele. +4. MADCATS always writes the canonical Ref and Alt rows, and reports any other sequence only if it + passes one of the [evidence routes](../reference/evidence-model.md). 5. Rare sequences that differ by one base from a much more common sequence in the same sample are treated as sequencing errors and merged into it (abundance-skew denoising). ## 3. Read the QC -`results/sweetpotato_MADC.csv.qc.json` accounts for every read. Look first at the per-sample totals: -samples with very few assignable tags (on the sweetpotato project, twelve samples had fewer than -100,000; the lowest had 23) usually contain mostly short or primer-less reads. Check them for +`results/sweetpotato_MADC.csv.qc.json` accounts for every read. Look first at the per-sample totals. +Samples with very few assignable tags (on the sweetpotato project, twelve samples had fewer than +100,000 and the lowest had 23) usually contain mostly short or primer-less reads. Check them for missing data before using them downstream. ## 4. Optional: Compare with the Vendor MADC @@ -77,9 +78,9 @@ The report gives row precision and recall by allele category plus cell-level cou this project every shared row had identical classification, and shared-cell count correlation was 0.9997. -Do not expect 100% row recall against a vendor table: 27.3% of the vendor's positive-count rows on -this project had fewer than 10 reads in fewer than 2 samples of the vendor's *own* counts, below -any within-project evidence threshold. +Do not expect 100% row recall against a vendor table, because 27.3% of the vendor's positive-count +rows on this project had fewer than 10 reads in fewer than 2 samples of the vendor's *own* counts, +below any within-project evidence threshold. ## Tuning @@ -106,4 +107,4 @@ madcats generate ... --leading-anchor auto madcats generate ... --leading-anchor GATGGATCCAGTGC ``` -Reads without the anchor are counted as `reads_no_leading_anchor` in the QC. +The QC counts reads without the anchor as `reads_no_leading_anchor`. diff --git a/docs/src/tutorials/genotype-calling.md b/docs/src/tutorials/genotype-calling.md new file mode 100644 index 0000000..c1f8666 --- /dev/null +++ b/docs/src/tutorials/genotype-calling.md @@ -0,0 +1,180 @@ +# Calling Genotypes from a MADC + +A MADC holds read counts. Most analyses need genotypes, the number of copies of each allele a +sample carries. In a diploid the counts alone nearly show the genotype. In a tetraploid or a +hexaploid, neighbouring dosages differ by a small share of the reads, and read counts scatter. +[`madcats call`](../commands/call.md) estimates each sample's genotype from the counts and writes +VCFs. It reads the MADC as it is, from `generate`, `filter-paralogs`, the vendor or HapApp, and +needs no other tool. + +The steps below use the [example project](../getting-started/example.md), which is diploid. They +are the same for polyploids. + +## 1. Call the Targets + +```bash +madcats call --madc results/demo_MADC.csv --output-prefix results/demo +``` + +```text +note: too few read counts to estimate the read model (96 sample-by-marker cells with reads; 100 needed). +Using background-read rate: 0.002 +Using overdispersion: 9 +results/demo.targets.vcf: 8 records, 96 of 96 genotypes called (100.0%) +Wrote QC to results/demo.qc.json +``` + +`call` called every genotype of the eight markers and twelve samples. All 96 equal the simulated +genotypes in `truth_genotypes.csv`. + +The note is about the size of the example. On a real table `call` measures from the +counts themselves how far read fractions scatter (the overdispersion) and how often a read shows +an allele it did not come from (the background-read rate). Eight markers are too few for that, so +it uses fixed values and prints them. + +## 2. Read the VCF + +One record of `results/demo.targets.vcf`, with two of its twelve samples: + +```text +#CHROM POS REF ALT INFO FORMAT Line_01 Line_02 +Chr01_000412883 34 G C TARGET;NS=12;AF=0.5;PMC=0.0001 GT:AD:DP:XD:GQ:MPP:DS:GP:PL 0/0:76,0:76:0:48:1:0:1,0,0:0,51,142 0/1:35,35:70:0:40:0.9999:1:0.0001,0.9999,0.0001:40,0,40 +``` + +`Line_01` has 76 reads of G and none of C, so it is `0/0`. `Line_02` has 35 of each and is `0/1`. + +| Field | Meaning | +|-------|---------| +| `GT` | The genotype, with one allele per chromosome copy, such as `0/0/0/1` in a tetraploid | +| `AD`, `DP` | Reads of each allele, and their sum | +| `GQ` | The model's confidence in `GT`, Phred-scaled, so 20 is one chance in 100 of a wrong genotype and 30 one in 1,000 | +| `MPP` | The same confidence as a probability, which is the largest value of `GP` | +| `DS` | The expected number of alternate copies, for analyses that take a dosage | +| `GP` | The probability of each genotype | +| `PMC`, in INFO | The share of the marker's samples expected to have a wrong genotype, which is one minus the mean of `MPP` | + +**Coordinates are marker-local.** `CHROM` is the marker and `POS` is a position in its tag, not on +a genome. A `##madcatsCoordinates` line in the header marks the coordinates as marker-local. + +`call` writes a genotype as missing (`./.`) when the sample has fewer than 10 reads or its `GQ` is +below 10. `--min-dp` and `--min-gq` change that. The read counts stay in the record either way. + +## 3. Give the Ploidy + +`call` takes every sample to be diploid unless told otherwise: + +```bash +madcats call --madc project_MADC.csv --ploidy 4 --output-prefix results/project +``` + +When ploidy differs between samples, list the exceptions in a two-column table and pass it with +`--ploidy-file`. Samples it does not list take `--ploidy`: + +```text +sample ploidy +Clone_017 2 +Clone_044 6 +``` + +## 4. Choose the Settings for Your Samples + +Start with the defaults for diploids, and add `--allele-bias auto` for polyploids: + +```bash +madcats call --madc project_MADC.csv --ploidy 4 --allele-bias auto --output-prefix results/project +``` + +`call` estimates how the read counts scatter from the table itself. With `auto` it also fits an +allelic bias at every marker, and uses it only where the bias gains enough and stays within the +search limits. Markers left without a bias get the same calls as without the option. + +Change this when one of these describes your samples: + +| Your samples | Change | Why | +|--------------|--------|-----| +| Inbred lines, doubled haploids, selfing species | `--selfing-rate` of `0.95` to `0.99` for inbred lines, or the species' own rate for partly selfing ones. Leave the bias off for inbred lines | The default prior expects the heterozygotes of random mating, and a bias has nothing to correct where there are none | +| Families (F1, half-sibs) | Add `--prior norm` | The default prior assumes random mating. It takes two to five times as long | +| Hexaploids, or fewer than about 50 reads, on a panel known to be biased | `--allele-bias fit` | `auto` keeps a bias only where it gains enough, which takes depth, while `fit` uses every bias | +| Subpopulations at 25 reads or fewer, on a panel without bias | `--prior structure` | Each sample's prior comes from the samples most like it. You cannot combine it with a bias | + +[Which settings to use](../commands/call.md#which-settings-to-use) has the measurements behind +each row. + +If you come from updog or polyRAD: + +| You used | The same model in `call` | +|----------|--------------------------| +| updog, `norm` | `--prior norm --allele-bias fit`. updog always fits a bias | +| polyRAD, `IterateHWE` | The defaults. polyRAD's own settings are `--overdispersion 9 --contamination-rate 0.001 --background-rate 0` | +| polyRAD, `IteratePopStruct` | `--prior structure`. The method differs, and the purpose is the same | + +`MPP` and `PMC` are updog's `maxpostprob` and `prop_mis`, named as BIGr's `updog2vcf` names them, +and `INFO/BIAS` is updog's bias. BIGr's `filterVCF` filters on all three by these names. + +## 5. Write Other Record Types + +`--vcf` takes one or more record types, and each goes to its own file: + +```bash +madcats call --madc results/demo_MADC.csv --output-prefix results/demo --vcf target all haplotype +``` + +| `--vcf` | File | A record is | +|---------|------|-------------| +| `target` | `PREFIX.targets.vcf` | The variant a marker was designed for | +| `all` | `PREFIX.all.vcf` | A target, or any other base substitution seen in the tags | +| `haplotype` | `PREFIX.haplotypes.vcf` | A marker, with whole tag sequences as its alleles | + +Records with more than two alleles, which is every haplotype record, carry `MPC` in place of `GQ` +and `MPP`, and have no `PMC`. `MPC` is the lowest probability among the allele dosages called, +and `--min-marginal-posterior` (default 0.9) is its threshold. + +## 6. Check the Run + +`call` prints what to look at, and `PREFIX.qc.json` keeps it: + +- **How many genotypes were called.** A low share at the default thresholds usually means low + depth. +- **Markers without a target record.** `call` does not guess a target when a marker's Ref and Alt + tags differ in more than one place. It reports how many markers that left out. Pass the panel's + reference tags with `--reference-tags` to settle them. +- **The overdispersion estimate.** A warning that the estimate is the lowest value tried means the + table has no heterozygous samples to measure scatter from, as with inbred lines. +- **The allelic bias**, when one is fitted. The `allele_bias` block counts the records called with + their bias and those called without it, by reason. A warning that most fitted biases ended at a + limit of the search means the samples have few heterozygotes, so call them without a bias. + +## 7. Filter Markers Afterwards + +No genotype threshold removes a marker that does not behave as one locus. On two potato panels +with SNP-array genotypes for every marker: + +- **Filter markers on call rate first.** The share of samples that pass the default thresholds + separated the markers that disagreed with the array from the rest better than depth, `GQ`, the + fitted bias or allele frequency. Removing markers with a call rate under 0.9 raised agreement + with the array from 91.3% to 94.1% on one panel and from 93.3% to 95.0% on the other. +- **`PMC` finds fewer of them.** Removing markers with a `PMC` of 0.05 or more raised agreement + to 92.7% and 94.2%, and every marker it removed had a call rate under 0.9 as well. `PMC` also + rises with ploidy and falls with depth, so a cut that suits one data set removes good markers + in another. +- **Raise `--min-gq` to 20 at most.** Beyond that, most of the genotypes lost were right ones. +- **`INFO/BIAS` is a weak filter.** Fitting the bias gives a strongly biased marker the right + calls. Removing markers with a bias outside 0.5 to 2 took out 62 good markers among 162 on one + panel and 121 among 179 on the other. + +These figures are from runs with the bias fitted. + +## Limits + +- **Allopolyploids.** `call` takes every chromosome copy to be able to carry any allele. A marker + that amplifies all subgenomes but varies on one is outside that model. A marker that amplifies + one subgenome is a diploid marker, so call it with `--ploidy 2`. +- **Pedigrees.** There is no prior for a cross with known parents, and the model does not use + linkage between markers. polyRAD has both. +- **Haplotype records.** Their accuracy has been measured on simulated tables only. + +## Next Steps + +- [madcats call](../commands/call.md) for every option, the model and its measurements +- [Output Formats](../reference/output-formats.md) for the VCF fields and the QC JSON +- [Validation & Accuracy](../reference/validation.md) for what `call` was checked against diff --git a/docs/src/tutorials/paired-end.md b/docs/src/tutorials/paired-end.md index b2a63cd..16eb16c 100644 --- a/docs/src/tutorials/paired-end.md +++ b/docs/src/tutorials/paired-end.md @@ -28,17 +28,18 @@ is needed. Disagreeing overlap bases take the higher-quality call. | `pool` | Keep both mates as independent reads | | `drop` | Discard the pair | -Mates are paired by read ID. Mismatched names or unequal read counts stop the run with an error. +The merger pairs mates by read ID. Mismatched names or unequal read counts stop the run with an +error. On a synthetic paired-end truth set with modeled adapter read-through, 100% of 513,715 pairs merged with 98.6% molecular count recovery. ## Non-Overlapping Pairs: Pool -Dual-anchored capture designs (Flex-Seq: R1 anchors the left probe, R2 the right) have mates +Dual-anchored capture designs (in Flex-Seq, R1 anchors the left probe and R2 the right) have mates covering opposite halves of a fragment neither read spans. The mates cannot be merged, and pooling -them cannot count a window twice, because no window is covered by both mates. Concatenate both -mates into one FASTQ per sample and feed that to `normalize-reads`. +them cannot count a window twice, because the two mates never cover the same window. Concatenate +both mates into one FASTQ per sample and feed that to `normalize-reads`. ## Multi-Window Tiling @@ -52,23 +53,24 @@ amp17_snp2 ACGT...C...ACG[A/G]TACGT... amp17 ``` Tiles must share an identical Ref expansion frame (validated at load), reads locate to the group, -and then they are routed to member windows with `--group-routing`: +and `--group-routing` then routes them to member windows: | Mode | Behaviour | Use when | |------|-----------|----------| -| `nearest` (default) | Each read feeds exactly one tile: the fully covered window whose center is nearest the read's center | Reads are short relative to tile spacing | -| `covered` | The read is excised at every member window it fully covers | Reads span several tiles, e.g. **merged fragments** | +| `nearest` (default) | Each read feeds only one tile, the fully covered window whose center is nearest the read's center | Reads are short relative to tile spacing | +| `covered` | MADCATS excises the read at every member window it fully covers | Reads span several tiles, e.g. merged fragments | Use `covered` with merged full-span fragments. They all share the amplicon's center, so with -`nearest` one tile gets almost every read: on the GT-seq benchmark one tile took effectively all -depth at 163 of 164 tiled loci. Each window still sees each fragment at most once in both modes. +`nearest` one tile gets almost every read. On the GT-seq benchmark, one tile took nearly all depth +at 163 of 164 tiled loci. Each window still sees each fragment at most once in both modes. Fresh placement forces tiles to excise distinct sequences, since twin windows would make every -read an exact tie. Nudged members are listed in the QC as `locus_groups.window_dedup_nudged`. +read an exact tie. The QC lists nudged members as `locus_groups.window_dedup_nudged`. ## Putting It Together -The recipe validated on GT-seq: merge, normalize against a frozen panel with tile rows, generate. +The recipe validated on GT-seq is to merge, normalize against a frozen panel with tile rows, and +generate. ```bash madcats merge-pairs --fastq-dir raw/ --out-dir merged/ --jobs 8 diff --git a/docs/src/tutorials/paralog-filter.md b/docs/src/tutorials/paralog-filter.md new file mode 100644 index 0000000..0e3b8a3 --- /dev/null +++ b/docs/src/tutorials/paralog-filter.md @@ -0,0 +1,170 @@ +# Filtering Paralogs with a Reference Genome + +Some panels co-amplify paralogous copies of their targets. A paralog read that carries the target +base lands in the marker's RefMatch or AltMatch rows. Tools that sum every row at the target SNP +(`BIGr::madc2vcf_all` does) then count those reads as alleles of the target, and homozygous samples +look heterozygous. This tutorial filters such a MADC with +[`madcats filter-paralogs`](../commands/filter-paralogs.md), which uses a reference genome to find +where each haplotype comes from. + +The filter is opt-in. Use it on panels that show the problem. + +## Is My Panel Affected? + +Signs of co-amplified paralogs: + +- Inbred or doubled-haploid lines that should be homozygous show many heterozygous calls. +- Many markers have RefMatch/AltMatch rows that carry a large share of the marker's reads. +- Heterozygosity drops sharply when you raise `generate --min-identity` from 0.80 to 0.95 or 0.97. + Stricter identity rejects the more diverged copies at read assignment. + +The filter's QC JSON measures it in `rows.by_label.paralog.reads`, the number of +reads in rows the genome places elsewhere. In the lettuce 3K DArTag panel that was 63% of the reads +in the vendor MADC and 16% of a MADCATS MADC generated at `--min-identity 0.95`. + +## 1. Install the Aligner + +```bash +pip install 'madcats[genome]' +``` + +This adds `mappy`, minimap2's Python binding. No other command needs it. + +## 2. Index the Genome Once + +Use the assembly the panel was designed on, or one close to it. To compare against another +assembly, see [Using Another Assembly](../commands/filter-paralogs.md#using-another-assembly). The +first run with both `--genome` and `--genome-index` builds the index and saves it. Later runs load +it in seconds: + +```bash +madcats filter-paralogs \ + --madc project_MADC.csv \ + --genome ref/genome.fa \ + --genome-index ref/genome.sr.mmi \ + --output project_MADC_filtered.csv +``` + +On the 2.4 Gb lettuce genome indexing took 50 s and 10.6 GB of memory. Run it on a machine with +enough memory, then reuse the `.mmi` anywhere. + +## 3a. A MADCATS MADC + +Generate as usual. On a paralog-heavy panel, a stricter identity gate removes the most diverged +copies before the genome filter sees them: + +```bash +madcats generate --fastq-dir fastqs/ --reference-tags tags.csv \ + --min-identity 0.95 --jobs 8 --output project_MADC.csv + +madcats filter-paralogs \ + --madc project_MADC.csv \ + --genome-index ref/genome.sr.mmi \ + --output project_MADC_filtered.csv \ + --jobs 8 +``` + +On lettuce this pair (identity 0.95, then the genome filter dropping paralog and ambiguous rows) +matched the published realignment-based genotypes for PCA, kinship and linkage mapping. That +measurement used the bwa-mem prototype of this filter, and this command keeps the same rows for +99.7% of the reads. + +## 3b. A Vendor MADC + +The filter reads vendor DArT MADCs as they are delivered, with or without the starred preamble, and +keeps the preamble, the statistics columns and the column order: + +```bash +madcats filter-paralogs \ + --madc DLet22-7611_MADC.csv \ + --genome-index ref/genome.sr.mmi \ + --output DLet22-7611_MADC_filtered.csv \ + --jobs 8 +``` + +The lettuce vendor MADC (732,534 rows, 376 samples) took 2 min 15 s and 9.4 GB of memory. The filter +kept 113,667 rows, with 36% of the reads. Genotype agreement with the published realignment VCF +rose from 89.9% to 98.5%. That figure comes from the bwa-mem prototype of this filter, and this +command keeps the same rows for 99.4% of the reads. + +## 4. Read the QC + +`OUTPUT.qc.json` summarises the run. On the lettuce MADCATS MADC (17,847 rows): + +| Label | Rows | Reads | +|-------|-----:|------:| +| `on_target` | 12,687 | 59.8 M (82.4%) | +| `paralog` | 4,135 | 11.6 M (16.0%) | +| `ambiguous` | 1,025 | 1.1 M (1.6%) | + +Check: + +- In `markers.placement` and `markers.clone_id_check`, nearly every marker should be placed by its + Ref tag, and CloneID positions should agree. Many disagreements mean the genome is not the panel's + assembly, and the command warns when more than 10% disagree. +- `markers.multicopy_ref_tags` lists markers whose own Ref tag has several equal-best sites. The + filter keeps their canonical rows, but their counts mix copies, so consider dropping these markers + downstream. +- `rows.by_outcome.protected` counts canonical Ref/Alt rows that the genome would have dropped. The + filter always keeps them, so dosage tools still see both designed alleles. +- `rows.by_category` shows which allele categories lost reads. Paralogs concentrate in RefMatch and + AltMatch rows. + +## 5. Review Before Dropping (Optional) + +To see the labels without removing anything, flag instead of dropping: + +```bash +madcats filter-paralogs --madc project_MADC.csv --genome-index ref/genome.sr.mmi \ + --paralogs flag --ambiguous flag --output project_MADC_flagged.csv +``` + +Every row then carries its label in the `GenomeLabel` column (the `PICRef` position). The +`OUTPUT.genome_labels.csv` sidecar lists, for every row, its score at the marker and its best score +elsewhere. A perfect 81-base match scores 162 and each mismatch costs 10, so a row with +`TargetScore` 132 and `OffTargetScore` 162 is three mismatches from the target and a perfect copy +of another locus. + +`--ambiguous keep` keeps ambiguous rows without a label. On lettuce, keeping them left false +heterozygous calls at 3.4% of inbred genotypes, against 0.87% when the filter dropped them, which is +why `drop` is the default. + +## 6. Continue Downstream + +The filtered file keeps the layout it came in. A filtered raw MADC continues as any MADCATS or +vendor MADC does, through HapApp or HapApp_utils for fixed allele IDs, then `BIGr::madc2vcf_all()` +and dosage calling. See [Output Formats](../reference/output-formats.md#downstream-processing). + +The filter also reads HapApp's fixed-allele-ID output, so it can run after HapApp instead: + +```bash +madcats filter-paralogs \ + --madc project_MADC_rename_v1.csv \ + --genome-index ref/genome.sr.mmi \ + --output project_MADC_rename_v1_filtered.csv +``` + +The filter keeps the fixed allele IDs and the column layout, so the filtered file goes wherever +HapApp's output would have gone. On the lettuce MADCATS MADC, rewritten in HapApp's layout, every +row got the same label as in the raw MADC. + +## How the Labels Are Decided + +The filter places each marker by aligning its Ref tag. The CloneID's `CHROM_POS` checks the +placement and breaks ties between equal-best sites. Where the Ref tag's best sites are all away from +the CloneID position, the filter aligns the tag again near that position and places the marker +there if it aligns (otherwise the marker stays `unplaced`). It then aligns every distinct haplotype +genome-wide, and again at its own marker's site alone with short seeds, because real alleles that +differ from the reference by several SNPs are too diverged for the genome-wide seeds and would +otherwise be judged only by their paralogs. It then compares the haplotype's best score at its +marker with its best score anywhere else. Higher at the marker is `on_target`, higher elsewhere is +`paralog`, and a tie is `ambiguous`. + +Labels were checked against an independent bwa-mem classification of the same lettuce haplotypes. +The drop or keep decision agreed for 98.2% of the rows of the MADCATS MADC and 94.9% of the vendor +MADC. The vendor MADC's rows are more diverged, and bwa mem finds more distant copies than +minimap2. Weighted by reads, the filtered vendor MADC agreed with the bwa-based prototype for 99.4% +of reads. blastn, on a sample of the vendor MADC's rows, agreed with more than 99% of the rows the +filter dropped as `paralog`. Like bwa mem, it would also drop some rows the filter keeps far from +the reference, which hold about 1% of the kept reads +(see [how the filter was checked](../commands/filter-paralogs.md#how-it-was-checked)). diff --git a/docs/src/tutorials/primer-free.md b/docs/src/tutorials/primer-free.md index 8dd9845..51b2537 100644 --- a/docs/src/tutorials/primer-free.md +++ b/docs/src/tutorials/primer-free.md @@ -1,7 +1,7 @@ # Panels Without a Primer File Some providers do not share the primer file. The only thing MADCATS needs from it is where each tag -starts inside the probe-design target, and that can be recovered from the reads. +starts inside the probe-design target, and MADCATS can recover that from the reads. ## Option A: Bootstrap From a Delivered MADC @@ -14,9 +14,10 @@ python3 scripts/madc_to_reference_tags.py \ --output reference_tags.csv ``` -Markers that cannot become loci are announced with a `WARNING:` and listed with their reason in -`reference_tags.csv.skipped.tsv`. The most common reason is Ref and Alt rows carrying the same -sequence, which means the variant lies one base past the window or the design is presence/absence. +The script announces markers that cannot become loci with a `WARNING:` and lists them with their +reason in `reference_tags.csv.skipped.tsv`. The most common reason is Ref and Alt rows carrying the +same sequence, which means the variant lies one base past the window or the design is +presence/absence. Prefer this route when a vendor MADC is available. It reproduces the vendor's tags, including IUPAC curation. @@ -36,11 +37,11 @@ madcats infer-design \ ### How it works The tag must be a tag-length window of the bracket-expanded target that contains the variant. So -inference only has to choose among a fixed set of candidate windows (about 4x tag length per locus); -it never assembles new sequences. Because reads are primer-started, exact read prefixes almost -always point clearly to one window. Ref/Alt identity comes from bracket order. +inference only has to choose among a fixed set of candidate windows (about 4x tag length per locus) +and never assembles new sequences. Because reads are primer-started, exact read prefixes almost +always point to only one window. Ref/Alt identity comes from bracket order. -Votes are counted in two tiers, in order: +Inference counts votes in two tiers, in order: 1. The exact tier counts reads whose prefix exactly matches a candidate window. When a locus has any exact votes, they decide. @@ -80,19 +81,19 @@ madcats generate \ --output project_MADC.csv ``` -Inference is a one-time operation per panel: freeze the CSV and reuse it for every later order. +Inference is a one-time operation per panel, so freeze the CSV and reuse it for every later order. ## How Accurate Is Inference? | Project | Result | |---------|--------| -| Sweetpotato DArTag (blinded) | 3,068/3,120 loci inferred: exact tier 100.0% correct (2,742), fuzzy tier 97.9% (319/326); zero Ref/Alt swaps. `generate` on the inferred panel reproduced 99.1% of rows with 100% classification agreement. | +| Sweetpotato DArTag (blinded) | 3,068/3,120 loci inferred, with the exact tier 100.0% correct (2,742), the fuzzy tier 97.9% (319/326) and zero Ref/Alt swaps. `generate` on the inferred panel reproduced 99.1% of rows with 100% classification agreement. | | Delta smelt GT-seq | 368/368 loci exactly as the physical primers define them | | Alfalfa DArTag (raw reads) | 2,999/3,000 loci in the exact tier, all identical to the design tags | -| Blackberry DArTag | 2,943/3,015 inferred; 18 wrong windows (0.6%), all at read-starved or short-amplicon loci | +| Blackberry DArTag | 2,943/3,015 inferred, with 18 wrong windows (0.6%), all at read-starved or short-amplicon loci | ## Raw Deliveries and Tag Length `infer-design` takes the same `--leading-anchor` option as `generate` for raw DArT deliveries. `--tag-length` sets the panel's reported tag length (54, 81, 109, ...). Reads must reach it, and the -design flanks must contain a full window; both are validated per locus. +design flanks must contain a full window. MADCATS validates both per locus. diff --git a/madcats.py b/madcats.py index 532075c..09b08f3 100644 --- a/madcats.py +++ b/madcats.py @@ -7,9 +7,13 @@ FASTQ-to-MADC workflow, where MADC denotes the Missing Allele Discovery Counts format. -The implementation intentionally uses only the Python standard library. +The implementation intentionally uses only the Python standard library. The +one exception is optional: ``filter-paralogs``, in its own module +``madcats_paralogs.py``, imports the ``mappy`` aligner on first use +(``pip install madcats[genome]``). -Seven subcommands cover the read models the pipeline understands: +Seven subcommands cover the read models the pipeline understands, an eighth +cleans a finished table, and a ninth calls genotypes from it: - ``design`` reconstructs each marker's canonical Ref/Alt tags from vendor primer + probe-design files. This is the primer-started, single-end model @@ -47,6 +51,18 @@ panel routes above into the MADC table, via the stages listed below. - ``validate`` compares a generated MADC against a reference MADC row-by-row and cell-by-cell, reporting precision/recall by allele category. +- ``filter-paralogs`` (optional; needs ``mappy``; in ``madcats_paralogs.py``) + cleans a MADC from ``generate``, a vendor delivery or HapApp with a + reference genome. Each marker is placed by aligning its Ref tag, every + haplotype is aligned, and rows whose best genome hit lies at another locus + (paralogs) or ties between loci (ambiguous) are dropped or flagged. It is + meant for panels where co-amplified paralogous copies inflate + heterozygosity when counts are summed per SNP; canonical rows are never + dropped. +- ``call`` (in ``madcats_call.py``) estimates each sample's genotype from the + read counts of a MADC and writes VCFs of the designed targets, of every + base substitution in the tags, or of whole tag sequences as alleles. The + model is adapted from polyRAD and ploidy is set per sample. Pipeline stages (the ``generate`` command; see ``run_generate``): @@ -60,7 +76,10 @@ length tag, and accept it within the substitution-only identity envelope of the locus's Ref/Alt tags — or, when every candidate rejects it, under one gapped reading within the same edit budget, so undesigned indel - haplotypes are counted rather than dropped (``select_marker``). Tags + haplotypes are counted rather than dropped (``select_markers``). Tiles + are the one exception to one read per locus: they read the same tag + window of one amplicon (``tile_families``), so a read that fits several + tiles is credited to each, and the extra credits are counted apart. Tags carrying an N are credited to the one N-free sequence they fit or dropped; they never become rows. Counting exact observed sequences — never consensus-corrected ones — is what makes the output auditable @@ -106,13 +125,14 @@ primer-free inference. 4. Platform intake: pair merging, window location and freezing, tiling and routing, window-scan. -5. Per-sample read assignment: ``PrimerIndex``, ``select_marker``, - ``process_sample`` and its cache. +5. Per-sample read assignment: ``PrimerIndex``, ``tile_families``, + ``select_markers``, ``process_sample`` and its cache. 6. Evidence, denoising and partial-read salvage: ``EvidenceRule``, ``collect_evidence``, ``finalize_counts``, ``apply_partial_reads``. 7. Classification and annotation: ``classify_sequence``, ``variant_events``, ``marker_type_label``. 8. MADC input/output and QC: ``read_madc``, ``write_madc``, ``validate_madc``. + The genome paralog filter is in ``madcats_paralogs.py``. 9. Commands and command line: one ``run_*`` per subcommand, ``build_parser``. Vocabulary. *tag*: the fixed-length sequence a read is measured as (intake @@ -152,7 +172,7 @@ # ============================================================================ -__version__ = "0.13.0" +__version__ = "0.14.0" # Increment only when the serialized per-sample cache or raw-read processing # changes. Output-row filtering does not invalidate reusable FASTQ counts. @@ -237,8 +257,9 @@ class Locus: # Markers sharing a locus_group are windows (tiles) of one physical # locus: normalize-reads locates reads to the group and routes them # among the member windows (--group-routing: one tile per read, or - # every fully covered tile). Empty = the marker is its own group - # (the universal pre-0.9 case). + # every fully covered tile), and generate credits a read to every + # member that reads the same tag window (tile_families). Empty = the + # marker is its own group (the universal pre-0.9 case). locus_group: str = "" @property @@ -309,9 +330,10 @@ class AssignmentSettings: partial_min_length: int gap_band: int # --max-indel-length, or 0 with --gapped-rescue off leading_anchor: str = "" # --leading-anchor: constant sequence preceding the amplicon in raw deliveries + tile_families: Tuple[Tuple[int, ...], ...] = () # --tile-groups: marker indices credited together (tile_families) def cache_arguments(self) -> Dict[str, object]: - return { + arguments: Dict[str, object] = { "tag_length": self.tag_length, "max_offtarget_mismatches": self.max_offtarget_mismatches, "gap_band": self.gap_band, @@ -320,6 +342,12 @@ def cache_arguments(self) -> Dict[str, object]: "leading_anchor": self.leading_anchor, "quality_thresholds": QUALITY_THRESHOLDS, } + # Keyed only when present: a panel without tiles is read exactly as + # before tile crediting existed, so its caches stay valid, while any + # tile family (and any change to the families) gives a new fingerprint. + if self.tile_families: + arguments["tile_families"] = [list(family) for family in self.tile_families] + return arguments @dataclasses.dataclass(frozen=True) @@ -1062,6 +1090,7 @@ def build_panel(primer_csv: Path, probe_design: Path, tag_length: int) -> List[L variant_pos=variant_pos, ref_tag=ref_tag, alt_tag=alt_tag, + locus_group=(designs[marker].get("LocusGroup") or "").strip(), ) ) return loci @@ -2180,13 +2209,88 @@ def _write_cache(path: Path, result: SampleResult, source_stat: os.stat_result) temporary.replace(path) -def select_marker( +def same_tag_window(a: Locus, b: Locus) -> bool: + """Do two markers read the same tag window of one amplicon? + + True when their Ref tags have the same length and are IUPAC-compatible + at every position other than the two markers' own variant positions. That + is what a DArTag tile looks like: tiles share a left primer (or one + primer extends the other), and each tile's tag carries the neighbouring + tiles' SNPs as ambiguity codes, so every read from the amplicon fits each + tile's tags equally well. normalize-reads tiles excise different + windows of their locus and never pass. + """ + if a.index == b.index or len(a.ref_tag) != len(b.ref_tag): + return False + exempt = (a.variant_pos, b.variant_pos) + return all( + position in exempt or bases_compatible(left, right) + for position, (left, right) in enumerate(zip(a.ref_tag, b.ref_tag)) + ) + + +def tile_families(loci: Sequence[Locus], mode: str = "auto") -> Tuple[Tuple[int, ...], ...]: + """Markers whose reads ``generate`` credits together (``--tile-groups``). + + Two markers are linked when they read the same tag window + (``same_tag_window``) and one's primer matches the start of the other's + tag, so one read reaches both; a family is a connected set of links. + ``declared`` links only markers that share a ``LocusGroup``; ``auto`` + also links undeclared markers, but never two + markers declared in different groups; ``off`` links none. Families are + sorted tuples of marker indices, members of two or more only. + """ + if mode == "off": + return () + primer_index = PrimerIndex(loci) + parent = list(range(len(loci))) + + def root(index: int) -> int: + while parent[index] != index: + parent[index] = parent[parent[index]] + index = parent[index] + return index + + for locus in loci: + for other in primer_index.match(locus.ref_tag): + mate = loci[other] + declared = bool(locus.locus_group) and locus.locus_group == mate.locus_group + conflicting = bool(locus.locus_group and mate.locus_group) and not declared + if not (declared or (mode == "auto" and not conflicting)): + continue + if same_tag_window(locus, mate): + parent[root(other)] = root(locus.index) + members: Dict[int, List[int]] = defaultdict(list) + for locus in loci: + members[root(locus.index)].append(locus.index) + return tuple(sorted(tuple(group) for group in members.values() if len(group) > 1)) + + +def _gapped_edits(tag: str, locus: Locus, max_mismatches: int, gap_band: int) -> Optional[int]: + """Fewest edits of an accepted gapped reading of ``tag`` against the locus's + Ref or Alt tag, or ``None`` when neither reading is believed.""" + best: Optional[int] = None + for template in (locus.ref_tag, locus.alt_tag): + if len(template) != len(tag): + continue + if gapped_prescreen(tag, template, gap_band) > max_mismatches: + continue + edits, substitutions, gap_bases, gap_events, _, _ = gapped_stats(tag, template, gap_band) + if edits > max_mismatches or not gapped_explanation_ok(substitutions, gap_bases, gap_events): + continue + if best is None or edits < best: + best = edits + return best + + +def select_markers( tag: str, candidates: Sequence[int], loci: Sequence[Locus], max_mismatches: int, gap_band: int = 0, -) -> Tuple[Optional[int], str]: + family_of: Optional[Mapping[int, Tuple[int, ...]]] = None, +) -> Tuple[Tuple[int, ...], str]: """Resolve primer candidates by identity to the available tag prefix. Substitution-only identity decides first and is never overturned. When @@ -2199,7 +2303,16 @@ def select_marker( (few substitutions, few gap events); a co-amplified paralog reaches the budget through many substitutions and stays off-target. A cheap exact lower bound (``gapped_prescreen``) skips the alignment for hopeless tags. + + The best-fitting marker takes the read alone, and a tie is dropped as + ambiguous. The exception is a tie in which every marker belongs to one + tile family (``family_of``, from ``tile_families``). Tiles read one + amplicon, so the read is then credited to every family member it fits + within the budget (substitution-only, or by one gapped reading), as + running each tile in a panel of its own would. A tie that crosses + families stays ambiguous. """ + family_of = family_of or {} scored = [] for marker_index in candidates: locus = loci[marker_index] @@ -2212,34 +2325,52 @@ def select_marker( ) scored.append((distance, marker_index)) scored.sort() + + def credited(ranked: Sequence[Tuple[int, int]]) -> Optional[Tuple[int, ...]]: + best_distance, best_marker = ranked[0] + family = family_of.get(best_marker, (best_marker,)) + if any(distance == best_distance and marker not in family for distance, marker in ranked): + return None + if len(family) == 1: + return (best_marker,) + members = [marker for distance, marker in ranked if marker in family and distance <= max_mismatches] + if gap_band > 0: + fitted = set(members) + members += [ + marker for _, marker in ranked + if marker in family and marker not in fitted + and _gapped_edits(tag, loci[marker], max_mismatches, gap_band) is not None + ] + return tuple(sorted(members)) + if scored and scored[0][0] <= max_mismatches: - if len(scored) > 1 and scored[0][0] == scored[1][0]: - return None, "ambiguous" - return scored[0][1], "assigned" + markers = credited(scored) + return (markers, "assigned") if markers else ((), "ambiguous") if gap_band <= 0 or not scored: - return None, "offtarget" + return (), "offtarget" gapped = [] for marker_index in candidates: - locus = loci[marker_index] - best: Optional[int] = None - for template in (locus.ref_tag, locus.alt_tag): - if len(template) != len(tag): - continue - if gapped_prescreen(tag, template, gap_band) > max_mismatches: - continue - edits, substitutions, gap_bases, gap_events, _, _ = gapped_stats(tag, template, gap_band) - if edits > max_mismatches or not gapped_explanation_ok(substitutions, gap_bases, gap_events): - continue - if best is None or edits < best: - best = edits - if best is not None: - gapped.append((best, marker_index)) + edits = _gapped_edits(tag, loci[marker_index], max_mismatches, gap_band) + if edits is not None: + gapped.append((edits, marker_index)) gapped.sort() if not gapped: - return None, "offtarget" - if len(gapped) > 1 and gapped[0][0] == gapped[1][0]: - return None, "ambiguous" - return gapped[0][1], "assigned_gapped" + return (), "offtarget" + markers = credited(gapped) + return (markers, "assigned_gapped") if markers else ((), "ambiguous") + + +def select_marker( + tag: str, + candidates: Sequence[int], + loci: Sequence[Locus], + max_mismatches: int, + gap_band: int = 0, +) -> Tuple[Optional[int], str]: + """``select_markers`` without tile families: the one marker a read goes to, or + ``None`` with the reason it went nowhere.""" + markers, status = select_markers(tag, candidates, loci, max_mismatches, gap_band) + return (markers[0] if markers else None), status def resolve_n_tags( @@ -2313,8 +2444,13 @@ def process_sample( accepted for exactly one locus when its tag lies within the identity envelope of that locus's Ref/Alt tags, or — when every candidate rejects it and ``settings.gap_band`` is positive — under one gapped reading within the - same edit budget (``select_marker``); ambiguous reads are dropped rather - than guessed. Tags carrying an N are resolved at the end of the sample + same edit budget (``select_markers``); ambiguous reads are dropped rather + than guessed. The one exception is a tile family + (``settings.tile_families``): a read that fits several tiles of one + amplicon is credited to each of them, ``reads_assigned`` still counts it + once, and the credits beyond the first are counted in + ``tile_extra_credits`` (``partial_tile_extra_credits`` for reads shorter + than the tag). Tags carrying an N are resolved at the end of the sample (``resolve_n_tags``) and never counted as haplotypes of their own. Alongside each count, a per-threshold vector of whole-tag-clean read tallies is kept so quality thresholds can change @@ -2340,8 +2476,9 @@ def process_sample( # Amplicon reads repeat heavily, so the (deterministic) assignment decision # is memoized per unique sequence once a primer candidate list was found; # primer-less reads fail the trie walk cheaply and are not worth storing. - full_decisions: Dict[str, Tuple[Optional[int], str]] = {} - partial_decisions: Dict[str, Tuple[Optional[int], str]] = {} + full_decisions: Dict[str, Tuple[Tuple[int, ...], str]] = {} + partial_decisions: Dict[str, Tuple[Tuple[int, ...], str]] = {} + family_of = {member: family for family in settings.tile_families for member in family} for sequence, quality in fastq_records(fastq_path): qc["reads_total"] += 1 if settings.leading_anchor: @@ -2366,21 +2503,19 @@ def process_sample( partial_max_mismatches = math.floor( settings.max_offtarget_mismatches * len(sequence) / settings.tag_length ) - marker_index, status = select_marker( - sequence, candidates, loci, partial_max_mismatches - ) - decision = ( - marker_index, - "reads_partial" if marker_index is not None else f"reads_partial_{status}", + markers, status = select_markers( + sequence, candidates, loci, partial_max_mismatches, family_of=family_of ) + decision = (markers, "reads_partial" if markers else f"reads_partial_{status}") partial_decisions[sequence] = decision - marker_index, qc_key = decision + markers, qc_key = decision qc[qc_key] += 1 - if marker_index is None: - continue - key = (marker_index, sequence) - partial_counts[key] += 1 - add_quality_support(partial_quality_support, key, quality) + if len(markers) > 1: + qc["partial_tile_extra_credits"] += len(markers) - 1 + for marker_index in markers: + key = (marker_index, sequence) + partial_counts[key] += 1 + add_quality_support(partial_quality_support, key, quality) continue tag = sequence[:settings.tag_length] decision = full_decisions.get(tag) @@ -2389,19 +2524,23 @@ def process_sample( if not candidates: qc["reads_no_primer"] += 1 continue - marker_index, status = select_marker(tag, candidates, loci, settings.max_offtarget_mismatches, settings.gap_band) - decision = (marker_index, status) + decision = select_markers( + tag, candidates, loci, settings.max_offtarget_mismatches, settings.gap_band, family_of + ) full_decisions[tag] = decision - marker_index, status = decision - if marker_index is None: + markers, status = decision + if not markers: qc[f"reads_{status}"] += 1 continue qc["reads_assigned"] += 1 if status == "assigned_gapped": qc["reads_assigned_gapped"] += 1 - key = (marker_index, tag) - exact_counts[key] += 1 - add_quality_support(exact_quality_support, key, quality[:settings.tag_length]) + if len(markers) > 1: + qc["tile_extra_credits"] += len(markers) - 1 + for marker_index in markers: + key = (marker_index, tag) + exact_counts[key] += 1 + add_quality_support(exact_quality_support, key, quality[:settings.tag_length]) resolve_n_tags(exact_counts, exact_quality_support, loci, qc) qc["unique_full_tags_raw"] = len(exact_counts) @@ -3289,30 +3428,117 @@ def build_dart_preamble( ] +def is_madc_header(row: Sequence[str]) -> bool: + """The header row names AlleleID: first in vendor and MADCATS files, after + a leading Code_version column in HapApp's fixed-allele-ID output.""" + return any(cell.strip() == "AlleleID" for cell in row) + + def find_madc_header(rows: Sequence[Sequence[str]]) -> int: for index, row in enumerate(rows): - if row and row[0].strip() == "AlleleID": + if is_madc_header(row): return index raise MadcError("Could not find an AlleleID header row in MADC file") -def read_madc(path: Path): - with open_text(path, "rt") as handle: - rows = list(csv.reader(handle)) - header_index = find_madc_header(rows) - header = rows[header_index] +def madc_identity_columns(header: Sequence[str], path: Path) -> Tuple[int, int, int]: + """Indices of AlleleID, CloneID and AlleleSequence in a MADC header row.""" try: - allele_index = header.index("AlleleID") - clone_index = header.index("CloneID") - sequence_index = header.index("AlleleSequence") + return header.index("AlleleID"), header.index("CloneID"), header.index("AlleleSequence") except ValueError as exc: raise MadcError(f"MADC is missing a required identifier column: {path}") from exc + + +# Processed MADCs can carry a per-row read total between the sequence columns and the samples. +READ_TOTAL_COLUMN = "readCountSum" + + +def madc_sample_start(header: Sequence[str]) -> int: + """First per-sample column: after the statistics block, or after the sequence columns of a + short layout, and after the per-row read total where there is one.""" if "RatioAvgCountRefAvgCountSnp" in header: - sample_start = header.index("RatioAvgCountRefAvgCountSnp") + 1 + start = header.index("RatioAvgCountRefAvgCountSnp") + 1 elif "ClusterConsensusSequence" in header: - sample_start = header.index("ClusterConsensusSequence") + 1 + start = header.index("ClusterConsensusSequence") + 1 else: - sample_start = sequence_index + 1 + start = header.index("AlleleSequence") + 1 + if [cell.strip() for cell in header[start : start + 1]] == [READ_TOTAL_COLUMN]: + start += 1 + return start + + +def madc_read_counts(cells: Sequence[str], path: Path) -> List[int]: + """A MADC row's per-sample counts. Blank cells are 0; HapApp writes counts of merged rows as floats (12.0).""" + try: + return list(map(int, cells)) + except ValueError: + counts = [] + for value in cells: + try: + counts.append(int(float(value or 0))) + except ValueError as exc: + raise MadcError(f"Non-numeric read count {value!r} in {path}") from exc + return counts + + +class CountOverflowCheck: + """Signed 8-bit overflow in a MADC's counts. + + Some vendor MADCs store counts in a signed byte: counts of 128-255 come + out as count - 256 (negative) and counts of 256 or more wrap to small + positives with no trace. A negative read count is impossible, so any is + reported, and a maximum of exactly 127 beside negatives points to overflow. + Adding 256 to the negatives would recover only the 128-255 band, so the + counts are reported, never corrected. + """ + + def __init__(self) -> None: + self.negative_cells = 0 + self.markers: Set[str] = set() + self.minimum: Optional[int] = None + self.maximum: Optional[int] = None + + def add(self, clone_id: str, counts: Sequence[int]) -> None: + if not counts: + return + low, high = min(counts), max(counts) + self.minimum = low if self.minimum is None else min(self.minimum, low) + self.maximum = high if self.maximum is None else max(self.maximum, high) + if low < 0: + self.negative_cells += sum(value < 0 for value in counts) + self.markers.add(clone_id) + + def summary(self) -> Dict[str, object]: + return { + "negative_cells": self.negative_cells, + "markers_with_negative_cells": len(self.markers), + "minimum": self.minimum, + "maximum": self.maximum, + "signed_8bit_signature": bool(self.negative_cells) and self.maximum == 127, + } + + def warning(self, path: Path) -> Optional[str]: + if not self.negative_cells: + return None + message = ( + f"warning: {path} has {self.negative_cells:,} negative read counts across {len(self.markers):,} " + f"markers (minimum {self.minimum})" + ) + if self.maximum == 127: + message += ( + ", and its largest count is exactly 127: the counts look like signed 8-bit overflow, where " + "128-255 are stored as count - 256 and counts of 256 or more wrap silently" + ) + return message + ". Counts are used as stored; regenerate them from the FASTQs or ask the vendor." + + +def read_madc(path: Path): + with open_text(path, "rt") as handle: + rows = list(csv.reader(handle)) + header_index = find_madc_header(rows) + header = rows[header_index] + allele_index, clone_index, sequence_index = madc_identity_columns(header, path) + sample_start = madc_sample_start(header) sample_names = list(header[sample_start:]) for preamble in reversed(rows[:header_index]): values = [value.strip() for value in preamble[sample_start:]] @@ -3327,16 +3553,13 @@ def read_madc(path: Path): sample_names = values break records = [] + overflow = CountOverflowCheck() for row in rows[header_index + 1 :]: if not row or not row[allele_index].strip(): continue padded = list(row) + [""] * max(0, len(header) - len(row)) - counts = [] - for value in padded[sample_start : sample_start + len(sample_names)]: - try: - counts.append(int(float(value or 0))) - except ValueError as exc: - raise MadcError(f"Non-numeric read count {value!r} in {path}") from exc + counts = madc_read_counts(padded[sample_start : sample_start + len(sample_names)], path) + overflow.add(padded[clone_index], counts) records.append( { "allele_id": padded[allele_index], @@ -3345,7 +3568,12 @@ def read_madc(path: Path): "counts": counts, } ) - return {"samples": sample_names, "records": records, "header_index": header_index} + warning = overflow.warning(path) + if warning: + print(warning, file=sys.stderr) + return { + "samples": sample_names, "records": records, "header_index": header_index, "count_overflow": overflow.summary() + } def allele_type(allele_id: str) -> str: @@ -3591,6 +3819,7 @@ def validate_madc(generated_path: Path, expected_path: Path) -> Dict[str, object "generated": len(generated_samples), "expected": len(expected_samples), }, + "count_overflow": {"generated": generated["count_overflow"], "expected": expected["count_overflow"]}, "allele_rows": { "common": len(common_rows), "generated": len(generated_rows), @@ -4389,7 +4618,7 @@ def _evidence_rule(args: argparse.Namespace) -> EvidenceRule: ) -def _assignment_settings(args: argparse.Namespace) -> AssignmentSettings: +def _assignment_settings(args: argparse.Namespace, families: Tuple[Tuple[int, ...], ...] = ()) -> AssignmentSettings: """The read-pass settings, which are also the cache key.""" return AssignmentSettings( tag_length=args.tag_length, @@ -4398,9 +4627,40 @@ def _assignment_settings(args: argparse.Namespace) -> AssignmentSettings: partial_min_length=args.min_partial_length, gap_band=args.max_indel_length if args.gapped_rescue == "on" else 0, leading_anchor=(getattr(args, "leading_anchor", None) or "").upper(), + tile_families=families, ) +def _report_tile_families(loci: Sequence[Locus], families: Tuple[Tuple[int, ...], ...], mode: str) -> None: + """Name the tile families on stderr, and the tiles a narrower mode leaves tied.""" + def describe(groups: Sequence[Tuple[int, ...]]) -> str: + shown = ", ".join("+".join(loci[index].marker for index in group) for group in groups[:5]) + return shown + (f" and {len(groups) - 5} more" if len(groups) > 5 else "") + + if families: + detected = sum( + 1 for family in families + if not loci[family[0]].locus_group or len({loci[index].locus_group for index in family}) > 1 + ) + print( + f"generate: {len(families)} tile famil{'y' if len(families) == 1 else 'ies'} " + f"({sum(map(len, families))} markers; {detected} detected, {len(families) - detected} declared by " + f"LocusGroup); a read is credited to every tile it fits: " + f"{describe(families)}", + file=sys.stderr, + ) + if mode != "auto": + linked = {index for family in families for index in family} + missed = [family for family in tile_families(loci, "auto") if not linked.issuperset(family)] + if missed: + print( + f"warning: {sum(map(len, missed))} markers read a tag window shared with another marker but are " + f"not linked with --tile-groups {mode}, so reads that fit them equally are dropped as ambiguous: " + f"{describe(missed)}; declare a LocusGroup or use --tile-groups auto", + file=sys.stderr, + ) + + def _discover_samples(args: argparse.Namespace) -> Tuple[List[Sample], Path, List[Path]]: """The run's samples: one per FASTQ, named from the target manifest or the lookup.""" fastq_directory = Path(args.fastq_dir) @@ -4494,6 +4754,7 @@ def _generate_provenance(args: argparse.Namespace) -> str: f"primer_mismatches={args.primer_mismatches}", f"gapped_rescue={args.gapped_rescue}", f"max_indel_length={args.max_indel_length}", + f"tile_groups={args.tile_groups}", f"annotations={args.annotations}", f"partial_read_mode={args.partial_read_mode}", f"min_partial_length={args.min_partial_length}", @@ -4536,6 +4797,11 @@ def _read_accounting( "reads_no_leading_anchor": total("reads_no_leading_anchor"), "reads_assigned_full_length": total("reads_assigned"), "reads_assigned_gapped": total("reads_assigned_gapped"), + # A read credited to several tiles of one family counts once above; + # these are the further credits. From here on the ledger counts + # (marker, read) credits, which equal reads for a panel without tiles. + "tile_extra_credits_full_length": total("tile_extra_credits"), + "tile_extra_credits_partial": total("partial_tile_extra_credits"), "reads_n_mapped": total("reads_n_mapped"), "reads_n_dropped": total("reads_n_dropped"), "partial_reads_added_to_counts": partial_summary.get("reads_assigned", 0), @@ -4556,6 +4822,7 @@ def _generate_parameters(args: argparse.Namespace, settings: AssignmentSettings, "primer_mismatches": args.primer_mismatches, "gapped_rescue": args.gapped_rescue, "max_indel_length": args.max_indel_length, + "tile_groups": args.tile_groups, "annotations": args.annotations, "partial_read_mode": args.partial_read_mode, "minimum_partial_length": args.min_partial_length, @@ -4594,7 +4861,9 @@ def run_generate(args: argparse.Namespace) -> int: args, [Path(sample.fastq) for sample in samples], {tag[:LEADING_ANCHOR_PREFIX] for locus in loci for tag in (locus.ref_tag, locus.alt_tag)}, "generate", ) - settings = _assignment_settings(args) + families = tile_families(loci, args.tile_groups) + _report_tile_families(loci, families, args.tile_groups) + settings = _assignment_settings(args, families) _warn_about_evidence_routes(rule, len(samples)) results = _assign_samples(samples, loci, settings, args.cache_dir, args.jobs) # Stage 3: optional partial-read salvage; stages 4-5: evidence and denoising @@ -4626,6 +4895,7 @@ def run_generate(args: argparse.Namespace) -> int: "panel_fingerprint_sha256": panel_fingerprint(loci), "parameters": _generate_parameters(args, settings, output), "loci": len(loci), + "tile_families": [[loci[index].marker for index in family] for family in families], "samples": len(samples), "read_accounting": _read_accounting(results, partial_summary, output_summary), "output_summary": output_summary, @@ -5449,6 +5719,17 @@ def _add_generate_command(subparsers) -> None: default=GAP_BAND_DEFAULT, help=f"widest gap (alignment band) a gapped alignment may use (default: {GAP_BAND_DEFAULT})", ) + assignment.add_argument( + "--tile-groups", + choices=("auto", "declared", "off"), + default="auto", + help=( + "link tiled markers (markers in one amplicon with a shared or nested left primer and tags that " + "agree except at their variant positions), so a read is credited to every tile it fits instead " + "of being dropped as a tie: auto links tiles declared with LocusGroup and detects undeclared " + "ones, declared links only the declared ones, off links none (default: auto)" + ), + ) assignment.add_argument( "--partial-read-mode", type=partial_read_mode, @@ -5475,7 +5756,8 @@ def _add_generate_command(subparsers) -> None: help=( "folder for a reusable per-sample cache, so re-runs skip the read pass. The cache is rebuilt " "when the panel or --tag-length, --min-identity, --leading-anchor, --primer-mismatches, " - "--gapped-rescue, --max-indel-length or --min-partial-length changes" + "--gapped-rescue, --max-indel-length, --min-partial-length or the tile families " + "(--tile-groups) change" ), ) _add_jobs_argument(assignment) @@ -5642,7 +5924,9 @@ def build_parser() -> argparse.ArgumentParser: " 1. build the panel design, or infer-design without a primer file\n" " 2. prepare reads if needed merge-pairs, window-scan, normalize-reads\n" " 3. count generate\n" - " 4. check (optional) validate\n" + " 4. clean (optional) filter-paralogs, with a reference genome\n" + " 5. check (optional) validate\n" + " 6. genotype (optional) call, from the MADC to VCFs\n" "\n" "Run 'madcats --help' for a command's options.\n" "Documentation: https://madcats.readthedocs.io" @@ -5657,6 +5941,12 @@ def build_parser() -> argparse.ArgumentParser: _add_normalize_reads_command(subparsers) _add_generate_command(subparsers) _add_validate_command(subparsers) + import madcats_paralogs # here, not at the top: it imports this module + + madcats_paralogs.add_filter_paralogs_command(subparsers) + import madcats_call + + madcats_call.add_call_command(subparsers) return parser @@ -5670,4 +5960,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if __name__ == "__main__": + # Run as a script, this file is __main__; madcats_paralogs's "import madcats" + # must find it rather than load a second copy with its own MadcError. + sys.modules.setdefault("madcats", sys.modules[__name__]) raise SystemExit(main()) diff --git a/madcats_call.py b/madcats_call.py new file mode 100644 index 0000000..0c016a2 --- /dev/null +++ b/madcats_call.py @@ -0,0 +1,3679 @@ +"""``madcats call``: genotype calls and VCFs from a MADC count table. + +The command reads a MADC (from ``generate``, a vendor delivery or HapApp), +estimates each sample's genotype at every marker, and writes one VCF per +requested type: + +``target`` one record per marker for the variant the panel was designed + for. Reads of the Ref and RefMatch rows count for the + reference allele, reads of the Alt and AltMatch rows for the + alternate allele, and reads of Other rows are left out. +``all`` the target records plus one record for every other position + of the tag where a row shows a base substitution. +``haplotype`` one record per marker whose alleles are the complete tag + sequences (microhaplotypes). + +Coordinates are marker-local: CHROM is the marker (CloneID) and POS is a +1-based position on its Ref tag. + +The statistical model is adapted from two R packages. Each function below +that follows one of them names the function it follows, and says where it +differs. + +polyRAD 2.0.1 (GPL >= 2; https://github.com/lvclark/polyRAD). Clark LV, + Lipka AE, Sacks EJ (2019) polyRAD: genotype calling with uncertainty from + sequencing data in polyploids and diploids. G3 9(3):663-673. + doi:10.1534/g3.118.200913. + From it: the beta-binomial likelihood of each allele against the rest of + the record with its contamination term, the Hardy-Weinberg prior and its + selfing adjustment (after de Silva HN, Hall AJ, Rikkerink E, McNeilage + MA, Fraser LG (2005) Heredity 95:327-334, doi:10.1038/sj.hdy.6800728), + the iteration of the allele frequencies, the marginal posteriors, the + correction of dosages to the ploidy, and the form of a prior with one + allele frequency per sample. +updog 2.1.7 (GPL >= 3; https://github.com/dcgerard/updog). Gerard D, Ferrão + LFV, Garcia AAF, Stephens M (2018) Genotyping polyploids from messy + sequencing data. Genetics 210(3):789-807. + doi:10.1534/genetics.118.301468. Gerard D, Ferrão LFV (2020) Priors for + genotyping polyploids. Bioinformatics 36(6):1795-1800. + doi:10.1093/bioinformatics/btz852. + From it: the allelic bias with its prior and starting values, the form + of the term for reads that show the wrong allele, the ``norm`` prior with + its limits, the free-weights prior (updog's ``flex``), and the two + summaries of the posterior written as MPP and PMC (updog's + ``maxpostprob`` and ``prop_mis``; see ``proportion_misgenotyped``). + +MADCATS's own, with no published counterpart: the estimate of the +overdispersion and of one background-read rate for the whole table +(``estimate_overdispersion``, ``estimate_read_model``) and its use of the +free-weights prior, the order of bias before background reads, the search for +the allelic bias (``fit_allele_bias``), the rules that use a fitted bias only +where it gains a margin and does not end at a limit of the search +(``BIAS_AUTO_MIN_GAIN``, ``BIAS_REJECTED_AT_LIMIT``) and the separate read +model of records called without a bias (``choose_read_model``), the fit of the +``norm`` prior by Newton's method and its fit per ploidy, the neighbour-based +``structure`` prior (``find_neighbours``), the exact search for dosages that +sum to the ploidy as a check on polyRAD's rule, and the VCF fields (GQ, GP, +PL, MPC). +The numerical agreement with both packages is checked by tests/test_call.py +against values stored in tests/fixtures/. + +In a record, each allele is considered on its own against all the other +alleles together, which turns a multiallelic record into one two-outcome +problem per allele: + +1. Likelihood. For a sample of ploidy P, an allele can be present in + 0, 1, ..., P copies (its *dosage*). The probability of the observed number + of reads of the allele, out of all the sample's reads in the record, is + beta-binomial. The expected read fraction for a dosage is dosage / P, + changed by an allelic bias if one is fitted (updog's; Gerard et al. 2018, + Genetics 210:789-807) and by the share of reads that show the wrong + allele (see ``expected_read_fraction``). +2. Prior. Either every dosage is equally probable (``uniform``), or dosages + follow Hardy-Weinberg proportions for the allele's frequency (``hwe``): + each of the P chromosome copies carries the allele independently. That is + the expectation for a diploid or an autopolyploid in a randomly mating + population. It is not the expectation under disomic inheritance in an + allopolyploid, or in a family. The ``norm`` prior makes no such + assumption: the dosage probabilities follow a normal shape whose mean and + spread are fitted to the samples of each ploidy. The ``structure`` prior + keeps Hardy-Weinberg proportions but gives every sample its own allele + frequency, taken from the samples most like it across the table's + markers (``find_neighbours``), for samples from several groups, clines or + families. +3. Posterior. Prior times likelihood, normalised over the dosages. +4. Iteration (``hwe``, ``norm`` and ``structure``). The allele frequencies, + and for ``norm`` the prior's two parameters, are re-estimated from the + posteriors and steps 1-3 repeat until they stop changing. Records do not + share any parameter, so each converges on its own (polyRAD stops all loci + together, on their mean change). With ``structure`` the lists of like + samples are made once for the whole table, and each record then iterates + on its own with them. + +The posteriors are *marginal*: one distribution per allele, not a joint +distribution over whole genotypes. With exactly two alleles the two +marginals mirror each other, so the posterior of the alternate allele is the +full genotype posterior and the record carries GQ, MPP, GP, PL and PMC. With +more alleles the most probable dosages need not add up to the ploidy; +``correct_dosages_to_ploidy`` (polyRAD's method) reconciles them, and the +record carries MPC, the lowest marginal posterior among the called dosages, +in place of those fields. + +Likelihoods are kept as logarithms and only normalised posteriors are +exponentiated, so a dosage that the reads make very improbable keeps a +usable value at any read depth. + +Reading map: the model (``expected_read_fraction``, ``dosage_shapes``, +``log_likelihoods_for_counts``, ``hwe_prior``, ``normal_prior``, ``fit_normal_prior``, ``dosage_posterior``, +``call_record``, ``fit_allele_bias``); from marginal posteriors to one +genotype (``correct_dosages_to_ploidy``, ``best_dosages_summing_to_ploidy``); +the estimate of the overdispersion and the background-read rate +(``estimate_overdispersion``, ``estimate_read_model``); the samples most +like each sample, for the ``structure`` prior (``find_neighbours``); input +(``read_count_table``, ``read_ploidy_table``); what each VCF type counts +(``resolve_target``, ``RowReading``, ``process_marker``); VCF text +(``format_record``, ``vcf_header``); the command line (``run_call``, +``add_call_command``). + +This module uses only the Python standard library. The MADC readers and the +tag alignment it shares with ``generate`` stay in ``madcats``. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import csv +import dataclasses +import datetime as dt +import functools +import heapq +import math +import operator +import re +import sys +import time +from array import array +from collections import Counter +from pathlib import Path +from typing import Dict, List, NamedTuple, Optional, Sequence, Set, Tuple + +from madcats import ( + GAP_BAND_DEFAULT, + IUPAC_BASES, + CountOverflowCheck, + Locus, + MadcError, + __version__, + _default_jobs, + _designed_events, + _write_json, + allele_type, + find_madc_header, + load_reference_tags, + madc_identity_columns, + madc_read_counts, + madc_sample_start, + nonnegative_int, + open_text, + positive_int, + proper_fraction, + variant_events, +) + +PRIOR_UNIFORM = "uniform" +PRIOR_HWE = "hwe" +PRIOR_NORM = "norm" +PRIOR_STRUCTURE = "structure" +PRIOR_CHOICES = (PRIOR_UNIFORM, PRIOR_HWE, PRIOR_NORM, PRIOR_STRUCTURE) +# A prior with one free probability per dosage (updog's "flex"). It is used to estimate +# the overdispersion and the background-read rate, and is not offered for calling: with +# as many parameters as dosages it follows the reads too closely to steady a call. +PRIOR_FREE = "free" + +# polyRAD's defaults: AddGenotypeLikelihood(overdispersion = 9), readDArTag(contamRate = 0.001) +# and IterateHWE(tol = 1e-5). polyRAD has no iteration limit. ModelSettings starts from +# these two, so that a bare ModelSettings() is polyRAD's model; the command sets its own. +POLYRAD_OVERDISPERSION = 9.0 +POLYRAD_CONTAMINATION_RATE = 0.001 +# The command estimates the overdispersion and the background-read rate unless they are +# given, and uses these when it cannot. The overdispersion stays at polyRAD's default +# until more real tables have been looked at; the two alfalfa tables the background rate +# was first estimated on gave 0.001 and 0.003. +FALLBACK_OVERDISPERSION = POLYRAD_OVERDISPERSION +FALLBACK_BACKGROUND_RATE = 0.002 +DEFAULT_TOLERANCE = 1e-5 +DEFAULT_MAX_ITERATIONS = 100 + +NEGATIVE_INFINITY = float("-inf") + + +@dataclasses.dataclass(frozen=True) +class ModelSettings: + """Parameters of the genotype model. + + overdispersion: how tightly read fractions follow the expected fraction. + The two beta-binomial shape parameters always add up to this value; + larger values mean less spread, and the distribution approaches the + binomial as it grows. + contamination_rate: fraction of a sample's reads expected to come from + other samples. Those reads carry an allele at its population frequency. + background_rate: probability that a read shows one allele although it + came from another (see ``expected_read_fraction``). + prior: ``uniform``, ``hwe``, ``norm`` (see ``normal_prior``) or + ``structure`` (see ``neighbours``). + selfing_rate: with the ``hwe`` prior, the share of individuals that come + from self-fertilisation, from 0 (random mating) to 1 (fully inbred). + fit_allele_bias: estimate an allelic bias for every record with two + alleles (see ``fit_allele_bias``). Without it both alleles are taken + to be read equally well. + min_bias_gain: with ``fit_allele_bias``, what a record's fitted bias must + add to the log-likelihood of its read counts, after the prior on the + bias, for the record to be called with it; a record that gains less + is called without a bias. 0 uses every fitted bias. + reject_bias_at_limit: with ``fit_allele_bias``, do not use a fitted bias + that ends at a limit of the search (``BIAS_LOG_LIMIT``); the record + is called without a bias. + read_model_without_bias: with ``fit_allele_bias``, the overdispersion and + the background rate for a record that is called without a bias: one + whose fitted bias is not used, and one with more than two alleles, + which gets none. ``overdispersion`` and ``background_rate`` are then + for the records called with their bias. None uses those for every + record. + tolerance: the passes of a record stop once the change they watch is at + or below this: the mean absolute change in the record's allele + frequencies (``hwe``), also the largest change in any prior + probability (``norm``), or the mean absolute change in the samples' + own frequencies (``structure`` with neighbours). + max_iterations: upper limit on the passes. The ``structure`` prior has + two stages (``hwe`` passes, then passes with the neighbours), and + each has this tolerance and this limit. + neighbours: with the ``structure`` prior, for each sample the indices of + the samples most like it (from ``find_neighbours``). A sample's + prior is then Hardy-Weinberg at the allele frequency among those + samples, not among all samples. Without neighbours the ``structure`` + prior is the ``hwe`` prior. + """ + + overdispersion: float = POLYRAD_OVERDISPERSION + contamination_rate: float = POLYRAD_CONTAMINATION_RATE + background_rate: float = 0.0 + prior: str = PRIOR_HWE + selfing_rate: float = 0.0 + fit_allele_bias: bool = False + min_bias_gain: float = 0.0 + reject_bias_at_limit: bool = False + read_model_without_bias: Optional[Tuple[float, float]] = None + tolerance: float = DEFAULT_TOLERANCE + max_iterations: int = DEFAULT_MAX_ITERATIONS + neighbours: Optional[Tuple[Tuple[int, ...], ...]] = None + + def __post_init__(self) -> None: + if self.overdispersion <= 0: + raise ValueError("overdispersion must be positive") + if not 0 <= self.contamination_rate < 1: + raise ValueError("contamination_rate must be at least 0 and below 1") + if not 0 <= self.background_rate < 0.5: + raise ValueError("background_rate must be at least 0 and below 0.5") + if self.prior not in PRIOR_CHOICES + (PRIOR_FREE,): + raise ValueError(f"prior must be one of {', '.join(PRIOR_CHOICES)}") + if not 0 <= self.selfing_rate <= 1: + raise ValueError("selfing_rate must be between 0 and 1") + if self.selfing_rate > 0 and self.prior != PRIOR_HWE: + raise ValueError("selfing_rate applies to the hwe prior only") + if self.tolerance <= 0: + raise ValueError("tolerance must be positive") + if self.max_iterations < 1: + raise ValueError("max_iterations must be at least 1") + if self.min_bias_gain < 0: + raise ValueError("min_bias_gain must be at least 0") + if self.fit_allele_bias and self.prior == PRIOR_STRUCTURE: + # The bias search scores a bias by the prior shared within a ploidy; it has not + # been worked out or tested for priors that differ from sample to sample. + raise ValueError("an allelic bias cannot be fitted with the structure prior") + + def without_bias(self) -> "ModelSettings": + """These settings for a record that is called without an allelic bias: those of a run that fits none.""" + overdispersion, background_rate = self.read_model_without_bias or (self.overdispersion, self.background_rate) + return dataclasses.replace( + self, fit_allele_bias=False, min_bias_gain=0.0, reject_bias_at_limit=False, read_model_without_bias=None, + overdispersion=overdispersion, background_rate=background_rate, + ) + + +@dataclasses.dataclass +class RecordCall: + """The model's result for one marker. + + ``log_likelihoods`` and ``posteriors`` are indexed + ``[sample][allele][dosage]``, with dosage running from 0 to the sample's + ploidy. ``allele_frequencies`` are the final estimates; the posteriors + were computed from the estimates of the pass before. ``final_change`` is + the last value of the change the passes stop on (see + ``ModelSettings.tolerance``). + + ``priors`` holds the prior the posteriors were computed with, as + ``[group][allele][dosage]``, and ``prior_groups`` each sample's group: + its ploidy, since samples of a ploidy share a prior, or under the + ``structure`` prior with neighbours the sample's own index. With the + ``norm`` prior, + ``normal_prior_parameters`` holds the last fitted + ``(mean, standard deviation)`` as ``[ploidy][allele]``, for the ploidy + groups large enough to be fitted. ``allele_biases`` holds the bias each + allele was called with (see ``expected_read_fraction``), 1 without one; + ``bias_fit`` describes the fit when a bias was fitted, whether or not the + record was called with it. + + With the ``structure`` prior and neighbours, ``sample_frequencies`` holds + the allele frequencies each sample's own prior was computed at, as + ``[sample][allele]``. + + ``has_reads`` is False when no sample has a read at the marker: there is + then no allele frequency to start from, and the other fields are empty. + """ + + has_reads: bool + allele_frequencies: List[float] + log_likelihoods: List[List[List[float]]] + posteriors: List[List[List[float]]] + iterations: int = 0 + final_change: Optional[float] = None + converged: bool = True + priors: Dict[int, List[List[float]]] = dataclasses.field(default_factory=dict) + prior_groups: List[int] = dataclasses.field(default_factory=list) + normal_prior_parameters: Dict[int, List[Tuple[float, float]]] = dataclasses.field(default_factory=dict) + allele_biases: List[float] = dataclasses.field(default_factory=list) + bias_fit: Optional["BiasFit"] = None + sample_frequencies: List[List[float]] = dataclasses.field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Likelihood +# --------------------------------------------------------------------------- + + +def log_binomial_coefficient(total: int, chosen: int) -> float: + """Natural log of the number of ways to choose ``chosen`` items out of ``total``.""" + return math.lgamma(total + 1) - math.lgamma(chosen + 1) - math.lgamma(total - chosen + 1) + + +def minimum_allele_frequency(sample_count: int) -> float: + """The smallest allele frequency the likelihood is given. + + An estimated frequency of exactly 0 or 1 would make the contamination + term vanish for that allele. polyRAD (``AddGenotypeLikelihood``) uses + ``1 / nTaxa / max(ploidies)``, where the ploidies are the locus ploidies + it is given as ``possiblePloidies``; with ``list(2)``, as in the stored + comparisons, that is the 1 / (2 * samples) used here, whatever the + samples' own ploidy. + """ + return 1.0 / (2.0 * sample_count) + + +def frequency_for_likelihood(allele_frequency: float, sample_count: int) -> float: + """``allele_frequency`` with exact 0 and 1 moved inside the open interval.""" + floor = minimum_allele_frequency(sample_count) + if allele_frequency == 0.0: + return floor + if allele_frequency == 1.0: + return 1.0 - floor + return allele_frequency + + +def expected_read_fraction( + dosage: int, + ploidy: int, + allele_frequency: float, + contamination_rate: float, + bias: float = 1.0, + background_rate: float = 0.0, +) -> float: + """Probability that one read of a sample shows the allele. + + A read comes from the sample itself with probability + 1 - contamination_rate, and then shows the allele in proportion to its + dosage. Otherwise it comes from another sample and shows the allele at + its population frequency (polyRAD's contamination). + + ``background_rate`` is the probability that a read shows the allele + although it came from another, and the reverse. It turns the fraction + ``f`` into ``f * (1 - background_rate) + (1 - f) * background_rate``, + which is how updog models sequencing error. Here it stands for every + source of such reads that is left after the MADC was built: sequencing + and amplification errors, index hopping and contamination alike. Unlike + polyRAD's contamination it does not depend on the allele's frequency, so + it also accounts for stray reads of an allele no other sample carries. + + ``bias`` is updog's allelic bias (Gerard et al. 2018, Genetics + 210:789-807): how readily a copy of any other allele yields a read, + relative to a copy of this allele. At 1 the two are read alike; above 1 + this allele is under-represented among the reads, and the fraction + ``f`` becomes ``f / (bias * (1 - f) + f)``. + + The bias is applied to the sample's own copies and the background reads + are added afterwards. updog applies its sequencing error first and the + bias second, and fits an error rate for every marker. With one + background rate for the whole table, that order would let the bias of a + marker stand in for its own rate of stray reads: samples that do not + carry an allele would pull the bias around, and at a marker with few + carriers they would outweigh them. In this order such samples say + nothing about the bias. The two orders differ by about the background + rate in the expected fraction, and not at all when it is 0. + """ + fraction = (dosage / ploidy) * (1.0 - contamination_rate) + contamination_rate * allele_frequency + if bias != 1.0: + fraction = fraction / (bias * (1.0 - fraction) + fraction) + if background_rate: + fraction = fraction * (1.0 - background_rate) + (1.0 - fraction) * background_rate + return fraction + + +class DosageShape(NamedTuple): + """The beta-binomial of one dosage: its two shape parameters, and the part of the + log-probability that depends only on them. ``fraction`` is the expected read fraction.""" + + fraction: float + shape_allele: float + shape_other: float + log_normaliser: float + + +def dosage_shapes( + ploidy: int, allele_frequency: float, settings: ModelSettings, bias: float = 1.0 +) -> List[DosageShape]: + """The beta-binomial of every dosage (0 to ``ploidy``) of one allele. + + This and ``log_likelihoods_for_counts`` are the likelihood of polyRAD's + ``AddGenotypeLikelihood``, kept as logarithms. The shape parameters are ``fraction * overdispersion`` and + ``(1 - fraction) * overdispersion``, where ``fraction`` is the expected + read fraction of the dosage. They are the same for every sample of the + ploidy, so they are worked out once and not per sample. + ``allele_frequency`` must already be inside the open interval (see + ``frequency_for_likelihood``). + """ + shapes = [] + for dosage in range(ploidy + 1): + fraction = expected_read_fraction( + dosage, ploidy, allele_frequency, settings.contamination_rate, bias, settings.background_rate + ) + if fraction <= 0.0 or fraction >= 1.0: + shapes.append(DosageShape(fraction, 0.0, 0.0, 0.0)) + continue + shape_allele = fraction * settings.overdispersion + shape_other = (1.0 - fraction) * settings.overdispersion + shapes.append( + DosageShape(fraction, shape_allele, shape_other, -math.lgamma(shape_allele) - math.lgamma(shape_other)) + ) + return shapes + + +def count_constant(allele_depth: int, record_depth: int, overdispersion: float) -> float: + """The part of the beta-binomial log-probability that is the same for every dosage. + + The two shape parameters add up to the overdispersion whatever the + dosage, so these terms depend on the read counts alone. + """ + return ( + log_binomial_coefficient(record_depth, allele_depth) + + math.lgamma(overdispersion) + - math.lgamma(record_depth + overdispersion) + ) + + +def log_likelihoods_for_counts( + allele_depth: int, record_depth: int, constant: float, shapes: Sequence[DosageShape] +) -> List[float]: + """Log-likelihood of each dosage for one sample's read counts. + + ``constant`` is ``count_constant`` for these counts and ``shapes`` is + ``dosage_shapes`` for the sample's ploidy. With no reads at the marker + every dosage has likelihood 1 (log 0.0). + """ + if record_depth == 0: + return [0.0] * len(shapes) + other_depth = record_depth - allele_depth + log_likelihoods = [] + for shape in shapes: + if shape.fraction <= 0.0 or shape.fraction >= 1.0: + # Only without contamination: a dosage of 0 cannot produce a read of the + # allele, and a dosage equal to the ploidy cannot produce any other read. + impossible = allele_depth > 0 if shape.fraction <= 0.0 else other_depth > 0 + log_likelihoods.append(NEGATIVE_INFINITY if impossible else 0.0) + else: + log_likelihoods.append( + constant + + math.lgamma(allele_depth + shape.shape_allele) + + math.lgamma(other_depth + shape.shape_other) + + shape.log_normaliser + ) + return log_likelihoods + + +def allele_log_likelihoods( + allele_depth: int, + record_depth: int, + ploidy: int, + allele_frequency: float, + settings: ModelSettings, + bias: float = 1.0, +) -> List[float]: + """Log-likelihood of each dosage (0 to ``ploidy``) of one allele in one sample. + + ``allele_depth`` reads out of ``record_depth`` showed the allele. The + likelihood is the beta-binomial probability of that count (see + ``dosage_shapes``). This is the likelihood written for one sample; the + calling code uses the two functions it is made of, so that samples of a + ploidy share their work. + """ + if record_depth == 0: + return [0.0] * (ploidy + 1) + return log_likelihoods_for_counts( + allele_depth, + record_depth, + count_constant(allele_depth, record_depth, settings.overdispersion), + dosage_shapes(ploidy, allele_frequency, settings, bias), + ) + + +# --------------------------------------------------------------------------- +# Priors and posteriors +# --------------------------------------------------------------------------- + + +def uniform_prior(ploidy: int) -> List[float]: + """Every dosage from 0 to ``ploidy`` equally probable.""" + return [1.0 / (ploidy + 1)] * (ploidy + 1) + + +def gamete_dosage_probabilities(parent_dosage: int, ploidy: int) -> List[float]: + """Probability that a gamete carries 0, 1, ..., ploidy / 2 copies of an allele. + + The gamete receives half of the parent's chromosome copies, drawn without + replacement from the ``parent_dosage`` copies that carry the allele and + the rest that do not (polysomic inheritance without double reduction). + The values are those of polyRAD's ``.gameteProb``, from the closed form. + """ + gamete_size = ploidy // 2 + return [ + math.comb(parent_dosage, copies) * math.comb(ploidy - parent_dosage, gamete_size - copies) + / math.comb(ploidy, gamete_size) + for copies in range(gamete_size + 1) + ] + + +def _solve_linear_system(matrix: Sequence[Sequence[float]], right_hand_side: Sequence[float]) -> List[float]: + """Solve ``matrix @ x = right_hand_side`` by Gaussian elimination with row pivoting.""" + size = len(right_hand_side) + rows = [list(matrix[index]) + [right_hand_side[index]] for index in range(size)] + for column in range(size): + pivot = max(range(column, size), key=lambda index: abs(rows[index][column])) + rows[column], rows[pivot] = rows[pivot], rows[column] + for index in range(column + 1, size): + factor = rows[index][column] / rows[column][column] + for position in range(column, size + 1): + rows[index][position] -= factor * rows[column][position] + solution = [0.0] * size + for index in range(size - 1, -1, -1): + already_solved = sum(rows[index][position] * solution[position] for position in range(index + 1, size)) + solution[index] = (rows[index][size] - already_solved) / rows[index][index] + return solution + + +@functools.lru_cache(maxsize=None) +def _selfing_system(ploidy: int, selfing_rate: float) -> Tuple[Tuple[float, ...], ...]: + """The matrix ``I - selfing_rate * S`` of ``hwe_prior``'s selfing adjustment. + + ``S[offspring][parent]`` is the probability that self-fertilising a + parent of one dosage gives an offspring of another: the offspring's + dosage is the sum of two gametes of the same parent. ``S`` is polyRAD's + ``.selfmat``. + """ + size = ploidy + 1 + selfed = [[0.0] * size for _ in range(size)] + for parent_dosage in range(size): + gamete = gamete_dosage_probabilities(parent_dosage, ploidy) + for first_copies, first_probability in enumerate(gamete): + for second_copies, second_probability in enumerate(gamete): + selfed[first_copies + second_copies][parent_dosage] += first_probability * second_probability + return tuple( + tuple((1.0 if row == column else 0.0) - selfing_rate * selfed[row][column] for column in range(size)) + for row in range(size) + ) + + +def hwe_prior(ploidy: int, allele_frequency: float, selfing_rate: float = 0.0) -> List[float]: + """Dosage probabilities under Hardy-Weinberg proportions, optionally with self-fertilisation. + + With random mating each of the ``ploidy`` chromosome copies carries the + allele independently with probability ``allele_frequency``, so the + dosage is binomial. + + With a ``selfing_rate`` between 0 and 1, a share of each generation comes + from self-fertilisation and the rest from random mating. The genotype + proportions that such a population settles at are those of equation 6 of + de Silva et al. (2005, Heredity 95:327-334), as in polyRAD: + ``(1 - s) * (I - s * S)^-1 * binomial``. Selfing moves probability from + the intermediate dosages to 0 and ``ploidy``. With a rate of 1 only + those two remain. As in polyRAD, an odd ploidy is left at random-mating + proportions unless the rate is 1, since its gametes cannot carry half + the copies. + """ + other_frequency = 1.0 - allele_frequency + if selfing_rate >= 1.0: + return [other_frequency] + [0.0] * (ploidy - 1) + [allele_frequency] + random_mating = [ + math.comb(ploidy, dosage) * allele_frequency**dosage * other_frequency ** (ploidy - dosage) + for dosage in range(ploidy + 1) + ] + if selfing_rate <= 0.0 or ploidy % 2 != 0: + return random_mating + settled = _solve_linear_system(_selfing_system(ploidy, selfing_rate), random_mating) + return [(1.0 - selfing_rate) * probability for probability in settled] + + +# Limits on the two parameters of the normal-shaped prior, as in updog. The largest +# standard deviation makes the prior practically uniform. +NORMAL_PRIOR_MIN_SD = 1e-8 +NORMAL_PRIOR_MAX_SD = 1000.0 +NORMAL_PRIOR_MEAN_MARGIN = 1.0 # the mean may lie this far below 0 or above the ploidy +# A ploidy group smaller than this uses the Hardy-Weinberg prior instead: two prior +# parameters cannot be estimated from a handful of samples. +FITTED_PRIOR_MIN_SAMPLES = 10 + + +def _log_normal_prior(ploidy: int, mean: float, standard_deviation: float) -> List[float]: + """Logarithms of ``normal_prior``; finite even where the probability itself rounds to zero.""" + exponents = [-((dosage - mean) ** 2) / (2.0 * standard_deviation**2) for dosage in range(ploidy + 1)] + largest = max(exponents) + log_total = largest + math.log(sum(math.exp(exponent - largest) for exponent in exponents)) + return [exponent - log_total for exponent in exponents] + + +def normal_prior(ploidy: int, mean: float, standard_deviation: float) -> List[float]: + """Dosage probabilities proportional to a normal density at 0, 1, ..., ``ploidy``. + + This is updog's ``norm`` genotype distribution (Gerard and Ferrao 2020, + Bioinformatics 36:1795-1800). With two parameters it can be narrower or + wider than Hardy-Weinberg proportions, so it does not assume random mating. + """ + return [math.exp(log_probability) for log_probability in _log_normal_prior(ploidy, mean, standard_deviation)] + + +def weighted_log_normal_prior(weights: Sequence[float], ploidy: int, mean: float, standard_deviation: float) -> float: + """Sum of ``weight * log(prior)`` over the dosages: what ``fit_normal_prior`` maximises.""" + log_prior = _log_normal_prior(ploidy, mean, standard_deviation) + return sum(weight * log_probability for weight, log_probability in zip(weights, log_prior) if weight > 0.0) + + +def _golden_section_maximum(function, low: float, high: float, tolerance: float = 1e-10) -> float: + """The argument between ``low`` and ``high`` at which a single-peaked function is highest.""" + ratio = (math.sqrt(5.0) - 1.0) / 2.0 + inner_low, inner_high = high - ratio * (high - low), low + ratio * (high - low) + value_low, value_high = function(inner_low), function(inner_high) + while high - low > tolerance: + if value_low < value_high: + low, inner_low, value_low = inner_low, inner_high, value_high + inner_high = low + ratio * (high - low) + value_high = function(inner_high) + else: + high, inner_high, value_high = inner_high, inner_low, value_low + inner_low = high - ratio * (high - low) + value_low = function(inner_low) + return (low + high) / 2.0 + + +# Range of the amount added to the curvature when a Newton step of the normal-shaped prior's +# fit overshoots. The curvature itself is at most a few hundred (the variance of a squared +# dosage), so the largest amount leaves a step too short to change the fit. +NEWTON_MIN_DAMPING = 1e-12 +NEWTON_MAX_DAMPING = 1e8 + + +def _normal_prior_newton_fit(shares: Sequence[float], ploidy: int, mean: float, standard_deviation: float): + """Fit the normal-shaped prior without limits on its parameters, by Newton's method. + + The quantity maximised is updog's (``obj_for_weighted_lnorm``); updog + maximises it with L-BFGS-B, and this method is MADCATS's own. + The prior is written as ``log probability = a * dosage + b * dosage**2 - + constant``, with ``a = mean / sd**2`` and ``b = -1 / (2 * sd**2)``. In + ``a`` and ``b`` the weighted log prior has a single peak, reached where + the prior's mean dosage and mean squared dosage equal those of + ``shares``. Returns ``(a, b)``, or None when the iteration does not + settle (the shares sit on one or two dosages and the peak is at infinity). + + A Newton step that does not improve the fit is shortened by adding to the + diagonal of the curvature (the Levenberg-Marquardt method): the more is + added, the shorter the step and the nearer its direction to the + gradient's. A start whose prior is almost entirely on one dosage has + next to no curvature, and its plain Newton step is too long by many + orders of magnitude; halving such a step does not rescue it. When no + amount added improves the fit, to within rounding, the peak has been + reached. + """ + dosages = range(ploidy + 1) + target_first = sum(share * dosage for share, dosage in zip(shares, dosages)) + target_second = sum(share * dosage**2 for share, dosage in zip(shares, dosages)) + linear, quadratic = mean / standard_deviation**2, -1.0 / (2.0 * standard_deviation**2) + + def prior_and_objective(linear_term: float, quadratic_term: float): + exponents = [linear_term * dosage + quadratic_term * dosage**2 for dosage in dosages] + largest = max(exponents) + unnormalised = [math.exp(exponent - largest) for exponent in exponents] + total = sum(unnormalised) + objective = linear_term * target_first + quadratic_term * target_second - largest - math.log(total) + return [value / total for value in unnormalised], objective + + prior, objective = prior_and_objective(linear, quadratic) + for _ in range(100): + moments = [ + sum(probability * dosage**power for probability, dosage in zip(prior, dosages)) for power in (1, 2, 3, 4) + ] + gradient_first, gradient_second = target_first - moments[0], target_second - moments[1] + # Variances and covariance of dosage and squared dosage under the current prior. + variance_first = moments[1] - moments[0] ** 2 + variance_second = moments[3] - moments[1] ** 2 + covariance = moments[2] - moments[0] * moments[1] + damping = 0.0 + while damping <= NEWTON_MAX_DAMPING: + damped_first, damped_second = variance_first + damping, variance_second + damping + determinant = damped_first * damped_second - covariance**2 + if determinant > 1e-300: + step_linear = (damped_second * gradient_first - covariance * gradient_second) / determinant + step_quadratic = (damped_first * gradient_second - covariance * gradient_first) / determinant + candidate = prior_and_objective(linear + step_linear, quadratic + step_quadratic) + if candidate[1] >= objective: + break + damping = max(NEWTON_MIN_DAMPING, 10.0 * damping) + else: + return linear, quadratic + linear, quadratic = linear + step_linear, quadratic + step_quadratic + improvement = candidate[1] - objective + prior, objective = candidate + if improvement < 1e-14: + return linear, quadratic + return None + + +def fit_normal_prior( + dosage_weights: Sequence[float], ploidy: int, mean: float, standard_deviation: float +) -> Tuple[float, float]: + """The mean and standard deviation of the normal-shaped prior that best fit dosage weights. + + ``dosage_weights[d]`` is the summed posterior probability of dosage ``d`` + over the samples: the expected number of samples with that dosage. The + fit maximises ``sum(weight * log(prior))``, which is the prior's update + in an EM iteration, and is exact because the prior depends on the samples + only through these sums. ``mean`` and ``standard_deviation`` are the + starting values, usually the previous fit. + + The parameters are kept within updog's limits: a mean from -1 to + ploidy + 1 and a standard deviation up to 1000. The search first ignores + the limits. If the best fit then lies outside them, the best fit within + them lies on their edge, because the function being maximised has a + single peak; the two edges for the mean are searched along the standard + deviation. (The third edge, the largest standard deviation, is a + practically uniform prior and is where those searches end if nothing + fits better.) + """ + total = sum(dosage_weights) + shares = [weight / total for weight in dosage_weights] + lowest_mean, highest_mean = -NORMAL_PRIOR_MEAN_MARGIN, ploidy + NORMAL_PRIOR_MEAN_MARGIN + + def objective(candidate_mean: float, candidate_sd: float) -> float: + return weighted_log_normal_prior(shares, ploidy, candidate_mean, candidate_sd) + + unlimited = _normal_prior_newton_fit(shares, ploidy, mean, standard_deviation) + if unlimited is not None and unlimited[1] < 0.0: + linear, quadratic = unlimited + fitted_sd = math.sqrt(-1.0 / (2.0 * quadratic)) + fitted_mean = -linear / (2.0 * quadratic) + if lowest_mean <= fitted_mean <= highest_mean and NORMAL_PRIOR_MIN_SD <= fitted_sd <= NORMAL_PRIOR_MAX_SD: + return fitted_mean, fitted_sd + + # The starting values are a candidate too, so that a fit is never worse than the one before. + start_mean = min(highest_mean, max(lowest_mean, mean)) + start_sd = min(NORMAL_PRIOR_MAX_SD, max(NORMAL_PRIOR_MIN_SD, standard_deviation)) + candidates = [(objective(start_mean, start_sd), start_mean, start_sd)] + for edge_mean in (lowest_mean, highest_mean): + log_sd = _golden_section_maximum( + lambda value: objective(edge_mean, math.exp(value)), + math.log(NORMAL_PRIOR_MIN_SD), math.log(NORMAL_PRIOR_MAX_SD), + ) + candidates.append((objective(edge_mean, math.exp(log_sd)), edge_mean, math.exp(log_sd))) + _, best_mean, best_sd = max(candidates) + return best_mean, best_sd + + +def log_probability(probability: float) -> float: + """Natural log of a probability, with minus infinity for a probability of zero.""" + return math.log(probability) if probability > 0.0 else NEGATIVE_INFINITY + + +def log_probabilities(probabilities: Sequence[float]) -> List[float]: + """``log_probability`` of each of several probabilities.""" + return [math.log(probability) if probability > 0.0 else NEGATIVE_INFINITY for probability in probabilities] + + +def log_sum_of_exponentials(log_values: Sequence[float]) -> float: + """Log of the sum of numbers given by their logs; minus infinity when all of them are zero.""" + largest = max(log_values) + if largest == NEGATIVE_INFINITY: + return NEGATIVE_INFINITY + return largest + math.log(sum(math.exp(log_value - largest) for log_value in log_values)) + + +def dosage_posterior( + log_likelihoods: Sequence[float], prior: Sequence[float], log_prior: Optional[Sequence[float]] = None +) -> List[float]: + """Posterior probability of each dosage: prior times likelihood, normalised. + + When the prior gives zero probability to every dosage the reads allow, + the product is zero everywhere and cannot be normalised; the prior is + returned unchanged, as polyRAD does. ``log_prior`` can be given when the + same prior is used for many samples, to take its logs once. + """ + if log_prior is None: + log_prior = log_probabilities(prior) + log_products = [log_likelihood + log_value for log_likelihood, log_value in zip(log_likelihoods, log_prior)] + largest = max(log_products) + if largest == NEGATIVE_INFINITY: + return list(prior) + # Subtract the largest term before exponentiating so that it becomes exp(0) = 1 and + # the sum cannot underflow to zero. + scaled = [math.exp(log_product - largest) for log_product in log_products] + total = sum(scaled) + return [value / total for value in scaled] + + +def posterior_mean_dosage(posterior: Sequence[float]) -> float: + """Expected number of copies of the allele (polyRAD's ``GetWeightedMeanGenotypes``, per sample).""" + return sum(dosage * probability for dosage, probability in enumerate(posterior)) + + +# --------------------------------------------------------------------------- +# One marker +# --------------------------------------------------------------------------- + + +def expected_allele_share(posterior: Sequence[float], ploidy: int) -> float: + """The share of a sample's chromosome copies expected to carry the allele. + + Shares can be compared and averaged between samples of different ploidy. + """ + return posterior_mean_dosage(posterior) / ploidy + + +def depth_ratio_frequencies(depths: Sequence[Sequence[int]]) -> Optional[List[float]]: + """Starting allele frequencies: each allele's mean share of a sample's reads. + + These are polyRAD's starting frequencies (the depth ratios averaged by + ``.alleleFreq``). + ``depths`` is ``[sample][allele]``. Samples with no reads at the marker + are left out. Returns None when no sample has a read. + """ + allele_count = len(depths[0]) + share_totals = [0.0] * allele_count + samples_with_reads = 0 + for sample_depths in depths: + record_depth = sum(sample_depths) + if record_depth == 0: + continue + samples_with_reads += 1 + for allele, allele_depth in enumerate(sample_depths): + share_totals[allele] += allele_depth / record_depth + if samples_with_reads == 0: + return None + return [total / samples_with_reads for total in share_totals] + + +# With the structure prior, the record's allele frequency over all samples counts as this +# many extra neighbours of every sample. It keeps a sample's frequency off 0 and 1 when +# all its neighbours agree. +NEIGHBOUR_RECORD_WEIGHT = 1.0 + + +def _frequency_priors( + ploidy_groups: Sequence[int], allele_frequencies: Sequence[float], settings: ModelSettings +) -> Dict[int, List[List[float]]]: + """Priors that follow from the allele frequencies alone, as ``[ploidy][allele][dosage]``. + + Uniform for the ``uniform`` prior and Hardy-Weinberg proportions for + every other prior: ``hwe`` uses them throughout, the fitted priors start + from them (``_fitted_priors``), and ``structure`` uses them until the + neighbours take over. Samples of the same ploidy share a prior. + """ + if settings.prior == PRIOR_UNIFORM: + return {ploidy: [uniform_prior(ploidy)] * len(allele_frequencies) for ploidy in ploidy_groups} + return { + ploidy: [hwe_prior(ploidy, frequency, settings.selfing_rate) for frequency in allele_frequencies] + for ploidy in ploidy_groups + } + + +def _fitted_priors( + posteriors: Sequence[Sequence[Sequence[float]]], + ploidies: Sequence[int], + allele_frequencies: Sequence[float], + previous_parameters: Dict[int, List[Tuple[float, float]]], + free: bool, +) -> Tuple[Dict[int, List[List[float]]], Dict[int, List[Tuple[float, float]]]]: + """Priors fitted to the current posteriors, one per ploidy group and allele. + + Returns the priors as ``[ploidy][allele][dosage]`` and, for the + normal-shaped prior, their ``(mean, standard deviation)`` as + ``[ploidy][allele]``. With ``free`` the prior of a dosage is simply its + share of the summed posteriors, with no shape imposed. Each ploidy group + is fitted on its own samples, because the dosages of one ploidy say + nothing about another. A group with fewer than + ``FITTED_PRIOR_MIN_SAMPLES`` samples gets the Hardy-Weinberg prior for + the marker's allele frequencies instead, and no parameters. + """ + priors: Dict[int, List[List[float]]] = {} + parameters: Dict[int, List[Tuple[float, float]]] = {} + for ploidy in set(ploidies): + group = [ + sample_posteriors + for sample_posteriors, sample_ploidy in zip(posteriors, ploidies) + if sample_ploidy == ploidy + ] + if len(group) < FITTED_PRIOR_MIN_SAMPLES: + priors[ploidy] = [hwe_prior(ploidy, frequency) for frequency in allele_frequencies] + continue + priors[ploidy] = [] + if not free: + parameters[ploidy] = [] + for allele, frequency in enumerate(allele_frequencies): + dosage_weights = [ + sum(sample_posteriors[allele][dosage] for sample_posteriors in group) for dosage in range(ploidy + 1) + ] + if free: + priors[ploidy].append([weight / len(group) for weight in dosage_weights]) + continue + if ploidy in previous_parameters: + start_mean, start_sd = previous_parameters[ploidy][allele] + else: + # The first fit starts from the binomial's mean and spread, as updog's does + # (flexdog_full); the floor of 0.1 on the spread is added here. + start_mean = ploidy * frequency + start_sd = max(0.1, math.sqrt(ploidy * frequency * (1.0 - frequency))) + mean, standard_deviation = fit_normal_prior(dosage_weights, ploidy, start_mean, start_sd) + parameters[ploidy].append((mean, standard_deviation)) + priors[ploidy].append(normal_prior(ploidy, mean, standard_deviation)) + return priors, parameters + + +def _neighbourhood_frequencies( + posteriors: Sequence[Sequence[Sequence[float]]], + ploidies: Sequence[int], + record_frequencies: Sequence[float], + neighbours: Sequence[Sequence[int]], +) -> List[List[float]]: + """Each sample's allele frequencies among its neighbours, as ``[allele][sample]``. + + A sample's frequency of an allele is the mean share of chromosome copies + expected to carry it among the sample's neighbours, with the record's + frequency over all samples counted as ``NEIGHBOUR_RECORD_WEIGHT`` more + neighbours. A sample is not among its own neighbours, so its own reads + do not enter its own frequency. + """ + frequencies = [] + for allele, record_frequency in enumerate(record_frequencies): + shares = [ + expected_allele_share(sample_posteriors[allele], ploidy) + for sample_posteriors, ploidy in zip(posteriors, ploidies) + ] + frequencies.append( + [ + (sum(map(shares.__getitem__, sample_neighbours)) + NEIGHBOUR_RECORD_WEIGHT * record_frequency) + / (len(sample_neighbours) + NEIGHBOUR_RECORD_WEIGHT) + for sample_neighbours in neighbours + ] + ) + return frequencies + + +def _neighbour_priors( + sample_frequencies: Sequence[Sequence[float]], ploidies: Sequence[int] +) -> Dict[int, List[List[float]]]: + """Hardy-Weinberg priors at each sample's own allele frequencies, as ``[sample][allele][dosage]``. + + This is the form of polyRAD's population-structure prior + (``AddGenotypePriorProb_ByTaxa``); where the frequencies come from is + different (``find_neighbours``). + """ + return { + sample: [hwe_prior(ploidy, of_allele[sample]) for of_allele in sample_frequencies] + for sample, ploidy in enumerate(ploidies) + } + + +def _log_likelihoods_of_samples( + depths: Sequence[Sequence[int]], + record_depths: Sequence[int], + count_constants: Sequence[Sequence[float]], + ploidies: Sequence[int], + allele_frequencies: Sequence[float], + settings: ModelSettings, + biases: Sequence[float], +) -> List[List[List[float]]]: + """Log-likelihood of every dosage for every sample and allele, as ``[sample][allele][dosage]``. + + ``depths`` holds the alleles being computed, which need not be all the + alleles of the record; ``record_depths`` is each sample's total over all + of them. ``count_constants`` is ``count_constant`` for every sample and + allele, which does not change between passes. The allele frequencies + matter only through the contamination term. + """ + sample_count = len(depths) + allele_count = len(allele_frequencies) + likelihood_frequencies = [frequency_for_likelihood(frequency, sample_count) for frequency in allele_frequencies] + # Samples of the same ploidy share the beta-binomial of each dosage. + shapes = { + ploidy: [ + dosage_shapes(ploidy, likelihood_frequencies[allele], settings, biases[allele]) + for allele in range(allele_count) + ] + for ploidy in set(ploidies) + } + return [ + [ + log_likelihoods_for_counts(sample_depths[allele], record_depth, constants[allele], shapes[ploidy][allele]) + for allele in range(allele_count) + ] + for sample_depths, record_depth, constants, ploidy in zip(depths, record_depths, count_constants, ploidies) + ] + + +def _posteriors_of_samples( + log_likelihoods: Sequence[Sequence[Sequence[float]]], + priors: Dict[int, List[List[float]]], + prior_groups: Sequence[int], +) -> List[List[List[float]]]: + """Posterior of every dosage for every sample and allele, as ``[sample][allele][dosage]``. + + ``priors`` is ``[group][allele][dosage]`` and ``prior_groups`` names each + sample's group: its ploidy when samples of a ploidy share a prior, or the + sample itself under the ``structure`` prior. + """ + # The samples of a group share the prior's logs. + log_priors = {group: [log_probabilities(prior) for prior in group_priors] for group, group_priors in priors.items()} + return [ + [ + dosage_posterior(allele_log_likelihoods, priors[group][allele], log_priors[group][allele]) + for allele, allele_log_likelihoods in enumerate(sample_log_likelihoods) + ] + for sample_log_likelihoods, group in zip(log_likelihoods, prior_groups) + ] + + +def frequencies_from_posteriors( + posteriors: Sequence[Sequence[Sequence[float]]], ploidies: Sequence[int] +) -> List[float]: + """Allele frequencies re-estimated from the posteriors. + + A sample's estimate of an allele's frequency is its posterior mean dosage + divided by its ploidy; the marker's estimate is the mean over samples. + Every sample counts equally, whatever its ploidy or read depth, as in + polyRAD. + """ + allele_count = len(posteriors[0]) + totals = [0.0] * allele_count + for sample_posteriors, ploidy in zip(posteriors, ploidies): + for allele, posterior in enumerate(sample_posteriors): + totals[allele] += expected_allele_share(posterior, ploidy) + return [total / len(posteriors) for total in totals] + + +def _iterate_record( + depths: Sequence[Sequence[int]], + record_depths: Sequence[int], + count_constants: Sequence[Sequence[float]], + ploidies: Sequence[int], + settings: ModelSettings, + frequencies: Sequence[float], + biases: Sequence[float], + warm_start: Optional[RecordCall] = None, + neighbours: Optional[Sequence[Sequence[int]]] = None, +) -> RecordCall: + """Run the passes of ``call_record`` for the alleles in ``depths`` with fixed biases. + + The passes are polyRAD's ``IterateHWE`` for one record, with a limit on + their number. For the ``norm`` prior they are the two steps of an EM + iteration, as in updog, but stopped on the change in the frequencies and + the prior where updog stops on the change in the log-likelihood. + + Each pass computes the posteriors from the current priors, re-estimates + the allele frequencies from them, and renews the priors: from the new + frequencies (``hwe``), by a fit to the posteriors (``norm`` and the + estimate's free weights), or, when ``neighbours`` is given, from each + sample's neighbours. The passes stop when the quantity the priors follow + from has settled: the record's allele frequencies, for the fitted priors + also every prior probability, and with neighbours the samples' own + frequencies. + + ``warm_start`` is an earlier result for the same alleles. Without + neighbours it is usually the result at a nearby bias, and its allele + frequencies and priors replace ``frequencies`` and the priors that follow + from them, so that fewer passes are needed. With neighbours it is the + record called with shared priors, which the samples' first frequencies + are taken from. + """ + sample_count = len(ploidies) + normal_parameters: Dict[int, List[Tuple[float, float]]] = {} + sample_frequencies: List[List[float]] = [] + if neighbours is not None: + # One prior per sample. + prior_groups = list(range(sample_count)) + frequencies = list(warm_start.allele_frequencies) + sample_frequencies = _neighbourhood_frequencies(warm_start.posteriors, ploidies, frequencies, neighbours) + priors = _neighbour_priors(sample_frequencies, ploidies) + else: + # One prior per ploidy. + prior_groups = list(ploidies) + if warm_start is None: + frequencies = list(frequencies) + priors = _frequency_priors(sorted(set(ploidies)), frequencies, settings) + else: + frequencies = list(warm_start.allele_frequencies) + priors = warm_start.priors + normal_parameters = warm_start.normal_prior_parameters + + # Only the contamination term makes the likelihood depend on the allele frequencies. + # Without it the likelihoods are the same in every pass and are computed once. + likelihoods_change = settings.contamination_rate > 0 + log_likelihoods = _log_likelihoods_of_samples( + depths, record_depths, count_constants, ploidies, frequencies, settings, biases + ) + if settings.prior == PRIOR_UNIFORM: + posteriors = _posteriors_of_samples(log_likelihoods, priors, prior_groups) + return RecordCall( + True, frequencies, log_likelihoods, posteriors, iterations=1, priors=priors, prior_groups=prior_groups, + allele_biases=list(biases), + ) + + for iterations in range(1, settings.max_iterations + 1): + if likelihoods_change and iterations > 1: + log_likelihoods = _log_likelihoods_of_samples( + depths, record_depths, count_constants, ploidies, frequencies, settings, biases + ) + posteriors = _posteriors_of_samples(log_likelihoods, priors, prior_groups) + new_frequencies = frequencies_from_posteriors(posteriors, ploidies) + used_priors, used_sample_frequencies = priors, sample_frequencies + if neighbours is not None: + sample_frequencies = _neighbourhood_frequencies(posteriors, ploidies, new_frequencies, neighbours) + priors = _neighbour_priors(sample_frequencies, ploidies) + change = sum( + abs(new - old) + for new_allele, old_allele in zip(sample_frequencies, used_sample_frequencies) + for new, old in zip(new_allele, old_allele) + ) / (len(new_frequencies) * sample_count) + else: + change = sum(abs(new - old) for new, old in zip(new_frequencies, frequencies)) / len(frequencies) + if settings.prior in (PRIOR_NORM, PRIOR_FREE): + priors, normal_parameters = _fitted_priors( + posteriors, ploidies, new_frequencies, normal_parameters, free=settings.prior == PRIOR_FREE + ) + largest_prior_change = max( + abs(new - old) + for ploidy in priors + for new_prior, old_prior in zip(priors[ploidy], used_priors[ploidy]) + for new, old in zip(new_prior, old_prior) + ) + change = max(change, largest_prior_change) + else: + priors = _frequency_priors(sorted(priors), new_frequencies, settings) + frequencies = new_frequencies + if change <= settings.tolerance: + break + return RecordCall( + True, + frequencies, + log_likelihoods, + posteriors, + iterations=iterations, + final_change=change, + converged=change <= settings.tolerance, + priors=used_priors, + prior_groups=prior_groups, + normal_prior_parameters=normal_parameters, + allele_biases=list(biases), + # As [sample][allele], the frequencies the posteriors were computed at. + sample_frequencies=[list(of_sample) for of_sample in zip(*used_sample_frequencies)], + ) + + +# updog's settings for the allelic bias: the five values each fit is started from, and +# the spread of the normal prior that pulls the log of the bias towards 0 (no bias). +BIAS_LOG_STARTS = (-1.0, -0.5, 0.0, 0.5, 1.0) +BIAS_LOG_PRIOR_SD = 0.7 +# The search goes no further than a bias of exp(-2) to exp(2), about 0.14 to 7.4, and +# stops when the log of the bias is known to this precision (about 1% of the bias). +BIAS_LOG_LIMIT = 2.0 +BIAS_LOG_TOLERANCE = 0.01 +# With --allele-bias auto a record is called with its fitted bias only when the fit raises +# the log-likelihood of its read counts, after the prior on the bias, by this much over no +# bias (``ModelSettings.min_bias_gain``). The value is MADCATS's own, chosen on potato, +# delta smelt and polyploids pooled from smelt reads: at 10 the records of biased data are +# called with their bias and nine in ten records of unbiased data without (docs/DEVELOPMENT.md). +BIAS_AUTO_MIN_GAIN = 10.0 +# Why a record was called without the bias fitted to it (``BiasFit.rejection``). A bias at +# a limit of the search is not used with --allele-bias auto +# (``ModelSettings.reject_bias_at_limit``): where samples have few heterozygotes, as inbred +# lines do, and the prior expects them, a bias at the limit makes homozygous samples pass +# for heterozygous ones and gains far more than the margin (docs/DEVELOPMENT.md). +BIAS_REJECTED_FOR_GAIN = "gain_below_margin" +BIAS_REJECTED_AT_LIMIT = "at_search_limit" +# A warning is given when this share of a run's fitted biases end at a limit of the search. +# On five real data sets the share was 0.3 to 2.5%; on simulated inbred lines, all of them. +BIAS_AT_LIMIT_WARNING_SHARE = 0.2 + + +class BiasFit(NamedTuple): + """How the allelic bias of a two-allele record was fitted. + + ``log_bias`` is the natural log of the alternate allele's fitted bias, and + ``objective`` the value that was maximised there: the log-likelihood of + the read counts plus the log of the prior on the bias. + ``nearest_log_start`` is the log bias of the starting value nearest the + result, ``fits`` the number of bias values tried, and ``at_limit`` + whether the result is the largest or smallest bias the search allows. + ``gain`` is how much the fitted bias raised the objective over no bias. + ``rejection`` is None when the record's call was made with the bias. + Otherwise it says why not, and the record's ``allele_biases`` are 1 + although a bias was fitted: ``BIAS_REJECTED_FOR_GAIN`` where the gain is + less than ``ModelSettings.min_bias_gain``, ``BIAS_REJECTED_AT_LIMIT`` + where the gain is enough but the bias is at a limit of the search and + ``ModelSettings.reject_bias_at_limit`` is set. + """ + + log_bias: float + objective: float + nearest_log_start: float + fits: int + at_limit: bool + gain: float + rejection: Optional[str] = None + + @property + def used(self) -> bool: + return self.rejection is None + + +def log_probability_of_counts(log_likelihoods: Sequence[float], log_prior: Sequence[float]) -> float: + """Log-probability of one sample's read count of an allele, with the sample's genotype summed out. + + The genotype is unknown, so the probability of the count is the average + over the dosages, weighted by their prior probabilities. Minus infinity + when the prior rules out every dosage the count allows. + """ + return log_sum_of_exponentials( + [log_likelihood + log_dosage_prior for log_likelihood, log_dosage_prior in zip(log_likelihoods, log_prior)] + ) + + +def marginal_log_likelihood( + log_likelihoods: Sequence[Sequence[Sequence[float]]], + priors: Dict[int, List[List[float]]], + prior_groups: Sequence[int], + allele: int = 0, +) -> float: + """Log-likelihood of the read counts of one allele, with the genotypes summed out. + + The sum over samples of ``log_probability_of_counts``. + ``log_likelihoods`` is ``[sample][allele][dosage]``, + ``priors`` is ``[group][allele][dosage]`` and ``prior_groups`` gives each + sample's group, as in ``RecordCall``. + """ + log_priors = {group: log_probabilities(prior[allele]) for group, prior in priors.items()} + total = 0.0 + for sample_log_likelihoods, group in zip(log_likelihoods, prior_groups): + sample_total = log_probability_of_counts(sample_log_likelihoods[allele], log_priors[group]) + # A sample whose counts the prior rules out entirely is left out, as its posterior + # falls back to the prior (see dosage_posterior). + if sample_total > NEGATIVE_INFINITY: + total += sample_total + return total + + +def fit_allele_bias( + depths: Sequence[Sequence[int]], + record_depths: Sequence[int], + count_constants: Sequence[Sequence[float]], + ploidies: Sequence[int], + settings: ModelSettings, + frequencies: Sequence[float], +) -> RecordCall: + """Call one allele of a two-allele record while estimating its allelic bias. + + ``depths`` holds the one allele (``[sample][0]``). The bias (see + ``expected_read_fraction``) is the value that makes the read counts most + probable, after a normal prior on its log that pulls it towards 1, as in + updog. For any given bias the record is called as usual + (``_iterate_record``), which also fits the allele frequency or the normal + prior, and the log-likelihood of the counts is taken from that result. + + The bias, its prior and the starting values are updog's (``flexdog``). + The search is MADCATS's own: updog runs its EM, which updates the bias, + the sequencing error and the overdispersion together, from each starting + value. + + The search has two stages. The record is first called at each of + updog's five starting values, each from scratch, so that a wrong first + guess of the genotypes at one bias cannot carry over to another. Every + starting value that scores at least as well as its neighbours is then + refined by a golden-section search between them, each call starting from + the nearest one already made; an outermost starting value is first moved + outwards in the same steps for as long as that improves the fit. The + result is the call with the highest value among all that were made. + + Several starting values are refined, not just the best, because their + scores before refinement can mislead. At a marker where all samples + share one or two dosages, the reads fit equally well with every dosage + shifted by one and a strong bias; only the prior on the bias tells the + two apart, and only once each has been refined to its best bias. + + With ``settings.min_bias_gain`` the best bias is used only when it raises + that value by as much over no bias, which is always among the calls made; + otherwise the record's result is its call without a bias, and the fitted + bias is only reported (``BiasFit``). A bias fitted to a record that has + none follows the noise in its counts, and on reads without bias this cost + accuracy, most of all in hexaploids. With + ``settings.reject_bias_at_limit`` a bias at a limit of the search is not + used either (see ``BIAS_REJECTED_AT_LIMIT``). Both rules are MADCATS's + own; updog always uses the bias it fits. + """ + tried: Dict[float, Tuple[float, RecordCall]] = {} + + def objective(log_bias: float, from_scratch: bool = False) -> float: + if log_bias not in tried: + warm_start = None + if tried and not from_scratch: + warm_start = tried[min(tried, key=lambda other: abs(other - log_bias))][1] + result = _iterate_record( + depths, record_depths, count_constants, ploidies, settings, frequencies, [math.exp(log_bias)], + warm_start, + ) + log_prior = -(log_bias**2) / (2.0 * BIAS_LOG_PRIOR_SD**2) + log_likelihood = marginal_log_likelihood(result.log_likelihoods, result.priors, result.prior_groups) + tried[log_bias] = (log_likelihood + log_prior, result) + return tried[log_bias][0] + + step = BIAS_LOG_STARTS[1] - BIAS_LOG_STARTS[0] + start_scores = [objective(log_bias, from_scratch=True) for log_bias in BIAS_LOG_STARTS] + last = len(BIAS_LOG_STARTS) - 1 + for index, centre in enumerate(BIAS_LOG_STARTS): + neighbour_scores = [start_scores[other] for other in (index - 1, index + 1) if 0 <= other <= last] + if start_scores[index] < max(neighbour_scores): + continue + if index in (0, last): + while abs(centre) + step <= BIAS_LOG_LIMIT + 1e-9: + further = centre + math.copysign(step, centre) + if objective(further) <= objective(centre): + break + centre = further + _golden_section_maximum( + objective, max(-BIAS_LOG_LIMIT, centre - step), min(BIAS_LOG_LIMIT, centre + step), BIAS_LOG_TOLERANCE + ) + log_bias = max(tried, key=lambda value: tried[value][0]) + best_objective, result = tried[log_bias] + gain = best_objective - tried[0.0][0] + at_limit = abs(log_bias) >= BIAS_LOG_LIMIT + rejection = None + if gain < settings.min_bias_gain: + rejection = BIAS_REJECTED_FOR_GAIN + elif at_limit and settings.reject_bias_at_limit: + rejection = BIAS_REJECTED_AT_LIMIT + if rejection is not None: + result = tried[0.0][1] + # The starting value whose refinement gave the fitted bias: the nearest one. + nearest_log_start = min(BIAS_LOG_STARTS, key=lambda start: abs(start - log_bias)) + result.bias_fit = BiasFit(log_bias, best_objective, nearest_log_start, len(tried), at_limit, gain, rejection) + return result + + +def _with_mirrored_reference_allele(alternate: RecordCall) -> RecordCall: + """Complete a two-allele record from the result for its alternate allele. + + With two alleles the reference allele's dosage is the ploidy minus the + alternate allele's, so everything about it is the alternate allele's + result read backwards: its likelihoods, prior and posterior in reverse + dosage order, its frequency one minus the other, and its bias the + reciprocal. + """ + def both(values: Sequence[float]) -> List[List[float]]: + return [list(reversed(values)), list(values)] + + return dataclasses.replace( + alternate, + allele_frequencies=[1.0 - alternate.allele_frequencies[0], alternate.allele_frequencies[0]], + log_likelihoods=[both(sample[0]) for sample in alternate.log_likelihoods], + posteriors=[both(sample[0]) for sample in alternate.posteriors], + priors={ploidy: both(prior[0]) for ploidy, prior in alternate.priors.items()}, + normal_prior_parameters={ + ploidy: [(ploidy - parameters[0][0], parameters[0][1]), parameters[0]] + for ploidy, parameters in alternate.normal_prior_parameters.items() + }, + allele_biases=[1.0 / alternate.allele_biases[0], alternate.allele_biases[0]], + sample_frequencies=[[1.0 - of_sample[0], of_sample[0]] for of_sample in alternate.sample_frequencies], + ) + + +def call_record( + depths: Sequence[Sequence[int]], + ploidies: Sequence[int], + settings: ModelSettings = ModelSettings(), +) -> RecordCall: + """Estimate dosage posteriors for every sample and allele of one marker. + + depths: read counts, ``[sample][allele]``. + ploidies: one ploidy per sample. + + With the ``uniform`` prior there is one pass, and the allele frequencies + are used only by the contamination term. With the ``hwe`` prior the + passes repeat until the frequencies settle (see ``ModelSettings``). With + the ``norm`` prior each pass also re-fits the prior to the posteriors + (the two steps of an EM iteration), and the passes repeat until neither + the frequencies nor any prior probability changes by more than the + tolerance. With the ``structure`` prior the record is first called with + the ``hwe`` prior, and the passes are then run again with each sample's + prior taken from its neighbours (``_neighbourhood_frequencies``). + + A record with two alleles is computed for its second (alternate) allele + only; the first is its mirror image (``_with_mirrored_reference_allele``). + With ``settings.fit_allele_bias`` such a record also gets an allelic bias + (``fit_allele_bias``); records with more alleles never do. A record that + is called without a bias, because it has more than two alleles or because + its fitted bias is not used, is called with ``settings.without_bias()``: + exactly as a run that fits no bias calls it. + """ + if len(depths) != len(ploidies): + raise ValueError("depths and ploidies must have one entry per sample") + # Without any neighbours the structure prior is the Hardy-Weinberg prior. + use_neighbours = settings.prior == PRIOR_STRUCTURE and settings.neighbours is not None and any(settings.neighbours) + if use_neighbours and len(settings.neighbours) != len(ploidies): + raise ValueError("settings.neighbours must have one entry per sample") + frequencies = depth_ratio_frequencies(depths) + if frequencies is None: + return RecordCall(has_reads=False, allele_frequencies=[], log_likelihoods=[], posteriors=[]) + + record_depths = [sum(sample_depths) for sample_depths in depths] + two_alleles = len(frequencies) == 2 + if two_alleles: + depths = [[sample_depths[1]] for sample_depths in depths] + frequencies = [frequencies[1]] + elif settings.fit_allele_bias: + settings = settings.without_bias() + + def count_constants_at(overdispersion: float) -> List[List[float]]: + return [ + [count_constant(allele_depth, record_depth, overdispersion) if record_depth else 0.0 + for allele_depth in sample_depths] + for sample_depths, record_depth in zip(depths, record_depths) + ] + + count_constants = count_constants_at(settings.overdispersion) + if two_alleles and settings.fit_allele_bias: + result = fit_allele_bias(depths, record_depths, count_constants, ploidies, settings, frequencies) + if not result.bias_fit.used and settings.read_model_without_bias is not None: + fit = result.bias_fit + settings = settings.without_bias() + count_constants = count_constants_at(settings.overdispersion) + result = _iterate_record(depths, record_depths, count_constants, ploidies, settings, frequencies, [1.0]) + result.bias_fit = fit + else: + # Without neighbours, _iterate_record gives the structure prior Hardy-Weinberg + # proportions, which is where its second run starts from. + result = _iterate_record( + depths, record_depths, count_constants, ploidies, settings, frequencies, [1.0] * len(frequencies) + ) + if use_neighbours: + shared = result + result = _iterate_record( + depths, record_depths, count_constants, ploidies, settings, shared.allele_frequencies, + shared.allele_biases, warm_start=shared, neighbours=settings.neighbours, + ) + result.iterations += shared.iterations + result.converged = result.converged and shared.converged + return _with_mirrored_reference_allele(result) if two_alleles else result + + +# --------------------------------------------------------------------------- +# From marginal posteriors to one genotype per sample +# --------------------------------------------------------------------------- + + +def most_probable_dosage(posterior: Sequence[float]) -> Optional[int]: + """The dosage with the highest posterior probability, or None when two tie for it. + + polyRAD's ``BestGenos`` takes the first of tied dosages, and compares in + single precision, so that it also ties posteriors such as 0.4, 0.4, 0.2. + """ + highest = max(posterior) + if sum(1 for probability in posterior if probability == highest) > 1: + return None + return list(posterior).index(highest) + + +def correct_dosages_to_ploidy( + dosages: Sequence[Optional[int]], + posteriors: Sequence[Sequence[float]], + ploidy: int, +) -> Optional[List[int]]: + """Adjust the dosages of one sample's alleles until they add up to ``ploidy``. + + ``dosages`` holds each allele's most probable dosage and ``posteriors`` + its dosage posterior. This is ``CorrectGenos`` of polyRAD 2.0 and later + (``GetProbableGenotypes(multiallelic = "correct")``): while the sum + is short, add one copy to the allele for which doing so leaves the product + of the posteriors of all the dosages highest; while it is over, remove one + copy by the same rule. One copy at a time is fast but not guaranteed to + reach the best possible set of dosages; ``best_dosages_summing_to_ploidy`` + is. + + Returns None (no call) when any allele's dosage is None, or when every + possible adjustment has zero probability. + """ + if any(dosage is None for dosage in dosages): + return None + corrected = [int(dosage) for dosage in dosages] + while sum(corrected) != ploidy: + step = 1 if sum(corrected) < ploidy else -1 + best_allele = None + best_log_probability = NEGATIVE_INFINITY + for allele, dosage in enumerate(corrected): + if not 0 <= dosage + step <= ploidy: + continue + # The log-probability of the whole genotype if this allele takes the step. + genotype_log_probability = sum( + log_probability(posteriors[other][corrected[other] + (step if other == allele else 0)]) + for other in range(len(corrected)) + ) + if genotype_log_probability > best_log_probability: + best_allele, best_log_probability = allele, genotype_log_probability + if best_allele is None: + return None + corrected[best_allele] += step + return corrected + + +def best_dosages_summing_to_ploidy(posteriors: Sequence[Sequence[float]], ploidy: int) -> Optional[List[int]]: + """The dosages that add up to ``ploidy`` with the highest product of marginal posteriors. + + It maximises the same quantity as ``correct_dosages_to_ploidy`` but over + every combination, so its result is never less probable. polyRAD had an + exhaustive search for the same quantity (``BestMultiGeno``, deprecated in + 2.0); this one is a dynamic programme and is used only to count how often + polyRAD's rule falls short of it. The search goes + through the alleles in order, keeping for each number of copies used so + far the best assignment that uses that many; the work grows with the + number of alleles times the square of the ploidy. + + Returns None when no combination has a probability above zero. + """ + # best_by_copies_used[copies] = (log probability, dosages of the alleles handled so far) + best_by_copies_used: Dict[int, Tuple[float, List[int]]] = {0: (0.0, [])} + for posterior in posteriors: + extended: Dict[int, Tuple[float, List[int]]] = {} + for copies_used, (log_probability, chosen) in best_by_copies_used.items(): + for dosage in range(ploidy - copies_used + 1): + if posterior[dosage] <= 0.0: + continue + candidate = log_probability + math.log(posterior[dosage]) + total_copies = copies_used + dosage + if total_copies not in extended or candidate > extended[total_copies][0]: + extended[total_copies] = (candidate, chosen + [dosage]) + best_by_copies_used = extended + if ploidy not in best_by_copies_used: + return None + return best_by_copies_used[ploidy][1] + + +# --------------------------------------------------------------------------- +# Estimating the read model from the data: overdispersion and background-read rate +# --------------------------------------------------------------------------- + +# Values tried by ``estimate_overdispersion``. Steps widen as the values grow, because +# the beta-binomial changes less and less as it approaches the binomial. +OVERDISPERSION_GRID = (2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 25, 30, 40, 50, 75, 100, 150, 200, 300, 500) +# The estimate uses at most this many sample-by-marker cells, from markers taken at even +# spacing through the table, so that it stays quick and the same table always gives the +# same value. +OVERDISPERSION_MAX_CELLS = 20000 +# However many samples there are, at least this many markers are used when the table has them. +OVERDISPERSION_MIN_MARKERS = 20 +# With fewer sample-by-marker cells that have reads, the estimate would be mostly noise. +OVERDISPERSION_MIN_CELLS = 100 +OVERDISPERSION_MAX_ROUNDS = 4 +# Background-read rates tried by ``estimate_read_model``. +BACKGROUND_RATE_GRID = (0.0002, 0.0005, 0.001, 0.002, 0.003, 0.005, 0.01, 0.02) + + +@dataclasses.dataclass +class OverdispersionEstimate: + """Result of ``estimate_overdispersion``. + + ``value`` is the chosen overdispersion, or None when fewer than + ``OVERDISPERSION_MIN_CELLS`` sample-by-marker cells had reads + (``cells_with_reads``), which is too little to estimate from. + ``log_likelihood_by_value`` maps every value tried to the log-likelihood + of the read counts under it, in the last round. ``at_limit`` is ``"lowest"`` or + ``"highest"`` when the chosen value is an end of the grid, so the best + value may lie beyond it. ``values_by_round`` is the value chosen in each + round, and ``markers_used`` the number of markers the estimate is based on. + """ + + value: Optional[float] + log_likelihood_by_value: Dict[float, float] + markers_used: int + values_by_round: List[float] + at_limit: Optional[str] = None + cells_with_reads: int = 0 + + +def marker_overdispersion_log_likelihoods( + depths: Sequence[Sequence[int]], ploidies: Sequence[int], settings: ModelSettings, grid: Sequence[float] +) -> Optional[List[float]]: + """Log-likelihood of one two-allele marker's read counts at each overdispersion in ``grid``. + + depths: ``[sample][allele]`` read counts of the reference and alternate allele. + + The marker is first called with ``settings``. The allele frequency, the + genotype prior and the allelic bias of that call are then held fixed + while the overdispersion takes each value of ``grid``, and the + log-likelihood of the counts is computed with the genotypes summed out + (as in ``marginal_log_likelihood``). Returns None for a marker without + reads. + """ + result = call_record(depths, ploidies, settings) + if not result.has_reads: + return None + frequency = frequency_for_likelihood(result.allele_frequencies[1], len(depths)) + bias = result.allele_biases[1] + log_priors = {group: log_probabilities(prior[1]) for group, prior in result.priors.items()} + # The samples with reads, each with the term of count_constant that no overdispersion changes. + samples_with_reads = [ + (alternate_depth, reference_depth + alternate_depth, + log_binomial_coefficient(reference_depth + alternate_depth, alternate_depth), ploidy, group) + for (reference_depth, alternate_depth), ploidy, group in zip(depths, ploidies, result.prior_groups) + if reference_depth + alternate_depth > 0 + ] + totals = [] + for value in grid: + at_value = dataclasses.replace(settings, overdispersion=value) + shapes = {ploidy: dosage_shapes(ploidy, frequency, at_value, bias) for ploidy in set(ploidies)} + log_gamma_of_value = math.lgamma(value) + total = 0.0 + for alternate_depth, record_depth, log_coefficient, ploidy, group in samples_with_reads: + # count_constant, with its terms in the same order. + constant = log_coefficient + log_gamma_of_value - math.lgamma(record_depth + value) + log_likelihoods = log_likelihoods_for_counts(alternate_depth, record_depth, constant, shapes[ploidy]) + sample_total = log_probability_of_counts(log_likelihoods, log_priors[group]) + if sample_total > NEGATIVE_INFINITY: + total += sample_total + totals.append(total) + return totals + + +def _evenly_spaced(items: Sequence, limit: int) -> List: + """At most ``limit`` of the items, taken at even spacing.""" + if len(items) <= limit: + return list(items) + return [items[index * len(items) // limit] for index in range(limit)] + + +def _markers_for_estimate( + depths_by_marker: Sequence[Sequence[Sequence[int]]], sample_count: int +) -> Tuple[List[Sequence[Sequence[int]]], int]: + """The markers the read-model estimate works on, and how many of their sample cells have reads. + + Markers without reads are left out, and at most enough markers to make up + ``OVERDISPERSION_MAX_CELLS`` cells are kept, at even spacing. Selecting + from an earlier selection returns it unchanged. + """ + samples_with_reads = [ + sum(1 for sample_depths in depths if sum(sample_depths) > 0) for depths in depths_by_marker + ] + usable = [(depths, count) for depths, count in zip(depths_by_marker, samples_with_reads) if count > 0] + marker_limit = max(OVERDISPERSION_MIN_MARKERS, OVERDISPERSION_MAX_CELLS // max(1, sample_count)) + chosen_markers = _evenly_spaced(usable, marker_limit) + return [depths for depths, _ in chosen_markers], sum(count for _, count in chosen_markers) + + +def estimate_overdispersion( + depths_by_marker: Sequence[Sequence[Sequence[int]]], + ploidies: Sequence[int], + settings: ModelSettings = ModelSettings(), + grid: Sequence[float] = OVERDISPERSION_GRID, + start: float = POLYRAD_OVERDISPERSION, + parallel_map=map, +) -> OverdispersionEstimate: + """Choose the overdispersion that makes the observed read counts most probable. + + depths_by_marker: ``[marker][sample][allele]`` read counts of two alleles + per marker (the reference-type and alternate-type rows). + settings: the model the markers are called with while estimating; its + overdispersion is ignored. The command passes the free-weights prior + whatever prior the run uses (``choose_read_model``). + parallel_map: a function like ``map``, used to work on the markers; a + process pool's ``map`` spreads them over its workers. + + For every value in ``grid`` the log-likelihood of the counts is added up + over the markers (``marker_overdispersion_log_likelihoods``), and the + value with the highest total is chosen. Each marker's allele frequency, + genotype prior and allelic bias come from calling it under the current + value, so the search is repeated with the chosen value until the choice + no longer changes, which usually takes two rounds. + + This is a maximum-likelihood estimate and MADCATS's own; updog instead + fits an overdispersion for every marker. polyRAD's ``TestOverdispersion`` + instead tests the read counts of samples that a rough genotype estimate + assigns one allele copy. That works in diploids, but in polyploids the + rough estimate assigns the wrong dosage often enough that the counts look + more scattered than they are (see docs/DEVELOPMENT.md). + """ + markers, cells_with_reads = _markers_for_estimate(depths_by_marker, len(ploidies)) + if cells_with_reads < OVERDISPERSION_MIN_CELLS: + return OverdispersionEstimate(None, {}, len(markers), [], cells_with_reads=cells_with_reads) + + current = start + values_by_round: List[float] = [] + log_likelihood: Dict[float, float] = {} + for _ in range(OVERDISPERSION_MAX_ROUNDS): + per_marker = parallel_map( + functools.partial( + marker_overdispersion_log_likelihoods, + ploidies=ploidies, + settings=dataclasses.replace(settings, overdispersion=current), + grid=grid, + ), + markers, + ) + totals = [0.0] * len(grid) + for marker_totals in per_marker: + if marker_totals is not None: + totals = [total + marker_total for total, marker_total in zip(totals, marker_totals)] + log_likelihood = dict(zip(grid, totals)) + best = max(grid, key=lambda value: log_likelihood[value]) + values_by_round.append(best) + if best == current: + break + current = best + chosen = values_by_round[-1] + at_limit = "lowest" if chosen == min(grid) else "highest" if chosen == max(grid) else None + return OverdispersionEstimate(chosen, log_likelihood, len(markers), values_by_round, at_limit, cells_with_reads) + + +@dataclasses.dataclass +class ReadModelEstimate: + """Result of ``estimate_read_model``. + + ``background_rate`` is the chosen rate and ``overdispersion_estimate`` + the overdispersion estimate made at that rate. ``by_background_rate`` maps + every rate tried to the overdispersion chosen at it and the + log-likelihood of the read counts there. + """ + + background_rate: float + overdispersion_estimate: OverdispersionEstimate + by_background_rate: Dict[float, Tuple[Optional[float], float]] + + +def estimate_read_model( + depths_by_marker: Sequence[Sequence[Sequence[int]]], + ploidies: Sequence[int], + settings: ModelSettings, + background_rates: Sequence[float] = BACKGROUND_RATE_GRID, + overdispersions: Sequence[float] = OVERDISPERSION_GRID, + parallel_map=map, + start_background_rate: float = FALLBACK_BACKGROUND_RATE, + start_overdispersion: float = POLYRAD_OVERDISPERSION, +) -> ReadModelEstimate: + """Choose the background-read rate and the overdispersion that make the read counts most probable. + + For a given background-read rate the overdispersion is chosen by + ``estimate_overdispersion``, which also gives the log-likelihood of the + counts at its choice. The rate with the highest such log-likelihood is + found by starting at the rate nearest ``start_background_rate`` and + moving to a neighbouring rate for as long as that raises it; the search + for the overdispersion at the first rate starts from + ``start_overdispersion``. Give a single value in ``background_rates`` or + ``overdispersions`` to hold that parameter fixed. + + The two parameters can be told apart because they act on different + samples: reads of an allele in samples that do not carry it show the + background rate, and the spread of read fractions among samples that + carry both alleles shows the overdispersion. In simulated diploids and + tetraploids with 20 or more reads per sample both were recovered; with 6 + to 25 reads the rate was still recovered, and the overdispersion was + within a factor of two of a simulated 40 and unmeasurable at 150 (see + docs/DEVELOPMENT.md). + """ + rates = sorted(background_rates) + tried: Dict[float, OverdispersionEstimate] = {} + # Every rate is judged on the same markers, selected once. + depths_by_marker, _ = _markers_for_estimate(depths_by_marker, len(ploidies)) + + def log_likelihood_at(index: int) -> float: + rate = rates[index] + if rate not in tried: + # Start each search from the overdispersion chosen at the rate tried before. + start = start_overdispersion + if tried: + previous = tried[min(tried, key=lambda other: abs(other - rate))] + start = previous.value if previous.value is not None else start + if len(overdispersions) == 1: + start = overdispersions[0] + tried[rate] = estimate_overdispersion( + depths_by_marker, ploidies, dataclasses.replace(settings, background_rate=rate), + overdispersions, start, parallel_map, + ) + estimate = tried[rate] + if not estimate.log_likelihood_by_value: + return NEGATIVE_INFINITY + return max(estimate.log_likelihood_by_value.values()) + + best = min(range(len(rates)), key=lambda index: abs(rates[index] - start_background_rate)) + log_likelihood_at(best) + while True: + neighbours = [index for index in (best - 1, best + 1) if 0 <= index < len(rates)] + better = [index for index in neighbours if log_likelihood_at(index) > log_likelihood_at(best)] + if not better: + break + best = max(better, key=log_likelihood_at) + return ReadModelEstimate( + rates[best], + tried[rates[best]], + {rate: (estimate.value, log_likelihood_at(rates.index(rate))) for rate, estimate in sorted(tried.items())}, + ) + + +# --------------------------------------------------------------------------- +# Population structure: the samples most like each sample +# --------------------------------------------------------------------------- + +# The numbers of neighbours tried; 0 is the plain Hardy-Weinberg prior. +NEIGHBOUR_COUNTS = (0, 5, 10, 20, 40, 80) +# Samples are compared on at most this many markers, taken at even spacing. +NEIGHBOUR_MAX_MARKERS = 1000 +# Below these sizes the table is too small to tell which samples are alike. +NEIGHBOUR_MIN_MARKERS = 100 +NEIGHBOUR_MIN_SAMPLES = 20 +# The number of neighbours is judged on at most this many markers of each half. +NEIGHBOUR_JUDGED_MARKERS = 250 +# The samples are handed to the worker processes in about this many blocks. +NEIGHBOUR_BLOCKS = 16 + + +@dataclasses.dataclass +class NeighbourChoice: + """Result of ``find_neighbours``. + + ``neighbours`` holds, for each sample, the indices of the samples most + like it, nearest first. Every sample has ``neighbours_per_sample`` of + them, which is 0 when neighbours predict a sample no better than the + whole table does, or when the table is too small to tell; ``reason`` + then says which. + ``markers_used`` is the number of markers the samples were compared on. + ``error_by_count`` maps each number of neighbours tried to the mean + squared error with which that many neighbours predicted the samples' + allele shares at markers they had not been chosen on. + """ + + neighbours: Tuple[Tuple[int, ...], ...] + neighbours_per_sample: int + markers_used: int + error_by_count: Dict[int, float] + reason: Optional[str] = None + + +def expected_alternate_shares( + depths: Sequence[Sequence[int]], ploidies: Sequence[int], settings: ModelSettings +) -> Optional[List[float]]: + """Each sample's expected share of chromosome copies that carry a marker's second allele. + + ``depths`` is ``[sample][allele]`` with two alleles. The share is the + posterior mean dosage divided by the ploidy, so that samples of different + ploidy can be compared. A sample without reads gets the marker's allele + frequency. Returns None when no sample has a read. + """ + result = call_record(depths, ploidies, settings) + if not result.has_reads: + return None + return [ + expected_allele_share(sample_posteriors[1], ploidy) + for sample_posteriors, ploidy in zip(result.posteriors, ploidies) + ] + + +def _sum_of_products(first: Sequence[float], second: Sequence[float]) -> float: + return sum(map(operator.mul, first, second)) + + +if hasattr(math, "sumprod"): # Python 3.12 and later: several times faster + _sum_of_products = math.sumprod # noqa: F811 + + +def _neighbour_candidates( + samples: Sequence[int], + shares_by_half: Sequence[Sequence[Sequence[float]]], + judged_by_half: Sequence[Sequence[Sequence[float]]], + judged_means: Sequence[Sequence[float]], + neighbour_counts: Sequence[int], +) -> List[Tuple[List[int], List[float]]]: + """For each of ``samples``: the samples most like it, and how well its nearest ones predict it. + + ``shares_by_half[half][sample]`` holds a sample's allele shares at the + markers of one half of the table, and ``judged_by_half`` the same for a + thinned set of each half's markers, whose means over all samples are + ``judged_means``. Two samples are alike when their shares differ little: + the measure is the sum of squared differences over markers. + + Returns, per sample, its ``max(neighbour_counts)`` most similar samples over both + halves together, nearest first, and for each entry of ``neighbour_counts`` + (ascending) the summed squared error of predicting the sample's shares at + the judged markers of one half from that many of its nearest samples as + found on the other half, both ways round. The prediction is the one the + prior uses: the neighbours' mean with the marker's mean counted as + ``NEIGHBOUR_RECORD_WEIGHT`` more of them. + """ + # The rows arrive as compact arrays; products of plain lists are about twice as fast. + shares_by_half = [[list(row) for row in half] for half in shares_by_half] + sample_count = len(shares_by_half[0]) + limit = max(neighbour_counts) + squared_lengths = [[_sum_of_products(row, row) for row in half] for half in shares_by_half] + results = [] + for sample in samples: + others = [other for other in range(sample_count) if other != sample] + differences_by_half = [] + for half, lengths in zip(shares_by_half, squared_lengths): + own = half[sample] + # |a - b|^2 = |a|^2 + |b|^2 - 2 a.b + differences_by_half.append( + [lengths[sample] + lengths[other] - 2.0 * _sum_of_products(own, half[other]) + for other in range(sample_count)] + ) + errors = [0.0] * len(neighbour_counts) + for measured, judged in ((0, 1), (1, 0)): + nearest = heapq.nsmallest(limit, others, key=differences_by_half[measured].__getitem__) + own = judged_by_half[judged][sample] + means = judged_means[judged] + neighbour_totals = [0.0] * len(own) + included = 0 + for index, count in enumerate(neighbour_counts): + for other in nearest[included:count]: + neighbour_totals = list(map(operator.add, neighbour_totals, judged_by_half[judged][other])) + included = count + weight = count + NEIGHBOUR_RECORD_WEIGHT + errors[index] += sum( + (value - (total + NEIGHBOUR_RECORD_WEIGHT * mean) / weight) ** 2 + for value, total, mean in zip(own, neighbour_totals, means) + ) + overall = [first + second for first, second in zip(*differences_by_half)] + results.append((heapq.nsmallest(limit, others, key=overall.__getitem__), errors)) + return results + + +def find_neighbours( + depths_by_marker: Sequence[Sequence[Sequence[int]]], + ploidies: Sequence[int], + settings: ModelSettings, + parallel_map=map, +) -> NeighbourChoice: + """Find the samples most like each sample across markers, and how many of them to use. + + This is the one step of the ``structure`` prior that looks at the whole + table, and it is MADCATS's own. polyRAD's population-structure prior + (``IteratePopStruct``) gets each sample's allele frequencies from a + principal component analysis of all markers and a regression on its + axes; here they come from the sample's neighbours. The prior built from + them has polyRAD's form (``_neighbour_priors``). ``depths_by_marker`` is ``[marker][sample][allele]`` with two + alleles per marker (reference and alternate). + + Every marker is called with the ``hwe`` prior, which gives each sample an + expected share of alternate copies per marker. Samples whose shares + differ little over the markers are alike: members of one subpopulation, + of one family, or close on a cline. The sum of squared differences used + here ranks pairs of samples as a genomic relationship matrix would. + + How many neighbours to use is chosen from ``NEIGHBOUR_COUNTS`` by + prediction. The markers are split in two halves; the neighbours found on + one half predict each sample's shares at the other half, and the number + with the smallest error wins. With too many neighbours a small group or + family is diluted, and with too few the frequency is noisy. When no + number predicts better than the table's own frequencies (0 neighbours) + there is no structure worth modelling and the prior stays Hardy-Weinberg. + The halves matter: judged on the markers they were found on, neighbours + always look better than they are, because each of those markers helped + pick them. + + On simulated groups, clines and families and on thinned potato DArTag + counts, the prior built from these lists called as many genotypes + correctly as polyRAD's population-structure prior or more (see + docs/DEVELOPMENT.md). + + ``parallel_map`` is a replacement for ``map``, for calling the markers and + comparing the samples in worker processes. + """ + sample_count = len(ploidies) + no_neighbours = tuple(() for _ in ploidies) + if sample_count < NEIGHBOUR_MIN_SAMPLES: + return NeighbourChoice(no_neighbours, 0, 0, {}, f"fewer than {NEIGHBOUR_MIN_SAMPLES} samples") + first_stage = dataclasses.replace(settings, prior=PRIOR_HWE, fit_allele_bias=False, neighbours=None) + shares_by_marker = [ + shares + for shares in parallel_map( + functools.partial(expected_alternate_shares, ploidies=ploidies, settings=first_stage), + _evenly_spaced(depths_by_marker, NEIGHBOUR_MAX_MARKERS), + ) + if shares is not None + ] + if len(shares_by_marker) < NEIGHBOUR_MIN_MARKERS: + return NeighbourChoice( + no_neighbours, 0, len(shares_by_marker), {}, f"fewer than {NEIGHBOUR_MIN_MARKERS} markers with reads" + ) + + def rows_by_sample(markers: Sequence[Sequence[float]]) -> List[array]: + """Turn ``[marker][sample]`` into one compact row of shares per sample.""" + return [array("d", (marker_shares[sample] for marker_shares in markers)) for sample in range(sample_count)] + + halves = (shares_by_marker[0::2], shares_by_marker[1::2]) + judged_halves = [_evenly_spaced(half, NEIGHBOUR_JUDGED_MARKERS) for half in halves] + neighbour_counts = [count for count in NEIGHBOUR_COUNTS if count < sample_count] + block_size = -(-sample_count // NEIGHBOUR_BLOCKS) + blocks = [range(start, min(start + block_size, sample_count)) for start in range(0, sample_count, block_size)] + candidates = [ + candidate + for block_candidates in parallel_map( + functools.partial( + _neighbour_candidates, + shares_by_half=[rows_by_sample(half) for half in halves], + judged_by_half=[rows_by_sample(half) for half in judged_halves], + judged_means=[[sum(marker_shares) / sample_count for marker_shares in half] for half in judged_halves], + neighbour_counts=neighbour_counts, + ), + blocks, + ) + for candidate in block_candidates + ] + judged_cells = sample_count * sum(len(half) for half in judged_halves) + error_by_count = { + count: sum(errors[index] for _, errors in candidates) / judged_cells + for index, count in enumerate(neighbour_counts) + } + best = min(neighbour_counts, key=lambda count: (error_by_count[count], count)) + reason = None if best else "neighbours predict the samples' allele shares no better than the whole table does" + return NeighbourChoice( + tuple(tuple(nearest[:best]) for nearest, _ in candidates), best, len(shares_by_marker), error_by_count, reason + ) + + +# --------------------------------------------------------------------------- +# Input: the MADC count table and the ploidy table +# --------------------------------------------------------------------------- + +VCF_TARGET = "target" +VCF_ALL = "all" +VCF_HAPLOTYPE = "haplotype" +VCF_TYPES = (VCF_TARGET, VCF_ALL, VCF_HAPLOTYPE) +VCF_FILE_SUFFIX = {VCF_TARGET: ".targets.vcf", VCF_ALL: ".all.vcf", VCF_HAPLOTYPE: ".haplotypes.vcf"} + +REFERENCE_CATEGORIES = ("Ref", "RefMatch") +ALTERNATE_CATEGORIES = ("Alt", "AltMatch") +CONCRETE_BASES = frozenset("ACGT") + +DEFAULT_PLOIDY = 2 +DEFAULT_MIN_DEPTH = 10 +DEFAULT_MIN_GENOTYPE_QUALITY = 10 +DEFAULT_MIN_MARGINAL_POSTERIOR = 0.9 +MAX_GENOTYPE_QUALITY = 99 +MAX_PHRED_LIKELIHOOD = 99999 +# A record is listed in the QC report when at least this share of its reads was left out. +NOTABLE_EXCLUDED_READ_SHARE = 0.10 + + +class MadcRow(NamedTuple): + """One allele row of a MADC: its ID, category (Ref, Alt, RefMatch, AltMatch or Other), + tag sequence and read count in each sample.""" + + allele_id: str + category: str + sequence: str + counts: Tuple[int, ...] + + +@dataclasses.dataclass +class MarkerTask: + """Everything needed to call one marker. + + ``design_ref_tag``, ``design_alt_tag`` and ``declared_target_position`` + (1-based) come from ``--reference-tags`` and are empty or None when the + marker has no entry there; the Ref and Alt rows are used instead. + """ + + clone_id: str + rows: List[MadcRow] + design_ref_tag: str = "" + design_alt_tag: str = "" + declared_target_position: Optional[int] = None + + +def read_count_table(path: Path) -> Tuple[List[str], Dict[str, List[MadcRow]], Dict[str, object]]: + """Read a MADC into sample names, rows grouped by marker, and a count-overflow summary. + + Accepts the layouts ``filter-paralogs`` does: a MADC from ``generate``, a + vendor MADC with or without its starred preamble, and HapApp's + fixed-allele-ID table. Sample names are the header row's. Markers keep + the order in which they first appear. + """ + with open_text(path, "rt") as handle: + table = list(csv.reader(handle)) + header_index = find_madc_header(table) + header = [cell.strip() for cell in table[header_index]] + allele_column, clone_column, sequence_column = madc_identity_columns(header, path) + sample_start = madc_sample_start(header) + sample_names = header[sample_start:] + if not sample_names or any(not name for name in sample_names): + raise MadcError(f"MADC has no sample columns, or a sample column without a name: {path}") + repeated = sorted(name for name, occurrences in Counter(sample_names).items() if occurrences > 1) + if repeated: + raise MadcError(f"MADC has repeated sample names ({', '.join(repeated[:5])}): {path}") + + rows_by_marker: Dict[str, List[MadcRow]] = {} + overflow = CountOverflowCheck() + for cells in table[header_index + 1 :]: + if len(cells) <= allele_column or not cells[allele_column].strip(): + continue + padded = list(cells) + [""] * max(0, len(header) - len(cells)) + clone_id = padded[clone_column].strip() + counts = madc_read_counts(padded[sample_start : sample_start + len(sample_names)], path) + overflow.add(clone_id, counts) + allele_id = padded[allele_column].strip() + rows_by_marker.setdefault(clone_id, []).append( + MadcRow(allele_id, allele_type(allele_id), padded[sequence_column].strip().upper(), tuple(counts)) + ) + if not rows_by_marker: + raise MadcError(f"MADC has no allele rows: {path}") + warning = overflow.warning(path) + if warning: + print(warning, file=sys.stderr) + return sample_names, rows_by_marker, overflow.summary() + + +def read_ploidy_table(path: Path) -> Dict[str, int]: + """Read sample ploidies from a two-column table: sample name, ploidy. + + Columns are separated by a tab or a comma. A first line whose second + column is not a whole number is taken as a header and skipped. + """ + ploidy_by_sample: Dict[str, int] = {} + with open_text(path, "rt") as handle: + for line_number, line in enumerate(handle, start=1): + fields = [field.strip() for field in re.split(r"[\t,]", line.rstrip("\r\n"))] + if not any(fields): + continue + if len(fields) < 2: + raise MadcError(f"{path} line {line_number}: expected a sample name and a ploidy") + name, value = fields[0], fields[1] + if not re.fullmatch(r"\d+", value): + if line_number == 1: + continue + raise MadcError(f"{path} line {line_number}: ploidy {value!r} is not a whole number") + if int(value) < 1: + raise MadcError(f"{path} line {line_number}: ploidy must be at least 1") + if name in ploidy_by_sample: + raise MadcError(f"{path} lists sample {name!r} more than once") + ploidy_by_sample[name] = int(value) + return ploidy_by_sample + + +# --------------------------------------------------------------------------- +# What each VCF type counts +# --------------------------------------------------------------------------- + + +class TargetVariant(NamedTuple): + """The designed variant of a marker: 1-based position on the Ref tag and its two alleles. + + A substitution has single-base alleles. An insertion or deletion is + written the VCF way, starting at the base before it (``C`` > ``CAAAA``). + """ + + position: int + ref_allele: str + alt_allele: str + + +def _is_concrete(sequence: str) -> bool: + return bool(sequence) and set(sequence) <= CONCRETE_BASES + + +def resolve_target( + ref_tag: str, alt_tag: str, declared_position: Optional[int], gap_band: int +) -> Tuple[Optional[TargetVariant], Optional[str]]: + """Find the designed variant from the marker's Ref and Alt tags. + + Returns ``(target, None)``, or ``(None, reason)`` when the target cannot + be identified with certainty. It is certain when the two tags differ by + exactly one substitution, insertion or deletion, or when they differ by + more but ``declared_position`` (``VariantPositionInTagSequence`` of the + reference tags) names one of the substitutions. The target is never + guessed among several differences. + """ + if not alt_tag: + return None, "no_alt_row" + differences = [ + event for event in _designed_events(ref_tag, alt_tag, gap_band) if event[0] in ("snp", "ins", "del") + ] + if len(differences) != 1 and declared_position is not None: + differences = [event for event in differences if event[0] == "snp" and event[1] == declared_position] + if len(differences) != 1: + return None, "ambiguous_design" + _, position, ref_allele, alt_allele = differences[0] + if not _is_concrete(ref_allele) or not _is_concrete(alt_allele): + return None, "ambiguous_design" + return TargetVariant(position, ref_allele, alt_allele), None + + +class RowReading: + """Which base a row's tag shows at each position of the marker's Ref tag. + + Built from the row's events against the Ref tag (``madcats.variant_events``, + the same reading that fills the MADC's Variants column). A position is + not shown when the row deleted it, when an insertion pushed it out of the + fixed-length window, or when the row's tag is not the Ref tag's length. + """ + + def __init__(self, ref_tag: str, sequence: str, events) -> None: + self.ref_tag = ref_tag + self.comparable = len(sequence) == len(ref_tag) + self.substitutions: Dict[int, str] = {} + self.positions_not_shown: Set[int] = set() + for kind, position, ref_allele, alt_allele in events: + if kind == "snp": + self.substitutions[position] = alt_allele + elif kind == "del": + # Written from the base before the deleted ones: that base is still shown. + # A deletion at the very start of the tag is written from the base after it + # instead; the two cannot always be told apart there, so the whole span is + # treated as not shown. + first_missing = position if position == 1 else position + 1 + self.positions_not_shown.update(range(first_missing, position + len(ref_allele))) + elif kind == "clip": + self.positions_not_shown.update(range(position, len(ref_tag) + 1)) + + def base_at(self, position: int) -> Optional[str]: + """The base at a 1-based Ref-tag position, or None when the row does not show one + (including an N or other ambiguous base in the read).""" + if not self.comparable or position in self.positions_not_shown: + return None + base = self.substitutions.get(position, self.ref_tag[position - 1]) + return base if base in CONCRETE_BASES else None + + +def vcf_reference_sequence(tag: str) -> str: + """A tag as a VCF REF allele. + + VCF allows only A, C, G, T and N in REF, and says to replace any other + ambiguity code by the alphabetically first base it stands for. + """ + return "".join( + base if base in CONCRETE_BASES or base == "N" else min(IUPAC_BASES.get(base, "N")) for base in tag + ) + + +def _summed_counts(counts_by_row: Sequence[Sequence[int]], row_indices: Sequence[int], sample_count: int) -> List[int]: + """Per-sample read totals over the given rows.""" + totals = [0] * sample_count + for row_index in row_indices: + for sample, count in enumerate(counts_by_row[row_index]): + totals[sample] += count + return totals + + +def target_allele_depths( + rows: Sequence[MadcRow], counts_by_row: Sequence[Sequence[int]], sample_count: int +) -> Tuple[List[List[int]], List[int]]: + """Reads for and against the designed variant, and the reads that show neither. + + Returns ``[sample][allele]`` depths of the reference allele (Ref and + RefMatch rows) and the alternate allele (Alt and AltMatch rows), and the + per-sample depth of every other row. + """ + reference_rows = [index for index, row in enumerate(rows) if row.category in REFERENCE_CATEGORIES] + alternate_rows = [index for index, row in enumerate(rows) if row.category in ALTERNATE_CATEGORIES] + other_rows = [ + index for index, row in enumerate(rows) if row.category not in REFERENCE_CATEGORIES + ALTERNATE_CATEGORIES + ] + reference_depth = _summed_counts(counts_by_row, reference_rows, sample_count) + alternate_depth = _summed_counts(counts_by_row, alternate_rows, sample_count) + depths = [list(pair) for pair in zip(reference_depth, alternate_depth)] + return depths, _summed_counts(counts_by_row, other_rows, sample_count) + + +def _without_negative_cells(rows: Sequence[MadcRow]) -> Tuple[List[List[int]], int]: + """The rows' counts with every sample that has a negative count at the marker set to zero. + + A negative read count (signed 8-bit overflow in some vendor MADCs) makes + the sample's depth at the marker unknown, so the sample is treated as + having no reads there. Returns the counts and the number of samples + cleared. + """ + counts_by_row = [list(row.counts) for row in rows] + samples_with_negative = { + sample for counts in counts_by_row for sample, count in enumerate(counts) if count < 0 + } + for counts in counts_by_row: + for sample in samples_with_negative: + counts[sample] = 0 + return counts_by_row, len(samples_with_negative) + + +# --------------------------------------------------------------------------- +# VCF records +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class CallOptions: + """Settings of one ``madcats call`` run that the per-marker work needs.""" + + settings: ModelSettings + vcf_types: Tuple[str, ...] + min_depth: int = DEFAULT_MIN_DEPTH + min_genotype_quality: int = DEFAULT_MIN_GENOTYPE_QUALITY + min_marginal_posterior: float = DEFAULT_MIN_MARGINAL_POSTERIOR + max_other_fraction: Optional[float] = None + gap_band: int = GAP_BAND_DEFAULT + overdispersion_estimated: bool = False + background_rate_estimated: bool = False + + +def _decimal(value: float, places: int = 4) -> str: + """A number with at most ``places`` decimals and no trailing zeros.""" + text = f"{value:.{places}f}".rstrip("0").rstrip(".") + return text if text not in ("", "-0") else "0" + + +def genotype_text(dosages: Optional[Sequence[int]], ploidy: int) -> str: + """The VCF GT value for allele dosages (``[1, 0, 2]`` gives ``0/2/2``), or all-missing. + + Alleles are joined by ``/`` because the copies are not phased. + """ + if dosages is None: + return "/".join(["."] * ploidy) + return "/".join(str(allele) for allele, dosage in enumerate(dosages) for _ in range(dosage)) + + +def genotype_quality(posterior: Sequence[float], called_dosage: int) -> int: + """Phred-scaled probability that the called dosage is wrong, capped at 99. + + The VCF 4.3 definition of GQ, applied to the dosage posterior. polyRAD's + VCF export writes no such field. + """ + chance_wrong = sum(probability for dosage, probability in enumerate(posterior) if dosage != called_dosage) + if chance_wrong <= 0.0: + return MAX_GENOTYPE_QUALITY + return min(MAX_GENOTYPE_QUALITY, int(round(-10.0 * math.log10(chance_wrong)))) + + +def phred_scaled_likelihoods(log_likelihoods: Sequence[float]) -> List[int]: + """VCF PL values (VCF 4.3): each dosage's likelihood relative to the most likely one, Phred-scaled.""" + best = max(log_likelihoods) + scaled = [] + for log_likelihood in log_likelihoods: + if log_likelihood == NEGATIVE_INFINITY: + scaled.append(MAX_PHRED_LIKELIHOOD) + else: + scaled.append(min(MAX_PHRED_LIKELIHOOD, int(round(-10.0 * (log_likelihood - best) / math.log(10.0))))) + return scaled + + +def _two_allele_genotype( + posteriors: Sequence[Sequence[float]], log_likelihoods: Sequence[Sequence[float]], ploidy: int +) -> Tuple[Optional[List[int]], Optional[float], List[str]]: + """Call a sample at a record with a reference and one alternate allele. + + Returns the allele dosages (None on a tie), the genotype quality, and the + GQ, MPP, DS, GP and PL values. The alternate allele's posterior is the + genotype posterior, because the reference dosage is the ploidy minus it. + MPP is its largest value (updog's ``maxpostprob``): the probability of the + called genotype, which GQ gives on the Phred scale. + """ + alternate_posterior = posteriors[1] + alternate_dosage = most_probable_dosage(alternate_posterior) + quality = None if alternate_dosage is None else genotype_quality(alternate_posterior, alternate_dosage) + values = [ + "." if quality is None else str(quality), + _decimal(max(alternate_posterior)), + _decimal(posterior_mean_dosage(alternate_posterior), 3), + ",".join(_decimal(probability) for probability in alternate_posterior), + ",".join(str(value) for value in phred_scaled_likelihoods(log_likelihoods[1])), + ] + dosages = None if alternate_dosage is None else [ploidy - alternate_dosage, alternate_dosage] + return dosages, quality, values + + +def proportion_misgenotyped(genotype_posteriors: Sequence[Sequence[float]]) -> float: + """The share of the samples expected to have a wrong genotype at a record (updog's ``prop_mis``). + + A sample's most probable genotype is wrong with a probability of one + minus that genotype's posterior, so the mean of those probabilities over + the samples is the expected share of wrong genotypes, as far as the model + fits the record. + + updog averages over every sample it was given a depth for, and a sample + with a depth of 0 enters at the largest probability of the prior. The + posteriors passed here are those of the samples with reads, which equals + updog's value when it is given the other samples as missing. + """ + chance_wrong = [1.0 - max(posterior) for posterior in genotype_posteriors] + return sum(chance_wrong) / len(chance_wrong) + + +def _multiallelic_genotype( + posteriors: Sequence[Sequence[float]], ploidy: int, tallies: Counter +) -> Tuple[Optional[List[int]], Optional[float], List[str]]: + """Call a sample at a record with any number of alleles from the marginal posteriors. + + Returns the allele dosages (None when there is no call), the lowest + marginal posterior among the called dosages (MPC), and the DS and MPC + values. ``tallies`` counts how often the dosages had to be corrected to + the ploidy, and how often an exact search would have chosen differently. + """ + most_probable = [most_probable_dosage(posterior) for posterior in posteriors] + dosages = correct_dosages_to_ploidy(most_probable, posteriors, ploidy) + lowest_posterior = None + if dosages is not None: + if sum(dosage for dosage in most_probable if dosage is not None) != ploidy: + tallies["genotypes_corrected_to_ploidy"] += 1 + if best_dosages_summing_to_ploidy(posteriors, ploidy) != dosages: + tallies["corrections_where_exact_search_differs"] += 1 + lowest_posterior = min(posterior[dosage] for posterior, dosage in zip(posteriors, dosages)) + mean_dosages = ",".join(_decimal(posterior_mean_dosage(posterior), 3) for posterior in posteriors[1:]) + values = [mean_dosages or ".", "." if lowest_posterior is None else _decimal(lowest_posterior)] + return dosages, lowest_posterior, values + + +def format_record( + marker: str, + position: int, + record_id: str, + alleles: Sequence[str], + depths: Sequence[Sequence[int]], + excluded_depths: Optional[Sequence[int]], + ploidies: Sequence[int], + options: CallOptions, + two_allele_model: bool, + is_target: bool = False, +) -> Tuple[str, Counter, Optional["BiasRow"]]: + """Call every sample at one record and format its VCF line. + + alleles: the reference allele, then the alternate alleles. + depths: ``[sample][allele]`` read counts, in the order of ``alleles``. + excluded_depths: reads per sample left out of ``depths`` (written as XD), + or None for a record that uses every read. + two_allele_model: write GQ, MPP, GP, PL and INFO/PMC from the alternate + allele's posterior (requires exactly two alleles); otherwise write MPC. + + A genotype is written as missing, with its AD and DP kept, when the + sample has fewer than ``min_depth`` reads, when no single dosage is most + probable, when its confidence is below the threshold, or when too large a + share of its reads was excluded. Those thresholds do not enter PMC, + which is taken over every sample with reads. Returns the line, the + tallies for the QC report, and a ``BiasRow`` when an allelic bias was + fitted (None otherwise). + """ + result = call_record(depths, ploidies, options.settings) + tallies: Counter = Counter(records=1) + if result.has_reads and not result.converged: + tallies["records_not_converged"] += 1 + used_reads = sum(sum(sample_depths) for sample_depths in depths) + excluded_reads = sum(excluded_depths) if excluded_depths is not None else 0 + tallies["reads_used"] += used_reads + tallies["reads_excluded"] += excluded_reads + if excluded_reads and excluded_reads >= NOTABLE_EXCLUDED_READ_SHARE * (used_reads + excluded_reads): + tallies["records_with_many_excluded_reads"] += 1 + + # What follows GT, AD, DP and XD in a sample's column, and the confidence a genotype needs. + if two_allele_model: + value_keys = ["GQ", "MPP", "DS", "GP", "PL"] + confidence_threshold = options.min_genotype_quality + else: + value_keys = ["DS", "MPC"] + confidence_threshold = options.min_marginal_posterior + sample_columns = [] + samples_called = 0 + genotype_posteriors = [] # of the samples with reads at a two-allele record, for PMC + for sample, (sample_depths, ploidy) in enumerate(zip(depths, ploidies)): + depth = sum(sample_depths) + excluded = excluded_depths[sample] if excluded_depths is not None else 0 + dosages = None + if not result.has_reads or depth == 0: + missing_reason = "no_reads" + values = ["."] * len(value_keys) + else: + if two_allele_model: + dosages, confidence, values = _two_allele_genotype( + result.posteriors[sample], result.log_likelihoods[sample], ploidy + ) + genotype_posteriors.append(result.posteriors[sample][1]) + else: + dosages, confidence, values = _multiallelic_genotype(result.posteriors[sample], ploidy, tallies) + if depth < options.min_depth: + missing_reason = "low_depth" + elif dosages is None: + missing_reason = "no_single_best_dosage" + elif confidence < confidence_threshold: + missing_reason = "low_confidence" + elif options.max_other_fraction is not None and excluded > options.max_other_fraction * (depth + excluded): + missing_reason = "excluded_read_share" + else: + missing_reason = None + tallies["genotypes"] += 1 + if missing_reason is None: + samples_called += 1 + tallies["genotypes_called"] += 1 + else: + dosages = None + tallies[f"genotypes_missing_{missing_reason}"] += 1 + column = [genotype_text(dosages, ploidy), ",".join(str(count) for count in sample_depths), str(depth)] + if excluded_depths is not None: + column.append(str(excluded)) + sample_columns.append(":".join(column + values)) + + info = ["TARGET"] if is_target else [] + info.append(f"NS={samples_called}") + if result.has_reads and len(alleles) > 1: + info.append("AF=" + ",".join(_decimal(frequency) for frequency in result.allele_frequencies[1:])) + if genotype_posteriors: + info.append("PMC=" + _decimal(proportion_misgenotyped(genotype_posteriors))) + bias_row = None + if result.bias_fit is not None: + # The fitted bias, whether or not the genotypes were called with it. Written for the + # reference allele, which is what updog reports when it is given reference read + # counts; the search itself works on the alternate allele. + fit = result.bias_fit + reference_bias = math.exp(-fit.log_bias) + info.append("BIAS=" + _decimal(reference_bias, 3)) + if not fit.used: + info.append("BIASUNUSED") + bias_row = BiasRow( + position, record_id, reference_bias, fit.objective, fit.gain, fit.rejection, + math.exp(-fit.nearest_log_start), fit.fits, result.converged, fit.at_limit, + ) + format_keys = ["GT", "AD", "DP"] + (["XD"] if excluded_depths is not None else []) + value_keys + line = "\t".join( + [marker, str(position), record_id, alleles[0], ",".join(alleles[1:]) or ".", ".", ".", ";".join(info), + ":".join(format_keys)] + + sample_columns + ) + return line, tallies, bias_row + + +class BiasRow(NamedTuple): + """One line of the bias table: a record's fitted allelic bias and how the fit went. + + ``bias`` and ``best_start`` are for the reference allele, as in the VCF's + BIAS field. ``objective`` is the log-likelihood of the record's read + counts at the fitted bias plus the log of the prior on the bias, ``gain`` + what the fitted bias added to it over no bias, and ``rejection`` why the + genotypes were called without the bias, or None when they were called + with it (see ``BiasFit``). + """ + + position: int + record_id: str + bias: float + objective: float + gain: float + rejection: Optional[str] + best_start: float + fits: int + converged: bool + at_limit: bool + + +@dataclasses.dataclass +class MarkerOutput: + """What ``process_marker`` produced for one marker. + + ``lines`` holds the VCF lines per VCF type, ``tallies`` the QC counts + keyed by ``(vcf type or "markers", name)``, and ``haplotype_alleles`` one + ``(allele index, MADC allele IDs, sequence, total reads)`` entry per + allele of the haplotype record. ``bias_rows`` has one entry per record + with a fitted allelic bias. ``tag_length`` is 0 for a marker without a + Ref tag, which gets no record. + """ + + clone_id: str + tag_length: int = 0 + lines: Dict[str, List[str]] = dataclasses.field(default_factory=dict) + tallies: Counter = dataclasses.field(default_factory=Counter) + haplotype_alleles: List[Tuple[int, str, str, int]] = dataclasses.field(default_factory=list) + bias_rows: List[BiasRow] = dataclasses.field(default_factory=list) + + +def _first_sequence(rows: Sequence[MadcRow], category: str) -> str: + return next((row.sequence for row in rows if row.category == category and row.sequence), "") + + +def process_marker(task: MarkerTask, ploidies: Sequence[int], options: CallOptions) -> MarkerOutput: + """Call one marker and format its records for every requested VCF type.""" + output = MarkerOutput(task.clone_id, lines={vcf_type: [] for vcf_type in options.vcf_types}) + sample_count = len(ploidies) + rows = [row for row in task.rows if row.sequence] + ref_tag = task.design_ref_tag or _first_sequence(rows, "Ref") + alt_tag = task.design_alt_tag or _first_sequence(rows, "Alt") + if not ref_tag: + output.tallies[("markers", "skipped_no_ref_row")] += 1 + return output + output.tag_length = len(ref_tag) + counts_by_row, samples_cleared = _without_negative_cells(rows) + output.tallies[("markers", "sample_cells_cleared_for_negative_counts")] += samples_cleared + + def add(vcf_type: str, line: str, tallies: Counter) -> None: + output.lines[vcf_type].append(line) + for name, count in tallies.items(): + output.tallies[(vcf_type, name)] += count + + site_types = [vcf_type for vcf_type in (VCF_TARGET, VCF_ALL) if vcf_type in options.vcf_types] + target = None + if site_types: + target, skip_reason = resolve_target(ref_tag, alt_tag, task.declared_target_position, options.gap_band) + if target is None: + for vcf_type in site_types: + output.tallies[(vcf_type, f"targets_skipped_{skip_reason}")] += 1 + else: + depths, other_depths = target_allele_depths(rows, counts_by_row, sample_count) + line, tallies, bias_row = format_record( + task.clone_id, target.position, task.clone_id, [target.ref_allele, target.alt_allele], + depths, other_depths, ploidies, options, two_allele_model=True, is_target=True, + ) + for vcf_type in site_types: + add(vcf_type, line, tallies) + if bias_row is not None: + output.bias_rows.append(bias_row) + + if VCF_ALL in options.vcf_types: + for line, tallies, bias_row in _off_target_records( + task.clone_id, rows, counts_by_row, ref_tag, alt_tag, target, ploidies, options + ): + add(VCF_ALL, line, tallies) + if bias_row is not None: + output.bias_rows.append(bias_row) + output.bias_rows.sort(key=lambda row: row.position) + # The target was added first; put the marker's records in position order. + output.lines[VCF_ALL].sort(key=lambda record: int(record.split("\t", 2)[1])) + + if VCF_HAPLOTYPE in options.vcf_types: + line, tallies, output.haplotype_alleles = _haplotype_record( + task.clone_id, rows, counts_by_row, ref_tag, ploidies, options + ) + add(VCF_HAPLOTYPE, line, tallies) + return output + + +def _off_target_records( + clone_id: str, + rows: Sequence[MadcRow], + counts_by_row: Sequence[Sequence[int]], + ref_tag: str, + alt_tag: str, + target: Optional[TargetVariant], + ploidies: Sequence[int], + options: CallOptions, +): + """Yield ``format_record``'s result for each substitution site away from the target. + + A site is a Ref-tag position where at least one row with reads shows a + base other than the Ref tag's. Reads are grouped by the base their row + shows there; reads of rows that do not show the position are excluded. + Insertions and deletions are not written as records. + """ + sample_count = len(ploidies) + # variant_events needs a Locus but reads only its two tags. + locus = Locus( + index=0, marker=clone_id, left_primer="", right_primer="", ref_base="", alt_base="", variant_pos=-1, + ref_tag=ref_tag, alt_tag=alt_tag or ref_tag, + ) + readings = [ + RowReading(ref_tag, row.sequence, variant_events(locus, row.sequence, options.gap_band)) for row in rows + ] + rows_with_reads = [index for index, counts in enumerate(counts_by_row) if any(counts)] + target_is_substitution = target is not None and len(target.ref_allele) == len(target.alt_allele) == 1 + site_positions = sorted( + { + position + for index in rows_with_reads + if readings[index].comparable + for position, base in readings[index].substitutions.items() + if base in CONCRETE_BASES + and ref_tag[position - 1] in CONCRETE_BASES + and not (target_is_substitution and position == target.position) + } + ) + for position in site_positions: + rows_by_base: Dict[str, List[int]] = {} + rows_not_showing = [] + for index in rows_with_reads: + base = readings[index].base_at(position) + if base is None: + rows_not_showing.append(index) + else: + rows_by_base.setdefault(base, []).append(index) + depth_by_base = { + base: _summed_counts(counts_by_row, indices, sample_count) for base, indices in rows_by_base.items() + } + reference_base = ref_tag[position - 1] + depth_by_base.setdefault(reference_base, [0] * sample_count) + # Alternate bases with the most reads first, then alphabetically. + alternate_bases = sorted( + (base for base in depth_by_base if base != reference_base), + key=lambda base: (-sum(depth_by_base[base]), base), + ) + alleles = [reference_base] + alternate_bases + yield format_record( + clone_id, position, f"{clone_id}_{position}", alleles, + [[depth_by_base[base][sample] for base in alleles] for sample in range(sample_count)], + _summed_counts(counts_by_row, rows_not_showing, sample_count), + ploidies, options, two_allele_model=len(alleles) == 2, + ) + + +def _haplotype_record( + clone_id: str, + rows: Sequence[MadcRow], + counts_by_row: Sequence[Sequence[int]], + ref_tag: str, + ploidies: Sequence[int], + options: CallOptions, +) -> Tuple[str, Counter, List[Tuple[int, str, str, int]]]: + """The marker's haplotype record: the Ref tag, then every other tag sequence with reads. + + Rows with the same sequence are added together. Alternate alleles keep + the order of their first row in the MADC. + """ + sample_count = len(ploidies) + reference_sequence = vcf_reference_sequence(ref_tag) + rows_by_sequence: Dict[str, List[int]] = {reference_sequence: []} + for index, row in enumerate(rows): + sequence = reference_sequence if row.sequence == ref_tag else row.sequence + rows_by_sequence.setdefault(sequence, []).append(index) + depth_by_sequence = { + sequence: _summed_counts(counts_by_row, indices, sample_count) for sequence, indices in rows_by_sequence.items() + } + alleles = [reference_sequence] + [ + sequence for sequence in rows_by_sequence if sequence != reference_sequence and any(depth_by_sequence[sequence]) + ] + # Haplotype records are called without an allelic bias, including those that happen to + # have two alleles, so that every record of the file is called the same way. + without_bias = dataclasses.replace(options, settings=options.settings.without_bias()) + line, tallies, _ = format_record( + clone_id, 1, clone_id, alleles, + [[depth_by_sequence[sequence][sample] for sequence in alleles] for sample in range(sample_count)], + None, ploidies, without_bias, two_allele_model=False, + ) + allele_table = [ + ( + allele_index, + ";".join(rows[index].allele_id for index in rows_by_sequence[sequence]) or ".", + sequence, + sum(depth_by_sequence[sequence]), + ) + for allele_index, sequence in enumerate(alleles) + ] + return line, tallies, allele_table + + +def model_description( + settings: ModelSettings, overdispersion_estimated: bool, background_rate_estimated: bool +) -> str: + """One sentence stating the genotype model and its inheritance assumption, for the VCF header.""" + source = ", estimated from the read counts" if overdispersion_estimated else "" + background_source = ", estimated from the read counts" if background_rate_estimated else "" + if settings.prior == PRIOR_HWE: + mating = ( + f"a selfing rate of {settings.selfing_rate:g}" if settings.selfing_rate > 0 else "random mating" + ) + prior = ( + "Hardy-Weinberg prior with one allele frequency per record estimated over all samples together; " + f"it assumes {mating} and that each chromosome copy is drawn independently " + "(polysomic inheritance in polyploids)" + ) + elif settings.prior == PRIOR_STRUCTURE: + neighbour_count = len(settings.neighbours[0]) if settings.neighbours else 0 + if neighbour_count: + prior = ( + "Hardy-Weinberg prior at each sample's own allele frequency, which is the mean over the " + f"{neighbour_count} samples most like it across the table's markers and the record's frequency " + "counted as one more sample; it assumes random mating among like samples and that each chromosome " + "copy is drawn independently (polysomic inheritance in polyploids)" + ) + else: + prior = ( + "Hardy-Weinberg prior with one allele frequency per record estimated over all samples together " + "(the structure prior was asked for and found no like samples to use); it assumes random mating " + "and that each chromosome copy is drawn independently (polysomic inheritance in polyploids)" + ) + elif settings.prior == PRIOR_NORM: + prior = ( + "normal-shaped prior over dosages (updog's norm; Gerard and Ferrao 2020), with a mean and standard " + f"deviation fitted per record and ploidy; a ploidy with fewer than {FITTED_PRIOR_MIN_SAMPLES} samples " + "uses Hardy-Weinberg proportions" + ) + else: + prior = "uniform prior over dosages" + if settings.fit_allele_bias: + prior += ( + "; allelic bias fitted for each record with two alleles (updog's bias; Gerard et al. 2018), with a " + f"normal prior of standard deviation {BIAS_LOG_PRIOR_SD:g} on its log" + ) + if settings.min_bias_gain > 0: + prior += f", and used where it raises the log-likelihood by {settings.min_bias_gain:g} or more" + if settings.reject_bias_at_limit: + prior += " and does not end at a limit of the search" + without_bias = "" + if settings.read_model_without_bias is not None: + overdispersion, background_rate = settings.read_model_without_bias + without_bias = ( + f"; for records called without a bias, overdispersion {overdispersion:g} and background-read rate " + f"{background_rate:g}" + ) + return ( + f"beta-binomial read-count likelihood (overdispersion {settings.overdispersion:g}{source}, contamination rate " + f"{settings.contamination_rate:g}, background-read rate {settings.background_rate:g}{background_source}" + f"{without_bias}) adapted from polyRAD; {prior}" + ) + + +def vcf_header( + vcf_type: str, sample_names: Sequence[str], contigs: Sequence[Tuple[str, int]], options: CallOptions +) -> List[str]: + """Header lines of one VCF. It carries no date, so the same input gives the same file.""" + multiallelic_scope = "this record's alleles" if vcf_type == VCF_HAPLOTYPE else "records with more than two alleles" + other_fraction = "none" if options.max_other_fraction is None else format(options.max_other_fraction, "g") + lines = [ + "##fileformat=VCFv4.3", + f"##source=MADCATS call v{__version__}", + f"##madcatsVcfType={vcf_type}", + "##madcatsCoordinates=marker-local: CHROM is a marker (CloneID) and POS is a 1-based position on its " + "Ref tag. These are not genome coordinates.", + "##madcatsModel=" + + model_description(options.settings, options.overdispersion_estimated, options.background_rate_estimated), + f"##madcatsFilters=min_dp={options.min_depth}, min_gq={options.min_genotype_quality}, " + f"min_marginal_posterior={options.min_marginal_posterior:g}, " + f"max_other_fraction={other_fraction}" + "; a genotype that fails is written as missing with its AD and DP kept", + ] + lines += [f"##contig=" for marker, length in contigs] + if vcf_type != VCF_HAPLOTYPE: + lines.append('##INFO=') + if vcf_type != VCF_HAPLOTYPE and options.settings.fit_allele_bias: + lines.append( + '##INFO=' + ) + if options.settings.min_bias_gain > 0 or options.settings.reject_bias_at_limit: + lines.append( + '##INFO=' + ) + lines += [ + '##INFO=', + '##INFO=', + ] + if vcf_type != VCF_HAPLOTYPE: + lines.append( + '##INFO=" + ) + lines += [ + '##FORMAT=', + '##FORMAT=', + '##FORMAT=', + ] + if vcf_type != VCF_HAPLOTYPE: + lines += [ + '##FORMAT=', + '##FORMAT=', + '##FORMAT=", + ] + lines.append( + '##FORMAT=' + ) + if vcf_type != VCF_HAPLOTYPE: + lines += [ + '##FORMAT=', + '##FORMAT=', + ] + if vcf_type != VCF_TARGET: + lines.append( + '##FORMAT=' + ) + columns = ["#CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO", "FORMAT"] + lines.append("\t".join(columns + list(sample_names))) + return lines + + +# --------------------------------------------------------------------------- +# Command line +# --------------------------------------------------------------------------- + + +def _fraction(value: str) -> float: + parsed = float(value) + if not 0 <= parsed <= 1: + raise argparse.ArgumentTypeError("must be between 0 and 1") + return parsed + + +def _positive_float(value: str) -> float: + parsed = float(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("must be greater than 0") + return parsed + + +AUTO = "auto" + + +def _number_or_auto(parse): + """An argparse type that accepts ``auto`` or whatever ``parse`` accepts.""" + def parse_value(value: str): + return AUTO if value.strip().lower() == AUTO else parse(value) + + return parse_value + + +def target_depths_by_marker(tasks: Sequence[MarkerTask], sample_count: int) -> List[List[List[int]]]: + """Reference-type and alternate-type read counts of every marker that has an alternate row. + + Returns ``[marker][sample][allele]``. These are the counts the target + records are called from; the read-model estimate and the search for like + samples use them because every marker has them, whatever VCF types were + asked for. + """ + depths_by_marker = [] + for task in tasks: + rows = [row for row in task.rows if row.sequence] + if not any(row.category in ALTERNATE_CATEGORIES for row in rows): + continue + counts_by_row, _ = _without_negative_cells(rows) + depths_by_marker.append(target_allele_depths(rows, counts_by_row, sample_count)[0]) + return depths_by_marker + + +@dataclasses.dataclass +class ReadModelChoice: + """The overdispersion and background-read rate a run uses, and where each came from. + + ``overdispersion_estimated`` and ``background_rate_estimated`` are False + when the value was given on the command line, and also when the estimate + could not be used and the fallback value took its place. The two reports + go into the QC JSON. + + ``without_bias`` is set when the settings can leave a record without the + bias fitted to it and the read model is estimated: the read model for + such records, estimated without a bias. The values of this object are + then for the records called with their bias. + """ + + overdispersion: float + background_rate: float + overdispersion_estimated: bool + background_rate_estimated: bool + overdispersion_report: Dict[str, object] + background_report: Dict[str, object] + without_bias: Optional["ReadModelChoice"] = None + + +def _read_model_from_estimate( + estimate: ReadModelEstimate, requested_overdispersion, requested_background_rate, used_for: Optional[str] = "" +) -> ReadModelChoice: + """The read model a run takes from an estimate, with its reports for the QC JSON. + + A value given on the command line is kept. A value that could not be + estimated is replaced by its fallback: both when the table is too small, + and the overdispersion also when its estimate is the lowest value tried. + What is used is said on stderr, followed by ``used_for`` when the run has + two read models, and nothing is said when ``used_for`` is None. + """ + def say(message: str) -> None: + if used_for is not None: + print(message + (f", {used_for}" if used_for else ""), file=sys.stderr) + + overdispersion_report: Dict[str, object] = {"requested": requested_overdispersion} + background_report: Dict[str, object] = {"requested": requested_background_rate} + overdispersion_estimate = estimate.overdispersion_estimate + too_small = overdispersion_estimate.value is None + + background_rate = requested_background_rate + background_rate_estimated = requested_background_rate == AUTO and not too_small + if requested_background_rate == AUTO: + background_report["estimate"] = estimate.background_rate if background_rate_estimated else None + background_report["by_rate"] = ( + { + format(rate, "g"): {"overdispersion": value, "log_likelihood": round(total, 3)} + for rate, (value, total) in estimate.by_background_rate.items() + } + if background_rate_estimated + else {} + ) + background_rate = estimate.background_rate if background_rate_estimated else FALLBACK_BACKGROUND_RATE + at_end = background_rate_estimated and background_rate in ( + min(BACKGROUND_RATE_GRID), max(BACKGROUND_RATE_GRID) + ) + say( + f"{'Estimated' if background_rate_estimated else 'Using'} background-read rate: {background_rate:g}" + + (" (the end of the range tried)" if at_end else "") + ) + background_report["used"] = background_rate + + overdispersion = requested_overdispersion + overdispersion_estimated = False + if requested_overdispersion == AUTO: + overdispersion_report.update( + estimate=overdispersion_estimate.value, + markers_used=overdispersion_estimate.markers_used, + cells_with_reads=overdispersion_estimate.cells_with_reads, + values_by_round=overdispersion_estimate.values_by_round, + at_limit=overdispersion_estimate.at_limit, + log_likelihood_by_value={ + format(value, "g"): round(total, 3) + for value, total in overdispersion_estimate.log_likelihood_by_value.items() + }, + ) + if too_small: + overdispersion = FALLBACK_OVERDISPERSION + say(f"Using overdispersion: {overdispersion:g}") + elif overdispersion_estimate.at_limit == "lowest": + overdispersion = FALLBACK_OVERDISPERSION + say( + f"warning: the overdispersion estimate is the lowest value tried ({overdispersion_estimate.value:g}), " + "as it is when no sample is heterozygous (inbred lines) and the reads show no scatter to measure. " + f"Using {overdispersion:g}; set --overdispersion to a number to choose it yourself." + ) + else: + overdispersion = float(overdispersion_estimate.value) + overdispersion_estimated = True + limit_note = "" + if overdispersion_estimate.at_limit == "highest": + limit_note = ( + "; this is the highest value tried, so the counts scatter no more than random sampling of reads" + ) + say( + f"Estimated overdispersion: {overdispersion:g} " + f"(from {overdispersion_estimate.markers_used:,} markers{limit_note})" + ) + overdispersion_report["used"] = overdispersion + return ReadModelChoice( + overdispersion, background_rate, overdispersion_estimated, background_rate_estimated, overdispersion_report, + background_report, + ) + + +def choose_read_model( + depths_by_marker: Sequence[Sequence[Sequence[int]]], + ploidies: Sequence[int], + settings: ModelSettings, + requested_overdispersion, + requested_background_rate, + parallel_map=map, +) -> ReadModelChoice: + """Settle the overdispersion and the background-read rate of a run. + + ``requested_overdispersion`` and ``requested_background_rate`` are + numbers or ``auto``. Whatever is ``auto`` is estimated from each + marker's reference-type and alternate-type read counts, + ``depths_by_marker`` (``estimate_read_model``). + + The estimate is made with the free-weights prior whatever prior the run + uses, because the estimate is only as good as the genotype proportions + it assumes: with Hardy-Weinberg proportions it was badly off for + samples from two subpopulations, and with those or the ``norm`` prior, + neither of which can describe an excess of homozygotes, it failed for + inbred samples. An overdispersion that cannot be estimated falls back + to ``FALLBACK_OVERDISPERSION``: when the table is too small, and when the + best value is the lowest one tried, which is what a table without + heterozygous samples and without stray reads gives, since it has no + scatter to measure. + + With an allelic bias there are two estimates. The first is made without + a bias, as for a run that fits none. The second fits every record's bias + and keeps it, and its search starts from the first: started from + polyRAD's value it stopped at an overdispersion of 10 on a simulated + hexaploid table whose counts are far more probable at 20, with or without + a bias, because a bias and dosage weights fitted at one overdispersion + make that value look best. The second estimate is the read model of the + records called with their bias. Where the settings can leave a record + without its bias (``min_bias_gain``, ``reject_bias_at_limit``), the + first is the read model of the records called without one, so that such + a record is called exactly as a run without a bias calls it + (docs/DEVELOPMENT.md). + """ + if AUTO not in (requested_overdispersion, requested_background_rate): + return ReadModelChoice( + requested_overdispersion, requested_background_rate, False, False, + {"requested": requested_overdispersion, "used": requested_overdispersion}, + {"requested": requested_background_rate, "used": requested_background_rate}, + ) + + background_rates = BACKGROUND_RATE_GRID if requested_background_rate == AUTO else (requested_background_rate,) + overdispersions = OVERDISPERSION_GRID if requested_overdispersion == AUTO else (requested_overdispersion,) + estimate_settings = dataclasses.replace( + settings, prior=PRIOR_FREE, selfing_rate=0.0, min_bias_gain=0.0, reject_bias_at_limit=False, + read_model_without_bias=None, + ) + estimate_without_bias = estimate_read_model( + depths_by_marker, ploidies, dataclasses.replace(estimate_settings, fit_allele_bias=False), background_rates, + overdispersions, parallel_map, + ) + baseline = estimate_without_bias.overdispersion_estimate + if baseline.value is None: + print( + f"note: too few read counts to estimate the read model ({baseline.cells_with_reads:,} " + f"sample-by-marker cells with reads; {OVERDISPERSION_MIN_CELLS} needed).", + file=sys.stderr, + ) + if not settings.fit_allele_bias: + return _read_model_from_estimate(estimate_without_bias, requested_overdispersion, requested_background_rate) + + some_records_without_bias = settings.min_bias_gain > 0 or settings.reject_bias_at_limit + without_bias = _read_model_from_estimate( + estimate_without_bias, requested_overdispersion, requested_background_rate, + used_for="for records called without a bias" if some_records_without_bias else None, + ) + starts = {} + if baseline.value is not None: + starts = dict( + start_background_rate=estimate_without_bias.background_rate, start_overdispersion=baseline.value + ) + with_bias = _read_model_from_estimate( + estimate_read_model( + depths_by_marker, ploidies, estimate_settings, background_rates, overdispersions, parallel_map, **starts + ), + requested_overdispersion, requested_background_rate, + used_for="for records called with their bias" if some_records_without_bias else "", + ) + # The estimate without a bias is reported beside the one with it; its "used" is null when + # no record is called with it. + for report, baseline_report in ( + (with_bias.overdispersion_report, without_bias.overdispersion_report), + (with_bias.background_report, without_bias.background_report), + ): + if report["requested"] == AUTO: + report["without_bias"] = { + key: value if key != "used" or some_records_without_bias else None + for key, value in baseline_report.items() if key != "requested" + } + if some_records_without_bias: + with_bias.without_bias = without_bias + return with_bias + + +def _background_rate(value: str) -> float: + parsed = float(value) + if not 0 <= parsed < 0.5: + raise argparse.ArgumentTypeError("must be at least 0 and below 0.5") + return parsed + + +def write_vcfs( + prefix: str, + vcf_types: Sequence[str], + sample_names: Sequence[str], + contigs: Sequence[Tuple[str, int]], + options: CallOptions, + outputs: Sequence[MarkerOutput], +) -> Dict[str, Path]: + """Write one VCF per requested type; return each type's path.""" + written = {} + for vcf_type in vcf_types: + vcf_path = Path(prefix + VCF_FILE_SUFFIX[vcf_type]) + with vcf_path.open("w") as handle: + for line in vcf_header(vcf_type, sample_names, contigs, options): + handle.write(line + "\n") + for output in outputs: + for line in output.lines[vcf_type]: + handle.write(line + "\n") + written[vcf_type] = vcf_path + return written + + +def write_haplotype_allele_table(prefix: str, outputs: Sequence[MarkerOutput]) -> Path: + """Write the table that ties each allele number of the haplotype VCF to the MADC rows it was counted from.""" + alleles_path = Path(prefix + ".haplotypes.alleles.csv") + with alleles_path.open("w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(["CloneID", "AlleleIndex", "AlleleIDs", "AlleleSequence", "TotalReads"]) + for output in outputs: + for allele_index, allele_ids, sequence, total_reads in output.haplotype_alleles: + writer.writerow([output.clone_id, allele_index, allele_ids, sequence, total_reads]) + return alleles_path + + +def write_bias_table(prefix: str, outputs: Sequence[MarkerOutput]) -> Tuple[Optional[Path], Dict[str, object]]: + """Write the fitted allelic biases, one row per record; return the path and a summary for the QC JSON. + + Nothing is written, and the summary is empty, when no bias was fitted. + The summary counts the records called without their fitted bias by + reason, and names those whose bias was rejected for ending at a limit of + the search. + """ + bias_rows = [(output.clone_id, row) for output in outputs for row in output.bias_rows] + if not bias_rows: + return None, {} + bias_path = Path(prefix + ".bias.csv") + with bias_path.open("w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow( + ["CloneID", "Position", "RecordID", "Bias", "LogLikelihood", "Gain", "Used", "Rejected", "BestStart", + "Fits", "Converged", "AtLimit"] + ) + for clone_id, row in bias_rows: + writer.writerow( + [clone_id, row.position, row.record_id, _decimal(row.bias), _decimal(row.objective, 3), + _decimal(row.gain, 3), int(row.rejection is None), row.rejection or "", _decimal(row.best_start), + row.fits, int(row.converged), int(row.at_limit)] + ) + biases = sorted(row.bias for _, row in bias_rows) + summary = { + "records_fitted": len(biases), + "records_called_with_the_bias": sum(1 for _, row in bias_rows if row.rejection is None), + "records_called_without_it": { + reason: sum(1 for _, row in bias_rows if row.rejection == reason) + for reason in (BIAS_REJECTED_FOR_GAIN, BIAS_REJECTED_AT_LIMIT) + }, + "rejected_at_search_limit": [ + f"{clone_id}:{row.position}" for clone_id, row in bias_rows if row.rejection == BIAS_REJECTED_AT_LIMIT + ], + "median": round(biases[len(biases) // 2], 4), + "outside_0.8_to_1.25": sum(1 for bias in biases if not 0.8 <= bias <= 1.25), + "outside_0.67_to_1.5": sum(1 for bias in biases if not 2 / 3 <= bias <= 1.5), + "at_search_limit": sum(1 for _, row in bias_rows if row.at_limit), + "not_converged": sum(1 for _, row in bias_rows if not row.converged), + } + return bias_path, summary + + +def _warn_about_biases_at_limit(bias_summary: Dict[str, object]) -> None: + """Say on stderr when fitted biases ended at a limit of the search (``write_bias_table``'s summary).""" + fitted = bias_summary.get("records_fitted", 0) + at_limit = bias_summary.get("at_search_limit", 0) + if not at_limit: + return + rejected = bias_summary["records_called_without_it"][BIAS_REJECTED_AT_LIMIT] + not_used = ( + f" The {rejected:,} that gained enough to be used were not used: those records were called without a " + "bias, are marked in the bias table and are listed in the QC JSON." + if rejected else "" + ) + if at_limit >= BIAS_AT_LIMIT_WARNING_SHARE * fitted: + print( + f"warning: {at_limit:,} of {fitted:,} fitted biases ended at a limit of the search.{not_used} Samples " + "with few heterozygotes give this, as inbred lines do, and a bias has nothing to correct in them: call " + "them with --allele-bias off, and with --selfing-rate if they are inbred", + file=sys.stderr, + ) + elif rejected: + print( + f"warning: the fitted bias of {rejected:,} of {fitted:,} records ended at a limit of the search and " + "was not used; those records were called without a bias. They are marked in the bias table and " + "listed in the QC JSON; --allele-bias fit uses such a bias", + file=sys.stderr, + ) + + +def run_call(args: argparse.Namespace) -> int: + """Call genotypes from a MADC and write the requested VCFs (see the module docstring).""" + madc = Path(args.madc) + if not madc.is_file(): + raise MadcError(f"MADC not found: {madc}") + started = time.time() + vcf_types = tuple(vcf_type for vcf_type in VCF_TYPES if vcf_type in args.vcf) + if args.selfing_rate > 0 and args.prior != PRIOR_HWE: + raise MadcError("--selfing-rate applies to --prior hwe only") + if args.allele_bias != "off" and args.prior == PRIOR_STRUCTURE: + raise MadcError(f"--allele-bias {args.allele_bias} cannot be combined with --prior structure") + + sample_names, rows_by_marker, count_overflow = read_count_table(madc) + ploidy_by_sample = read_ploidy_table(Path(args.ploidy_file)) if args.ploidy_file else {} + ploidies = [ploidy_by_sample.get(name, args.ploidy) for name in sample_names] + samples_not_in_madc = sorted(set(ploidy_by_sample) - set(sample_names)) + if samples_not_in_madc: + print( + f"note: {len(samples_not_in_madc):,} samples in {args.ploidy_file} are not in the MADC " + f"(first: {samples_not_in_madc[0]})", + file=sys.stderr, + ) + + if 0 < args.selfing_rate < 1 and any(ploidy % 2 for ploidy in ploidies): + print( + "note: --selfing-rate is not applied to samples of odd ploidy; they keep random-mating proportions", + file=sys.stderr, + ) + + small_ploidy_groups = sorted( + ploidy for ploidy, count in Counter(ploidies).items() if count < FITTED_PRIOR_MIN_SAMPLES + ) + if args.prior == PRIOR_NORM and small_ploidy_groups: + print( + f"note: fewer than {FITTED_PRIOR_MIN_SAMPLES} samples have ploidy " + f"{', '.join(str(ploidy) for ploidy in small_ploidy_groups)}; --prior norm uses Hardy-Weinberg " + "proportions for them", + file=sys.stderr, + ) + + designs = {} + if args.reference_tags: + designs = {locus.marker: locus for locus in load_reference_tags(Path(args.reference_tags))} + tasks = [] + for clone_id, rows in rows_by_marker.items(): + design = designs.get(clone_id) + if design is None: + tasks.append(MarkerTask(clone_id, rows)) + else: + tasks.append(MarkerTask(clone_id, rows, design.ref_tag, design.alt_tag, design.variant_pos + 1)) + + model = ModelSettings( + contamination_rate=args.contamination_rate, + prior=args.prior, + selfing_rate=args.selfing_rate, + fit_allele_bias=args.allele_bias != "off", + min_bias_gain=BIAS_AUTO_MIN_GAIN if args.allele_bias == "auto" else 0.0, + reject_bias_at_limit=args.allele_bias == "auto", + tolerance=args.tol, + max_iterations=args.max_iterations, + ) + # One pool of worker processes serves the read-model estimate, the search for like + # samples and the calling. + executor = concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) if args.jobs > 1 else None + + def parallel_map(function, items): + if executor is None or len(items) < 2: + return list(map(function, items)) + return list(executor.map(function, items, chunksize=max(1, len(items) // (args.jobs * 8)))) + + try: + # The read-model estimate and the search for like samples both work on these counts. + needs_target_depths = AUTO in (args.overdispersion, args.background_rate) or args.prior == PRIOR_STRUCTURE + target_depths = target_depths_by_marker(tasks, len(ploidies)) if needs_target_depths else [] + read_model = choose_read_model( + target_depths, ploidies, model, args.overdispersion, args.background_rate, parallel_map + ) + settings = dataclasses.replace( + model, overdispersion=read_model.overdispersion, background_rate=read_model.background_rate, + read_model_without_bias=( + (read_model.without_bias.overdispersion, read_model.without_bias.background_rate) + if read_model.without_bias is not None else None + ), + ) + structure_report: Dict[str, object] = {} + if args.prior == PRIOR_STRUCTURE: + choice = find_neighbours(target_depths, ploidies, settings, parallel_map) + settings = dataclasses.replace(settings, neighbours=choice.neighbours) + structure_report = { + "neighbours_per_sample": choice.neighbours_per_sample, + "markers_compared": choice.markers_used, + "prediction_error_by_neighbours": { + str(count): round(error, 6) for count, error in choice.error_by_count.items() + }, + "reason_none_used": choice.reason, + } + if choice.neighbours_per_sample: + print( + f"Structure prior: each sample's allele frequencies are taken from the " + f"{choice.neighbours_per_sample} samples most like it " + f"(compared on {choice.markers_used:,} markers)", + file=sys.stderr, + ) + else: + print( + f"note: --prior structure uses Hardy-Weinberg proportions for the whole table: {choice.reason}", + file=sys.stderr, + ) + del target_depths + options = CallOptions( + settings=settings, + vcf_types=vcf_types, + min_depth=args.min_dp, + min_genotype_quality=args.min_gq, + min_marginal_posterior=args.min_marginal_posterior, + max_other_fraction=args.max_other_fraction, + gap_band=args.max_indel_length, + overdispersion_estimated=read_model.overdispersion_estimated, + background_rate_estimated=read_model.background_rate_estimated, + ) + outputs = parallel_map(functools.partial(process_marker, ploidies=ploidies, options=options), tasks) + finally: + if executor is not None: + executor.shutdown() + + contigs = [(output.clone_id, output.tag_length) for output in outputs if output.tag_length] + tallies: Counter = Counter() + for output in outputs: + tallies.update(output.tallies) + + prefix = str(args.output_prefix) + Path(prefix).parent.mkdir(parents=True, exist_ok=True) + written = write_vcfs(prefix, vcf_types, sample_names, contigs, options, outputs) + alleles_path = write_haplotype_allele_table(prefix, outputs) if VCF_HAPLOTYPE in vcf_types else None + bias_path, bias_summary = write_bias_table(prefix, outputs) + + per_type = { + vcf_type: {name: count for (group, name), count in sorted(tallies.items()) if group == vcf_type} + for vcf_type in vcf_types + } + qc_path = Path(args.qc_output) if args.qc_output else Path(prefix + ".qc.json") + qc = { + "pipeline": "MADCATS", + "command": "call", + "version": __version__, + "created_utc": dt.datetime.now(dt.timezone.utc).isoformat(), + "inputs": { + "madc": str(madc.resolve()), + "reference_tags": str(Path(args.reference_tags).resolve()) if args.reference_tags else None, + "ploidy_file": str(Path(args.ploidy_file).resolve()) if args.ploidy_file else None, + }, + "parameters": { + "vcf": list(vcf_types), + "ploidy": args.ploidy, + "prior": args.prior, + "selfing_rate": args.selfing_rate, + "allele_bias": args.allele_bias, + "overdispersion": args.overdispersion, + "contamination_rate": args.contamination_rate, + "background_rate": args.background_rate, + "tol": args.tol, + "max_iterations": args.max_iterations, + "min_dp": args.min_dp, + "min_gq": args.min_gq, + "min_marginal_posterior": args.min_marginal_posterior, + "max_other_fraction": args.max_other_fraction, + "max_indel_length": args.max_indel_length, + "jobs": args.jobs, + }, + "model": model_description( + options.settings, options.overdispersion_estimated, options.background_rate_estimated + ), + "overdispersion": read_model.overdispersion_report, + "background_rate": read_model.background_report, + "structure_prior": structure_report, + "coordinates": "marker-local", + "samples": { + "count": len(sample_names), + "by_ploidy": {str(ploidy): count for ploidy, count in sorted(Counter(ploidies).items())}, + "ploidy_file_samples_not_in_madc": len(samples_not_in_madc), + "ploidies_using_hwe_under_norm_prior": small_ploidy_groups if args.prior == PRIOR_NORM else [], + }, + "markers": { + "in_madc": len(tasks), + "with_reference_tags": sum(1 for task in tasks if task.design_ref_tag), + **{name: count for (group, name), count in sorted(tallies.items()) if group == "markers"}, + }, + "count_overflow": count_overflow, + "vcfs": { + vcf_type: {"path": str(written[vcf_type]), **per_type[vcf_type]} for vcf_type in vcf_types + }, + "allele_bias": bias_summary, + "outputs": { + "haplotype_alleles": str(alleles_path) if alleles_path else None, + "bias_table": str(bias_path) if bias_path else None, + }, + "elapsed_seconds": round(time.time() - started, 1), + } + _write_json(qc_path, qc) + + for vcf_type in vcf_types: + counts = per_type[vcf_type] + genotypes = counts.get("genotypes", 0) + called = counts.get("genotypes_called", 0) + share = called / genotypes if genotypes else 0.0 + print( + f"{written[vcf_type]}: {counts.get('records', 0):,} records, {called:,} of {genotypes:,} genotypes " + f"called ({share:.1%})", + file=sys.stderr, + ) + skipped = sum(count for name, count in counts.items() if name.startswith("targets_skipped_")) + if skipped: + print( + f" {skipped:,} markers have no target record because the designed variant could not be " + "identified; see the QC JSON, and give --reference-tags if you have them", + file=sys.stderr, + ) + if counts.get("records_not_converged"): + print( + f" {counts['records_not_converged']:,} records reached --max-iterations before their allele " + "frequencies settled", + file=sys.stderr, + ) + _warn_about_biases_at_limit(bias_summary) + print(f"Wrote QC to {qc_path}", file=sys.stderr) + return 0 + + +def add_call_command(subparsers) -> None: + calling = subparsers.add_parser( + "call", + help="call genotypes from a MADC and write target, all-variant or haplotype VCFs", + description=( + "Estimate each sample's genotype from the read counts in a MADC and write one VCF for each record " + "type chosen with --vcf. Ploidy can differ between samples. The model is adapted from polyRAD " + "(beta-binomial likelihood, Hardy-Weinberg prior) and updog (allelic bias, read-error term, norm " + "prior). Coordinates are marker-local: CHROM is the marker and POS a position on its Ref tag." + ), + ) + + files = calling.add_argument_group("input and output") + files.add_argument( + "--madc", + required=True, + help=( + "MADC to call from, plain or gzipped: one written by generate or filter-paralogs, a vendor DArT " + "MADC (with or without its starred preamble) or a HapApp fixed-allele-ID MADC" + ), + ) + files.add_argument( + "--output-prefix", + required=True, + metavar="PREFIX", + help=( + "path prefix of the outputs: PREFIX.targets.vcf, PREFIX.all.vcf and PREFIX.haplotypes.vcf (with " + "PREFIX.haplotypes.alleles.csv) as chosen by --vcf, PREFIX.bias.csv when an allelic bias is " + "fitted, and PREFIX.qc.json" + ), + ) + files.add_argument( + "--vcf", + nargs="+", + choices=VCF_TYPES, + default=[VCF_TARGET], + metavar="TYPE", + help=( + "record types to write, one VCF each; one or more of target (the variant each marker was designed " + "for), all (the targets plus every other base substitution in the tags) and haplotype (whole tag " + f"sequences as alleles) (default: {VCF_TARGET})" + ), + ) + files.add_argument("--qc-output", help="QC JSON (default: PREFIX.qc.json)") + + panel = calling.add_argument_group("samples and panel") + panel.add_argument( + "--ploidy", type=positive_int, default=DEFAULT_PLOIDY, + help=f"ploidy of every sample not listed in --ploidy-file (default: {DEFAULT_PLOIDY})", + ) + panel.add_argument( + "--ploidy-file", + help="two-column table of sample name and ploidy, tab- or comma-separated, with or without a header line", + ) + panel.add_argument( + "--reference-tags", + help=( + "reference-tag CSV of the panel (from design or infer-design); its tags and " + "VariantPositionInTagSequence column identify each marker's target. Without it the MADC's Ref and " + "Alt rows are used, and a marker whose two tags differ in more than one place gets no target record" + ), + ) + panel.add_argument( + "--max-indel-length", type=nonnegative_int, default=GAP_BAND_DEFAULT, + help=( + "widest insertion or deletion considered when a row is compared with the Ref tag; use the value " + f"given to generate (default: {GAP_BAND_DEFAULT})" + ), + ) + + model = calling.add_argument_group("model") + model.add_argument( + "--prior", + choices=PRIOR_CHOICES, + default=PRIOR_HWE, + help=( + "genotype prior: uniform (every dosage equally probable), hwe (Hardy-Weinberg proportions at " + "allele frequencies estimated from all samples; assumes random mating and polysomic inheritance), " + "norm (a normal shape over the dosages, fitted per record and ploidy as in updog; does not assume " + "random mating) or structure (Hardy-Weinberg proportions at each sample's own allele frequencies, " + "estimated from the samples most like it; for subpopulations, clines and families) " + f"(default: {PRIOR_HWE})" + ), + ) + model.add_argument( + "--selfing-rate", type=_fraction, default=0.0, + help=( + "with --prior hwe, the share of individuals that come from self-fertilisation, from 0 (random " + "mating) to 1 (fully inbred lines); it moves prior probability from the intermediate dosages to " + "the two homozygous ones (default: 0)" + ), + ) + model.add_argument( + "--allele-bias", + choices=("off", "fit", "auto"), + default="off", + help=( + "allelic bias, where one allele yields more reads per copy than the other: off (none assumed), fit " + "(estimated for every record with two alleles, as in updog, and used) or auto (estimated the same " + f"way, but used only for records where it raises the log-likelihood by {BIAS_AUTO_MIN_GAIN:g} or " + "more and does not end at a limit of the search; the rest are called without it). fit and auto " + "take several times as long and cannot be combined with --prior structure (default: off)" + ), + ) + model.add_argument( + "--overdispersion", type=_number_or_auto(_positive_float), default=AUTO, + help=( + "beta-binomial overdispersion parameter, as in polyRAD: lower values allow read fractions to stray " + "further from the expected fraction. 'auto' estimates it from the read counts by maximum " + f"likelihood, and uses {FALLBACK_OVERDISPERSION:g} when it cannot; or give a number " + f"(default: {AUTO})" + ), + ) + model.add_argument( + "--background-rate", type=_number_or_auto(_background_rate), default=AUTO, + help=( + "probability that a read shows one allele although it came from another: sequencing and " + "amplification errors, index hopping and contamination together. 'auto' estimates it from the read " + f"counts along with the overdispersion, and uses {FALLBACK_BACKGROUND_RATE:g} when it cannot; or give " + f"a number (default: {AUTO})" + ), + ) + model.add_argument( + "--contamination-rate", type=proper_fraction, default=0.0, + help=( + "polyRAD's contamination term: the expected share of a sample's reads that come from other " + "samples. --background-rate already covers such reads, so this is 0 unless set; polyRAD's own " + f"default is {POLYRAD_CONTAMINATION_RATE:g} (default: 0)" + ), + ) + model.add_argument( + "--tol", type=_positive_float, default=DEFAULT_TOLERANCE, + help=( + "with --prior hwe, norm or structure, stop iterating a record once its allele frequencies (and, " + f"with norm, its prior probabilities) change by no more than this (default: {DEFAULT_TOLERANCE:g})" + ), + ) + model.add_argument( + "--max-iterations", type=positive_int, default=DEFAULT_MAX_ITERATIONS, + help=( + "with --prior hwe, norm or structure, the most iterations a record may take; with structure, in " + f"each of its two stages (default: {DEFAULT_MAX_ITERATIONS})" + ), + ) + + filters = calling.add_argument_group("filters (a genotype that fails one is written as missing)") + filters.add_argument( + "--min-dp", type=nonnegative_int, default=DEFAULT_MIN_DEPTH, + help=f"fewest reads a sample needs in a record (default: {DEFAULT_MIN_DEPTH})", + ) + filters.add_argument( + "--min-gq", type=nonnegative_int, default=DEFAULT_MIN_GENOTYPE_QUALITY, + help=f"two-allele records: lowest genotype quality, GQ (default: {DEFAULT_MIN_GENOTYPE_QUALITY})", + ) + filters.add_argument( + "--min-marginal-posterior", type=_fraction, default=DEFAULT_MIN_MARGINAL_POSTERIOR, + help=( + "haplotype records and other records with more than two alleles: lowest MPC, the smallest " + f"marginal posterior among the called dosages (default: {DEFAULT_MIN_MARGINAL_POSTERIOR:g})" + ), + ) + filters.add_argument( + "--max-other-fraction", type=_fraction, default=None, + help=( + "target and all records: largest share of the sample's reads that may be left out of the call, " + "XD (default: no limit)" + ), + ) + + performance = calling.add_argument_group("performance") + performance.add_argument( + "--jobs", type=positive_int, default=_default_jobs(), + help="worker processes; markers are called independently (default: up to 4)", + ) + calling.set_defaults(func=run_call) diff --git a/madcats_paralogs.py b/madcats_paralogs.py new file mode 100644 index 0000000..e3c68ab --- /dev/null +++ b/madcats_paralogs.py @@ -0,0 +1,1043 @@ +"""``madcats filter-paralogs``: clean a finished MADC with a reference genome. + +In panels where paralogous copies co-amplify with a target, a paralog read +that differs from the Ref tag only at non-target bases becomes a RefMatch or +AltMatch row of that marker. This module places each marker on the genome +by its Ref tag and, for CHROM_POS CloneIDs, by that position (see +``--clone-positions``); aligns every haplotype; and drops or flags the rows +whose best alignment lies at another locus (paralogs) or ties between loci +(ambiguous). + +It is kept apart from ``madcats.py`` because it is the only command with a +dependency outside the standard library: the ``mappy`` aligner (minimap2), +imported on first use (``pip install 'madcats[genome]'``), so ``madcats`` +runs without it. The MADC reading helpers it shares with ``validate`` +(header detection, sample columns, allele categories, the count-overflow +check) stay in ``madcats``. + +What is minimap2's and what is this module's. Alignment is minimap2's, +through its Python binding mappy (MIT licence; https://github.com/lh3/minimap2): + + Li H (2018) Minimap2: pairwise alignment for nucleotide sequences. + Bioinformatics 34(18):3094-3100. doi:10.1093/bioinformatics/bty191 + Li H (2021) New strategies to improve minimap2 alignment accuracy. + Bioinformatics 37(23):4572-4574. doi:10.1093/bioinformatics/btab705 + +The index (k=21, w=11) and the alignment scores are those of minimap2's +short-read preset (``-x sr``). Three of the preset's mapping options are +changed for the genome-wide search (``GENOME_WIDE_MAP_OPTIONS``). The layout +of an index file read by ``_index_part_contigs`` is that written by +minimap2's ``mm_idx_dump`` (index.c). Everything else is MADCATS's own, with +no published counterpart: the score recomputed from an alignment +(``hit_score``), the rule for when two alignments are one locus, the +realignment of every haplotype at its own marker's site with short seeds, +marker placement with the CloneID position, and the labels. The label rule +began as "a read whose best alignment score exceeds its second best", as +bwa mem reports them, in a prototype that is not part of MADCATS; here the +comparison is between the marker's own site and anywhere else. The +measurements behind the choices are in docs/DEVELOPMENT.md ("Genome paralog +filter"). + +Reading map: scoring and alignment (``hit_score``, ``GenomeAligner``, +``WindowIndex``, ``map_sequences``); marker placement and labels +(``place_marker``, ``score_at_sites``, ``genome_label``); the two passes over +the table (``_read_haplotypes``, ``_write_filtered_table``) and the function +that joins them (``filter_paralogs``); the command line +(``add_filter_paralogs_command``, ``run_filter_paralogs``). +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import csv +import datetime as dt +import re +import struct +import sys +import time +from collections import Counter, defaultdict +from pathlib import Path +from typing import Callable, Dict, Iterable, List, Mapping, NamedTuple, Optional, Sequence, Set, Tuple + +from madcats import ( + CountOverflowCheck, + MadcError, + __version__, + _default_jobs, + _write_json, + allele_type, + is_madc_header, + madc_identity_columns, + madc_read_counts, + madc_sample_start, + open_text, + positive_int, +) + +# In flag mode the label goes into the column of this vendor statistic, which is renamed. +REPLACED_STATISTICS_COLUMN = "PICRef" +GENOME_LABEL_COLUMN = "GenomeLabel" + +# The alignment scores of minimap2's short-read preset (-x sr): match, mismatch, gap open, gap +# extend, long-gap open, long-gap extend; and minimap2's general score for an ambiguous base. +# They are passed to the aligner, although they are its preset's values, so that hit_score and +# the aligner cannot come to differ. +SHORT_READ_SCORING = (2, 8, 12, 2, 24, 1, 1) +# The index of that preset: minimizers of 21 bases in windows of 11. +SHORT_READ_INDEX_K_W = (21, 11) +# Changes to the preset's mapping options for the genome-wide search (the index is unchanged): +# chains of a single seed (the preset needs 2), a lower least chain score (25) and up to 50 +# secondary hits (20), so 51 hits in all. An 81-base allele with several SNPs then still finds +# its copies. Chosen on the lettuce MADCATS MADC, where the rows kept or dropped agreed with a +# bwa mem classification for 98.2% of rows, against 97.3% with the preset's values +# (docs/DEVELOPMENT.md). The realignment at a marker's site keeps the preset's values. +GENOME_WIDE_MAP_OPTIONS = {"min_cnt": 1, "min_chain_score": 20, "best_n": 50} +# Distinct genome sites remembered per haplotype, best first. Ref tags keep all of theirs. +MAX_SITES_PER_HAPLOTYPE = 8 +# Sequences handed to a worker thread at a time. +MAPPING_BATCH = 2000 + +# A CloneID of the form CHROM_POS; the position may be zero-padded. +CLONE_POSITION_RE = re.compile(r"(.+)_0*(\d+)") +# A CloneID position up to this many bases outside a site still counts as at that site. +CLONE_POSITION_SLACK = 10 +# More than this share of CloneID positions away from their Ref tags' sites brings a warning +# that the positions may refer to another assembly. +CLONE_DISAGREEMENT_WARNING_SHARE = 0.1 + +# The realignment of a haplotype at its own marker's site: the site with this many genome bases +# either side, indexed with minimizers of 11 bases in windows of 3, so that an allele too +# diverged to seed in the genome-wide search (21 bases) still aligns where it belongs. The +# rest of the genome is searched with the longer seeds only, so a copy elsewhere that is as +# diverged as that may go unfound. +SITE_FLANK = 20 +SITE_INDEX_K, SITE_INDEX_W = 11, 3 + +# CIGAR operations as mappy numbers them. +CIGAR_INSERTION, CIGAR_DELETION = 1, 2 +# A minimap2 index file: the bytes each part starts with, and the flag of a part stored +# without its sequences (MM_I_NO_SEQ). +INDEX_PART_MAGIC = b"MMI\x02" +INDEX_WITHOUT_SEQUENCES = 2 + +# The labels sidecar's Action column holds what was done with the row: kept, flagged, +# protected or dropped. +LABELS_SIDECAR_COLUMNS = [ + "CloneID", "AlleleID", "AlleleSequence", "Label", "Action", + "TargetScore", "TargetFrom", "OffTargetScore", "OffTargetSites", "OffTargetHit", +] +MARKERS_SIDECAR_COLUMNS = ["CloneID", "Placement", "RefTagSites", "CloneIDCheck", "Sites"] + + +class GenomeHit(NamedTuple): + """One alignment to the genome: a 0-based, half-open interval, its strand (1 or -1) and its score. + + Also used for a genome window that stands in for a site, with strand 0 and score 0. + """ + + contig: str + start: int + end: int + strand: int + score: int + + +class GenomePlacement(NamedTuple): + """A sequence's distinct genome sites, best score first (hits overlapping a better one are merged into it).""" + + sites: Tuple[GenomeHit, ...] + + @property + def equal_best(self) -> Tuple[GenomeHit, ...]: + """The sites that share the best score.""" + return tuple(site for site in self.sites if site.score == self.sites[0].score) + + +class MarkerPlacement(NamedTuple): + """Where a marker's own tag lies in the genome. + + ``placed_by`` (the markers sidecar's Placement column): ``ref_tag`` (the + Ref tag has one best site), ``ref_tag+clone_id`` (several equal-best + sites; the CloneID position picks one), ``ref_tag_multicopy`` (several, + none picked: every site is the marker's), ``clone_id_site`` (no + equal-best site is at the CloneID position; the Ref tag's alignment + within a window around it), ``clone_id`` (the Ref tag did not align; the + window around the CloneID position stands in) or ``unplaced``. + ``ref_tag_site_count`` is the number of equal-best genome sites of the Ref + tag, whichever of them became ``sites``. ``clone_id_check`` compares the + CloneID position with the Ref-tag sites: ``agree``, ``disagree``, + ``no_position`` (the CloneID is not CHROM_POS), ``contig_absent``, + ``fallback``, or ``ignored`` (``--clone-positions ignore``). + """ + + sites: Tuple[GenomeHit, ...] + placed_by: str + ref_tag_site_count: int + clone_id_check: str + + +class GenomeLabel(NamedTuple): + """A haplotype's label and the two scores that decided it. + + The fields after the label are the labels sidecar's columns of the same + names. ``target_score`` is the best alignment at the marker's own site + and ``target_from`` says which search gave it: ``genome`` (the + genome-wide search), ``site`` (the realignment at the site, which scored + higher or was the only one to align there) or ``.`` (none). + ``off_target_score`` is the best alignment anywhere else, + ``off_target_sites`` the number of distinct sites elsewhere with that + score, among the at most ``MAX_SITES_PER_HAPLOTYPE`` sites kept, and + ``off_target_hit`` the first of them. + """ + + label: str + target_score: Optional[int] + target_from: str + off_target_score: Optional[int] + off_target_sites: int + off_target_hit: Optional[GenomeHit] + + +def hit_score(cigar: Sequence[Sequence[int]], edit_distance: int, matches: int, block_length: int) -> int: + """The score of an alignment under ``SHORT_READ_SCORING``, from what mappy reports of it. + + mappy returns an alignment's CIGAR, edit distance (NM), number of matching + bases and block length, but not minimap2's own alignment score (AS), so + the score is worked out here. As minimap2 defines them (``mm_update_extra`` + in align.c), the block length counts aligned columns and gap bases but + not columns with an ambiguous base, and NM counts all three kinds of + difference; hence mismatches = block - matches - gap bases, and ambiguous + columns = NM - (block - matches). A gap costs the cheaper of minimap2's + two affine pieces (Li 2018). Both differences are held at zero or above, + since a gap base that is itself ambiguous is not in the block length. + """ + (match_score, mismatch_penalty, gap_open_penalty, gap_extend_penalty, long_gap_open_penalty, + long_gap_extend_penalty, ambiguous_penalty) = SHORT_READ_SCORING + gap_bases = gap_cost = 0 + for length, operation in cigar: + if operation in (CIGAR_INSERTION, CIGAR_DELETION): + gap_bases += length + gap_cost += min( + gap_open_penalty + gap_extend_penalty * length, long_gap_open_penalty + long_gap_extend_penalty * length + ) + differences = block_length - matches + mismatches = max(0, differences - gap_bases) + ambiguous_bases = max(0, edit_distance - differences) + return match_score * matches - mismatch_penalty * mismatches - ambiguous_penalty * ambiguous_bases - gap_cost + + +def _same_locus(hit: GenomeHit, site: GenomeHit) -> bool: + """Whether two alignments are at one locus: their intervals share at least half of the shorter one. + + Adjacent tandem copies, which share a few bases, stay distinct. + """ + if hit.contig != site.contig: + return False + overlap = min(hit.end, site.end) - max(hit.start, site.start) + return overlap > 0 and 2 * overlap >= min(hit.end - hit.start, site.end - site.start) + + +def distinct_sites( + hits: Iterable[GenomeHit], max_sites: Optional[int] = MAX_SITES_PER_HAPLOTYPE +) -> GenomePlacement: + """A sequence's distinct genome sites, best first, at most ``max_sites`` of them (None: all). + + A hit at the same locus as a better-scoring one (``_same_locus``) is + merged into it. + """ + sites: List[GenomeHit] = [] + for hit in sorted(hits, key=lambda hit: (-hit.score, hit.contig, hit.start)): + if any(_same_locus(hit, site) for site in sites): + continue + sites.append(hit) + if len(sites) == max_sites: + break + return GenomePlacement(tuple(sites)) + + +def _index_part_contigs(path: Path) -> Optional[List[List[str]]]: + """The contig names in each part of a minimap2 index file, or None when the file is not one. + + minimap2 writes an index in one part for every ``-I`` bases of genome, + and mappy loads only the first part. The file is read as minimap2's + ``mm_idx_dump`` (index.c) writes it. Each part is: 4 bytes ``MMI\\2``; + five 32-bit words (window, k-mer length, bucket bits, number of + sequences, flags); for every sequence its name, preceded by the name's + length in one byte, and its length in 4 bytes; for every bucket two + counts, of 8-byte positions and of 16-byte hash entries; and, unless the + flags say the sequences are left out, the sequences at 4 bits a base in + 32-bit words. + """ + file_size = path.stat().st_size + with path.open("rb", buffering=0) as handle: + + def read_exactly(count: int) -> bytes: + data = handle.read(count) + if len(data) != count: + raise MadcError(f"{path} ends inside a minimap2 index part; rebuild the index") + return data + + def read_word() -> int: + return struct.unpack(" GenomeHit: + """A mappy alignment as a GenomeHit. ``contig`` and ``offset`` put a hit within a window on the genome.""" + return GenomeHit( + hit.ctg if contig is None else contig, offset + hit.r_st, offset + hit.r_en, hit.strand, + hit_score(hit.cigar, hit.NM, hit.mlen, hit.blen), + ) + + +def _load_mappy(): + try: + import mappy + except ImportError as exc: + raise MadcError( + "filter-paralogs needs the optional minimap2 aligner: pip install 'madcats[genome]' (or pip install mappy)" + ) from exc + return mappy + + +class GenomeAligner: + """The genome-wide search: minimap2's short-read preset through mappy, with + ``SHORT_READ_SCORING`` and the mapping options of ``GENOME_WIDE_MAP_OPTIONS``.""" + + def __init__(self, aligner, mappy_module) -> None: + self._aligner = aligner + self._mappy = mappy_module + self.contigs = frozenset(aligner.seq_names) + self.description = { + "mappy": getattr(mappy_module, "__version__", None), + "preset": "sr", + "k": aligner.k, + "w": aligner.w, + "scoring": list(SHORT_READ_SCORING), + "map_options": dict(GENOME_WIDE_MAP_OPTIONS), + "site_realignment": {"k": SITE_INDEX_K, "w": SITE_INDEX_W, "pad": SITE_FLANK}, + } + + @classmethod + def open(cls, genome: Optional[Path], index: Optional[Path], jobs: int) -> "GenomeAligner": + """Load ``index`` when it exists; otherwise index ``genome`` (saving to ``index`` when given).""" + mappy = _load_mappy() + if index is not None and index.exists(): + source, index_to_write = index, None + elif genome is not None: + source, index_to_write = genome, index + else: + raise MadcError(f"Genome index not found: {index} (pass --genome to build it)") + if not source.exists(): + raise MadcError(f"Genome file not found: {source}") + # The file may be an index whichever option named it; an index is loaded, never rebuilt. + parts = _index_part_contigs(source) + if parts is not None: + index_to_write = None + if index_to_write is not None: + index_to_write.parent.mkdir(parents=True, exist_ok=True) + if parts is not None and len(parts) > 1: + raise MadcError( + f"{source} is a minimap2 index in {len(parts)} parts ({sum(map(len, parts)):,} contigs), and mappy " + f"searches only the first ({len(parts[0]):,} contigs), so the rest of the genome would be missed. " + "Rebuild it as one part: give --genome and a new --genome-index path, or run minimap2 -x sr " + "with -I above the genome size" + ) + verb = "Loading index" if parts is not None else "Indexing genome" + print(f"{verb} {source}" + (f" (saving {index_to_write})" if index_to_write else ""), file=sys.stderr) + aligner = mappy.Aligner( + str(source), preset="sr", n_threads=jobs, fn_idx_out=str(index_to_write) if index_to_write else None, + scoring=list(SHORT_READ_SCORING), **GENOME_WIDE_MAP_OPTIONS, + ) + if not aligner: + raise MadcError(f"Could not load or build a minimap2 index from {source}") + if (aligner.k, aligner.w) != SHORT_READ_INDEX_K_W: + print( + f"warning: {source} was indexed with k={aligner.k}, w={aligner.w}; " + "short-read indexes (minimap2 -x sr) use k=21, w=11", + file=sys.stderr, + ) + return cls(aligner, mappy) + + def new_buffer(self): + return self._mappy.ThreadBuffer() + + def hits(self, sequence: str, buffer) -> List[GenomeHit]: + return [_genome_hit(hit) for hit in self._aligner.map(sequence, buf=buffer)] + + def fetch(self, contig: str, start: int, end: int) -> str: + """The genome sequence of an interval; empty when the index holds no sequence there.""" + return self._aligner.seq(contig, start, end) or "" + + def window(self, contig: str, start: int, end: int) -> "WindowIndex": + return WindowIndex(self._mappy, contig, start, self.fetch(contig, start, end)) + + +class WindowIndex: + """An index of one genome window with short seeds (``SITE_INDEX_K``, + ``SITE_INDEX_W``), so that a tag too diverged to seed in the genome-wide + search still aligns at a site it belongs to. Each window is an index of + its own, so windows never compete for the aligner's limit on hits. The + mapping options are the short-read preset's, not + ``GENOME_WIDE_MAP_OPTIONS``. Hits come back in genome coordinates.""" + + def __init__(self, mappy_module, contig: str, start: int, sequence: str) -> None: + self._contig, self._start = contig, start + self._aligner = ( + mappy_module.Aligner( + seq=sequence, preset="sr", k=SITE_INDEX_K, w=SITE_INDEX_W, scoring=list(SHORT_READ_SCORING) + ) + if sequence + else None + ) + + def hits(self, sequence: str, buffer=None) -> List[GenomeHit]: + if self._aligner is None: + return [] + return [_genome_hit(hit, self._contig, self._start) for hit in self._aligner.map(sequence, buf=buffer)] + + +def map_sequences( + aligner, sequences: Sequence[str], jobs: int = 1, max_sites: Optional[int] = MAX_SITES_PER_HAPLOTYPE +) -> Dict[str, GenomePlacement]: + """Every sequence's distinct genome sites (``distinct_sites``). + + Worker threads share the index; mappy releases Python's interpreter lock + while it maps, and each batch has a buffer of its own. + """ + + def align_batch(batch: Sequence[str]) -> List[GenomePlacement]: + buffer = aligner.new_buffer() + return [distinct_sites(aligner.hits(sequence, buffer), max_sites) for sequence in batch] + + batches = [sequences[start : start + MAPPING_BATCH] for start in range(0, len(sequences), MAPPING_BATCH)] + with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool: + placements = [placement for found in pool.map(align_batch, batches) for placement in found] + return dict(zip(sequences, placements)) + + +def clone_position(clone_id: str, contigs: Set[str]) -> Tuple[Optional[Tuple[str, int]], Optional[str]]: + """The (contig, 1-based position) a CHROM_POS CloneID names, with None; or None, with why it names none. + + Any CloneID that ends in an underscore and digits is read as CHROM_POS + when what precedes them is a contig of the genome. + """ + match = CLONE_POSITION_RE.fullmatch(clone_id) + if not match: + return None, "no_position" + if match.group(1) not in contigs: + return None, "contig_absent" + return (match.group(1), int(match.group(2))), None + + +def _site_is_at(site: GenomeHit, contig: str, position: int) -> bool: + """Whether a 1-based position lies in a site (0-based, half-open) or within ``CLONE_POSITION_SLACK`` of it.""" + return site.contig == contig and site.start - CLONE_POSITION_SLACK < position <= site.end + CLONE_POSITION_SLACK + + +def place_marker( + clone_id: str, + ref_placement: Optional[GenomePlacement], + contigs: Set[str], + tag_length: int, + use_clone_position: bool = True, + align_ref_tag: Optional[Callable[[str, int, int], Sequence[GenomeHit]]] = None, +) -> MarkerPlacement: + """Locate a marker's tag from its Ref tag's equal-best genome sites and its CloneID position. + + ``ref_placement`` is the Ref tag's genome sites, or None for a marker + without a Ref row. ``tag_length`` sizes the window around a CloneID + position. + + With ``use_clone_position`` the CloneID position is taken to be on this + genome. It picks one of several equal-best sites, and a window around it + stands in for a Ref tag that does not align. If two or more equal-best + sites are at the position (tandem copies), none is picked and every + equal-best site stays the marker's. When no equal-best site is at the + position, ``align_ref_tag(contig, start, end)`` aligns the Ref tag within + a window around it, and an alignment at the position becomes the marker's + site even if the tag aligns better elsewhere: the assembly may carry the + Alt allele at the target and a Ref-like copy elsewhere, or the + genome-wide search may have missed the site. With no such alignment the + marker is unplaced. Either way the disagreement is reported. Without + ``use_clone_position`` (a genome other than the one the CloneIDs refer + to) the position is never read: an unaligned Ref tag leaves the marker + unplaced, and equal-best sites stay multi-copy. + """ + position, no_position_reason = clone_position(clone_id, contigs) if use_clone_position else (None, "ignored") + equal_best = ref_placement.equal_best if ref_placement is not None else () + if position is None: + if not equal_best: + return MarkerPlacement((), "unplaced", 0, no_position_reason) + placed_by = "ref_tag" if len(equal_best) == 1 else "ref_tag_multicopy" + return MarkerPlacement(equal_best, placed_by, len(equal_best), no_position_reason) + contig, clone_base = position + # The genome interval a tag of this length lies in if it contains the CloneID position. + window_start, window_end = max(0, clone_base - 1 - tag_length), clone_base + tag_length + if not equal_best: + return MarkerPlacement((GenomeHit(contig, window_start, window_end, 0, 0),), "clone_id", 0, "fallback") + at_position = [site for site in equal_best if _site_is_at(site, contig, clone_base)] + if len(at_position) == 1: + placed_by = "ref_tag" if len(equal_best) == 1 else "ref_tag+clone_id" + return MarkerPlacement(tuple(at_position), placed_by, len(equal_best), "agree") + if at_position: + return MarkerPlacement(equal_best, "ref_tag_multicopy", len(equal_best), "agree") + realigned = () + if align_ref_tag is not None: + realigned = align_ref_tag(contig, max(0, window_start - SITE_FLANK), window_end + SITE_FLANK) + realigned_at_position = [hit for hit in realigned if _site_is_at(hit, contig, clone_base)] + if realigned_at_position: + best = max(realigned_at_position, key=lambda hit: hit.score) + return MarkerPlacement((best,), "clone_id_site", len(equal_best), "disagree") + return MarkerPlacement((), "unplaced", len(equal_best), "disagree") + + +def score_at_sites( + aligner, markers: Mapping[str, MarkerPlacement], haplotypes: Iterable[Tuple[str, str]], jobs: int = 1 +) -> Dict[Tuple[str, str], int]: + """Each (CloneID, sequence) pair's best score within its own marker's sites. + + A site is searched with ``SITE_FLANK`` genome bases either side of it. + Every placed marker's sites are indexed on their own with short seeds + (``WindowIndex``), so an allele of the marker that carries several SNPs, + which the genome-wide search misses or aligns only in part at the + marker's site, is still scored there. Markers that share a site do not + compete for the aligner's limit on hits. Pairs that do not align at + their marker's sites, and pairs of unplaced markers, are left out. + """ + sequences_by_marker: Dict[str, Set[str]] = defaultdict(set) + for clone_id, sequence in haplotypes: + if markers[clone_id].sites: + sequences_by_marker[clone_id].add(sequence) + + def score_marker(clone_id: str) -> Dict[Tuple[str, str], int]: + windows = [ + aligner.window(site.contig, max(0, site.start - SITE_FLANK), site.end + SITE_FLANK) + for site in markers[clone_id].sites + ] + buffer = aligner.new_buffer() + best_scores = {} + for sequence in sequences_by_marker[clone_id]: + scores = [hit.score for window in windows for hit in window.hits(sequence, buffer)] + if scores: + best_scores[(clone_id, sequence)] = max(scores) + return best_scores + + site_scores: Dict[Tuple[str, str], int] = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool: + for best_scores in pool.map(score_marker, list(sequences_by_marker)): + site_scores.update(best_scores) + return site_scores + + +def genome_label(placement: GenomePlacement, marker: MarkerPlacement, site_score: Optional[int] = None) -> GenomeLabel: + """Label a haplotype by its best score at its marker's own site against its best score anywhere else. + + ``on_target``: the marker's site scores strictly best. ``paralog``: + somewhere else scores strictly better (or the haplotype aligns only + elsewhere). ``ambiguous``: a tie between the marker's site and another + site, or a marker whose own Ref tag is multi-copy and unresolved. + ``unmapped``: no alignment. ``unplaced``: the marker itself could not be + located, so the haplotype cannot be judged. + + ``placement`` is the haplotype's sites from the genome-wide search and + ``marker`` its marker's placement. ``site_score`` is the haplotype's + score in the realignment at the marker's site (``score_at_sites``), or + None when it did not align there. The target score is the higher of it + and the genome-wide hit at the site, so the label does not depend on + which search found the site. + """ + hits_at_marker = [hit for hit in placement.sites if any(_same_locus(hit, site) for site in marker.sites)] + hits_elsewhere = [hit for hit in placement.sites if hit not in hits_at_marker] + off_target_score = hits_elsewhere[0].score if hits_elsewhere else None + off_target_sites = sum(hit.score == off_target_score for hit in hits_elsewhere) + off_target_hit = hits_elsewhere[0] if hits_elsewhere else None + if not marker.sites: + return GenomeLabel("unplaced", None, ".", off_target_score, off_target_sites, off_target_hit) + if site_score is not None and (not hits_at_marker or site_score > hits_at_marker[0].score): + target_score, target_from = site_score, "site" + elif hits_at_marker: + target_score, target_from = hits_at_marker[0].score, "genome" + else: + target_score, target_from = None, "." + if target_score is None and off_target_score is None: + label = "unmapped" + elif target_score is None or (off_target_score is not None and off_target_score > target_score): + label = "paralog" + elif off_target_score == target_score or len(marker.sites) > 1: + label = "ambiguous" + else: + label = "on_target" + return GenomeLabel(label, target_score, target_from, off_target_score, off_target_sites, off_target_hit) + + +def _preamble_header_and_rows(handle, path: Path): + """(preamble rows, header row, reader positioned at the first data row) of an open MADC.""" + reader = csv.reader(handle) + preamble = [] + for row in reader: + if is_madc_header(row): + return preamble, row, reader + preamble.append(row) + raise MadcError(f"Could not find an AlleleID header row in MADC file: {path}") + + +def _haplotype_rows(reader, header: Sequence[str], path: Path): + """The haplotype rows of a MADC, each as (row, AlleleID, CloneID, sequence in upper case). + + A row with an empty AlleleID, or too short to hold the three identity + columns, is not a haplotype row and is passed over. + """ + allele_index, clone_index, sequence_index = madc_identity_columns(header, path) + width = max(allele_index, clone_index, sequence_index) + 1 + for row in reader: + if len(row) < width or not row[allele_index].strip(): + continue + yield row, row[allele_index].strip(), row[clone_index].strip(), row[sequence_index].strip().upper() + + +def _label_column_index(header: Sequence[str], madc_path: Path) -> int: + """The column that carries the flag: a GenomeLabel column from an earlier run, or PICRef.""" + stripped = [column.strip() for column in header] + for name in (GENOME_LABEL_COLUMN, REPLACED_STATISTICS_COLUMN): + if name in stripped: + return stripped.index(name) + raise MadcError( + f"{madc_path} has no {REPLACED_STATISTICS_COLUMN} column to carry the flag; use drop mode, " + "or read the labels sidecar" + ) + + +def check_madc_header(madc_path: Path, flagging: bool) -> None: + """Stop with an error when the MADC cannot be filtered as asked. + + That is a MADC without a header row or without its identity columns, or, + when rows are to be flagged, without a column to carry the flag. + """ + with open_text(madc_path, "rt") as handle: + _, header, _ = _preamble_header_and_rows(handle, madc_path) + madc_identity_columns(header, madc_path) + if flagging: + _label_column_index(header, madc_path) + + +def _read_haplotypes(madc_path: Path): + """Pass 1 over the MADC: its distinct haplotypes, and each marker's Ref tag and tag length. + + Returns the header row and three dictionaries, each in the order of first + appearance. ``haplotypes`` maps (CloneID, sequence as written) to the + sequence that is aligned, which is the sequence without gap characters. + ``ref_tags`` maps a CloneID to its first Ref row's sequence without gaps, + and ``tag_lengths`` to the length of its first row as written. + """ + haplotypes: Dict[Tuple[str, str], str] = {} + ref_tags: Dict[str, str] = {} + tag_lengths: Dict[str, int] = {} + with open_text(madc_path, "rt") as handle: + _, header, reader = _preamble_header_and_rows(handle, madc_path) + for _, allele_id, clone_id, sequence in _haplotype_rows(reader, header, madc_path): + ungapped = sequence.replace("-", "") + haplotypes[(clone_id, sequence)] = ungapped + tag_lengths.setdefault(clone_id, len(sequence)) + if clone_id not in ref_tags and allele_type(allele_id) == "Ref": + ref_tags[clone_id] = ungapped + return header, haplotypes, ref_tags, tag_lengths + + +def _place_markers( + aligner, ref_tags: Mapping[str, str], tag_lengths: Mapping[str, int], use_clone_position: bool, jobs: int +) -> Dict[str, MarkerPlacement]: + """Every marker's placement (``place_marker``), from the genome sites of its Ref tag.""" + # Ref tags keep every equal-best site, so a capped list cannot hide the one the CloneID names. + ref_placements = map_sequences(aligner, sorted(set(ref_tags.values())), jobs, max_sites=None) + markers: Dict[str, MarkerPlacement] = {} + for clone_id, tag_length in tag_lengths.items(): + ref_tag = ref_tags.get(clone_id) # None for a marker without a Ref row, which place_marker never aligns + + def align_ref_tag(contig: str, start: int, end: int, ref_tag=ref_tag) -> List[GenomeHit]: + return aligner.window(contig, start, end).hits(ref_tag) + + markers[clone_id] = place_marker( + clone_id, ref_placements.get(ref_tag), aligner.contigs, tag_length, use_clone_position, align_ref_tag + ) + return markers + + +def _format_site(site: GenomeHit) -> str: + """A site as contig:start-end(strand), 1-based and inclusive, as the two sidecars show it.""" + strand = {1: "+", -1: "-"}.get(site.strand, ".") + return f"{site.contig}:{site.start + 1}-{site.end}({strand})" + + +def _dot_if_none(value) -> object: + return "." if value is None else value + + +def _write_markers_sidecar(markers_path: Path, markers: Mapping[str, MarkerPlacement]) -> None: + markers_path.parent.mkdir(parents=True, exist_ok=True) + with markers_path.open("w", newline="") as handle: + writer = csv.writer(handle, lineterminator="\n") + writer.writerow(MARKERS_SIDECAR_COLUMNS) + for clone_id, marker in markers.items(): + sites = ";".join(_format_site(site) for site in marker.sites) or "." + writer.writerow([clone_id, marker.placed_by, marker.ref_tag_site_count, marker.clone_id_check, sites]) + + +class _RowTallies(NamedTuple): + """Rows and reads counted while the table is written, in total and by label, outcome and allele category.""" + + totals: Dict[str, Dict[str, int]] + by_label: Dict[str, Dict[str, int]] + by_outcome: Dict[str, Dict[str, int]] + by_category: Dict[str, Dict[str, Dict[str, int]]] + overflow: CountOverflowCheck + + +def _write_filtered_table( + madc_path: Path, + output: Path, + labels_path: Path, + labels: Mapping[Tuple[str, str], GenomeLabel], + actions: Mapping[str, str], + label_index: Optional[int], + provenance: str, +) -> _RowTallies: + """Pass 2 over the MADC: write the rows that are kept, and every row's label to the labels sidecar. + + ``actions`` gives what was asked for ``paralog`` and ``ambiguous`` rows + (``drop``, ``flag`` or ``keep``); rows with any other label are kept. + What is done with a row is its outcome, the sidecar's Action column: + ``kept``, ``flagged``, ``dropped``, or ``protected`` for a canonical Ref + or Alt row that would have been dropped and is kept. With ``label_index`` + every row written carries its label in that column, whatever its outcome. + """ + + def tally(): + return defaultdict(lambda: {"rows": 0, "reads": 0}) + + by_label, by_outcome = tally(), tally() + by_category: Dict[str, Dict[str, Dict[str, int]]] = defaultdict(tally) + overflow = CountOverflowCheck() + totals = {"input": {"rows": 0, "reads": 0}, "output": {"rows": 0, "reads": 0}} + output.parent.mkdir(parents=True, exist_ok=True) + labels_path.parent.mkdir(parents=True, exist_ok=True) + with open_text(madc_path, "rt") as handle, open_text(output, "wt") as out_handle, labels_path.open( + "w", newline="" + ) as label_handle: + preamble, header, reader = _preamble_header_and_rows(handle, madc_path) + writer = csv.writer(out_handle, lineterminator="\n") + label_writer = csv.writer(label_handle, lineterminator="\n") + label_writer.writerow(LABELS_SIDECAR_COLUMNS) + if provenance and preamble and preamble[0] and preamble[0][0].startswith("* MADCATS"): + preamble[0] = [f"{preamble[0][0]} | {provenance}"] + preamble[0][1:] + writer.writerows(preamble) + if label_index is not None: + header[label_index] = GENOME_LABEL_COLUMN + writer.writerow(header) + sample_start = madc_sample_start(header) + for row, allele_id, clone_id, sequence in _haplotype_rows(reader, header, madc_path): + haplotype_label = labels[(clone_id, sequence)] + category = allele_type(allele_id) + requested = actions.get(haplotype_label.label, "keep") + if requested == "drop": + outcome = "protected" if category in ("Ref", "Alt") else "dropped" + else: + outcome = "flagged" if requested == "flag" else "kept" + counts = madc_read_counts(row[sample_start:], madc_path) + overflow.add(clone_id, counts) + reads = sum(counts) + label = haplotype_label.label + for counted in (totals["input"], by_label[label], by_outcome[outcome], by_category[category][label]): + counted["rows"] += 1 + counted["reads"] += reads + if outcome != "dropped": + if label_index is not None: + row = row + [""] * max(0, label_index + 1 - len(row)) + row[label_index] = label + writer.writerow(row) + totals["output"]["rows"] += 1 + totals["output"]["reads"] += reads + off_target_hit = haplotype_label.off_target_hit + label_writer.writerow([ + clone_id, allele_id, sequence, label, outcome, + _dot_if_none(haplotype_label.target_score), haplotype_label.target_from, + _dot_if_none(haplotype_label.off_target_score), haplotype_label.off_target_sites, + "." if off_target_hit is None else _format_site(off_target_hit), + ]) + return _RowTallies(totals, by_label, by_outcome, by_category, overflow) + + +def filter_paralogs( + madc_path: Path, + output: Path, + aligner, + labels_path: Path, + markers_path: Path, + paralog_action: str = "drop", + ambiguous_action: str = "drop", + jobs: int = 1, + provenance: str = "", + clone_positions: str = "use", +) -> Dict[str, object]: + """Label every MADC haplotype by where the genome places it; drop or flag paralogous rows. + + Pass 1 reads only row identities (``_read_haplotypes``). Every distinct + haplotype is aligned to the genome once; each marker is located from its + Ref row's tag, all of whose equal-best sites are kept (``place_marker``); + every haplotype is realigned to its own marker's sites + (``score_at_sites``) and labelled by ``genome_label``. Pass 2 streams the + table again and writes the kept rows unchanged + (``_write_filtered_table``), so a vendor MADC keeps its preamble, + statistics and column layout, and a HapApp fixed-allele-ID MADC keeps its + Code_version column. Canonical Ref and Alt rows are never dropped, since + downstream dosage tools need both designed alleles; they are reported as + ``protected`` instead. In flag mode the PICRef column becomes + ``GenomeLabel`` on every row. The labels sidecar has one line per + haplotype row, dropped rows included. ``clone_positions="use"`` takes + CloneID coordinates to be on this genome; ``"ignore"`` keeps them out of + marker placement, for a genome other than the one they refer to. + + Returns the summary that goes into the QC JSON. + """ + actions = {"paralog": paralog_action, "ambiguous": ambiguous_action} + header, haplotypes, ref_tags, tag_lengths = _read_haplotypes(madc_path) + label_index = _label_column_index(header, madc_path) if "flag" in actions.values() else None + + sequences = sorted(set(haplotypes.values())) + print(f"Aligning {len(sequences):,} distinct haplotype sequences on {jobs} thread(s)", file=sys.stderr) + started = time.time() + placements = map_sequences(aligner, sequences, jobs) + markers = _place_markers(aligner, ref_tags, tag_lengths, clone_positions == "use", jobs) + # Realign every haplotype at its own marker's sites, and label it by the better of the two scores there. + aligned_pairs = {(clone_id, ungapped) for (clone_id, _), ungapped in haplotypes.items()} + site_scores = score_at_sites(aligner, markers, aligned_pairs, jobs) + labels = { + (clone_id, sequence): genome_label( + placements[ungapped], markers[clone_id], site_scores.get((clone_id, ungapped)) + ) + for (clone_id, sequence), ungapped in haplotypes.items() + } + alignment_seconds = round(time.time() - started, 1) + + _write_markers_sidecar(markers_path, markers) + try: + tallies = _write_filtered_table(madc_path, output, labels_path, labels, actions, label_index, provenance) + except BaseException: + # A run that fails part of the way through leaves no partial files behind. + for partial in (output, labels_path, markers_path): + partial.unlink(missing_ok=True) + raise + + warning = tallies.overflow.warning(madc_path) + if warning: + print(warning, file=sys.stderr) + sources = Counter(marker.placed_by for marker in markers.values()) + checks = Counter(marker.clone_id_check for marker in markers.values()) + checked = checks["agree"] + checks["disagree"] + if checked and checks["disagree"] > CLONE_DISAGREEMENT_WARNING_SHARE * checked: + print( + f"warning: CloneID positions disagree with the Ref tag's best genome sites for {checks['disagree']:,} " + f"of {checked:,} markers, so they may refer to another assembly. Those markers were placed at their " + "CloneID position, or left unplaced where the Ref tag does not align there. If the positions refer " + "to another assembly, rerun with --clone-positions ignore", + file=sys.stderr, + ) + if markers and not sources.keys() - {"unplaced"}: + print("warning: no marker could be placed on the genome; every row was kept", file=sys.stderr) + + def plain(tally: Mapping[str, Dict[str, int]]) -> Dict[str, Dict[str, int]]: + return {key: dict(counts) for key, counts in tally.items()} + + return { + "markers": { + "total": len(markers), + "placement": dict(sources), + "clone_id_check": dict(checks), + "multicopy_ref_tags": [clone_id for clone_id, marker in markers.items() if marker.ref_tag_site_count > 1], + "clone_id_disagreements": [ + clone_id for clone_id, marker in markers.items() if marker.clone_id_check == "disagree" + ], + "unplaced": [clone_id for clone_id, marker in markers.items() if marker.placed_by == "unplaced"], + }, + "haplotypes": { + "distinct": len(haplotypes), + "distinct_sequences": len(sequences), + "labels": dict(Counter(result.label for result in labels.values())), + "realigned_at_site": { + "haplotypes": sum(bool(markers[clone_id].sites) for clone_id, _ in haplotypes), + "found_at_site": sum(pair in site_scores for pair in + ((clone_id, ungapped) for (clone_id, _), ungapped in haplotypes.items())), + "target_from_site": sum(result.target_from == "site" for result in labels.values()), + }, + "alignment_seconds": alignment_seconds, + }, + "rows": { + **tallies.totals, + "by_label": plain(tallies.by_label), + "by_outcome": plain(tallies.by_outcome), + "by_category": {category: plain(by_label) for category, by_label in tallies.by_category.items()}, + }, + "genome_label_column": GENOME_LABEL_COLUMN if label_index is not None else None, + "count_overflow": tallies.overflow.summary(), + } + + +def run_filter_paralogs(args: argparse.Namespace) -> int: + """Clean a finished MADC with a reference genome (see ``filter_paralogs``).""" + if not args.genome and not args.genome_index: + raise MadcError("filter-paralogs needs --genome, --genome-index, or both") + madc = Path(args.madc) + output = Path(args.output) + if not madc.is_file(): + raise MadcError(f"MADC not found: {madc}") + if output.resolve() == madc.resolve(): + raise MadcError("--output must be a new file, not the input MADC") + genome = Path(args.genome) if args.genome else None + index = Path(args.genome_index) if args.genome_index else None + # A MADC that cannot be filtered as asked is refused before the index is loaded, which can take minutes. + check_madc_header(madc, flagging="flag" in (args.paralogs, args.ambiguous)) + started = time.time() + aligner = GenomeAligner.open(genome, index, args.jobs) + labels_path = output.with_suffix(output.suffix + ".genome_labels.csv") + markers_path = output.with_suffix(output.suffix + ".genome_markers.csv") + provenance = ( + f"filter-paralogs v{__version__} paralogs={args.paralogs} ambiguous={args.ambiguous} " + f"clone_positions={args.clone_positions}" + ) + summary = filter_paralogs( + madc, output, aligner, labels_path, markers_path, + paralog_action=args.paralogs, ambiguous_action=args.ambiguous, jobs=args.jobs, provenance=provenance, + clone_positions=args.clone_positions, + ) + qc_path = Path(args.qc_output) if args.qc_output else output.with_suffix(output.suffix + ".qc.json") + qc = { + "pipeline": "MADCATS", + "command": "filter-paralogs", + "version": __version__, + "created_utc": dt.datetime.now(dt.timezone.utc).isoformat(), + "inputs": { + "madc": str(madc.resolve()), + "genome": str(genome.resolve()) if genome else None, + "genome_index": str(index.resolve()) if index else None, + }, + "parameters": { + "paralogs": args.paralogs, "ambiguous": args.ambiguous, "clone_positions": args.clone_positions, + "jobs": args.jobs, + }, + "aligner": aligner.description, + "outputs": {"madc": str(output.resolve()), "labels": str(labels_path), "markers": str(markers_path)}, + "elapsed_seconds": round(time.time() - started, 1), + **summary, + } + _write_json(qc_path, qc) + rows = summary["rows"] + dropped_reads = rows["input"]["reads"] - rows["output"]["reads"] + dropped_share = dropped_reads / rows["input"]["reads"] if rows["input"]["reads"] else 0.0 + print( + f"Kept {rows['output']['rows']:,} of {rows['input']['rows']:,} rows; " + f"dropped {dropped_reads:,} reads ({dropped_share:.1%})", + file=sys.stderr, + ) + print( + f"Wrote {output}, its labels {labels_path}, its markers {markers_path}, and QC to {qc_path}", file=sys.stderr + ) + return 0 + + +def add_filter_paralogs_command(subparsers) -> None: + filtering = subparsers.add_parser( + "filter-paralogs", + help="drop or flag MADC rows that a reference genome places at another locus (needs mappy)", + description=( + "Drop or flag the rows of a MADC that a reference genome places at another locus, for panels where " + "paralogous copies of a target co-amplify with it. Each marker is placed on the genome by its Ref " + "tag and, for CHROM_POS CloneIDs, by that position. Every row's sequence is then aligned and " + "labelled on_target, paralog (it aligns better elsewhere) or ambiguous (it aligns equally well at " + "the marker and elsewhere). Needs the mappy aligner: pip install 'madcats[genome]'." + ), + ) + filtering.add_argument( + "--madc", + required=True, + help=( + "MADC to clean, plain or gzipped: one written by generate, a vendor DArT MADC (with or without its " + "starred preamble) or a HapApp fixed-allele-ID MADC" + ), + ) + filtering.add_argument( + "--genome", + help=( + "reference genome FASTA, plain or gzipped. It should hold each locus once: of a haplotype-resolved " + "assembly, give one set of chromosomes. Not needed when --genome-index exists" + ), + ) + filtering.add_argument( + "--genome-index", + help=( + "minimap2 index (.mmi) built with -x sr, in one part; loaded when it exists, otherwise built from " + "--genome and saved here for the next run" + ), + ) + filtering.add_argument( + "--clone-positions", + choices=("use", "ignore"), + default="use", + help=( + "CloneIDs of the form CHROM_POS: use takes them as positions on this genome and places each marker " + "there, leaving it unplaced when its Ref tag aligns only elsewhere; ignore places markers by " + "their Ref tag alone, for CloneIDs that refer to another assembly (default: use)" + ), + ) + filtering.add_argument( + "--output", + required=True, + help=( + "cleaned MADC to write. Written beside it: OUTPUT.genome_labels.csv (every input row's label and " + "scores, and what was done with it) and OUTPUT.genome_markers.csv (where each marker was placed)" + ), + ) + filtering.add_argument( + "--paralogs", + choices=("drop", "flag"), + default="drop", + help=( + "paralog rows: drop them, or flag them (keep them, and write every row's label in a GenomeLabel " + "column that takes the place of PICRef). Ref and Alt rows are never dropped, nor are rows that do " + "not align or whose marker could not be placed (default: drop)" + ), + ) + filtering.add_argument( + "--ambiguous", + choices=("drop", "flag", "keep"), + default="drop", + help=( + "ambiguous rows, which include the rows of a marker whose Ref tag has several equal-best sites: " + "drop them, flag them as --paralogs flag does, or keep them (default: drop)" + ), + ) + filtering.add_argument("--qc-output", help="QC JSON (default: OUTPUT.qc.json)") + filtering.add_argument( + "--jobs", type=positive_int, default=_default_jobs(), help="alignment threads (default: up to 4)" + ) + filtering.set_defaults(func=run_filter_paralogs) diff --git a/pyproject.toml b/pyproject.toml index 0a4a751..4766bb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,11 @@ classifiers = [ "Topic :: Scientific/Engineering :: Bio-Informatics", ] +[project.optional-dependencies] +# filter-paralogs aligns haplotypes to a reference genome with minimap2; every other command +# needs only the standard library. +genome = ["mappy>=2.24"] + [project.urls] Homepage = "https://github.com/Breeding-Insight/MADCATS" Documentation = "https://madcats.readthedocs.io" @@ -48,7 +53,7 @@ Changelog = "https://madcats.readthedocs.io/en/latest/development/changelog.html madcats = "madcats:main" [tool.setuptools] -py-modules = ["madcats"] +py-modules = ["madcats", "madcats_paralogs", "madcats_call"] [tool.setuptools.dynamic] version = { attr = "madcats.__version__" } diff --git a/scripts/make_polyrad_fixture.R b/scripts/make_polyrad_fixture.R new file mode 100644 index 0000000..2a151b2 --- /dev/null +++ b/scripts/make_polyrad_fixture.R @@ -0,0 +1,147 @@ +#!/usr/bin/env Rscript +# Writes tests/fixtures/polyrad_parity.json: reference values from polyRAD that +# tests/test_call.py compares the madcats_call genotype engine against. +# +# Usage (from the repository root, with polyRAD installed): +# Rscript scripts/make_polyrad_fixture.R +# +# The fixture holds two sets of values for one small simulated data set: +# single_pass one round of likelihood, HWE prior and posterior, starting +# from the depth-ratio allele frequencies. madcats_call must +# reproduce these almost exactly. +# iterated IterateHWE run to a very tight tolerance, plus polyRAD's most +# probable genotypes after its correction to the ploidy. +# selfing the Hardy-Weinberg prior with a selfing rate, for several +# ploidies, and IterateHWE with that selfing rate. +# The committed JSON was made with the polyRAD version recorded inside it. + +suppressMessages(library(polyRAD)) + +overdispersion <- 9 +contamination_rate <- 0.001 +iteration_tolerance <- 1e-10 +selfing_rate <- 0.4 + +# Three loci with 2, 3 and 4 alleles; 16 samples of ploidy 2, 3, 4 and 6. +set.seed(20240611) +alleles_per_locus <- c(2L, 3L, 4L) +alleles2loc <- rep(seq_along(alleles_per_locus), alleles_per_locus) +sample_ploidy <- rep(c(2L, 3L, 4L, 6L), each = 4) +sample_names <- sprintf("sample%02d", seq_along(sample_ploidy)) +allele_names <- unlist(lapply(seq_along(alleles_per_locus), function(locus) { + sprintf("locus%d_allele%d", locus, seq_len(alleles_per_locus[locus])) +})) + +depth <- matrix(0L, nrow = length(sample_names), ncol = length(allele_names), + dimnames = list(sample_names, allele_names)) +for (locus in seq_along(alleles_per_locus)) { + columns <- which(alleles2loc == locus) + frequencies <- rev(seq_along(columns)) / sum(seq_along(columns)) + for (sample in seq_along(sample_names)) { + copies <- rmultinom(1, sample_ploidy[sample], frequencies)[, 1] + total_reads <- sample(c(4:60, 150, 400), 1) + depth[sample, columns] <- rmultinom(1, total_reads, copies + 0.02)[, 1] + } +} +# One sample with no reads at the first locus, and one with a single read. +depth["sample03", alleles2loc == 1] <- 0L +depth["sample09", alleles2loc == 1] <- c(1L, 0L) + +loc_table <- data.frame(row.names = sprintf("locus%d", seq_along(alleles_per_locus))) +nucleotides <- unlist(lapply(alleles_per_locus, function(n) c("A", "C", "G", "T")[seq_len(n)])) +rad <- RADdata(depth, alleles2loc, loc_table, list(2L), contamination_rate, + nucleotides, setNames(sample_ploidy, sample_names)) + +# Flatten polyRAD's [dosage, sample, allele] arrays (one per sample ploidy) +# into one record per sample and allele. +records <- function(object, with_likelihood) { + out <- list() + for (h in seq_len(ncol(object$posteriorProb))) { + posterior <- object$posteriorProb[[1, h]] + for (sample in dimnames(posterior)[[2]]) { + for (allele in dimnames(posterior)[[3]]) { + record <- list(sample = sample, allele = allele, + posterior = unname(posterior[, sample, allele])) + if (with_likelihood) { + record$likelihood <- unname(object$genotypeLikelihood[[1, h]][, sample, allele]) + record$prior <- unname(object$priorProb[[1, h]][, allele]) + } + out[[length(out) + 1]] <- record + } + } + } + out +} + +single <- AddAlleleFreqHWE(rad) +start_frequency <- single$alleleFreq +single <- AddGenotypePriorProb_HWE(single) +single <- AddGenotypeLikelihood(single, overdispersion = overdispersion) +single <- AddPloidyChiSq(single) +single <- AddGenotypePosteriorProb(single) + +iterated <- suppressMessages(IterateHWE(rad, tol = iteration_tolerance, + overdispersion = overdispersion)) +genotypes <- GetProbableGenotypes(iterated, omit1allelePerLocus = FALSE, + multiallelic = "correct")$genotypes + +selfed <- suppressMessages(suppressWarnings( + IterateHWE(rad, selfing.rate = selfing_rate, tol = iteration_tolerance, overdispersion = overdispersion))) +selfing_priors <- list() +for (ploidy in c(2L, 4L, 6L, 8L)) { + for (rate in c(0.1, selfing_rate, 0.95)) { + for (frequency in c(0.05, 0.3, 0.5)) { + selfing_priors[[length(selfing_priors) + 1]] <- list( + ploidy = ploidy, selfing_rate = rate, allele_frequency = frequency, + prior = as.vector(polyRAD:::.HWEpriors(c(allele = frequency), ploidy, rate))) + } + } +} + +# Minimal JSON writer, so the script needs no package beyond polyRAD. +to_json <- function(value) { + if (is.list(value)) { + if (is.null(names(value))) { + return(paste0("[", paste(vapply(value, to_json, ""), collapse = ","), "]")) + } + fields <- vapply(names(value), function(name) { + paste0("\"", name, "\":", to_json(value[[name]])) + }, "") + return(paste0("{", paste(fields, collapse = ","), "}")) + } + if (is.character(value)) { + text <- paste0("\"", value, "\"") + } else { + text <- ifelse(is.na(value), "null", sprintf("%.17g", value)) + } + if (length(value) == 1) text else paste0("[", paste(text, collapse = ","), "]") +} + +fixture <- list( + polyrad_version = as.character(packageVersion("polyRAD")), + overdispersion = overdispersion, + contamination_rate = contamination_rate, + iteration_tolerance = iteration_tolerance, + samples = as.list(sample_names), + ploidy = as.list(sample_ploidy), + alleles = as.list(allele_names), + allele_locus = as.list(alleles2loc), + depth = lapply(seq_len(nrow(depth)), function(row) unname(depth[row, ])), + single_pass = list( + allele_frequency = unname(start_frequency), + records = records(single, TRUE) + ), + iterated = list( + allele_frequency = unname(iterated$alleleFreq), + records = records(iterated, FALSE), + genotypes = lapply(seq_len(nrow(genotypes)), function(row) unname(genotypes[row, ])) + ), + selfing = list( + selfing_rate = selfing_rate, + priors = selfing_priors, + allele_frequency = unname(selfed$alleleFreq), + records = records(selfed, FALSE) + ) +) +writeLines(to_json(fixture), "tests/fixtures/polyrad_parity.json") +cat("Wrote tests/fixtures/polyrad_parity.json\n") diff --git a/scripts/make_updog_fixture.R b/scripts/make_updog_fixture.R new file mode 100644 index 0000000..615270c --- /dev/null +++ b/scripts/make_updog_fixture.R @@ -0,0 +1,176 @@ +#!/usr/bin/env Rscript +# Writes tests/fixtures/updog_norm.json: reference values from updog that +# tests/test_call.py compares the normal-shaped prior of madcats_call against. +# +# Usage (from the repository root, with updog installed): +# Rscript scripts/make_updog_fixture.R +# +# The fixture holds two sets of values: +# prior_fits updog's fit of the norm genotype distribution to given dosage +# weights (its internal flex_update_pivec), for weights of many +# shapes. madcats_call.fit_normal_prior must find the same +# distribution. +# markers complete updog fits (flexdog_full, model = "norm") of simulated +# read counts, with updog's allelic bias, sequencing error and +# overdispersion held fixed at no bias, no error and the given +# overdispersion. With those fixed, updog's likelihood is the +# beta-binomial that madcats_call uses with a contamination rate +# of 0, so the posteriors must agree. +# bias_markers complete updog fits (flexdog, from its five starting values +# of the bias) of read counts simulated with an allelic bias, +# with the bias estimated and the sequencing error and +# overdispersion held fixed, for the norm and the hw (binomial) +# genotype distributions. +# no_read_marker one marker in which some samples have no reads, fitted as +# the markers above, twice: with those samples given a depth of +# 0, which updog keeps in the fit with a posterior equal to the +# prior, and with them given NA, which updog leaves out. +# Every fit also records updog's maxpostprob (each sample's largest posterior +# probability) and prop_mis (one minus the mean of maxpostprob), which +# madcats_call writes to its VCFs as MPP and PMC. +# The committed JSON was made with the updog version recorded inside it. + +suppressMessages(library(updog)) + +# madcats_call and polyRAD give the overdispersion as the sum of the two +# beta-binomial shape parameters; updog as rho = 1 / (that sum + 1). +overdispersion <- 30 +updog_overdispersion <- 1 / (overdispersion + 1) + +weight_sets <- list( + list(ploidy = 4L, weights = c(10, 40, 30, 15, 5)), # one peak inside the range + list(ploidy = 4L, weights = c(90, 9, 0.9, 0.09, 0.01)), # rare allele: mean below 0 + list(ploidy = 4L, weights = c(0.01, 0.5, 3, 30, 66)), # common allele + list(ploidy = 4L, weights = c(50, 0.5, 0.2, 0.5, 50)), # two peaks: inbred lines + list(ploidy = 4L, weights = c(20, 20, 20, 20, 20)), # flat + list(ploidy = 4L, weights = c(0, 1, 98, 1, 0)), # one F1 class + list(ploidy = 4L, weights = c(60, 40, 0.001, 0.001, 0.001)), # two neighbouring dosages + list(ploidy = 2L, weights = c(30, 50, 20)), + list(ploidy = 2L, weights = c(49, 2, 49)), + list(ploidy = 2L, weights = c(77, 51, 45)), + list(ploidy = 6L, weights = c(1, 2, 5, 20, 40, 25, 7)), + list(ploidy = 6L, weights = c(61, 13, 7, 21, 8, 0.4, 2.8)), + list(ploidy = 3L, weights = c(5, 30, 40, 10)), + list(ploidy = 8L, weights = c(2, 9, 20, 30, 22, 11, 4, 1.5, 0.5)) +) +prior_fits <- lapply(weight_sets, function(set) { + fit <- updog:::flex_update_pivec(weight_vec = set$weights, model = "norm", + control = list(ploidy = set$ploidy, mu = set$ploidy / 2, sigma = 1)) + list(ploidy = set$ploidy, weights = set$weights, prior = as.vector(fit$pivec), + mean = fit$par$mu, standard_deviation = fit$par$sigma) +}) + +# Simulated read counts of one allele: genotype classes of three kinds of population. +set.seed(20240930) +simulate <- function(ploidy, dosages) { + depth <- sample(15:90, length(dosages), replace = TRUE) + fraction <- dosages / ploidy + shape1 <- pmax(fraction * overdispersion, 1e-9); shape2 <- pmax((1 - fraction) * overdispersion, 1e-9) + read_fraction <- ifelse(fraction %in% c(0, 1), fraction, rbeta(length(dosages), shape1, shape2)) + list(ploidy = ploidy, allele_depth = rbinom(length(dosages), depth, read_fraction), marker_depth = depth) +} +marker_sets <- list( + random_mating_tetraploid = simulate(4L, rbinom(90, 4, 0.3)), + one_family_hexaploid = simulate(6L, sample(c(2, 3, 4), 80, replace = TRUE, prob = c(0.25, 0.5, 0.25))), + inbred_diploid = simulate(2L, sample(c(0, 2), 70, replace = TRUE, prob = c(0.6, 0.4))), + rare_allele_tetraploid = simulate(4L, rbinom(90, 4, 0.03)) +) +markers <- lapply(names(marker_sets), function(name) { + data <- marker_sets[[name]] + fit <- flexdog_full(refvec = data$allele_depth, sizevec = data$marker_depth, ploidy = data$ploidy, + model = "norm", verbose = FALSE, seq = 0, bias = 1, od = updog_overdispersion, + update_bias = FALSE, update_seq = FALSE, update_od = FALSE, + itermax = 20000, tol = 1e-13) + list(name = name, ploidy = data$ploidy, allele_depth = data$allele_depth, marker_depth = data$marker_depth, + prior = as.vector(fit$gene_dist), mean = fit$par$mu, standard_deviation = fit$par$sigma, + genotypes = as.vector(fit$geno), + posteriors = lapply(seq_len(nrow(fit$postmat)), function(row) as.vector(fit$postmat[row, ])), + max_posterior = as.vector(fit$maxpostprob), proportion_misgenotyped = fit$prop_mis) +}) + +# Read counts with an allelic bias, as updog defines it: with bias h the expected +# read fraction xi of the allele becomes xi / (h * (1 - xi) + xi). +simulate_biased <- function(ploidy, dosages, bias) { + depth <- sample(25:110, length(dosages), replace = TRUE) + fraction <- dosages / ploidy + fraction <- fraction / (bias * (1 - fraction) + fraction) + shape1 <- pmax(fraction * overdispersion, 1e-9); shape2 <- pmax((1 - fraction) * overdispersion, 1e-9) + read_fraction <- ifelse(fraction %in% c(0, 1), fraction, rbeta(length(dosages), shape1, shape2)) + list(ploidy = ploidy, allele_depth = rbinom(length(dosages), depth, read_fraction), marker_depth = depth) +} +bias_sets <- list( + tetraploid_bias_0.6 = list(data = simulate_biased(4L, rbinom(120, 4, 0.4), 0.6), simulated_bias = 0.6), + tetraploid_bias_1.9 = list(data = simulate_biased(4L, rbinom(120, 4, 0.55), 1.9), simulated_bias = 1.9), + tetraploid_no_bias = list(data = simulate_biased(4L, rbinom(120, 4, 0.3), 1), simulated_bias = 1), + diploid_bias_0.5 = list(data = simulate_biased(2L, rbinom(100, 2, 0.5), 0.5), simulated_bias = 0.5), + hexaploid_family_bias_1.5 = list( + data = simulate_biased(6L, sample(c(1, 2, 3), 110, replace = TRUE, prob = c(0.25, 0.5, 0.25)), 1.5), + simulated_bias = 1.5) +) +bias_markers <- list() +for (name in names(bias_sets)) { + data <- bias_sets[[name]]$data + for (model in c("norm", "hw")) { + fit <- flexdog(refvec = data$allele_depth, sizevec = data$marker_depth, ploidy = data$ploidy, + model = model, verbose = FALSE, seq = 0, od = updog_overdispersion, + update_seq = FALSE, update_od = FALSE, itermax = 20000, tol = 1e-13) + bias_markers[[length(bias_markers) + 1]] <- list( + name = name, model = model, ploidy = data$ploidy, simulated_bias = bias_sets[[name]]$simulated_bias, + allele_depth = data$allele_depth, marker_depth = data$marker_depth, + bias = fit$bias, log_likelihood = fit$llike, prior = as.vector(fit$gene_dist), + genotypes = as.vector(fit$geno), + posteriors = lapply(seq_len(nrow(fit$postmat)), function(row) as.vector(fit$postmat[row, ])), + max_posterior = as.vector(fit$maxpostprob), proportion_misgenotyped = fit$prop_mis) + } +} + +# Samples without reads. This marker is simulated after the others so that their +# random draws, and with them the values above, stay as they were. +no_read_data <- simulate(4L, rbinom(100, 4, 0.35)) +samples_without_reads <- c(3, 17, 18, 40, 41, 42, 77, 90) +no_read_data$allele_depth[samples_without_reads] <- 0L +no_read_data$marker_depth[samples_without_reads] <- 0L +fit_fixed_read_model <- function(allele_depth, marker_depth) { + flexdog_full(refvec = allele_depth, sizevec = marker_depth, ploidy = no_read_data$ploidy, + model = "norm", verbose = FALSE, seq = 0, bias = 1, od = updog_overdispersion, + update_bias = FALSE, update_seq = FALSE, update_od = FALSE, itermax = 20000, tol = 1e-13) +} +given_depth_0 <- fit_fixed_read_model(no_read_data$allele_depth, no_read_data$marker_depth) +given_na <- fit_fixed_read_model(replace(no_read_data$allele_depth, samples_without_reads, NA), + replace(no_read_data$marker_depth, samples_without_reads, NA)) +no_read_marker <- list( + ploidy = no_read_data$ploidy, allele_depth = no_read_data$allele_depth, marker_depth = no_read_data$marker_depth, + prior = as.vector(given_depth_0$gene_dist), prior_given_na = as.vector(given_na$gene_dist), + max_posterior = as.vector(given_depth_0$maxpostprob), + proportion_misgenotyped = given_depth_0$prop_mis, + proportion_misgenotyped_given_na = given_na$prop_mis) + +# Minimal JSON writer, so the script needs no package beyond updog. +to_json <- function(value) { + if (is.list(value)) { + if (is.null(names(value))) { + return(paste0("[", paste(vapply(value, to_json, ""), collapse = ","), "]")) + } + fields <- vapply(names(value), function(name) { + paste0("\"", name, "\":", to_json(value[[name]])) + }, "") + return(paste0("{", paste(fields, collapse = ","), "}")) + } + if (is.character(value)) { + text <- paste0("\"", value, "\"") + } else { + text <- ifelse(is.na(value), "null", sprintf("%.17g", value)) + } + if (length(value) == 1) text else paste0("[", paste(text, collapse = ","), "]") +} + +fixture <- list( + updog_version = as.character(packageVersion("updog")), + overdispersion = overdispersion, + prior_fits = prior_fits, + markers = markers, + bias_markers = bias_markers, + no_read_marker = no_read_marker +) +writeLines(to_json(fixture), "tests/fixtures/updog_norm.json") +cat("Wrote tests/fixtures/updog_norm.json\n") diff --git a/tests/fixtures/polyrad_parity.json b/tests/fixtures/polyrad_parity.json new file mode 100644 index 0000000..71ed584 --- /dev/null +++ b/tests/fixtures/polyrad_parity.json @@ -0,0 +1 @@ +{"polyrad_version":"2.0.1","overdispersion":9,"contamination_rate":0.001,"iteration_tolerance":1e-10,"samples":["sample01","sample02","sample03","sample04","sample05","sample06","sample07","sample08","sample09","sample10","sample11","sample12","sample13","sample14","sample15","sample16"],"ploidy":[2,2,2,2,3,3,3,3,4,4,4,4,6,6,6,6],"alleles":["locus1_allele1","locus1_allele2","locus2_allele1","locus2_allele2","locus2_allele3","locus3_allele1","locus3_allele2","locus3_allele3","locus3_allele4"],"allele_locus":[1,1,2,2,2,3,3,3,3],"depth":[[18,0,27,29,1,5,4,0,0],[56,0,28,0,0,18,27,0,2],[0,0,0,0,19,0,23,19,0],[0,38,0,0,30,28,0,0,0],[13,5,44,1,0,3,1,3,0],[12,16,42,0,14,19,6,0,0],[12,14,25,17,0,1,14,9,0],[7,11,4,7,4,37,0,15,0],[1,0,10,7,0,0,2,2,0],[5,8,35,0,20,32,0,0,11],[8,7,24,30,0,35,15,0,0],[54,0,12,13,6,13,15,0,2],[2,2,16,23,6,24,6,8,6],[3,5,29,21,0,10,3,8,0],[4,5,35,13,0,30,0,29,0],[15,1,9,12,0,2,0,2,0]],"single_pass":{"allele_frequency":[0.61174094424094427,0.38825905575905578,0.5133000343826114,0.28515442746849728,0.20154553814889128,0.48674686334359607,0.25699642703632064,0.22491936911958099,0.031337340500502305],"records":[{"sample":"sample01","allele":"locus1_allele1","posterior":[8.0480933827748891e-11,0.0026077359399369342,0.99739226397958225],"likelihood":[1.995234281465539e-10,0.0020515804549146276,0.99603586444515824],"prior":[0.15074509437891354,0.47502792276028438,0.37422698286080208]},{"sample":"sample01","allele":"locus1_allele2","posterior":[0.99739226397958203,0.0026077359399369294,8.0480933827755056e-11],"likelihood":[0.99603586444516001,0.0020515804549146276,1.9952342814656949e-10],"prior":[0.37422698286080208,0.47502792276028444,0.15074509437891359]},{"sample":"sample01","allele":"locus2_allele1","posterior":[2.1117869138908025e-05,0.99996923257335435,9.6495575067955508e-06],"likelihood":[1.6757894864120412e-06,0.03761976092549562,6.8842528038315616e-07],"prior":[0.23687685653196724,0.4996462181708427,0.26347692529719002]},{"sample":"sample01","allele":"locus2_allele2","posterior":[1.7658350520183648e-05,0.99997294824999816,9.3933994817135658e-06],"likelihood":[5.3392883644241037e-07,0.037898638809811154,1.7849261864866441e-06],"prior":[0.51100419256789187,0.40768275992722164,0.081313047504886479]},{"sample":"sample01","allele":"locus2_allele3","posterior":[0.9642650923360031,0.035734907654836452,9.1603797610842833e-12],"likelihood":[0.0015848241225949525,0.00011633871443588446,2.3629364112671915e-13],"prior":[0.63752952764994353,0.32184986840233015,0.040620603947726194]},{"sample":"sample01","allele":"locus3_allele1","posterior":[5.360295611217866e-05,0.99978136689086805,0.00016503015301972036],"likelihood":[1.7972987742296558e-05,0.1767400732183661,6.1525016768340708e-05],"prior":[0.26342878228763722,0.49964870873753336,0.23692250897482939]},{"sample":"sample01","allele":"locus3_allele2","posterior":[0.00025103789565660646,0.99972202607761917,2.693602672409043e-05],"likelihood":[3.0709928200355092e-05,0.17678792444060518,2.754233035031337e-05],"prior":[0.55205430943679357,0.38189852705377153,0.066047163509434875]},{"sample":"sample01","allele":"locus3_allele3","posterior":[0.99037610608736104,0.009623891198861239,2.7137776551406853e-09],"likelihood":[0.99854003363036492,0.016718837000291482,3.2492239401078178e-08],"prior":[0.60074998436598837,0.34866129302886134,0.050588722605150321]},{"sample":"sample01","allele":"locus3_allele4","posterior":[0.99891698847740085,0.0010830114799364081,4.2662834990437527e-11],"likelihood":[0.99979647418783479,0.01675311957701783,4.079930426999173e-08],"prior":[0.93830734790863979,0.060710623181715766,0.00098202890964442209]},{"sample":"sample02","allele":"locus1_allele1","posterior":[9.2413223624540142e-15,3.7538599715771183e-05,0.999962461400275],"likelihood":[2.2780718464526183e-14,2.9365365757988425e-05,0.99294474949454958],"prior":[0.15074509437891354,0.47502792276028438,0.37422698286080208]},{"sample":"sample02","allele":"locus1_allele2","posterior":[0.999962461400275,3.7538599715771156e-05,9.241322362454732e-15],"likelihood":[0.99294474949455047,2.9365365757988425e-05,2.2780718464527962e-14],"prior":[0.37422698286080208,0.47502792276028444,0.15074509437891359]},{"sample":"sample02","allele":"locus2_allele1","posterior":[5.0185211422239483e-12,0.00082353712953905163,0.99917646286544248],"likelihood":[5.5511377013494721e-12,0.00043186643665510459,0.9936386632490275],"prior":[0.23687685653196724,0.4996462181708427,0.26347692529719002]},{"sample":"sample02","allele":"locus2_allele2","posterior":[0.99965297858498392,0.00034702141377303074,1.2430548845357775e-12],"likelihood":[0.99626821830112311,0.00043349630854752218,7.7854109486756453e-12],"prior":[0.51100419256789187,0.40768275992722164,0.081313047504886479]},{"sample":"sample02","allele":"locus2_allele3","posterior":[0.99978028020803333,0.00021971979140964715,5.5703934396688555e-13],"likelihood":[0.99736101380128184,0.0004341741551787132,8.7214284098646367e-12],"prior":[0.63752952764994353,0.32184986840233015,0.040620603947726194]},{"sample":"sample02","allele":"locus3_allele1","posterior":[0.00010663296554972762,0.99989081613122954,2.5509032206760185e-06],"likelihood":[7.7834488782363708e-06,0.038479695284254892,2.0702913767628023e-07],"prior":[0.26342878228763722,0.49964870873753336,0.23692250897482939]},{"sample":"sample02","allele":"locus3_allele2","posterior":[7.4935376582328218e-06,0.99996579654931383,2.6709913028004131e-05],"likelihood":[2.2002090572768019e-07,0.042442017531469908,6.5550673971356611e-06],"prior":[0.55205430943679357,0.38189852705377153,0.066047163509434875]},{"sample":"sample02","allele":"locus3_allele3","posterior":[0.9999653208054361,3.4679194553282729e-05,1.062566817267369e-14],"likelihood":[0.99620995373634691,5.952850445823698e-05,1.2570780467116384e-13],"prior":[0.60074998436598837,0.34866129302886134,0.050588722605150321]},{"sample":"sample02","allele":"locus3_allele4","posterior":[0.71290801126578029,0.2870919886802446,5.397510634541977e-11],"likelihood":[0.00010262923662757285,0.00063876204642702213,7.4242391751072045e-12],"prior":[0.93830734790863979,0.060710623181715766,0.00098202890964442209]},{"sample":"sample03","allele":"locus1_allele1","posterior":[0.15074509437891354,0.47502792276028438,0.37422698286080208],"likelihood":[1,1,1],"prior":[0.15074509437891354,0.47502792276028438,0.37422698286080208]},{"sample":"sample03","allele":"locus1_allele2","posterior":[0.37422698286080208,0.47502792276028444,0.15074509437891359],"likelihood":[1,1,1],"prior":[0.37422698286080208,0.47502792276028444,0.15074509437891359]},{"sample":"sample03","allele":"locus2_allele1","posterior":[0.99639421983897281,0.0036057800435290877,1.1749813793377587e-10],"likelihood":[0.99459213080675091,0.0017063694872453323,1.0544472240783754e-10],"prior":[0.23687685653196724,0.4996462181708427,0.26347692529719002]},{"sample":"sample03","allele":"locus2_allele2","posterior":[0.99863150931104494,0.0013684906640932515,2.4861886012803087e-11],"likelihood":[0.99699235065757563,0.001712499751491289,1.559857255678916e-10],"prior":[0.51100419256789187,0.40768275992722164,0.081313047504886479]},{"sample":"sample03","allele":"locus2_allele3","posterior":[6.7578547207433404e-10,0.013391867969224456,0.98660813135499015],"likelihood":[4.3275954905383989e-11,0.0016987361767082118,0.99159979185440517],"prior":[0.63752952764994353,0.32184986840233015,0.040620603947726194]},{"sample":"sample03","allele":"locus3_allele1","posterior":[0.9998236997729526,0.00017630022685798401,1.8937795601662241e-13],"likelihood":[0.99222584768049904,9.2244065469704575e-05,2.0896494421049025e-13],"prior":[0.26342878228763722,0.49964870873753336,0.23692250897482939]},{"sample":"sample03","allele":"locus3_allele2","posterior":[1.2354520166631496e-05,0.99996910246826387,1.854301156951762e-05],"likelihood":[4.1827305097018746e-07,0.048938922572438474,5.2473726130765849e-06],"prior":[0.55205430943679357,0.38189852705377153,0.066047163509434875]},{"sample":"sample03","allele":"locus3_allele3","posterior":[5.5316147267414135e-05,0.99994090285268189,3.7810000507333923e-06],"likelihood":[1.5724125173604821e-06,0.048975506222374607,1.2763250887167632e-06],"prior":[0.60074998436598837,0.34866129302886134,0.050588722605150321]},{"sample":"sample03","allele":"locus3_allele4","posterior":[0.99999396863484613,6.0313651533906631e-06,4.202902812058777e-16],"likelihood":[0.99949771786514829,9.3170939594526116e-05,4.013789735526259e-13],"prior":[0.93830734790863979,0.060710623181715766,0.00098202890964442209]},{"sample":"sample04","allele":"locus1_allele1","posterior":[0.99956809052490525,0.00043190947419906452,8.9575798337906026e-13],"likelihood":[0.99068896776691662,0.000135844394732797,3.5762158236398676e-13],"prior":[0.15074509437891354,0.47502792276028438,0.37422698286080208]},{"sample":"sample04","allele":"locus1_allele2","posterior":[8.9575798337916891e-13,0.00043190947419906474,0.99956809052490525],"likelihood":[3.5762158236402992e-13,0.000135844394732797,0.99068896776691573],"prior":[0.37422698286080208,0.47502792276028444,0.15074509437891359]},{"sample":"sample04","allele":"locus2_allele1","posterior":[0.99929052842158272,0.00070947157501700421,3.4002761025042355e-12],"likelihood":[0.9930473505122791,0.00033425138282884776,3.0378925832683544e-12],"prior":[0.23687685653196724,0.4996462181708427,0.26347692529719002]},{"sample":"sample04","allele":"locus2_allele2","posterior":[0.99973119121255372,0.00026880878672796277,7.1837392651931121e-13],"likelihood":[0.99613183538059458,0.00033572150042736451,4.4982994634507949e-12],"prior":[0.51100419256789187,0.40768275992722164,0.081313047504886479]},{"sample":"sample04","allele":"locus2_allele3","posterior":[1.9705483286438783e-11,0.0026555609560001566,0.99734443902429437],"likelihood":[1.2453008871713856e-12,0.00033242252957084401,0.98920500576031323],"prior":[0.63752952764994353,0.32184986840233015,0.040620603947726194]},{"sample":"sample04","allele":"locus3_allele1","posterior":[5.8815602501503869e-12,0.00091562419032264629,0.99908437580379583],"likelihood":[5.2590915940801362e-12,0.00043165213903775492,0.99329272114493183],"prior":[0.26342878228763722,0.49964870873753336,0.23692250897482939]},{"sample":"sample04","allele":"locus3_allele2","posterior":[0.99969903723544817,0.00030096276357990091,9.7205841620852341e-13],"likelihood":[0.99663612446115446,0.00043372448303725106,8.1000434614492095e-12],"prior":[0.55205430943679357,0.38189852705377153,0.066047163509434875]},{"sample":"sample04","allele":"locus3_allele3","posterior":[0.99974744541475935,0.00025255458452637825,7.142673919302435e-13],"likelihood":[0.99705539466447557,0.0004339845540322271,8.4592111075740543e-12],"prior":[0.60074998436598837,0.34866129302886134,0.050588722605150321]},{"sample":"sample04","allele":"locus3_allele4","posterior":[0.9999718076707067,2.8192329282188959e-05,1.1143864744555762e-14],"likelihood":[0.99958923828707869,0.00043555720423251273,1.0643647729096226e-11],"prior":[0.93830734790863979,0.060710623181715766,0.00098202890964442209]},{"sample":"sample05","allele":"locus1_allele1","posterior":[3.7725312290879072e-07,0.085568901515549092,0.91404973560135583,0.00038098562997215536],"likelihood":[3.5399073878103583e-07,0.016986652540822621,0.11516372901170124,9.1396546627549611e-05],"prior":[0.058528148003866715,0.27665083912514049,0.43589104501528608,0.22892996785570674]},{"sample":"sample05","allele":"locus1_allele2","posterior":[0.00038098562997220193,0.91404973560135561,0.085568901515549245,3.7725312290882084e-07],"likelihood":[9.1396546627560643e-05,0.11516372901170104,0.016986652540822621,3.5399073878106352e-07],"prior":[0.22892996785570674,0.43589104501528614,0.2766508391251406,0.058528148003866735]},{"sample":"sample05","allele":"locus2_allele1","posterior":[4.2086217676135391e-11,0.00086197830799149115,0.83159029117378436,0.16754773047613794],"likelihood":[1.08719820252389e-12,7.0377515788201389e-06,0.0064377933007353991,0.0036895843603735137],"prior":[0.11528795792966355,0.36476669580691107,0.38470263144935296,0.13524271481407238]},{"sample":"sample05","allele":"locus2_allele2","posterior":[0.21902132716556957,0.78063770566730684,0.00034096715734355866,9.7800598127591169e-12],"likelihood":[0.0021688544514718517,0.0064595815308033057,7.0729326270389334e-06,1.5257446315812727e-12],"prior":[0.36528908460219295,0.43714532389709682,0.17437881599373556,0.023186775506974625]},{"sample":"sample05","allele":"locus2_allele3","posterior":[0.99818201377681226,0.0018177498493001753,2.3637388450544269e-07,2.9946396036417485e-15],"likelihood":[0.99666951633695788,0.0023967970522114105,1.2347345508646e-06,1.8591605315253907e-13],"prior":[0.50903829591392724,0.38547369520804897,0.097301107395446215,0.0081869014825774536]},{"sample":"sample05","allele":"locus3_allele1","posterior":[0.00012405222482461068,0.61244263470043969,0.38740535946144905,2.7953613286611621e-05],"likelihood":[0.00011287463343073525,0.19586841429030902,0.13064508218464882,2.982051229022722e-05],"prior":[0.13520564879470676,0.38466940047879145,0.36480366262750857,0.1153212880989932]},{"sample":"sample05","allele":"locus3_allele2","posterior":[0.0046062003350986593,0.95520155008312047,0.040191969953274058,2.7962850678698176e-07],"likelihood":[0.0010780532363605872,0.21544427876505429,0.026208553791170597,1.5815039423848296e-06],"prior":[0.41017832438153429,0.42562795516577789,0.14721983541487937,0.016973885037808421]},{"sample":"sample05","allele":"locus3_allele3","posterior":[0.00025561106757295411,0.83737223161129626,0.16236673075218258,5.4265689482639713e-06],"likelihood":[5.2021579611842682e-05,0.19575876150232002,0.13080360918881781,4.5194885311109783e-05],"prior":[0.46562967688379209,0.40536092244658872,0.11763101709670327,0.011378383572915896]},{"sample":"sample05","allele":"locus3_allele4","posterior":[0.98815068107080684,0.011831826529901553,1.749239267570071e-05,6.6158480810967755e-12],"likelihood":[0.99983068571267952,0.12335144696862246,0.0056370526966950073,1.9770610383848357e-07],"prior":[0.90890329105310352,0.088212170566608974,0.0028537642059646737,3.0774174322864263e-05]},{"sample":"sample06","allele":"locus1_allele1","posterior":[2.156658076946068e-05,0.55396437245340624,0.44600719894543456,6.8620203897900097e-06],"likelihood":[1.1271990068979277e-05,0.061253925495822432,0.031300264319799494,9.1692421086326666e-07],"prior":[0.058528148003866715,0.27665083912514049,0.43589104501528608,0.22892996785570674]},{"sample":"sample06","allele":"locus1_allele2","posterior":[6.8620203897908364e-06,0.4460071989454345,0.55396437245340624,2.1566580769462293e-05],"likelihood":[9.1692421086337741e-07,0.031300264319799494,0.061253925495822432,1.1271990068980118e-05],"prior":[0.22892996785570674,0.43589104501528614,0.2766508391251406,0.058528148003866735]},{"sample":"sample06","allele":"locus2_allele1","posterior":[5.5026336308657688e-08,0.05375930867744802,0.94593519263362857,0.00030544366258714712],"likelihood":[8.0695204806917961e-09,0.0024917219200032673,0.041571653906215242,3.8183726918538503e-05],"prior":[0.11528795792966355,0.36476669580691107,0.38470263144935296,0.13524271481407238]},{"sample":"sample06","allele":"locus2_allele2","posterior":[0.99838387461130418,0.0016159445260702796,1.8086262381638627e-07,1.7030168635093739e-15],"likelihood":[0.99481356778047236,0.001345493015015894,3.7751650118379177e-07,2.6733722032553953e-14],"prior":[0.36528908460219295,0.43714532389709682,0.17437881599373556,0.023186775506974625]},{"sample":"sample06","allele":"locus2_allele3","posterior":[0.00049241629292721707,0.98456059986709688,0.014946977476540816,6.3634349551518919e-09],"likelihood":[1.5755670560891869e-05,0.041600891601753363,0.0025020185806361723,1.2659803898071947e-08],"prior":[0.50903829591392724,0.38547369520804897,0.097301107395446215,0.0081869014825774536]},{"sample":"sample06","allele":"locus3_allele1","posterior":[2.0111149371782516e-07,0.079297745738836564,0.92028192047328439,0.00042013267638527888],"likelihood":[5.051905731660175e-08,0.007001424077263306,0.085679097459157319,0.00012373431514795243],"prior":[0.13520564879470676,0.38466940047879145,0.36480366262750857,0.1153212880989932]},{"sample":"sample06","allele":"locus3_allele2","posterior":[0.00067505971228397376,0.97178561679923681,0.027539288391364786,3.5097114365546647e-08],"likelihood":[6.1798715947514659e-05,0.085733447988235414,0.0070241916778608395,7.7642609373037651e-08],"prior":[0.41017832438153429,0.42562795516577789,0.14721983541487937,0.016973885037808421]},{"sample":"sample06","allele":"locus3_allele3","posterior":[0.99109825911955662,0.0088953329345123726,6.407945429902066e-06,5.0108833487670835e-13],"likelihood":[0.99722850723921319,0.010281085049849807,2.5522051248193905e-05,2.0632523410768606e-11],"prior":[0.46562967688379209,0.40536092244658872,0.11763101709670327,0.011378383572915896]},{"sample":"sample06","allele":"locus3_allele4","posterior":[0.99899973998871128,0.0010001795925125457,8.0418775214035728e-08,8.782691802622733e-16],"likelihood":[0.99961341642839618,0.010311785196078721,2.5628532868191574e-05,2.5955273565131551e-11],"prior":[0.90890329105310352,0.088212170566608974,0.0028537642059646737,3.0774174322864263e-05]},{"sample":"sample07","allele":"locus1_allele1","posterior":[1.4076637750927609e-05,0.47510749676209646,0.52486666473300925,1.1761867143411412e-05],"likelihood":[8.1719391364324681e-06,0.058351343823960129,0.040913064889979638,1.7456798261604322e-06],"prior":[0.058528148003866715,0.27665083912514049,0.43589104501528608,0.22892996785570674]},{"sample":"sample07","allele":"locus1_allele2","posterior":[1.176186714341281e-05,0.52486666473300836,0.4751074967620974,1.4076637750928687e-05],"likelihood":[1.7456798261606429e-06,0.040913064889979638,0.058351343823960337,8.1719391364331067e-06],"prior":[0.22892996785570674,0.43589104501528614,0.2766508391251406,0.058528148003866735]},{"sample":"sample07","allele":"locus2_allele1","posterior":[1.8287230319211248e-06,0.26472790417095166,0.73523200460647553,3.8262499540809875e-05],"likelihood":[3.7689392152740566e-07,0.017244066440690219,0.045410302183742986,6.7222491152251339e-06],"prior":[0.11528795792966355,0.36476669580691107,0.38470263144935296,0.13524271481407238]},{"sample":"sample07","allele":"locus2_allele2","posterior":[6.2727123394325368e-05,0.86811970403160832,0.13181703370596262,5.3513903474528312e-07],"likelihood":[3.9242323791459593e-06,0.045382646339434327,0.017274832700155038,5.2742694600579989e-07],"prior":[0.36528908460219295,0.43714532389709682,0.17437881599373556,0.023186775506974625]},{"sample":"sample07","allele":"locus2_allele3","posterior":[0.9978282317959396,0.0021714269619059982,3.4124214903828183e-07,5.2918353307439137e-15],"likelihood":[0.99677384945673431,0.0028644535466167353,1.7833483732349756e-06,3.2868362027878688e-13],"prior":[0.50903829591392724,0.38547369520804897,0.097301107395446215,0.0081869014825774536]},{"sample":"sample07","allele":"locus3_allele1","posterior":[0.039199496560749184,0.95524059351360602,0.0055599081375241098,1.7881206755982903e-09],"likelihood":[0.0032667465086887756,0.027980463984646573,0.00017172685202361808,1.7470974220835695e-10],"prior":[0.13520564879470676,0.38466940047879145,0.36480366262750857,0.1153212880989932]},{"sample":"sample07","allele":"locus3_allele2","posterior":[1.1192178970755522e-05,0.57536759265662307,0.42460742218804509,1.3792976361060778e-05],"likelihood":[6.9113882612628939e-07,0.034240382190050232,0.073054101716126793,2.0582593024286389e-05],"prior":[0.41017832438153429,0.42562795516577789,0.14721983541487937,0.016973885037808421]},{"sample":"sample07","allele":"locus3_allele3","posterior":[0.00013638987544326202,0.91475996540863624,0.085103298173270711,3.4654264976320835e-07],"likelihood":[1.0520828627784335e-05,0.081053866323883048,0.025985615334836301,1.093917544666974e-06],"prior":[0.46562967688379209,0.40536092244658872,0.11763101709670327,0.011378383572915896]},{"sample":"sample07","allele":"locus3_allele4","posterior":[0.99889992317694565,0.0010999785649084505,9.8258144658049426e-08,1.2070506820794439e-15],"likelihood":[0.99962195974486479,0.011341936149906121,3.131713016405488e-05,3.5675542065009945e-11],"prior":[0.90890329105310352,0.088212170566608974,0.0028537642059646737,3.0774174322864263e-05]},{"sample":"sample08","allele":"locus1_allele1","posterior":[4.9813631777937466e-05,0.61767208849370514,0.38227140301301482,6.6948615020960836e-06],"likelihood":[3.8391627560110552e-05,0.10071150469774788,0.039559151081623126,1.3191435233474032e-06],"prior":[0.058528148003866715,0.27665083912514049,0.43589104501528608,0.22892996785570674]},{"sample":"sample08","allele":"locus1_allele2","posterior":[6.6948615020968755e-06,0.38227140301301604,0.61767208849370392,4.9813631777941092e-05],"likelihood":[1.3191435233475624e-06,0.039559151081623334,0.10071150469774788,3.8391627560113419e-05],"prior":[0.22892996785570674,0.43589104501528614,0.2766508391251406,0.058528148003866735]},{"sample":"sample08","allele":"locus2_allele1","posterior":[0.00036511408478255873,0.86288117242377871,0.13675274000870447,9.7348273431923329e-07],"likelihood":[0.00017885641964855811,0.13359669257260184,0.020075703476908329,4.0651283837979939e-07],"prior":[0.11528795792966355,0.36476669580691107,0.38470263144935296,0.13524271481407238]},{"sample":"sample08","allele":"locus2_allele2","posterior":[6.5390688849936005e-05,0.76473664443765055,0.23519337507916396,4.589794335583449e-06],"likelihood":[9.6675963159817237e-06,0.094476838967444243,0.072840119562764302,1.0690355770163595e-05],"prior":[0.36528908460219295,0.43714532389709682,0.17437881599373556,0.023186775506974625]},{"sample":"sample08","allele":"locus2_allele3","posterior":[0.00066614309949668022,0.96271551511157938,0.036618239034899795,1.027540241347023e-07],"likelihood":[7.0030828250407202e-05,0.13365198385371249,0.020139647176094982,6.7166305490987303e-07],"prior":[0.50903829591392724,0.38547369520804897,0.097301107395446215,0.0081869014825774536]},{"sample":"sample08","allele":"locus3_allele1","posterior":[1.7065549304974593e-07,0.094955077198507071,0.90487940241208809,0.00016534973391182337],"likelihood":[2.3004546413945078e-08,0.0044990289713193301,0.045208464006198185,2.6132577973857786e-05],"prior":[0.13520564879470676,0.38466940047879145,0.36480366262750857,0.1153212880989932]},{"sample":"sample08","allele":"locus3_allele2","posterior":[0.99829412711994481,0.0017056692788870449,2.0360116593093192e-07,2.1496597123933665e-15],"likelihood":[0.99547193896912634,0.0016391092287788909,5.6566218345743715e-07,5.1800228272787711e-14],"prior":[0.41017832438153429,0.42562795516577789,0.14721983541487937,0.016973885037808421]},{"sample":"sample08","allele":"locus3_allele3","posterior":[0.00028147664502320224,0.97156363779540866,0.028154863286565537,2.2273002655127271e-08],"likelihood":[1.1405368424820091e-05,0.045220668454004846,0.0045158465471048142,3.6932253160296281e-08],"prior":[0.46562967688379209,0.40536092244658872,0.11763101709670327,0.011378383572915896]},{"sample":"sample08","allele":"locus3_allele4","posterior":[0.99984009459235479,0.00015990361976067554,1.7878845233227317e-09,2.3084881442982081e-18],"likelihood":[0.99944679055921704,0.0016469355030881145,5.6920431514623195e-07,6.8153481232819223e-14],"prior":[0.90890329105310352,0.088212170566608974,0.0028537642059646737,3.0774174322864263e-05]},{"sample":"sample09","allele":"locus1_allele1","posterior":[2.272408347930754e-05,0.05861283611396105,0.27671266557711821,0.43581069050288923,0.22884108372255219],"likelihood":[0.00061174094424094416,0.25036174094424085,0.50011174094424105,0.74986174094424118,0.99961174094424166],"prior":[0.022724083479307551,0.14321625809823665,0.33847729110292601,0.35553653261843071,0.14004583470109905]},{"sample":"sample09","allele":"locus1_allele2","posterior":[0.22884108372255185,0.43581069050288934,0.27671266557711838,0.058612836113961141,2.2724083479309255e-05],"likelihood":[0.99961174094423988,0.74986174094424118,0.50011174094424105,0.25036174094424107,0.00061174094424098981],"prior":[0.14004583470109905,0.35553653261843077,0.33847729110292613,0.14321625809823671,0.022724083479307565]},{"sample":"sample09","allele":"locus2_allele1","posterior":[2.4840917393540702e-06,0.089447920157418384,0.61296613195719074,0.29755523968607878,2.8224107572607949e-05],"likelihood":[2.7769598801681407e-06,0.02370294192502248,0.10267559967581853,0.070889130954526217,2.5502472170434568e-05],"prior":[0.05611064516046619,0.23670925107678942,0.37446951499863801,0.26329049860004528,0.06942009016406106]},{"sample":"sample09","allele":"locus2_allele2","posterior":[6.8617172196879013e-05,0.52041185976572346,0.45174058484790769,0.027778485341575579,4.5287259630572537e-07],"likelihood":[1.4896614719906508e-05,0.070806654594560522,0.10272043593872135,0.023751972255066839,3.8829273202989744e-06],"prior":[0.26112528482196318,0.4166551991209192,0.24930784911281489,0.066299855249770806,0.0066118116945319255]},{"sample":"sample09","allele":"locus2_allele3","posterior":[0.93523874696225595,0.063868148829491836,0.00089124301283494394,1.8611938939468283e-06,1.5233510234420776e-12],"likelihood":[0.99800999758099351,0.067501543227267174,0.0024877756703341777,3.0872721860832501e-05,4.0042413422123837e-10],"prior":[0.40644389862556018,0.41037758915346811,0.15538100668589588,0.026147472069997697,0.0016500334650780288]},{"sample":"sample09","allele":"locus3_allele1","posterior":[0.33089940760739489,0.47185147496194663,0.18167402085479109,0.015574465769557809,6.3080630960180354e-07],"likelihood":[0.99831293761721984,0.37527055190240138,0.10157087382839307,0.013772387349575818,2.3527757802416212e-06],"prior":[0.069394723337547384,0.26324370182863749,0.37447324821462663,0.23675605136026034,0.05613227525892811]},{"sample":"sample09","allele":"locus3_allele2","posterior":[0.00020195061271369399,0.51253358366035628,0.42580981006051938,0.061446269883305822,8.3857831048957593e-06],"likelihood":[0.00010532662648185042,0.19320535183850385,0.309374878006558,0.19360652148393043,0.00030555675660288078],"prior":[0.30476396056773508,0.42165745525519693,0.21876972744876036,0.050446628920665579,0.0043622278076420259]},{"sample":"sample09","allele":"locus3_allele3","posterior":[0.00023064758702083459,0.56118902057087905,0.39120625874284215,0.04736841493690129,5.6581623566990983e-06],"likelihood":[9.2159331030406398e-05,0.19317886044979052,0.30937484367401369,0.19363298542603422,0.00031882026088305543],"prior":[0.36090054371573521,0.41891653267222745,0.18234704588483627,0.035276658872380191,0.0025592188548208472]},{"sample":"sample09","allele":"locus3_allele4","posterior":[0.95300992499566273,0.04637864764938443,0.00060964096026870759,1.786390013568183e-06,4.6705901752228015e-12],"likelihood":[0.99989132081780097,0.37603120537811946,0.10185892947381846,0.013838943378262808,4.473720267625649e-06],"prior":[0.8804206791393453,0.11393044765503302,0.0055286696506684253,0.0001192391741739474,9.6438077937741234e-07]},{"sample":"sample10","allele":"locus1_allele1","posterior":[2.4187413517503758e-05,0.22268256222276989,0.62885153395282367,0.14843580556309305,5.910847795985849e-06],"likelihood":[7.0187900441975239e-05,0.10253050233729905,0.12251176195791912,0.02753047084059888,2.7831638699779889e-06],"prior":[0.022724083479307551,0.14321625809823665,0.33847729110292601,0.35553653261843071,0.14004583470109905]},{"sample":"sample10","allele":"locus1_allele2","posterior":[5.9108477959865402e-06,0.14843580556309302,0.62885153395282367,0.22268256222276989,2.4187413517505564e-05],"likelihood":[2.7831638699783154e-06,0.02753047084059888,0.12251176195791912,0.10253050233729905,7.018790044198047e-05],"prior":[0.14004583470109905,0.35553653261843077,0.33847729110292613,0.14321625809823671,0.022724083479307565]},{"sample":"sample10","allele":"locus2_allele1","posterior":[2.996487653083871e-07,0.038517375228985372,0.5800861982797193,0.38136757151822265,2.855532430737325e-05],"likelihood":[1.0735790950584249e-07,0.0032712088236178838,0.031141740421083461,0.029118928518641389,8.2692998389640008e-06],"prior":[0.05611064516046619,0.23670925107678942,0.37446951499863801,0.26329049860004528,0.06942009016406106]},{"sample":"sample10","allele":"locus2_allele2","posterior":[0.98709067048331389,0.01287932483830239,2.9995730262470113e-05,8.9481203410755662e-09,7.814395639878962e-16],"likelihood":[0.99485346124644669,0.0081351822421092835,3.1664634721264379e-05,3.551977701647869e-08,3.1104692629403044e-14],"prior":[0.26112528482196318,0.4166551991209192,0.24930784911281489,0.066299855249770806,0.0066118116945319255]},{"sample":"sample10","allele":"locus2_allele3","posterior":[8.2133532215998087e-05,0.70751537857175018,0.28730934537045749,0.0050931260627185716,1.6462857832431807e-08],"likelihood":[3.4076215790133059e-06,0.029072589502340922,0.03118051540213836,0.0032846279631886413,1.6824568830970221e-07],"prior":[0.40644389862556018,0.41037758915346811,0.15538100668589588,0.026147472069997697,0.0016500334650780288]},{"sample":"sample10","allele":"locus3_allele1","posterior":[5.2130284892379441e-08,0.012787334227928739,0.3902514251843634,0.59683058407827883,0.00013060437914409536],"likelihood":[1.6478295667505504e-08,0.0010655422352360828,0.022859795519410735,0.055296624761335483,5.1038010176630319e-05],"prior":[0.069394723337547384,0.26324370182863749,0.37447324821462663,0.23675605136026034,0.05613227525892811]},{"sample":"sample10","allele":"locus3_allele2","posterior":[0.98197856896025326,0.017961568301525562,5.9836638293501171e-05,2.6099924193361298e-08,3.54807519349243e-15],"likelihood":[0.99584284914642285,0.013165485496448718,8.4534184780197369e-05,1.5990406499351657e-07,2.5138356538172703e-13],"prior":[0.30476396056773508,0.42165745525519693,0.21876972744876036,0.050446628920665579,0.0043622278076420259]},{"sample":"sample10","allele":"locus3_allele3","posterior":[0.98484350362444117,0.015114233249948003,4.2247661870902057e-05,1.5463738110466246e-08,1.8403670066118599e-15],"likelihood":[0.99636080790472936,0.01317332989602904,8.4594259796302293e-05,1.6005306251627145e-07,2.6256326247326467e-13],"prior":[0.36090054371573521,0.41891653267222745,0.18234704588483627,0.035276658872380191,0.0025592188548208472]},{"sample":"sample10","allele":"locus3_allele4","posterior":[0.00042447762545719296,0.97981089271396027,0.019744689243436361,1.9940412155606335e-05,4.9904808451427955e-12],"likelihood":[3.0978078231309487e-06,0.055257643867733727,0.022946664491677409,0.0010744966720998162,3.324939524264786e-08],"prior":[0.8804206791393453,0.11393044765503302,0.0055286696506684253,0.0001192391741739474,9.6438077937741234e-07]},{"sample":"sample11","allele":"locus1_allele1","posterior":[3.0396357230730669e-06,0.085645549115737185,0.59894942106315174,0.3153749421692616,2.7048016126473589e-05],"likelihood":[9.1316288624553664e-06,0.040824940833766123,0.12080188222955371,0.060555840934527795,1.3184944576734451e-05],"prior":[0.022724083479307551,0.14321625809823665,0.33847729110292601,0.35553653261843071,0.14004583470109905]},{"sample":"sample11","allele":"locus1_allele2","posterior":[2.7048016126476849e-05,0.31537494216926149,0.59894942106315163,0.085645549115737185,3.0396357230732939e-06],"likelihood":[1.3184944576736043e-05,0.060555840934527795,0.12080188222955371,0.040824940833766123,9.1316288624560474e-06],"prior":[0.14004583470109905,0.35553653261843077,0.33847729110292613,0.14321625809823671,0.022724083479307565]},{"sample":"sample11","allele":"locus2_allele1","posterior":[7.5803811683748186e-06,0.21088851969032973,0.69127747270674755,0.097824895284582103,1.5319371722488689e-06],"likelihood":[2.8128028900654571e-06,0.018549458016766572,0.038435227318651378,0.0077358443443263621,4.5946168273748054e-07],"prior":[0.05611064516046619,0.23670925107678942,0.37446951499863801,0.26329049860004528,0.06942009016406106]},{"sample":"sample11","allele":"locus2_allele2","posterior":[4.987844031816063e-06,0.22925010379897703,0.68291724764783379,0.087825806744173118,1.8539649843267417e-06],"likelihood":[2.6792175289325054e-07,0.007717502403532102,0.03842166378348115,0.018580337799934968,3.9330103470222302e-06],"prior":[0.26112528482196318,0.4166551991209192,0.24930784911281489,0.066299855249770806,0.0066118116945319255]},{"sample":"sample11","allele":"locus2_allele3","posterior":[0.99149618493183067,0.0084908144895360881,1.2998025499553961e-05,2.553133473624192e-09,1.6428540197528157e-16],"likelihood":[0.99638846231125489,0.008450919553231467,3.4167854165897775e-05,3.9882471435612257e-08,4.0667218888415973e-14],"prior":[0.40644389862556018,0.41037758915346811,0.15538100668589588,0.026147472069997697,0.0016500334650780288]},{"sample":"sample11","allele":"locus3_allele1","posterior":[1.1254021777266706e-07,0.021599083246832084,0.48196975890875338,0.49636520902746201,6.5836276734760268e-05],"likelihood":[3.2454124476470817e-08,0.0016419726690818624,0.025756543844190193,0.041955494134220485,2.3471519768865632e-05],"prior":[0.069394723337547384,0.26324370182863749,0.37447324821462663,0.23675605136026034,0.05613227525892811]},{"sample":"sample11","allele":"locus3_allele2","posterior":[0.00015245353359014227,0.75516578082816077,0.24112821671263618,0.0035535396270272236,9.2985856691098529e-09],"likelihood":[1.1709016771209349e-05,0.041920751630034239,0.02579926601121918,0.0016488287625471474,4.9894778232986398e-08],"prior":[0.30476396056773508,0.42165745525519693,0.21876972744876036,0.050446628920665579,0.0043622278076420259]},{"sample":"sample11","allele":"locus3_allele3","posterior":[0.98865187447740177,0.011324791943397951,2.3327374224872206e-05,6.2049748861734248e-09,5.2851171129361017e-16],"likelihood":[0.99610379876226962,0.009829953655318564,4.6517446107902818e-05,6.3958949379877122e-08,7.5092390017124615e-14],"prior":[0.36090054371573521,0.41891653267222745,0.18234704588483627,0.035276658872380191,0.0025592188548208472]},{"sample":"sample11","allele":"locus3_allele4","posterior":[0.9987237257402104,0.0012759810303799328,2.932207028893112e-07,8.7068724251834174e-12,1.0352400387089077e-19],"likelihood":[0.99945626625504336,0.0098676350017618802,4.6728587389839707e-05,6.4335678324257418e-08,9.4580406654822408e-14],"prior":[0.8804206791393453,0.11393044765503302,0.0055286696506684253,0.0001192391741739474,9.6438077937741234e-07]},{"sample":"sample12","allele":"locus1_allele1","posterior":[4.9454281766536007e-15,3.9969911105537527e-08,8.1034809869272919e-05,0.021067307835387669,0.97885161738482707],"likelihood":[3.0920303059058644e-14,3.9652180911898288e-08,3.4014829620047048e-05,0.0084188161910594191,0.99305404649337092],"prior":[0.022724083479307551,0.14321625809823665,0.33847729110292601,0.35553653261843071,0.14004583470109905]},{"sample":"sample12","allele":"locus1_allele2","posterior":[0.97885161738482707,0.021067307835387652,8.1034809869272865e-05,3.9969911105537646e-08,4.9454281766539841e-15],"likelihood":[0.99305404649337181,0.0084188161910594191,3.4014829620047048e-05,3.9652180911898434e-08,3.0920303059061055e-14],"prior":[0.14004583470109905,0.35553653261843077,0.33847729110292613,0.14321625809823671,0.022724083479307565]},{"sample":"sample12","allele":"locus2_allele1","posterior":[2.2828118044082774e-05,0.30936321209193102,0.62379682654067881,0.066816170753880008,9.6249546607364185e-07],"likelihood":[1.401051324705391e-05,0.045007266257545725,0.057366174617841027,0.0087392787776972135,4.7746629187206499e-07],"prior":[0.05611064516046619,0.23670925107678942,0.37446951499863801,0.26329049860004528,0.06942009016406106]},{"sample":"sample12","allele":"locus2_allele2","posterior":[4.1263200052570986e-05,0.49427988310818571,0.48102640187804441,0.024652202677194091,2.4913652312846266e-07],"likelihood":[5.0382949726059467e-06,0.037823850759826749,0.061518051805668834,0.011855305799796554,1.2013969758039384e-06],"prior":[0.26112528482196318,0.4166551991209192,0.24930784911281489,0.066299855249770806,0.0066118116945319255]},{"sample":"sample12","allele":"locus2_allele3","posterior":[0.0007503138747751003,0.91488493429223527,0.083891439563164749,0.00047331162973383823,6.4009102467813061e-10],"likelihood":[6.8337454238586259e-05,0.082527609838091293,0.019986472167536785,0.00067009116971357646,1.4360362806055419e-08],"prior":[0.40644389862556018,0.41037758915346811,0.15538100668589588,0.026147472069997697,0.0016500334650780288]},{"sample":"sample12","allele":"locus3_allele1","posterior":[1.4080325241834647e-05,0.25658730262159196,0.65450707072190084,0.088889814067201728,1.732264063534927e-06],"likelihood":[7.5184275983145214e-06,0.036117522806520239,0.064764099099886077,0.01391206805405727,1.1435159680984799e-06],"prior":[0.069394723337547384,0.26324370182863749,0.37447324821462663,0.23675605136026034,0.05613227525892811]},{"sample":"sample12","allele":"locus3_allele2","posterior":[1.8278647126524117e-05,0.38064215636747317,0.57359520143367382,0.045743599743282008,7.6380844441191501e-07],"likelihood":[1.5540180149756303e-06,0.023390134045946649,0.067935048976263776,0.023494910614527291,4.5368218163954186e-06],"prior":[0.30476396056773508,0.42165745525519693,0.21876972744876036,0.050446628920665579,0.0043622278076420259]},{"sample":"sample12","allele":"locus3_allele3","posterior":[0.97069090388143919,0.029143618926833575,0.00016534918567971305,1.2800601382047501e-07,3.374750485101866e-14],"likelihood":[0.99694773503655099,0.025786651249777005,0.00033611066715864298,1.3449987959939889e-06,4.8877975474690927e-12],"prior":[0.36090054371573521,0.41891653267222745,0.18234704588483627,0.035276658872380191,0.0025592188548208472]},{"sample":"sample12","allele":"locus3_allele4","posterior":[0.010069107634293199,0.98751741082466571,0.0024129740508481794,5.0749015547476131e-07,3.7396756720194111e-14],"likelihood":[8.7247426068015464e-05,0.066123652117098225,0.0033295373079688566,3.2468373802912412e-05,2.9582663114640412e-10],"prior":[0.8804206791393453,0.11393044765503302,0.0055286696506684253,0.0001192391741739474,9.6438077937741234e-07]},{"sample":"sample13","allele":"locus1_allele1","posterior":[3.842276084436364e-06,0.017505795752715207,0.14499992238295017,0.3699769251193406,0.35976656067670337,0.10770970319502632,3.7250597179926146e-05],"likelihood":[0.00025134310157242355,0.12113316199240449,0.25471997467602864,0.30937497420498022,0.25457990338640057,0.12093505478204077,0.00015927070825520469],"prior":[0.0034255441087625267,0.032383722515420287,0.12755947798107395,0.26797694082358503,0.31666833854086351,0.19957704584787972,0.052408930182414923]},{"sample":"sample13","allele":"locus1_allele2","posterior":[3.7250597179930612e-05,0.10770970319502624,0.35976656067670348,0.36997692511934038,0.14499992238295015,0.017505795752715329,3.8422760844366503e-06],"likelihood":[0.00015927070825522393,0.12093505478204077,0.2545799033864008,0.30937497420498022,0.25471997467602864,0.12113316199240536,0.00025134310157244225],"prior":[0.052408930182414923,0.19957704584787975,0.31666833854086363,0.26797694082358514,0.12755947798107403,0.032383722515420307,0.0034255441087625298]},{"sample":"sample13","allele":"locus2_allele1","posterior":[6.3671490985748639e-06,0.062032986652550429,0.40240228256415456,0.43218241013727676,0.099615360205900977,0.0037605056145593708,8.7676459321263299e-08],"likelihood":[1.2683885717661168e-05,0.019528494497913551,0.048045920845099087,0.036695639332009054,0.010693070743575707,0.0009568688189934046,1.2692010918027e-07],"prior":[0.013291313243591871,0.084106414960659084,0.22175790394981931,0.3118371283155652,0.24666019107342782,0.10405645654665609,0.018290591910280516]},{"sample":"sample13","allele":"locus2_allele2","posterior":[4.9894401532263148e-06,0.077817926605401946,0.43877935174271815,0.4025161271632246,0.078392521136232116,0.002489012272487373,7.1639782563423978e-08],"likelihood":[7.4141895238540617e-07,0.0048313931926839875,0.027316874911672982,0.047115249276558843,0.03067070005014165,0.0061030681034639775,2.6421580947774319e-06],"prior":[0.13343611532950808,0.31936883040899933,0.31849339034182955,0.16939751999638616,0.050679952445628472,0.0080865649192375655,0.00053762655841083844]},{"sample":"sample13","allele":"locus2_allele3","posterior":[0.00076097710529538138,0.75288541677366527,0.23263754461614342,0.013529530956873533,0.00018607874209617309,4.5180475099357937e-07,1.1752620789048058e-12],"likelihood":[0.00010736855305113574,0.070139086978812992,0.034343771341160644,0.0059345684943093325,0.0004311405467286722,1.0367877870420082e-05,6.4106492701236166e-10],"prior":[0.25911998670695491,0.39244174585669966,0.24764994949557984,0.08334892759012863,0.01577917583396693,0.0015931891607844132,6.7025355885428904e-05]},{"sample":"sample13","allele":"locus3_allele1","posterior":[4.7219378018531503e-07,0.012111408778970798,0.19197445249302231,0.49690406748529781,0.2742172637573988,0.024790865579457547,1.4697120725299401e-06],"likelihood":[7.6225171837210663e-07,0.0034359751373577328,0.02297138759661051,0.047022537104985572,0.036483415245871675,0.0086948249537839046,3.2612276451664259e-06],"prior":[0.018280567465997589,0.10401895172641176,0.24661757944118809,0.31184179154531344,0.22180285377740847,0.084139256554869674,0.013298999488810989]},{"sample":"sample13","allele":"locus3_allele2","posterior":[0.00061734927342764934,0.67780535160875777,0.29663227180935298,0.024465585402223697,0.00047779272633147527,1.6491742223792314e-06,5.6840347679095325e-12],"likelihood":[0.00013384552385378604,0.070809327586906001,0.035836672907157266,0.0064089956502575568,0.00048247681920436002,1.2036695710278249e-05,7.1963542943688451e-10],"prior":[0.16824625779244318,0.34916672291967521,0.30193192703100241,0.13924639446106107,0.036122799899306945,0.0049977851232344784,0.00028811277327673657]},{"sample":"sample13","allele":"locus3_allele3","posterior":[0.00036310791366428192,0.62005349903610507,0.34444184630714497,0.034355050939039622,0.00078341973296165018,3.0760587541722337e-06,1.2330244634230017e-11],"likelihood":[5.9705964578281891e-05,0.058557151779142676,0.044838049191393482,0.011558532226665174,0.0012110591683930914,4.0966184624479672e-05,3.3952664531343579e-09],"prior":[0.21681099599490464,0.377496150680242,0.27386246241124884,0.10596226133033353,0.023061760305341499,0.0026769016651970352,0.00012946761273240233]},{"sample":"sample13","allele":"locus3_allele4","posterior":[0.0011377790716417626,0.95922288850020199,0.039334977974486873,0.00030379867939110092,5.5559456188932021e-07,1.7971635792802278e-10,7.5628999048716151e-17],"likelihood":[1.6301408876159051e-05,0.070802061744275988,0.035898449901987393,0.0064276815475002164,0.00048447973430727997,1.2110321384043338e-05,9.4518775121802389e-10],"prior":[0.82610519248716263,0.16035266427785724,0.012968978393454194,0.00055941496642986489,1.3573283572075301e-05,1.7564447430141435e-07,9.4704980525403814e-10]},{"sample":"sample14","allele":"locus1_allele1","posterior":[5.7873501075272139e-06,0.030804058503707515,0.2287034708713003,0.4366390904638765,0.26494963741509148,0.038893630329292966,4.3250666236706689e-06],"likelihood":[0.00018457020644109819,0.10391842983622665,0.19587158084070078,0.1780068007233071,0.091405029287615347,0.021290178285685857,9.0156888160115222e-06],"prior":[0.0034255441087625267,0.032383722515420287,0.12755947798107395,0.26797694082358503,0.31666833854086351,0.19957704584787972,0.052408930182414923]},{"sample":"sample14","allele":"locus1_allele2","posterior":[4.325066623671194e-06,0.038893630329293001,0.26494963741509103,0.43663909046387694,0.22870347087130022,0.030804058503707613,5.7873501075276543e-06],"likelihood":[9.0156888160126115e-06,0.021290178285685857,0.091405029287615111,0.1780068007233071,0.19587158084070044,0.10391842983622683,0.00018457020644111196],"prior":[0.052408930182414923,0.19957704584787975,0.31666833854086363,0.26797694082358514,0.12755947798107403,0.032383722515420307,0.0034255441087625298]},{"sample":"sample14","allele":"locus2_allele1","posterior":[1.8310181265561705e-07,0.0062189907521989583,0.13519385086996036,0.47197369719388654,0.34574989352219526,0.040860426117792628,2.9584421536556741e-06],"likelihood":[3.6115185460355981e-07,0.0019384557362099122,0.015982434370961262,0.039678484214310715,0.036747524703157765,0.010294350662255272,4.2403374635444507e-06],"prior":[0.013291313243591871,0.084106414960659084,0.22175790394981931,0.3118371283155652,0.24666019107342782,0.10405645654665609,0.018290591910280516]},{"sample":"sample14","allele":"locus2_allele2","posterior":[1.4659876223905714e-05,0.14557089397097467,0.5191734751814211,0.29852376079868465,0.036018846061256882,0.00069835204951768039,1.2061921109242232e-08],"likelihood":[2.4749046551833029e-06,0.010267946102556278,0.036720926581889246,0.03969843766563054,0.016010140957549414,0.00194541335857806,5.0540243303437037e-07],"prior":[0.13343611532950808,0.31936883040899933,0.31849339034182955,0.16939751999638616,0.050679952445628472,0.0080865649192375655,0.00053762655841083844]},{"sample":"sample14","allele":"locus2_allele3","posterior":[0.92908069025299855,0.069283694552958741,0.0016216173299497762,1.3958061317078425e-05,3.9778197412906652e-08,2.4578281321443811e-11,1.8673177592642085e-17],"likelihood":[0.99650800690571295,0.049066390476195661,0.0018198617815571834,4.6542891106852427e-05,7.0063052458717943e-07,4.2875820016816772e-09,7.7429663232102513e-14],"prior":[0.25911998670695491,0.39244174585669966,0.24764994949557984,0.08334892759012863,0.01577917583396693,0.0015931891607844132,6.7025355885428904e-05]},{"sample":"sample14","allele":"locus3_allele1","posterior":[2.3659882247875627e-06,0.030044548869570697,0.27568678492161064,0.4767557242038144,0.20152679632270765,0.015982823316778774,9.5637729303002008e-07],"likelihood":[7.7819116282917311e-06,0.017366672053336703,0.067213313604975775,0.09192316901448605,0.054629730757736084,0.011421370422448366,4.3238804294380307e-06],"prior":[0.018280567465997589,0.10401895172641176,0.24661757944118809,0.31184179154531344,0.22180285377740847,0.084139256554869674,0.013298999488810989]},{"sample":"sample14","allele":"locus3_allele2","posterior":[0.00065212205690187426,0.63278145624259563,0.32680936016021411,0.038505664064234446,0.0012433665531223661,8.0308649462570306e-06,5.7985266407878707e-11],"likelihood":[0.00028086328531345853,0.13132045893208474,0.078432659788445755,0.02003789764953413,0.0024941884873285612,0.0001164384022898641,1.4583663638130186e-08],"prior":[0.16824625779244318,0.34916672291967521,0.30193192703100241,0.13924639446106107,0.036122799899306945,0.0049977851232344784,0.00028811277327673657]},{"sample":"sample14","allele":"locus3_allele3","posterior":[5.4948936975139738e-05,0.28700439828151098,0.51787534600439933,0.17947482596072492,0.015344143457411972,0.00024633289774795859,4.4612296773932016e-09],"likelihood":[1.2036791139737461e-05,0.036108442746480195,0.089810158305102208,0.080442407001708849,0.031599697883357612,0.0043704155431575241,1.6365383118111538e-06],"prior":[0.21681099599490464,0.377496150680242,0.27386246241124884,0.10596226133033353,0.023061760305341499,0.0026769016651970352,0.00012946761273240233]},{"sample":"sample14","allele":"locus3_allele4","posterior":[0.97330057789114233,0.026463359573168987,0.00023525751481173418,8.0406553491973937e-07,9.550825489954284e-10,2.5947081674718265e-13,1.1139189426910874e-19],"likelihood":[0.99964926268350063,0.14002475400342032,0.015391245267322105,0.0012195325226824045,5.9702430000552687e-05,1.2534010319980862e-06,9.9796840254534776e-11],"prior":[0.82610519248716263,0.16035266427785724,0.012968978393454194,0.00055941496642986489,1.3573283572075301e-05,1.7564447430141435e-07,9.4704980525403814e-10]},{"sample":"sample15","allele":"locus1_allele1","posterior":[2.2948452383415652e-06,0.018304714653081415,0.18111792321595716,0.43211386603931651,0.31458504981265262,0.053869305957622335,6.8454761316975577e-06],"likelihood":[7.3419723753696768e-05,0.061947687844031603,0.15561003630361817,0.17672162046045192,0.10887353989453252,0.029581466337326431,1.4314865557062959e-05],"prior":[0.0034255441087625267,0.032383722515420287,0.12755947798107395,0.26797694082358503,0.31666833854086351,0.19957704584787972,0.052408930182414923]},{"sample":"sample15","allele":"locus1_allele2","posterior":[6.8454761316983895e-06,0.053869305957622377,0.31458504981265212,0.4321138660393169,0.18111792321595707,0.018304714653081502,2.2948452383417397e-06],"likelihood":[1.4314865557064689e-05,0.029581466337326431,0.10887353989453223,0.17672162046045192,0.15561003630361789,0.061947687844031825,7.3419723753702243e-05],"prior":[0.052408930182414923,0.19957704584787975,0.31666833854086363,0.26797694082358514,0.12755947798107403,0.032383722515420307,0.0034255441087625298]},{"sample":"sample15","allele":"locus2_allele1","posterior":[1.0911988848623991e-08,0.00079918203177964831,0.038206207108299854,0.30083957364711689,0.51317640738417181,0.14695152781003598,2.7091106606937154e-05],"likelihood":[1.9188594008715228e-08,0.00022208729960563104,0.0040268175398654339,0.022548324495360771,0.048626694948373697,0.033007467703476207,3.4618330454881029e-05],"prior":[0.013291313243591871,0.084106414960659084,0.22175790394981931,0.3118371283155652,0.24666019107342782,0.10405645654665609,0.018290591910280516]},{"sample":"sample15","allele":"locus2_allele2","posterior":[8.9816924755236818e-05,0.35025223731609811,0.51550037219873246,0.12728851719994563,0.0068090261098239322,6.0029769821699358e-05,4.8082301108058757e-10],"likelihood":[2.0228821688164826e-05,0.0329590120654576,0.048642328278860607,0.022582289702612141,0.0040377050955808531,0.00022309425965229146,2.6877589981780602e-08],"prior":[0.13343611532950808,0.31936883040899933,0.31849339034182955,0.16939751999638616,0.050679952445628472,0.0080865649192375655,0.00053762655841083844]},{"sample":"sample15","allele":"locus2_allele3","posterior":[0.92543117180207479,0.072754626046270282,0.0017977908225932095,1.6361914871227066e-05,4.9381821981863325e-08,3.2368478684144716e-11,2.613474975992704e-17],"likelihood":[0.99657088683349626,0.051730943366233365,0.0020256569570015481,5.4777107667668307e-05,8.7326846688827099e-07,5.6691758654895327e-09,1.0880383470775897e-13],"prior":[0.25911998670695491,0.39244174585669966,0.24764994949557984,0.08334892759012863,0.01577917583396693,0.0015931891607844132,6.7025355885428904e-05]},{"sample":"sample15","allele":"locus3_allele1","posterior":[7.0848020852843301e-07,0.016612366083803454,0.23169294712007707,0.50767584192577997,0.22793278606532658,0.016084637955771031,7.1236903327350964e-07],"likelihood":[8.7378444241828043e-07,0.0036006862350204359,0.021181418506239065,0.036704426197050631,0.023168927341618944,0.0043100159547763877,1.2076812433579772e-06],"prior":[0.018280567465997589,0.10401895172641176,0.24661757944118809,0.31184179154531344,0.22180285377740847,0.084139256554869674,0.013298999488810989]},{"sample":"sample15","allele":"locus3_allele2","posterior":[0.9221913990220606,0.075843079987602008,0.0019471839487247185,1.8280523179484959e-05,5.6480841642199165e-08,3.7591619099651625e-11,2.8442860884936542e-17],"likelihood":[0.9952198846213588,0.039439055435340799,0.0011709580392059813,2.38368054950308e-05,2.8389853069538912e-07,1.3657048197593603e-09,1.7924793240379019e-14],"prior":[0.16824625779244318,0.34916672291967521,0.30193192703100241,0.13924639446106107,0.036122799899306945,0.0049977851232344784,0.00028811277327673657]},{"sample":"sample15","allele":"locus3_allele3","posterior":[9.2384038943018151e-06,0.13123895898393242,0.51318781889223475,0.31512911504533381,0.039650371809742957,0.000784482176385354,1.4688476299338608e-08],"likelihood":[5.2593228140614649e-07,0.0042910585159901381,0.023129061888116094,0.036707220406404006,0.021221154039207383,0.0036171361040587698,1.4003280697370798e-06],"prior":[0.21681099599490464,0.377496150680242,0.27386246241124884,0.10596226133033353,0.023061760305341499,0.0026769016651970352,0.00012946761273240233]},{"sample":"sample15","allele":"locus3_allele4","posterior":[0.99234570279695666,0.0076359368931419947,1.8344187943369424e-05,1.6117296005671246e-08,4.6617292733882117e-12,2.9070120229344836e-16,2.6852029079461642e-23],"likelihood":[0.99941593017862651,0.039619115576536031,0.0011768235499953642,2.3970479799199821e-05,2.8574638991209186e-07,1.3769905584389258e-09,2.358972705253392e-14],"prior":[0.82610519248716263,0.16035266427785724,0.012968978393454194,0.00055941496642986489,1.3573283572075301e-05,1.7564447430141435e-07,9.4704980525403814e-10]},{"sample":"sample16","allele":"locus1_allele1","posterior":[2.4897727935375407e-10,2.3768729296108713e-05,0.0022610066517223084,0.04759699816285913,0.31836786002996537,0.62981540979618855,0.0019349563809911742],"likelihood":[4.5697706560328468e-09,4.6146997201570947e-05,0.0011144314467492757,0.011167253896578532,0.063210441899814312,0.19841154740896083,0.0023212939335088396],"prior":[0.0034255441087625267,0.032383722515420287,0.12755947798107395,0.26797694082358503,0.31666833854086351,0.19957704584787972,0.052408930182414923]},{"sample":"sample16","allele":"locus1_allele2","posterior":[0.0019349563809914032,0.62981540979618822,0.31836786002996553,0.047596998162859123,0.0022610066517223157,2.3768729296108838e-05,2.4897727935377356e-10],"likelihood":[0.0023212939335091158,0.19841154740896083,0.063210441899814368,0.011167253896578532,0.0011144314467492796,4.6146997201571191e-05,4.5697706560332042e-09],"prior":[0.052408930182414923,0.19957704584787975,0.31666833854086363,0.26797694082358514,0.12755947798107403,0.032383722515420307,0.0034255441087625298]},{"sample":"sample16","allele":"locus2_allele1","posterior":[3.4580496493820535e-06,0.036720675112795691,0.3012953327372665,0.46964992555907603,0.17948107536125887,0.012848879078319931,6.5410163361448313e-07],"likelihood":[1.5193023337268783e-05,0.025495434016370635,0.079340360619047867,0.087948271681689477,0.042491344511311126,0.0072106942879575425,2.088326250132734e-06],"prior":[0.013291313243591871,0.084106414960659084,0.22175790394981931,0.3118371283155652,0.24666019107342782,0.10405645654665609,0.018290591910280516]},{"sample":"sample16","allele":"locus2_allele2","posterior":[4.654263892413388e-06,0.065719796188261875,0.38685700063253753,0.42632131643304982,0.11518143242066974,0.0059154733433887611,3.2671819973370495e-07],"likelihood":[1.2184595793133074e-06,0.0071884820570360369,0.042431027570064894,0.087915107874688467,0.079392556989450844,0.02555400274924326,2.1228833258068001e-05],"prior":[0.13343611532950808,0.31936883040899933,0.31849339034182955,0.16939751999638616,0.050679952445628472,0.0080865649192375655,0.00053762655841083844]},{"sample":"sample16","allele":"locus2_allele3","posterior":[0.81486191919070394,0.1728308614119306,0.011984777245236529,0.00031947682569487577,2.9590618364283876e-06,6.2645804428324027e-09,1.7282691736129891e-14],"likelihood":[0.99774627090083301,0.13972788937357514,0.015354254194841517,0.0012161197676099787,5.9498597497610923e-05,1.2475606007478657e-06,8.1810539868507352e-11],"prior":[0.25911998670695491,0.39244174585669966,0.24764994949557984,0.08334892759012863,0.01577917583396693,0.0015931891607844132,6.7025355885428904e-05]},{"sample":"sample16","allele":"locus3_allele1","posterior":[1.5312642028685849e-05,0.052773867208157466,0.26326571684216171,0.40444583565203496,0.23679129890394893,0.042696220070068763,1.1748681599468442e-05],"likelihood":[0.00019981110493927652,0.12102235833500986,0.25464165103634168,0.30937499963713211,0.25465826432018485,0.12104585502291025,0.00021073143732595743],"prior":[0.018280567465997589,0.10401895172641176,0.24661757944118809,0.31184179154531344,0.22180285377740847,0.084139256554869674,0.013298999488810989]},{"sample":"sample16","allele":"locus3_allele2","posterior":[0.37597104890267202,0.41787483139567544,0.17196232415913351,0.031678883734846973,0.0024568807286317507,5.6028875921328152e-05,2.203118981305998e-09],"likelihood":[0.99910898638258694,0.53507818088010961,0.25464088898928067,0.10171612005586357,0.030409308384721907,0.0050123132398217259,3.418844229738376e-06],"prior":[0.16824625779244318,0.34916672291967521,0.30193192703100241,0.13924639446106107,0.036122799899306945,0.0049977851232344784,0.00028811277327673657]},{"sample":"sample16","allele":"locus3_allele3","posterior":[0.00012950133133855974,0.29552763229893148,0.45168471754520717,0.21246618988985908,0.038087583876192502,0.0021041075357533063,2.6752271787248921e-07],"likelihood":[9.2159331030406398e-05,0.12079026992580714,0.25447743163460562,0.30937484367401369,0.25482225501522682,0.12127796438404706,0.00031882026088305543],"prior":[0.21681099599490464,0.377496150680242,0.27386246241124884,0.10596226133033353,0.023061760305341499,0.0026769016651970352,0.00012946761273240233]},{"sample":"sample16","allele":"locus3_allele4","posterior":[0.90249380121808775,0.093831294269157844,0.0036121945537031172,6.225715429692216e-05,4.5183978018956317e-07,9.6496952502566646e-10,4.6291123484625862e-15],"likelihood":[0.99989132081780097,0.53556902287586139,0.25492315789990283,0.10185892947381846,0.030467966140603986,0.0050283214023546952,4.473720267625649e-06],"prior":[0.82610519248716263,0.16035266427785724,0.012968978393454194,0.00055941496642986489,1.3573283572075301e-05,1.7564447430141435e-07,9.4704980525403814e-10]}]},"iterated":{"allele_frequency":[0.61429761701774155,0.38570238298225856,0.49280332847609504,0.26388398200484564,0.22552857001995399,0.50375999654593473,0.27895643509565604,0.19810719327512916,0.056601571518492044],"records":[{"sample":"sample01","allele":"locus1_allele1","posterior":[7.9100318469745252e-11,0.0025798878868182619,0.99742011203408143]},{"sample":"sample01","allele":"locus1_allele2","posterior":[0.99742011203408143,0.002579887886818258,7.9100318469746945e-11]},{"sample":"sample01","allele":"locus2_allele1","posterior":[2.1997349359279874e-05,0.99996873425804866,9.2683925919444443e-06]},{"sample":"sample01","allele":"locus2_allele2","posterior":[1.8175310396702283e-05,0.99997312792592197,8.6967636812744744e-06]},{"sample":"sample01","allele":"locus2_allele3","posterior":[0.9632027054752047,0.036797294514244641,1.0550637073963406e-11]},{"sample":"sample01","allele":"locus3_allele1","posterior":[5.1838429448380185e-05,0.9997774026024715,0.00017075896808015672]},{"sample":"sample01","allele":"locus3_allele2","posterior":[0.00024369102165784257,0.99972708023407286,2.9228744269232987e-05]},{"sample":"sample01","allele":"locus3_allele3","posterior":[0.99179414685758427,0.0082058511035856627,2.0388301498052946e-09]},{"sample":"sample01","allele":"locus3_allele4","posterior":[0.99799354066148283,0.002006459195804645,1.4271260411439411e-10]},{"sample":"sample02","allele":"locus1_allele1","posterior":[9.0825967957213844e-15,3.7136799568452272e-05,0.99996286320042249]},{"sample":"sample02","allele":"locus1_allele2","posterior":[0.99996286320042249,3.7136799568452204e-05,9.0825967957215816e-15]},{"sample":"sample02","allele":"locus2_allele1","posterior":[5.6738599698551145e-12,0.00089374937470883189,0.99910625061961733]},{"sample":"sample02","allele":"locus2_allele2","posterior":[0.99968809530406366,0.00031190469490201982,1.0342817907860111e-12]},{"sample":"sample02","allele":"locus2_allele3","posterior":[0.99974656324941302,0.00025343674986838294,7.1869656977176485e-13]},{"sample":"sample02","allele":"locus3_allele1","posterior":[0.00010313584915252795,0.99989422503852365,2.6391123238331854e-06]},{"sample":"sample02","allele":"locus3_allele2","posterior":[7.2754733828470883e-06,0.99996374589180959,2.8978634807500832e-05]},{"sample":"sample02","allele":"locus3_allele3","posterior":[0.99997047125186356,2.9528748128432975e-05,7.9726169137059095e-15]},{"sample":"sample02","allele":"locus3_allele4","posterior":[0.70752796037227617,0.29247203952845319,9.9270615337158801e-11]},{"sample":"sample03","allele":"locus1_allele1","posterior":[0.14876632841200355,0.47387210959114423,0.37736156199685222]},{"sample":"sample03","allele":"locus1_allele2","posterior":[0.37736156199685222,0.47387210959114429,0.14876632841200357]},{"sample":"sample03","allele":"locus2_allele1","posterior":[0.99667680717300311,0.0033231927229979579,1.039989907382134e-10]},{"sample":"sample03","allele":"locus2_allele2","posterior":[0.99876987714936916,0.0012301228299428557,2.0687983257124974e-11]},{"sample":"sample03","allele":"locus2_allele3","posterior":[5.6948582714082271e-10,0.011630447272433187,0.98836955215808098]},{"sample":"sample03","allele":"locus3_allele1","posterior":[0.99981130339434621,0.00018869660544409123,2.097189855497646e-13]},{"sample":"sample03","allele":"locus3_allele2","posterior":[1.1994933290470692e-05,0.99996788689367533,2.0118173034261635e-05]},{"sample":"sample03","allele":"locus3_allele3","posterior":[5.7197272484240224e-05,0.9999394705715603,3.3321559554198387e-06]},{"sample":"sample03","allele":"locus3_allele4","posterior":[0.99998881612709334,1.1183872905246598e-05,1.4070502050163699e-15]},{"sample":"sample04","allele":"locus1_allele1","posterior":[0.99956341906917312,0.00043658092991756531,9.091918693014366e-13]},{"sample":"sample04","allele":"locus1_allele2","posterior":[9.0919186930146144e-13,0.00043658092991756266,0.99956341906917312]},{"sample":"sample04","allele":"locus2_allele1","posterior":[0.99934627254391284,0.00065372745307816049,3.0090112315067685e-12]},{"sample":"sample04","allele":"locus2_allele2","posterior":[0.99975839480697293,0.00024160519242949927,5.9771904845898661e-13]},{"sample":"sample04","allele":"locus2_allele3","posterior":[1.6582582619391201e-11,0.002303008025031751,0.99769699195838568]},{"sample":"sample04","allele":"locus3_allele1","posterior":[5.3147138730736267e-12,0.00085550834127548036,0.99914449165340991]},{"sample":"sample04","allele":"locus3_allele2","posterior":[0.99966342425550814,0.0003365757433123034,1.1795512826871771e-12]},{"sample":"sample04","allele":"locus3_allele3","posterior":[0.99978494951625108,0.00021505048321293708,5.3593042355094048e-13]},{"sample":"sample04","allele":"locus3_allele4","posterior":[0.99994772390845355,5.227609150922595e-05,3.7307523410837042e-14]},{"sample":"sample05","allele":"locus1_allele1","posterior":[3.7111208581270443e-07,0.084731108369602279,0.91488560395331409,0.00038291656499783976]},{"sample":"sample05","allele":"locus1_allele2","posterior":[0.00038291656499785115,0.91488560395331375,0.084731108369602542,3.711120858127129e-07]},{"sample":"sample05","allele":"locus2_allele1","posterior":[4.7902770745483247e-11,0.00094171769865652435,0.83712841754344447,0.16192986470999621]},{"sample":"sample05","allele":"locus2_allele2","posterior":[0.22408432986385959,0.77561117550050351,0.00030449462755215335,8.0847114374273057e-12]},{"sample":"sample05","allele":"locus2_allele3","posterior":[0.99790339867992772,0.0020962868917449918,3.1442832286570896e-07,4.4561678443866295e-15]},{"sample":"sample05","allele":"locus3_allele1","posterior":[0.00011677545365810123,0.59620319513590847,0.40364989029382531,3.0139116608083392e-05]},{"sample":"sample05","allele":"locus3_allele2","posterior":[0.0044493842728137774,0.95080778635931718,0.044742491555048412,3.3781282068576919e-07]},{"sample":"sample05","allele":"locus3_allele3","posterior":[0.00027092820065290708,0.85805533079914753,0.14166956958131374,4.1714188858567537e-06]},{"sample":"sample05","allele":"locus3_allele4","posterior":[0.97821909757092906,0.021721352915601289,5.9549472800013531e-05,4.0669700802744897e-11]},{"sample":"sample06","allele":"locus1_allele1","posterior":[2.1322561778575797e-05,0.55130706491586468,0.44866468102006118,6.931502295610222e-06]},{"sample":"sample06","allele":"locus1_allele2","posterior":[6.9315022956104634e-06,0.44866468102005946,0.55130706491586634,2.1322561778576322e-05]},{"sample":"sample06","allele":"locus2_allele1","posterior":[6.1926288464653046e-08,0.058073824524537236,0.94163411027201904,0.00029200327715521771]},{"sample":"sample06","allele":"locus2_allele2","posterior":[0.99854731238315531,0.0014525414965065144,1.4612033694074276e-07,1.2736006137589704e-15]},{"sample":"sample06","allele":"locus2_allele3","posterior":[0.00047671229573056196,0.98232385953140511,0.017199419982184612,8.1906797312698996e-09]},{"sample":"sample06","allele":"locus3_allele1","posterior":[1.8267320916449044e-07,0.074479344557348212,0.9250834788778699,0.00043699389157265229]},{"sample":"sample06","allele":"locus3_allele2","posterior":[0.00065317232661118833,0.96865015718761238,0.030696628034946057,4.2450830288489621e-08]},{"sample":"sample06","allele":"locus3_allele3","posterior":[0.99241172060696414,0.0075836277252168708,4.6516674985678987e-06,3.2049525365503666e-13]},{"sample":"sample06","allele":"locus3_allele4","posterior":[0.99814654593589502,0.0018531777688993631,2.7629520010415335e-07,5.4484669330024785e-15]},{"sample":"sample07","allele":"locus1_allele1","posterior":[1.3905562485362197e-05,0.47242761145809592,0.5275466120777792,1.1870901639553866e-05]},{"sample":"sample07","allele":"locus1_allele2","posterior":[1.1870901639554266e-05,0.52754661207777653,0.47242761145809853,1.3905562485362522e-05]},{"sample":"sample07","allele":"locus2_allele1","posterior":[2.0218533746126409e-06,0.28094513843839036,0.71901690418917164,3.5935519063357862e-05]},{"sample":"sample07","allele":"locus2_allele2","posterior":[6.5445052622416496e-05,0.87984427347959471,0.12008983015846499,4.5130931799987448e-07]},{"sample":"sample07","allele":"locus2_allele3","posterior":[0.99749552165526412,0.0025040244428851041,4.539018428311566e-07,7.8740983574478372e-15]},{"sample":"sample07","allele":"locus3_allele1","posterior":[0.037934596114708105,0.95610923304613771,0.0059561688570702086,1.9820839837906203e-09]},{"sample":"sample07","allele":"locus3_allele2","posterior":[1.0345278362389244e-05,0.54785930354292389,0.45211441447107198,1.5936707641758282e-05]},{"sample":"sample07","allele":"locus3_allele3","posterior":[0.00014286095495800319,0.92645757561634634,0.073399300075905297,2.6335279030300321e-07]},{"sample":"sample07","allele":"locus3_allele4","posterior":[0.99796174496088919,0.0020379174813479907,3.3755775535661614e-07,7.4874904912771773e-15]},{"sample":"sample08","allele":"locus1_allele1","posterior":[4.9283531941154491e-05,0.6151302234524284,0.38481372569112066,6.7673245097736497e-06]},{"sample":"sample08","allele":"locus1_allele2","posterior":[6.7673245097738487e-06,0.38481372569111899,0.61513022345243007,4.9283531941155602e-05]},{"sample":"sample08","allele":"locus2_allele1","posterior":[0.00038453544115479014,0.87223885331696926,0.12737574049154538,8.707503305117463e-07]},{"sample":"sample08","allele":"locus2_allele2","posterior":[6.8959513635862943e-05,0.78337274359277009,0.2165543850424588,3.9118511352456312e-06]},{"sample":"sample08","allele":"locus2_allele3","posterior":[0.00064269687627887238,0.95735671728675531,0.042000453995339894,1.3184162594264548e-07]},{"sample":"sample08","allele":"locus3_allele1","posterior":[1.5517945879994205e-07,0.089280812658647166,0.91054687524824163,0.00017215691365233098]},{"sample":"sample08","allele":"locus3_allele2","posterior":[0.99809247228575093,0.0019072731048166086,2.5460942965318376e-07,2.9170921399567675e-15]},{"sample":"sample08","allele":"locus3_allele3","posterior":[0.00029231444506078652,0.9756301832155887,0.024077485555601557,1.6783749050623291e-08]},{"sample":"sample08","allele":"locus3_allele4","posterior":[0.99970350737932834,0.00029648647374653912,6.1469251051124399e-09,1.4330821598790754e-17]},{"sample":"sample09","allele":"locus1_allele1","posterior":[2.2131420469188088e-05,0.057463140309902586,0.27422307476872393,0.43656875585690225,0.23172289764400195]},{"sample":"sample09","allele":"locus1_allele2","posterior":[0.23172289764400208,0.4365687558569023,0.27422307476872393,0.05746314030990253,2.2131420469188528e-05]},{"sample":"sample09","allele":"locus2_allele1","posterior":[2.8538710356070655e-06,0.098629887380397852,0.62277910162956751,0.27856278480166247,2.53723173366531e-05]},{"sample":"sample09","allele":"locus2_allele2","posterior":[7.4444863529742904e-05,0.54840375538296926,0.42787272447857816,0.023648718415016048,3.5685990684562159e-07]},{"sample":"sample09","allele":"locus2_allele3","posterior":[0.92588220864256987,0.072940813456725803,0.0011741495926096623,2.8283055048125406e-06,2.5898110780748546e-12]},{"sample":"sample09","allele":"locus3_allele1","posterior":[0.31121897245659724,0.47503818801729525,0.19577772987291525,0.017964356693337253,7.5295985499825786e-07]},{"sample":"sample09","allele":"locus3_allele2","posterior":[0.00018388919915340963,0.48087924313556901,0.4468145727056832,0.072111613364338481,1.0681595255857748e-05]},{"sample":"sample09","allele":"locus3_allele3","posterior":[0.00025688321621779016,0.60417010336841381,0.35859939836947574,0.036969725145546826,3.8899003458506613e-06]},{"sample":"sample09","allele":"locus3_allele4","posterior":[0.91536250593864743,0.082612700879713188,0.0020138504110220434,1.0942718949124789e-05,5.1668191681808852e-11]},{"sample":"sample10","allele":"locus1_allele1","posterior":[2.3790003780035283e-05,0.22047105251391316,0.62934102273311576,0.15015813046601717,6.00428317385354e-06]},{"sample":"sample10","allele":"locus1_allele2","posterior":[6.0042831738537636e-06,0.15015813046601795,0.62934102273311343,0.22047105251391474,2.3790003780035927e-05]},{"sample":"sample10","allele":"locus2_allele1","posterior":[3.4808729833870633e-07,0.042945914211290064,0.59598308759671692,0.36104468979585558,2.5960308839120638e-05]},{"sample":"sample10","allele":"locus2_allele2","posterior":[0.98838572176291473,0.011590012171793474,2.425956072023978e-05,6.5045711065555709e-09,5.2578318696587239e-16]},{"sample":"sample10","allele":"locus2_allele3","posterior":[7.6202814020471851e-05,0.6765659260737964,0.31687900827296428,0.0064788394118079923,2.3427410854684305e-08]},{"sample":"sample10","allele":"locus3_allele1","posterior":[4.5242519543153987e-08,0.011476060048012392,0.37484132535309156,0.61354364036707176,0.00013892898930477772]},{"sample":"sample10","allele":"locus3_allele2","posterior":[0.97987912553606782,0.020046148718768298,7.4689311438923048e-05,3.6433719595893787e-08,5.3749302424807198e-15]},{"sample":"sample10","allele":"locus3_allele3","posterior":[0.98707198572755595,0.012897310152956818,3.0694552893704671e-05,9.5665924723998646e-09,1.0030781612061496e-15]},{"sample":"sample10","allele":"locus3_allele4","posterior":[0.00040664650558969965,0.96352512175878668,0.036000819916398266,6.7411788761366696e-05,3.0463973780117503e-11]},{"sample":"sample11","allele":"locus1_allele1","posterior":[2.9799465870046478e-06,0.084518290847769781,0.59745878933711372,0.31799255400843501,2.738586009456601e-05]},{"sample":"sample11","allele":"locus1_allele2","posterior":[2.7385860094566962e-05,0.31799255400843596,0.59745878933711227,0.084518290847770197,2.9799465870047207e-06]},{"sample":"sample11","allele":"locus2_allele1","posterior":[8.483567628815931e-06,0.22653192042795225,0.684234870588463,0.089223383648750834,1.3417672051523227e-06]},{"sample":"sample11","allele":"locus2_allele2","posterior":[5.6177551582890957e-06,0.25080382140699747,0.67155905152753026,0.077629992435796311,1.5168745176306832e-06]},{"sample":"sample11","allele":"locus2_allele3","posterior":[0.99020068429865293,0.0097820378071808607,1.7273980452134189e-05,3.9137137765751129e-09,2.8173435293531874e-16]},{"sample":"sample11","allele":"locus3_allele1","posterior":[9.839443380955257e-08,0.019527757116423666,0.46636378597261352,0.51403780865547666,7.0549861052360589e-05]},{"sample":"sample11","allele":"locus3_allele2","posterior":[0.00014375737809247207,0.73359536758254351,0.2619440131136353,0.004316849667670165,1.2258058521321505e-08]},{"sample":"sample11","allele":"locus3_allele3","posterior":[0.99032485056682795,0.0096582069634671402,1.6938633185456161e-05,3.8365191434427114e-09,2.8789906343495575e-16]},{"sample":"sample11","allele":"locus3_allele4","posterior":[0.99763531630120639,0.0023636763815226823,1.0072618112589053e-06,5.5459700183060352e-11,1.1907336742298142e-18]},{"sample":"sample12","allele":"locus1_allele1","posterior":[4.7579214089563981e-15,3.8708363691918323e-08,7.9326091033643628e-05,0.020846347381781753,0.97907428781881611]},{"sample":"sample12","allele":"locus1_allele2","posterior":[0.97907428781881611,0.020846347381781771,7.9326091033643561e-05,3.8708363691918502e-08,4.7579214089565054e-15]},{"sample":"sample12","allele":"locus2_allele1","posterior":[2.5277906200815673e-05,0.32879079982429343,0.61088958227700907,0.06029350595090982,8.3404158674311032e-07]},{"sample":"sample12","allele":"locus2_allele2","posterior":[4.4877404056055484e-05,0.52215973272771221,0.45675478354776383,0.021040409500135342,1.9682033245176777e-07]},{"sample":"sample12","allele":"locus2_allele3","posterior":[0.00071859570957624502,0.90314027164794775,0.095519542533608393,0.00062158916846308373,9.4040444850389285e-10]},{"sample":"sample12","allele":"locus3_allele1","posterior":[1.285843403963124e-05,0.24230984764913122,0.66152008799946649,0.09615526692958079,1.9389877818636869e-06]},{"sample":"sample12","allele":"locus3_allele2","posterior":[1.643893270903128e-05,0.35267295646160551,0.59430821400653466,0.053001430209033411,9.6039011735454744e-07]},{"sample":"sample12","allele":"locus3_allele3","posterior":[0.9749578063500477,0.024921727036724187,0.0001203872551667559,7.9358042934419082e-08,1.8432638220081021e-14]},{"sample":"sample12","allele":"locus3_allele4","posterior":[0.0097899502813122187,0.98574205422542682,0.0044662537807189538,1.7417123101819064e-06,2.3175964610498205e-13]},{"sample":"sample13","allele":"locus1_allele1","posterior":[3.7195975243498643e-06,0.01705920583921057,0.14283014598308455,0.36838723215501279,0.36210001681813653,0.10958162320629297,3.8056400738265755e-05]},{"sample":"sample13","allele":"locus1_allele2","posterior":[3.80564007382669e-05,0.10958162320629297,0.36210001681813619,0.36838723215501279,0.14283014598308488,0.01705920583921058,3.7195975243499456e-06]},{"sample":"sample13","allele":"locus2_allele1","posterior":[7.5358598336440707e-06,0.070461043612502036,0.42116667249443451,0.41677417494986696,0.088511654761399647,0.0030788493960209637,6.8925942269928217e-08]},{"sample":"sample13","allele":"locus2_allele2","posterior":[6.0018614050727472e-06,0.090915322465197554,0.46077257094251839,0.37990784089344365,0.066500374674186361,0.0018978386068193998,5.0556429582244228e-08]},{"sample":"sample13","allele":"locus2_allele3","posterior":[0.00070964808584739159,0.72376101513528113,0.25794965904772604,0.017304381131451028,0.00027452778026268334,7.6881719418504955e-07,2.2375569497988274e-12]},{"sample":"sample13","allele":"locus3_allele1","posterior":[3.9512767617135482e-07,0.010481079098499694,0.17780665589507583,0.49259855765683402,0.29095877145182225,0.0281528135902466,1.7271798454217225e-06]},{"sample":"sample13","allele":"locus3_allele2","posterior":[0.00057534625153629366,0.65085933587858102,0.31853866436143957,0.029382437479356982,0.00064173893204318806,2.4770877770445459e-06,9.2658403458161853e-12]},{"sample":"sample13","allele":"locus3_allele3","posterior":[0.00039996209655033795,0.66024409980229848,0.31231538430129169,0.026523828890210824,0.00051500297021748152,1.7219333506115718e-06,6.0804600982135514e-12]},{"sample":"sample13","allele":"locus3_allele4","posterior":[0.0010714718662098873,0.92740020384846844,0.070515006547850742,0.0010098909208470545,3.4247625827028339e-06,2.0540396537370216e-09,1.5610310747013935e-15]},{"sample":"sample14","allele":"locus1_allele1","posterior":[5.6233843095902972e-06,0.030129576335136746,0.22611618972342198,0.43637332074959678,0.26765501973188544,0.039715835158344918,4.4349173045966608e-06]},{"sample":"sample14","allele":"locus1_allele2","posterior":[4.4349173045967853e-06,0.039715835158344918,0.26765501973188516,0.43637332074959689,0.2261161897234221,0.030129576335136767,5.6233843095904209e-06]},{"sample":"sample14","allele":"locus2_allele1","posterior":[2.2947206592939182e-07,0.007479938995366581,0.1498318444635138,0.48195506011709782,0.32530612569806761,0.035424338507515377,2.4627463728421754e-06]},{"sample":"sample14","allele":"locus2_allele2","posterior":[1.7152041782140303e-05,0.16541833626023006,0.53027974239045905,0.27404794816316674,0.029718894949980604,0.00051791791507482538,8.2793065040618069e-09]},{"sample":"sample14","allele":"locus2_allele3","posterior":[0.9188026948074961,0.079041880983186347,0.002134163546950832,2.1190949652971043e-05,6.9663065664298563e-08,4.9648038168338676e-11,4.2202973475327416e-17]},{"sample":"sample14","allele":"locus3_allele1","posterior":[2.007960953337819e-06,0.026369983828564869,0.2589762648968717,0.47936034960470492,0.21688078307597111,0.018409470649178496,1.1399837555449911e-06]},{"sample":"sample14","allele":"locus3_allele2","posterior":[0.00060342871544391487,0.60333167339926552,0.34847486237884706,0.045919737697489131,0.0016583194468103024,1.1978268278006414e-05,9.3866127845419675e-11]},{"sample":"sample14","allele":"locus3_allele3","posterior":[6.5503525632461696e-05,0.3307365535295278,0.508178796686094,0.14995398135317906,0.01091593779110309,0.00014922473374256789,2.380720964364368e-09]},{"sample":"sample14","allele":"locus3_allele4","posterior":[0.95123994955015434,0.047964272612520432,0.00079075482695306458,5.0119650046045784e-06,1.1039806199866645e-08,5.5612477141426634e-12,4.3117220667700568e-18]},{"sample":"sample15","allele":"locus1_allele1","posterior":[2.2261913057176432e-06,0.017874669200796268,0.17877621712123887,0.43114450731012172,0.31727729863278725,0.054918073691062393,7.0078526877449759e-06]},{"sample":"sample15","allele":"locus1_allele2","posterior":[7.0078526877451978e-06,0.0549180736910624,0.31727729863278692,0.43114450731012255,0.17877621712123831,0.017874669200796282,2.2261913057176919e-06]},{"sample":"sample15","allele":"locus2_allele1","posterior":[1.4234104481361229e-08,0.0010004882606403178,0.044072537820696457,0.31974895962713873,0.50255129210179139,0.13260323533641236,2.3472619216317369e-05]},{"sample":"sample15","allele":"locus2_allele2","posterior":[0.00010035473067798437,0.38008532012937146,0.50281722142788698,0.11158954932316055,0.0053650395783788928,4.2514495353603432e-05,3.1517052994853793e-10]},{"sample":"sample15","allele":"locus2_allele3","posterior":[0.91465738959692311,0.08295306276174387,0.0023646352323062874,2.482591249515301e-05,8.6431185517637264e-08,6.5345976933619192e-11,5.9032241088792698e-17]},{"sample":"sample15","allele":"locus3_allele1","posterior":[5.9741976036027105e-07,0.014486911481287612,0.2162462905630122,0.50715022667171006,0.24370879771948536,0.018406332549391169,8.4359535331634387e-07]},{"sample":"sample15","allele":"locus3_allele2","posterior":[0.91353016557486144,0.08403143517122956,0.0024129750577623994,2.5336579930458001e-05,8.7551050779315651e-08,6.516527903800148e-11,5.3512817312190446e-17]},{"sample":"sample15","allele":"locus3_allele3","posterior":[1.1630754765473762e-05,0.15972727999620182,0.53186959433647485,0.2780954881582558,0.029794028981392339,0.00050196949312817415,8.2797814641212314e-09]},{"sample":"sample15","allele":"locus3_allele4","posterior":[0.98586880180487368,0.014068419658834522,6.2676361166782361e-05,1.021203455792646e-07,5.4773106660535713e-11,6.3332446800079686e-15,1.0564903395487506e-21]},{"sample":"sample16","allele":"locus1_allele1","posterior":[2.3798424470925225e-10,2.2869591410638206e-05,0.0021990067673106828,0.046792953488679297,0.3163786349095255,0.63265470810966107,0.0019518268954285202]},{"sample":"sample16","allele":"locus1_allele2","posterior":[0.0019518268954285749,0.63265470810966107,0.31637863490952567,0.046792953488679213,0.002199006767310688,2.2869591410638216e-05,2.3798424470925742e-10]},{"sample":"sample16","allele":"locus2_allele1","posterior":[4.1766894214280937e-06,0.042563936715606897,0.32179784592350413,0.46216596146463679,0.16273302485079302,0.010734529654059168,5.2470197859718608e-07]},{"sample":"sample16","allele":"locus2_allele2","posterior":[5.6690635008130108e-06,0.077744869670950692,0.4113402490540306,0.40741263762070717,0.098929592532442795,0.0045667486208305665,2.3343753741415582e-07]},{"sample":"sample16","allele":"locus2_allele3","posterior":[0.79060084969228983,0.19344361746599548,0.015474578465325,0.00047585770843584682,5.0842524897032623e-06,1.2415425756045794e-08,3.8322961892381984e-14]},{"sample":"sample16","allele":"locus3_allele1","posterior":[1.2937048339341971e-05,0.046115379899507584,0.24623263056057124,0.40490564903570181,0.25374727013741921,0.04897218661142147,1.3946707039382599e-05]},{"sample":"sample16","allele":"locus3_allele2","posterior":[0.33970483978559241,0.42230533528565883,0.19437673470541336,0.040050419012640635,0.0034740639105296368,8.8603518661275677e-05,3.7815038590396401e-09]},{"sample":"sample16","allele":"locus3_allele3","posterior":[0.00015599523508991723,0.34408265084777973,0.44777595474409165,0.17932788735900829,0.027369951351865294,0.0012874162898299612,1.4417233524078545e-07]},{"sample":"sample16","allele":"locus3_allele4","posterior":[0.82848215210882015,0.15974375031125357,0.011404645288323611,0.00036452668966763892,4.9061725889844723e-06,1.9429177711234397e-08,1.6833715435921281e-13]}],"genotypes":[[2,0,1,1,0,1,1,0,0],[2,0,2,0,0,1,1,0,0],[1,1,0,0,2,0,1,1,0],[0,2,0,0,2,2,0,0,0],[2,1,2,1,0,1,1,1,0],[1,2,2,0,1,2,1,0,0],[2,1,2,1,0,1,1,1,0],[1,2,1,1,1,2,0,1,0],[3,1,2,2,0,1,2,1,0],[2,2,3,0,1,3,0,0,1],[2,2,2,2,0,3,1,0,0],[4,0,2,1,1,2,1,0,1],[3,3,3,2,1,3,1,1,1],[3,3,4,2,0,3,1,2,0],[3,3,4,2,0,3,0,3,0],[5,1,3,3,0,3,1,2,0]]},"selfing":{"selfing_rate":0.40000000000000002,"priors":[{"ploidy":2,"selfing_rate":0.10000000000000001,"allele_frequency":0.050000000000000003,"prior":[0.90500000000000003,0.090000000000000011,0.005000000000000001]},{"ploidy":2,"selfing_rate":0.10000000000000001,"allele_frequency":0.29999999999999999,"prior":[0.50105263157894731,0.39789473684210525,0.10105263157894737]},{"ploidy":2,"selfing_rate":0.10000000000000001,"allele_frequency":0.5,"prior":[0.26315789473684209,0.47368421052631576,0.26315789473684209]},{"ploidy":2,"selfing_rate":0.40000000000000002,"allele_frequency":0.050000000000000003,"prior":[0.91437499999999994,0.071249999999999994,0.014375000000000001]},{"ploidy":2,"selfing_rate":0.40000000000000002,"allele_frequency":0.29999999999999999,"prior":[0.54249999999999998,0.31499999999999995,0.14249999999999999]},{"ploidy":2,"selfing_rate":0.40000000000000002,"allele_frequency":0.5,"prior":[0.3125,0.375,0.3125]},{"ploidy":2,"selfing_rate":0.94999999999999996,"allele_frequency":0.050000000000000003,"prior":[0.94547619047619058,0.0090476190476190561,0.045476190476190476]},{"ploidy":2,"selfing_rate":0.94999999999999996,"allele_frequency":0.29999999999999999,"prior":[0.67999999999999994,0.040000000000000036,0.27999999999999997]},{"ploidy":2,"selfing_rate":0.94999999999999996,"allele_frequency":0.5,"prior":[0.47619047619047616,0.047619047619047658,0.47619047619047616]},{"ploidy":4,"selfing_rate":0.10000000000000001,"allele_frequency":0.050000000000000003,"prior":[0.81908276194144825,0.16285077041602464,0.017132935285053937,0.00085077041602465368,8.2761941448382159e-05]},{"ploidy":4,"selfing_rate":0.10000000000000001,"allele_frequency":0.29999999999999999,"prior":[0.25191473522017677,0.39609225529154157,0.26314391371340529,0.077776465817857424,0.011072629957018895]},{"ploidy":4,"selfing_rate":0.10000000000000001,"allele_frequency":0.5,"prior":[0.07045454545454545,0.24545454545454548,0.36818181818181828,0.24545454545454545,0.07045454545454545]},{"ploidy":4,"selfing_rate":0.40000000000000002,"allele_frequency":0.050000000000000003,"prior":[0.83694151785714266,0.13160892857142856,0.027024107142857141,0.0033589285714285704,0.001066517857142857]},{"ploidy":4,"selfing_rate":0.40000000000000002,"allele_frequency":0.29999999999999999,"prior":[0.30084999999999995,0.3365999999999999,0.25109999999999993,0.084599999999999995,0.026849999999999999]},{"ploidy":4,"selfing_rate":0.40000000000000002,"allele_frequency":0.5,"prior":[0.10625000000000002,0.22499999999999995,0.33749999999999991,0.22499999999999995,0.10625000000000002]},{"ploidy":4,"selfing_rate":0.94999999999999996,"allele_frequency":0.050000000000000003,"prior":[0.92540716053748218,0.021828500707213594,0.013671534653465354,0.0055427864214992965,0.033550017680339453]},{"ploidy":4,"selfing_rate":0.94999999999999996,"allele_frequency":0.29999999999999999,"prior":[0.60330099009900962,0.068396039603960429,0.072605940594059462,0.036396039603960414,0.21930099009900983]},{"ploidy":4,"selfing_rate":0.94999999999999996,"allele_frequency":0.5,"prior":[0.39499999999999991,0.060000000000000039,0.090000000000000066,0.060000000000000039,0.39499999999999991]},{"ploidy":6,"selfing_rate":0.10000000000000001,"allele_frequency":0.050000000000000003,"prior":[0.74137995234256115,0.22080458334823602,0.034604922124246387,0.002885094409411996,0.00029973573517463068,2.2926007880573815e-05,2.7860324889620746e-06]},{"ploidy":6,"selfing_rate":0.10000000000000001,"allele_frequency":0.29999999999999999,"prior":[0.12732115454269768,0.29550278460301654,0.31788344057249451,0.18334284425021791,0.062441044259130538,0.01211476616983696,0.0013939656026055562]},{"ploidy":6,"selfing_rate":0.10000000000000001,"allele_frequency":0.5,"prior":[0.019410952816868381,0.096121643437196105,0.23202364210819143,0.30488752327548829,0.23202364210819151,0.096121643437196119,0.019410952816868377]},{"ploidy":6,"selfing_rate":0.40000000000000002,"allele_frequency":0.050000000000000003,"prior":[0.76624171504838201,0.17964483555394506,0.044906771261823637,0.0069027584456112538,0.0018082658270410321,0.00037363990177117715,0.00012201396142564248]},{"ploidy":6,"selfing_rate":0.40000000000000002,"allele_frequency":0.29999999999999999,"prior":[0.16997446797925581,0.26570201019115947,0.29065326916118905,0.17462746185852979,0.07243587785684133,0.020275923234637876,0.0063309897183862996]},{"ploidy":6,"selfing_rate":0.40000000000000002,"allele_frequency":0.5,"prior":[0.039121322072518332,0.10295410639984148,0.2204666757479691,0.2749157915593422,0.2204666757479691,0.10295410639984148,0.039121322072518325]},{"ploidy":6,"selfing_rate":0.94999999999999996,"allele_frequency":0.050000000000000003,"prior":[0.90179445489031651,0.031750829067837633,0.020252285911544904,0.010408187375029512,0.0071469532001021069,0.0039515131474396033,0.024695776407729596]},{"ploidy":6,"selfing_rate":0.94999999999999996,"allele_frequency":0.29999999999999999,"prior":[0.52847413058286463,0.071922152589058416,0.086935493633987032,0.065766960484365353,0.048545775557701773,0.027410046453071664,0.17094544069895096]},{"ploidy":6,"selfing_rate":0.94999999999999996,"allele_frequency":0.5,"prior":[0.3245533544109509,0.0511912492897151,0.081151364866718409,0.086208062865231125,0.081151364866718423,0.051191249289715114,0.32455335441095101]},{"ploidy":8,"selfing_rate":0.10000000000000001,"allele_frequency":0.050000000000000003,"prior":[0.67109882780448937,0.26610484097393716,0.055447413627646519,0.0064969407669589084,0.00076455186765403893,7.4853304614299953e-05,1.1118762436823265e-05,1.3002365160774897e-06,1.5265574647940354e-07]},{"ploidy":8,"selfing_rate":0.10000000000000001,"allele_frequency":0.29999999999999999,"prior":[0.064733355758942027,0.19644500292806943,0.29094161404949198,0.24882072328021029,0.13623813931819512,0.048960216165964979,0.011779261907035812,0.0018731031492248056,0.00020858344286509665]},{"ploidy":8,"selfing_rate":0.10000000000000001,"allele_frequency":0.5,"prior":[0.0055470387174803223,0.034042565511449185,0.11111621123117928,0.21551755622877564,0.26755325662223128,0.21551755622877558,0.11111621123117926,0.034042565511449178,0.0055470387174803223]},{"ploidy":8,"selfing_rate":0.40000000000000002,"allele_frequency":0.050000000000000003,"prior":[0.70174002218840859,0.21779178222645135,0.064708256128637134,0.011665128347802632,0.0030740451529205951,0.00072242922449946949,0.00021981342501725592,5.9073261519286419e-05,1.9450044743508945e-05]},{"ploidy":8,"selfing_rate":0.40000000000000002,"allele_frequency":0.29999999999999999,"prior":[0.098027693687442588,0.18839476495206664,0.26749399227712628,0.22744803794495538,0.13473882544423835,0.057135035230023334,0.019478918068981572,0.0054927088434694382,0.0017900235516960187]},{"ploidy":8,"selfing_rate":0.40000000000000002,"allele_frequency":0.5,"prior":[0.015622489533340729,0.044636602907315442,0.11622680401050479,0.20158534344050821,0.24385752021666157,0.20158534344050827,0.11622680401050479,0.044636602907315442,0.015622489533340729]},{"ploidy":8,"selfing_rate":0.94999999999999996,"allele_frequency":0.050000000000000003,"prior":[0.87783836281704386,0.039987593250391307,0.026462737984150532,0.013957842959377888,0.0093921734897962367,0.0063285935044845917,0.0046856378138463361,0.0028885519087814325,0.018458506272127223]},{"ploidy":8,"selfing_rate":0.94999999999999996,"allele_frequency":0.29999999999999999,"prior":[0.46458915744881002,0.067691006682932264,0.088840980257069269,0.078949754461343127,0.063532525569128431,0.045963685408968911,0.034248538435128373,0.021136804406182305,0.13504754733043672]},{"ploidy":8,"selfing_rate":0.94999999999999996,"allele_frequency":0.5,"prior":[0.26927719982299075,0.042048840422466073,0.066043844568329735,0.07925289105966235,0.086754448253102107,0.079252891059662364,0.066043844568329749,0.042048840422466066,0.26927719982299075]}],"allele_frequency":[0.61359980606110809,0.38640019393889202,0.49320022463071145,0.27057002922819506,0.22531911715433864,0.49977546523533273,0.28007094275049665,0.20410773792537654,0.055265988627452869],"records":[{"sample":"sample01","allele":"locus1_allele1","posterior":[9.6012628949880518e-11,0.0016781712104660612,0.99832182869352137]},{"sample":"sample01","allele":"locus1_allele2","posterior":[0.99832182869352137,0.0016781712104660625,9.6012628949874469e-11]},{"sample":"sample01","allele":"locus2_allele1","posterior":[3.643651477098034e-05,0.99994801892252205,1.5544562706958973e-05]},{"sample":"sample01","allele":"locus2_allele2","posterior":[2.6243839685878134e-05,0.9999538567475883,1.989941272585736e-05]},{"sample":"sample01","allele":"locus2_allele3","posterior":[0.97399158755945259,0.02600841242207521,1.8472190919029519e-11]},{"sample":"sample01","allele":"locus3_allele1","posterior":[8.7057766731668285e-05,0.99963057073007,0.00028237150319824383]},{"sample":"sample01","allele":"locus3_allele2","posterior":[0.00035592384885175951,0.99957981481548019,6.4261335668060527e-05]},{"sample":"sample01","allele":"locus3_allele3","posterior":[0.99398319956111092,0.0060167963836410621,4.0552480743920047e-09]},{"sample":"sample01","allele":"locus3_allele4","posterior":[0.998553071073496,0.0014469282199253398,7.0657866256991779e-10]},{"sample":"sample02","allele":"locus1_allele1","posterior":[1.1014699262401461e-14,2.4135305323612237e-05,0.99997586469466537]},{"sample":"sample02","allele":"locus1_allele2","posterior":[0.99997586469466537,2.4135305323612237e-05,1.1014699262400759e-14]},{"sample":"sample02","allele":"locus2_allele1","posterior":[5.6012399427410899e-12,0.00053265597821554116,0.9994673440161832]},{"sample":"sample02","allele":"locus2_allele2","posterior":[0.99977847570434975,0.00022152429396938417,1.6809143709371942e-12]},{"sample":"sample02","allele":"locus2_allele3","posterior":[0.99982300546795966,0.0001769945307969691,1.2433043323993778e-12]},{"sample":"sample02","allele":"locus3_allele1","posterior":[0.00017321511526456848,0.99982242031175972,4.364572975654474e-06]},{"sample":"sample02","allele":"locus3_allele2","posterior":[1.0627476903894803e-05,0.99992565461755278,6.3717905543300758e-05]},{"sample":"sample02","allele":"locus3_allele3","posterior":[0.99997839632296326,2.1603677020894753e-05,1.5822330643022908e-14]},{"sample":"sample02","allele":"locus3_allele4","posterior":[0.76620562377757262,0.2337943756776053,5.4482204315608703e-10]},{"sample":"sample03","allele":"locus1_allele1","posterior":[0.20857888100930386,0.35564262614261594,0.43577849284808018]},{"sample":"sample03","allele":"locus1_allele2","posterior":[0.43577849284807996,0.35564262614261605,0.20857888100930397]},{"sample":"sample03","allele":"locus2_allele1","posterior":[0.99798949938648174,0.0020105005079917903,1.0552659887707541e-10]},{"sample":"sample03","allele":"locus2_allele2","posterior":[0.99912609388574414,0.00087390608062459568,3.363128235078081e-11]},{"sample":"sample03","allele":"locus2_allele3","posterior":[3.3129899512421208e-10,0.0047292809236955417,0.9952707187450055]},{"sample":"sample03","allele":"locus3_allele1","posterior":[0.99988854099653546,0.00011145900325977266,2.0487820645270061e-13]},{"sample":"sample03","allele":"locus3_allele2","posterior":[1.7521459485049774e-05,0.99993824253509334,4.4236005421653721e-05]},{"sample":"sample03","allele":"locus3_allele3","posterior":[8.0553055690169665e-05,0.99991040861248703,9.0383318228103894e-06]},{"sample":"sample03","allele":"locus3_allele4","posterior":[0.99999193939061182,8.0606093813028774e-06,6.9625520530521801e-15]},{"sample":"sample04","allele":"locus1_allele1","posterior":[0.99976625537835562,0.00023374462089404611,7.5037426271138463e-13]},{"sample":"sample04","allele":"locus1_allele2","posterior":[7.503742627113091e-13,0.00023374462089404606,0.99976625537835562]},{"sample":"sample04","allele":"locus2_allele1","posterior":[0.99960491906794469,0.00039508092900532489,3.0499814062983146e-12]},{"sample":"sample04","allele":"locus2_allele2","posterior":[0.99982840814243046,0.00017159185659811993,9.7139199099907184e-13]},{"sample":"sample04","allele":"locus2_allele3","posterior":[9.5932133272984283e-12,0.00093125569736573718,0.99906874429304093]},{"sample":"sample04","allele":"locus3_allele1","posterior":[5.4428498255585261e-12,0.00052162076885031445,0.99947837922570681]},{"sample":"sample04","allele":"locus3_allele2","posterior":[0.99976864311712532,0.00023135688109180569,1.7828671709759324e-12]},{"sample":"sample04","allele":"locus3_allele3","posterior":[0.99984265759190938,0.00015734240702689666,1.0636581774020484e-12]},{"sample":"sample04","allele":"locus3_allele4","posterior":[0.9999623223724482,3.7677627367234313e-05,1.8461195125870929e-13]},{"sample":"sample05","allele":"locus1_allele1","posterior":[3.7278105461366387e-07,0.084959234787930091,0.91465800294244681,0.00038238948856841252]},{"sample":"sample05","allele":"locus1_allele2","posterior":[0.00038238948856837756,0.91465800294244637,0.084959234787930563,3.7278105461364233e-07]},{"sample":"sample05","allele":"locus2_allele1","posterior":[4.7783585705610767e-11,0.00094010533330888917,0.837020525504373,0.16203936911453462]},{"sample":"sample05","allele":"locus2_allele2","posterior":[0.22249994756878752,0.77718435881167658,0.00031569361094223422,8.5936700436897236e-12]},{"sample":"sample05","allele":"locus2_allele3","posterior":[0.99790590580219229,0.0020937805214913167,3.1367631195954486e-07,4.4413934283302666e-15]},{"sample":"sample05","allele":"locus3_allele1","posterior":[0.00011846422260057859,0.60002632047322713,0.39982559679055679,2.9618513615578152e-05]},{"sample":"sample05","allele":"locus3_allele2","posterior":[0.0044414301381392168,0.95057857496780673,0.044979653937616181,3.4095643791045385e-07]},{"sample":"sample05","allele":"locus3_allele3","posterior":[0.00026747351353570637,0.85346043701343488,0.1462676524557284,4.4370173010115578e-06]},{"sample":"sample05","allele":"locus3_allele4","posterior":[0.9787529154179867,0.021190441217646357,5.6643326594297416e-05,3.7772615355602948e-11]},{"sample":"sample06","allele":"locus1_allele1","posterior":[2.1389066719282566e-05,0.55203328584969369,0.4479384125934357,6.9124901513874573e-06]},{"sample":"sample06","allele":"locus1_allele2","posterior":[6.9124901513867737e-06,0.44793841259343292,0.55203328584969646,2.1389066719281316e-05]},{"sample":"sample06","allele":"locus2_allele1","posterior":[6.1785985977765726e-08,0.057987272114943644,0.94172040260511802,0.00029226349395240374]},{"sample":"sample06","allele":"locus2_allele2","posterior":[0.99849695815663897,0.0015028854140318566,1.564293277784556e-07,1.3978798750110326e-15]},{"sample":"sample06","allele":"locus2_allele3","posterior":[0.00047684947169634545,0.98234394729932384,0.0171791950555031,8.1734767238454891e-09]},{"sample":"sample06","allele":"locus3_allele1","posterior":[1.8685803622866871e-07,0.075582913019901707,0.92398385660207472,0.00043304351998738413]},{"sample":"sample06","allele":"locus3_allele2","posterior":[0.00065206155756851849,0.96848648171872653,0.030861413875139958,4.2848564899671898e-08]},{"sample":"sample06","allele":"locus3_allele3","posterior":[0.99212516807806905,0.0078698211815650329,5.0107400103165771e-06,3.5566554413534277e-13]},{"sample":"sample06","allele":"locus3_allele4","posterior":[0.99819275453494605,0.0018069827839251049,2.6268112380326211e-07,5.0578560623073152e-15]},{"sample":"sample07","allele":"locus1_allele1","posterior":[1.395216747136654e-05,0.47315955144230076,0.52681465530840321,1.1841081824651862e-05]},{"sample":"sample07","allele":"locus1_allele2","posterior":[1.1841081824650656e-05,0.52681465530840221,0.47315955144230171,1.3952167471365734e-05]},{"sample":"sample07","allele":"locus2_allele1","posterior":[2.0179842550578318e-06,0.2806254154469664,0.71933658633187758,3.5980236901061164e-05]},{"sample":"sample07","allele":"locus2_allele2","posterior":[6.4588389217070627e-05,0.87619845240183469,0.12373648246139773,4.7674755055140536e-07]},{"sample":"sample07","allele":"locus2_allele3","posterior":[0.99749851540551804,0.0025010317780027797,4.5281647130038055e-07,7.847995457335241e-15]},{"sample":"sample07","allele":"locus3_allele1","posterior":[0.038231137240220141,0.95590792941397651,0.0058609314107403131,1.9350630426616898e-09]},{"sample":"sample07","allele":"locus3_allele2","posterior":[1.0303650735406562e-05,0.54649101729728244,0.45348263049430537,1.6048557676846037e-05]},{"sample":"sample07","allele":"locus3_allele3","posterior":[0.00014140918240626249,0.9238822615852087,0.075976048401021831,2.8083136310793183e-07]},{"sample":"sample07","allele":"locus3_allele4","posterior":[0.99801255256421029,0.0019871265092784164,3.2092650430164953e-07,6.9507293174435079e-15]},{"sample":"sample08","allele":"locus1_allele1","posterior":[4.9428051553569399e-05,0.61582524136281969,0.38411858309733155,6.7474882951285082e-06]},{"sample":"sample08","allele":"locus1_allele2","posterior":[6.7474882951278365e-06,0.38411858309733099,0.61582524136282035,4.9428051553566647e-05]},{"sample":"sample08","allele":"locus2_allele1","posterior":[0.00038415862797729219,0.87206298133329796,0.1275519873856697,8.7265305505522865e-07]},{"sample":"sample08","allele":"locus2_allele2","posterior":[6.7829474946347229e-05,0.77753523791435919,0.22239281384748791,4.118763206546919e-06]},{"sample":"sample08","allele":"locus2_allele3","posterior":[0.00064290151100215268,0.95740468498070785,0.0419522819398351,1.3156845497790799e-07]},{"sample":"sample08","allele":"locus3_allele1","posterior":[1.5869452325089944e-07,0.090581454056165592,0.90924782546943472,0.00017056177987648157]},{"sample":"sample08","allele":"locus3_allele2","posterior":[0.99808191263438628,0.0019178299280042394,2.5743760647908698e-07,2.9612363590739631e-15]},{"sample":"sample08","allele":"locus3_allele3","posterior":[0.00028988882123233482,0.97474086862084097,0.024969224627028545,1.7930898189432233e-08]},{"sample":"sample08","allele":"locus3_allele4","posterior":[0.99971090942574925,0.00028908473042798264,5.8438228482508355e-09,1.3302924861698276e-17]},{"sample":"sample09","allele":"locus1_allele1","posterior":[5.1409425001937921e-05,0.057307613200654278,0.25126045615285059,0.3691334546622852,0.32224706655920804]},{"sample":"sample09","allele":"locus1_allele2","posterior":[0.32224706655920787,0.36913345466228509,0.2512604561528507,0.057307613200654313,5.1409425001934851e-05]},{"sample":"sample09","allele":"locus2_allele1","posterior":[5.2957060430365935e-06,0.097917670917936891,0.62207203709490855,0.27995630831264967,4.8687968461889402e-05]},{"sample":"sample09","allele":"locus2_allele2","posterior":[9.9265189780960186e-05,0.49182831015631756,0.47484274923329389,0.033227978458585751,1.6969620218582663e-06]},{"sample":"sample09","allele":"locus2_allele3","posterior":[0.94778228991325697,0.051173214630412224,0.0010412019103831916,3.29353376507812e-06,1.2182531953586403e-11]},{"sample":"sample09","allele":"locus3_allele1","posterior":[0.46566526901784494,0.37046830318758617,0.15029063963238212,0.013574721565018683,1.0665971680377531e-06]},{"sample":"sample09","allele":"locus3_allele2","posterior":[0.000248159445348299,0.43036069445976977,0.47600695301907769,0.093340024626530238,4.4168449273917597e-05]},{"sample":"sample09","allele":"locus3_allele3","posterior":[0.00031563263284749808,0.51963047824705266,0.41856625763835442,0.061457433621093634,3.0197860651694829e-05]},{"sample":"sample09","allele":"locus3_allele4","posterior":[0.9350492395933061,0.061333440851447402,0.003556096587500137,6.1216884850231855e-05,6.0828960936088533e-09]},{"sample":"sample10","allele":"locus1_allele1","posterior":[5.9836396518372039e-05,0.23807602728601771,0.62437987370449177,0.13747520500297694,9.0576099952718775e-06]},{"sample":"sample10","allele":"locus1_allele2","posterior":[9.0576099952709712e-06,0.13747520500297683,0.62437987370449177,0.23807602728601776,5.9836396518368244e-05]},{"sample":"sample10","allele":"locus2_allele1","posterior":[6.4537524035608245e-07,0.042599905949269157,0.59480485769283309,0.36254481675374717,4.9774228910140004e-05]},{"sample":"sample10","allele":"locus2_allele2","posterior":[0.99195712265308256,0.008022091458363188,2.077883478338998e-05,7.053768965183924e-09,1.9296864828918697e-15]},{"sample":"sample10","allele":"locus2_allele3","posterior":[0.00010210208368441036,0.62186679934416933,0.36814658112827253,0.0098843730628498036,1.4438102393512231e-07]},{"sample":"sample10","allele":"locus3_allele1","posterior":[8.8299622541376127e-08,0.011767556373228658,0.37835582159144393,0.60961775837235666,0.00025877536334827653]},{"sample":"sample10","allele":"locus3_allele2","posterior":[0.98650310587817114,0.013437261304656833,5.9597494892130665e-05,3.5322263195928063e-08,1.6646685503248596e-14]},{"sample":"sample10","allele":"locus3_allele3","posterior":[0.99063485294806486,0.0093349830114948899,3.0150657249597236e-05,1.3383184033149336e-08,6.5528184714266267e-15]},{"sample":"sample10","allele":"locus3_allele4","posterior":[0.00052018379859956648,0.91746230356884007,0.081533821636326873,0.00048368639622216265,4.6000113883748344e-09]},{"sample":"sample11","allele":"locus1_allele1","posterior":[7.6857410116580961e-06,0.093588287924226979,0.60782378677046778,0.29853787665248999,4.2362911803515278e-05]},{"sample":"sample11","allele":"locus1_allele2","posterior":[4.2362911803511036e-05,0.29853787665248988,0.607823786770468,0.09358828792422702,7.6857410116576641e-06]},{"sample":"sample11","allele":"locus2_allele1","posterior":[1.5773198842838123e-05,0.22533742792290629,0.68479867345716217,0.089845545603763302,2.579817325363114e-06]},{"sample":"sample11","allele":"locus2_allele2","posterior":[6.9405786974620635e-06,0.20840615867117265,0.69052062878687204,0.10105958898549637,6.6829777613725661e-06]},{"sample":"sample11","allele":"locus2_allele3","posterior":[0.99326004580508198,0.0067249393748116037,1.5010354175063874e-05,4.4659300257390293e-09,1.2986628070682315e-15]},{"sample":"sample11","allele":"locus3_allele1","posterior":[1.9172091968493773e-07,0.019990938376138839,0.46996512136771279,0.50991255406155778,0.00013119447367084187]},{"sample":"sample11","allele":"locus3_allele2","posterior":[0.00020608692539693827,0.69742055750181819,0.29643768409584897,0.0059356176338718715,5.3843063913204913e-08]},{"sample":"sample11","allele":"locus3_allele3","posterior":[0.99299916273109312,0.0069842085012674507,1.6623405406163955e-05,5.3622314409312124e-09,1.8790548435234046e-15]},{"sample":"sample11","allele":"locus3_allele4","posterior":[0.99827924809276003,0.0017190092770521803,1.7423262631598949e-06,3.0392454002762326e-10,1.3732398048306724e-16]},{"sample":"sample12","allele":"locus1_allele1","posterior":[8.0130663213154545e-15,2.7988524776564799e-08,5.2697491467849355e-05,0.012779544466229677,0.98716773005376979]},{"sample":"sample12","allele":"locus1_allele2","posterior":[0.98716773005376957,0.012779544466229687,5.2697491467849308e-05,2.7988524776564776e-08,8.0130663213149954e-15]},{"sample":"sample12","allele":"locus2_allele1","posterior":[4.7035327158445537e-05,0.32731474950277817,0.61187480338371747,0.060761806908795797,1.6048775500853368e-06]},{"sample":"sample12","allele":"locus2_allele2","posterior":[5.9554161501345303e-05,0.46605173769501623,0.50446652336201336,0.029421253348935632,9.3143253341186686e-07]},{"sample":"sample12","allele":"locus2_allele3","posterior":[0.0010210174863859549,0.88029293806666775,0.117680404509311,0.0010056337917434322,6.1458918644342789e-09]},{"sample":"sample12","allele":"locus3_allele1","posterior":[2.480424212692019e-05,0.24557792549994151,0.65996387754676966,0.094429823031294158,3.5696798678179499e-06]},{"sample":"sample12","allele":"locus3_allele2","posterior":[2.1805512574129228e-05,0.31022937312198179,0.62231390178283408,0.06743101630685025,3.9032757597735836e-06]},{"sample":"sample12","allele":"locus3_allele3","posterior":[0.9817821171204526,0.018099117789393541,0.0001186536967846192,1.1139324842533932e-07,1.2082230537079317e-13]},{"sample":"sample12","allele":"locus3_allele4","posterior":[0.013028051293273604,0.97643635845971877,0.01052258979291907,1.3000417683379043e-05,3.640511165542151e-11]},{"sample":"sample13","allele":"locus1_allele1","posterior":[1.7779052585198816e-05,0.026076717299772342,0.16035811358584917,0.35452464878403533,0.34835312954101988,0.11059883719783888,7.0774538899166743e-05]},{"sample":"sample13","allele":"locus1_allele2","posterior":[7.0774538899159885e-05,0.11059883719783878,0.34835312954101977,0.35452464878403539,0.16035811358584909,0.026076717299772591,1.7779052585197759e-05]},{"sample":"sample13","allele":"locus2_allele1","posterior":[1.9809561053446516e-05,0.082544332485495503,0.42630527494843834,0.39679189488120958,0.090625552295208564,0.0037129432012466589,1.9262734792274857e-07]},{"sample":"sample13","allele":"locus2_allele2","posterior":[8.1149738178202976e-06,0.077876980136024385,0.42807302321603025,0.39244047323938064,0.096527244133488316,0.0050734678215010025,6.9647975767871497e-07]},{"sample":"sample13","allele":"locus2_allele3","posterior":[0.0010511223872056445,0.70312659170329084,0.2747107056308728,0.020615901551905947,0.0004927805009055711,2.8981717123425464e-06,5.4106795634209185e-11]},{"sample":"sample13","allele":"locus3_allele1","posterior":[1.1246981417128631e-06,0.012987950428756865,0.18579691503414006,0.47385945735545382,0.29458988126538599,0.032760122760694205,4.5484574273348139e-06]},{"sample":"sample13","allele":"locus3_allele2","posterior":[0.00090747903064326081,0.63902402826815141,0.32678273940601626,0.032331978567004122,0.00094741946849667691,6.3551451930668558e-06,1.1449518619631165e-10]},{"sample":"sample13","allele":"locus3_allele3","posterior":[0.00055828491389561395,0.6206005877281332,0.34300131739167311,0.034704688184733337,0.0011260770164614809,9.0445246024600105e-06,2.4050076631855422e-10]},{"sample":"sample13","allele":"locus3_allele4","posterior":[0.0013718049550507688,0.87774387850980951,0.11746230638254862,0.003356031228700295,6.5638148443482994e-05,3.4076699196503336e-07,8.4553862258402045e-12]},{"sample":"sample14","allele":"locus1_allele1","posterior":[2.6416805470649759e-05,0.045264495437119495,0.24950177087314054,0.41273498662324515,0.25306853817401415,0.039395686058825957,8.1060281841060449e-06]},{"sample":"sample14","allele":"locus1_allele2","posterior":[8.1060281841052232e-06,0.039395686058825874,0.25306853817401426,0.4127349866232447,0.24950177087314102,0.045264495437119481,2.6416805470648109e-05]},{"sample":"sample14","allele":"locus2_allele1","posterior":[6.0620147834480062e-07,0.0088060535800993118,0.15241084744961647,0.46111956788734487,0.33472445566934411,0.04293155249679622,6.9167153206416925e-06]},{"sample":"sample14","allele":"locus2_allele2","posterior":[2.410752307593203e-05,0.1472960768814843,0.51211990286686548,0.29427765336256895,0.044842874162168873,0.0014392666377669892,1.1856606947600144e-07]},{"sample":"sample14","allele":"locus2_allele3","posterior":[0.94512723251725417,0.05327820025899456,0.001576963733029918,1.7516600216194627e-05,8.6760650205891878e-08,1.2985420773010647e-10,7.0806579352049823e-16]},{"sample":"sample14","allele":"locus3_allele1","posterior":[5.6846612682968525e-06,0.032500763864750953,0.26915223060451832,0.45863227735596818,0.21839974587506344,0.021306311795255106,2.985843175700015e-06]},{"sample":"sample14","allele":"locus3_allele2","posterior":[0.0009481530876701387,0.59010928255427908,0.35613555938942354,0.05033745620427979,0.0024389331740666278,3.0614434807836968e-05,1.1554729076600091e-09]},{"sample":"sample14","allele":"locus3_allele3","posterior":[8.3887870687195687e-05,0.28522538818333121,0.51205696177059523,0.18001569527843594,0.021898840607362923,0.00071913989364172067,8.6395945798029008e-08]},{"sample":"sample14","allele":"locus3_allele4","posterior":[0.96388814844640414,0.035080910968633805,0.0010179055377583432,1.2870828415497664e-05,1.6350580873179509e-07,7.1296146666303876e-10,1.8047505167735681e-14]},{"sample":"sample15","allele":"locus1_allele1","posterior":[1.0602152882555393e-05,0.027224005304396729,0.19998674746860753,0.4134143452856272,0.30412450431468424,0.055226810015735824,1.2985458065886573e-05]},{"sample":"sample15","allele":"locus1_allele2","posterior":[1.2985458065885258e-05,0.055226810015735719,0.30412450431468435,0.41341434528562682,0.19998674746860792,0.027224005304396722,1.060215288255479e-05]},{"sample":"sample15","allele":"locus2_allele1","posterior":[3.6514168551266333e-08,0.001143771797154683,0.043533536908260555,0.29707104175717913,0.50213420325429503,0.15605339413098263,6.401563795937393e-05]},{"sample":"sample15","allele":"locus2_allele2","posterior":[0.00014812679969779464,0.35542552773835201,0.50996165709533614,0.12583910931357051,0.0085015006451255593,0.00012407366795859283,4.7399594935712764e-09]},{"sample":"sample15","allele":"locus2_allele3","posterior":[0.94223355876795067,0.055995974610464488,0.0017498074714460724,2.0551177977659734e-05,1.0780099916657435e-07,1.7116102934080655e-10,9.9186355546868895e-16]},{"sample":"sample15","allele":"locus3_allele1","posterior":[1.7005925608900533e-06,0.017952833027690983,0.22597589278763056,0.48788381009390885,0.24676375181829596,0.021419789979726822,2.2217001858827536e-06]},{"sample":"sample15","allele":"locus3_allele2","posterior":[0.94408017204227568,0.054272989049561057,0.0016284133787598841,1.8340391386141802e-05,8.5028035896335684e-08,1.0998100010703928e-10,4.3498858883989396e-16]},{"sample":"sample15","allele":"locus3_allele3","posterior":[1.3924421417376397e-05,0.12877039170665097,0.50099629363370257,0.31208358254726276,0.055874170849463251,0.0022613559620239753,2.8087947905720953e-07]},{"sample":"sample15","allele":"locus3_allele4","posterior":[0.98972550264439907,0.010194303237787245,7.9933494646359377e-05,2.5981865377444904e-07,8.0370910683420037e-10,8.0441484864079652e-13,4.3811908936369828e-18]},{"sample":"sample16","allele":"locus1_allele1","posterior":[1.1443135173190716e-09,3.5167233544612527e-05,0.0024836140006135013,0.045301152355828898,0.30618609172207223,0.64234243169145089,0.003651541852176267]},{"sample":"sample16","allele":"locus1_allele2","posterior":[0.0036515418521759191,0.64234243169145133,0.30618609172207206,0.045301152355828968,0.002483614000613497,3.5167233544612601e-05,1.1443135173190091e-09]},{"sample":"sample16","allele":"locus2_allele1","posterior":[1.1032544786822759e-05,0.050105063481279875,0.32730423736391334,0.44214210420501893,0.16742795243608319,0.013008136468406322,1.4735005115049039e-06]},{"sample":"sample16","allele":"locus2_allele2","posterior":[7.4749135394731758e-06,0.064944142488245635,0.37267574301571743,0.41042250835265276,0.14004117644978042,0.011905818516870783,3.1362631935140805e-06]},{"sample":"sample16","allele":"locus2_allele3","posterior":[0.85114786034607648,0.13646660304361843,0.011967198896740108,0.00041167657505179929,6.627152224514233e-06,3.3985615729405598e-08,6.7292891209789624e-13]},{"sample":"sample16","allele":"locus3_allele1","posterior":[3.6178015873718869e-05,0.056141016453044243,0.25277181250205355,0.38264556000917871,0.25238762923093189,0.055981724311821093,3.6079477096853924e-05]},{"sample":"sample16","allele":"locus3_allele2","posterior":[0.44579159521263706,0.34634836704183958,0.16657171802622631,0.036814039223341422,0.0042843523784512052,0.00018988908434959596,3.9033154838450822e-08]},{"sample":"sample16","allele":"locus3_allele3","posterior":[0.00019498541465119281,0.2896267283334279,0.44039325275126656,0.21012861037664718,0.053595224621097858,0.0060560913099849852,5.1071929243419101e-06]},{"sample":"sample16","allele":"locus3_allele4","posterior":[0.86365806853232852,0.12019837611534674,0.015103188939651463,0.0009630491650706143,7.4754003586100488e-05,2.5625191404971673e-06,7.2487612777111997e-10]}]}} diff --git a/tests/fixtures/updog_norm.json b/tests/fixtures/updog_norm.json new file mode 100644 index 0000000..356ce77 --- /dev/null +++ b/tests/fixtures/updog_norm.json @@ -0,0 +1 @@ +{"updog_version":"2.1.7","overdispersion":30,"prior_fits":[{"ploidy":4,"weights":[10,40,30,15,5],"prior":[0.1312296741165917,0.32175630489059664,0.34735299135162001,0.16510642846598489,0.034554601175206819],"mean":1.5933161563284968,"standard_deviation":1.1041144317266938},{"ploidy":4,"weights":[90,9,0.90000000000000002,0.089999999999999997,0.01],"prior":[0.88647743487006292,0.11009293060422506,0.0034033949079678681,2.6189452646272775e-05,5.0165097869294761e-08],"mean":-1,"standard_deviation":0.84799965812119571},{"ploidy":4,"weights":[0.01,0.5,3,30,66],"prior":[2.4791131457725731e-05,0.0017752171236168842,0.040350142783363858,0.29112341735503783,0.66672643160652378],"mean":4.2221073625621273,"standard_deviation":0.93351198751283382},{"ploidy":4,"weights":[50,0.5,0.20000000000000001,0.5,50],"prior":[0.19999980000002995,0.20000009999995491,0.20000020000002983,0.20000009999995491,0.19999980000002995],"mean":2.0000000000434706,"standard_deviation":1000},{"ploidy":4,"weights":[20,20,20,20,20],"prior":[0.19998303263041067,0.20000848346886743,0.20001696780144387,0.20000848346886743,0.19998303263041067],"mean":2.0000000000000018,"standard_deviation":108.56878413744364},{"ploidy":4,"weights":[0,1,98,1,0],"prior":[1.0624642002301963e-08,0.0099999575014443932,0.980000063747827,0.009999957501444473,1.0624642002302152e-08],"mean":2.0000000000000004,"standard_deviation":0.33023010911314243},{"ploidy":4,"weights":[60,40,0.001,0.001,0.001],"prior":[0.6000223180841463,0.39987768338452045,9.9998521949613576e-05,9.3835385971408161e-12,3.3040480839077368e-22],"mean":0.44855341122914377,"standard_deviation":0.35605567598128257},{"ploidy":2,"weights":[30,50,20],"prior":[0.30000002371639861,0.49999998964194148,0.19999998664165977],"mean":0.85794246292263099,"standard_deviation":0.83708645852950136},{"ploidy":2,"weights":[49,2,49],"prior":[0.33333327777777327,0.33333344444445367,0.33333327777777327],"mean":1,"standard_deviation":1000},{"ploidy":2,"weights":[77,51,45],"prior":[0.41660543157782054,0.34036775085402354,0.24302681756815622],"mean":-1,"standard_deviation":2.7242602799818805},{"ploidy":6,"weights":[1,2,5,20,40,25,7],"prior":[0.0010244040066018636,0.012743420461425903,0.077771602713713209,0.232850072462219,0.3420203900860887,0.24646083223496973,0.087129278034981758],"mean":4.0398854703350882,"standard_deviation":1.1849947940577517},{"ploidy":6,"weights":[61,13,7,21,8,0.40000000000000002,2.7999999999999998],"prior":[0.34048192450262638,0.27187911498251977,0.18685819887630328,0.11053580884470431,0.056279256714316238,0.02466313254126673,0.009302563538263477],"mean":-1.0000000000000002,"standard_deviation":2.5819630680837995},{"ploidy":3,"weights":[5,30,40,10],"prior":[0.05640631106773971,0.36019267771509422,0.46333682003933691,0.12006419117782922],"mean":1.6571641598980118,"standard_deviation":0.79001588312605475},{"ploidy":8,"weights":[2,9,20,30,22,11,4,1.5,0.5],"prior":[0.026607266983295748,0.091138491554769344,0.19725720206076663,0.26976982166208197,0.23312203451484756,0.12729248009374661,0.043918918313101359,0.009574806061775798,0.0013189787556150466],"mean":3.1819487906089261,"standard_deviation":1.4759160891310141}],"markers":[{"name":"random_mating_tetraploid","ploidy":4,"allele_depth":[0,40,44,25,10,19,14,16,18,8,36,30,8,15,0,14,23,28,13,0,14,8,30,3,0,0,21,12,0,20,17,49,20,8,0,0,17,0,25,62,14,0,6,0,24,0,23,7,0,0,3,30,5,12,0,0,0,8,49,27,10,8,0,37,0,16,0,5,18,26,18,0,30,0,58,10,5,1,27,0,12,35,0,0,0,33,18,12,0,0],"marker_depth":[79,55,70,39,40,75,50,31,77,36,86,35,15,43,42,83,50,63,34,74,43,41,75,27,63,87,90,34,15,72,67,73,86,63,72,71,68,20,64,87,44,47,50,30,64,24,26,29,35,16,27,85,15,81,90,25,85,23,85,46,62,47,52,89,37,61,25,15,50,82,53,68,84,17,78,54,60,21,65,74,40,46,48,55,81,77,85,75,20,76],"prior":[0.31667605025806073,0.42043487848948802,0.21638099280825182,0.043169432108396535,0.0033386463358029881],"mean":0.79906350672312965,"standard_deviation":1.0272447177524846,"genotypes":[0,3,2,2,1,1,1,2,1,1,2,3,2,1,0,1,2,2,1,0,1,1,2,1,0,0,1,1,0,1,1,2,1,1,0,0,1,0,1,3,1,0,1,0,1,0,3,1,0,0,1,1,1,1,0,0,0,1,2,2,1,1,0,2,0,1,0,1,1,1,1,0,1,0,3,1,1,1,2,0,1,3,0,0,0,2,1,1,0,0],"posteriors":[[0.99996516181658468,3.4838122897014025e-05,6.0518153630393196e-11,1.2685502496862858e-18,2.3910771156345458e-35],[3.879163399326984e-16,0.00053262976305656115,0.3868700926112128,0.61259727650059337,1.1251366090388243e-09],[1.273628638731333e-14,0.0056416737592802635,0.84370063709844301,0.15065768913012681,1.2137363480836706e-11],[1.8298169654925595e-13,0.012262374788018393,0.80434395194313135,0.18339367319469035,7.3977230553445083e-11],[5.7044183970562704e-08,0.95045488701715131,0.049534384264070946,1.0671674594105388e-05,2.1553800257346267e-18],[1.9781226304097182e-08,0.96977322648670128,0.030225470622518864,1.2831095534662254e-06,1.8225338919165922e-20],[1.5846871312910211e-08,0.92417492958332459,0.075809275026047049,1.5779543756987471e-05,2.030884984481449e-18],[4.6619691157784093e-11,0.14228415573548725,0.83336677363505851,0.02434907058136019,1.474499435533381e-12],[3.720631647887194e-08,0.98242394826453217,0.017575577489655943,4.3703949511921128e-07,3.6146227364708763e-21],[1.5681804320669395e-07,0.96786879914362134,0.032125237720399324,5.806317936212715e-06,1.2131630500192809e-18],[5.7607506782379234e-11,0.3618909354644862,0.63718285703071265,0.00092620744719330167,3.2961411007005676e-16],[1.786163306944008e-17,2.8243420882612955e-05,0.05862284092836266,0.94134867555961732,2.4009113734619647e-07],[8.91835248071333e-10,0.22849408780571653,0.72377432450157897,0.047731586751170854,4.9698264782466595e-11],[2.8803764284953861e-09,0.73137170763165016,0.26833998915731444,0.00028830033065872042,2.4879762985080318e-16],[0.9990838139139675,0.00091612820268494922,5.7883281826781703e-08,6.5588662670924132e-14,1.0991330230087e-28],[4.2548297611869919e-07,0.9976893214164072,0.0023102443878083826,8.7128082568014323e-09,1.1450512245476593e-23],[5.0110377048307781e-11,0.22999917808184811,0.76493273467058387,0.0050680871974357463,2.150393651106032e-14],[4.4703479156967709e-11,0.26465931939116377,0.73263715426914777,0.0027035262949811328,4.081402319294386e-15],[1.9825691444141017e-09,0.593077540217822,0.40569835541292093,0.0012241023866829907,4.9043691329069633e-15],[0.99994971625555207,5.0283614649432967e-05,1.2979850762203297e-10,4.1832627404021112e-18,1.2624946201836103e-34],[5.6392994863944071e-09,0.81183496254214293,0.18803251893654585,0.000132512882011963,7.4822025970134713e-17],[3.1124955435530476e-07,0.98488399064837973,0.01511460143953096,1.096662535257679e-06,7.4932689535087969e-20],[1.5741855535005948e-10,0.4736758641015007,0.52572032892471021,0.00060380681637011249,2.0919581627371633e-16],[1.0560482516179621e-05,0.9939004184758875,0.0060885157074184546,5.0533417772040104e-07,8.8590347615397912e-20],[0.99987916834574353,0.00012083084544420453,8.0881235534898819e-10,7.4004813672787851e-17,7.075715574535882e-33],[0.99997994375977783,2.0056220970395677e-05,1.9251775011515199e-11,2.1263256383642125e-19,1.9970713903095232e-36],[3.1712118163726146e-08,0.98506341966938127,0.014936301489149949,2.4712935050785707e-07,1.0996247202111947e-21],[4.518001863717582e-09,0.70940805198096102,0.29004972491375208,0.00054221858728381882,1.3148752766083748e-15],[0.95967098790256589,0.040107748330113505,0.00022120835698385762,5.5410336810752884e-08,8.8022619110010774e-20],[9.4626789316352598e-09,0.94285830210929766,0.057136979956845031,4.7084711785834958e-06,1.334845276889758e-19],[2.2713003803011557e-08,0.96645293838965285,0.033545132257771226,1.90663957229873e-06,4.2822126126979865e-20],[1.2977175723228961e-15,0.0016458941372579409,0.67182926905213491,0.32652483673513294,7.5473222794083679e-11],[3.4239950245432943e-08,0.98474493403425523,0.015254754552179084,2.7717361506365624e-07,1.4412976459275601e-21],[3.4379450594550454e-06,0.9989476377556018,0.0010489207319590778,3.5673801720549126e-09,6.8949457782233424e-24],[0.99994146355737401,5.8536264510581085e-05,1.7811534330680324e-10,6.8672922259906655e-18,2.5235236435806041e-34],[0.99993676840955859,6.3231381260501053e-05,2.0918092862259472e-10,8.8355785929270734e-18,3.5897818007438666e-34],[2.5149905229862054e-08,0.96981876423794611,0.030179693182673533,1.5174294749958215e-06,2.9635376314152735e-20],[0.98276245497346082,0.017204486926439354,3.3055929315422068e-05,2.170784442254198e-09,5.4967561645666533e-22],[3.1085572335441695e-10,0.53271006099222284,0.46673582793332208,0.00055411076359887415,2.5715665805689476e-16],[4.6368522890694811e-17,0.00026865628171325189,0.40323896151911592,0.59649238184920983,3.4996111257435309e-10],[6.6427112176544522e-09,0.83472472111583385,0.16517870728146872,9.6564959986359576e-05,4.332078653313458e-17],[0.99946297860985422,0.00053700274382806616,1.8646306999137633e-08,1.0701632621650544e-14,8.1582356144888949e-30],[5.3354341770615687e-06,0.99848751599226437,0.0015071374347214914,1.1138837089889721e-08,6.5469741803999742e-23],[0.99605048275474861,0.0039481941107342121,1.3231240839881356e-06,1.0433106881508028e-11,1.7761208975230735e-25],[5.2980736168732497e-10,0.61656256363093587,0.38310894832258019,0.00032848751667619551,1.101009836632793e-16],[0.99075335713819945,0.0092382421670519511,8.4004751570545064e-06,2.1959155002018806e-10,1.6757426560109105e-23],[1.5201898349291946e-16,6.7142267070029044e-05,0.066668821312586252,0.93326316232621565,8.7409412819534402e-07],[1.2983806451602898e-07,0.94142770881127646,0.05854416501874455,2.7996331914194386e-05,2.3295859043984884e-17],[0.99792440631942592,0.0020752621438995234,3.315355794306835e-07,1.0950419122818796e-12,6.4941810883767826e-27],[0.96620094056565176,0.03365055574856788,0.00014847579205856968,2.7893721703532267e-08,2.9540597851008153e-20],[1.0560482516179621e-05,0.9939004184758875,0.0060885157074184546,5.0533417772040104e-07,8.8590347615397912e-20],[6.5492331471610761e-10,0.73054558573731387,0.26936478967420152,8.962393356169524e-05,7.5435707568057346e-18],[6.4585943774970321e-08,0.73404707133725589,0.26382951054348458,0.0021233535331670328,1.490170385761628e-13],[1.0914840512993706e-06,0.99877598574523174,0.0012229199238123717,2.8469049856541249e-09,2.5372874709761966e-24],[0.99998353022790321,1.6469759293296492e-05,1.2803496048678584e-11,1.127144334116328e-19,8.2827208389270493e-37],[0.99202853133581814,0.0079653937140124857,6.0748219091983171e-06,1.2826007118161024e-10,7.4573444114413842e-24],[0.99997706145549414,2.293851907840434e-05,2.5427505445028608e-11,3.2796737530122204e-19,3.6444354339665263e-36],[1.4453299819394613e-08,0.71481726349454511,0.28406573038816285,0.0011169916639785274,1.3647348662920506e-14],[5.999336708306401e-14,0.015011740722895409,0.93612322297888773,0.04886503629743378,7.2286331567809052e-13],[5.8613347177346745e-13,0.02699311944473862,0.90104049081435711,0.071966389734699335,5.6185845130672705e-12],[7.2698173951575754e-07,0.99690675421636754,0.00309249280799874,2.5993894082687161e-08,1.1528277676617993e-22],[6.5156090969620953e-07,0.99336266121826178,0.0066365039418825452,1.8327894576518063e-07,3.9564729124686737e-21],[0.99967386677288339,0.00032612672181387434,6.5053004944107208e-09,2.0013234573251258e-15,7.481315277289367e-31],[6.0134233520742712e-11,0.37536816243807819,0.62381628276576118,0.00081555473602589498,2.4823063859263672e-16],[0.99837187837578467,0.0016279244685942092,1.971551489771415e-07,4.7211416709978859e-13,1.9082421073512477e-27],[1.9753901049334039e-08,0.95548584659394409,0.044510250879598146,3.8827725567174008e-06,1.5310497241716955e-19],[0.99202853133581814,0.0079653937140124857,6.0748219091983171e-06,1.2826007118161024e-10,7.4573444114413842e-24],[6.4585943774970321e-08,0.73404707133725589,0.26382951054348458,0.0021233535331670328,1.490170385761628e-13],[1.4643185331471683e-09,0.68854151180613998,0.31115249398481698,0.00030599274472438074,1.8185189173009601e-16],[2.2401769567437507e-09,0.86722385581623374,0.13275556280267992,2.0579140909775676e-05,8.7129667165435487e-19],[2.4068852326695954e-09,0.77370595701887146,0.22616046715035701,0.00013357342388623913,4.2930908044818112e-17],[0.99991991025941174,8.0089398136591456e-05,3.424517075185641e-10,1.9150415583705011e-17,1.0603884654750661e-33],[5.794693261351731e-10,0.70974951174384837,0.29014217783753832,0.00010830983914417817,1.0382503799082654e-17],[0.97157855583679309,0.028320813089437298,0.00010061674438563401,1.4329383946004879e-08,1.0332204509480317e-20],[1.2446769778872797e-17,9.777881467385092e-05,0.2418353820847543,0.75806683773580963,1.3647623405879561e-09],[3.2030971515619696e-07,0.99236913525598713,0.0076303658152783701,1.7861901931794163e-07,2.5656836977055356e-21],[4.0087166263341071e-05,0.99968193917277137,0.00027797328314690262,3.7781819514174419e-10,3.6215295299930933e-25],[0.00022756280751462529,0.99594494000526557,0.0038271201967242387,3.7699049528345379e-07,1.2727958906495071e-19],[1.2351575793784968e-10,0.39849422976300636,0.60033955687948704,0.0011662132339901867,8.6640694063601454e-16],[0.99994971625555207,5.0283614649432967e-05,1.2979850762203297e-10,4.1832627404021112e-18,1.2624946201836103e-34],[1.3548534685284536e-08,0.87419637402386297,0.12573735964693172,6.6252780670519144e-05,3.2164400229619627e-17],[2.4745382439031614e-16,0.00029340611520070994,0.25679717834942362,0.74290941054885007,4.9865249091918115e-09],[0.99951526226392695,0.00048472272136488865,1.5014700785487533e-08,7.5749334533826667e-15,4.9799678740640391e-30],[0.99975451701772811,0.00024547940489640476,3.5773748105434115e-09,7.7504932271175982e-16,1.9476725523063068e-31],[0.99996977069023762,3.0229264689082367e-05,4.5073386063267112e-11,8.0078336341066883e-19,1.2603583575289169e-35],[4.965693569673836e-11,0.3186182573290321,0.67997400658949136,0.0014077360318191627,8.4623991241795702e-16],[7.389302671430888e-08,0.99153293866834458,0.0084668942499269064,9.3188701680834444e-08,3.1094800836568243e-22],[6.6748938810331251e-07,0.99791368554705773,0.0020856382302555203,8.7332986412677318e-09,1.4922326998233294e-23],[0.98276245497346082,0.017204486926439354,3.3055929315422068e-05,2.170784442254198e-09,5.4967561645666533e-22],[0.99995667487026774,4.3325034522169875e-05,9.5210062102373511e-11,2.5757724384659699e-18,6.4165783598828404e-35]],"max_posterior":[0.99996516181658468,0.61259727650059337,0.84370063709844301,0.80434395194313135,0.95045488701715131,0.96977322648670128,0.92417492958332459,0.83336677363505851,0.98242394826453217,0.96786879914362134,0.63718285703071265,0.94134867555961732,0.72377432450157897,0.73137170763165016,0.9990838139139675,0.9976893214164072,0.76493273467058387,0.73263715426914777,0.593077540217822,0.99994971625555207,0.81183496254214293,0.98488399064837973,0.52572032892471021,0.9939004184758875,0.99987916834574353,0.99997994375977783,0.98506341966938127,0.70940805198096102,0.95967098790256589,0.94285830210929766,0.96645293838965285,0.67182926905213491,0.98474493403425523,0.9989476377556018,0.99994146355737401,0.99993676840955859,0.96981876423794611,0.98276245497346082,0.53271006099222284,0.59649238184920983,0.83472472111583385,0.99946297860985422,0.99848751599226437,0.99605048275474861,0.61656256363093587,0.99075335713819945,0.93326316232621565,0.94142770881127646,0.99792440631942592,0.96620094056565176,0.9939004184758875,0.73054558573731387,0.73404707133725589,0.99877598574523174,0.99998353022790321,0.99202853133581814,0.99997706145549414,0.71481726349454511,0.93612322297888773,0.90104049081435711,0.99690675421636754,0.99336266121826178,0.99967386677288339,0.62381628276576118,0.99837187837578467,0.95548584659394409,0.99202853133581814,0.73404707133725589,0.68854151180613998,0.86722385581623374,0.77370595701887146,0.99991991025941174,0.70974951174384837,0.97157855583679309,0.75806683773580963,0.99236913525598713,0.99968193917277137,0.99594494000526557,0.60033955687948704,0.99994971625555207,0.87419637402386297,0.74290941054885007,0.99951526226392695,0.99975451701772811,0.99996977069023762,0.67997400658949136,0.99153293866834458,0.99791368554705773,0.98276245497346082,0.99995667487026774],"proportion_misgenotyped":0.11308999889967752},{"name":"one_family_hexaploid","ploidy":6,"allele_depth":[36,26,42,42,8,21,34,9,18,33,13,22,30,42,31,35,31,40,47,28,26,42,19,38,24,15,27,29,25,36,26,37,12,13,23,29,15,38,37,17,12,7,13,35,31,34,4,30,13,25,26,49,13,29,18,14,47,21,38,9,32,36,41,18,55,13,22,10,14,18,51,14,5,28,26,27,36,18,13,40],"marker_depth":[70,33,68,84,18,29,65,20,87,71,29,51,65,74,77,52,49,80,85,48,47,52,41,81,55,87,60,54,60,74,40,60,32,39,79,51,46,56,89,37,24,17,21,64,57,47,18,70,50,35,49,75,40,68,28,32,67,32,82,18,69,71,66,42,69,39,43,23,27,33,71,28,15,47,68,70,52,37,38,78],"prior":[4.0190448757549455e-09,0.00014235258588856544,0.082846138400906127,0.79221595275838219,0.12447418777661345,0.00032135082765540713,1.3631509439152121e-08],"mean":3.0495442201195413,"standard_deviation":0.49334858393017123,"genotypes":[3,4,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3,3,3,3,4,3,3,3,3,3,3,3,3,3],"posteriors":[[6.5772882961701224e-21,4.3162297712163933e-07,0.022593373036869601,0.92648508921020334,0.050918895090582339,2.2110393674735468e-06,7.7985357837014697e-20],[6.8806385935533073e-24,1.8232254164548582e-09,0.00068385859066539731,0.37993688872082609,0.61708843315677875,0.00229081770846771,3.6970800257999042e-14],[7.6608489807662755e-23,2.1160265500260359e-08,0.004605632390284488,0.79975092226226674,0.19560221265646671,4.1211530716517048e-05,8.0702625621990954e-18],[6.8805478744498778e-21,5.0779075061797516e-07,0.026202789764770251,0.93442654322300289,0.039369012920206573,1.1463012695930611e-06,2.3336944973953903e-20],[1.7028452168100833e-17,1.6420264967311183e-05,0.071688094362633278,0.88292405832457832,0.04536541427927393,6.0127685473156009e-06,2.8059776076994367e-18],[2.2433819807343563e-22,1.9167749962811385e-08,0.0026628928457379381,0.5947284801081687,0.40200116225261145,0.00060744562572859553,3.482981415900929e-15],[5.8627792592083704e-21,3.7770690421387265e-07,0.020517874710188736,0.92096101718369583,0.058517611719292095,3.1186799190270212e-06,1.4473019029051705e-19],[9.5296342069833019e-18,1.3065972903071077e-05,0.067700311662024185,0.88785280522031429,0.044428612036374472,5.2051083841520207e-06,1.864741058009004e-18],[9.3595606186380814e-15,0.0025441890950089587,0.7758589174926348,0.22150693130493029,8.9962079673228874e-05,2.7743606085284047e-11,6.0728450899348035e-27],[5.5928339237931755e-20,1.7695210293409861e-06,0.045666946472598821,0.92919430677366766,0.02513645211085452,5.2512184954564604e-07,8.5618630991933312e-21],[2.3369315504917721e-18,8.2971034345575282e-06,0.065188656097880091,0.90038189470570174,0.034418990915466748,2.1611775167880387e-06,2.5415558896012645e-19],[6.3392205167030946e-19,6.3793554250661726e-06,0.074151997484312185,0.90674487982287111,0.019096344214964311,3.9912242703375501e-07,8.4670581930881268e-21],[8.4091959741675424e-20,2.1512883667344864e-06,0.048570370811702603,0.92636695408790026,0.025059965303964447,5.5850806590935187e-07,1.0409048482916402e-20],[5.0317914840845547e-22,8.1271280386738035e-08,0.0098117171296085177,0.88614026964097414,0.10403843644413471,9.4955140022378229e-06,7.028936276503414e-19],[7.6395637374800673e-19,1.0072714052563447e-05,0.10750844335984626,0.8832124248436326,0.0092689881437184353,7.0938749947026471e-08,3.9193364638420523e-22],[2.819142626597763e-23,8.3005414903065041e-09,0.0023614718244288793,0.66341176368401,0.33403219083672714,0.00019456535429244649,1.562522760644933e-16],[2.2084875133750559e-22,3.2763372998009562e-08,0.0049752641127321787,0.77450922177282711,0.22044278162825071,7.2699722817299873e-05,3.2316273856274293e-17],[7.9993755082159545e-21,5.4186438273965354e-07,0.0266047885596301,0.93342044065897112,0.039973005696961335,1.2232200549651717e-06,2.7131703676454108e-20],[5.8003404724819665e-22,9.8897567544197368e-08,0.011460740621772115,0.90440431299587853,0.084129382399571129,5.4650852109544858e-06,2.5439791016288171e-19],[1.7989045472641793e-21,1.331688639698824e-07,0.010493264336245752,0.86107469877699061,0.12841039592924325,2.1507788656552952e-05,4.5412271605892912e-18],[6.5375223605254273e-21,3.0823092643103926e-07,0.016078275018060036,0.89299524295902688,0.090915828652429045,1.0345139557741145e-05,1.4518270107931024e-18],[4.2082502757995699e-26,7.0231188252251494e-11,0.00011639490619113499,0.20793673220522957,0.78731585524885184,0.0046310175694314277,6.4551750262669187e-14],[3.6805414065132439e-19,3.6957567818414214e-06,0.052019702346414641,0.9150181398149877,0.032957033120180168,1.4289616353983126e-06,7.9726382624940216e-20],[3.147131314208686e-20,1.3412820116070302e-06,0.04182290570446006,0.93293755576435811,0.025237716329612123,4.8091955796998298e-07,6.4898231245918123e-21],[4.0994094585494841e-19,5.1711400180717043e-06,0.06926022349141657,0.91131089546223787,0.019423323073549728,3.8683277766843302e-07,7.3292972837499516e-21],[6.1941162976009791e-14,0.0063742474537012402,0.87018025784758091,0.12341881872727151,2.6675966796692909e-05,4.5878336078564131e-12,5.7804442397022684e-28],[1.7662824770554653e-19,3.2510397938317276e-06,0.057464372890611917,0.92028491792491129,0.022246987974261792,4.701704209279388e-07,8.8310064076954841e-21],[6.7881482860026091e-21,3.5710365782578117e-07,0.018453590623000834,0.90845320536142027,0.073087078197580591,5.7687143404723795e-06,4.7707627841864419e-19],[7.4497742615416915e-19,8.1338412900207883e-06,0.088751268184096191,0.89741431465101895,0.013826097525464074,1.8579813091459248e-07,2.1874130891229305e-21],[1.8752937343333141e-20,8.9565327718089339e-07,0.0331979482131423,0.93301621096699716,0.03378402457663103,9.2058995232414447e-07,1.9159178175499845e-20],[3.6700383899507143e-22,3.8086402146430771e-08,0.0048357581596849197,0.74463080159500006,0.25041272847090046,0.0001206736880120262,1.0501438857638137e-16],[1.478547535978462e-22,2.9885326213456264e-08,0.0052428273435573425,0.80409673563125483,0.190617309134549,4.3098005312336745e-05,1.0225960980458151e-17],[2.2722865902548237e-17,3.6486992643013728e-05,0.13421734715204031,0.85167726553365763,0.014068558516125736,3.4180553322368669e-07,1.2916838234689173e-20],[6.9918871664012807e-17,8.7183951586348519e-05,0.21273175375708442,0.78071748181511813,0.0064635140542785904,6.6421932487296455e-08,8.4983453267472326e-22],[1.4649291790880885e-16,0.00025273880516886823,0.41562830219206792,0.5830104396967376,0.0011085177304857364,1.575539269262889e-09,1.6069980146733949e-24],[2.4216038211437814e-21,1.7150772026593194e-07,0.01232919638782499,0.87958262468011916,0.10807403787754243,1.3969546793060143e-05,2.085683332005668e-18],[6.7592876768651829e-17,0.00010088086511480959,0.24197723542640626,0.75328082160869869,0.0046410310203296602,3.1079450755758895e-08,2.2284958014632054e-22],[1.4014086787225864e-23,5.4478620352523756e-09,0.0019285445682705018,0.63863613618065329,0.35921495127283909,0.00022036253037496504,1.7321454635623376e-16],[3.0067154198206141e-19,6.1646729435163761e-06,0.089793909280533352,0.89979593956852144,0.010403906853306085,7.9624695611258698e-08,3.9929308482777734e-22],[6.2485987783366539e-19,4.6742898831513658e-06,0.055582186941806638,0.91102176608379515,0.033389760099178205,1.612585336779477e-06,1.1146703864872085e-19],[9.6214722076034603e-19,3.9312228455260815e-06,0.041657998130231681,0.89573913302231212,0.062590063170451585,8.8744541593079462e-06,3.2633414020859658e-18],[5.2846846313082469e-17,2.972637786336083e-05,0.091609742433685054,0.87185542249327719,0.036501026736696404,4.081958477970145e-06,1.647504585255741e-18],[5.7532159044025418e-20,5.7252208404401192e-07,0.014698073961354844,0.8163387648458682,0.16887154795950302,9.1040711189633086e-05,2.0958407879050991e-16],[2.2357631238106876e-21,1.9692887161653187e-07,0.01463069677668999,0.90493608652702573,0.080426880437484735,6.1393299281912829e-06,4.2273142839482235e-19],[4.0850509313598715e-21,2.6722381526078091e-07,0.016292273177658217,0.90488066624613861,0.078820329070677847,6.4642817099988269e-06,5.310002708014494e-19],[6.0094917000505925e-24,2.4856542609256386e-09,0.0010775995275309263,0.50890698008893953,0.48936374935410953,0.00065166854376410112,1.5865385665593642e-15],[1.6100049076169932e-14,0.00065659286129989057,0.33051022182512224,0.66269516393121375,0.0061378841803751225,1.3720197298448585e-07,8.1458650969787716e-21],[2.9344330556377448e-19,5.0929783868153356e-06,0.075501785513825262,0.90965521713184661,0.014837716043402003,1.8833253934484812e-07,1.8608607443821602e-21],[1.0886754063466479e-15,0.00052841790889747475,0.45919921345815751,0.53902233744268735,0.0012500281479996219,3.0422570766805739e-09,7.4018690127285368e-24],[7.7153673552968697e-23,1.1155277044693351e-08,0.002189963918031577,0.58970216534792197,0.40759630162596255,0.00051155795280503522,1.7320138713870492e-15],[1.3261521605852204e-20,5.097282267380743e-07,0.021177736819418615,0.90983057312328908,0.068985625364693631,5.5549643716497664e-06,5.1305652632523938e-19],[8.9031431610769305e-24,5.1482717300431409e-09,0.0021857522028132415,0.6984052519009728,0.299303005116975,0.00010598563096699393,3.3652572548494282e-17],[9.3153671596096636e-17,0.00010578505756647961,0.23281332277030481,0.7615449322985538,0.0055359108854004533,4.8988174078607911e-08,5.2568549647850949e-22],[3.4763425856873611e-19,5.5482820919893805e-06,0.077783284397756447,0.90756155346762335,0.014649425953471254,1.8789905710405175e-07,1.917091554299032e-21],[4.513486966295738e-21,1.4471145222473865e-07,0.008242354380868417,0.77627663431659255,0.21536028899156451,0.00012057759952195403,1.9801578304410905e-16],[2.3623364177738913e-18,9.2497179429920228e-06,0.071829080836837894,0.89948251672244983,0.028677788099217796,1.3646235514097266e-06,1.0691399826740006e-19],[1.5616700436139313e-24,1.3485511065254349e-09,0.00093117578434523266,0.53438555794782383,0.46429368596755832,0.00038957895172138688,3.6294318372563175e-16],[1.2064018649959603e-21,6.8283040378250361e-08,0.0058845343942709695,0.74452520197945482,0.24943975874509175,0.00015043659814179921,2.2749382515897961e-16],[3.9553871366404276e-20,1.5702917032334845e-06,0.045363456048963149,0.93160545016561314,0.023029126059454237,3.9743426611482352e-07,4.8045539022558432e-21],[3.6480506330186058e-18,6.5569800275691339e-06,0.046470813514631885,0.88368663396773139,0.069821193624574882,1.4801913034358403e-05,1.2373194466256116e-17],[6.3693344517354101e-20,1.8834293961385368e-06,0.046576293479318506,0.92830879078730266,0.025112496725632739,5.355783500665872e-07,9.1130100303867146e-21],[8.6156452602619894e-21,5.212158100872905e-07,0.024970998738746582,0.92917105956312651,0.045855655572717216,1.7649095997206076e-06,5.4269213783260375e-20],[7.5381644091708627e-23,2.0414022667797586e-08,0.0044528895552630923,0.79221317826349369,0.20328772806664078,4.6183700579831745e-05,1.0088446627912239e-17],[1.3027541705482276e-18,8.3802924511719721e-06,0.077566196171593452,0.90125224785620905,0.021172594888557709,5.80791188954796e-07,1.9429339876523265e-20],[6.4090254835817907e-27,2.3365680306760185e-11,6.9677180564079385e-05,0.18324919595959779,0.8125161685050708,0.004164958331369535,3.2165162205337929e-14],[6.9918871664012807e-17,8.7183951586348519e-05,0.21273175375708442,0.78071748181511813,0.0064635140542785904,6.6421932487296455e-08,8.4983453267472326e-22],[4.8350441315029892e-20,1.0522651509033317e-06,0.028728919955956358,0.91417738062618115,0.057088444503556025,4.2026491556144512e-06,3.9826434645132769e-19],[8.7753784178494559e-18,1.4477291023590581e-05,0.075817426771548707,0.88879677820106195,0.035368457568367324,2.8601679983066601e-06,5.7039033002907381e-19],[3.0882801327917637e-19,2.1804747587436223e-06,0.033044458023420972,0.89570830111370736,0.071234668955359889,1.03914327531493e-05,3.4646667390561463e-18],[4.7297214783042105e-20,8.0395706444331518e-07,0.022130286867947595,0.88952942919376088,0.088325950967672137,1.3529013554776098e-05,3.8252629069003781e-18],[4.5616050775021324e-25,5.8334552540638812e-10,0.00057591744083234969,0.45888348839144866,0.53994002100367056,0.00060057258070231041,7.2805222861315054e-16],[4.7752925170307619e-19,2.9795424106813562e-06,0.039212775353601771,0.90186133971586657,0.058916179284417572,6.7261037031766405e-06,1.6196492014582601e-18],[7.0069769383238247e-16,0.00010749426516726155,0.15183346528787953,0.82572004428647972,0.022337250242448568,1.7459180241894055e-06,5.2010911583559362e-19],[1.2227823203804147e-21,1.0080615760575038e-07,0.0089615465087440459,0.84352655036045676,0.147481953928355,2.9848396286438878e-05,7.9607829916728003e-18],[2.5397130790659187e-18,1.9486001449565663e-05,0.13895837977510361,0.85363599933584455,0.0073860839013520467,5.0986250125674759e-08,2.7734213503074848e-22],[2.0468076198762595e-18,1.7423908535213258e-05,0.13352579966700576,0.85885063124981031,0.0076060925599214279,5.2614727534415672e-08,2.8082738407355044e-22],[1.2049679467258662e-23,4.5491532381037829e-09,0.0016582139954699584,0.60108938233048526,0.39694011889930597,0.00031228022558535,3.5658764311703899e-16],[2.3099035971848289e-19,2.4902536242748321e-06,0.041000806721133105,0.91360224158894365,0.04539145454638463,3.0068899143244907e-06,2.9379682581867146e-19],[5.2032868968467138e-17,7.1369480046820871e-05,0.1934932044986763,0.79887664872677799,0.0075586866720536144,9.0622445273077693e-08,1.3913604643000182e-21],[4.858457998895769e-21,3.8297493149987288e-07,0.022155417924115507,0.92941070274764315,0.048431653957008894,1.8423963008986451e-06,5.2170881676362201e-20]],"max_posterior":[0.92648508921020334,0.61708843315677875,0.79975092226226674,0.93442654322300289,0.88292405832457832,0.5947284801081687,0.92096101718369583,0.88785280522031429,0.7758589174926348,0.92919430677366766,0.90038189470570174,0.90674487982287111,0.92636695408790026,0.88614026964097414,0.8832124248436326,0.66341176368401,0.77450922177282711,0.93342044065897112,0.90440431299587853,0.86107469877699061,0.89299524295902688,0.78731585524885184,0.9150181398149877,0.93293755576435811,0.91131089546223787,0.87018025784758091,0.92028491792491129,0.90845320536142027,0.89741431465101895,0.93301621096699716,0.74463080159500006,0.80409673563125483,0.85167726553365763,0.78071748181511813,0.5830104396967376,0.87958262468011916,0.75328082160869869,0.63863613618065329,0.89979593956852144,0.91102176608379515,0.89573913302231212,0.87185542249327719,0.8163387648458682,0.90493608652702573,0.90488066624613861,0.50890698008893953,0.66269516393121375,0.90965521713184661,0.53902233744268735,0.58970216534792197,0.90983057312328908,0.6984052519009728,0.7615449322985538,0.90756155346762335,0.77627663431659255,0.89948251672244983,0.53438555794782383,0.74452520197945482,0.93160545016561314,0.88368663396773139,0.92830879078730266,0.92917105956312651,0.79221317826349369,0.90125224785620905,0.8125161685050708,0.78071748181511813,0.91417738062618115,0.88879677820106195,0.89570830111370736,0.88952942919376088,0.53994002100367056,0.90186133971586657,0.82572004428647972,0.84352655036045676,0.85363599933584455,0.85885063124981031,0.60108938233048526,0.91360224158894365,0.79887664872677799,0.92941070274764315],"proportion_misgenotyped":0.17282181671777053},{"name":"inbred_diploid","ploidy":2,"allele_depth":[77,0,41,0,33,0,0,78,0,37,62,0,0,51,0,0,0,0,85,0,18,87,0,24,0,0,0,0,0,30,0,0,64,35,22,0,0,0,41,0,81,0,0,39,17,0,0,0,0,24,86,20,71,0,0,23,29,0,79,0,42,0,0,0,0,0,28,0,75,75],"marker_depth":[77,62,41,86,33,48,75,78,82,37,62,78,20,51,18,18,16,78,85,46,18,87,75,24,62,52,69,82,30,30,30,76,64,35,22,64,81,81,41,57,81,89,67,39,17,38,29,47,55,24,86,20,71,78,68,23,29,67,79,25,42,72,64,43,57,37,28,52,75,75],"prior":[0.36055845342992626,0.33739437694521396,0.30204716962485989],"mean":-1,"standard_deviation":4.7528751566544463,"genotypes":[2,0,2,0,2,0,0,2,0,2,2,0,0,2,0,0,0,0,2,0,2,2,0,2,0,0,0,0,0,2,0,0,2,2,2,0,0,0,2,0,2,0,0,2,2,0,0,0,0,2,2,2,2,0,0,2,2,0,2,0,2,0,0,0,0,0,2,0,2,2],"posteriors":[[5.2092612799512234e-33,1.3362770937333264e-10,0.99999999986637222],[0.99999999867640388,1.3235961734408149e-09,8.3437750617492237e-31],[2.1570890425453715e-26,1.2008272059386978e-07,0.99999987991727923],[0.99999999996971867,3.0281283381437319e-11,2.1404521810313669e-34],[2.647881985909664e-24,9.3105234583066213e-07,0.99999906894765411],[0.99999997942757313,2.0572426973123844e-08,3.9589594856596592e-28],[0.9999999998478728,1.5212711057066688e-10,7.138287106925066e-33],[3.7487232458251811e-33,1.1489488513479872e-10,0.99999999988510502],[0.99999999994661248,5.3387547588388331e-11,7.3082508831060129e-34],[2.1641471945664297e-25,3.2283071892469597e-07,0.99999967716928106],[1.188952304823712e-30,1.5799975942360814e-09,0.99999999842000231],[0.9999999999037501,9.624978596863725e-11,2.6307618394428902e-33],[0.99995393839186741,4.6061608132560692e-05,4.4440799775535903e-20],[1.3457978800636521e-28,1.3055970178219464e-08,0.99999998694402981],[0.99990344804093134,9.6551959068740851e-05,3.0561115436338611e-19],[0.99990344804093134,9.6551959068740851e-05,3.0561115436338611e-19],[0.99978959546605983,0.00021040453394009895,2.4288706814934145e-18],[0.9999999999037501,9.624978596863725e-11,2.6307618394428902e-33],[4.126539661942101e-34,4.1569320681970103e-11,0.99999999995843059],[0.99999996816770309,3.1832296896807679e-08,1.0715903136084176e-27],[4.3547469555828237e-19,0.00011525343826656864,0.99988474656173354],[2.2612475088977999e-34,3.1473040254893398e-11,0.99999999996852695],[0.9999999998478728,1.5212711057066688e-10,7.138287106925066e-33],[1.9150563317765692e-21,1.3860901863565537e-05,0.99998613909813638],[0.99999999867640388,1.3235961734408149e-09,8.3437750617492237e-31],[0.9999999910881614,8.911838463778218e-09,5.9465293873381726e-29],[0.99999999960276775,3.9723230081684452e-10,5.846916551559335e-32],[0.99999999994661248,5.3387547588388331e-11,7.3082508831060129e-34],[0.9999981808156102,1.8191843897372383e-06,1.4168841399843341e-23],[2.0189986645181146e-23,2.1715882870851743e-06,0.99999782841171281],[0.9999981808156102,1.8191843897372383e-06,1.4168841399843341e-23],[0.99999999986960519,1.3039470346189247e-10,5.0987800013622371e-33],[5.4278344738758573e-31,1.1091014154934898e-09,0.99999999889089852],[7.3683850950556635e-25,5.431145923066303e-07,0.99999945688540759],[1.0430444457312092e-20,2.7169341021853351e-05,0.99997283065897813],[0.99999999907088322,9.2911685103075506e-10,3.8091208314433819e-31],[0.99999999993827071,6.1729335214950909e-11,1.001500405333778e-33],[0.99999999993827071,6.1729335214950909e-11,1.001500405333778e-33],[2.1570890425453715e-26,1.2008272059386978e-07,0.99999987991727923],[0.99999999666827866,3.3317213445842704e-09,6.5222979856280975e-30],[1.4270952976109409e-33,7.3687279484732847e-11,0.99999999992631272],[0.99999999997993649,2.0063538217980645e-11,8.7999626937038923e-35],[0.99999999944518247,5.5481750325613639e-10,1.2199452396425441e-31],[6.6786631208178101e-26,1.9528585727467992e-07,0.99999980471414274],[1.2038915191433981e-18,0.00016926923464348676,0.99983073076535667],[0.99999979010469953,2.098953004466515e-07,8.3871124777995091e-26],[0.99999756064186685,2.4393581330560282e-06,2.8826201182666964e-23],[0.99999997445038302,2.5549616965701258e-08,6.4859482027940036e-28],[0.99999999509962456,4.900375432390184e-09,1.5479841945740292e-29],[1.9150563317765692e-21,1.3860901863565537e-05,0.99998613909813638],[3.0500529266500741e-34,3.6147244805538115e-11,0.99999999996385269],[6.3325676306133074e-20,5.498397633653393e-05,0.99994501602366348],[4.0648199580632155e-32,3.4198633874468849e-10,0.99999999965801356],[0.9999999999037501,9.624978596863725e-11,2.6307618394428902e-33],[0.99999999953097896,4.6902112723237482e-10,8.4264322619297107e-32],[4.4129234961857671e-21,1.933219377698286e-05,0.99998066780622297],[4.1076085938527724e-23,2.9118988395201313e-06,0.9999970881011605],[0.99999999944518247,5.5481750325613639e-10,1.2199452396425441e-31],[2.7074130265103262e-33,9.8937289683569626e-11,0.9999999999010627],[0.99999161382285473,8.3861771452429924e-06,5.973107780324293e-22],[1.245644437606563e-26,9.4713174158489888e-08,0.99999990528682581],[0.99999999975605891,2.4394113568416959e-10,2.0052880051485191e-32],[0.99999999907088322,9.2911685103075506e-10,3.8091208314433819e-31],[0.99999993718664038,6.2813359638203777e-08,5.0992880628388734e-27],[0.99999999666827866,3.3317213445842704e-09,6.5222979856280975e-30],[0.99999972955811567,2.7044188426156212e-07,1.518745391222287e-25],[8.5085956413485834e-23,3.9276714832747492e-06,0.99999607232851673],[0.9999999910881614,8.911838463778218e-09,5.9465293873381726e-29],[1.0171754208842248e-32,1.8159652739811551e-10,0.99999999981840348],[1.0171754208842248e-32,1.8159652739811551e-10,0.99999999981840348]],"max_posterior":[0.99999999986637222,0.99999999867640388,0.99999987991727923,0.99999999996971867,0.99999906894765411,0.99999997942757313,0.9999999998478728,0.99999999988510502,0.99999999994661248,0.99999967716928106,0.99999999842000231,0.9999999999037501,0.99995393839186741,0.99999998694402981,0.99990344804093134,0.99990344804093134,0.99978959546605983,0.9999999999037501,0.99999999995843059,0.99999996816770309,0.99988474656173354,0.99999999996852695,0.9999999998478728,0.99998613909813638,0.99999999867640388,0.9999999910881614,0.99999999960276775,0.99999999994661248,0.9999981808156102,0.99999782841171281,0.9999981808156102,0.99999999986960519,0.99999999889089852,0.99999945688540759,0.99997283065897813,0.99999999907088322,0.99999999993827071,0.99999999993827071,0.99999987991727923,0.99999999666827866,0.99999999992631272,0.99999999997993649,0.99999999944518247,0.99999980471414274,0.99983073076535667,0.99999979010469953,0.99999756064186685,0.99999997445038302,0.99999999509962456,0.99998613909813638,0.99999999996385269,0.99994501602366348,0.99999999965801356,0.9999999999037501,0.99999999953097896,0.99998066780622297,0.9999970881011605,0.99999999944518247,0.9999999999010627,0.99999161382285473,0.99999990528682581,0.99999999975605891,0.99999999907088322,0.99999993718664038,0.99999999666827866,0.99999972955811567,0.99999607232851673,0.9999999910881614,0.99999999981840348,0.99999999981840348],"proportion_misgenotyped":1.271110678546794e-05},{"name":"rare_allele_tetraploid","ploidy":4,"allele_depth":[0,0,16,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0,8,0,0,0,9,0,0,0,0,17,0,0,0,10,0,0,12,0,0,0,0,0,0,0,0,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,12,0,0,0],"marker_depth":[83,26,67,32,38,69,35,49,71,68,58,86,46,81,49,27,79,65,40,17,88,47,56,74,42,26,30,48,40,58,60,73,71,37,49,51,40,24,30,24,32,48,50,83,54,35,28,41,33,23,83,66,87,76,57,39,36,45,54,71,40,85,75,58,19,16,20,34,20,31,69,75,34,15,77,15,63,62,42,43,50,54,27,85,90,49,28,38,85,62],"prior":[0.86628433470248822,0.13371565138948965,1.390802220033504e-08,9.7478728072382646e-22,4.6037918512023317e-41],"mean":0.36851068893797617,"standard_deviation":0.2652766029969047,"genotypes":[0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"posteriors":[[0.99999694235517467,3.0576448253170762e-06,7.9322939704126524e-19,4.2106646861833036e-39,3.3931004778192861e-74],[0.99919444347532504,0.00080555652457051053,1.0454940179104917e-13,6.2999421789135864e-31,1.7203649129459146e-62],[3.2761295202991193e-07,0.99999966761984926,4.7671986467684893e-09,6.7794052170427033e-26,6.7659822769812268e-58],[0.99964646886091124,0.00035353113907127633,1.7644931104996396e-14,3.4086982580361504e-32,2.2823025177464459e-64],[0.99983167719612787,0.00016832280386852978,3.6002779229383497e-15,2.5915838527955616e-33,5.3177503665963202e-66],[0.9999914007505144,8.599249485638349e-06,6.8147064120629643e-18,1.2176668369869693e-37,3.7077603566778264e-72],[0.99975828575971215,0.00024171424028009158,7.8035483548929767e-15,9.0537029332162625e-33,3.2783416650270478e-65],[1.6419241041973243e-05,0.99998358015261457,6.0634361068343973e-10,2.8253396492623556e-27,1.7727265374593105e-59],[0.99999264828194057,7.3517180594293701e-06,4.9148579621707712e-18,7.2921802346874594e-38,1.8089425378898029e-72],[0.99999068811347269,9.3118865274066687e-06,8.046271662754669e-18,1.5805457582236311e-37,5.3435291833738825e-72],[0.99997829067963284,2.1709320367092926e-05,4.7301941537677753e-17,2.5751483822221418e-36,2.7097903178717365e-70],[0.99999750703123225,2.4929687677539093e-06,5.1949278952382058e-19,2.1771168073550131e-39,1.3573534239610453e-74],[0.99993069979092575,6.9300209073788695e-05,5.4606484998765123e-16,1.2551940619651093e-34,6.7949644810582269e-68],[0.99999648543895603,3.5145610438013609e-06,1.0590010794088767e-18,6.6088329571638774e-39,6.3509412993290691e-74],[0.99994904247680594,5.0957523193878785e-05,2.8504659014328528e-16,4.4500414502182001e-35,1.5448791216465324e-68],[0.99930225507439541,0.00069774492552801873,7.6553397853186428e-14,3.7691257562465862e-31,7.9882838506023185e-63],[0.99999594958033033,4.0504196696259102e-06,1.4218823270361092e-18,1.0469336938052563e-38,1.2048678192484735e-73],[0.99998811419544864,1.1885804551286786e-05,1.3395868655229746e-17,3.5220071785218706e-37,1.64460098477285e-71],[0.99986651552950656,0.00013348447049115519,2.1961558985740528e-15,1.1686579224184926e-33,1.6797101098024331e-66],[0.99662253850010563,0.0033774614974694727,2.4248655314572637e-12,1.2130492507514021e-28,5.3404546109587564e-59],[0.9999978176867822,2.1823132177257609e-06,3.9432796467739567e-19,1.4173653001303609e-39,7.4828571678445322e-75],[0.99993753818920406,6.2461810795589898e-05,4.3829206316500328e-16,8.8359719606670319e-35,4.1127740918071797e-68],[0.99997397293872414,2.6027061275921521e-05,6.9231485318969889e-17,4.7041822879695272e-36,6.3516691791394108e-70],[0.99999415374580813,5.8462541918624182e-06,3.0496753832991406e-18,3.4524911390960644e-38,6.3618186722148811e-73],[3.0855876714433989e-06,0.99999691173012739,2.6822012605516736e-09,5.7678655934567063e-26,1.960680890504818e-57],[0.99919444347532504,0.00080555652457051053,1.0454940179104917e-13,6.2999421789135864e-31,1.7203649129459146e-62],[0.99953937345937138,0.00046062654059721202,3.119491066912565e-14,8.6403302684262295e-32,8.9809759003104834e-64],[0.99994362177901508,5.6378220984460712e-05,3.5291277701384548e-16,6.2540743093292444e-35,2.5104124291555309e-68],[0.99986651552950656,0.00013348447049115519,2.1961558985740528e-15,1.1686579224184926e-33,1.6797101098024331e-66],[0.99997829067963284,2.1709320367092926e-05,4.7301941537677753e-17,2.5751483822221418e-36,2.7097903178717365e-70],[0.9999818143552992,1.8185644700756264e-05,3.2625914432736898e-17,1.4321699175660434e-36,1.1839825001932415e-70],[0.99999369461963894,6.3053803610894035e-06,3.5695037360183497e-18,4.4174689965332308e-38,8.976254277313615e-73],[0.99999264828194057,7.3517180594293701e-06,4.9148579621707712e-18,7.2921802346874594e-38,1.8089425378898029e-72],[0.9998104640944675,0.00018953590552792679,4.6387191341201384e-15,3.9018493906285667e-33,9.6292234292737904e-66],[0.99994904247680594,5.0957523193878785e-05,2.8504659014328528e-16,4.4500414502182001e-35,1.5448791216465324e-68],[0.99995820354902087,4.1796450978847532e-05,1.8762746418912444e-16,2.2875353056345796e-35,5.9889176726016238e-69],[0.0001323369372158053,0.99986766273861538,3.2416887823712886e-10,1.4379619711504929e-27,1.2748067661938048e-59],[0.99891710799729017,0.0010828920025109186,1.9899051038042742e-13,1.8271645201909065e-30,8.5134124485637559e-62],[0.99953937345937138,0.00046062654059721202,3.119491066912565e-14,8.6403302684262295e-32,8.9809759003104834e-64],[0.99891710799729017,0.0010828920025109186,1.9899051038042742e-13,1.8271645201909065e-30,8.5134124485637559e-62],[0.99964646886091124,0.00035353113907127633,1.7644931104996396e-14,3.4086982580361504e-32,2.2823025177464459e-64],[6.3297480935229535e-06,0.99999366906646481,1.1854417970919152e-09,9.9025331639685968e-27,1.094682290886549e-58],[0.99995387998539498,4.6120014604778459e-05,2.3092502470885057e-16,3.1826420317904096e-35,9.5822175022188305e-69],[0.99999694235517467,3.0576448253170762e-06,7.9322939704126524e-19,4.2106646861833036e-39,3.3931004778192861e-74],[0.99996865575883198,3.1344241167940541e-05,1.0234157883172346e-16,8.7382485425782258e-36,1.5269560491423075e-69],[5.409923718983687e-07,0.99999944513139594,1.3876232612963188e-08,1.7406048963193027e-24,4.5547306558966866e-55],[0.9993940078467769,0.00060599215316683086,5.6412975712969656e-14,2.2815246134371421e-31,3.7842775536307069e-63],[0.99988081571002518,0.00011918428997298624,1.7255764816840764e-15,7.9302980803976385e-34,9.5984920734544677e-67],[0.99968922136897764,0.00031077863100893813,1.3376574692399961e-14,2.171765417171153e-32,1.1780146428644153e-64],[0.99873883472959302,0.0012611652701295043,2.7748970749487107e-13,3.1745027704567725e-30,1.9614325650083964e-61],[8.5163361724885479e-07,0.99999914692285918,1.4435235698224585e-09,4.9824982971731782e-27,9.5005476860264123e-60],[0.99998905253658654,1.0947463413348021e-05,1.1280745646781796e-17,2.6878515106588568e-37,1.1252552359468542e-71],[0.99999766821390113,2.3317860987783437e-06,4.5231718765947497e-19,1.7548322379303289e-39,1.0063145188446509e-74],[0.99999496282021971,5.0371797804348072e-06,2.2369902573495635e-18,2.1257992582511642e-38,3.2333489842755499e-73],[3.8084973453907431e-06,0.99999619041141419,1.0912407208819325e-09,6.0106855715615714e-27,3.3192269507428564e-59],[0.99985023940201811,0.00014976059797900011,2.8061523035131767e-15,1.7341080201359792e-33,2.9717428558621144e-66],[0.99978616985771795,0.00021383014227613147,6.0028996954987635e-15,5.9198987687921346e-33,1.7653124590441554e-65],[2.6001321567402813e-07,0.9999997265477002,1.3439084620701635e-08,9.9853851913160087e-25,9.679444675548544e-56],[0.99996865575883198,3.1344241167940541e-05,1.0234157883172346e-16,8.7382485425782258e-36,1.5269560491423075e-69],[0.99999264828194057,7.3517180594293701e-06,4.9148579621707712e-18,7.2921802346874594e-38,1.8089425378898029e-72],[0.99986651552950656,0.00013348447049115519,2.1961558985740528e-15,1.1686579224184926e-33,1.6797101098024331e-66],[0.99999733310396599,2.6668960338838324e-06,5.9741644819726593e-19,2.7066841447142752e-39,1.8364178916826787e-74],[0.999994575347669,5.4246523309060606e-06,2.6098202100052235e-18,2.7055603790453805e-38,4.5266837199967364e-73],[0.99997829067963284,2.1709320367092926e-05,4.7301941537677753e-17,2.5751483822221418e-36,2.7097903178717365e-70],[0.99760265495721767,0.0023973450416462289,1.1361611203953282e-12,3.3625872756096692e-29,7.2508595046541003e-60],[0.99596724584793139,0.0040327541484727007,3.5958193612462749e-12,2.3729150703058818e-28,1.5343672896836825e-58],[0.99796884970553212,0.0020311502936793068,7.8864656446885415e-13,1.8192117824699767e-29,2.8125959819129398e-60],[0.99972620863910777,0.00027379136088196017,1.0192057888935073e-14,1.3961877482684835e-32,6.1707897423250953e-65],[0.99796884970553212,0.0020311502936793068,7.8864656446885415e-13,1.8192117824699767e-29,2.8125959819129398e-60],[9.3080638204924304e-09,0.99999956675058999,4.2394134577604918e-07,1.5723490693091353e-21,1.8241756864202692e-50],[0.9999914007505144,8.599249485638349e-06,6.8147064120629643e-18,1.2176668369869693e-37,3.7077603566778264e-72],[0.999994575347669,5.4246523309060606e-06,2.6098202100052235e-18,2.7055603790453805e-38,4.5266837199967364e-73],[0.99972620863910777,0.00027379136088196017,1.0192057888935073e-14,1.3961877482684835e-32,6.1707897423250953e-65],[0.99516459629955589,0.004835403695054542,5.3893786451774654e-12,4.741999155599979e-28,4.5993799663365592e-58],[0.99999531922216911,4.680777830872734e-06,1.9204362602228525e-18,1.6745696552003496e-38,2.3182526125823137e-73],[0.99516459629955589,0.004835403695054542,5.3893786451774654e-12,4.741999155599979e-28,4.5993799663365592e-58],[0.99998595065434215,1.4049345657778628e-05,1.9004601633436158e-17,6.1080766495311243e-37,3.5657360841804665e-71],[0.9999847036923466,1.5296307653369937e-05,2.2706761037541569e-17,8.0854970719547035e-37,5.291081370506882e-71],[0.99989340423792461,0.00010659576207400231,1.3610357804024038e-15,5.4172476160927434e-34,5.5428673790541762e-67],[0.99990450688523247,9.5493114766494284e-05,1.077499072672889e-15,3.724401771142253e-34,3.2333788636961364e-67],[0.99995387998539498,4.6120014604778459e-05,2.3092502470885057e-16,3.1826420317904096e-35,9.5822175022188305e-69],[0.99996865575883198,3.1344241167940541e-05,1.0234157883172346e-16,8.7382485425782258e-36,1.5269560491423075e-69],[0.00050387009843925231,0.99949612932163878,5.799219512864468e-10,9.4754438228592357e-27,5.9164184955339845e-58],[0.99999733310396599,2.6668960338838324e-06,5.9741644819726593e-19,2.7066841447142752e-39,1.8364178916826787e-74],[0.99999808519091882,1.9148090811193182e-06,3.0081501398445415e-19,9.3021638332834691e-40,4.1736093451230047e-75],[0.99994904247680594,5.0957523193878785e-05,2.8504659014328528e-16,4.4500414502182001e-35,1.5448791216465324e-68],[1.9962956514944221e-08,0.99999970163525032,2.7840179299183746e-07,8.4078217490524442e-22,9.7157979884121223e-51],[0.99983167719612787,0.00016832280386852978,3.6002779229383497e-15,2.5915838527955616e-33,5.3177503665963202e-66],[0.99999733310396599,2.6668960338838324e-06,5.9741644819726593e-19,2.7066841447142752e-39,1.8364178916826787e-74],[0.9999847036923466,1.5296307653369937e-05,2.2706761037541569e-17,8.0854970719547035e-37,5.291081370506882e-71]],"max_posterior":[0.99999694235517467,0.99919444347532504,0.99999966761984926,0.99964646886091124,0.99983167719612787,0.9999914007505144,0.99975828575971215,0.99998358015261457,0.99999264828194057,0.99999068811347269,0.99997829067963284,0.99999750703123225,0.99993069979092575,0.99999648543895603,0.99994904247680594,0.99930225507439541,0.99999594958033033,0.99998811419544864,0.99986651552950656,0.99662253850010563,0.9999978176867822,0.99993753818920406,0.99997397293872414,0.99999415374580813,0.99999691173012739,0.99919444347532504,0.99953937345937138,0.99994362177901508,0.99986651552950656,0.99997829067963284,0.9999818143552992,0.99999369461963894,0.99999264828194057,0.9998104640944675,0.99994904247680594,0.99995820354902087,0.99986766273861538,0.99891710799729017,0.99953937345937138,0.99891710799729017,0.99964646886091124,0.99999366906646481,0.99995387998539498,0.99999694235517467,0.99996865575883198,0.99999944513139594,0.9993940078467769,0.99988081571002518,0.99968922136897764,0.99873883472959302,0.99999914692285918,0.99998905253658654,0.99999766821390113,0.99999496282021971,0.99999619041141419,0.99985023940201811,0.99978616985771795,0.9999997265477002,0.99996865575883198,0.99999264828194057,0.99986651552950656,0.99999733310396599,0.999994575347669,0.99997829067963284,0.99760265495721767,0.99596724584793139,0.99796884970553212,0.99972620863910777,0.99796884970553212,0.99999956675058999,0.9999914007505144,0.999994575347669,0.99972620863910777,0.99516459629955589,0.99999531922216911,0.99516459629955589,0.99998595065434215,0.9999847036923466,0.99989340423792461,0.99990450688523247,0.99995387998539498,0.99996865575883198,0.99949612932163878,0.99999733310396599,0.99999808519091882,0.99994904247680594,0.99999970163525032,0.99983167719612787,0.99999733310396599,0.9999847036923466],"proportion_misgenotyped":0.0003972082780744346}],"bias_markers":[{"name":"tetraploid_bias_0.6","model":"norm","ploidy":4,"simulated_bias":0.59999999999999998,"allele_depth":[34,17,26,20,13,33,28,46,42,60,36,45,24,30,71,0,30,14,22,17,19,48,62,40,20,22,21,13,33,21,39,32,30,22,8,11,41,29,10,12,25,53,0,32,0,0,43,50,25,88,0,36,34,0,45,55,33,9,0,31,22,11,33,50,50,66,47,37,53,91,0,0,96,60,62,74,31,33,88,91,33,45,0,21,71,38,25,26,70,15,32,38,13,0,19,31,51,38,66,0,40,60,0,26,8,32,0,0,27,41,16,0,12,16,0,23,69,9,16,12],"marker_depth":[45,29,47,57,76,63,59,79,64,88,50,80,54,45,110,45,97,30,83,61,40,92,102,62,33,62,48,26,69,86,50,98,52,76,32,41,105,55,48,30,43,67,93,106,58,29,96,96,26,101,55,98,43,94,87,94,86,27,48,83,68,26,66,90,104,74,99,90,94,91,108,108,106,102,95,89,43,43,97,91,34,91,89,53,93,82,71,46,107,35,78,42,41,77,32,109,71,62,104,27,109,90,27,26,26,42,59,71,35,59,31,79,64,55,71,31,96,79,27,38],"bias":0.62507718757580477,"log_likelihood":-482.85975640534161,"prior":[0.14191783819483544,0.361389936370626,0.34988181036266214,0.12878726436256446,0.018023150709312034],"genotypes":[2,2,2,1,1,2,1,2,2,2,2,2,1,2,2,0,1,1,1,1,1,2,2,2,2,1,1,2,1,1,2,1,2,1,1,1,1,2,1,1,2,3,0,1,0,0,1,2,3,3,0,1,3,0,2,2,1,1,0,1,1,1,2,2,1,3,1,1,2,4,0,0,3,2,2,3,2,2,3,4,3,2,0,1,2,1,1,2,2,1,1,3,1,0,2,1,2,2,2,0,1,2,0,4,1,2,0,0,2,2,2,0,1,1,0,2,2,1,2,1],"posteriors":[[6.8353431472431771e-17,0.0025935682058012147,0.64379858625839648,0.35360784209206547,3.443737025749868e-09],[2.4008297888211741e-12,0.15195163876836046,0.81733689282251132,0.030711468369345813,3.7381546559432328e-11],[7.9225002514998808e-13,0.19223263339054728,0.79720942545844986,0.010557941147636569,2.5737379411782088e-12],[9.1812157089054109e-10,0.94507568805703024,0.054906043240500246,1.8267784347779545e-05,1.5567025637563401e-16],[1.3620877421007834e-06,0.99965180791677333,0.00034682809050774023,1.9049767476595367e-09,5.015724903538127e-22],[8.9657284753101109e-13,0.28229610617567646,0.71366851158435018,0.0040353822387374644,3.3919002825847977e-13],[9.2251648478248613e-12,0.53816738248993956,0.46064337025288837,0.0011892472478925496,5.4305346518946331e-14],[2.1019467507365562e-14,0.084082095032862431,0.90354269396825082,0.012375210997056929,1.8084556130526857e-12],[1.1644621417296292e-15,0.018368015498880192,0.91861936567478397,0.063012618777954749,4.8379855389763891e-11],[4.798593635064851e-17,0.0063968311506716257,0.898968556586351,0.094634612186835942,7.6141895719125903e-11],[2.0107330434308572e-16,0.00549241468243062,0.78480577534549512,0.20970180915136669,8.2070711204377129e-10],[5.6363039623562431e-14,0.12781688285184725,0.86425195483664163,0.0079311623106455044,8.0966590013860392e-13],[3.8412393466235484e-11,0.68658466891738634,0.31286016081868456,0.00055517022549655775,2.0092875485839236e-14],[5.8621458184673994e-15,0.022901808074260472,0.88914761302929979,0.087950578746824437,1.4960931276952031e-10],[1.4765090780280781e-16,0.013775536683089578,0.94563516950180648,0.040589293803067042,1.2036495204889154e-11],[0.99995789389782763,4.2105542507675899e-05,5.59663918110504e-10,9.2766523196686297e-16,1.6922836046849314e-28],[1.7162267587507997e-09,0.98814936935845255,0.01185002509572481,6.0382959599105951e-07,7.3316166781855944e-19],[1.3008972384687262e-10,0.56171256289224447,0.43529783558196072,0.0029896013950205459,6.8458151802328635e-13],[1.3432698348955308e-08,0.99564372783651234,0.004356130961716634,1.2776907302695665e-07,1.1333025926919667e-19],[1.1770114709793265e-08,0.99011206340594626,0.0098871197242976563,8.0509964130501201e-07,2.075871528961203e-18],[3.3673803301822318e-11,0.53133402620099424,0.46644971538799912,0.0022162583770733747,2.5914017937194498e-13],[2.847916819069048e-13,0.27204783966079338,0.72533323949226625,0.0026189208465487593,1.0720604455257826e-13],[1.854278132294705e-15,0.039120345986125668,0.94250560548293372,0.018374048528017412,2.9210051262061678e-12],[2.481658678323617e-15,0.024748606637527516,0.92364203229750574,0.051609361030941728,3.4022792896706124e-11],[5.4113961417306168e-13,0.10165311675931969,0.86012749277797418,0.038219390416393613,4.5771433064589778e-11],[6.640451306681382e-10,0.94450951950876794,0.055473946346837458,1.6533480349599648e-05,1.1923678656619419e-16],[7.0227134353166313e-11,0.70952585907345411,0.28991214075178412,0.00056200010450984392,2.4707711727444479e-14],[7.9549573929216551e-11,0.42936418666053627,0.56302556444409657,0.0076102488118707321,3.9469766007590621e-12],[4.9369773822667707e-12,0.51903896605532618,0.47989945736331946,0.0010615765763812833,3.6703655061998701e-14],[3.2522429432115704e-08,0.99769642021605154,0.0023035066613199665,4.0600199372989106e-08,2.3540697143110554e-20],[6.8332096297787192e-18,0.00092703284945941779,0.51104455976474861,0.48802840023626892,7.1495232769996547e-09],[8.4336935964629074e-10,0.98141737369779591,0.018581308207181825,1.3172516527862863e-06,2.0623689310168741e-18],[1.72131498056075e-13,0.12047017917850315,0.86406046174487183,0.015469359072040773,4.4122333169978951e-12],[5.4618140074914528e-09,0.9905844570433423,0.0094149907399740749,5.4675486929441772e-07,8.9145076920830544e-19],[1.0151544867651104e-07,0.98303888304457232,0.016955472567638828,5.5428723398904803e-06,8.3187281019309694e-17],[3.4097834674806663e-08,0.98481269317312192,0.015184187842256088,3.0848867875560004e-06,2.4253185007014005e-17],[6.2125231292830922e-11,0.91109317059713568,0.088886488032094285,2.0341308644531916e-05,7.7561231085172294e-17],[1.2759992938971467e-12,0.27562453847827773,0.71928925555317946,0.0050862059666705256,5.9629950861843257e-13],[2.9721305780707499e-07,0.99703611349315102,0.0029634444550339935,1.4483875685526828e-07,3.1284513416766098e-19],[1.031656769236485e-09,0.79513582573926112,0.20428765324613243,0.00057651998289055145,5.8998787380092103e-14],[3.5721270127177888e-13,0.12545728944018827,0.8546231243680158,0.019919586182369976,9.06892712245391e-12],[2.6146816444909988e-19,0.00027610416394076813,0.4157948652941334,0.58392902266198143,7.8799445507080515e-09],[0.9999998276905171,1.7230946243097862e-07,2.0440528265034683e-14,5.5248549375701034e-22,2.6526793608066029e-36],[2.0674503963115376e-09,0.99118898914023634,0.0088106843821931669,3.2441012046548102e-07,2.8583258151091076e-19],[0.99999301965299003,6.9803277996941194e-06,1.9210178426763038e-11,7.8418659732059953e-18,4.0491880566864165e-31],[0.99934393193913451,0.00065596268325350624,1.0537587088070605e-07,1.7410733325179191e-12,2.729719950532987e-24],[7.5610441934469648e-12,0.69744824726577681,0.30230775289266576,0.00024399983399375091,2.7742372100975186e-15],[2.6320328279973702e-13,0.27454148397524375,0.72299894182293245,0.0024595742014682197,9.2606125895541411e-14],[2.1060063329625143e-19,1.6600972258262435e-05,0.039719772515660279,0.96024897135735632,1.4655154725162561e-05],[3.7118148052331765e-24,7.3380364124614708e-07,0.04300083207224107,0.95699831587825923,1.1824585804423586e-07],[0.99998971017395399,1.0289786382628575e-05,3.9663467026102628e-11,2.1791724277557901e-17,1.4668185843333821e-30],[1.7125020763155519e-10,0.94727695165838333,0.052714586851427145,8.4613189394204578e-06,2.5461652761861644e-17],[1.4122305612438234e-17,0.00098974498309077727,0.46902936274162532,0.52998087994268561,1.2332598072548428e-08],[0.99999984230728645,1.5769269626064804e-07,1.7372684294507066e-14,4.4096873872829395e-22,2.0056856111042728e-36],[4.2065888654886312e-13,0.29689802685387712,0.70065154298606225,0.0024504301595374076,1.0215579226734721e-13],[9.1585866556214711e-15,0.071424288108017778,0.91689628631935916,0.011679425571237868,1.3758354444487951e-12],[1.169630094840245e-10,0.91596643886149021,0.084010534769148784,2.3026252397620856e-05,1.1776713545665517e-16],[1.0186047472288423e-08,0.91313921984385504,0.086727253574439653,0.0001335163956486504,9.0166911714857611e-15],[0.99997296534840441,2.7034408748447889e-05,2.4284686105733009e-10,2.8313180070174995e-16,3.7513161717724217e-29],[1.841213359167237e-10,0.93214772100564258,0.067836022758385878,1.6256051850080766e-05,7.6679898395014973e-17],[1.7425731649143854e-09,0.97506564110333704,0.024930857596266617,3.4995578232272669e-06,1.2656937105218093e-17],[8.3148000401152561e-10,0.71179320988902384,0.28674121828317023,0.0014655709960216085,3.0421157481963951e-13],[2.2214860190587645e-12,0.39838050751901627,0.59953552555633061,0.0020839669223214675,1.1022066936164196e-13],[5.317217479079243e-14,0.14098308524802594,0.8528566754069925,0.0061602393444656674,4.6249591885145833e-13],[1.493610461950431e-12,0.50497025034645227,0.49433402150353606,0.0006957281485065475,1.1808877524000382e-14],[1.4502221550305391e-23,7.687470227556158e-07,0.031314346443508383,0.96868455483815985,3.2997130859433418e-07],[2.2088653899802524e-12,0.54239596816466007,0.45700134534554565,0.00060268648757511587,1.004793484569068e-14],[3.766581086764578e-11,0.84951939788657693,0.15041567010806323,6.4931967693915131e-05,4.6401752979622258e-16],[2.9384987066961664e-14,0.11589768895376996,0.87688007732291695,0.0072222337226988433,5.8526862586761543e-13],[7.3980276311687685e-34,1.8737751358417004e-14,2.1959047313677411e-07,0.0033922086444693432,0.99660757176503856],[0.99999995095189098,4.9048106993263449e-08,2.0486737153404482e-15,2.2899729502783704e-23,5.1528692406864766e-38],[0.99999995095189098,4.9048106993263449e-08,2.0486737153404482e-15,2.2899729502783704e-23,5.1528692406864766e-38],[3.4374371466008803e-26,3.8945736895918526e-08,0.010449913770810434,0.98954949436523243,5.5291822012561579e-07],[5.6880870667070668e-15,0.063366699928136636,0.9247419864089248,0.011891313661611665,1.3213266168656182e-12],[1.8486468212814451e-16,0.013025533089768167,0.93690162609395156,0.050072840795852787,2.0427013770330023e-11],[7.6588170155629183e-22,1.5820671949482815e-05,0.16228774471982349,0.83769640700250725,2.7605719702443535e-08],[5.8962329185214907e-16,0.007149833579463233,0.77814671421863946,0.21470345110688138,1.0950154679707193e-09],[5.2525857626800627e-17,0.002048622601925365,0.58915445340961448,0.40879691844379129,5.5446689013076622e-09],[8.5260271985411919e-26,5.5563999605519035e-08,0.011304171182210951,0.98869515909247485,6.1416131490361913e-07],[7.3980276311687685e-34,1.8737751358417004e-14,2.1959047313677411e-07,0.0033922086444693432,0.99660757176503856],[1.4116615241839224e-21,1.366237298077884e-06,0.014707371733163367,0.98525968534096575,3.1576688573230102e-05],[1.1023200172917839e-12,0.42164496608295754,0.57712089209955608,0.0012341418163513669,3.2815354695952323e-14],[0.9999997524527251,2.4754723520112421e-07,3.9755621653831229e-14,1.3904502720471978e-21,8.3386423374917689e-36],[2.2484923731245967e-10,0.85795136808683825,0.14193167121715769,0.00011696047115257661,2.2593783755064995e-15],[1.6046860842621016e-19,0.00039565746777324434,0.56737346890055118,0.43223087158862211,2.0430529410469122e-09],[5.7525243914888595e-12,0.60670625952876012,0.3927695947355801,0.00052414572989702233,1.0221629174276276e-14],[5.5419267898086155e-10,0.95348510465280822,0.046504872912293313,1.0021880705908785e-05,4.9329466193023599e-17],[5.1858017672691775e-13,0.15901122018372238,0.82726201181155212,0.013726767999995315,4.21169738544775e-12],[9.7812189887426079e-17,0.011111821203714995,0.93922073783476012,0.049667440943438827,1.8086422420936822e-11],[2.5790823359294824e-10,0.72015061077462983,0.27899929838296444,0.00085009058441934675,7.8032582581721972e-14],[5.2991229395139811e-11,0.84414275256041038,0.15577464664532642,8.2600741271053466e-05,7.8255337565370867e-16],[8.3720988275665145e-21,9.1467059356948881e-06,0.054811943963238734,0.94517814883513174,7.6049569429475932e-07],[6.1608216871431615e-09,0.96016504224002963,0.039818133999071532,1.6817600077279955e-05,2.4091203783425366e-16],[0.99999920303516698,7.9696449053453485e-07,3.4242980911640602e-13,2.7809003566660053e-20,3.4640842769099294e-34],[1.0437759335989503e-12,0.12656821304588106,0.84147271164088777,0.031959075277954177,3.4233495455769508e-11],[4.2066490130096062e-09,0.99479487006314593,0.0052050004314424811,1.2529876263740336e-07,7.7505374081731092e-20],[1.9395475472672903e-17,0.0030747344789906006,0.79772180554343386,0.19920345952473803,4.5283733551813144e-10],[1.2893036400549997e-14,0.051092986343450775,0.92138182156142745,0.02752519208485851,1.0250556974813598e-11],[3.5847256172078239e-16,0.019345468329160263,0.94792921566214083,0.032725316000292208,8.4065370877082813e-12],[0.99902101507520658,0.00097875637568269622,2.2854371904892933e-07,5.3914272614348343e-12,1.1933215245933501e-23],[1.4537700152078572e-10,0.95108428788937593,0.048909136332042219,6.5756332051942419e-06,1.591978336955036e-17],[1.050584259973687e-16,0.0094543613124702958,0.92190927031558445,0.06863636833238837,3.9556504747487113e-11],[0.99902101507520658,0.00097875637568269622,2.2854371904892933e-07,5.3914272614348343e-12,1.1933215245933501e-23],[3.3504047095392553e-21,5.7394146635182147e-07,0.0028558769231492994,0.17657438392650254,0.82056916520888168],[2.4106622529200199e-08,0.93786777236303975,0.062055197265453922,7.7006264879087808e-05,4.4924895632127929e-15],[8.6767832881741759e-17,0.0025520232239739367,0.61718086589942145,0.38026710614922693,4.7273775393269162e-09],[0.99999384729019902,6.1526946206876687e-06,1.5180283343701902e-11,5.6294831565469389e-18,2.668787564543499e-31],[0.99999849194730084,1.5080515857595538e-06,1.1134321061688224e-12,1.4425559168481594e-19,2.7030449006408383e-33],[2.6223703981274117e-16,0.0031138301840301099,0.58452179206125299,0.41236436935548892,8.3992278080453781e-09],[2.4366380237924561e-16,0.007902280517584518,0.85863777431476274,0.13345994492355939,2.440928625940536e-10],[2.1571517646169577e-11,0.35883925975180569,0.63301775163767238,0.0081429885857394989,3.2108925951699706e-12],[0.99999935023674957,6.4976301549599168e-07,2.3491710311978258e-13,1.6447101443175949e-20,1.8003913011568217e-34],[6.2297523977712358e-07,0.99913414134821454,0.00086522416063633555,1.1515909307510177e-08,6.9315446358166979e-21],[8.7318794795884131e-09,0.98463702841079914,0.015360947785994727,2.0150713264987177e-06,8.3584031148966219e-18],[0.99999849194730084,1.5080515857595538e-06,1.1134321061688224e-12,1.4425559168481594e-19,2.7030449006408383e-33],[2.9996288298426187e-15,0.0085985135867834798,0.7052365323682982,0.28616495013426529,3.9106499424768084e-09],[2.9447939337401222e-18,0.0018743893088707315,0.80079623526942545,0.19732937510670248,3.1500119626085646e-10],[4.9019644445830013e-05,0.99990360284407298,4.737744705151262e-05,6.442987976537756e-11,5.7854970320192746e-24],[2.8439568167369743e-12,0.14614038337100546,0.81771617247558182,0.036143444093160566,5.7408003743635858e-11],[7.6317482513804256e-09,0.95710564850382462,0.042872738640800885,2.1605223625858737e-05,3.8770571451477321e-16]],"max_posterior":[0.64379858625839648,0.81733689282251132,0.79720942545844986,0.94507568805703024,0.99965180791677333,0.71366851158435018,0.53816738248993956,0.90354269396825082,0.91861936567478397,0.898968556586351,0.78480577534549512,0.86425195483664163,0.68658466891738634,0.88914761302929979,0.94563516950180648,0.99995789389782763,0.98814936935845255,0.56171256289224447,0.99564372783651234,0.99011206340594626,0.53133402620099424,0.72533323949226625,0.94250560548293372,0.92364203229750574,0.86012749277797418,0.94450951950876794,0.70952585907345411,0.56302556444409657,0.51903896605532618,0.99769642021605154,0.51104455976474861,0.98141737369779591,0.86406046174487183,0.9905844570433423,0.98303888304457232,0.98481269317312192,0.91109317059713568,0.71928925555317946,0.99703611349315102,0.79513582573926112,0.8546231243680158,0.58392902266198143,0.9999998276905171,0.99118898914023634,0.99999301965299003,0.99934393193913451,0.69744824726577681,0.72299894182293245,0.96024897135735632,0.95699831587825923,0.99998971017395399,0.94727695165838333,0.52998087994268561,0.99999984230728645,0.70065154298606225,0.91689628631935916,0.91596643886149021,0.91313921984385504,0.99997296534840441,0.93214772100564258,0.97506564110333704,0.71179320988902384,0.59953552555633061,0.8528566754069925,0.50497025034645227,0.96868455483815985,0.54239596816466007,0.84951939788657693,0.87688007732291695,0.99660757176503856,0.99999995095189098,0.99999995095189098,0.98954949436523243,0.9247419864089248,0.93690162609395156,0.83769640700250725,0.77814671421863946,0.58915445340961448,0.98869515909247485,0.99660757176503856,0.98525968534096575,0.57712089209955608,0.9999997524527251,0.85795136808683825,0.56737346890055118,0.60670625952876012,0.95348510465280822,0.82726201181155212,0.93922073783476012,0.72015061077462983,0.84414275256041038,0.94517814883513174,0.96016504224002963,0.99999920303516698,0.84147271164088777,0.99479487006314593,0.79772180554343386,0.92138182156142745,0.94792921566214083,0.99902101507520658,0.95108428788937593,0.92190927031558445,0.99902101507520658,0.82056916520888168,0.93786777236303975,0.61718086589942145,0.99999384729019902,0.99999849194730084,0.58452179206125299,0.85863777431476274,0.63301775163767238,0.99999935023674957,0.99913414134821454,0.98463702841079914,0.99999849194730084,0.7052365323682982,0.80079623526942545,0.99990360284407298,0.81771617247558182,0.95710564850382462],"proportion_misgenotyped":0.1437203909907494},{"name":"tetraploid_bias_0.6","model":"hw","ploidy":4,"simulated_bias":0.59999999999999998,"allele_depth":[34,17,26,20,13,33,28,46,42,60,36,45,24,30,71,0,30,14,22,17,19,48,62,40,20,22,21,13,33,21,39,32,30,22,8,11,41,29,10,12,25,53,0,32,0,0,43,50,25,88,0,36,34,0,45,55,33,9,0,31,22,11,33,50,50,66,47,37,53,91,0,0,96,60,62,74,31,33,88,91,33,45,0,21,71,38,25,26,70,15,32,38,13,0,19,31,51,38,66,0,40,60,0,26,8,32,0,0,27,41,16,0,12,16,0,23,69,9,16,12],"marker_depth":[45,29,47,57,76,63,59,79,64,88,50,80,54,45,110,45,97,30,83,61,40,92,102,62,33,62,48,26,69,86,50,98,52,76,32,41,105,55,48,30,43,67,93,106,58,29,96,96,26,101,55,98,43,94,87,94,86,27,48,83,68,26,66,90,104,74,99,90,94,91,108,108,106,102,95,89,43,43,97,91,34,91,89,53,93,82,71,46,107,35,78,42,41,77,32,109,71,62,104,27,109,90,27,26,26,42,59,71,35,59,31,79,64,55,71,31,96,79,27,38],"bias":0.62378987863331636,"log_likelihood":-483.04372005284938,"prior":[0.14804649299133335,0.36249569850560392,0.33284256379537663,0.13582894942611515,0.020786295281570902],"genotypes":[2,2,2,1,1,2,1,2,2,2,2,2,1,2,2,0,1,1,1,1,1,2,2,2,2,1,1,2,1,1,3,1,2,1,1,1,1,2,1,1,2,3,0,1,0,0,1,2,3,3,0,1,3,0,2,2,1,1,0,1,1,1,2,2,1,3,1,1,2,4,0,0,3,2,2,3,2,2,3,4,3,2,0,1,2,1,1,2,2,1,1,3,1,0,2,1,2,2,2,0,1,2,0,4,1,2,0,0,2,2,2,0,1,1,0,2,2,1,2,1],"posteriors":[[7.2155409448714625e-17,0.0026653565710566679,0.62183371941880428,0.37550092000762453,4.0025147818112334e-09],[2.6076660059657825e-12,0.15950135039639554,0.80706257394482694,0.033436075611466068,4.4703577025723684e-11],[8.610396131404339e-13,0.20200498892617413,0.7865170324064773,0.011477978663407716,3.0797792881226841e-12],[9.5976559336584039e-10,0.94829741757742203,0.05168358230016231,1.8999162649973719e-05,1.7916989405130567e-16],[1.4320112800863428e-06,0.99967327040856713,0.00032529559908486968,1.9810675281821011e-09,5.8058992519570157e-22],[9.7054550647935195e-13,0.29536473286421355,0.70027223996491061,0.004363027169501139,4.0426653136664768e-13],[9.8446860227704153e-12,0.55397621424490184,0.44475755051582982,0.001266235229359878,6.380647029861648e-14],[2.2977981660741817e-14,0.089097841094261887,0.89737580954275176,0.013526349360786812,2.1766690545008566e-12],[1.2680336447698178e-15,0.019432809630101418,0.91178609130879873,0.068781099003093857,5.8004855758515876e-11],[5.2064187710290121e-17,0.0067583412001267695,0.89028517466607771,0.10295648404283693,9.0958476934909786e-11],[2.1547195653166374e-16,0.0057258536962415665,0.76849835582509374,0.22577578951034438,9.6832005108337751e-10],[6.1516481624558895e-14,0.13511983682938455,0.85623011055516574,0.0086500526144153381,9.7296521204186847e-13],[4.0657997244409568e-11,0.70009787052678807,0.29931609970689416,0.00058602972563646146,2.3415913697699114e-14],[6.3643262193187915e-15,0.024130795290725312,0.8800676970131649,0.095801507517269652,1.7883348388387608e-10],[1.612406938364016e-16,0.014635487824921958,0.94098457633567623,0.044379935824929946,1.4472146540331391e-11],[0.99996019627531296,3.9803226309766674e-05,4.983764119375516e-10,9.185461899247906e-16,1.867084740583069e-28],[1.7928527974919544e-09,0.98889335336244844,0.011106019139274819,6.2570542356652875e-07,8.4326507994156004e-19],[1.3864229357292602e-10,0.57660390361047897,0.42021059409736666,0.0031855021527086897,8.0328995237351598e-13],[1.4055525057939923e-08,0.99591663300250688,0.0040832204102177663,1.3253175012579747e-07,1.3056419806409794e-19],[1.2308192510669356e-08,0.99071882992112947,0.0092803218141700207,8.3595650773204665e-07,2.3900581936491292e-18],[3.5947568848057948e-11,0.54676283907675838,0.45087369415118184,0.0023634667358075675,3.0458360224351457e-13],[3.0859349748770344e-13,0.28511761996622181,0.71205068966863727,0.0028316903647041891,1.2790075964308232e-13],[2.0296367502865855e-15,0.041579934769404227,0.93830514126255127,0.020114923964522548,3.5202226452219265e-12],[2.705622371154219e-15,0.026200195772573872,0.91741364272363479,0.056386161462947955,4.0840352037646852e-11],[5.8879667544855694e-13,0.10700285890795649,0.85130932213585686,0.041687818900764631,5.4833370342024462e-11],[6.9407859367745214e-10,0.94777820770835253,0.052204602160134027,1.7189437434582945e-05,1.3721946593579062e-16],[7.424697270402527e-11,0.7223583108622289,0.27704899540952105,0.00059269365397403837,2.8760785039027624e-14],[8.5347075898384687e-11,0.44398925226832581,0.5478381366666375,0.0081726109750270334,4.6624025302560759e-12],[5.274230917821311e-12,0.53507937346603074,0.46378950077651848,0.0011311257521331001,4.3171954398744287e-14],[3.4064476143629728e-08,0.99784107736057737,0.0021588464492366503,4.2125709872376569e-08,2.7147664213919111e-20],[7.1147701181831696e-18,0.00094110286926385976,0.48752223289911439,0.51153665603551501,8.1961066093331073e-09],[8.8065342886321916e-10,0.98257758582672083,0.017421048289014904,1.3650036111193949e-06,2.3710896155037482e-18],[1.8769002816896194e-13,0.12715321473669125,0.85596862885823899,0.016878156399584992,5.2970456335611871e-12],[5.7095323513647175e-09,0.99116961669566073,0.0088298104442134529,5.6715059382613736e-07,1.0260190055162446e-18],[1.0623564811374788e-07,0.98403139143608187,0.015962728286982125,5.7740412878230649e-06,9.5849424092551631e-17],[3.5667985962348937e-08,0.98571792994971319,0.01427882508527378,3.2092970267523774e-06,2.7932833871613626e-17],[6.4965291278407486e-11,0.91628767633507813,0.083691181517465241,2.1142082490923782e-05,8.9300204913783855e-17],[1.3815018931231694e-12,0.28837977331587261,0.7061177217345711,0.0055025049474636779,7.1082042245969576e-13],[3.1154276844492013e-07,0.9972157229330193,0.0027838148467548302,1.5067745757200239e-07,3.6105522254149041e-19],[1.0864323628507641e-09,0.80491459379455477,0.19447917885911742,0.00060622625982690952,6.8407697885142565e-14],[3.8912657597224304e-13,0.1322319603945788,0.84604259150270755,0.021725448091447719,1.087712982445531e-11],[2.6963663857093114e-19,0.00027822259586387068,0.39339069773815838,0.60633107071897441,8.9470038164697138e-09],[0.99999983838599493,1.616139871322456e-07,1.8038001492514693e-14,5.438005017703903e-22,2.926676058235009e-36],[2.1602346204457668e-09,0.99174577994328728,0.00825388182837632,3.3606810173809695e-07,3.2883039630077896e-19],[0.99999341843672596,6.5815462197893556e-06,1.7054327992741509e-11,7.7488605387544117e-18,4.4674325656070928e-31],[0.99937728574870144,0.00062261994734706509,9.4302221743740642e-08,1.7299336144391276e-12,3.0117746134818276e-24],[7.9954706393600089e-12,0.71109962232570578,0.2886434614882824,0.00025691617801295712,3.2299829911225214e-15],[2.851723934392503e-13,0.28771333208242506,0.70962783989067602,0.0026588280265035392,1.1047150617643762e-13],[2.0855261634517624e-19,1.6043778779603067e-05,0.036208734267391761,0.96375924327688012,1.5978676948782666e-05],[3.6861423708872177e-24,7.1681487786273432e-07,0.03943254115521768,0.96056661273955724,1.2929034714494807e-07],[0.99999029261750794,9.7073472567933773e-06,3.5235169245567909e-11,2.1542593771621527e-17,1.6183279744472664e-30],[1.7886060268240236e-10,0.95046290641057551,0.049528313594694742,8.7798158691238913e-06,2.9279561513553279e-17],[1.4642090822517497e-17,0.00099997572936195143,0.44557206248030218,0.55342794771215731,1.4078178288305388e-08],[0.99999985211392084,1.4788606381160981e-07,1.5328597685653116e-14,4.3399778139768199e-22,2.2128539702785861e-36],[4.551583234005346e-13,0.31062145571625921,0.68673286531514177,0.0026456789680223435,1.2169978196111108e-13],[1.0020160729700625e-14,0.075789671483339635,0.9114371009602148,0.012773227554777949,1.6573227739429145e-12],[1.2231539035165982e-10,0.92085759133810008,0.079118465477244396,2.3943062339563525e-05,1.3559731138374026e-16],[1.0673075781895607e-08,0.91783796531517103,0.082022438177210333,0.00013958583453225454,1.0402207961554337e-14],[0.99997446003644463,2.5539747467047327e-05,2.1608808199283516e-10,2.8020339696237175e-16,4.1387977253393175e-29],[1.9243882485490621e-10,0.93615586238357207,0.063827246301194573,1.6891122794146367e-05,8.8239797983400914e-17],[1.8204042191758839e-09,0.97658556902685623,0.023410796753468747,3.6323992707908249e-06,1.4557937225316092e-17],[8.7936819286481829e-10,0.7240090730355826,0.2744416976364264,0.0015492284482684518,3.5423307544816236e-13],[2.3894743568521695e-12,0.41382676084779058,0.58393603989285303,0.0022371992568360343,1.3053176956724625e-13],[5.8012888310516448e-14,0.14898158616921203,0.8443051375206938,0.0067132763094803552,5.5557455665423296e-13],[1.5969620538247453e-12,0.52137724925743067,0.47788152674453566,0.00074122399642283752,1.3901454733370203e-14],[1.4373221124897788e-23,7.4856092272902637e-07,0.028658822998386568,0.97134006836805187,3.6007263909371849e-07],[2.3565789964791593e-12,0.55861742981152684,0.44074190384299694,0.0006406663431078682,1.1802770046003849e-14],[3.9507781116725569e-11,0.85769276656337079,0.14223947298133657,6.7760415784448621e-05,5.3587574099325301e-16],[3.2098997178566675e-14,0.12267655238404555,0.86944208846645354,0.0078813591487652511,7.0390748234227194e-13],[6.7070822586542128e-34,1.6839506616177535e-14,1.8601488423108133e-07,0.0031457398684183457,0.99685407411668037],[0.99999995407740261,4.5922595556685075e-08,1.8043176126801641e-15,2.2511365884909661e-23,5.6851119029322435e-38],[0.99999995407740261,4.5922595556685075e-08,1.8043176126801641e-15,2.2511365884909661e-23,5.6851119029322435e-38],[3.3963877785503438e-26,3.7945566443643917e-08,0.0095630686955133198,0.99043629185596815,6.0150295165048794e-07],[6.2256048040233294e-15,0.0672882571455162,0.91970232973599242,0.013009413116892665,1.5922815761129603e-12],[2.0163938559244028e-16,0.013820490934464399,0.93147204846534615,0.054707460575658132,2.4531305974415694e-11],[7.7029490665807443e-22,1.5604602645221834e-05,0.15023271442751004,0.84975165040031375,3.0569530641745497e-08],[6.3153894282166523e-16,0.0074445783529210894,0.76145222636311671,0.23110319399262286,1.2913386576860144e-09],[5.5134428422992761e-17,0.0020937587679786241,0.5660649442145913,0.43184129060947057,6.4079591492393624e-09],[8.4247273562439909e-26,5.4107333200817899e-08,0.010341818440556661,0.98965745928366788,6.681684422607233e-07],[6.7070822586542128e-34,1.6839506616177535e-14,1.8601488423108133e-07,0.0031457398684183457,0.99685407411668037],[1.3928504044496293e-21,1.3193913603180668e-06,0.013391979547845836,0.98657239783914408,3.4303221649567644e-05],[1.1842829176837312e-12,0.43762254725563271,0.5610553650467005,0.0013220876964436215,3.881675777100311e-14],[0.99999976769984944,2.323001155482096e-07,3.5102985937818868e-14,1.3690974340791329e-21,9.1999452781209748e-36],[2.3589895030077456e-10,0.86552540918483367,0.13435234523787423,0.00012224534139059596,2.609864530615514e-15],[1.6799693055953227e-19,0.00040495071421758682,0.54455207798812433,0.45504296894268847,2.3549692780823202e-09],[6.1147815437194385e-12,0.62210779737597444,0.37733697820857315,0.00055522440932591235,1.1962914188163348e-14],[5.7900257698584916e-10,0.95626957168899052,0.043720018640259116,1.0409091747459881e-05,5.6744076480898716e-17],[5.6438206122832405e-13,0.16738372698185061,0.81766758513919191,0.014948687873346096,5.0467157897625919e-12],[1.0669343801805998e-16,0.01179581593527544,0.93394261851388938,0.054261565529113209,2.1721597642590669e-11],[2.7258578506241158e-10,0.73242466301425146,0.26667824635750209,0.00089709035557027496,9.0804714471773858e-14],[5.5606180392768491e-11,0.85250023096324035,0.14741349990876843,8.6269072384061917e-05,9.0412282513525402e-16],[8.3152049216862181e-21,8.8878943131441388e-06,0.050145290430706972,0.94984499004570799,8.3162927144300691e-07],[6.4421592292902556e-09,0.96248823110084814,0.037494255357646979,1.7507099345560113e-05,2.7736122231095408e-16],[0.99999925088597597,7.4911372127394905e-07,3.0292095114465205e-13,2.741507474280588e-20,3.821891547556408e-34],[1.1350878013020472e-12,0.1330939996148314,0.83207239768517949,0.034833602657864976,4.0989048221148461e-11],[4.3985133758672139e-09,0.99512511372252921,0.0048747520639411254,1.2981501615204057e-07,8.9226911116790926e-20],[2.0803713236435934e-17,0.0032136176931498502,0.78222114453659808,0.21456523723546941,5.3478230640675238e-10],[1.4085054013301915e-14,0.054132029302878047,0.9157618596209397,0.030106111063838783,1.2329491268596677e-11],[3.918349613156878e-16,0.020559091750553853,0.94363537628080596,0.035805531958522874,1.0117163454534866e-11],[0.99907022028898851,0.00092957502223020062,2.0468342155630734e-07,5.3599108581152283e-12,1.3166451501345513e-23],[1.5180145476411896e-10,0.95406470168035629,0.04592847878574529,6.819382096392951e-06,1.8302557181129078e-17],[1.1433560505746068e-16,0.01001334650310309,0.91512560663704878,0.074861046812449539,4.7398313118446597e-11],[0.99907022028898851,0.00092957502223020062,2.0468342155630734e-07,5.3599108581152283e-12,1.3166451501345513e-23],[3.0815197256396802e-21,5.1574618325053701e-07,0.0024218942363387025,0.16490812574402056,0.83266946427345767],[2.5241228663070259e-08,0.9413050106726466,0.058614536252089039,8.0427834030294678e-05,5.1791160816071143e-15],[9.1344651394222136e-17,0.0026150521331539042,0.59456351581860678,0.40282142656876274,5.479476303853838e-09],[0.99999419984117122,5.8001453549817152e-06,1.3473848481994338e-11,5.5619399768048164e-18,2.9444490509889288e-31],[0.99999858120913965,1.4187898744711938e-06,9.8598470162638682e-13,1.4230788027480849e-19,2.9822440892521961e-33],[2.7516889184283074e-16,0.0031778435809469928,0.56121894657572546,0.43560320013958925,9.7037380185670181e-09],[2.6325931273860108e-16,0.0083046111789675352,0.84697023443808506,0.14472515409258357,2.9036361152490634e-10],[2.3232269623780443e-11,0.3728094053457468,0.61841470109683394,0.0087758935303793133,3.8074230465257296e-12],[0.99999938942597699,6.105738150587696e-07,2.0774452958526153e-13,1.6210660299121819e-20,1.9863547455448852e-34],[6.5406420118166513e-07,0.99918741148289258,0.00081192247746379915,1.1975442373418086e-08,8.0126108553137542e-21],[9.1283770167963839e-09,0.98556995160154393,0.01442794603443154,2.0932356476812262e-06,9.6206312333952191e-18],[0.99999858120913965,1.4187898744711938e-06,9.8598470162638682e-13,1.4230788027480849e-19,2.9822440892521961e-33],[3.1888938312830749e-15,0.0088762555535789635,0.68508043667680951,0.30604330319223988,4.5773686500971775e-09],[3.1585889550964852e-18,0.0019616283901971003,0.78555939057085433,0.21247898066694729,3.7200156932745126e-10],[5.1733118244113788e-05,0.99990380987819072,4.445693643067772e-05,6.7134069443466811e-11,6.7225435334194487e-24],[3.0878205079509472e-12,0.15333334626783315,0.8073183695717101,0.039348284088742191,6.862714207504223e-11],[7.981765442146579e-09,0.95958758219958762,0.040389907409510026,2.2502409136730909e-05,4.4644869310187723e-16]],"max_posterior":[0.62183371941880428,0.80706257394482694,0.7865170324064773,0.94829741757742203,0.99967327040856713,0.70027223996491061,0.55397621424490184,0.89737580954275176,0.91178609130879873,0.89028517466607771,0.76849835582509374,0.85623011055516574,0.70009787052678807,0.8800676970131649,0.94098457633567623,0.99996019627531296,0.98889335336244844,0.57660390361047897,0.99591663300250688,0.99071882992112947,0.54676283907675838,0.71205068966863727,0.93830514126255127,0.91741364272363479,0.85130932213585686,0.94777820770835253,0.7223583108622289,0.5478381366666375,0.53507937346603074,0.99784107736057737,0.51153665603551501,0.98257758582672083,0.85596862885823899,0.99116961669566073,0.98403139143608187,0.98571792994971319,0.91628767633507813,0.7061177217345711,0.9972157229330193,0.80491459379455477,0.84604259150270755,0.60633107071897441,0.99999983838599493,0.99174577994328728,0.99999341843672596,0.99937728574870144,0.71109962232570578,0.70962783989067602,0.96375924327688012,0.96056661273955724,0.99999029261750794,0.95046290641057551,0.55342794771215731,0.99999985211392084,0.68673286531514177,0.9114371009602148,0.92085759133810008,0.91783796531517103,0.99997446003644463,0.93615586238357207,0.97658556902685623,0.7240090730355826,0.58393603989285303,0.8443051375206938,0.52137724925743067,0.97134006836805187,0.55861742981152684,0.85769276656337079,0.86944208846645354,0.99685407411668037,0.99999995407740261,0.99999995407740261,0.99043629185596815,0.91970232973599242,0.93147204846534615,0.84975165040031375,0.76145222636311671,0.5660649442145913,0.98965745928366788,0.99685407411668037,0.98657239783914408,0.5610553650467005,0.99999976769984944,0.86552540918483367,0.54455207798812433,0.62210779737597444,0.95626957168899052,0.81766758513919191,0.93394261851388938,0.73242466301425146,0.85250023096324035,0.94984499004570799,0.96248823110084814,0.99999925088597597,0.83207239768517949,0.99512511372252921,0.78222114453659808,0.9157618596209397,0.94363537628080596,0.99907022028898851,0.95406470168035629,0.91512560663704878,0.99907022028898851,0.83266946427345767,0.9413050106726466,0.59456351581860678,0.99999419984117122,0.99999858120913965,0.56121894657572546,0.84697023443808506,0.61841470109683394,0.99999938942597699,0.99918741148289258,0.98556995160154393,0.99999858120913965,0.68508043667680951,0.78555939057085433,0.99990380987819072,0.8073183695717101,0.95958758219958762],"proportion_misgenotyped":0.14516056596638127},{"name":"tetraploid_bias_1.9","model":"norm","ploidy":4,"simulated_bias":1.8999999999999999,"allele_depth":[31,52,47,0,15,0,35,0,68,21,45,32,32,10,27,30,0,30,63,28,3,0,38,42,58,34,28,40,21,0,19,21,5,12,17,22,10,64,28,67,24,4,15,16,19,14,10,21,23,77,9,62,12,36,20,78,60,31,17,65,24,16,67,59,11,32,17,28,27,27,10,15,0,95,21,22,37,26,53,48,82,8,107,23,32,51,38,19,81,31,35,47,71,12,33,5,10,61,32,7,64,32,12,12,52,15,5,8,15,12,48,63,17,29,0,8,22,0,11,30],"marker_depth":[51,77,66,48,37,49,68,89,80,53,45,80,75,28,56,66,89,49,63,54,28,92,68,62,103,53,54,74,107,39,95,42,30,28,106,84,52,108,110,99,83,48,34,61,51,43,83,46,98,77,62,96,28,97,69,78,98,53,82,99,88,50,106,73,34,84,99,89,48,92,56,34,62,95,109,75,99,109,107,102,110,28,107,39,110,72,94,52,81,79,70,93,103,31,85,25,76,99,48,56,64,110,56,33,52,79,35,80,59,33,91,63,66,55,86,43,41,109,25,108],"bias":1.6115987550707052,"log_likelihood":-489.00547206998971,"prior":[0.068018208584793322,0.23563231899062101,0.36505451503100222,0.25292618301793274,0.078368774375650735],"genotypes":[3,3,3,0,2,0,2,0,3,2,4,2,2,2,2,2,0,3,4,2,1,0,3,3,3,3,2,3,1,0,1,2,1,2,1,2,1,3,2,3,2,1,2,2,2,2,1,2,1,4,1,3,2,2,2,4,3,3,1,3,2,2,3,3,2,2,1,2,3,2,1,2,0,4,1,2,2,1,2,2,3,2,4,3,2,3,2,2,4,2,2,2,3,2,2,1,1,3,3,1,4,2,1,2,4,1,1,1,2,2,2,4,2,2,0,1,2,0,2,2],"posteriors":[[1.1359918475075274e-14,0.00026245325593296737,0.18257574328351023,0.8171618032472695,2.1327614025108194e-10],[3.5136575001200754e-17,6.0582336518668045e-06,0.031765401992262972,0.968228538717821,1.0562637515887526e-09],[1.0909610348407485e-17,2.1476594671191295e-06,0.015665675533935084,0.98433217103694082,5.7696573155374782e-09],[0.98244594307128819,0.017539486402246719,1.4570467677648869e-05,5.8787306082108255e-11,8.6217559234569152e-28],[8.1509725981639015e-11,0.050002990936955941,0.87153325742330245,0.078463751557986822,2.4500596109806025e-13],[0.98358523398177655,0.016402326917257629,1.2439056835986559e-05,4.4129553854471874e-11,5.3118553076101976e-28],[2.2593857373439259e-13,0.0026707930165572303,0.6063679653064652,0.39096124167432156,2.4299136961865477e-12],[0.99811148608886446,0.0018884321633971469,8.1747733058909973e-08,5.5822650169945283e-15,1.9470124897322642e-34],[3.9889522632006779e-22,1.1499025842079102e-09,0.00022230635186370402,0.99977393574556084,3.7567526733054544e-06],[3.9420761364470387e-11,0.044078917122296078,0.90658813864425158,0.049332944194007404,2.3893166321050877e-14],[1.148067352895004e-27,2.8537828288585118e-15,9.7150282165721082e-10,5.0221009147147001e-05,0.99994977801934726],[1.2653174267050661e-11,0.030694580066999207,0.93003689890125774,0.03926852101908597,4.1640712892483627e-15],[5.3674547521703685e-12,0.018573831890394573,0.90619468232693445,0.07523148577728074,2.2751760603208934e-14],[7.9597310668235547e-10,0.12506547003180599,0.83065846748082395,0.044276061691213099,1.8416186591608489e-13],[1.5676432709105603e-12,0.0075568168926157609,0.74942819912172065,0.24301498398302329,1.0728539619068966e-12],[2.6524309318368529e-12,0.011562776800088067,0.84637399303652183,0.14206323016058151,1.5599448664337838e-13],[0.99811148608886446,0.0018884321633971469,8.1747733058909973e-08,5.5822650169945283e-15,1.9470124897322642e-34],[1.1411265912095044e-14,0.0002514512223520526,0.17323097087155706,0.82651757762982259,2.7625663423422477e-10],[3.6147396104533174e-31,4.754682728752742e-18,1.0425625889033445e-11,4.4401245284392383e-06,0.99999555986504607],[4.3745366302355193e-13,0.0033336574220412278,0.58568925602791178,0.41097708654365656,5.9529031914427487e-12],[1.2830487287870291e-06,0.91497961509315828,0.084955453802147321,6.3648055965620252e-05,1.3579190110545897e-18],[0.9983436181612485,0.0016563213786060787,6.0460141867031814e-08,3.278279084907517e-15,8.2154425473751668e-35],[3.1507470383603397e-14,0.00072413958277883523,0.36309604054809314,0.63617981985316707,1.592977408775611e-11],[1.1118655041030593e-16,1.1268380203363062e-05,0.038287604050698497,0.96170112587736067,1.6917376008118363e-09],[5.8127650091768293e-15,0.00032223831077698002,0.31536177154886114,0.68431599013308497,7.2707455332832713e-12],[1.8121122131353886e-15,7.3155090379263345e-05,0.095520883323965383,0.90440596094705483,6.385987168945413e-10],[4.3745366302355193e-13,0.0033336574220412278,0.58568925602791178,0.41097708654365656,5.9529031914427487e-12],[5.4152782267597825e-14,0.0011277305164236738,0.45997225562008137,0.53890001385748609,5.954733909800168e-12],[2.0456213805812529e-08,0.8062486482956418,0.19373083028491828,2.0500963226066966e-05,7.3713332430710258e-22],[0.96660458598616494,0.033328327183675628,6.7085874662474636e-05,9.554970764929921e-10,1.0042493487339353e-25],[1.9608604660647238e-08,0.77945856809739889,0.22050710112812161,3.4311165874621727e-05,2.6106393060812406e-21],[2.2915639118403911e-12,0.0074189276263120375,0.66530802819444312,0.3272730441700949,6.8580827512694599e-12],[1.7658045663329522e-07,0.78100680080064744,0.21860651950783447,0.00038650311106154078,1.7473463394721672e-17],[1.012122392812147e-10,0.044666913775407109,0.81840714457219277,0.13692594154841276,2.7750244780817145e-12],[7.2049854713836381e-08,0.92783235099333716,0.072165247770343041,2.3291864649877278e-06,2.2456014498142441e-23],[2.3914688447872677e-09,0.40089647788230959,0.59838723560023299,0.00071628412598848503,8.1581406333201743e-19],[4.3632577043931164e-08,0.75803547168025021,0.24182232744099694,0.00014215724617596681,2.887341725654026e-19],[9.6232899103909487e-16,9.3559969282939342e-05,0.17551616589032604,0.82439027411767907,2.2710674726697645e-11],[2.418319238957213e-09,0.45002830562584573,0.54963383461089554,0.00033785734493948513,8.8582427884018562e-20],[8.4992140075875364e-18,2.6941678582920261e-06,0.023862983978115787,0.97613432110335319,7.5067259007550403e-10],[8.5887103069702466e-10,0.25928472094351535,0.73884694521429783,0.0018683329833155463,5.3866788488515992e-18],[2.523040227067205e-06,0.98182299891654601,0.018173610499507799,8.6754371877248202e-07,1.4770629319609466e-22],[3.5082717684059207e-11,0.029805711014612678,0.82107289022733465,0.14912139872131897,1.6511638647269574e-12],[3.4907952323819257e-09,0.39610203412638118,0.60257282645039456,0.0013251359324289174,7.6803342735905831e-18],[9.885139458875502e-11,0.069201404410252396,0.89991233614686406,0.030886259344022639,9.2928877110974316e-15],[7.2448945942496505e-10,0.16336697315913468,0.82316731766683948,0.013465708449533693,2.7286826065221867e-15],[3.7914967113108949e-07,0.97532659868353289,0.024672601855395174,4.2031140066806191e-07,4.0627282321359912e-24],[7.551321095233896e-12,0.016184306670481787,0.81602280506098446,0.16779288826022307,7.5933942123705457e-13],[5.6283150566947701e-09,0.57706837901762065,0.42275521470857141,0.00017640064549259531,3.8825499026075461e-20],[2.3500990056398698e-33,8.4225076546346125e-20,5.8058086077785866e-13,9.3149424090966422e-07,0.99999906850517828],[1.7943326589756367e-07,0.92637235157238984,0.073620116690383181,7.3523039611298452e-06,1.1820394611112272e-21],[6.5932163770240234e-17,1.1959159566956425e-05,0.054002257437808218,0.94598578317243853,2.3018665347955696e-10],[1.012122392812147e-10,0.044666913775407109,0.81840714457219277,0.13692594154841276,2.7750244780817145e-12],[2.650787763984446e-11,0.050021374632149458,0.93351315419021008,0.016465471151131804,3.5464995376822862e-16],[1.0809860632366558e-09,0.26104687908824703,0.73640897650653292,0.002544143324233672,1.7734612459354973e-17],[1.6911940697480279e-33,6.4657799532198479e-20,4.801309556539324e-13,8.4022558888926038e-07,0.99999915977393072],[4.4270743369538121e-16,4.918248303810997e-05,0.1183928046295319,0.88155801282791246,5.9516792664306011e-11],[2.7259876351644343e-14,0.00050807034643927571,0.26145347357614074,0.73803845599309736,8.4295117304457371e-11],[1.7193887478646019e-08,0.72991445660630361,0.27001209494131156,7.3431258497181921e-05,1.5793145283586921e-20],[2.9765167107728779e-17,6.8319094087540169e-06,0.040297422528876532,0.95969574522738399,3.343308785922005e-10],[1.505125027695112e-09,0.33970125595722772,0.65931976191210817,0.00097898062553944051,1.2766990194866312e-18],[6.3803240974222436e-10,0.1700231542797882,0.82048899607159154,0.0094878490105874212,7.6505227656342751e-16],[9.8801519472477122e-17,1.7580667709866853e-05,0.070996054987383025,0.92898636423130521,1.1360200135122096e-10],[1.5514417529342833e-20,1.7037186831843988e-08,0.0010173568717983701,0.99898224848289641,3.7760811837993674e-07],[1.3172904890913225e-09,0.18207519854702506,0.79976763303551568,0.018157167100156395,1.2352783981224692e-14],[2.3680437089017064e-11,0.043962244639989989,0.93222670201794033,0.023811053318387641,1.1705535416384742e-15],[5.0423508287378748e-08,0.894467267706334,0.10552668697852217,5.9948916351496794e-06,1.3051634127587676e-22],[2.8804638402202346e-10,0.16125677930694218,0.83501266546940855,0.0037305549356026861,1.7970301952254464e-17],[1.1583649543755067e-13,0.0012356435194861828,0.3694751876233377,0.62928916880817132,4.8889002916640219e-11],[6.3554829554310032e-10,0.23775962261923683,0.76038902603520131,0.0018513507100137395,3.913224838789307e-18],[6.326496335840332e-08,0.82229890454069909,0.17764107223035694,5.9959963980714324e-05,5.1320106896956676e-20],[3.5082717684059207e-11,0.029805711014612678,0.82107289022733465,0.14912139872131897,1.6511638647269574e-12],[0.9926548864765149,0.0073432203256570093,1.8931963412672418e-06,1.4868449808666335e-12,1.8359806690301145e-30],[1.031494833585986e-35,1.0615031055478976e-21,2.4827841468816581e-14,1.6735779897746045e-07,0.99999983264217618],[2.287973355125352e-08,0.82392630278100221,0.17605778356750093,1.5890771763618739e-05,4.5451291757564301e-22],[8.4084507470422392e-10,0.24401081370984637,0.75350523431443617,0.0024839511348725457,1.2963807750236342e-17],[2.3053802827313341e-11,0.046907583963659044,0.93574540128782113,0.017347014725465595,3.7905069092778477e-16],[4.4681997650817879e-09,0.55639018866477141,0.44343889490144889,0.00017091196557991398,2.6353498119886553e-20],[1.4055685761972682e-13,0.0027172197068494369,0.71652863531419775,0.28075414497854395,2.6861289718413044e-13],[4.5302657971565103e-13,0.0053665677162430779,0.82061093043320732,0.17402250185001375,8.2465237926947782e-14],[5.2320349993872116e-20,6.5429621213456696e-08,0.0030385015632770304,0.99696142021364542,1.2793456467155307e-08],[6.2471547223937588e-09,0.30392218271751847,0.68469199052790675,0.011385820507409631,1.0210858559283315e-14],[4.4759318369809561e-37,8.38793676858757e-23,3.9535075755357062e-15,6.1215052777849214e-08,0.99999993878494309],[1.0212775059087911e-13,0.00089524467123989441,0.27456913276847605,0.72453562232042246,2.3975947743436116e-10],[5.7046839370229691e-10,0.24488277821437213,0.75378262884438851,0.0013345923707708618,1.2475357636643855e-18],[7.6877764667592549e-18,1.8189093249281389e-06,0.015240045496620834,0.98475813126336542,4.3306891323354292e-09],[7.6343376215223544e-12,0.025339886214739656,0.93551252384713202,0.039147589930490882,2.728639958217451e-15],[1.2087395240116008e-10,0.077601133567033465,0.89651976589451232,0.025879100417574352,5.8648875185221458e-15],[6.4381808202740348e-34,2.9740569726674878e-20,2.7467939729515605e-13,6.2033897965491281e-07,0.99999937966074548],[1.7356651437134898e-11,0.036114614103702238,0.93082985929913942,0.033055526579798586,2.902799718750878e-15],[3.7868948756819321e-13,0.0037808374970365703,0.68185009105521421,0.31436907144623677,1.1333330592005509e-12],[1.300728847266703e-13,0.0023723309953751432,0.66066121312586723,0.33696645587798468,6.4315837428933506e-13],[3.1413212604028332e-18,1.3302064469323e-06,0.016399840684326431,0.98359882792267983,1.1865470956923504e-09],[2.4576181626946396e-10,0.076607535692426246,0.85696683854152311,0.066425625519970929,3.1790466801983192e-13],[1.746047294823627e-11,0.037623247669119619,0.93411255093039358,0.028264201383025041,1.6722403255892421e-15],[8.4831127367483151e-08,0.63172967650209444,0.36631278721635679,0.0019574514504210163,6.2458008469340681e-16],[2.5696848563456253e-07,0.96033381244284566,0.039664604812772147,1.3257758969191922e-06,3.451790316546072e-23],[3.3809640525121307e-16,4.0779423983426354e-05,0.10772736817310601,0.89223185233473079,6.8179552908714129e-11],[9.5983554302437877e-16,4.0893803066401779e-05,0.064041929936128492,0.93591717432249122,1.938313045350576e-09],[4.0360883631089768e-07,0.95233635541356798,0.047659385998613524,3.8549789818247664e-06,6.8417933201230071e-22],[2.4486977970346462e-31,3.4837840876332291e-18,8.3502865128962736e-12,3.9395875986747571e-06,0.9999960604040512],[5.7046839370229691e-10,0.24488277821437213,0.75378262884438851,0.0013345923707708618,1.2475357636643855e-18],[1.9772444320769944e-08,0.65881991487699865,0.34089835362047444,0.00028171173008258242,6.7028556570666164e-19],[4.1568228563803181e-10,0.10405185219465336,0.85473121753631154,0.041216929853269427,8.3399607807235878e-14],[3.8226781238406163e-29,1.9393666465165158e-16,1.4556039299723021e-10,1.8270505515097433e-05,0.99998172934892426],[3.204162052143527e-08,0.81100378150727837,0.18896237575586625,3.3810695235124655e-05,5.0118648579815637e-21],[3.1667349328843506e-07,0.87278546060976059,0.12712914808880871,8.507462793776524e-05,7.4332759073984765e-19],[9.4095781531090338e-07,0.98755476227448458,0.012444178403119259,1.1836458114592898e-07,6.9395584676873983e-25],[4.8365886019433294e-09,0.43923271406966052,0.55968358074597346,0.0010837003477775441,5.9655859044297193e-18],[4.1568228563803181e-10,0.10405185219465336,0.85473121753631154,0.041216929853269427,8.3399607807235878e-14],[5.076796023229001e-14,0.0012749710385225198,0.53231025115739472,0.46641477780207452,1.957013093468328e-12],[3.6147396104533174e-31,4.754682728752742e-18,1.0425625889033445e-11,4.4401245284392383e-06,0.99999555986504607],[3.7254925920577758e-09,0.4228547699722861,0.57617908423640374,0.00096614206581742521,3.2146065625574511e-18],[2.8641433288573816e-13,0.0025834409321880414,0.54078937284872419,0.45662718621072806,8.0734808839319687e-12],[0.99783947286606722,0.0021604157027334129,1.1143118969292497e-07,9.6485772638772442e-15,4.7345086550735427e-34],[6.5211098157665071e-08,0.76143123077150998,0.23835997310208112,0.00020873091531050213,1.2401481898873363e-18],[6.5885872295432585e-13,0.0033066055153036554,0.50183098584713814,0.49486240860602504,3.0874016917159327e-11],[0.99916606105798866,0.00083392641836702985,1.2523644042936571e-08,2.0560616721133302e-16,9.4259812524305469e-37],[1.0965214817811709e-10,0.042157713490703873,0.78815460236139945,0.16968768403063947,7.6050773769993903e-12],[9.8780497944294297e-10,0.31049381090715406,0.68864698191238505,0.00085920619265576,5.4890314046981151e-19]],"max_posterior":[0.8171618032472695,0.968228538717821,0.98433217103694082,0.98244594307128819,0.87153325742330245,0.98358523398177655,0.6063679653064652,0.99811148608886446,0.99977393574556084,0.90658813864425158,0.99994977801934726,0.93003689890125774,0.90619468232693445,0.83065846748082395,0.74942819912172065,0.84637399303652183,0.99811148608886446,0.82651757762982259,0.99999555986504607,0.58568925602791178,0.91497961509315828,0.9983436181612485,0.63617981985316707,0.96170112587736067,0.68431599013308497,0.90440596094705483,0.58568925602791178,0.53890001385748609,0.8062486482956418,0.96660458598616494,0.77945856809739889,0.66530802819444312,0.78100680080064744,0.81840714457219277,0.92783235099333716,0.59838723560023299,0.75803547168025021,0.82439027411767907,0.54963383461089554,0.97613432110335319,0.73884694521429783,0.98182299891654601,0.82107289022733465,0.60257282645039456,0.89991233614686406,0.82316731766683948,0.97532659868353289,0.81602280506098446,0.57706837901762065,0.99999906850517828,0.92637235157238984,0.94598578317243853,0.81840714457219277,0.93351315419021008,0.73640897650653292,0.99999915977393072,0.88155801282791246,0.73803845599309736,0.72991445660630361,0.95969574522738399,0.65931976191210817,0.82048899607159154,0.92898636423130521,0.99898224848289641,0.79976763303551568,0.93222670201794033,0.894467267706334,0.83501266546940855,0.62928916880817132,0.76038902603520131,0.82229890454069909,0.82107289022733465,0.9926548864765149,0.99999983264217618,0.82392630278100221,0.75350523431443617,0.93574540128782113,0.55639018866477141,0.71652863531419775,0.82061093043320732,0.99696142021364542,0.68469199052790675,0.99999993878494309,0.72453562232042246,0.75378262884438851,0.98475813126336542,0.93551252384713202,0.89651976589451232,0.99999937966074548,0.93082985929913942,0.68185009105521421,0.66066121312586723,0.98359882792267983,0.85696683854152311,0.93411255093039358,0.63172967650209444,0.96033381244284566,0.89223185233473079,0.93591717432249122,0.95233635541356798,0.9999960604040512,0.75378262884438851,0.65881991487699865,0.85473121753631154,0.99998172934892426,0.81100378150727837,0.87278546060976059,0.98755476227448458,0.55968358074597346,0.85473121753631154,0.53231025115739472,0.99999555986504607,0.57617908423640374,0.54078937284872419,0.99783947286606722,0.76143123077150998,0.50183098584713814,0.99916606105798866,0.78815460236139945,0.68864698191238505],"proportion_misgenotyped":0.16657367422995606},{"name":"tetraploid_bias_1.9","model":"hw","ploidy":4,"simulated_bias":1.8999999999999999,"allele_depth":[31,52,47,0,15,0,35,0,68,21,45,32,32,10,27,30,0,30,63,28,3,0,38,42,58,34,28,40,21,0,19,21,5,12,17,22,10,64,28,67,24,4,15,16,19,14,10,21,23,77,9,62,12,36,20,78,60,31,17,65,24,16,67,59,11,32,17,28,27,27,10,15,0,95,21,22,37,26,53,48,82,8,107,23,32,51,38,19,81,31,35,47,71,12,33,5,10,61,32,7,64,32,12,12,52,15,5,8,15,12,48,63,17,29,0,8,22,0,11,30],"marker_depth":[51,77,66,48,37,49,68,89,80,53,45,80,75,28,56,66,89,49,63,54,28,92,68,62,103,53,54,74,107,39,95,42,30,28,106,84,52,108,110,99,83,48,34,61,51,43,83,46,98,77,62,96,28,97,69,78,98,53,82,99,88,50,106,73,34,84,99,89,48,92,56,34,62,95,109,75,99,109,107,102,110,28,107,39,110,72,94,52,81,79,70,93,103,31,85,25,76,99,48,56,64,110,56,33,52,79,35,80,59,33,91,63,66,55,86,43,41,109,25,108],"bias":1.6223205411055459,"log_likelihood":-489.46622262108667,"prior":[0.057299467680516775,0.23926257990215066,0.37465388723562015,0.26073731356508439,0.068046751616628043],"genotypes":[3,3,3,0,2,0,2,0,3,2,4,2,2,2,2,2,0,3,4,2,1,0,3,3,3,3,2,3,1,0,1,2,1,2,1,2,1,3,2,3,2,1,2,2,2,2,1,2,1,4,1,3,2,2,2,4,3,3,1,3,2,2,3,3,2,2,1,2,3,2,1,2,0,4,1,2,2,1,2,2,3,2,4,3,2,3,2,2,4,2,2,2,3,2,2,1,1,3,3,1,4,2,1,2,4,1,1,1,2,2,2,4,2,2,0,1,3,0,2,2],"posteriors":[[9.2310840700396097e-15,0.0002484815300348727,0.17692693890685676,0.82282457938208231,1.8101677325051248e-10],[2.8647028833959285e-17,5.6972524151809054e-06,0.03047830486059502,0.96951599698750945,8.9948019157801566e-10],[8.9347769820186661e-18,2.0266776998133419e-06,0.01505099209381844,0.98494697629306804,4.9354133897892191e-09],[0.9782738852192987,0.021707328347195961,1.8786355941537261e-05,7.7563993369570015e-11,8.9077251960127182e-28],[6.6584703873308428e-11,0.04845772352181274,0.87048216273240853,0.081060113678984955,2.0904546967148111e-13],[0.9796707021586224,0.020313243925901756,1.6053857190526009e-05,5.828522692358745e-11,5.4895108597527754e-28],[1.8460971922288192e-13,0.0025533594696477077,0.59650398173833152,0.40094265878976243,2.0737392096966663e-12],[0.99762402668746775,0.002375865086415181,1.08226109312957e-07,7.5741474038779257e-15,2.01918419497464e-34],[3.347546785083573e-22,1.0958245186529541e-09,0.00021359059243621838,0.99978311540533349,3.2929064053020114e-06],[3.2203014300933339e-11,0.042631447437433141,0.90614955512527251,0.051218997405071007,2.0386578206686041e-14],[1.1064857872060771e-27,3.1215655392019709e-15,1.0636674144097014e-09,5.643507808379963e-05,0.99994356385824545],[1.0343648335797925e-11,0.029617643358386462,0.92943657614903186,0.040945780482234337,3.5554253805913588e-15],[4.3953246100205513e-12,0.017916873574334181,0.90382301829771572,0.078260108123535649,1.9459725729454696e-14],[6.4919062842610858e-10,0.12170472397949489,0.83257391716706775,0.045721358204089638,1.568818106311869e-13],[1.2831655818466264e-12,0.0072687136937759182,0.74235893315932044,0.25037235314470324,9.1722349302571775e-13],[2.1731958480286999e-12,0.011142931846792112,0.84167431837101014,0.14718274977989085,1.3349443333627557e-13],[0.99762402668746775,0.002375865086415181,1.08226109312957e-07,7.5741474038779257e-15,2.01918419497464e-34],[9.2744856313602153e-15,0.00023814632319501058,0.16788174239174972,0.83188011105053239,2.3451345906232724e-10],[3.4838382832258479e-31,5.1556938290492043e-18,1.1266219729072485e-11,4.9338793182522041e-06,0.99999506610941558],[3.5723296371882966e-13,0.0031910558774635727,0.5763159008423423,0.42049304327475956,5.0774701275238947e-12],[1.0481485816200693e-06,0.91227112136449839,0.087660492732859904,6.7337754059980156e-05,1.1586509017287207e-18],[0.9979144201590765,0.0020854996888514752,8.0152067674581003e-08,4.4543208445986292e-15,8.5204704081999409e-35],[2.5628558963156442e-14,0.00068673579855220326,0.35352472077371627,0.64578854341417258,1.3533787906720521e-11],[9.0677038746576126e-17,1.0626185116951083e-05,0.036832247945924559,0.96315712442791479,1.4410436056767254e-09],[4.7209007659440801e-15,0.00030391286552528739,0.30555945767315223,0.69413662945514998,6.1676565421124976e-12],[1.4738493411608349e-15,6.9120759614799734e-05,0.092233334303635284,0.90769754439425432,5.4249419175402967e-10],[3.5723296371882966e-13,0.0031910558774635727,0.5763159008423423,0.42049304327475956,5.0774701275238947e-12],[4.4123285767840584e-14,0.0010720545099035192,0.4494498078923877,0.54947813759259745,5.0676725203009014e-12],[1.6792834593271111e-08,0.79858378897821147,0.2013939894774977,2.2204751455875044e-05,6.3203920973917977e-22],[0.95897116641425106,0.040943243624519191,8.5588714718165464e-05,1.2465113992879169e-09,1.0337547636601465e-25],[1.6086979003797004e-08,0.77126834658382315,0.22869459636559891,3.7040963599089157e-05,2.2370406954089883e-21],[1.8728207276826094e-12,0.0071346988200715776,0.65756952379069122,0.33529577738151028,5.8541792906610469e-12],[1.4451699739012123e-07,0.77502630552088281,0.22456550104372219,0.00040804891839766325,1.4936689278725111e-17],[8.2684267763898032e-11,0.043322323500813879,0.81594850604996305,0.14072917036417118,2.3678582351970595e-12],[5.9163938330569371e-08,0.92438244182204643,0.075614959926531805,2.5390874836761521e-06,1.9259981686229063e-23],[1.9471443653912116e-09,0.39070432328288912,0.60853700840521274,0.00075866636475377576,6.9378323130341377e-19],[3.5748465071191064e-08,0.75045751045584141,0.24939079791780872,0.0001516558778848678,2.4708356570211295e-19],[7.7996719100778316e-16,8.7796034595890061e-05,0.16892217761510092,0.83099002633107699,1.9225689610315353e-11],[1.970121115697259e-09,0.43889832415834962,0.56074185989295511,0.00035981397857454584,7.5374785126235442e-20],[6.9305102364593561e-18,2.5267183978690088e-06,0.022831939307487403,0.97716553333476819,6.3934620209244187e-10],[6.9800007260828904e-10,0.25145930601987038,0.74657352834137469,0.0019671649407549062,4.572437499509598e-18],[2.0538531574371064e-06,0.98104882085382161,0.018948194442531163,9.3085048982734708e-07,1.2558660705138217e-22],[2.8679193502333006e-11,0.028850706213410383,0.81766920639910545,0.15348008735739532,1.4098176166231542e-12],[2.8435686303761854e-09,0.38660810752021485,0.61199472213225314,0.0013971675039634599,6.5345940764206626e-18],[8.0636508572060268e-11,0.066986310339514257,0.90091810511565085,0.032095584464190481,7.9176940955319923e-15],[5.8995041508205839e-10,0.15867582248426587,0.82731343971503013,0.014010737210751339,2.3207885002834861e-15],[3.1001599872901675e-07,0.97410413964890663,0.025895093267812988,4.570672815355021e-07,3.4696877423591334e-24],[6.1795496256508797e-12,0.015621852274695957,0.81129836501224351,0.17307978270623198,6.4903619849205152e-13],[4.5982574576678636e-09,0.5659653347721878,0.43384597611102232,0.00018868451853205104,3.3130739988329316e-20],[2.2649953218907704e-33,9.0845588015277972e-20,6.2230471365828901e-13,1.0277193850626146e-06,0.99999897227999279],[1.4706462884949718e-07,0.92326148181943624,0.076730443648701033,7.9274672340554115e-06,1.0118972263134934e-21],[5.3575798824048079e-17,1.1210638983314565e-05,0.051744742503127578,0.9482440466625226,1.9536670973556498e-10],[8.2684267763898032e-11,0.043322323500813879,0.81594850604996305,0.14072917036417118,2.3678582351970595e-12],[2.1615257603866279e-11,0.048229103922190729,0.93454662552789058,0.017224270528303533,3.0205396701221188e-16],[8.7895231529376422e-10,0.25345748550078467,0.74386958233671951,0.0026729312835433259,1.5061425709952601e-17],[1.6299512022139537e-33,6.9716293454837275e-20,5.1436275179377783e-13,9.2658771646037391e-07,0.9999990734117693],[3.589542100355887e-16,4.6129160330735096e-05,0.11373863784415134,0.88621523294511406,5.0403519970584564e-11],[2.2156217911124728e-14,0.00048172583909535338,0.25402987391497772,0.74548840017434403,7.1560477876245393e-11],[1.4090827118847601e-08,0.72090930603101822,0.27901178749692385,7.8892381230488749e-05,1.3518553628270928e-20],[2.4212281802546286e-17,6.4030768651573499e-06,0.038580315140551685,0.96141328149852756,2.8405535393750836e-10],[1.2241751815188929e-09,0.33025121108851613,0.66871393315342587,0.0010348545338828142,1.0845727962848715e-18],[5.1927198440121012e-10,0.16499518335521215,0.82511057781863539,0.0098942383068796654,6.5034287277669021e-16],[8.0177389875355076e-17,1.6462746411761716e-05,0.068018162497786028,0.93196537465951412,9.6288269543505846e-11],[1.2905054509426141e-20,1.6177418844252569e-08,0.0009773344942462076,0.99902232125972135,3.2806861344977963e-07],[1.0732727770083035e-09,0.17722774566694954,0.8039307498478987,0.018841503411868656,1.051217241410655e-14],[1.9327443051772711e-11,0.042416776543355357,0.93271958832320889,0.024863635114107809,9.9787268496001648e-16],[4.1414971749925296e-08,0.8897474841746118,0.11024595942407568,6.5149863407186163e-06,1.1196658091401482e-22],[2.3402116153083791e-10,0.15587023466655911,0.84021380800516587,0.0039159570942533537,1.5249207002085587e-17],[9.4285719336053801e-14,0.0011764920369302465,0.36077107595808627,0.63805243196332606,4.1563369864151582e-11],[5.162922672813042e-10,0.23029221835369684,0.76775790146178824,0.0019498796682226732,3.3203284998919048e-18],[5.187266058857007e-08,0.81603993106407469,0.18389578718806429,6.4229875200296022e-05,4.3950312087550607e-20],[2.8679193502333006e-11,0.028850706213410383,0.81766920639910545,0.15348008735739532,1.4098176166231542e-12],[0.99084175447552048,0.009155776410155617,2.4691123382885023e-06,1.9858522470496472e-12,1.9014873303837171e-30],[9.9414161335449902e-36,1.1385294282312743e-21,2.6380434006894992e-14,1.8321524821272612e-07,0.99999981678472538],[1.8787799127276353e-08,0.816743886188254,0.18323886252847513,1.7232495471759751e-05,3.8982563843078274e-22],[6.8346418885586827e-10,0.23667060821740604,0.76071873307024662,0.0026106580288833771,1.1006051257676538e-17],[1.880319438235084e-11,0.045221518081307276,0.93663102852914293,0.018147453370746316,3.2291300249338329e-16],[3.648634717562881e-09,0.54501812926493109,0.454798971521089,0.0001828955653456083,2.2476841897735675e-20],[1.1515779775157048e-13,0.0026002059499298453,0.70725633950229527,0.29014345454743007,2.2986207690764791e-13],[3.7165682444173239e-13,0.0051554644537575153,0.814175289290614,0.18066924625518618,7.0662419181520932e-14],[4.3130605095221022e-20,6.1575283075562606e-08,0.0029034696182161489,0.99709645779106038,1.1015440482214642e-08],[5.0927835904969796e-09,0.29723175527021223,0.69094123962561382,0.01182700001138156,8.6942929003032008e-15],[4.3138463375258813e-37,8.9677992586890439e-23,4.1796248581836065e-15,6.6712338320510376e-08,0.99999993328765768],[8.3090950850967314e-14,0.00085245775266773384,0.26765962322932169,0.73148791881418362,2.037439758303837e-10],[4.632424453583705e-10,0.23702543544771038,0.76156588855545759,0.0014086755335895522,1.0581054907173539e-18],[6.2939092144422162e-18,1.7144069062774023e-06,0.014627129216631741,0.98537115267327369,3.7031882100163819e-09],[6.2442388092895438e-12,0.024431708140227049,0.93469340155241387,0.040874890301112278,2.3310604936668507e-15],[9.8558523536129786e-11,0.075121978976688555,0.89796860832503012,0.026909412599717759,4.9948249339380355e-15],[6.2050364572174666e-34,3.2035098889265953e-20,2.9380515350796889e-13,6.8315752412019467e-07,0.99999931684218191],[1.4179666591081185e-11,0.034852702517826149,0.93067074224141078,0.034476555226580531,2.4769446681855275e-15],[3.0983839010810092e-13,0.0036229890135084748,0.67288035991560702,0.32349665106960668,9.6852013425921973e-13],[1.0643024697593114e-13,0.0022673736349628957,0.65067934221060442,0.3470532841537769,5.4966156134187242e-13],[2.5659675018501952e-18,1.2477809709066208e-06,0.015681568098875068,0.98431718310782246,1.0123317615880374e-09],[2.0060217545316772e-10,0.074388550701390069,0.85706080724484324,0.068550641852893446,2.7102988744976987e-13],[1.4260005082230618e-11,0.036296382441030853,0.93419696598570023,0.029506651559007186,1.4264663526542343e-15],[6.9345508172285475e-08,0.62427023160225004,0.37367991882940044,0.0020497802228409347,5.3327381575158909e-16],[2.104097074738804e-07,0.95846603522475038,0.041532316363353361,1.4380021886970068e-06,2.9520893622615071e-23],[2.7416202028201376e-16,3.8235142761294969e-05,0.10343507616376728,0.89652668863572516,5.7745746169211188e-11],[7.8216254260681947e-16,3.8692520066913012e-05,0.061843110297411089,0.93811819553275244,1.6497689724670339e-09],[3.3022077980998767e-07,0.95029869085530827,0.049696828053733294,4.1508701787447007e-06,5.8467271784458917e-22],[2.3600227282944285e-31,3.7760448117417409e-18,9.0178613926765271e-12,4.3752798626276093e-06,0.99999562471111958],[4.632424453583705e-10,0.23702543544771038,0.76156588855545759,0.0014086755335895522,1.0581054907173539e-18],[1.617813773145936e-08,0.64956112449573378,0.35013911749623761,0.00029974182989087831,5.7283188115847809e-19],[3.3903825291715956e-10,0.1010886115287624,0.85628012444974722,0.042631263682381113,7.1047737716390748e-14],[3.6842404574166769e-29,2.1134789792338511e-16,1.5849036289588575e-10,2.0435027087165136e-05,0.99997956481442229],[2.6290358650306022e-08,0.80389826863314062,0.19606526428311782,3.6440793383345583e-05,4.2951721910260363e-21],[2.5921823338972723e-07,0.86861548124945653,0.13129388180014534,9.0377732164452376e-05,6.3552577081901868e-19],[7.6690024238198766e-07,0.98692585621449214,0.013073248240298466,1.2864496750019893e-07,5.9074458636971029e-25],[3.9426367950513632e-09,0.42943488602407331,0.5694211395656672,0.0011439704676227637,5.0792533488717572e-18],[3.3903825291715956e-10,0.1010886115287624,0.85628012444974722,0.042631263682381113,7.1047737716390748e-14],[4.1426693362378954e-14,0.0012130877364897022,0.52123997184595572,0.47754694041584572,1.6679515026011733e-12],[3.4838382832258479e-31,5.1556938290492043e-18,1.1266219729072485e-11,4.9338793182522041e-06,0.99999506610941558],[3.0355989794002227e-09,0.41293652428438327,0.58604258842530366,0.0010208842547141462,2.7358210227310103e-18],[2.3372134154685589e-13,0.0024690312492100671,0.53111525561198114,0.46641571313169361,6.8811897186227524e-12],[0.99728404038629703,0.0027158122964499802,1.4731723978538492e-07,1.307229980398511e-14,4.90967172074845e-34],[5.3409596815605189e-08,0.7543158576882647,0.24546222259405459,0.0002218663080840094,1.0608903079438532e-18],[5.3732024129742717e-13,0.003165724776026356,0.4930057764376185,0.50382849875951918,2.6298627957881409e-11],[0.99894570949062955,0.0010542737908584656,1.6718511616165573e-08,2.8137989760444752e-16,9.7780028133078369e-37],[8.9567940988050589e-11,0.040896445080111692,0.78509683347545034,0.17400672134838166,6.4884071325706322e-12],[8.0275170425888721e-10,0.3012383683474813,0.6978523886565402,0.00090924219322646249,4.6591312778659347e-19]],"max_posterior":[0.82282457938208231,0.96951599698750945,0.98494697629306804,0.9782738852192987,0.87048216273240853,0.9796707021586224,0.59650398173833152,0.99762402668746775,0.99978311540533349,0.90614955512527251,0.99994356385824545,0.92943657614903186,0.90382301829771572,0.83257391716706775,0.74235893315932044,0.84167431837101014,0.99762402668746775,0.83188011105053239,0.99999506610941558,0.5763159008423423,0.91227112136449839,0.9979144201590765,0.64578854341417258,0.96315712442791479,0.69413662945514998,0.90769754439425432,0.5763159008423423,0.54947813759259745,0.79858378897821147,0.95897116641425106,0.77126834658382315,0.65756952379069122,0.77502630552088281,0.81594850604996305,0.92438244182204643,0.60853700840521274,0.75045751045584141,0.83099002633107699,0.56074185989295511,0.97716553333476819,0.74657352834137469,0.98104882085382161,0.81766920639910545,0.61199472213225314,0.90091810511565085,0.82731343971503013,0.97410413964890663,0.81129836501224351,0.5659653347721878,0.99999897227999279,0.92326148181943624,0.9482440466625226,0.81594850604996305,0.93454662552789058,0.74386958233671951,0.9999990734117693,0.88621523294511406,0.74548840017434403,0.72090930603101822,0.96141328149852756,0.66871393315342587,0.82511057781863539,0.93196537465951412,0.99902232125972135,0.8039307498478987,0.93271958832320889,0.8897474841746118,0.84021380800516587,0.63805243196332606,0.76775790146178824,0.81603993106407469,0.81766920639910545,0.99084175447552048,0.99999981678472538,0.816743886188254,0.76071873307024662,0.93663102852914293,0.54501812926493109,0.70725633950229527,0.814175289290614,0.99709645779106038,0.69094123962561382,0.99999993328765768,0.73148791881418362,0.76156588855545759,0.98537115267327369,0.93469340155241387,0.89796860832503012,0.99999931684218191,0.93067074224141078,0.67288035991560702,0.65067934221060442,0.98431718310782246,0.85706080724484324,0.93419696598570023,0.62427023160225004,0.95846603522475038,0.89652668863572516,0.93811819553275244,0.95029869085530827,0.99999562471111958,0.76156588855545759,0.64956112449573378,0.85628012444974722,0.99997956481442229,0.80389826863314062,0.86861548124945653,0.98692585621449214,0.5694211395656672,0.85628012444974722,0.52123997184595572,0.99999506610941558,0.58604258842530366,0.53111525561198114,0.99728404038629703,0.7543158576882647,0.50382849875951918,0.99894570949062955,0.78509683347545034,0.6978523886565402],"proportion_misgenotyped":0.16681867603143885},{"name":"tetraploid_no_bias","model":"norm","ploidy":4,"simulated_bias":1,"allele_depth":[31,12,0,0,31,70,20,0,31,0,0,61,33,0,31,42,42,46,0,0,39,13,0,42,7,0,24,10,59,16,0,0,16,8,40,0,0,51,39,21,25,38,40,14,36,46,31,0,0,26,0,0,26,41,0,3,41,38,7,75,13,0,17,17,13,10,7,18,0,78,3,23,35,11,0,28,11,13,0,6,0,14,38,0,0,22,9,12,57,6,69,28,13,0,0,0,49,13,29,23,70,18,26,28,25,0,0,21,21,21,27,21,7,0,11,30,27,35,17,0],"marker_depth":[53,26,25,80,98,70,92,73,101,55,88,83,62,36,91,54,75,75,45,75,101,35,45,82,35,43,102,66,109,54,30,38,85,29,62,57,90,71,68,100,62,61,108,83,57,77,86,37,25,67,75,99,29,71,44,50,83,76,35,95,50,47,64,63,98,57,29,82,36,102,52,51,46,104,66,99,45,75,56,61,61,84,84,87,28,110,48,44,57,35,94,94,67,44,37,83,97,71,37,65,98,53,39,64,76,83,109,82,79,54,38,98,65,89,81,93,90,41,92,28],"bias":1.1147870819156167,"log_likelihood":-447.27662597110537,"prior":[0.28009526819927782,0.35207395022349075,0.24810379140751834,0.098017652278417786,0.021709337891295283],"genotypes":[2,2,0,0,1,4,1,0,1,0,0,3,2,0,1,3,2,2,0,0,2,2,0,2,1,0,1,1,2,1,0,0,1,1,2,0,0,3,2,1,2,2,2,1,2,2,2,0,0,2,0,0,3,2,0,1,2,2,1,3,1,0,1,1,1,1,1,1,0,3,1,2,3,1,0,1,1,1,0,1,0,1,2,0,0,1,1,1,4,1,3,1,1,0,0,0,2,1,3,1,3,1,3,2,1,0,0,1,1,2,3,1,1,0,1,1,1,3,1,0],"posteriors":[[2.3678351553706237e-13,0.0097050719827897498,0.79576719420988151,0.19452773378352553,2.3566331802281304e-11],[3.8556398265372058e-10,0.18416890154000079,0.78163030815802259,0.034200789913204542,3.2080293825122658e-12],[0.98823542038666456,0.011747089931604143,1.7488848549716132e-05,8.3318151490046054e-10,6.088395083170422e-23],[0.99992403122843865,7.596846621746157e-05,3.0534392832462067e-10,1.2250247704696394e-17,1.4202336352170944e-34],[1.7179078798379805e-09,0.73058763298335594,0.26931942175281975,9.294354591607368e-05,2.8495009747906721e-18],[5.6294227523984802e-31,3.1855998544857416e-16,9.6138130274392895e-10,9.4052909542590075e-05,0.99990594612907568],[4.9880269919045773e-08,0.97561654827135313,0.024382680264379807,7.2158399698925822e-07,2.1678928200765866e-21],[0.99987824452224361,0.00012175464268840708,8.3506793397851404e-10,6.1803212459404295e-17,1.459807870809796e-33],[2.2745345404217885e-09,0.77688407097759771,0.22305824583634049,5.7680911527127037e-05,1.2602661591552743e-18],[0.99951344461950564,0.00048653906588426145,1.6314602676082994e-08,7.630252265628245e-15,1.5958577942927904e-30],[0.99995403107488867,4.5968820227455208e-05,1.0488400943435135e-10,2.2081405713638462e-18,1.2170242546100383e-35],[6.2363160713661141e-18,2.0423198522795646e-05,0.091476437069239705,0.90850313619958456,3.5326527915247367e-09],[1.0374455655996013e-12,0.024678741447627568,0.90550908050164514,0.069812178048139759,1.5502096453256213e-12],[0.99692466693672643,0.0030744353748935471,8.9768277448138998e-07,5.60587142922868e-12,2.8630374778268679e-26],[8.1296304899149488e-10,0.59348966575979367,0.40622266066067642,0.00028767276656680178,2.0637829872584376e-17],[1.0027332229450703e-17,1.4421619727592924e-05,0.051675250543293208,0.94831029853776927,2.9299209977336251e-08],[1.5557389058331249e-13,0.010698056674386343,0.8741636745443212,0.11513826877795148,3.1854857690226164e-12],[1.3585387367013157e-14,0.0027362942767955884,0.68812736713292733,0.30913633855506201,3.5201867308682676e-11],[0.99879205781494418,0.0012078254428356974,1.1674202870086065e-07,1.9149273718186094e-13,1.8398865733851877e-28],[0.9998939483169571,0.00010605106119324584,6.2184974131462508e-10,3.8442016066453819e-17,7.3618532946951114e-34],[1.198488233728185e-10,0.32307300275745138,0.67567401858189091,0.0012529785408085365,2.0551477655338724e-16],[2.1201751775985611e-09,0.45246774958014974,0.54380689812798855,0.0037253501716669572,1.9731236291630246e-14],[0.99879205781494418,0.0012078254428356974,1.1674202870086065e-07,1.9149273718186094e-13,1.8398865733851877e-28],[9.0383631598541225e-13,0.028650638692154933,0.93155577389941158,0.039793587407266039,2.6383689387640337e-13],[2.860779250061651e-07,0.95409012202760146,0.045893384062837758,1.6207831635780398e-05,3.8254571491436771e-18],[0.99852889088986796,0.0014709298920311791,1.7921771289954901e-07,3.880320709626317e-13,5.2518107373919096e-28],[2.4283498699271186e-08,0.96233579200188335,0.037662756832687418,1.4268819302266996e-06,4.5786913573616229e-21],[8.1658709580001005e-07,0.99476951512341494,0.0052295956324475571,7.2657041736653334e-08,1.9928232121687177e-22],[1.026625542303104e-13,0.011288792412656975,0.91952145236995908,0.06918975521675047,5.3056893718785748e-13],[8.087126927216999e-09,0.78976749748533204,0.21008650478426588,0.00014598964327489734,2.6893329017642595e-17],[0.99380495305614847,0.0061908468668574381,4.2000026988994506e-06,7.4295246369757851e-11,1.4523619526824063e-24],[0.99752890541309036,0.002470538722704905,5.558616787121519e-07,2.5260504363947324e-12,8.6293247731146442e-27],[1.549178583671275e-07,0.98891270585542701,0.011086957002231175,1.8222448339221404e-07,3.5914226851939147e-22],[4.6694971008114777e-08,0.80739364596297725,0.19218913570505244,0.00041717163699837178,9.8259712834564721e-16],[7.2038675786926584e-15,0.0015041165405104313,0.52388172889278195,0.4746141543749624,1.9173807069926722e-10],[0.99958883900968576,0.00041114963961323838,1.1350696925829962e-08,4.2259068870018977e-15,6.7245168093296037e-31],[0.99995923246624752,4.0767452476907367e-05,8.1275461320975662e-11,1.4682218780966981e-18,6.7880608718539824e-36],[5.2230703605946175e-17,7.0849411688667395e-05,0.15140108181858475,0.84852806640226253,2.367464326568955e-09],[1.2728329294313729e-13,0.0088144734576643229,0.83791959809567262,0.15326592843899509,7.540601939491104e-12],[6.0085356384749502e-08,0.98183179707602142,0.018167788360155413,3.5447846661736147e-07,6.1857554994392599e-22],[1.6765699451495372e-10,0.27263454618685362,0.72368198846999687,0.0036834651754888473,4.10616715147833e-15],[2.2784768850956345e-14,0.0029659573722948785,0.64337475708729186,0.35365928545935055,8.1039865734137274e-11],[2.0359223319955413e-10,0.40966485022801208,0.58965099560998135,0.00068415395841386689,6.2378605013648311e-17],[3.3949610281560467e-07,0.99379713890515331,0.0062024581970230308,6.3401720411789743e-08,8.3474956525197482e-23],[2.1572923701376001e-14,0.0026483008155225589,0.60173580385092362,0.39561589520085966,1.3267203651418155e-10],[2.5884000182263512e-14,0.0040734315661453118,0.76039735811289433,0.23552921030445081,1.648323453177087e-11],[4.3054433664232329e-10,0.47532915221984856,0.5240224540261017,0.00064839332350485944,9.0617669089337597e-17],[0.99724564747824895,0.0027536475530504957,7.0496495130915956e-07,3.7493008328477713e-12,1.5621617020241832e-26],[0.98823542038666456,0.011747089931604143,1.7488848549716132e-05,8.3318151490046054e-10,6.088395083170422e-23],[2.4759720298786534e-10,0.33587763328575881,0.66193409057857489,0.002188275888067367,1.3297313174060131e-15],[0.9998939483169571,0.00010605106119324584,6.2184974131462508e-10,3.8442016066453819e-17,7.3618532946951114e-34],[0.99997566385441783,2.4336118363615654e-05,2.7218526718692104e-11,2.5579588559548617e-19,5.6038021302450806e-37],[1.0188646786545323e-17,4.5164296051553503e-06,0.013748311579318275,0.98624106674069756,6.1052503786638629e-06],[8.9676177353798098e-14,0.0075684265572276066,0.82837699092548467,0.16405458250911309,8.0849978357522784e-12],[0.99866788409520824,0.0013319714933831518,1.4441113673842641e-07,2.7180751888586872e-13,3.0939665125451307e-28],[0.00011083904683035231,0.99933566288983811,0.00055349568452875701,2.3788029962286108e-09,4.099772980330266e-24],[1.9073158152351418e-12,0.042103840486360219,0.93184448015757937,0.026051679354046977,1.0609205310045999e-13],[1.9344317758252824e-12,0.039794199927977461,0.92869291751236172,0.031512882557539777,1.8633204043110495e-13],[2.860779250061651e-07,0.95409012202760146,0.045893384062837758,1.6207831635780398e-05,3.8254571491436771e-18],[4.2131520420053697e-20,7.2724991846797567e-07,0.017406600784282238,0.98259264669908153,2.526671716584245e-08],[2.7298218766814517e-08,0.890136467163958,0.10982020252251974,4.3303015303594799e-05,5.2709589268080545e-18],[0.9990027361944841,0.00099718679718236499,7.7008236811976245e-08,9.6629479652912163e-14,6.6831173263421593e-29],[1.5840817419679226e-08,0.89257689924367167,0.10739723060293609,2.5854312574927601e-05,1.1711857493428611e-18],[1.4221504199186712e-08,0.88142153502640708,0.11854585995689164,3.2590795197036898e-05,1.73719678106787e-18],[1.5301828825210957e-06,0.99864759673493575,0.0013508704137009772,2.6684805446793287e-09,5.633759137632094e-25],[3.6294440021603752e-07,0.98682298472761565,0.013176101384618694,5.5094336519440535e-07,5.7354705519390457e-21],[1.175294195474127e-07,0.88448133645387395,0.11537315242560379,0.000145393591102315,1.9266391864972379e-16],[5.2035552142821878e-08,0.97132379266452917,0.028674948012832963,1.2072870855580921e-06,6.246658408208247e-21],[0.99692466693672643,0.0030744353748935471,8.9768277448138998e-07,5.60587142922868e-12,2.8630374778268679e-26],[1.7743064920141416e-19,2.2345292910591483e-06,0.03373085908013948,0.96626689765951213,8.7310575026635239e-09],[0.00013359654430694098,0.99942114937266802,0.00044525260588142542,1.4771436015844995e-09,1.8561665062658044e-24],[4.854630158429325e-11,0.13674915941006305,0.84874124418623031,0.0145095963550702,9.0160126006036471e-14],[1.0854041890346201e-16,5.8171273804411357e-05,0.092414870613319947,0.9075269381293064,1.9983569004279049e-08],[6.3509581569276409e-06,0.99958918940482278,0.00040445938057269952,2.5644811984056459e-10,1.7327014611248188e-26],[0.99979799467564157,0.00020200285787296966,2.4664849283973907e-09,3.551010254747006e-16,1.8298635152234181e-32],[5.1831283124454827e-09,0.86798382209214608,0.13199716464148345,1.9008083241945128e-05,2.3851079504619521e-19],[5.1289835929603282e-08,0.91351331833127036,0.086453599677642212,3.3030701251288598e-05,4.8653733443412715e-18],[3.0578533756288528e-07,0.99170761383021322,0.0082919473078033653,1.3307664574699285e-07,3.243768327561525e-22],[0.99955295663443211,0.00044702977319561294,1.3592366591705287e-08,5.667050892322776e-15,1.0326556422715298e-30],[1.059887003863151e-05,0.99886938603029285,0.00112000992563852,5.1740297993217823e-09,6.1237710888929671e-24],[0.99970289916564692,0.00029709519111629373,5.6432354319208265e-09,1.3574058781327698e-15,1.2834644059926968e-31],[3.6513145363078053e-07,0.99428464190945354,0.0057149398351015917,5.3123990861803895e-08,6.2572697696238635e-23],[1.0911479560828444e-11,0.099509315354940425,0.89095748235626093,0.0095332022778736591,1.3428705181911057e-14],[0.99995114788470185,4.885199594313107e-05,1.1935504939994285e-10,2.7158408515691957e-18,1.6366820659768563e-35],[0.99204766711111814,0.0079450258441539478,7.3068555223034441e-06,1.8920565112185545e-10,6.1098254517152183e-24],[8.0271089149842587e-08,0.98784927599650207,0.012150504437232295,1.3929517646007344e-07,1.2464090647881461e-22],[2.8488113091884267e-07,0.97703120721133496,0.022966269023774917,2.2388837593122053e-06,7.5073448868359901e-20],[2.319898875011886e-08,0.85005956968365259,0.14983473857672025,0.00010566854063829781,3.0339660972408065e-17],[9.0080318352362713e-29,1.312976959336858e-14,1.0893358668405025e-08,0.00030939700096548707,0.99969059210566291],[7.1328620302274267e-07,0.97430037853234031,0.025693489460532581,5.4187209242250621e-06,7.6744043401468614e-19],[2.994517452700107e-18,1.456705900726794e-05,0.08474844914044731,0.91523698085383165,2.9467138343392612e-09],[3.3668057471665137e-09,0.8125362307973103,0.18742010119857186,4.3664637312084197e-05,9.6374454018965296e-19],[1.5686179692984036e-07,0.98246749597495753,0.017531664043034877,6.8312021079385062e-07,4.8021899880300654e-21],[0.99866788409520824,0.0013319714933831518,1.4441113673842641e-07,2.7180751888586872e-13,3.0939665125451307e-28],[0.99724564747824895,0.0027536475530504957,7.0496495130915956e-07,3.7493008328477713e-12,1.5621617020241832e-26],[0.99993734579459725,6.265400280612465e-05,2.025966249841724e-10,6.3416231431404396e-18,5.5185070555510118e-35],[7.4397220024380751e-13,0.029120182772715116,0.9398207883396662,0.031059028886758717,1.1645122040071567e-13],[2.2095328332103136e-07,0.98804912489517238,0.011950359111359464,2.9504018497071247e-07,1.2004045202174486e-21],[2.2155684458862262e-16,6.469717339712101e-05,0.078398770666415846,0.92153647191549748,6.0244689765686105e-08],[9.0535922529079423e-10,0.51885519071752184,0.48035289030310419,0.00079191807401434464,2.4615811537614346e-16],[9.5002259681103805e-18,3.4252789593993321e-05,0.13490750017327288,0.86505824569626932,1.340863870761508e-09],[2.1736036902116028e-09,0.59534461104672121,0.40394462918945073,0.00071075759022406126,3.5422033929241558e-16],[4.0996887300721774e-14,0.0023391963138827115,0.45516466673189365,0.54249613585222489,1.1019573284826282e-09],[4.1309421355955714e-11,0.15217088316550945,0.83934704322593301,0.0084820735672290967,1.8554584648635732e-14],[1.6158355147982995e-09,0.65715005214817857,0.3426027160624307,0.00024723017355498644,2.4609638584811569e-17],[0.99993734579459725,6.265400280612465e-05,2.025966249841724e-10,6.3416231431404396e-18,5.5185070555510118e-35],[0.99998569528575854,1.4304705395952176e-05,8.8455356797793246e-12,4.2694528894163896e-20,4.3894728925288124e-38],[1.5448401576852415e-08,0.92488173365053949,0.075110176776562884,8.0741244960119671e-06,1.0419062248855252e-19],[1.1838232822718691e-08,0.90308131434901251,0.096904252492653858,1.4421320101119662e-05,2.7410361260030282e-19],[3.9561421396646119e-10,0.34620795961787842,0.65078872252184317,0.0030033174646603858,3.9832717087274177e-15],[6.7249116945508561e-15,0.00069888695010241744,0.26644361017544943,0.73285749779553055,5.0789106353230871e-09],[5.245918135692784e-08,0.97894813282369253,0.021051326923092577,4.8779403329336369e-07,1.0391236528771711e-21],[6.3332007036953585e-06,0.99868270232826029,0.0013109585170575271,5.9539787443782553e-09,6.0606842699194182e-24],[0.99995672111784029,4.3278789884345778e-05,9.2275242815808065e-11,1.7988478996182611e-18,9.0761430206851534e-36],[1.4388961215941698e-06,0.9978430564418509,0.0021554954100403726,9.2519869305816992e-09,5.714585423654205e-24],[1.4871049809737254e-09,0.69644059944945413,0.30342600107404083,0.0001333979894004449,5.6847010598202998e-18],[3.3194301249785363e-09,0.80204989884188427,0.19789744907327961,5.2648765405846994e-05,1.4262526733205183e-18],[1.598033167666836e-18,2.3385513682365341e-06,0.014151740982018376,0.98584515224403768,7.6822257590682241e-07],[1.655125313229797e-07,0.99094641712689346,0.0090533099874629853,1.0737311196072699e-07,1.3467899991592639e-22],[0.99204766711111814,0.0079450258441539478,7.3068555223034441e-06,1.8920565112185545e-10,6.1098254517152183e-24]],"max_posterior":[0.79576719420988151,0.78163030815802259,0.98823542038666456,0.99992403122843865,0.73058763298335594,0.99990594612907568,0.97561654827135313,0.99987824452224361,0.77688407097759771,0.99951344461950564,0.99995403107488867,0.90850313619958456,0.90550908050164514,0.99692466693672643,0.59348966575979367,0.94831029853776927,0.8741636745443212,0.68812736713292733,0.99879205781494418,0.9998939483169571,0.67567401858189091,0.54380689812798855,0.99879205781494418,0.93155577389941158,0.95409012202760146,0.99852889088986796,0.96233579200188335,0.99476951512341494,0.91952145236995908,0.78976749748533204,0.99380495305614847,0.99752890541309036,0.98891270585542701,0.80739364596297725,0.52388172889278195,0.99958883900968576,0.99995923246624752,0.84852806640226253,0.83791959809567262,0.98183179707602142,0.72368198846999687,0.64337475708729186,0.58965099560998135,0.99379713890515331,0.60173580385092362,0.76039735811289433,0.5240224540261017,0.99724564747824895,0.98823542038666456,0.66193409057857489,0.9998939483169571,0.99997566385441783,0.98624106674069756,0.82837699092548467,0.99866788409520824,0.99933566288983811,0.93184448015757937,0.92869291751236172,0.95409012202760146,0.98259264669908153,0.890136467163958,0.9990027361944841,0.89257689924367167,0.88142153502640708,0.99864759673493575,0.98682298472761565,0.88448133645387395,0.97132379266452917,0.99692466693672643,0.96626689765951213,0.99942114937266802,0.84874124418623031,0.9075269381293064,0.99958918940482278,0.99979799467564157,0.86798382209214608,0.91351331833127036,0.99170761383021322,0.99955295663443211,0.99886938603029285,0.99970289916564692,0.99428464190945354,0.89095748235626093,0.99995114788470185,0.99204766711111814,0.98784927599650207,0.97703120721133496,0.85005956968365259,0.99969059210566291,0.97430037853234031,0.91523698085383165,0.8125362307973103,0.98246749597495753,0.99866788409520824,0.99724564747824895,0.99993734579459725,0.9398207883396662,0.98804912489517238,0.92153647191549748,0.51885519071752184,0.86505824569626932,0.59534461104672121,0.54249613585222489,0.83934704322593301,0.65715005214817857,0.99993734579459725,0.99998569528575854,0.92488173365053949,0.90308131434901251,0.65078872252184317,0.73285749779553055,0.97894813282369253,0.99868270232826029,0.99995672111784029,0.9978430564418509,0.69644059944945413,0.80204989884188427,0.98584515224403768,0.99094641712689346,0.99204766711111814],"proportion_misgenotyped":0.10256821175211306},{"name":"tetraploid_no_bias","model":"hw","ploidy":4,"simulated_bias":1,"allele_depth":[31,12,0,0,31,70,20,0,31,0,0,61,33,0,31,42,42,46,0,0,39,13,0,42,7,0,24,10,59,16,0,0,16,8,40,0,0,51,39,21,25,38,40,14,36,46,31,0,0,26,0,0,26,41,0,3,41,38,7,75,13,0,17,17,13,10,7,18,0,78,3,23,35,11,0,28,11,13,0,6,0,14,38,0,0,22,9,12,57,6,69,28,13,0,0,0,49,13,29,23,70,18,26,28,25,0,0,21,21,21,27,21,7,0,11,30,27,35,17,0],"marker_depth":[53,26,25,80,98,70,92,73,101,55,88,83,62,36,91,54,75,75,45,75,101,35,45,82,35,43,102,66,109,54,30,38,85,29,62,57,90,71,68,100,62,61,108,83,57,77,86,37,25,67,75,99,29,71,44,50,83,76,35,95,50,47,64,63,98,57,29,82,36,102,52,51,46,104,66,99,45,75,56,61,61,84,84,87,28,110,48,44,57,35,94,94,67,44,37,83,97,71,37,65,98,53,39,64,76,83,109,82,79,54,38,98,65,89,81,93,90,41,92,28],"bias":1.0543050125840154,"log_likelihood":-448.93758781758339,"prior":[0.24156942221490793,0.41201502952230551,0.26352111796031485,0.074909219470375474,0.0079852108320962924],"genotypes":[2,2,0,0,1,4,1,0,1,0,0,3,2,0,1,3,2,2,0,0,2,1,0,2,1,0,1,1,2,1,0,0,1,1,2,0,0,3,2,1,2,2,1,1,2,2,1,0,0,2,0,0,3,2,0,1,2,2,1,3,1,0,1,1,1,1,1,1,0,3,1,2,3,1,0,1,1,1,0,1,0,1,2,0,0,1,1,1,4,1,3,1,1,0,0,0,2,1,3,1,3,1,2,2,1,0,0,1,1,2,3,1,1,0,1,1,1,3,1,0],"posteriors":[[1.9802265987769511e-13,0.01433593980445457,0.86911836583984081,0.11654569434798895,7.5180147769721267e-12],[3.2115708610661153e-10,0.2326442286047754,0.74780846005828217,0.019547311014765967,1.0193134626662027e-12],[0.98727369599669146,0.012711991786960249,1.4311759686660571e-05,4.5666186966468896e-10,2.4533268761674621e-23],[0.99993472808923622,6.5271736369975513e-05,1.7439379949989617e-10,4.8639482197895539e-18,5.7284802428952016e-35],[1.323542375445636e-09,0.80548169930919966,0.19448039120073468,3.790816652334732e-05,8.3744073369196932e-19],[1.3953813484224265e-30,2.1226319038302045e-15,5.8809896696945788e-09,0.00031326333048558371,0.99968673078852222],[3.967440393456665e-08,0.98422963062356694,0.015770054187956302,2.7551407319629365e-07,6.5775905278129814e-22],[0.99989313568133875,0.00010686382624491608,4.9241631891349484e-10,2.5216075598134239e-17,5.888127113311635e-34],[1.7406176612179699e-09,0.84236463836053721,0.15761229127969387,2.3068619151002854e-05,3.678920497630786e-19],[0.99954522384193012,0.00045476557414868876,1.0583918033582134e-08,3.3813850322608781e-15,6.4369939472115953e-31],[0.99996138770623799,3.8612235846004194e-05,5.7916192998069171e-11,8.5205402398796744e-19,4.9088230842443515e-36],[6.9124794238730768e-18,4.7713069386860205e-05,0.16397140232914459,0.83598088310779961,1.4936687940148477e-09],[8.5738045162377683e-13,0.035015360107192424,0.92595611597634497,0.039028523915116159,4.8870445859928834e-13],[0.99687187006627509,0.0031274654351068957,6.644958217991951e-07,2.7960881949530789e-12,1.1547269530006103e-26],[6.4229110155879571e-10,0.68765725000984268,0.31221849128961054,0.00012425805825557818,6.2197432413465265e-18],[1.0996371587232116e-17,3.2586421977296343e-05,0.09246584615254308,0.90750155516893782,1.2256541714056109e-08],[1.2744467535503068e-13,0.015646788247767045,0.9195281199375116,0.064825091813598176,9.9542408088863329e-13],[1.1587638952352213e-14,0.0043647172248738268,0.8043806628014345,0.19125461996222692,1.1453445710365055e-11],[0.9988230021236123,0.0011769171134814935,8.0762816300765995e-08,8.9831652810505708e-14,7.4212948071346245e-29],[0.99990749812319402,9.2501513540648781e-05,3.6326548951694218e-10,1.5559402708262182e-17,2.9693954833773975e-34],[1.0025174533531995e-10,0.41702754490228983,0.58237800991530964,0.00059444508214894615,6.5576537403509694e-17],[1.720366034941706e-09,0.53285811248799508,0.4652533534163048,0.0018885323753278812,6.1073314863068486e-15],[0.9988230021236123,0.0011769171134814935,8.0762816300765995e-08,8.9831652810505708e-14,7.4212948071346245e-29],[7.497502942425202e-13,0.040940688716525828,0.93752311027381641,0.021536201008824779,8.348525757948284e-14],[2.316991501903183e-07,0.96686273857498772,0.033129775855618278,7.2538702435984108e-06,1.1818754580918334e-18],[0.99855354957059161,0.0014463246813824747,1.2574784168253849e-07,1.8435268069516977e-13,2.1183366905790984e-28],[1.8956196776167313e-08,0.97563971518580705,0.024359727815091187,5.3804290517494721e-07,1.3634168005255679e-21],[7.0312924128032754e-07,0.99655515868346489,0.003444108646873484,2.9540420412369097e-08,6.5455963491517446e-23],[8.3549328404107279e-14,0.016565092891228803,0.94614354062942718,0.037291366479096012,1.647100035913141e-13],[6.2682555258445045e-09,0.84544728319397344,0.15448965473019011,6.3055807580926704e-05,7.9514206935187974e-18],[0.99349209415478412,0.0065046312137165309,3.2745926123901055e-06,3.8887013297100001e-11,5.8561672828589112e-25],[0.99751167717842515,0.0024879178821677816,4.0493816528709333e-07,1.2416252020125509e-12,3.4805234270615771e-27],[1.2755869727997641e-07,0.99285200769990845,0.0071477941754198174,7.0565974403312445e-08,1.1280351293483568e-22],[3.6828374745205178e-08,0.85145200443786939,0.1483504130032651,0.00019754573049043673,2.9562168165218225e-16],[6.5533430763320054e-15,0.0025782328112153741,0.67173819788592426,0.32568356923631825,6.6535470752597077e-11],[0.99961863191833267,0.0003813608040430437,7.277622451962885e-09,1.8535692785139517e-15,2.7123711631266653e-31],[0.99996594118408832,3.4058771390632392e-05,4.4521027304628011e-11,5.6272181174501547e-19,2.7379378586140647e-36],[5.6333176225087862e-17,0.00015561838875336831,0.25455108287527151,0.74529329776195163,9.7402365658461745e-10],[1.0490509051160938e-13,0.013013928157861971,0.89857552206436608,0.088410549775296124,2.3707116965048459e-12],[4.8187917681518068e-08,0.98838813656873936,0.011611681447517804,1.3379582550953557e-07,1.8923855921789272e-22],[1.4049158219235686e-10,0.35084043632806627,0.64730114936365768,0.0018584141677835302,1.3125408589469976e-15],[1.9824722233628351e-14,0.0047667571247332454,0.76809289742530484,0.22714034542304437,2.6897312153438509e-11],[1.6726703178311456e-10,0.51209366375747956,0.48759437892898055,0.00031195714627294907,1.9549336686258912e-17],[2.8695797176426551e-07,0.99601183062300824,0.0039878576615679281,2.4757452030639898e-08,2.6914559230638543e-23],[1.9076813266339067e-14,0.0043245389269429653,0.73458148177481342,0.26109397925347161,4.4753186520049948e-11],[2.159885149853541e-14,0.006284122322730212,0.8536227686612442,0.14009310901075719,5.2467358829591407e-12],[3.4851406909801797e-10,0.5750064140105009,0.42469933508000507,0.0002942505609796285,2.7980964549168383e-17],[0.99721254982294727,0.0027869325318648527,5.1764333172509544e-07,1.8562796353014016e-12,6.3006714132429618e-27],[0.98727369599669146,0.012711991786960249,1.4311759686660571e-05,4.5666186966468896e-10,2.4533268761674621e-23],[2.0560928704491623e-10,0.42397820236782541,0.57495061626935184,0.001071181157213514,4.2121944854050489e-16],[0.99990749812319402,9.2501513540648781e-05,3.6326548951694218e-10,1.5559402708262182e-17,2.9693954833773975e-34],[0.99998013441967726,1.986556591594116e-05,1.4406786931267746e-11,9.5253024474144998e-20,2.2602665186304439e-37],[1.037939610542148e-17,9.2075090339898406e-06,0.023493978078835163,0.97649444190102386,2.3725111063407232e-06],[7.3919655288562562e-14,0.011255762407935511,0.893929271102474,0.094814966486974031,2.5422033882430935e-12],[0.99869616807307449,0.0013037313201101487,1.0060668692425574e-07,1.2831114533169165e-13,1.2479669854309881e-28],[0.00010836678043345705,0.99951268943030569,0.00037894272566617727,1.0635942833377245e-09,1.5290151878965671e-24],[1.5980140264794427e-12,0.059818105173695769,0.92620449661100246,0.013977398213669535,3.3906910880442581e-14],[1.6158127474746365e-12,0.056346779479719451,0.92660257653921119,0.017050643979394382,5.9370759492404957e-14],[2.316991501903183e-07,0.96686273857498772,0.033129775855618278,7.2538702435984108e-06,1.1818754580918334e-18],[4.6330798340090703e-20,1.8087995939008135e-06,0.034030053423614696,0.96596812717793246,1.0598858994997926e-08],[2.1257838094040571e-08,0.92173164854484102,0.078249909681062366,1.8420516258385518e-05,1.5657493954492114e-18],[0.99903678461918854,0.00096316282921332544,5.2551553269691743e-08,4.4780264090586926e-14,2.6956844231596984e-29],[1.2235701706153164e-08,0.92524735834303218,0.074742047446268944,1.0581974997142949e-05,3.450838981099037e-19],[1.0976612254214605e-08,0.91698734221899969,0.082999236886748942,1.3409917639333213e-05,5.1146914203306058e-19],[1.3758596090656039e-06,0.99915251236250135,0.00084611074572727048,1.0321622659200951e-09,1.9323090847857495e-25],[3.0203260778230125e-07,0.99112539412730805,0.0088740762790675623,2.27561016742294e-07,1.8206675480130255e-21],[9.3333597613511484e-08,0.91299400238502249,0.086938123865448552,6.7780415931265956e-05,5.8363348857941479e-17],[4.1316745760377625e-08,0.98120883852889784,0.018790651389357364,4.6876499934115591e-07,1.8920002438037844e-21],[0.99687187006627509,0.0031274654351068957,6.644958217991951e-07,2.7960881949530789e-12,1.1547269530006103e-26],[1.9874358519108971e-19,5.5698633140423876e-06,0.065106854012881979,0.93488757239320208,3.7306015946255991e-09],[0.00013173328714276359,0.99956486734719008,0.0003033987073628189,6.5830465816227255e-10,6.9817745747509629e-25],[4.0972960636676689e-11,0.18258581557152406,0.80958934062552645,0.0078248437619474129,2.9027086061815542e-14],[1.1790914423260061e-16,0.00012535037745343725,0.1573554098658152,0.84251923147583796,8.2808931557451992e-09],[6.0256537837992013e-06,0.99974332728020132,0.00025064696610563477,9.9909318613863852e-11,6.2709921359225367e-27],[0.99981858827966474,0.00018141021474337614,1.5055916238239902e-09,1.4925122150627242e-16,7.3807872026766659e-33],[3.9464617349247038e-09,0.91027482706161378,0.089717780031532204,7.3889603922295241e-06,6.9274161755062496e-20],[4.0292882451638806e-08,0.93830666879254621,0.061679068970140312,1.4221944430956967e-05,1.4580121713493219e-18],[2.5633383074687521e-07,0.99459988048694759,0.0053998104959191674,5.2683302549003352e-08,1.0372578686040518e-22],[0.99958376558473783,0.00041622564937746081,8.7658822060739684e-09,2.4983992709225198e-15,4.1652781240531046e-31],[9.8658931457131423e-06,0.99924708088150316,0.0007430510383756476,2.186975277440591e-09,2.174422216397773e-24],[0.99972847210587301,0.00027152435659356965,3.5375327131686036e-09,5.8380274646934133e-16,5.1769027379293445e-32],[3.0958579541216078e-07,0.99633198615972052,0.0036676835338669226,2.0720617132581161e-08,2.0237843065517497e-23],[9.3100292121180251e-12,0.13864216715651173,0.85636032622060654,0.0049975066135672765,4.3706790318368709e-15],[0.99995885342533364,4.1146508490507298e-05,6.6175839663259542e-11,1.0515673280089568e-18,6.6014999632427455e-36],[0.99154979064653848,0.0084444045080455978,5.8047446136787483e-06,1.0080223989407367e-10,2.4631234267789251e-24],[6.5201786640417016e-08,0.99233904984524057,0.0076608330035530438,5.1949419294047371e-08,3.8619643057770497e-23],[2.3349750644580633e-07,0.98414658920942821,0.015852228061450709,9.4923161432304785e-07,2.3472152809693108e-20],[1.8078133781651894e-08,0.89027062293820425,0.10968303893689699,4.6320046764970169e-05,9.0186846475927294e-18],[2.2318385798615421e-28,8.2683417962817979e-14,6.2084286169630676e-08,0.00097983179482189929,0.99902010612080894],[5.9120555379144014e-07,0.98157770636540131,0.018419271448116142,2.430980928827937e-06,2.4264296877978714e-19],[3.3378269246547752e-18,3.4706255709800715e-05,0.15428661176682115,0.84567868072455177,1.2529174710344633e-09],[2.5712797345536174e-09,0.86897509605444401,0.13100750652887605,1.7394845400347783e-05,2.8076342061051278e-19],[1.280061979174691e-07,0.9883653499557733,0.011634248050299854,2.7398772903431055e-07,1.4948610883239089e-21],[0.99869616807307449,0.0013037313201101487,1.0060668692425574e-07,1.2831114533169165e-13,1.2479669854309881e-28],[0.99721254982294727,0.0027869325318648527,5.1764333172509544e-07,1.8562796353014016e-12,6.3006714132429618e-27],[0.99994663358502578,5.3366300754785636e-05,1.142194855786715e-10,2.4903762661380406e-18,2.2258741447367857e-35],[6.1884377389593873e-13,0.041904302160277296,0.94151968311309686,0.016576014725969965,3.695010421820707e-14],[1.8275940822474709e-07,0.9921481502865066,0.007851549451757452,1.1750232758236686e-07,3.7875167936956588e-22],[2.3900353105964569e-16,0.00013588387494392908,0.13207062829664196,0.86779346303785498,2.4790558754911938e-08],[7.2681104322592092e-10,0.61211362848635231,0.38752194160011488,0.00036442918672139463,7.5381070358465972e-17],[1.0392232647153048e-17,7.8935804254213359e-05,0.23505066802447905,0.76487039561175774,5.5950902414494047e-10],[1.7245302511785429e-09,0.67959681910792835,0.32007637217356594,0.00032680699397509371,1.0720416797766986e-16],[3.8555226359007736e-14,0.0039844199572401113,0.59758470203271485,0.39843087761468959,3.9531679222775875e-10],[3.5058109256153104e-11,0.2048609300695268,0.79068276831825623,0.0044563015771529038,6.0067308487683739e-15],[1.2641803883820049e-09,0.74085315747580749,0.25904033311845936,0.00010650814155285148,7.3445397855380019e-18],[0.99994663358502578,5.3366300754785636e-05,1.142194855786715e-10,2.4903762661380406e-18,2.2258741447367857e-35],[0.99998859789996053,1.1402095518912106e-05,4.5205181625113777e-12,1.543956981272394e-20,1.7704698048167479e-38],[1.1914146940441785e-08,0.9495529407315626,0.050443884825916732,3.1625283734762872e-06,3.0651764682190865e-20],[9.0968472196646536e-09,0.93405080378049132,0.065943476045932217,5.7110767293467609e-06,8.0346367895771329e-20],[3.2710724427210893e-10,0.43096795374550773,0.56753478030902005,0.0014972656183640516,1.2563362669034714e-15],[6.8288051822962107e-15,0.0013200100463357479,0.39358705043258152,0.60509293755374727,1.9673286368406451e-09],[4.1863092178452271e-08,0.9864912077586141,0.013508565772332564,1.8460596118198373e-07,3.1631835825812487e-22],[5.8275609673552019e-06,0.99913185472733956,0.00086231523303740849,2.4786556761224556e-09,2.1273219142353186e-24],[0.99996374574801361,3.625420123782234e-05,5.0748715542521411e-11,6.9176416652333053e-19,3.6608280466234676e-36],[1.2776786190013956e-06,0.99861721635730738,0.0013815022920130517,3.6720601742356494e-09,1.9356400140265769e-24],[1.1527882768826477e-09,0.77694494542437231,0.22299966315255018,5.5390270289419268e-05,1.6809812800971194e-18],[2.539603649389571e-09,0.86061636093760518,0.1393624788309383,2.1157691852737254e-05,4.1624232288297226e-19],[1.6678051786352699e-18,5.0831549677226492e-06,0.025299571094938987,0.97469503990948547,3.0584060767645657e-07],[1.3701949713337751e-07,0.99421914465237793,0.0057806771896540448,4.1138470854142629e-08,4.2530337831158985e-23],[0.99154979064653848,0.0084444045080455978,5.8047446136787483e-06,1.0080223989407367e-10,2.4631234267789251e-24]],"max_posterior":[0.86911836583984081,0.74780846005828217,0.98727369599669146,0.99993472808923622,0.80548169930919966,0.99968673078852222,0.98422963062356694,0.99989313568133875,0.84236463836053721,0.99954522384193012,0.99996138770623799,0.83598088310779961,0.92595611597634497,0.99687187006627509,0.68765725000984268,0.90750155516893782,0.9195281199375116,0.8043806628014345,0.9988230021236123,0.99990749812319402,0.58237800991530964,0.53285811248799508,0.9988230021236123,0.93752311027381641,0.96686273857498772,0.99855354957059161,0.97563971518580705,0.99655515868346489,0.94614354062942718,0.84544728319397344,0.99349209415478412,0.99751167717842515,0.99285200769990845,0.85145200443786939,0.67173819788592426,0.99961863191833267,0.99996594118408832,0.74529329776195163,0.89857552206436608,0.98838813656873936,0.64730114936365768,0.76809289742530484,0.51209366375747956,0.99601183062300824,0.73458148177481342,0.8536227686612442,0.5750064140105009,0.99721254982294727,0.98727369599669146,0.57495061626935184,0.99990749812319402,0.99998013441967726,0.97649444190102386,0.893929271102474,0.99869616807307449,0.99951268943030569,0.92620449661100246,0.92660257653921119,0.96686273857498772,0.96596812717793246,0.92173164854484102,0.99903678461918854,0.92524735834303218,0.91698734221899969,0.99915251236250135,0.99112539412730805,0.91299400238502249,0.98120883852889784,0.99687187006627509,0.93488757239320208,0.99956486734719008,0.80958934062552645,0.84251923147583796,0.99974332728020132,0.99981858827966474,0.91027482706161378,0.93830666879254621,0.99459988048694759,0.99958376558473783,0.99924708088150316,0.99972847210587301,0.99633198615972052,0.85636032622060654,0.99995885342533364,0.99154979064653848,0.99233904984524057,0.98414658920942821,0.89027062293820425,0.99902010612080894,0.98157770636540131,0.84567868072455177,0.86897509605444401,0.9883653499557733,0.99869616807307449,0.99721254982294727,0.99994663358502578,0.94151968311309686,0.9921481502865066,0.86779346303785498,0.61211362848635231,0.76487039561175774,0.67959681910792835,0.59758470203271485,0.79068276831825623,0.74085315747580749,0.99994663358502578,0.99998859789996053,0.9495529407315626,0.93405080378049132,0.56753478030902005,0.60509293755374727,0.9864912077586141,0.99913185472733956,0.99996374574801361,0.99861721635730738,0.77694494542437231,0.86061636093760518,0.97469503990948547,0.99421914465237793,0.99154979064653848],"proportion_misgenotyped":0.094832589625738817},{"name":"diploid_bias_0.5","model":"norm","ploidy":2,"simulated_bias":0.5,"allele_depth":[78,0,0,0,0,62,52,69,86,0,69,35,0,74,85,0,79,34,78,0,0,0,70,0,105,105,44,109,68,41,20,110,24,70,0,0,0,41,41,63,0,0,53,73,21,43,68,86,0,42,0,34,68,0,72,36,36,22,48,0,57,35,0,42,49,0,43,33,0,45,77,50,40,106,104,35,58,68,0,24,32,29,61,60,0,0,0,61,0,0,22,0,0,66,65,108,28,36,32,31],"marker_depth":[78,101,41,93,70,80,76,109,86,98,108,54,26,96,85,55,79,47,85,68,73,60,99,42,105,105,44,109,89,72,36,110,43,93,64,61,101,81,56,83,107,33,85,109,32,54,96,86,34,67,46,75,93,93,87,49,62,36,81,43,88,53,60,67,69,53,71,55,31,65,110,86,40,106,104,55,58,84,105,40,50,38,88,84,36,47,99,109,53,50,37,94,25,102,65,108,61,36,32,43],"bias":0.50147057543220319,"log_likelihood":-302.58363230824909,"prior":[0.31000000124451971,0.51001086901828119,0.17998912973719908],"genotypes":[2,0,0,0,0,1,1,1,2,0,1,1,0,1,2,0,2,1,1,0,0,0,1,0,2,2,2,2,1,1,1,2,1,1,0,0,0,1,1,1,0,0,1,1,1,1,1,2,0,1,0,1,1,0,1,1,1,1,1,0,1,1,0,1,1,0,1,1,0,1,1,1,2,2,2,1,2,1,0,1,1,1,1,1,0,0,0,1,0,0,1,0,0,1,2,2,1,2,2,1],"posteriors":[[1.0787149680957383e-32,1.5651591187209286e-06,0.99999843484088136],[1,1.0330443775708367e-16,1.1147394678200765e-36],[0.99999999989483102,1.0516902941852894e-10,5.2611535318366718e-27],[0.99999999999999956,4.0892129277297956e-16,9.7294133613318831e-36],[0.99999999999996003,4.0060908045206137e-14,1.416304920379804e-32],[3.8036363107681632e-19,0.99999997965985465,2.0340145371430031e-08],[1.6317093142731335e-16,0.99999999946163998,5.3835982121412507e-10],[6.8119945718104135e-16,0.99999999995494271,4.5056396938130559e-11],[8.7767123973617248e-34,7.3665816371028238e-07,0.9999992633418362],[0.99999999999999978,1.7115292813449007e-16,2.4663478993049718e-36],[4.8394414502708709e-16,0.99999999994336797,5.6631927788829989e-11],[9.4426483135829179e-15,0.99999999969444353,3.0554709097159364e-10],[0.99999996652147594,3.3478524216810866e-08,9.4361896037761006e-23],[1.3333149121918038e-19,0.99999998579782379,1.4202176251435545e-08],[1.1874364850859014e-33,8.0696646690546591e-07,0.99999919303353313],[0.99999999999840661,1.5934442838738744e-12,5.3799921070177127e-30],[7.7907274057928828e-33,1.4199535018054868e-06,0.99999858004649811],[4.6238886406822354e-16,0.99999999474573364,5.2542658028778811e-09],[4.9743996088302011e-24,0.99988127069693911,0.00011872930306104605],[0.99999999999993694,6.3043906031506719e-14,2.9285949722820856e-32],[0.99999999999997935,2.0677781772250807e-14,4.919547423333912e-33],[0.9999999999995679,4.3207010579357231e-13,6.4890328773851518e-31],[8.4804757070506371e-18,0.9999999991242654,8.7573475680233124e-10],[0.9999999999244269,7.5573029884139805e-11,3.0381345875277243e-27],[4.7095146927848663e-36,1.5010119271299481e-07,0.9999998498988073],[4.7095146927848663e-36,1.5010119271299481e-07,0.9999998498988073],[1.2315093906657477e-26,8.9251765833816518e-05,0.99991074823416637],[1.745049327713505e-36,1.1064638440170393e-07,0.99999988935361561],[3.6265212382539779e-19,0.99999998883098773,1.1169012022776031e-08],[1.8045877932057991e-13,0.9999999999874909,1.2328983865664856e-11],[8.3051726321397409e-12,0.99999999992541833,6.6276598373474002e-11],[1.368420886477343e-36,1.0267063203632541e-07,0.99999989732936789],[3.1453345388595255e-12,0.99999999995769517,3.9159750727862893e-11],[5.8034911061901341e-19,0.99999999356147651,6.4385237944397508e-09],[0.9999999999998388,1.6124579552680376e-13,1.3238542753118983e-31],[0.99999999999966382,3.3614879007609846e-13,4.3260258785010488e-31],[1,1.0330443775708367e-16,1.1147394678200765e-36],[4.9583829488316803e-12,0.99999999999377476,1.2667399936060353e-12],[7.6563912454086808e-17,0.99999999461873568,5.3812643743079243e-09],[8.3207839940806953e-19,0.99999999039924159,9.6007581709269326e-09],[1,3.9045924481903331e-17,2.4198467085954095e-37],[0.99999999815698071,1.8430193033689314e-09,6.4582038413133422e-25],[3.6178009808483579e-15,0.99999999994957278,5.0423954414463156e-11],[6.1625798644541137e-17,0.99999999982303667,1.7696355405090221e-10],[1.8144525527587504e-13,0.9999999985270982,1.472720690049459e-09],[2.740049269238393e-18,0.99999992246830993,7.753169009772858e-08],[9.2751665645206715e-18,0.99999999904050885,9.5949119803139072e-10],[8.7767123973617248e-34,7.3665816371028238e-07,0.9999992633418362],[0.99999999874149204,1.2585080524638799e-09,3.3828734352844806e-25],[9.3317291818869766e-15,0.99999999990690513,9.3085801006103686e-11],[0.99999999997876299,2.1236932298710893e-11,3.7242911153771972e-28],[1.4339621492703066e-10,0.99999999985627852,3.2557006970860351e-13],[2.4670554341719775e-18,0.99999999744703727,2.5529628216031207e-09],[0.99999999999999956,4.0892129277297956e-16,9.7294133613318831e-36],[4.9519305492503279e-21,0.99999973726418101,2.627358188213929e-07],[1.8540082218528876e-16,0.99999999263519923,7.3648005298167486e-09],[1.8061903782934177e-13,0.99999999997410782,2.5711692575572415e-11],[6.4618759584927748e-13,0.99999999970416376,2.9519030083546579e-10],[2.8697205452224414e-14,0.9999999999803082,1.9663645962176525e-11],[0.99999999994539879,5.4601127505678213e-11,1.7722472834419088e-27],[6.9719827445730362e-16,0.9999999998894471,1.105524139144355e-10],[5.5120718647044865e-15,0.99999999952436713,4.7562758832376042e-10],[0.9999999999995679,4.3207010579357231e-13,6.4890328773851518e-31],[9.3317291818869766e-15,0.99999999990690513,9.3085801006103686e-11],[6.2683978343309528e-17,0.99999999833068332,1.6693166141639058e-09],[0.99999999999724642,2.7536275481630464e-12,1.3105950427599535e-29],[2.377389765411537e-14,0.99999999995933608,4.0640392578874675e-11],[1.1128935130352534e-13,0.99999999993588862,6.4000182427044115e-11],[0.99999999595603417,4.0439657513770476e-09,2.4621830661596123e-24],[2.620438391413052e-16,0.99999999904806325,9.5193644163010497e-10],[7.7487030586935189e-18,0.9999999994250155,5.7498447825873488e-10],[4.4247357289789578e-14,0.999999999988054,1.1902103736889139e-11],[1.0860575843701726e-25,0.0001632812440551067,0.99983671875594482],[3.6629584081428576e-36,1.3896082323122738e-07,0.9999998610391766],[6.068024301095766e-36,1.6223173542423199e-07,0.99999983776826451],[1.5944030461501786e-14,0.99999999979993293,2.0005105627599242e-10],[1.7521607386269786e-29,1.3808064961988639e-05,0.9999861919350379],[2.4477483818845469e-20,0.99999989697923386,1.0302076631742821e-07],[1,5.3721097100126007e-17,3.9917640312163761e-37],[5.9705244559795124e-13,0.99999999984184562,1.575572019563603e-10],[2.2822378552903677e-14,0.99999999971713383,2.8284360760475976e-10],[3.6607782041096786e-16,0.99999996823867765,3.1761322147533319e-08],[4.0409677168758017e-17,0.99999999940257167,5.9742849355326317e-10],[1.3898025787121069e-17,0.99999999853957044,1.4604294643364968e-09],[0.99999999940047002,5.9953007331092161e-10,9.6769960822959327e-26],[0.9999999999843463,1.5653751897074676e-11,2.2541787130390758e-28],[0.99999999999999978,1.4443654299209559e-16,1.8882987828242329e-36],[7.5067291716889027e-14,0.99999999999643929,3.4854956306087356e-12],[0.99999999999724642,2.7536275481630464e-12,1.3105950427599535e-29],[0.99999999999355826,6.4417212218083665e-12,5.2518523553495984e-29],[1.1837095211251172e-12,0.99999999982676213,1.7205409903468982e-10],[0.99999999999999956,3.424953153398326e-16,7.356390480187016e-36],[0.99999994742037601,5.2579623989749516e-08,2.0759581658966632e-22],[3.6592062820730578e-16,0.99999999991617905,8.3820666150661434e-11],[1.0634054294412125e-30,6.0997729257190119e-06,0.99999390022707424],[2.2297837545471824e-36,1.1930887756831149e-07,0.99999988069112233],[1.6921242517354329e-10,0.99999999983001864,7.6875003810553433e-13],[1.1413268708332887e-24,0.00031052597349112736,0.99968947402650887],[1.475325972099421e-23,0.00061714987397681312,0.99938285012602301],[1.0396764835048881e-15,0.99999999431885156,5.6811475192875519e-09]],"max_posterior":[0.99999843484088136,1,0.99999999989483102,0.99999999999999956,0.99999999999996003,0.99999997965985465,0.99999999946163998,0.99999999995494271,0.9999992633418362,0.99999999999999978,0.99999999994336797,0.99999999969444353,0.99999996652147594,0.99999998579782379,0.99999919303353313,0.99999999999840661,0.99999858004649811,0.99999999474573364,0.99988127069693911,0.99999999999993694,0.99999999999997935,0.9999999999995679,0.9999999991242654,0.9999999999244269,0.9999998498988073,0.9999998498988073,0.99991074823416637,0.99999988935361561,0.99999998883098773,0.9999999999874909,0.99999999992541833,0.99999989732936789,0.99999999995769517,0.99999999356147651,0.9999999999998388,0.99999999999966382,1,0.99999999999377476,0.99999999461873568,0.99999999039924159,1,0.99999999815698071,0.99999999994957278,0.99999999982303667,0.9999999985270982,0.99999992246830993,0.99999999904050885,0.9999992633418362,0.99999999874149204,0.99999999990690513,0.99999999997876299,0.99999999985627852,0.99999999744703727,0.99999999999999956,0.99999973726418101,0.99999999263519923,0.99999999997410782,0.99999999970416376,0.9999999999803082,0.99999999994539879,0.9999999998894471,0.99999999952436713,0.9999999999995679,0.99999999990690513,0.99999999833068332,0.99999999999724642,0.99999999995933608,0.99999999993588862,0.99999999595603417,0.99999999904806325,0.9999999994250155,0.999999999988054,0.99983671875594482,0.9999998610391766,0.99999983776826451,0.99999999979993293,0.9999861919350379,0.99999989697923386,1,0.99999999984184562,0.99999999971713383,0.99999996823867765,0.99999999940257167,0.99999999853957044,0.99999999940047002,0.9999999999843463,0.99999999999999978,0.99999999999643929,0.99999999999724642,0.99999999999355826,0.99999999982676213,0.99999999999999956,0.99999994742037601,0.99999999991617905,0.99999390022707424,0.99999988069112233,0.99999999983001864,0.99968947402650887,0.99938285012602301,0.99999999431885156],"proportion_misgenotyped":1.3257148564149723e-05},{"name":"diploid_bias_0.5","model":"hw","ploidy":2,"simulated_bias":0.5,"allele_depth":[78,0,0,0,0,62,52,69,86,0,69,35,0,74,85,0,79,34,78,0,0,0,70,0,105,105,44,109,68,41,20,110,24,70,0,0,0,41,41,63,0,0,53,73,21,43,68,86,0,42,0,34,68,0,72,36,36,22,48,0,57,35,0,42,49,0,43,33,0,45,77,50,40,106,104,35,58,68,0,24,32,29,61,60,0,0,0,61,0,0,22,0,0,66,65,108,28,36,32,31],"marker_depth":[78,101,41,93,70,80,76,109,86,98,108,54,26,96,85,55,79,47,85,68,73,60,99,42,105,105,44,109,89,72,36,110,43,93,64,61,101,81,56,83,107,33,85,109,32,54,96,86,34,67,46,75,93,93,87,49,62,36,81,43,88,53,60,67,69,53,71,55,31,65,110,86,40,106,104,55,58,84,105,40,50,38,88,84,36,47,99,109,53,50,37,94,25,102,65,108,61,36,32,43],"bias":0.50147216157658892,"log_likelihood":-302.6542427559836,"prior":[0.31923051282135773,0.49154873152110251,0.18922075565753979],"genotypes":[2,0,0,0,0,1,1,1,2,0,1,1,0,1,2,0,2,1,1,0,0,0,1,0,2,2,2,2,1,1,1,2,1,1,0,0,0,1,1,1,0,0,1,1,1,1,1,2,0,1,0,1,1,0,1,1,1,1,1,0,1,1,0,1,1,0,1,1,0,1,1,1,2,2,2,1,2,1,0,1,1,1,1,1,0,0,0,1,0,0,1,0,0,1,2,2,1,2,2,1],"posteriors":[[1.0566365241709882e-32,1.4348565144401152e-06,0.99999856514348551],[1,9.66909660658002e-17,1.1380321213932034e-36],[0.99999999990156541,9.8434561540998809e-11,5.3710861489447959e-27],[0.99999999999999956,3.8274185499610184e-16,9.9327109582376557e-36],[0.99999999999996247,3.7495978592630464e-14,1.4458988307696763e-32],[4.0640266657403429e-19,0.9999999778132278,2.2186772661753039e-08],[1.7434012360479417e-16,0.99999999941276796,5.8723200035733165e-10],[7.2782543550588868e-16,0.99999999995085265,4.9146422690063499e-11],[8.5970757018256953e-34,6.753287380770912e-07,0.99999932467126196],[0.99999999999999978,1.6019575453959548e-16,2.5178826200858895e-36],[5.1706883507392243e-16,0.99999999993822719,6.177275699854007e-11],[1.0088982368850366e-14,0.99999999966670616,3.3328381053153771e-10],[0.99999996866549612,3.1334503850688834e-08,9.6333602664254615e-23],[1.4245913937187382e-19,0.99999998450844541,1.5491554335225674e-08],[1.1631327256699959e-33,7.3978377675310749e-07,0.99999926021622321],[0.99999999999850853,1.4914162475165322e-12,5.4924078743949403e-30],[7.6312717175186345e-33,1.3017392572817875e-06,0.99999869826074261],[4.9404090579853872e-16,0.99999999426873964,5.7312598419090828e-09],[5.3149584559423073e-24,0.9998704910927807,0.00012950890721943062],[0.99999999999994094,5.9007441302847948e-14,2.9897884172396694e-32],[0.99999999999998068,1.9353886532623938e-14,5.0223421276976841e-33],[0.99999999999959566,4.0440534855570361e-13,6.6246222231863662e-31],[9.0609880091658344e-18,0.99999999904476455,9.5523547916402318e-10],[0.99999999992926614,7.0733762397843874e-11,3.1016168798152989e-27],[4.6131228324771784e-36,1.3760424405980302e-07,0.99999986239575589],[4.6131228324771784e-36,1.3760424405980302e-07,0.99999986239575589],[1.2063124542199475e-26,8.1822717114474821e-05,0.99991817728288568],[1.70933257376138e-36,1.0143424975013335e-07,0.99999989856575011],[3.8747839546938017e-19,0.99999998781699173,1.2183008369656517e-08],[1.9280988790763313e-13,0.99999999998635936,1.3448104886717682e-11],[8.8736088122419628e-12,0.99999999991883382,7.2292682808591499e-11],[1.3404127649385574e-36,9.4122522639250208e-08,0.99999990587747734],[3.3606116209698923e-12,0.9999999999539253,4.2714361229846606e-11],[6.2007784750307476e-19,0.99999999297695119,7.023049262832977e-09],[0.99999999999984901,1.5092166355811087e-13,1.3515164151826394e-31],[0.99999999999968536,3.1462581597977743e-13,4.4164188584459385e-31],[1,9.66909660658002e-17,1.1380321213932034e-36],[5.2977262553522662e-12,0.9999999999933209,1.3817181013870262e-12],[8.180503009247486e-17,0.99999999413020779,5.869792327509557e-09],[8.8903995911654843e-19,0.9999999895276277,1.0472372477652132e-08],[1,3.654627481472977e-17,2.4704097797834769e-37],[0.99999999827500408,1.7249958277107712e-09,6.5931490106729623e-25],[3.8654262766517344e-15,0.99999999994499511,5.5001196916757161e-11],[6.584407086153359e-17,0.99999999980697218,1.9302806696777915e-10],[1.9386502205952892e-13,0.99999999839339493,1.6064112896734161e-09],[2.9276300104121764e-18,0.99999991542936595,8.457063395610161e-08],[9.9100784538544873e-18,0.99999999895340475,1.0465955572859589e-09],[8.5970757018256953e-34,6.753287380770912e-07,0.99999932467126196],[0.99999999882208379,1.1779161513538711e-09,3.4535590994402505e-25],[9.970456260602299e-15,0.99999999989845456,1.0153572236752402e-10],[0.99999999998012301,1.9877071176145897e-11,3.8021107544453799e-28],[1.532095458883828e-10,0.99999999984643573,3.5511999553315563e-13],[2.6359371264242565e-18,0.99999999721526978,2.7847303359003279e-09],[0.99999999999999956,3.8274185499610184e-16,9.9327109582376557e-36],[5.2909568391894846e-21,0.99999971340970828,2.8659029179911976e-07],[1.9809227299061362e-16,0.99999999196659983,8.0333998706251207e-09],[1.9298129991492648e-13,0.9999999999717617,2.8045609737528872e-11],[6.9041693114248313e-13,0.99999999967732345,3.2198631678196528e-10],[3.0661362948514892e-14,0.99999999997852118,2.1448573426660853e-11],[0.9999999999488951,5.1104802237349101e-11,1.8092786646386608e-27],[7.4492018402458951e-16,0.99999999987941157,1.2058802569005287e-10],[5.8893687345841132e-15,0.99999999948119012,5.1880413567649893e-10],[0.99999999999959566,4.0440534855570361e-13,6.6246222231863662e-31],[9.970456260602299e-15,0.99999999989845456,1.0153572236752402e-10],[6.6974870189802046e-17,0.99999999817914009,1.8208600229763523e-09],[0.99999999999742273,2.5773114108986608e-12,1.3379801289352526e-29],[2.5401109043190031e-14,0.99999999995564526,4.432948656576559e-11],[1.1890658313757132e-13,0.99999999993007149,6.980973945004594e-11],[0.9999999962150059,3.7849939733313151e-09,2.5136307629744289e-24],[2.7998110970206086e-16,0.99999999896164615,1.0383535360875146e-09],[8.2791193727655951e-18,0.99999999937281814,6.2718225721277216e-10],[4.7275793727032564e-14,0.99999999998697042,1.2982482315511318e-11],[1.0638431776930214e-25,0.00014969133216261623,0.99985030866783742],[3.5879869059987848e-36,1.2739136473197011e-07,0.99999987260863532],[5.9438271893720277e-36,1.487248617758595e-07,0.99999985127513824],[1.7035361254092162e-14,0.9999999997817719,2.182109789712519e-10],[1.7163003892744133e-29,1.2658590391379248e-05,0.99998734140960854],[2.615325064347021e-20,0.99999988762590786,1.1237409223837151e-07],[1,5.0281950973428849e-17,4.0751725579540685e-37],[6.3791818285035369e-13,0.99999999982750287,1.7185939388950246e-10],[2.4384522792817064e-14,0.99999999969145659,3.0851923134070733e-10],[3.9113785665859965e-16,0.99999996535524194,3.4644757351887962e-08],[4.3175784677266509e-17,0.99999999934833683,6.5166336520460767e-10],[1.484939098891122e-17,0.99999999840698983,1.5930104777299965e-09],[0.99999999943886198,5.6113810704041657e-10,9.8791984131489657e-26],[0.99999999998534861,1.4651403201291389e-11,2.3012801260043524e-28],[0.99999999999999978,1.3518977303567247e-16,1.9277550779180513e-36],[8.0204999772077559e-14,0.99999999999611777,3.8018742974790406e-12],[0.99999999999742273,2.5773114108986608e-12,1.3379801289352526e-29],[0.99999999999397082,6.0292483601457885e-12,5.3615906228311228e-29],[1.2647292861124221e-12,0.99999999981106313,1.876722127985273e-10],[0.99999999999999978,3.2056858249987744e-16,7.5101033969791544e-36],[0.9999999507876921,4.9212307827667345e-08,2.1193356401616121e-22],[3.909671643481343e-16,0.99999999990857003,9.1429641353486259e-11],[1.0416407245519749e-30,5.5919737231329144e-06,0.99999440802627693],[2.1841457122817016e-36,1.0937554403388935e-07,0.99999989062445604],[1.8079269422157048e-10,0.99999999981836929,8.3852553773045089e-13],[1.1179956798043394e-24,0.00028468477780926518,0.99971531522219081],[1.4452040033821986e-23,0.00056580753836906372,0.9994341924616309],[1.1108454767073298e-15,0.99999999380310589,6.196893227429395e-09]],"max_posterior":[0.99999856514348551,1,0.99999999990156541,0.99999999999999956,0.99999999999996247,0.9999999778132278,0.99999999941276796,0.99999999995085265,0.99999932467126196,0.99999999999999978,0.99999999993822719,0.99999999966670616,0.99999996866549612,0.99999998450844541,0.99999926021622321,0.99999999999850853,0.99999869826074261,0.99999999426873964,0.9998704910927807,0.99999999999994094,0.99999999999998068,0.99999999999959566,0.99999999904476455,0.99999999992926614,0.99999986239575589,0.99999986239575589,0.99991817728288568,0.99999989856575011,0.99999998781699173,0.99999999998635936,0.99999999991883382,0.99999990587747734,0.9999999999539253,0.99999999297695119,0.99999999999984901,0.99999999999968536,1,0.9999999999933209,0.99999999413020779,0.9999999895276277,1,0.99999999827500408,0.99999999994499511,0.99999999980697218,0.99999999839339493,0.99999991542936595,0.99999999895340475,0.99999932467126196,0.99999999882208379,0.99999999989845456,0.99999999998012301,0.99999999984643573,0.99999999721526978,0.99999999999999956,0.99999971340970828,0.99999999196659983,0.9999999999717617,0.99999999967732345,0.99999999997852118,0.9999999999488951,0.99999999987941157,0.99999999948119012,0.99999999999959566,0.99999999989845456,0.99999999817914009,0.99999999999742273,0.99999999995564526,0.99999999993007149,0.9999999962150059,0.99999999896164615,0.99999999937281814,0.99999999998697042,0.99985030866783742,0.99999987260863532,0.99999985127513824,0.9999999997817719,0.99998734140960854,0.99999988762590786,1,0.99999999982750287,0.99999999969145659,0.99999996535524194,0.99999999934833683,0.99999999840698983,0.99999999943886198,0.99999999998534861,0.99999999999999978,0.99999999999611777,0.99999999999742273,0.99999999999397082,0.99999999981106313,0.99999999999999978,0.9999999507876921,0.99999999990857003,0.99999440802627693,0.99999989062445604,0.99999999981836929,0.99971531522219081,0.9994341924616309,0.99999999380310589],"proportion_misgenotyped":1.2361647756309324e-05},{"name":"hexaploid_family_bias_1.5","model":"norm","ploidy":6,"simulated_bias":1.5,"allele_depth":[23,22,6,10,15,36,2,3,12,24,10,21,4,11,2,14,29,4,4,8,18,24,12,4,11,4,23,18,13,9,9,16,35,53,24,30,4,11,16,8,4,28,48,12,41,5,8,51,32,4,31,30,6,11,12,25,24,7,12,2,1,43,21,18,8,1,4,35,17,5,21,35,28,10,26,5,19,15,26,21,22,24,2,42,9,20,23,2,39,14,6,20,25,33,51,13,6,9,4,39,8,27,9,7,21,9,5,17,17,10],"marker_depth":[84,80,26,55,48,65,36,28,82,63,39,63,59,86,53,45,73,80,80,85,82,61,81,64,62,50,59,92,39,65,93,58,97,103,80,102,48,51,48,32,62,95,99,35,65,31,30,105,68,65,62,70,96,88,61,63,63,42,54,39,48,99,76,53,31,76,49,100,41,52,96,79,44,45,74,30,61,70,93,106,69,66,58,82,44,42,74,48,87,60,43,69,91,99,106,35,50,79,25,77,100,72,75,40,69,96,47,52,66,102],"bias":2.1363204862115874,"log_likelihood":-428.70658142916307,"prior":[0.0074333268791506694,0.10999414055056382,0.40829613247622687,0.38018972697319708,0.088806526816019166,0.0052036585190779559,7.64877857644132e-05],"genotypes":[3,3,2,2,3,4,2,2,2,3,3,3,2,2,1,3,3,1,1,2,2,3,2,1,2,2,3,2,3,2,2,3,3,4,3,3,2,2,3,3,1,3,3,3,4,2,3,3,3,1,3,3,1,2,2,3,3,2,2,1,1,3,3,3,3,1,2,3,3,2,2,3,4,2,3,2,3,2,3,2,3,3,1,4,2,3,3,1,3,2,2,3,3,3,3,3,2,2,2,4,2,3,2,2,3,2,2,3,3,2],"posteriors":[[6.5696146111981593e-11,0.008813200546810564,0.3923883374590113,0.58022958938042113,0.018567091882568415,1.7806654927602563e-06,1.2402239008979408e-21],[6.7269357953590213e-11,0.0088229381412266022,0.38920611051133258,0.58242963501937195,0.019539223736252446,2.0925245468414783e-06,1.8445054669541717e-21],[1.466279544502329e-09,0.037660894556226836,0.51725556001243489,0.42673665334652566,0.018335994026870278,1.0896591662644683e-05,1.1859914499822034e-18],[2.2373312269040223e-09,0.065138086079322935,0.67635521942691723,0.25606931491737156,0.0024372849716607302,9.2367396070368099e-08,4.7472707131676221e-23],[4.8350881757538627e-11,0.0059462043183552962,0.29445058073173686,0.649608949220423,0.049967433272228343,2.6832408905899436e-05,5.7902569743134591e-19],[9.5629112692749252e-15,1.231104656512261e-05,0.0080496294604090034,0.31738000536310856,0.65740250783938892,0.017155546290472626,4.6501157200300926e-14],[3.6655179577848803e-07,0.46424536353118573,0.50298594150420761,0.032687893936485406,8.0433202074026559e-05,1.2742512186830243e-09,6.2009359160185044e-25],[4.9861931991556688e-08,0.21547020217764154,0.65657586296204928,0.12680180677304501,0.0011519718461471145,1.0637918502898469e-07,8.2820205252033178e-22],[5.7291920805673333e-09,0.12160210453703708,0.75056365490037491,0.12747717861299138,0.00035705422988844724,1.9905162972522292e-09,4.1354649908647198e-26],[3.4086796449344245e-12,0.0010519147955963485,0.13158377498608859,0.73034109493670496,0.1368619449868716,0.00016127029132987944,5.5834529447218827e-18],[3.5981718114369838e-10,0.019734946498195613,0.45850584031129238,0.50115376979317183,0.020598666238176414,6.776799346623008e-06,1.2322592393524919e-19],[1.5105307956281258e-11,0.0029908296942033558,0.22659800475118624,0.70433294231136379,0.066045533808794757,3.2689419346717844e-05,3.488786905468785e-19],[1.8273666550104032e-07,0.46487392727678584,0.51401950388461148,0.021087377771073966,1.9008281828582076e-05,4.9034507749578284e-11,8.6527475451471616e-28],[1.1463573726968466e-08,0.17364262507763256,0.74283767872019446,0.083376616200037154,0.00014306810708077531,4.3148155089588739e-10,3.8887913675157978e-27],[9.7561573975521844e-07,0.66441508500742053,0.32796869617306978,0.0076106750339694721,4.5681605866689363e-06,9.2139887468243196e-12,1.6014446984556391e-28],[5.7358904013615151e-11,0.0064914713354343101,0.30159583924304034,0.64202241002606264,0.049860905018140826,2.9374319963245918e-05,8.332562847730383e-19],[1.4569675536710558e-12,0.00059966440733439914,0.099858209793972724,0.72742658280001382,0.17188409710251773,0.00023144589470449933,7.1307139530212217e-18],[4.5367780580553411e-07,0.6475380890674185,0.34687984628912638,0.0055801807466346011,1.4302183939935593e-06,6.2027982883819313e-13,7.0758547201472707e-31],[4.5367780580553411e-07,0.6475380890674185,0.34687984628912638,0.0055801807466346011,1.4302183939935593e-06,6.2027982883819313e-13,7.0758547201472707e-31],[4.8377206286121406e-08,0.32686725537733119,0.64027583522621712,0.03283191562811158,2.4945360023375189e-05,3.1110564182378396e-11,1.0208088770028491e-28],[4.2097099488822415e-10,0.028278322783224473,0.59080440264835288,0.37630439974772034,0.0046127313871331234,1.4301259822981458e-07,2.4642535006758959e-23],[2.4846668511982943e-12,0.00082862170344470318,0.11438665169962872,0.72268128911253282,0.16185551092470332,0.00024792655720571867,1.3188079863039801e-17],[5.3911624784080141e-09,0.11747481584072245,0.74934596396470221,0.13278554398058173,0.00039366844488344314,2.3779474394883216e-09,5.5834036394697642e-26],[2.3474190226972982e-07,0.51406489455776339,0.47079666352646782,0.015128400822032078,9.8063360386653818e-06,1.5795977768933828e-11,1.340658046971258e-28],[2.3131887833060803e-09,0.069140317583148328,0.69418555293997808,0.23489446123189581,0.0017796216050891241,4.4326699728275703e-08,1.0382112565501737e-23],[1.081578420875977e-07,0.36917954368969919,0.5914352581198008,0.039316831691067936,6.8257888836646178e-05,4.5275344216466139e-10,3.5452488770019927e-26],[3.0100526332150651e-12,0.00093956637888342059,0.12157248155176488,0.72314118763880053,0.15411649748426109,0.00023026694327991424,1.27110321765641e-17],[8.703326530161055e-10,0.044879998007277322,0.67334961273932803,0.27983466609752139,0.0019356955403049421,2.6745235821547686e-08,1.3418084641330517e-24],[4.1885828953377612e-11,0.004965009576680484,0.25679450742148946,0.66551411988566811,0.07264840592381118,7.7957150465003669e-05,7.0500973620253461e-18],[8.9567282243564173e-09,0.14161450769736075,0.73582404962104142,0.12214951426238682,0.00041191561246216443,3.850020640696548e-09,2.4596783508580923e-25],[4.1723275998283505e-08,0.31562829271619569,0.65146885339709781,0.032880328877089054,2.2483264297050762e-05,2.2044496950390929e-11,4.4958333282331709e-29],[1.0430115623423584e-10,0.010562220296952347,0.39365932751945626,0.57228315138662356,0.023490642373783465,4.6583188832872263e-06,1.7627900220571266e-20],[2.8465115565905177e-12,0.0010600528646609213,0.14554289993632288,0.75423835201827216,0.099111001822477854,4.7693355420122477e-05,2.1573682246598182e-19],[9.5954228001578214e-15,1.4684661263733227e-05,0.010558356838221266,0.39549368921360967,0.58773365252275256,0.006199616764141604,1.8060287972128717e-15],[2.9454796188087761e-11,0.0050939203851425238,0.3065593977075331,0.65492174033711403,0.033418957730404819,5.9838103506901293e-06,1.0310968843639772e-20],[2.5867076945026713e-11,0.0049954773058550556,0.31717064625597563,0.65069870255984719,0.02713221587468697,2.957977767915346e-06,1.7195311484327102e-21],[9.4778452829168071e-08,0.3470248687593504,0.60753087998718636,0.045351918420218504,9.2237284866983143e-05,7.6992502709451562e-10,8.7081260587344439e-26],[8.1636651134657736e-10,0.035822898950614523,0.58449287991078858,0.37296779124683777,0.0067158411127678603,5.8796262482829475e-07,1.0277394656378865e-21],[2.6266555863601186e-11,0.0039468019505727384,0.2429288028926759,0.6833689051498385,0.069702534359189508,5.2955621456961667e-05,1.8283424517923008e-18],[6.0949054441370367e-10,0.024899967442715009,0.47530978745198471,0.47849108801208184,0.02128915569162753,1.000079210045134e-05,4.7279116878756044e-19],[2.1301446062275432e-07,0.49478366863903045,0.48794635791676289,0.017257030642322289,1.2729762768794309e-05,2.4654771799760171e-11,2.7843772107919344e-28],[2.7705663122709198e-11,0.0051347757041271527,0.31734263451936867,0.64933591814997216,0.028183210273025099,3.4613258012101274e-06,2.6587132686063938e-21],[3.3541201490167237e-14,3.8668944925064908e-05,0.019992867062073261,0.50696249104027435,0.4702451844171085,0.0027607885355847833,3.7583087990222274e-16],[4.2010903495617494e-11,0.0047477964379302002,0.24402872819826971,0.66675167909119348,0.084346996376298167,0.00012479985429753,2.3044779573269249e-17],[5.0264507128472476e-16,1.1354427777976234e-06,0.0014541744557550701,0.13103107840606246,0.78143779145342507,0.086075820239882625,2.095991861518751e-12],[7.9559644296659927e-09,0.099887931890467505,0.65905061372097895,0.23740754290088112,0.0036533783631936424,5.2516851448042162e-07,5.6162310706708363e-21],[4.4115572821900772e-10,0.019924662912088976,0.43276130004226565,0.51787007175766842,0.029423829620220793,2.0135226600397483e-05,1.8638900710720535e-18],[2.8106459788813703e-14,3.4332224306179956e-05,0.018774346983439744,0.50000094614770185,0.47841384947897458,0.002776525165549484,3.3658687383156618e-16],[1.5362389630856653e-13,0.00010938770264479039,0.035148569227113541,0.57635506584414653,0.38623454795544804,0.0021524292704933543,5.580333625521942e-16],[2.4607144696959939e-07,0.5234988912199332,0.46231915571116178,0.014173084161757124,8.6228230098374231e-06,1.2691205306407504e-11,9.3691004511067948e-29],[7.5454569732817905e-14,6.1101257805729059e-05,0.023421061726877003,0.49308814701774506,0.47862695321992393,0.0048027367775689937,3.544031467585721e-15],[5.7675266075347797e-13,0.00029912948190064144,0.065889264295309155,0.68027999271120398,0.25289531532409387,0.00063629818691514177,5.0649270968931032e-17],[2.2151752776628513e-07,0.56850736909812172,0.4237460667064486,0.0077445232666267672,1.8194107330277985e-06,5.42327880496965e-13,2.6534541648693561e-31],[1.2748452604666726e-08,0.18381799756963241,0.73939355951867214,0.076670070902263313,0.00011835895329546989,3.0768434298358744e-10,2.2147904059887165e-27],[1.2260817304691363e-09,0.048271424840186643,0.64765571960834833,0.30080029979006206,0.0032724281451834893,1.263901380153885e-07,5.2909508935170041e-23],[2.0695842109816786e-12,0.00073476247618463927,0.10785585939673532,0.72167339188388735,0.16947014272276661,0.00026584351835661716,1.3649162935731521e-17],[3.4086796449344245e-12,0.0010519147955963485,0.13158377498608859,0.73034109493670496,0.1368619449868716,0.00016127029132987944,5.5834529447218827e-18],[4.8456916981080115e-09,0.088636423857550065,0.68077870433973675,0.22814219660479387,0.0024425175541927546,1.527980347907239e-07,2.7361850663592784e-22],[6.1789751121121749e-10,0.031051784946823275,0.56724735039072027,0.39426537307505927,0.0074348528954781964,6.3807402178120787e-07,9.5673848576826602e-22],[4.5043374345365466e-07,0.50549083878496182,0.46954811370260846,0.024914267657440221,4.6328936316748453e-05,4.8492913811987326e-10,1.1889611763589409e-25],[3.4821736484998557e-06,0.75922135512791766,0.23644866510094209,0.0043241162058568639,2.381386437857296e-06,5.1971291705992845e-12,1.2662617667344011e-28],[2.1358394882045723e-13,0.00015938630408124214,0.049186600478515731,0.66700403498953564,0.28305045570886345,0.00059952251879024916,1.9705516589978602e-17],[6.9010025918106087e-11,0.0088329428211867626,0.38578066802156674,0.58474766691790947,0.020636234908841689,2.4872614850141554e-06,2.8225013785502324e-21],[1.7641297341785813e-11,0.0031252757683333052,0.22221523844859092,0.69917370974800952,0.075431339104116493,5.4436913308258914e-05,1.4018554211300112e-18],[5.2011647032486293e-10,0.022330748802780912,0.4543922869909689,0.49824953924943843,0.025013281013556352,1.4143423138982891e-05,9.2971470132253153e-19],[9.291109213267012e-06,0.89868400969718376,0.10081529714500972,0.00049135653375853659,4.5514826730005537e-08,7.8276029350512632e-15,3.7184845007258481e-33],[1.0132785603486849e-07,0.35812949086112511,0.59957263134312444,0.042218494727255196,7.9281151314598155e-05,5.8932451538346205e-10,5.5356116887232804e-26],[3.9228301120601962e-12,0.001343156900293212,0.16596548634481348,0.75053974993011552,0.082121134040652077,3.0472780202969415e-05,9.2411481482182037e-20],[3.845016675735722e-12,0.00096694488211653701,0.11111336070439999,0.68713475353948761,0.20009524760759095,0.00068969326255982079,2.8723250201226809e-16],[5.3332764157981448e-08,0.28881938422712428,0.65358139091508483,0.057473175544228296,0.00012599497059822637,1.0101999310518054e-09,8.9938225480437838e-26],[3.7252595829381043e-10,0.027422831465625082,0.59849101029689711,0.37016883239179954,0.0039172399390985197,8.5534053871574573e-08,7.0260163475051975e-24],[2.6313159904248803e-13,0.00017354681314875038,0.048963143453746805,0.64726728377287401,0.30269304714521289,0.00090297881475398761,7.1791625294706374e-17],[3.1076508959210832e-15,4.1306241643325408e-06,0.0031346672823481417,0.17513924702311232,0.73947058957678513,0.082251365487648595,5.9384659873798734e-12],[7.8751683489294934e-10,0.033432939281664791,0.5601968569360023,0.39740866181541601,0.0089603385564097013,1.2026229900113463e-06,4.8379838833614023e-21],[6.283599788287076e-12,0.001714174159334717,0.17728282330224118,0.73412069334549246,0.086834263072872481,4.804611377576189e-05,4.3035258197311791e-19],[7.0268188910434008e-09,0.092543594331441964,0.64924724639285303,0.25382583453550855,0.0043825669662936944,7.507470837229409e-07,1.0912703407799183e-20],[3.1685872128800614e-11,0.0048978225978791113,0.28477354082775025,0.66477665679749132,0.045536106277558663,1.5873467634950293e-05,1.1350886679591827e-19],[5.9092882298474182e-10,0.033029194596002891,0.60256608309572279,0.35977719184699247,0.0046273468225851329,1.8304776801160823e-07,5.8952675120300797e-23],[4.7636778467578218e-11,0.0073371357110000885,0.36975093159241718,0.60267619297825192,0.020233889279145316,1.8503915485666178e-06,1.0053586356715499e-21],[7.1401121659947783e-10,0.04144939104459467,0.67251063260312938,0.28422913883379547,0.0018108173848080846,1.9419661385546533e-08,5.4632244997078678e-25],[2.024005521334777e-11,0.0037692238906710217,0.25862193791298466,0.68735727996327067,0.050235085151322814,1.6473061510614595e-05,8.4234307980061972e-20],[5.3163478423237155e-12,0.0014665680155681296,0.15913283202501743,0.73297110201176008,0.10634334447150057,8.6153470837516396e-05,1.6241845957140972e-18],[1.2167590008630106e-06,0.70810055022161311,0.28675115827453967,0.0051448950080048373,2.1797341744476126e-06,2.6669743914203106e-12,2.1182215099002066e-29],[2.0145911964705242e-14,2.43399512934778e-05,0.013905032316891022,0.42533893171396298,0.55462838987089824,0.0061033061469312824,2.8675632734966013e-15],[1.3930631650153753e-09,0.046011322812527518,0.6071279942261002,0.34079119118595036,0.0060688525824364814,6.3779992248348957e-07,1.9749434894277938e-21],[6.1111247871035754e-13,0.00025367278557011913,0.050706897462618902,0.58752906755992884,0.35839685448952568,0.0031135077017404671,4.7275908510285127e-15],[2.3323683874207498e-11,0.0042461209348705377,0.27759395136051113,0.67573330941053533,0.042415950849371449,1.0667421388101302e-05,3.3408709278943074e-20],[7.6266525539485582e-07,0.6140705443671417,0.37447207202998506,0.011446626081359138,9.9948216133002943e-06,3.4645360577255482e-11,1.4204200819201192e-27],[1.7387624391125151e-13,0.00013109156361843698,0.042233253042670227,0.63152128794043771,0.3251159265930138,0.00099844086008599392,6.898428693251689e-17],[3.7942493917176494e-10,0.024044358995226722,0.53478452514908159,0.43231722519021148,0.0088531572844144889,7.3300164099766773e-07,8.3374073608078211e-22],[1.1726872997100833e-08,0.13866287981538689,0.70508740354784072,0.15520135949671812,0.0010483075465210808,3.7866660157552418e-08,3.2414360482666083e-23],[5.1157515766832663e-11,0.0070324462388558677,0.34377558756667503,0.62042859855283017,0.028757979073215791,5.388517265454708e-06,1.3172028777211441e-20],[5.7655336421624175e-11,0.0082743236920213691,0.38751398641014734,0.58595431430853495,0.018255817834467211,1.557697173763889e-06,8.0449439189498362e-22],[7.037550201342667e-12,0.0020224404916595233,0.20452026451724428,0.73265925081583538,0.060781972526003647,1.6071642219709435e-05,3.1565433372954248e-20],[3.2739776574222093e-14,3.8769177093838123e-05,0.020387960006371095,0.51610294110915167,0.46104539470292216,0.002424935004428233,2.5204715636242524e-16],[1.9656285649932342e-11,0.0028369430404281974,0.18899338710486038,0.68700655233784569,0.12089343730893658,0.00026968018827282464,8.9198622026782214e-17],[2.1257338911819002e-08,0.19344619497374074,0.70569244068139547,0.10048796944539877,0.00037336802152825049,5.6205978492318581e-09,1.1620815240368917e-24],[2.113178149089859e-08,0.22509442350725284,0.71250284285111221,0.062313968086231658,8.8744179837903659e-05,2.4378394527870125e-10,2.361009566869922e-27],[1.058396895968219e-08,0.10375816063219506,0.63888602706294673,0.25214851754400541,0.0052058835318884115,1.4006449953421438e-06,5.7552240758579507e-20],[3.0097045795156319e-14,3.2564409337118138e-05,0.016576537621867112,0.45109804858450447,0.52694846052207356,0.0053443888621848108,2.5296634658637466e-15],[8.9980998215574725e-08,0.43174841663338492,0.55187714409365429,0.016368345338241198,6.0039510150781848e-06,2.7063430860124475e-12,1.9280745588505802e-30],[3.0891242861727585e-12,0.0010265565827655505,0.1341640615074442,0.73914152401208633,0.12555501796087556,0.0001128399337392979,2.055703837812862e-18],[1.6890807438187154e-08,0.20019617400508735,0.72438446932436185,0.075284218722009702,0.00013512054297974579,5.147536887794855e-10,8.2174608219891466e-27],[3.9141074282669327e-09,0.077758892283230449,0.66347738983981286,0.25544916770403059,0.0033142704615490319,2.757972698378291e-07,7.94233575921118e-22],[3.2157449372972309e-11,0.0051580321562151856,0.29941361108338854,0.65710554649338238,0.038313300588028662,9.5096468276354493e-06,3.3636954904680337e-20],[4.7424794880242988e-08,0.33525568681652962,0.63592580464961346,0.02880140781096786,1.7053284418993143e-05,1.3675238177546358e-11,2.0633570518885967e-29],[3.7163864749480617e-08,0.24014907611766864,0.67955449650709077,0.080034883811279087,0.00026150265487646788,3.7452201875278167e-09,8.4467312782162275e-25],[2.6836718892200204e-11,0.004130348592787058,0.25327493004487323,0.68064042780518885,0.061917264815770673,3.7028714543558327e-05,7.7135471210405314e-19],[1.5379506416819869e-10,0.014159808836617086,0.45276918541077521,0.51842059950409647,0.014648819637497654,1.5864572183042688e-06,2.0827072674549779e-21],[3.8311196285616116e-08,0.31212946500834421,0.65628672515581732,0.031564989531553303,1.8781979147301296e-05,1.3941468568018264e-11,1.6897075775229704e-29]],"max_posterior":[0.58022958938042113,0.58242963501937195,0.51725556001243489,0.67635521942691723,0.649608949220423,0.65740250783938892,0.50298594150420761,0.65657586296204928,0.75056365490037491,0.73034109493670496,0.50115376979317183,0.70433294231136379,0.51401950388461148,0.74283767872019446,0.66441508500742053,0.64202241002606264,0.72742658280001382,0.6475380890674185,0.6475380890674185,0.64027583522621712,0.59080440264835288,0.72268128911253282,0.74934596396470221,0.51406489455776339,0.69418555293997808,0.5914352581198008,0.72314118763880053,0.67334961273932803,0.66551411988566811,0.73582404962104142,0.65146885339709781,0.57228315138662356,0.75423835201827216,0.58773365252275256,0.65492174033711403,0.65069870255984719,0.60753087998718636,0.58449287991078858,0.6833689051498385,0.47849108801208184,0.49478366863903045,0.64933591814997216,0.50696249104027435,0.66675167909119348,0.78143779145342507,0.65905061372097895,0.51787007175766842,0.50000094614770185,0.57635506584414653,0.5234988912199332,0.49308814701774506,0.68027999271120398,0.56850736909812172,0.73939355951867214,0.64765571960834833,0.72167339188388735,0.73034109493670496,0.68077870433973675,0.56724735039072027,0.50549083878496182,0.75922135512791766,0.66700403498953564,0.58474766691790947,0.69917370974800952,0.49824953924943843,0.89868400969718376,0.59957263134312444,0.75053974993011552,0.68713475353948761,0.65358139091508483,0.59849101029689711,0.64726728377287401,0.73947058957678513,0.5601968569360023,0.73412069334549246,0.64924724639285303,0.66477665679749132,0.60256608309572279,0.60267619297825192,0.67251063260312938,0.68735727996327067,0.73297110201176008,0.70810055022161311,0.55462838987089824,0.6071279942261002,0.58752906755992884,0.67573330941053533,0.6140705443671417,0.63152128794043771,0.53478452514908159,0.70508740354784072,0.62042859855283017,0.58595431430853495,0.73265925081583538,0.51610294110915167,0.68700655233784569,0.70569244068139547,0.71250284285111221,0.63888602706294673,0.52694846052207356,0.55187714409365429,0.73914152401208633,0.72438446932436185,0.66347738983981286,0.65710554649338238,0.63592580464961346,0.67955449650709077,0.68064042780518885,0.51842059950409647,0.65628672515581732],"proportion_misgenotyped":0.36024977327164609},{"name":"hexaploid_family_bias_1.5","model":"hw","ploidy":6,"simulated_bias":1.5,"allele_depth":[23,22,6,10,15,36,2,3,12,24,10,21,4,11,2,14,29,4,4,8,18,24,12,4,11,4,23,18,13,9,9,16,35,53,24,30,4,11,16,8,4,28,48,12,41,5,8,51,32,4,31,30,6,11,12,25,24,7,12,2,1,43,21,18,8,1,4,35,17,5,21,35,28,10,26,5,19,15,26,21,22,24,2,42,9,20,23,2,39,14,6,20,25,33,51,13,6,9,4,39,8,27,9,7,21,9,5,17,17,10],"marker_depth":[84,80,26,55,48,65,36,28,82,63,39,63,59,86,53,45,73,80,80,85,82,61,81,64,62,50,59,92,39,65,93,58,97,103,80,102,48,51,48,32,62,95,99,35,65,31,30,105,68,65,62,70,96,88,61,63,63,42,54,39,48,99,76,53,31,76,49,100,41,52,96,79,44,45,74,30,61,70,93,106,69,66,58,82,44,42,74,48,87,60,43,69,91,99,106,35,50,79,25,77,100,72,75,40,69,96,47,52,66,102],"bias":2.6531468168139885,"log_likelihood":-434.83970671639929,"prior":[0.027224971964624693,0.13447263152349226,0.27675095270911082,0.30376873552204836,0.18755107118792894,0.061758217469024081,0.0084734196237707948],"genotypes":[3,3,3,2,3,4,1,2,2,4,3,3,2,2,1,3,4,1,1,2,3,4,2,1,2,2,4,2,3,2,2,3,3,4,3,3,2,3,3,3,1,3,4,3,5,2,3,4,4,1,4,4,1,2,2,4,4,2,3,1,1,4,3,3,3,1,2,3,4,2,3,4,5,3,3,2,3,3,3,2,3,3,1,4,3,4,3,1,4,3,2,3,3,3,4,3,2,2,2,4,2,3,2,2,3,2,2,3,3,2],"posteriors":[[2.5183250257266361e-10,0.0072395163466324047,0.2142346890023285,0.65020538674673567,0.12815545756198973,0.00016495009048124963,2.2180671851021661e-19],[2.5769239675433054e-10,0.0072645634293397902,0.21253953728791394,0.64777619676407538,0.13223142553083012,0.00018827673014848231,3.2966037974942207e-19],[5.357978544345637e-09,0.037693468200350369,0.35425300561851852,0.50948434265463238,0.098001179031258073,0.00056799913726216715,2.0219132619839932e-16],[7.9280618578274354e-09,0.063609487368826365,0.50740818360199247,0.40711304428786405,0.021860118839887095,9.1579733683159621e-06,7.8484249759265751e-21],[1.8151100266878731e-10,0.004798179669282681,0.14921282194619767,0.60114963823149914,0.24325413234203708,0.0015852276294722318,1.0141354290455156e-16],[2.1110136480216926e-14,3.8820032954440486e-06,0.001140294613065852,0.065335610482838716,0.66878274852827879,0.26473746436771106,4.7891880486095734e-12],[8.8705343634024574e-07,0.48378522561142112,0.45553000107479708,0.059977637162592544,0.00070615166633509877,9.7431417926865774e-08,7.0011772103003593e-23],[1.5050168342236617e-07,0.23358795469117272,0.55733139667833009,0.20081763142209824,0.0082560345328449997,6.8321738706362377e-06,1.1662896268147206e-19],[1.8956275778184699e-08,0.11841727536504157,0.62230735844768714,0.25471639695500448,0.0045586623413520671,2.8793463873669616e-07,6.3839327542605551e-24],[1.188431506526346e-11,0.00067515409376802326,0.047861757686671343,0.46638502033093771,0.47736763563414314,0.0077104322425946325,9.0821935874569845e-16],[1.34840501909987e-09,0.01813773335587018,0.28475213657989967,0.57722465648675847,0.11945307127131027,0.0004324009577563529,2.1544657823627996e-17],[5.6323172505731617e-11,0.002219612324897426,0.10201180456696632,0.58634433374772488,0.30741023322721317,0.0020140160768747119,6.0692157244932907e-17],[4.4597320432235574e-07,0.46132450385116225,0.4906805610092082,0.04775141071167243,0.0002430725962087889,5.8585439295470229e-09,9.8523540840930361e-26],[3.5860221327456365e-08,0.16945439758164435,0.64721231713995631,0.18131365266857427,0.0020195293961187484,6.7353485083448131e-08,5.6756076677396553e-25],[2.0121216889780881e-06,0.66138336030697675,0.32138599747151642,0.017173767306200446,5.4861831874212305e-05,9.6174334734742499e-10,1.5409525581306306e-26],[2.1502379639642958e-10,0.0052979200386959659,0.15498297004787684,0.59787077239676223,0.24016249280402868,0.0016858444976126537,1.4573488332494023e-16],[4.9098247114181843e-12,0.0003544349817606026,0.032449190007249752,0.41228546502251129,0.54435650933270141,0.010554400650866411,1.1211130630169303e-15],[9.6762164479824873e-07,0.62430148042899114,0.36028712121325857,0.015386140693189472,2.4289946125396497e-05,9.6790592710482581e-11,7.0411206859858179e-29],[9.6762164479824873e-07,0.62430148042899114,0.36028712121325857,0.015386140693189472,2.4289946125396497e-05,9.6790592710482581e-11,7.0411206859858179e-29],[1.3167489370733988e-07,0.31696416834898694,0.60243213314255195,0.080210787743114881,0.00039277403793322457,5.0525192098642632e-09,1.2963184132433268e-26],[1.5662966589234601e-09,0.025424403153538954,0.38924264950497495,0.54354981643853362,0.041766852740128466,1.6276596527174894e-05,4.2777027702726395e-21],[8.4560201048071107e-12,0.00051178482486652951,0.039415158486884455,0.43014882282483674,0.51902926796767623,0.010894965887278328,2.0940112470296492e-15],[1.7922077383244892e-08,0.1143865566037362,0.61805487863527009,0.26260282757196424,0.0049553798429819024,3.3942397021949028e-07,8.6598264182753295e-24],[5.5210527832887754e-07,0.50532672110346988,0.45853487970130746,0.036002949675488144,0.00013489539132883989,2.0231273041831125e-09,1.4711356864677261e-26],[8.152524784958519e-09,0.067114786721342565,0.52641834684540212,0.38920214330781133,0.017259873433718809,4.8415392004720202e-06,1.7071404808497325e-21],[2.8463105236520369e-07,0.37429317522233985,0.54407788944802027,0.080872546580049623,0.00075605704647212398,4.7072065992089792e-08,4.3528411680150505e-24],[1.0319817563531668e-11,0.00059109835744582779,0.042954717141877242,0.44181125201770077,0.50443174078518782,0.010211191687466425,2.0331913585034231e-15],[3.1621884787621838e-09,0.04135790518645379,0.47880758376383964,0.4591294462736108,0.020701496559829065,3.5650540783049743e-06,2.2745591781236728e-22],[1.5406127235336517e-10,0.0039304198431280666,0.12474273046260534,0.56354107510071061,0.30399217379404619,0.0037936006454473842,1.2098169947629123e-15],[2.9018310824136318e-08,0.14072418003303303,0.61832453137758292,0.2362101174496008,0.0047406659138335078,4.7620763893871294e-07,3.7179554540804257e-23],[1.1455858217284213e-07,0.3042223783083256,0.61326633405964071,0.082138413063969781,0.000372756172016445,3.8374652852368073e-09,5.7592373525852269e-27],[3.9677111961168776e-10,0.0089712589221799862,0.22118025586244836,0.62559237165001824,0.14390361618909245,0.0003524969794898505,3.1286176350678039e-18],[1.0331526366343534e-11,0.00068996749869053504,0.05438722629430414,0.52494741769945341,0.41697004933188136,0.0030053391653390555,3.6532361486969507e-17],[2.3617581327391906e-14,5.036178691842913e-06,0.0016421053473363054,0.094376784006913475,0.76775254526944581,0.13622352919738115,2.0739364908621783e-13],[1.1275799205905594e-10,0.0039928002436032299,0.15285648832476334,0.64486618834880272,0.19780215828947278,0.00048236468059990233,1.8415898383934571e-18],[9.9656021352228231e-11,0.0038808698129389196,0.15798153475012958,0.66222598634322416,0.17563850099117448,0.00027310800287651618,3.0907894193275968e-19],[2.5403293579874541e-07,0.35374596159023813,0.55409562751969277,0.091171244272417989,0.00098683521852504738,7.7366190197984178e-08,1.0889468334811029e-23],[3.0069521958185525e-09,0.033866138934274405,0.39977208895014216,0.51537035752676108,0.050941013503574828,5.0398078295545565e-05,1.766141007942555e-19],[9.7204256539854075e-11,0.0030326652096003205,0.11399657224459409,0.57424170968420885,0.3058656760675007,0.002863376696891299,3.1567413069607317e-16],[2.2641143509862678e-09,0.023674342430485627,0.30584706882259638,0.55295798421157982,0.11694860112610857,0.00057200114511526034,8.194051174162481e-17],[5.0825832473516063e-07,0.48811725127408878,0.47144131767887948,0.040270764852868131,0.00017015486300098022,3.0728377799746791e-09,3.0996075697344076e-26],[1.0657897712705689e-10,0.0040075179995701146,0.15871406184961864,0.65806756686531531,0.17890181695297275,0.0003090362259440689,4.7717469031432665e-19],[8.9271925383246584e-14,1.5082794994463424e-05,0.0037047355613072652,0.14957609518949638,0.77238987883790255,0.074314207616163475,4.6669016321941732e-14],[1.5278726946183729e-10,0.0037365399408918628,0.11683476264721572,0.54434788087629482,0.32955167538434882,0.0055291409984575294,3.9101659699076868e-15],[7.1768622123212739e-16,2.0930822300903906e-07,0.00010909660232666675,0.013078257987160816,0.3635658403580706,0.62324659560459483,1.3962347895276163e-10],[2.7088158050128238e-08,0.10493841857475042,0.51873431307417084,0.35083587003274858,0.025456193514321303,3.5177715850754649e-05,8.9213381965588382e-19],[1.641184169689457e-09,0.018622948475469768,0.26808555766398334,0.56326824276461873,0.14896731549021067,0.0010559339645328872,3.2350654302140653e-16],[7.4436130622591972e-14,1.3196601148852167e-05,0.0034095905494518092,0.1446616145415138,0.77698157033375592,0.074934027974013592,4.1588655508740827e-14],[4.3288150071692999e-13,4.8982633509723683e-05,0.0078529111359995542,0.20575275523271888,0.7265107303448689,0.059834620652397037,7.3361678011136144e-14],[5.7471979024694905e-07,0.51373405838775266,0.452087077904887,0.034057984928905161,0.00012030241130218661,1.6473626761915285e-09,1.0209309156151554e-26],[1.9789311597745503e-13,2.4851838041405792e-05,0.00460971300065859,0.14955625429666955,0.7376470211028745,0.10816215976112385,4.336513420625571e-13],[1.8060480105143156e-12,0.00015757098408776042,0.018300854467035346,0.31706414650967307,0.64103138434679496,0.023446043690595494,7.3996728157169326e-15],[5.0356680967623333e-07,0.54051988743360857,0.43712407966922989,0.022320858431989644,3.4670798530968412e-05,9.9831215487200724e-11,2.8142752654563801e-29],[3.9464667261691907e-08,0.17912631351620134,0.64947864719544102,0.16968252554153165,0.0017124251422165114,4.913994211393551e-08,3.1988107492449634e-25],[4.4396089708678641e-09,0.045933871345711808,0.46458540228812312,0.46037966552689125,0.029088136751547745,1.2919648116990063e-05,8.9384292886158999e-21],[6.9923950105628681e-12,0.00044583661120866318,0.036280430363446789,0.41888950944303194,0.53280540427739109,0.011578819297926873,2.1515357147511261e-15],[1.188431506526346e-11,0.00067515409376802326,0.047861757686671343,0.46638502033093771,0.47736763563414314,0.0077104322425946325,9.0821935874569845e-16],[1.6699407525452075e-08,0.089969188426067756,0.53117242553508548,0.35890811962452279,0.019937428114618041,1.282160029835054e-05,4.3993757704245085e-20],[2.2909636539756948e-09,0.028928679385638187,0.3794062555495053,0.53532305156591653,0.056286334688685644,5.5676519290699437e-05,1.654993292048967e-19],[1.0543543454329006e-06,0.52099480765800177,0.43103587522629067,0.047535919807060976,0.00043230350286433078,3.9451436892251173e-08,1.2984441297691396e-23],[6.4447896215496533e-06,0.76106138733380935,0.22961516121474612,0.009291077514880933,2.5928674984145674e-05,4.719579389397652e-10,1.0934079483083897e-26],[6.5008255299294549e-13,7.686362421701888e-05,0.012144899785692644,0.27958549589638848,0.68515067770395055,0.023042062989098441,2.7982628371621894e-15],[2.6415235300966221e-10,0.0072907562645881463,0.21069835035498558,0.64506979359218553,0.13672419355339266,0.00021690597069588504,5.0405589416910501e-19],[6.5078596570171125e-11,0.002331402894101945,0.10012278112695303,0.56743597483129149,0.32710846066296823,0.0030013804196063663,2.412740378536269e-16],[1.934378098832877e-09,0.021062218055912314,0.28707159880572058,0.55904941940996322,0.13204129011254709,0.00077547168147853594,1.6131996491244591e-16],[1.4564654776702037e-05,0.88561111620427835,0.11294436109573527,0.0014291977697968655,7.6027434428550606e-07,1.0688184076204666e-12,2.719579888593846e-31],[2.6909210735673608e-07,0.36406743788039525,0.54921056662429846,0.085858480326571757,0.00086318583018548073,6.0246441532137789e-08,6.8586622623154622e-24],[1.4493626014726365e-11,0.00090373169941948561,0.065176896877110813,0.55804747310633229,0.37379927863674683,0.0020726196658967674,1.5929630312245718e-17],[1.2626841636865627e-11,0.00060079461036422128,0.038379947286876555,0.38818149024751075,0.54941558865700701,0.023422179185570623,4.4007581901491373e-14],[1.5022245656780879e-07,0.29306010088082657,0.58930937665327576,0.11623085421606946,0.0013994092522681659,1.0877510377938806e-07,1.1819174327263249e-23],[1.387105186558293e-09,0.024367156629267261,0.39219766752613122,0.54572830336647482,0.037696207507635686,1.0663583385962776e-05,1.2205795943840063e-21],[7.9002232549541669e-13,8.4135011467168462e-05,0.012167989374690234,0.2672207628310348,0.68940722116273101,0.031119891619276686,1.0056341314308625e-14],[4.6232316352133913e-15,8.6734481405298909e-07,0.00028270447553014764,0.021075383122982416,0.38843558934176031,0.59020545530273383,4.1217503881983774e-10],[2.9091485609271746e-09,0.03175375310357112,0.3788375489764777,0.5265963870804945,0.06271925725545717,9.3050674850843351e-05,8.338169038322621e-19],[2.3037169111639045e-11,0.0011858047030431341,0.072091585015767679,0.54984792349404854,0.37398365962351304,0.0028910271405903657,7.3611564697553309e-17],[2.412811539325776e-08,0.097187317223726227,0.50579823490308018,0.36734737931306066,0.029618362499292662,4.8681932724719146e-05,1.7482161928156943e-18],[1.1989051161434348e-10,0.0038491558660283527,0.14029330475703961,0.61778466600312276,0.23700709257649238,0.0010657806774261561,2.0037784040244418e-17],[2.1844077926402279e-09,0.030338609943517891,0.40714626517857755,0.52209423375937725,0.040401568854490318,1.9320079629094362e-05,1.0167279037132265e-20],[1.8311837355139014e-10,0.0059099580361944683,0.19598616668607463,0.65886544754017562,0.13906301477316776,0.00017541278126927647,1.8030741980628676e-19],[2.6053076155535573e-09,0.037694506782807218,0.4723005136767105,0.46979530353966648,0.020206891136676799,2.7822588316021344e-06,9.3005333778633498e-23],[7.6530523978946236e-11,0.0028724081215362506,0.12193718806668453,0.61533298306683171,0.25872234949893547,0.0011350711694812924,1.4859808809113126e-17],[1.9102931941822549e-11,0.00099033469842548676,0.062288204245612246,0.51553929001690846,0.41655701440262433,0.0046251566173262145,2.7228417802099726e-16],[2.420825472108232e-06,0.69987283251734111,0.28782570715859801,0.012270671539104423,2.8367658732350508e-05,3.007516460079369e-10,1.9662228038362924e-27],[5.0354536505608172e-14,8.8191992391213659e-06,0.0023373243871075882,0.10950704660880224,0.75601162800163546,0.13213518180283088,3.3439785522430967e-13],[5.0616838074568713e-09,0.044784612827088725,0.43121756923726512,0.47908903384501395,0.044857830753813367,5.0948275135166177e-05,3.3479513770860404e-19],[1.7491879249409908e-12,0.00012589454482702414,0.013007537830071835,0.23374249961432614,0.67762847637992474,0.075495591628469944,6.3132333885803103e-13],[8.8758397406635861e-11,0.0032740578365606805,0.13393856777026864,0.62985893231003198,0.23213972889692358,0.00078871309745689241,5.9316243332178816e-18],[1.63768878734271e-06,0.61716131804766561,0.35834285944532918,0.024383929394797711,0.00011025209440793643,3.3290123907250966e-09,1.4230346322358175e-25],[5.1256012637775005e-13,6.100908391304118e-05,0.0099367152383908284,0.24642633993134691,0.70969182267312203,0.033884113072704856,9.4875671399596752e-15],[1.4209200501750731e-09,0.021811543610045597,0.34342857801750548,0.56802182798029954,0.066672125926508424,6.5923044720888287e-05,1.4567212631019065e-19],[3.8196268225899067e-08,0.14275012155761443,0.58279194004539248,0.26511398363609767,0.0093405413613199453,3.3752033070144162e-06,4.9258003906822735e-21],[1.95462980652656e-10,0.0057110037352872496,0.18071401505862089,0.64017493127196745,0.17297735168781764,0.00042269805084374761,2.3480609194092381e-18],[2.2135485452921634e-10,0.0067372768532881046,0.20941301765259721,0.65559293793163353,0.12810741826047189,0.00014934908065457808,1.441038533382766e-19],[2.6538577579445727e-11,0.0014326011776145189,0.08692113701348439,0.6020587316823961,0.30838881442585658,0.0011987156741098061,5.5535455499825103e-18],[8.7684510178432316e-14,1.5156195879599111e-05,0.0037889784265918085,0.15374926757003249,0.77477187166126016,0.067674726146116942,3.1494095776799002e-14],[6.9277467066923923e-11,0.0020750624581920584,0.08142595657636878,0.49369669772111541,0.41220683513330714,0.010595448041724442,1.4667153328220571e-14],[6.5398707113336037e-08,0.19738632805937303,0.60942212305248633,0.18935068704528851,0.0038402182999474916,5.7814419759106959e-07,1.6680096673967973e-22],[6.288997031716213e-08,0.22089268937329043,0.63894077792224646,0.13891634992773746,0.0012500833785163007,3.650823901975599e-08,3.2782909249682205e-25],[3.588857059399092e-08,0.11158702584472546,0.5025674362854714,0.35353858996817328,0.032227221923072938,7.969008998626592e-05,9.1047238536599323e-18],[7.6636104238222977e-14,1.2249692593058928e-05,0.0029320657825615201,0.12285473116880294,0.75494167176421789,0.11925928159144721,3.0051834335217805e-13],[2.2572848909581843e-07,0.41079296611973271,0.54437324632605277,0.044722642979124226,0.00011091833789687731,5.0870422486632887e-10,2.2566567285532467e-28],[1.0895804726894433e-11,0.00065945532920605942,0.048968264152645484,0.48226116677547404,0.46224257352241616,0.0058685402093619993,3.3828679970115435e-16],[5.149636490696521e-08,0.19736044263374106,0.63909361461275394,0.16174032707594385,0.0018054908120266188,7.3369169648311293e-08,1.168874598241484e-24],[1.3667788939098248e-08,0.078767157742087979,0.50801279918088416,0.38750396860378367,0.025694103344425085,2.19574610299033e-05,1.2939387135649462e-19],[1.2247258212416141e-10,0.0040642129961531177,0.14926778947257405,0.63240102101274409,0.21356657564143733,0.00070040075461894286,5.9768914919234468e-18],[1.2812355118795378e-07,0.32191828242700771,0.60423429339842027,0.073555502848898752,0.00029179075200881074,2.4501128424462838e-09,2.6007801695598222e-27],[1.0939977613404981e-07,0.24652714620045668,0.59836361531668691,0.15245312827511956,0.0026556315734304428,3.6923453036015862e-07,1.160073782494074e-22],[1.0007064740203282e-10,0.0031903227426807487,0.1202952479166074,0.58833626480622614,0.28603448081878752,0.0021436836156272163,1.3419341679407941e-16],[5.8421677474191684e-10,0.012246002895594344,0.2669700161705339,0.61860185268323276,0.10204384488910744,0.00013828277731478439,3.691150042539391e-19],[1.0543381755905392e-07,0.29893002335099483,0.61978257450823093,0.080958116856477899,0.00032917724556949513,2.6049091097170697e-09,2.1695607244674915e-27]],"max_posterior":[0.65020538674673567,0.64777619676407538,0.50948434265463238,0.50740818360199247,0.60114963823149914,0.66878274852827879,0.48378522561142112,0.55733139667833009,0.62230735844768714,0.47736763563414314,0.57722465648675847,0.58634433374772488,0.4906805610092082,0.64721231713995631,0.66138336030697675,0.59787077239676223,0.54435650933270141,0.62430148042899114,0.62430148042899114,0.60243213314255195,0.54354981643853362,0.51902926796767623,0.61805487863527009,0.50532672110346988,0.52641834684540212,0.54407788944802027,0.50443174078518782,0.47880758376383964,0.56354107510071061,0.61832453137758292,0.61326633405964071,0.62559237165001824,0.52494741769945341,0.76775254526944581,0.64486618834880272,0.66222598634322416,0.55409562751969277,0.51537035752676108,0.57424170968420885,0.55295798421157982,0.48811725127408878,0.65806756686531531,0.77238987883790255,0.54434788087629482,0.62324659560459483,0.51873431307417084,0.56326824276461873,0.77698157033375592,0.7265107303448689,0.51373405838775266,0.7376470211028745,0.64103138434679496,0.54051988743360857,0.64947864719544102,0.46458540228812312,0.53280540427739109,0.47736763563414314,0.53117242553508548,0.53532305156591653,0.52099480765800177,0.76106138733380935,0.68515067770395055,0.64506979359218553,0.56743597483129149,0.55904941940996322,0.88561111620427835,0.54921056662429846,0.55804747310633229,0.54941558865700701,0.58930937665327576,0.54572830336647482,0.68940722116273101,0.59020545530273383,0.5265963870804945,0.54984792349404854,0.50579823490308018,0.61778466600312276,0.52209423375937725,0.65886544754017562,0.4723005136767105,0.61533298306683171,0.51553929001690846,0.69987283251734111,0.75601162800163546,0.47908903384501395,0.67762847637992474,0.62985893231003198,0.61716131804766561,0.70969182267312203,0.56802182798029954,0.58279194004539248,0.64017493127196745,0.65559293793163353,0.6020587316823961,0.77477187166126016,0.49369669772111541,0.60942212305248633,0.63894077792224646,0.5025674362854714,0.75494167176421789,0.54437324632605277,0.48226116677547404,0.63909361461275394,0.50801279918088416,0.63240102101274409,0.60423429339842027,0.59836361531668691,0.58833626480622614,0.61860185268323276,0.61978257450823093],"proportion_misgenotyped":0.40598613345725731}],"no_read_marker":{"ploidy":4,"allele_depth":[19,0,0,17,46,0,13,39,0,5,0,31,15,0,10,16,0,0,46,4,45,43,3,4,0,19,12,52,0,16,18,27,8,10,38,33,16,49,33,0,0,0,2,0,0,17,9,50,17,9,0,18,32,54,35,22,26,11,25,29,21,17,52,49,5,39,12,50,44,37,41,14,15,16,12,23,0,29,17,19,8,12,0,18,12,10,21,22,18,0,7,12,21,0,0,9,63,30,0,0],"marker_depth":[23,61,0,77,86,49,59,50,24,32,76,69,49,41,48,90,0,0,69,43,88,90,21,38,78,46,74,70,72,69,79,63,15,31,68,66,68,77,62,0,0,0,60,76,90,60,35,87,59,34,79,57,52,84,56,39,39,36,70,48,81,39,85,67,73,89,28,71,49,76,62,75,61,66,29,27,0,67,69,47,68,43,26,47,72,57,25,35,80,0,53,43,42,78,85,18,80,88,60,54],"prior":[0.15902351099425685,0.40461705182323604,0.33663316326311454,0.091579757240208851,0.0081465166791837487],"prior_given_na":[0.1590235103022333,0.40461705186587721,0.33663316386988712,0.091579757307937257,0.0081465166540650412],"max_posterior":[0.83760255639080905,0.99972527265764677,0.40461705272867682,0.9802912042486428,0.95040055833809389,0.99916063156758705,0.97347101897306398,0.87170085467924685,0.98241843251727456,0.98352673479305264,0.99991697238459498,0.83507111561206604,0.80821590201239091,0.99803873664142173,0.97407883193898792,0.99558797707985813,0.40461705272867682,0.40461705272867682,0.62970935003006367,0.99826049617703849,0.94582955680959613,0.91317237859198175,0.96959204297884749,0.99656642787264882,0.99992825411098651,0.67322695439186753,0.99630565744489397,0.79317065270313269,0.99988782316271063,0.97012250774384323,0.97668415850909296,0.75960297690018819,0.77807913154378305,0.71502453363801877,0.93349501020210612,0.92377460943523992,0.9668885884343843,0.77226170130296923,0.93645558092171677,0.40461705272867682,0.40461705272867682,0.40461705272867682,0.99862127473214113,0.99991697238459498,0.99996843683170833,0.88772861185475671,0.9000444362439014,0.92775466375045235,0.87480230454807639,0.88347490601459411,0.99993323613828733,0.78469891705932393,0.83648689857420322,0.74363468721867243,0.81094091912471378,0.89930130284107468,0.66211307285839471,0.78481786641736562,0.59861153460393979,0.85961264700015727,0.94742404512215417,0.74333729036659402,0.85682970409470705,0.73426166704724583,0.99980400538221725,0.81757664919283357,0.68458159593725298,0.59355285304147387,0.9933053737228581,0.91720269570103019,0.66313553817469506,0.99235481256841396,0.95219225747243008,0.95919784622472704,0.63834778561943484,0.91752568772608423,0.40461705272867682,0.77995428711420944,0.9567774150022792,0.63671533090281418,0.99896419602039432,0.87332834203765519,0.98687121311457404,0.53641193444511226,0.9955387417174254,0.99134578229740811,0.88440963252359528,0.79257512913663586,0.97873390796171733,0.40461705272867682,0.9970139733212382,0.87332834203765519,0.88968336859373609,0.99992825411098651,0.99995604013267281,0.77700775641882824,0.93722515610871659,0.69389461753555937,0.99970030458416781,0.99948355293798408],"proportion_misgenotyped":0.15483878611784863,"proportion_misgenotyped_given_na":0.11653059803291843}} diff --git a/tests/test_call.py b/tests/test_call.py new file mode 100644 index 0000000..c0d8987 --- /dev/null +++ b/tests/test_call.py @@ -0,0 +1,1996 @@ +"""Tests of ``madcats call`` (madcats_call.py): the genotype model, then the command. + +The parity tests compare against values computed by polyRAD and updog and +stored in tests/fixtures/ (made by scripts/make_polyrad_fixture.R and +scripts/make_updog_fixture.R), so they run without R. +""" + +import contextlib +import csv +import dataclasses +import io +import itertools +import json +import math +import random +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import madcats # noqa: E402 +import madcats_call as call # noqa: E402 + +EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "madcats-demo" +POLYRAD_FIXTURE_PATH = Path(__file__).resolve().parent / "fixtures" / "polyrad_parity.json" +UPDOG_FIXTURE_PATH = Path(__file__).resolve().parent / "fixtures" / "updog_norm.json" + + +def load_polyrad_fixture(): + with open(POLYRAD_FIXTURE_PATH) as handle: + fixture = json.load(handle) + fixture["sample_index"] = {name: index for index, name in enumerate(fixture["samples"])} + fixture["allele_index"] = {name: index for index, name in enumerate(fixture["alleles"])} + # Columns of the depth table that belong to each locus, in fixture order. + fixture["locus_columns"] = {} + for column, locus in enumerate(fixture["allele_locus"]): + fixture["locus_columns"].setdefault(locus, []).append(column) + return fixture + + +class LikelihoodTests(unittest.TestCase): + def test_likelihoods_of_all_possible_counts_sum_to_one(self): + settings = call.ModelSettings() + marker_depth = 25 + for dosage in range(5): + total = sum( + math.exp(call.allele_log_likelihoods(allele_depth, marker_depth, 4, 0.3, settings)[dosage]) + for allele_depth in range(marker_depth + 1) + ) + self.assertAlmostEqual(total, 1.0, places=10) + + def test_no_reads_gives_equal_likelihoods(self): + self.assertEqual(call.allele_log_likelihoods(0, 0, 4, 0.3, call.ModelSettings()), [0.0] * 5) + + def test_very_deep_coverage_stays_finite(self): + log_likelihoods = call.allele_log_likelihoods(2000000, 2000000, 6, 0.5, call.ModelSettings()) + self.assertTrue(all(math.isfinite(value) for value in log_likelihoods)) + self.assertEqual(log_likelihoods, sorted(log_likelihoods)) + self.assertLess(log_likelihoods[0], -100) + + def test_without_contamination_impossible_dosages_have_zero_likelihood(self): + settings = call.ModelSettings(contamination_rate=0.0) + log_likelihoods = call.allele_log_likelihoods(3, 10, 2, 0.5, settings) + self.assertEqual(log_likelihoods[0], call.NEGATIVE_INFINITY) + self.assertEqual(log_likelihoods[2], call.NEGATIVE_INFINITY) + self.assertTrue(math.isfinite(log_likelihoods[1])) + + +class PriorAndPosteriorTests(unittest.TestCase): + def test_hwe_prior_is_binomial(self): + self.assertEqual(call.hwe_prior(2, 0.25), [0.5625, 0.375, 0.0625]) + self.assertAlmostEqual(sum(call.hwe_prior(6, 0.37)), 1.0) + + def test_selfing_moves_probability_to_the_homozygous_dosages(self): + for ploidy in (2, 4, 6): + random_mating = call.hwe_prior(ploidy, 0.3) + previous = random_mating + for selfing_rate in (0.2, 0.6, 0.95): + prior = call.hwe_prior(ploidy, 0.3, selfing_rate) + self.assertAlmostEqual(sum(prior), 1.0, places=12) + # Selfing changes genotype proportions, not the allele frequency. + self.assertAlmostEqual(call.posterior_mean_dosage(prior) / ploidy, 0.3, places=12) + self.assertGreater(prior[0], previous[0]) + self.assertGreater(prior[ploidy], previous[ploidy]) + previous = prior + self.assertEqual(call.hwe_prior(ploidy, 0.3, 1.0), [0.7] + [0.0] * (ploidy - 1) + [0.3]) + + def test_selfing_is_not_applied_to_odd_ploidy(self): + self.assertEqual(call.hwe_prior(3, 0.3, 0.5), call.hwe_prior(3, 0.3)) + self.assertEqual(call.hwe_prior(3, 0.3, 1.0), [0.7, 0.0, 0.0, 0.3]) + + def test_gamete_dosages(self): + # A tetraploid with two copies passes on 0, 1 or 2 of them in 1, 4 and 1 of 6 gametes. + self.assertEqual(call.gamete_dosage_probabilities(2, 4), [1 / 6, 4 / 6, 1 / 6]) + self.assertEqual(call.gamete_dosage_probabilities(1, 2), [0.5, 0.5]) + + def test_posterior_falls_back_to_prior_when_nothing_is_possible(self): + prior = [1.0, 0.0, 0.0] + self.assertEqual(call.dosage_posterior([call.NEGATIVE_INFINITY, -1.0, -2.0], prior), prior) + + def test_posterior_survives_extreme_log_likelihoods(self): + posterior = call.dosage_posterior([-5000.0, -5002.0, -9000.0], [1 / 3] * 3) + self.assertAlmostEqual(sum(posterior), 1.0) + self.assertAlmostEqual(posterior[0] / posterior[1], math.exp(2.0)) + + +class AllelicBiasTests(unittest.TestCase): + def test_expected_read_fraction_with_bias_and_background_reads(self): + # Two of four copies, no bias: half the reads. + self.assertEqual(call.expected_read_fraction(2, 4, 0.3, 0.0), 0.5) + # A bias of 3 means a copy of the other allele yields three times as many reads. + self.assertAlmostEqual(call.expected_read_fraction(2, 4, 0.3, 0.0, bias=3.0), 0.25) + self.assertAlmostEqual(call.expected_read_fraction(1, 4, 0.3, 0.0, bias=0.5), 0.25 / (0.5 * 0.75 + 0.25)) + # Background reads are added after the bias, so a sample without the allele shows + # the background rate whatever the bias. + for bias in (0.2, 1.0, 5.0): + self.assertAlmostEqual(call.expected_read_fraction(0, 4, 0.3, 0.0, bias, 0.01), 0.01) + self.assertAlmostEqual(call.expected_read_fraction(4, 4, 0.3, 0.0, bias, 0.01), 0.99) + self.assertAlmostEqual(call.expected_read_fraction(2, 4, 0.3, 0.0, 3.0, 0.01), 0.25 * 0.99 + 0.75 * 0.01) + + def test_bias_is_estimated_and_corrects_the_calls(self): + table = simulated_two_allele_markers(60, 4, markers=1, samples=150, bias=0.5, seed=21)[0] + ploidies = [4] * 150 + without = call.call_record(table, ploidies, call.ModelSettings(overdispersion=60)) + fitted = call.call_record(table, ploidies, call.ModelSettings(overdispersion=60, fit_allele_bias=True)) + self.assertEqual(without.allele_biases, [1.0, 1.0]) + self.assertIsNone(without.bias_fit) + self.assertAlmostEqual(fitted.allele_biases[1], 0.5, delta=0.06) + # The reference allele's bias is the reciprocal, and its results are the mirror image. + self.assertAlmostEqual(fitted.allele_biases[0] * fitted.allele_biases[1], 1.0, places=12) + self.assertEqual(fitted.posteriors[0][0], list(reversed(fitted.posteriors[0][1]))) + self.assertAlmostEqual(math.exp(fitted.bias_fit.log_bias), fitted.allele_biases[1], places=12) + self.assertFalse(fitted.bias_fit.at_limit) + # With the bias ignored, the alternate allele's excess of reads is read as extra copies. + def mean_dosage(result): + return sum(call.posterior_mean_dosage(sample[1]) for sample in result.posteriors) / 150 + + self.assertGreater(mean_dosage(without), mean_dosage(fitted) + 0.2) + + def test_fit_is_the_best_of_all_the_bias_values_tried(self): + table = simulated_two_allele_markers(60, 4, markers=1, samples=100, bias=1.6, seed=5)[0] + settings = call.ModelSettings(overdispersion=60, fit_allele_bias=True) + fitted = call.call_record(table, [4] * 100, settings) + chosen = fitted.bias_fit + self.assertGreaterEqual(chosen.fits, 15) + self.assertIn(chosen.nearest_log_start, call.BIAS_LOG_STARTS) + # No fixed bias on a fine grid gives a higher value of what the fit maximises. + marker_depths = [sum(sample) for sample in table] + alternate = [[sample[1]] for sample in table] + constants = [[call.count_constant(sample[1], sum(sample), 60)] for sample in table] + for step in range(-20, 21): + log_bias = step / 10 + at_bias = call._iterate_record( + alternate, marker_depths, constants, [4] * 100, settings, + [call.depth_ratio_frequencies(table)[1]], [math.exp(log_bias)], + ) + log_likelihood = call.marginal_log_likelihood(at_bias.log_likelihoods, at_bias.priors, [4] * 100) + objective = log_likelihood - log_bias**2 / (2 * call.BIAS_LOG_PRIOR_SD**2) + self.assertLessEqual(objective, chosen.objective + 1e-3) + + def test_samples_without_the_allele_say_nothing_about_its_bias(self): + # Every sample is a Ref homozygote, and stray Alt reads are far rarer than the + # background rate. Nothing here bears on the bias, so the prior keeps it at 1. + table = [[150, 0] for _ in range(60)] + [[149, 1] for _ in range(3)] + settings = call.ModelSettings( + overdispersion=60, contamination_rate=0.0, background_rate=0.005, fit_allele_bias=True + ) + fitted = call.call_record(table, [4] * 63, settings) + self.assertAlmostEqual(fitted.allele_biases[1], 1.0, delta=0.02) + + def test_records_with_more_than_two_alleles_get_no_bias(self): + depths = [[20, 10, 5], [0, 30, 6], [14, 0, 20]] * 5 + result = call.call_record(depths, [4] * 15, call.ModelSettings(overdispersion=40, fit_allele_bias=True)) + self.assertEqual(result.allele_biases, [1.0, 1.0, 1.0]) + self.assertIsNone(result.bias_fit) + + def test_a_bias_is_used_only_where_it_gains_the_margin(self): + ploidies = [4] * 150 + always = call.ModelSettings(overdispersion=60, fit_allele_bias=True) + with_margin = call.ModelSettings( + overdispersion=60, fit_allele_bias=True, min_bias_gain=call.BIAS_AUTO_MIN_GAIN + ) + # A marker simulated with a strong bias gains far more than the margin and is called with its fit. + biased = simulated_two_allele_markers(60, 4, markers=1, samples=150, bias=0.5, seed=21)[0] + with_bias = call.call_record(biased, ploidies, with_margin) + self.assertGreater(with_bias.bias_fit.gain, call.BIAS_AUTO_MIN_GAIN) + self.assertTrue(with_bias.bias_fit.used) + self.assertEqual(with_bias.allele_biases, call.call_record(biased, ploidies, always).allele_biases) + # A marker simulated without bias is fitted a bias near 1 that gains little. The + # record is then called exactly as it is without a bias, and the fit is still reported. + unbiased = simulated_two_allele_markers(60, 4, markers=1, samples=150, seed=9)[0] + fitted = call.call_record(unbiased, ploidies, always) + self.assertNotEqual(fitted.allele_biases, [1.0, 1.0]) + self.assertLess(fitted.bias_fit.gain, call.BIAS_AUTO_MIN_GAIN) + self.assertTrue(fitted.bias_fit.used) + unused = call.call_record(unbiased, ploidies, with_margin) + without = call.call_record(unbiased, ploidies, call.ModelSettings(overdispersion=60)) + self.assertEqual(unused.allele_biases, [1.0, 1.0]) + self.assertEqual(unused.bias_fit, fitted.bias_fit._replace(rejection=call.BIAS_REJECTED_FOR_GAIN)) + self.assertFalse(unused.bias_fit.used) + self.assertEqual(unused.posteriors, without.posteriors) + self.assertEqual(unused.allele_frequencies, without.allele_frequencies) + + def test_a_bias_at_a_limit_of_the_search_is_not_used_when_that_is_asked(self): + # Inbred samples: none is heterozygous, so a bias has nothing to correct. Under a + # prior that expects heterozygotes the fit runs to a limit of the search, where + # samples with all their reads on one allele pass for one to three copies. + table = simulated_two_allele_markers( + 40, 4, markers=1, samples=150, genotypes="inbred", background_rate=0.005, seed=3 + )[0] + ploidies = [4] * 150 + with_margin = dict( + overdispersion=40, background_rate=0.005, fit_allele_bias=True, min_bias_gain=call.BIAS_AUTO_MIN_GAIN + ) + + def called_heterozygous(result): + dosages = [max(range(5), key=lambda dosage: sample[1][dosage]) for sample in result.posteriors] + return sum(0 < dosage < 4 for dosage in dosages) + + used = call.call_record(table, ploidies, call.ModelSettings(**with_margin)) + self.assertTrue(used.bias_fit.at_limit) + self.assertGreater(used.bias_fit.gain, call.BIAS_AUTO_MIN_GAIN) + self.assertTrue(used.bias_fit.used) + self.assertGreater(called_heterozygous(used), 30) + # With the rule the fit is reported as it was and the record is called without a bias. + rejected = call.call_record(table, ploidies, call.ModelSettings(**with_margin, reject_bias_at_limit=True)) + without = call.call_record(table, ploidies, call.ModelSettings(overdispersion=40, background_rate=0.005)) + self.assertEqual(rejected.bias_fit, used.bias_fit._replace(rejection=call.BIAS_REJECTED_AT_LIMIT)) + self.assertFalse(rejected.bias_fit.used) + self.assertEqual(rejected.allele_biases, [1.0, 1.0]) + self.assertEqual(rejected.posteriors, without.posteriors) + self.assertLess(called_heterozygous(rejected), 3) + # A fitted bias inside the limits is not touched by the rule. + biased = simulated_two_allele_markers(60, 4, markers=1, samples=150, bias=0.5, seed=21)[0] + inside = call.call_record( + biased, ploidies, + call.ModelSettings(overdispersion=60, fit_allele_bias=True, min_bias_gain=call.BIAS_AUTO_MIN_GAIN, + reject_bias_at_limit=True), + ) + self.assertFalse(inside.bias_fit.at_limit) + self.assertTrue(inside.bias_fit.used) + + def test_a_record_without_its_bias_is_called_under_the_read_model_without_a_bias(self): + ploidies = [4] * 150 + with_bias = dict(overdispersion=60, background_rate=0.001, fit_allele_bias=True, + min_bias_gain=call.BIAS_AUTO_MIN_GAIN, reject_bias_at_limit=True) + two_models = call.ModelSettings(**with_bias, read_model_without_bias=(25.0, 0.004)) + plain = call.ModelSettings(overdispersion=25.0, background_rate=0.004) + self.assertEqual(two_models.without_bias(), plain) + self.assertEqual( + call.ModelSettings(**with_bias).without_bias(), call.ModelSettings(overdispersion=60, background_rate=0.001) + ) + # A record that gives its bias up is exactly the record of a run that fits none. + unbiased = simulated_two_allele_markers(60, 4, markers=1, samples=150, seed=9)[0] + rejected = call.call_record(unbiased, ploidies, two_models) + without = call.call_record(unbiased, ploidies, plain) + self.assertEqual(rejected.bias_fit.rejection, call.BIAS_REJECTED_FOR_GAIN) + self.assertEqual(rejected.posteriors, without.posteriors) + self.assertEqual(rejected.log_likelihoods, without.log_likelihoods) + self.assertEqual(rejected.allele_frequencies, without.allele_frequencies) + # The fit itself, and so the decision, is made under the read model with the bias. + one_model = call.ModelSettings(**with_bias) + self.assertEqual(rejected.bias_fit, call.call_record(unbiased, ploidies, one_model).bias_fit) + # A record that keeps its bias does not see the other read model. + biased = simulated_two_allele_markers(60, 4, markers=1, samples=150, bias=0.5, seed=21)[0] + kept = call.call_record(biased, ploidies, two_models) + self.assertTrue(kept.bias_fit.used) + self.assertEqual(kept.posteriors, call.call_record(biased, ploidies, one_model).posteriors) + # A record with more than two alleles gets no bias, and is called without one. + three_alleles = [[20, 10, 5], [0, 30, 6], [14, 0, 20]] * 5 + self.assertEqual( + call.call_record(three_alleles, [4] * 15, two_models).posteriors, + call.call_record(three_alleles, [4] * 15, plain).posteriors, + ) + + def test_the_margin_cannot_be_negative(self): + with self.assertRaisesRegex(ValueError, "min_bias_gain"): + call.ModelSettings(fit_allele_bias=True, min_bias_gain=-1.0) + + +class NormalPriorTests(unittest.TestCase): + def test_prior_is_a_normal_density_at_the_dosages(self): + prior = call.normal_prior(4, 1.5, 0.8) + self.assertAlmostEqual(sum(prior), 1.0, places=12) + # Ratio of two normal densities one dosage apart. + self.assertAlmostEqual(prior[2] / prior[1], math.exp(-(0.5**2 - 0.5**2) / (2 * 0.8**2)), places=12) + self.assertAlmostEqual(prior[3] / prior[2], math.exp(-(1.5**2 - 0.5**2) / (2 * 0.8**2)), places=12) + + def test_very_narrow_and_very_wide_priors(self): + self.assertEqual(call.normal_prior(4, 2.0, call.NORMAL_PRIOR_MIN_SD), [0.0, 0.0, 1.0, 0.0, 0.0]) + for probability in call.normal_prior(4, 2.0, call.NORMAL_PRIOR_MAX_SD): + self.assertAlmostEqual(probability, 0.2, places=5) + + def test_fit_recovers_a_normal_shaped_distribution(self): + for ploidy, mean, standard_deviation in ((2, 0.7, 0.6), (4, 1.3, 0.9), (6, 4.4, 1.7), (4, 2.0, 0.35)): + weights = [150 * probability for probability in call.normal_prior(ploidy, mean, standard_deviation)] + fitted_mean, fitted_sd = call.fit_normal_prior(weights, ploidy, ploidy / 2, 1.0) + self.assertAlmostEqual(fitted_mean, mean, places=6) + self.assertAlmostEqual(fitted_sd, standard_deviation, places=6) + + def test_fit_is_at_least_as_good_as_any_point_of_a_grid(self): + generator = random.Random(4) + for ploidy in (2, 4, 6): + for _ in range(12): + weights = [generator.random() ** 3 * 100 for _ in range(ploidy + 1)] + shares = [weight / sum(weights) for weight in weights] + mean, standard_deviation = call.fit_normal_prior(weights, ploidy, ploidy / 2, 1.0) + self.assertTrue(-1.0 <= mean <= ploidy + 1.0) + self.assertTrue(call.NORMAL_PRIOR_MIN_SD <= standard_deviation <= call.NORMAL_PRIOR_MAX_SD) + fitted = call.weighted_log_normal_prior(shares, ploidy, mean, standard_deviation) + for mean_step in range(41): + for sd_step in range(61): + grid_mean = -1.0 + (ploidy + 2.0) * mean_step / 40 + grid_sd = math.exp(math.log(1e-3) + math.log(1e6) * sd_step / 60) + on_grid = call.weighted_log_normal_prior(shares, ploidy, grid_mean, grid_sd) + self.assertLessEqual(on_grid, fitted + 1e-9) + + def test_fit_handles_weights_on_one_dosage_or_on_the_two_ends(self): + mean, standard_deviation = call.fit_normal_prior([100, 0, 0, 0, 0], 4, 2.0, 1.0) + self.assertAlmostEqual(call.normal_prior(4, mean, standard_deviation)[0], 1.0, places=9) + # Two peaks cannot be matched by one normal shape; the best it can do is flat. + mean, standard_deviation = call.fit_normal_prior([50, 0, 0, 0, 50], 4, 2.0, 1.0) + for probability in call.normal_prior(4, mean, standard_deviation): + self.assertAlmostEqual(probability, 0.2, places=4) + + def test_fit_is_never_worse_than_its_starting_values(self): + weights = [0.0, 0.0, 6.6, 63.4, 9.9, 0.1, 0.0] + shares = [weight / sum(weights) for weight in weights] + start = call.fit_normal_prior(weights, 6, 3.0, 1.0) + again = call.fit_normal_prior(weights, 6, *start) + self.assertGreaterEqual( + call.weighted_log_normal_prior(shares, 6, *again), call.weighted_log_normal_prior(shares, 6, *start) - 1e-13 + ) + self.assertAlmostEqual(again[0], start[0], places=7) + self.assertAlmostEqual(again[1], start[1], places=7) + + def test_fit_leaves_a_start_that_is_almost_entirely_on_one_dosage(self): + # 2% of the samples have dosage 1. Each start puts next to nothing there, so the + # fit has almost no curvature to work with at its first step. The prior's two + # parameters can match three dosages exactly, so the best fit reproduces the shares. + weights = [0.0003, 0.27, 14.0] + shares = [weight / sum(weights) for weight in weights] + for start_mean, start_sd in ((1.98, 0.137), (2.0, 0.05), (0.0, 0.1), (3.0, 0.2)): + with self.subTest(start_mean=start_mean, start_sd=start_sd): + mean, standard_deviation = call.fit_normal_prior(weights, 2, start_mean, start_sd) + for probability, share in zip(call.normal_prior(2, mean, standard_deviation), shares): + self.assertAlmostEqual(probability, share, places=6) + + +class NormalPriorMarkerTests(unittest.TestCase): + def test_inbred_tetraploids_keep_their_homozygous_calls(self): + # Every sample is homozygous. The Hardy-Weinberg prior expects mostly intermediate + # dosages at this allele frequency; the fitted normal prior becomes flat instead. + table = simulated_two_allele_markers(40, 4, markers=1, samples=80, genotypes="inbred")[0] + norm = call.call_record(table, [4] * 80, call.ModelSettings(prior=call.PRIOR_NORM, overdispersion=40)) + self.assertTrue(norm.converged) + prior = norm.priors[4][1] + self.assertLess(max(prior) - min(prior), 0.05) + hwe = call.call_record(table, [4] * 80, call.ModelSettings(prior=call.PRIOR_HWE, overdispersion=40)) + self.assertLess(hwe.priors[4][1][0] + hwe.priors[4][1][4], 0.3) + + def test_each_ploidy_group_gets_its_own_fit(self): + diploids = simulated_two_allele_markers(40, 2, markers=1, samples=30, seed=3)[0] + hexaploids = simulated_two_allele_markers(40, 6, markers=1, samples=30, seed=5)[0] + settings = call.ModelSettings(prior=call.PRIOR_NORM, overdispersion=40) + together = call.call_record(diploids + hexaploids, [2] * 30 + [6] * 30, settings) + self.assertEqual(sorted(together.normal_prior_parameters), [2, 6]) + self.assertEqual([len(together.priors[2][1]), len(together.priors[6][1])], [3, 7]) + # The groups share only the allele frequency used by the contamination term, so a + # group fitted with the other present is almost the group fitted alone. + alone = call.call_record(hexaploids, [6] * 30, settings) + for with_others, by_itself in zip(together.priors[6][1], alone.priors[6][1]): + self.assertAlmostEqual(with_others, by_itself, places=3) + + def test_small_ploidy_group_uses_hardy_weinberg_proportions(self): + tetraploids = simulated_two_allele_markers(40, 4, markers=1, samples=40, seed=7)[0] + triploids = [[20, 9], [31, 0], [8, 19]] + result = call.call_record( + tetraploids + triploids, [4] * 40 + [3] * 3, call.ModelSettings(prior=call.PRIOR_NORM, overdispersion=40) + ) + self.assertEqual(sorted(result.normal_prior_parameters), [4]) + # The triploids' prior is binomial for an allele frequency close to the final estimate. + binomial = call.hwe_prior(3, result.allele_frequencies[1]) + for used, expected in zip(result.priors[3][1], binomial): + self.assertAlmostEqual(used, expected, places=3) + + +class UpdogParityTests(unittest.TestCase): + """The normal-shaped prior of madcats_call against updog's norm model.""" + + @classmethod + def setUpClass(cls): + with open(UPDOG_FIXTURE_PATH) as handle: + cls.fixture = json.load(handle) + + def test_prior_fits_match_updog(self): + cases = self.fixture["prior_fits"] + self.assertEqual(len(cases), 14) + for case in cases: + ploidy, weights = case["ploidy"], case["weights"] + mean, standard_deviation = call.fit_normal_prior(weights, ploidy, ploidy / 2, 1.0) + for value, expected in zip(call.normal_prior(ploidy, mean, standard_deviation), case["prior"]): + self.assertAlmostEqual(value, expected, places=4, msg=str(weights)) + # Where the weights are flat or two-peaked, many parameter values give the same + # prior, so the comparison is on how well each fits, not on the parameters. + shares = [weight / sum(weights) for weight in weights] + ours = call.weighted_log_normal_prior(shares, ploidy, mean, standard_deviation) + theirs = call.weighted_log_normal_prior(shares, ploidy, case["mean"], case["standard_deviation"]) + self.assertGreaterEqual(ours, theirs - 1e-9, msg=str(weights)) + + def test_complete_fits_match_updog(self): + fixture = self.fixture + # updog's rho is 1 / (overdispersion + 1); the fixture stores the value on our scale. + settings = call.ModelSettings( + prior=call.PRIOR_NORM, overdispersion=fixture["overdispersion"], contamination_rate=0.0, + tolerance=1e-10, max_iterations=5000, + ) + self.assertEqual(len(fixture["markers"]), 4) + for marker in fixture["markers"]: + ploidy = marker["ploidy"] + depths = [ + [allele_depth, marker_depth - allele_depth] + for allele_depth, marker_depth in zip(marker["allele_depth"], marker["marker_depth"]) + ] + result = call.call_record(depths, [ploidy] * len(depths), settings) + self.assertTrue(result.converged, marker["name"]) + for value, expected in zip(result.priors[ploidy][0], marker["prior"]): + self.assertAlmostEqual(value, expected, places=4, msg=marker["name"]) + for sample, expected_posterior in enumerate(marker["posteriors"]): + # updog keeps a sequencing-error rate of at least 1e-6, so a dosage of 0 keeps a + # little probability (up to 5e-4 here) for a sample with a few reads of the allele. + for value, expected in zip(result.posteriors[sample][0], expected_posterior): + self.assertAlmostEqual(value, expected, delta=1e-3, msg=f"{marker['name']} sample {sample}") + self.assertEqual( + call.most_probable_dosage(result.posteriors[sample][0]), marker["genotypes"][sample], + msg=f"{marker['name']} sample {sample}", + ) + + + def test_fitted_bias_matches_updog(self): + fixture = self.fixture + self.assertEqual(len(fixture["bias_markers"]), 10) + for marker in fixture["bias_markers"]: + ploidy = marker["ploidy"] + settings = call.ModelSettings( + prior={"norm": call.PRIOR_NORM, "hw": call.PRIOR_HWE}[marker["model"]], + overdispersion=fixture["overdispersion"], contamination_rate=0.0, fit_allele_bias=True, + tolerance=1e-8, max_iterations=5000, + ) + depths = [ + [allele_depth, marker_depth - allele_depth] + for allele_depth, marker_depth in zip(marker["allele_depth"], marker["marker_depth"]) + ] + result = call.call_record(depths, [ploidy] * len(depths), settings) + label = f"{marker['name']} {marker['model']}" + # updog reports the bias of the allele it was given the read counts of: here the first. + self.assertAlmostEqual(result.allele_biases[0] / marker["bias"], 1.0, delta=0.01, msg=label) + for value, expected in zip(result.priors[ploidy][0], marker["prior"]): + self.assertAlmostEqual(value, expected, places=3, msg=label) + for sample, expected_posterior in enumerate(marker["posteriors"]): + for value, expected in zip(result.posteriors[sample][0], expected_posterior): + self.assertAlmostEqual(value, expected, delta=2e-3, msg=f"{label} sample {sample}") + self.assertEqual( + call.most_probable_dosage(result.posteriors[sample][0]), marker["genotypes"][sample], + msg=f"{label} sample {sample}", + ) + + def written_mpp_and_pmc(self, marker, settings): + """MPP per sample (None where it is written as missing) and PMC, from the record ``call`` writes.""" + depths = [ + [allele_depth, marker_depth - allele_depth] + for allele_depth, marker_depth in zip(marker["allele_depth"], marker["marker_depth"]) + ] + options = call.CallOptions(settings=settings, vcf_types=(call.VCF_TARGET,)) + line, _, _ = call.format_record( + "M1", 12, "M1", ["G", "C"], depths, None, [marker["ploidy"]] * len(depths), options, two_allele_model=True + ) + fields = line.split("\t") + mpp_column = fields[8].split(":").index("MPP") + mpp = [None if value == "." else float(value) for value in (cell.split(":")[mpp_column] for cell in fields[9:])] + pmc = float(dict(item.split("=") for item in fields[7].split(";") if "=" in item)["PMC"]) + return mpp, pmc + + def test_mpp_and_pmc_match_updog(self): + """MPP is updog's maxpostprob and PMC its prop_mis, where the two models are the same.""" + fixture = self.fixture + fits = [ + (marker, call.ModelSettings( + prior=call.PRIOR_NORM, overdispersion=fixture["overdispersion"], contamination_rate=0.0, + tolerance=1e-10, max_iterations=5000, + )) + for marker in fixture["markers"] + ] + [ + (marker, call.ModelSettings( + prior={"norm": call.PRIOR_NORM, "hw": call.PRIOR_HWE}[marker["model"]], + overdispersion=fixture["overdispersion"], contamination_rate=0.0, fit_allele_bias=True, + tolerance=1e-8, max_iterations=5000, + )) + for marker in fixture["bias_markers"] + ] + self.assertEqual(len(fits), 14) + for marker, settings in fits: + label = f"{marker['name']} {marker.get('model', 'norm')}" + mpp, pmc = self.written_mpp_and_pmc(marker, settings) + # updog's own two values are tied as its documentation says. + self.assertAlmostEqual( + marker["proportion_misgenotyped"], 1 - sum(marker["max_posterior"]) / len(mpp), places=12, msg=label + ) + for sample, (value, expected) in enumerate(zip(mpp, marker["max_posterior"])): + self.assertAlmostEqual(value, expected, delta=2e-3, msg=f"{label} sample {sample}") + self.assertAlmostEqual(pmc, marker["proportion_misgenotyped"], delta=5e-4, msg=label) + + def test_pmc_is_over_the_samples_with_reads_as_updog_gives_for_missing_samples(self): + marker = self.fixture["no_read_marker"] + settings = call.ModelSettings( + prior=call.PRIOR_NORM, overdispersion=self.fixture["overdispersion"], contamination_rate=0.0, + tolerance=1e-10, max_iterations=5000, + ) + mpp, pmc = self.written_mpp_and_pmc(marker, settings) + without_reads = [sample for sample, depth in enumerate(marker["marker_depth"]) if depth == 0] + self.assertEqual(len(without_reads), 8) + for sample, (value, expected) in enumerate(zip(mpp, marker["max_posterior"])): + if sample in without_reads: + self.assertIsNone(value) + else: + self.assertAlmostEqual(value, expected, delta=2e-3, msg=f"sample {sample}") + # updog told that those samples are missing (NA) leaves them out, as call does. + self.assertAlmostEqual(pmc, marker["proportion_misgenotyped_given_na"], delta=5e-4) + # updog given a depth of 0 keeps them, each at the largest probability of the prior, + # which raises its prop_mis. + largest_prior = max(marker["prior"]) + for sample in without_reads: + self.assertAlmostEqual(marker["max_posterior"][sample], largest_prior, places=6) + samples = len(marker["marker_depth"]) + with_reads = samples - len(without_reads) + self.assertAlmostEqual( + marker["proportion_misgenotyped"], + (with_reads * marker["proportion_misgenotyped_given_na"] + len(without_reads) * (1 - largest_prior)) + / samples, + places=6, + ) + self.assertGreater(marker["proportion_misgenotyped"], pmc + 0.02) + + +class MarkerTests(unittest.TestCase): + def test_marker_without_reads_is_not_callable(self): + result = call.call_record([[0, 0], [0, 0]], [2, 2]) + self.assertFalse(result.has_reads) + + def test_two_allele_marginals_mirror_each_other(self): + depths = [[30, 0], [14, 17], [0, 25], [22, 9], [5, 31]] + ploidies = [2, 4, 4, 6, 3] + result = call.call_record(depths, ploidies) + for sample_posteriors in result.posteriors: + first, second = sample_posteriors + for value, mirrored in zip(first, reversed(second)): + self.assertAlmostEqual(value, mirrored, places=12) + + def test_clear_diploid_genotypes_are_called(self): + depths = [[40, 0], [21, 19], [0, 38], [18, 22], [35, 0], [0, 41]] + result = call.call_record(depths, [2] * 6) + self.assertTrue(result.converged) + called = [call.most_probable_dosage(sample[1]) for sample in result.posteriors] + self.assertEqual(called, [0, 1, 2, 1, 0, 2]) + + def test_iteration_limit_is_reported(self): + depths = [[9, 3], [2, 8], [5, 5], [1, 0]] + result = call.call_record(depths, [4] * 4, call.ModelSettings(tolerance=1e-15, max_iterations=2)) + self.assertEqual(result.iterations, 2) + self.assertFalse(result.converged) + self.assertGreater(result.final_change, 1e-15) + + def test_uniform_prior_is_a_single_pass(self): + depths = [[9, 3], [2, 8], [0, 0]] + result = call.call_record(depths, [4] * 3, call.ModelSettings(prior=call.PRIOR_UNIFORM)) + self.assertEqual(result.iterations, 1) + # A sample without reads keeps the prior, so no dosage stands out. + self.assertEqual(result.posteriors[2][0], [0.2] * 5) + self.assertIsNone(call.most_probable_dosage(result.posteriors[2][0])) + + +class PloidyCorrectionTests(unittest.TestCase): + def test_dosages_that_already_fit_are_unchanged(self): + posteriors = [[0.1, 0.8, 0.1], [0.1, 0.8, 0.1]] + self.assertEqual(call.correct_dosages_to_ploidy([1, 1], posteriors, 2), [1, 1]) + self.assertEqual(call.best_dosages_summing_to_ploidy(posteriors, 2), [1, 1]) + + def test_copy_is_removed_where_it_costs_least(self): + # Sum is 3 in a diploid. Dropping a copy of the second allele keeps the most probability. + posteriors = [[0.01, 0.09, 0.90], [0.45, 0.55, 0.00]] + self.assertEqual(call.correct_dosages_to_ploidy([2, 1], posteriors, 2), [2, 0]) + + def test_copy_is_added_where_it_costs_least(self): + posteriors = [[0.05, 0.60, 0.30, 0.03, 0.02], [0.10, 0.80, 0.05, 0.03, 0.02], [0.70, 0.25, 0.03, 0.01, 0.01]] + # Sum is 2 in a tetraploid, so two copies are added one at a time. + self.assertEqual(call.correct_dosages_to_ploidy([1, 1, 0], posteriors, 4), [2, 1, 1]) + + def test_tied_dosage_gives_no_call(self): + self.assertIsNone(call.correct_dosages_to_ploidy([None, 1], [[0.5, 0.5, 0.0], [0.1, 0.8, 0.1]], 2)) + + def test_no_possible_correction_gives_no_call(self): + # Both alleles are certainly absent, so no copy can be added. + posteriors = [[1.0, 0.0, 0.0], [1.0, 0.0, 0.0]] + self.assertIsNone(call.correct_dosages_to_ploidy([0, 0], posteriors, 2)) + self.assertIsNone(call.best_dosages_summing_to_ploidy(posteriors, 2)) + + def test_exact_search_matches_brute_force(self): + posteriors = [ + [0.02, 0.10, 0.50, 0.30, 0.05, 0.02, 0.01], + [0.30, 0.40, 0.20, 0.05, 0.03, 0.01, 0.01], + [0.05, 0.15, 0.45, 0.25, 0.05, 0.03, 0.02], + [0.60, 0.30, 0.05, 0.02, 0.01, 0.01, 0.01], + ] + ploidy = 6 + + def log_probability(dosages): + return sum(math.log(posteriors[allele][dosage]) for allele, dosage in enumerate(dosages)) + + every_genotype = [ + dosages for dosages in itertools.product(range(ploidy + 1), repeat=4) if sum(dosages) == ploidy + ] + self.assertEqual(len(every_genotype), 84) + brute_force = max(every_genotype, key=log_probability) + exact = call.best_dosages_summing_to_ploidy(posteriors, ploidy) + self.assertEqual(exact, list(brute_force)) + greedy = call.correct_dosages_to_ploidy( + [call.most_probable_dosage(posterior) for posterior in posteriors], posteriors, ploidy + ) + self.assertLessEqual(log_probability(greedy), log_probability(exact) + 1e-12) + + +def simulated_two_allele_markers( + overdispersion, ploidy, markers, samples, genotypes="hwe", seed=11, bias=1.0, background_rate=0.0 +): + """Read counts ``[marker][sample][allele]`` drawn from the beta-binomial model. + + ``genotypes`` is ``hwe`` (random mating), ``inbred`` (every sample + homozygous) or ``partly inbred`` (three samples in four homozygous, the + rest as under random mating). ``bias`` is the alternate allele's allelic bias and + ``background_rate`` the share of reads that show the wrong allele. The + seed is fixed, so the counts are the same on every run. + """ + generator = random.Random(seed) + table = [] + for _ in range(markers): + frequency = generator.uniform(0.15, 0.85) + depths = [] + for _ in range(samples): + if genotypes == "hwe" or (genotypes == "partly inbred" and generator.random() < 0.25): + dosage = sum(generator.random() < frequency for _ in range(ploidy)) + else: + dosage = ploidy if generator.random() < frequency else 0 + read_fraction = dosage / ploidy + read_fraction = read_fraction / (bias * (1 - read_fraction) + read_fraction) + read_fraction = read_fraction * (1 - background_rate) + (1 - read_fraction) * background_rate + if 0 < read_fraction < 1: + read_fraction = generator.betavariate( + read_fraction * overdispersion, (1 - read_fraction) * overdispersion + ) + marker_depth = generator.randint(20, 120) + alternate_depth = sum(generator.random() < read_fraction for _ in range(marker_depth)) + depths.append([marker_depth - alternate_depth, alternate_depth]) + table.append(depths) + return table + + +class ReadModelEstimateTests(unittest.TestCase): + def test_marker_likelihood_is_the_prior_weighted_average_over_dosages(self): + depths = [[13, 7], [0, 0], [20, 0], [9, 11]] + ploidies = [4, 4, 4, 4] + settings = call.ModelSettings(overdispersion=12) + result = call.call_record(depths, ploidies, settings) + frequency = call.frequency_for_likelihood(result.allele_frequencies[1], len(depths)) + grid = (5, 12, 60) + for overdispersion, observed in zip( + grid, call.marker_overdispersion_log_likelihoods(depths, ploidies, settings, grid) + ): + at_value = call.ModelSettings(overdispersion=overdispersion) + expected = 0.0 + for reference_depth, alternate_depth in depths: + if reference_depth + alternate_depth == 0: + continue # a sample without reads adds nothing + by_dosage = call.allele_log_likelihoods( + alternate_depth, reference_depth + alternate_depth, 4, frequency, at_value + ) + expected += math.log( + sum(weight * math.exp(value) for weight, value in zip(result.priors[4][1], by_dosage)) + ) + self.assertAlmostEqual(observed, expected, places=10) + # The same quantity from the call's own likelihoods, which used the allele frequency + # of the pass before the last and so differ in the contamination term by a hair. + self.assertAlmostEqual( + call.marker_overdispersion_log_likelihoods(depths, ploidies, settings, (12,))[0], + call.marginal_log_likelihood(result.log_likelihoods, result.priors, ploidies, allele=1), + places=6, + ) + + def test_recovers_the_simulated_value_in_diploids_and_polyploids(self): + for ploidy in (2, 4, 6): + with self.subTest(ploidy=ploidy): + table = simulated_two_allele_markers(20, ploidy, markers=40, samples=60) + estimate = call.estimate_overdispersion(table, [ploidy] * 60) + self.assertIn(estimate.value, (16, 18, 20, 25)) + self.assertIsNone(estimate.at_limit) + self.assertEqual(estimate.markers_used, 40) + most_likely = max(estimate.log_likelihood_by_value, key=estimate.log_likelihood_by_value.get) + self.assertEqual(most_likely, estimate.value) + + def test_inbred_samples_end_at_the_lowest_value(self): + # Without heterozygotes the counts say nothing about scatter, and the + # Hardy-Weinberg weights of the estimate do not fit. + table = simulated_two_allele_markers(20, 2, markers=30, samples=60, genotypes="inbred") + estimate = call.estimate_overdispersion(table, [2] * 60) + self.assertEqual((estimate.value, estimate.at_limit), (min(call.OVERDISPERSION_GRID), "lowest")) + + def test_too_little_to_estimate_from(self): + self.assertIsNone(call.estimate_overdispersion([[[0, 0], [0, 0]]], [2, 2]).value) + self.assertIsNone(call.estimate_overdispersion([], [2, 2]).value) + # Three markers of ten samples are 30 cells with reads, below the minimum of 100. + few = simulated_two_allele_markers(20, 2, markers=3, samples=10) + estimate = call.estimate_overdispersion(few, [2] * 10) + self.assertEqual((estimate.value, estimate.cells_with_reads), (None, 30)) + + def test_background_rate_and_overdispersion_are_estimated_together(self): + table = simulated_two_allele_markers(40, 4, markers=40, samples=80, background_rate=0.005) + settings = call.ModelSettings(prior=call.PRIOR_NORM, contamination_rate=0.0) + estimate = call.estimate_read_model(table, [4] * 80, settings) + self.assertEqual(estimate.background_rate, 0.005) + self.assertIn(estimate.overdispersion_estimate.value, (30, 40, 50)) + # Each rate tried is recorded with its overdispersion and log-likelihood; the chosen one is the best. + best_rate = max(estimate.by_background_rate, key=lambda rate: estimate.by_background_rate[rate][1]) + self.assertEqual(best_rate, estimate.background_rate) + self.assertGreaterEqual(len(estimate.by_background_rate), 3) + + def test_excess_of_homozygotes_needs_the_free_weights_prior(self): + # Three samples in four are homozygous. Neither Hardy-Weinberg proportions nor a normal + # shape can describe that, and the estimate made with them ends at the lowest value; + # with one free probability per dosage it finds the simulated overdispersion. + table = simulated_two_allele_markers( + 40, 4, markers=40, samples=80, genotypes="partly inbred", background_rate=0.005 + ) + estimates = {} + for prior in (call.PRIOR_HWE, call.PRIOR_NORM, call.PRIOR_FREE): + settings = call.ModelSettings(prior=prior, contamination_rate=0.0, background_rate=0.005) + estimates[prior] = call.estimate_overdispersion(table, [4] * 80, settings) + self.assertIn(estimates[call.PRIOR_FREE].value, (30, 40, 50)) + self.assertIsNone(estimates[call.PRIOR_FREE].at_limit) + self.assertLess(estimates[call.PRIOR_HWE].value, 10) + self.assertLess(estimates[call.PRIOR_NORM].value, 10) + + def test_free_weights_prior_is_the_share_of_each_dosage(self): + table = simulated_two_allele_markers(60, 4, markers=1, samples=120, genotypes="partly inbred", seed=4)[0] + result = call.call_record(table, [4] * 120, call.ModelSettings(prior=call.PRIOR_FREE, overdispersion=60)) + self.assertTrue(result.converged) + prior = result.priors[4][1] + self.assertAlmostEqual(sum(prior), 1.0, places=9) + # At convergence the prior equals the mean of the posteriors, dosage by dosage. + for dosage, probability in enumerate(prior): + mean_posterior = sum(sample[1][dosage] for sample in result.posteriors) / 120 + self.assertAlmostEqual(probability, mean_posterior, places=4) + # Most of the weight is on the two homozygous dosages, which a normal shape cannot give. + self.assertGreater(prior[0] + prior[4], 0.7) + self.assertEqual(result.normal_prior_parameters, {}) + + def test_one_parameter_can_be_held_fixed(self): + table = simulated_two_allele_markers(40, 4, markers=30, samples=60, background_rate=0.005) + settings = call.ModelSettings(prior=call.PRIOR_NORM, contamination_rate=0.0) + fixed_rate = call.estimate_read_model(table, [4] * 60, settings, background_rates=(0.005,)) + self.assertEqual(list(fixed_rate.by_background_rate), [0.005]) + self.assertIn(fixed_rate.overdispersion_estimate.value, (30, 40, 50)) + fixed_scatter = call.estimate_read_model(table, [4] * 60, settings, overdispersions=(40,)) + self.assertEqual(fixed_scatter.overdispersion_estimate.value, 40) + self.assertIn(fixed_scatter.background_rate, (0.003, 0.005)) + + def test_too_low_a_background_rate_lowers_the_overdispersion_estimate(self): + # Reads of the wrong allele that the background rate does not allow for can only be + # explained by more scatter. This is why the two are estimated together. + table = simulated_two_allele_markers(40, 4, markers=30, samples=60, background_rate=0.005) + settings = call.ModelSettings(prior=call.PRIOR_NORM, contamination_rate=0.0, background_rate=0.0002) + too_low = call.estimate_overdispersion(table, [4] * 60, settings) + self.assertLess(too_low.value, 30) + + def test_marker_subset_is_limited_and_evenly_spaced(self): + self.assertEqual(call._evenly_spaced(list(range(10)), 20), list(range(10))) + self.assertEqual(call._evenly_spaced(list(range(10)), 5), [0, 2, 4, 6, 8]) + + +def simulated_group_markers(groups, samples_per_group, markers, ploidy=4, depth=(8, 16), overdispersion=50, seed=5): + """Read counts and true alternate dosages for samples from several groups. + + Returns ``[marker][sample][allele]`` counts and ``[marker][sample]`` + dosages; the samples of a group are next to each other. At every marker + each group has its own allele frequency, up to 0.35 away from the + marker's base frequency, and its samples are drawn at random-mating + proportions for it. The seed is fixed, so the counts are the same on + every run. + """ + generator = random.Random(seed) + table, truth = [], [] + for _ in range(markers): + base = generator.uniform(0.2, 0.8) + depths, dosages = [], [] + for _ in range(groups): + frequency = min(0.98, max(0.02, base + generator.uniform(-0.35, 0.35))) + for _ in range(samples_per_group): + dosage = sum(generator.random() < frequency for _ in range(ploidy)) + read_fraction = dosage / ploidy + if 0 < read_fraction < 1: + read_fraction = generator.betavariate( + read_fraction * overdispersion, (1 - read_fraction) * overdispersion + ) + marker_depth = generator.randint(*depth) + alternate_depth = sum(generator.random() < read_fraction for _ in range(marker_depth)) + depths.append([marker_depth - alternate_depth, alternate_depth]) + dosages.append(dosage) + table.append(depths) + truth.append(dosages) + return table, truth + + +class StructurePriorTests(unittest.TestCase): + """The structure prior: each sample's allele frequency comes from the samples most like it.""" + + settings = call.ModelSettings( + prior=call.PRIOR_STRUCTURE, overdispersion=50, contamination_rate=0.0, background_rate=0.001 + ) + + @classmethod + def setUpClass(cls): + cls.table, cls.truth = simulated_group_markers(groups=3, samples_per_group=20, markers=120) + cls.ploidies = [4] * 60 + cls.choice = call.find_neighbours(cls.table, cls.ploidies, cls.settings) + + def test_neighbours_come_from_the_sample_s_own_group(self): + choice = self.choice + self.assertIn(choice.neighbours_per_sample, (10, 20)) + self.assertIsNone(choice.reason) + self.assertEqual(choice.markers_used, 120) + self.assertEqual(sorted(choice.error_by_count), [0, 5, 10, 20, 40]) + self.assertLess(choice.error_by_count[choice.neighbours_per_sample], choice.error_by_count[0]) + own_group = total = 0 + for sample, sample_neighbours in enumerate(choice.neighbours): + self.assertEqual(len(sample_neighbours), choice.neighbours_per_sample) + self.assertNotIn(sample, sample_neighbours) + # A group has 19 other samples, so only the nearest 10 can all be from it. + for other in sample_neighbours[:10]: + total += 1 + own_group += other // 20 == sample // 20 + self.assertGreater(own_group / total, 0.95) + + def test_more_dosages_are_called_correctly_than_with_hwe(self): + def share_correct(settings): + right = total = 0 + for depths, dosages in zip(self.table, self.truth): + result = call.call_record(depths, self.ploidies, settings) + for sample_posteriors, dosage in zip(result.posteriors, dosages): + total += 1 + right += call.most_probable_dosage(sample_posteriors[1]) == dosage + return right / total + + with_neighbours = dataclasses.replace(self.settings, neighbours=self.choice.neighbours) + hwe = dataclasses.replace(self.settings, prior=call.PRIOR_HWE) + # 81.6% against 79.9% on this table. + self.assertGreater(share_correct(with_neighbours), share_correct(hwe) + 0.01) + + def test_one_population_gets_no_neighbours_and_the_hwe_result(self): + table = simulated_two_allele_markers(50, 4, markers=120, samples=40) + ploidies = [4] * 40 + choice = call.find_neighbours(table, ploidies, self.settings) + self.assertEqual(choice.neighbours_per_sample, 0) + self.assertIn("no better than the whole table", choice.reason) + self.assertEqual(choice.neighbours, tuple(() for _ in ploidies)) + without = call.call_record(table[0], ploidies, dataclasses.replace(self.settings, neighbours=choice.neighbours)) + hwe = call.call_record(table[0], ploidies, dataclasses.replace(self.settings, prior=call.PRIOR_HWE)) + self.assertEqual(without.posteriors, hwe.posteriors) + self.assertEqual(without.sample_frequencies, []) + + def test_a_table_too_small_to_compare_samples_gets_no_neighbours(self): + few_samples = call.find_neighbours([depths[:10] for depths in self.table], [4] * 10, self.settings) + self.assertEqual((few_samples.neighbours_per_sample, few_samples.reason), (0, "fewer than 20 samples")) + few_markers = call.find_neighbours(self.table[:30], self.ploidies, self.settings) + self.assertEqual(few_markers.neighbours_per_sample, 0) + self.assertEqual(few_markers.reason, "fewer than 100 markers with reads") + self.assertEqual(few_markers.markers_used, 30) + + def test_a_sample_s_prior_follows_its_neighbours(self): + # Samples 0 and 1 have the same reads; the neighbours of 0 carry the alternate allele + # and those of 1 do not. + depths = [[3, 1], [3, 1], [0, 12], [0, 14], [1, 11], [13, 0], [12, 0], [14, 1]] + neighbours = ((2, 3, 4), (5, 6, 7), (0, 3, 4), (0, 2, 4), (0, 2, 3), (1, 6, 7), (1, 5, 7), (1, 5, 6)) + settings = dataclasses.replace(self.settings, neighbours=neighbours) + result = call.call_record(depths, [2] * 8, settings) + self.assertTrue(result.converged) + self.assertGreater(result.sample_frequencies[0][1], 0.7) + self.assertLess(result.sample_frequencies[1][1], 0.2) + self.assertGreater(result.posteriors[0][1][1], result.posteriors[1][1][1]) + self.assertLess(result.posteriors[0][1][0], result.posteriors[1][1][0]) + for sample, sample_neighbours in enumerate(neighbours): + # The frequency is the neighbours' mean share with the record's frequency as one more. + shares = [call.posterior_mean_dosage(result.posteriors[other][1]) / 2 for other in sample_neighbours] + expected = (sum(shares) + result.allele_frequencies[1]) / 4 + self.assertAlmostEqual(result.sample_frequencies[sample][1], expected, delta=1e-3) + self.assertAlmostEqual(sum(result.sample_frequencies[sample]), 1.0, places=12) + for value, mirrored in zip(result.posteriors[sample][0], reversed(result.posteriors[sample][1])): + self.assertAlmostEqual(value, mirrored, places=12) + + def test_records_with_more_than_two_alleles(self): + depths = [[6, 2, 0], [5, 3, 1], [0, 7, 2], [1, 6, 3], [8, 0, 0], [7, 1, 0]] + neighbours = ((1, 4, 5), (0, 4, 5), (3, 1), (2, 1), (0, 5), (0, 4)) + result = call.call_record(depths, [4] * 6, dataclasses.replace(self.settings, neighbours=neighbours)) + self.assertEqual([len(of_sample) for of_sample in result.sample_frequencies], [3] * 6) + for sample_posteriors in result.posteriors: + self.assertEqual(len(sample_posteriors), 3) + for posterior in sample_posteriors: + self.assertAlmostEqual(sum(posterior), 1.0, places=12) + + def test_settings_are_checked(self): + with self.assertRaisesRegex(ValueError, "allelic bias"): + call.ModelSettings(prior=call.PRIOR_STRUCTURE, fit_allele_bias=True) + settings = dataclasses.replace(self.settings, neighbours=((1,), (0,))) + with self.assertRaisesRegex(ValueError, "one entry per sample"): + call.call_record([[4, 4], [5, 3], [2, 6]], [2] * 3, settings) + + +class PolyradParityTests(unittest.TestCase): + """madcats_call against polyRAD on the same read counts.""" + + @classmethod + def setUpClass(cls): + cls.fixture = load_polyrad_fixture() + cls.settings = call.ModelSettings( + overdispersion=cls.fixture["overdispersion"], + contamination_rate=cls.fixture["contamination_rate"], + tolerance=cls.fixture["iteration_tolerance"], + max_iterations=1000, + ) + + def locus_depths(self, columns): + return [[row[column] for column in columns] for row in self.fixture["depth"]] + + def position(self, record, columns): + """Sample index and index of the record's allele within its locus, or None for another locus.""" + column = self.fixture["allele_index"][record["allele"]] + if column not in columns: + return None + return self.fixture["sample_index"][record["sample"]], columns.index(column) + + def test_single_pass_matches_polyrad(self): + fixture = self.fixture + sample_count = len(fixture["samples"]) + compared = 0 + for columns in fixture["locus_columns"].values(): + depths = self.locus_depths(columns) + frequencies = call.depth_ratio_frequencies(depths) + for column, frequency in zip(columns, frequencies): + self.assertAlmostEqual(frequency, fixture["single_pass"]["allele_frequency"][column], places=14) + for record in fixture["single_pass"]["records"]: + position = self.position(record, columns) + if position is None: + continue + sample, allele = position + ploidy = fixture["ploidy"][sample] + log_likelihoods = call.allele_log_likelihoods( + depths[sample][allele], + sum(depths[sample]), + ploidy, + call.frequency_for_likelihood(frequencies[allele], sample_count), + self.settings, + ) + prior = call.hwe_prior(ploidy, frequencies[allele]) + posterior = call.dosage_posterior(log_likelihoods, prior) + for dosage in range(ploidy + 1): + expected_likelihood = record["likelihood"][dosage] + self.assertAlmostEqual( + math.exp(log_likelihoods[dosage]) / expected_likelihood, 1.0, places=9, + msg=f"likelihood {record['sample']} {record['allele']} dosage {dosage}", + ) + self.assertAlmostEqual(prior[dosage], record["prior"][dosage], places=13) + self.assertAlmostEqual(posterior[dosage], record["posterior"][dosage], places=11) + compared += 1 + self.assertEqual(compared, sum(len(record["posterior"]) for record in fixture["single_pass"]["records"])) + + def test_selfing_priors_match_polyrad(self): + cases = self.fixture["selfing"]["priors"] + self.assertEqual(len(cases), 36) + for case in cases: + prior = call.hwe_prior(case["ploidy"], case["allele_frequency"], case["selfing_rate"]) + for value, expected in zip(prior, case["prior"]): + self.assertAlmostEqual(value, expected, places=13, msg=str(case)) + + def test_iteration_with_selfing_matches_polyrad(self): + fixture = self.fixture + settings = call.ModelSettings( + overdispersion=fixture["overdispersion"], + contamination_rate=fixture["contamination_rate"], + selfing_rate=fixture["selfing"]["selfing_rate"], + tolerance=fixture["iteration_tolerance"], + max_iterations=1000, + ) + for columns in fixture["locus_columns"].values(): + result = call.call_record(self.locus_depths(columns), fixture["ploidy"], settings) + self.assertTrue(result.converged) + for column, frequency in zip(columns, result.allele_frequencies): + self.assertAlmostEqual(frequency, fixture["selfing"]["allele_frequency"][column], places=7) + for record in fixture["selfing"]["records"]: + position = self.position(record, columns) + if position is None: + continue + sample, allele = position + for value, expected in zip(result.posteriors[sample][allele], record["posterior"]): + self.assertAlmostEqual(value, expected, places=7) + # The selfing rate changed the result: the fixture's two runs differ. + self.assertNotEqual(fixture["selfing"]["allele_frequency"], fixture["iterated"]["allele_frequency"]) + + def test_iterated_frequencies_posteriors_and_genotypes_match_polyrad(self): + fixture = self.fixture + for columns in fixture["locus_columns"].values(): + depths = self.locus_depths(columns) + result = call.call_record(depths, fixture["ploidy"], self.settings) + self.assertTrue(result.converged) + for column, frequency in zip(columns, result.allele_frequencies): + self.assertAlmostEqual(frequency, fixture["iterated"]["allele_frequency"][column], places=7) + for record in fixture["iterated"]["records"]: + position = self.position(record, columns) + if position is None: + continue + sample, allele = position + for value, expected in zip(result.posteriors[sample][allele], record["posterior"]): + self.assertAlmostEqual(value, expected, places=7) + for sample, ploidy in enumerate(fixture["ploidy"]): + most_probable = [call.most_probable_dosage(posterior) for posterior in result.posteriors[sample]] + corrected = call.correct_dosages_to_ploidy(most_probable, result.posteriors[sample], ploidy) + expected = [fixture["iterated"]["genotypes"][sample][column] for column in columns] + self.assertEqual(corrected, expected, msg=f"{fixture['samples'][sample]} columns {columns}") + + + +# --------------------------------------------------------------------------- +# The command +# --------------------------------------------------------------------------- + +# A 30-base Ref tag and tags derived from it. Positions below are 1-based. +REF_TAG = "ACGTACGTACGGATCCTTAGCATGCAAGTC" + + +def with_base(tag, position, base): + return tag[: position - 1] + base + tag[position:] + + +ALT_TAG = with_base(REF_TAG, 12, "C") # the designed target: 12 G>C +REF_TAG_SITE_20 = with_base(REF_TAG, 20, "T") # Ref at the target, G>T at position 20 +ALT_TAG_SITE_20 = with_base(ALT_TAG, 20, "T") +OTHER_TAG = with_base(REF_TAG, 12, "T") # a third base at the target + + +def run_madcats(arguments): + """Run a madcats command quietly; return what it printed to stderr.""" + captured = io.StringIO() + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(captured): + madcats.main([str(argument) for argument in arguments]) + return captured.getvalue() + + +def read_vcf(path): + """Parse a VCF into (header lines, sample names, records). + + Each record is a dict of the fixed columns plus ``samples``: one dict of + FORMAT key to value per sample. + """ + header, records, sample_names = [], [], [] + with open(path) as handle: + for line in handle: + line = line.rstrip("\n") + if line.startswith("##"): + header.append(line) + elif line.startswith("#"): + sample_names = line.split("\t")[9:] + else: + fields = line.split("\t") + keys = fields[8].split(":") + records.append( + { + "CHROM": fields[0], "POS": int(fields[1]), "ID": fields[2], "REF": fields[3], + "ALT": [] if fields[4] == "." else fields[4].split(","), "INFO": fields[7], "FORMAT": keys, + "samples": [dict(zip(keys, column.split(":"))) for column in fields[9:]], + } + ) + return header, sample_names, records + + +class CommandTestCase(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.dir = Path(self.temporary.name) + + def tearDown(self): + self.temporary.cleanup() + + def write_madc(self, rows, samples, name="madc.csv", leading_columns=()): + """Write a short-layout MADC. ``rows`` are (marker, category, sequence, counts).""" + path = self.dir / name + with open(path, "w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow([*leading_columns, "AlleleID", "CloneID", "AlleleSequence", *samples]) + for marker, category, sequence, counts in rows: + writer.writerow([*["x" for _ in leading_columns], f"{marker}|{category}", marker, sequence, *counts]) + return path + + def call_madc(self, madc, *options): + prefix = self.dir / "calls" + self.stderr = run_madcats(["call", "--madc", madc, "--output-prefix", prefix, "--jobs", "1", *options]) + with open(str(prefix) + ".qc.json") as handle: + self.qc = json.load(handle) + return prefix + + def call_error(self, madc, *options): + """The message of a ``call`` command that is expected to stop with an error.""" + arguments = ["call", "--madc", str(madc), "--output-prefix", str(self.dir / "calls"), *options] + with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()) as stderr: + madcats.main(arguments) + return stderr.getvalue() + + def vcf(self, prefix, vcf_type): + return read_vcf(str(prefix) + call.VCF_FILE_SUFFIX[vcf_type]) + + def model_line(self, prefix): + """The header line of the target VCF that describes the model in words.""" + return next(line for line in self.vcf(prefix, "target")[0] if line.startswith("##madcatsModel=")) + + +class TargetVcfTests(CommandTestCase): + def test_default_writes_only_the_target_vcf(self): + madc = self.write_madc( + [("M1", "Ref", REF_TAG, [30, 15, 0]), ("M1", "Alt", ALT_TAG, [0, 16, 28])], ["s1", "s2", "s3"] + ) + prefix = self.call_madc(madc) + self.assertTrue(Path(str(prefix) + ".targets.vcf").is_file()) + self.assertFalse(Path(str(prefix) + ".all.vcf").exists()) + self.assertFalse(Path(str(prefix) + ".haplotypes.vcf").exists()) + header, samples, records = self.vcf(prefix, "target") + self.assertEqual(samples, ["s1", "s2", "s3"]) + self.assertIn("##contig=", header) + self.assertTrue(any(line.startswith("##madcatsCoordinates=marker-local") for line in header)) + (record,) = records + self.assertEqual((record["CHROM"], record["POS"], record["REF"], record["ALT"]), ("M1", 12, "G", ["C"])) + self.assertEqual([sample["GT"] for sample in record["samples"]], ["0/0", "0/1", "1/1"]) + self.assertEqual(record["FORMAT"], ["GT", "AD", "DP", "XD", "GQ", "MPP", "DS", "GP", "PL"]) + + def test_match_rows_count_for_their_allele_and_other_rows_are_excluded(self): + madc = self.write_madc( + [ + ("M1", "Ref", REF_TAG, [20, 10, 0]), + ("M1", "Alt", ALT_TAG, [0, 10, 20]), + ("M1", "RefMatch", REF_TAG_SITE_20, [5, 0, 0]), + ("M1", "AltMatch", ALT_TAG_SITE_20, [0, 4, 6]), + ("M1", "Other", OTHER_TAG, [3, 0, 7]), + ], + ["s1", "s2", "s3"], + ) + prefix = self.call_madc(madc) + (record,) = self.vcf(prefix, "target")[2] + self.assertEqual([sample["AD"] for sample in record["samples"]], ["25,0", "10,14", "0,26"]) + self.assertEqual([sample["DP"] for sample in record["samples"]], ["25", "24", "26"]) + self.assertEqual([sample["XD"] for sample in record["samples"]], ["3", "0", "7"]) + self.assertEqual(self.qc["vcfs"]["target"]["reads_excluded"], 10) + self.assertEqual(self.qc["vcfs"]["target"]["records_with_many_excluded_reads"], 1) + + def test_excluded_read_share_filter(self): + madc = self.write_madc( + [ + ("M1", "Ref", REF_TAG, [20, 20]), + ("M1", "Alt", ALT_TAG, [0, 20]), + ("M1", "Other", OTHER_TAG, [30, 2]), + ], + ["s1", "s2"], + ) + prefix = self.call_madc(madc, "--max-other-fraction", "0.5") + (record,) = self.vcf(prefix, "target")[2] + self.assertEqual([sample["GT"] for sample in record["samples"]], ["./.", "0/1"]) + self.assertEqual(self.qc["vcfs"]["target"]["genotypes_missing_excluded_read_share"], 1) + + def test_low_depth_and_no_reads_are_missing_but_keep_their_counts(self): + madc = self.write_madc( + [("M1", "Ref", REF_TAG, [30, 4, 0, 12]), ("M1", "Alt", ALT_TAG, [0, 3, 0, 14])], ["s1", "s2", "s3", "s4"] + ) + prefix = self.call_madc(madc) + (record,) = self.vcf(prefix, "target")[2] + low_depth, no_reads = record["samples"][1], record["samples"][2] + self.assertEqual((low_depth["GT"], low_depth["AD"], low_depth["DP"]), ("./.", "4,3", "7")) + # The posterior of a low-depth sample is still reported; only the call is withheld. + self.assertEqual(len(low_depth["GP"].split(",")), 3) + self.assertEqual( + no_reads, + {"GT": "./.", "AD": "0,0", "DP": "0", "XD": "0", "GQ": ".", "MPP": ".", "DS": ".", "GP": ".", "PL": "."}, + ) + self.assertIn("NS=2", record["INFO"]) + self.assertEqual(self.qc["vcfs"]["target"]["genotypes_missing_low_depth"], 1) + self.assertEqual(self.qc["vcfs"]["target"]["genotypes_missing_no_reads"], 1) + + prefix = self.call_madc(madc, "--min-dp", "0") + (record,) = self.vcf(prefix, "target")[2] + self.assertEqual(record["samples"][1]["GT"], "0/1") + + def info_values(self, record): + return dict(item.split("=") for item in record["INFO"].split(";") if "=" in item) + + def test_mpp_is_the_largest_genotype_probability_and_pmc_its_mean_shortfall(self): + # s2 and s5 lie between two dosages, s3 has too few reads for a call, s4 has none. + madc = self.write_madc( + [("M1", "Ref", REF_TAG, [30, 17, 4, 0, 10, 0]), ("M1", "Alt", ALT_TAG, [0, 3, 3, 0, 2, 26])], + ["s1", "s2", "s3", "s4", "s5", "s6"], + ) + prefix = self.call_madc(madc) + header, _, (record,) = self.vcf(prefix, "target") + self.assertTrue(any(line.startswith("##FORMAT== 0 and row["Used"] == "1" for row in table)) + self.assertFalse(any("BIASUNUSED" in line for line in header)) + self.assertFalse(any("BIASUNUSED" in record["INFO"] for record in records)) + # Haplotype records are not given a bias. + haplotype_header, _, haplotype_records = self.vcf(prefix, "haplotype") + self.assertFalse(any("ID=BIAS" in line for line in haplotype_header)) + self.assertFalse(any("BIAS=" in record["INFO"] for record in haplotype_records)) + + self.call_madc(madc, "--ploidy", "4", "--overdispersion", "60", "--background-rate", "0.001") + self.assertEqual(self.qc["allele_bias"], {}) + self.assertFalse(Path(str(self.dir / "calls") + ".bias.csv").exists() and self.qc["outputs"]["bias_table"]) + + def test_allele_bias_auto_calls_only_the_biased_records_with_their_bias(self): + biased = simulated_two_allele_markers(60, 4, markers=3, samples=60, bias=0.5, seed=8) + unbiased = simulated_two_allele_markers(60, 4, markers=3, samples=60, seed=9) + madc = self.madc_from_simulation(biased + unbiased) + options = ("--ploidy", "4", "--overdispersion", "60", "--background-rate", "0.001") + without_records = self.vcf(self.call_madc(madc, *options), "target")[2] + fit_prefix = self.call_madc(madc, *options, "--allele-bias", "fit") + fit_records = self.vcf(fit_prefix, "target")[2] + with open(str(fit_prefix) + ".bias.csv", newline="") as handle: + fit_table = list(csv.DictReader(handle)) + prefix = self.call_madc(madc, *options, "--allele-bias", "auto") + header, _, records = self.vcf(prefix, "target") + self.assertTrue(any("used where it raises the log-likelihood by 10 or more" in line for line in header)) + self.assertTrue(any(line.startswith("##INFO={name}\n{''.join(bases)}\n") + rows = [ + madc_row("chr1_001041", "Ref", cls.a_ref, [10, 0, 5]), + madc_row("chr1_001041", "Alt", cls.a_alt, [0, 10, 5]), + madc_row("chr1_001041", "RefMatch", cls.a_paralog, [6, 6, 6]), + madc_row("chr1_001041", "RefMatch", cls.a_match, [2, 0, 0]), + madc_row("chr1_004046", "Ref", cls.b_ref, [8, 8, 8]), + madc_row("chr1_004046", "Alt", cls.b_alt, [0, 4, 0]), + madc_row("chr1_004046", "RefMatch", cls.b_tie, [3, 3, 3]), + madc_row("chr1_005541", "Ref", cls.c_ref, [9, 9, 9]), + madc_row("chr1_005541", "Alt", cls.c_alt, [1, 1, 1]), + ] + cls.madc = cls.dir / "in_MADC.csv" + write_madc(cls.madc, rows) + cls.index = cls.dir / "genome.mmi" + cls.output = cls.dir / "out_MADC.csv" + with contextlib.redirect_stderr(io.StringIO()): + fp.run_filter_paralogs(mp.build_parser().parse_args([ + "filter-paralogs", "--madc", str(cls.madc), "--genome", str(cls.genome), + "--genome-index", str(cls.index), "--output", str(cls.output), "--jobs", "2", + ])) + cls.qc = json.loads(cls.output.with_suffix(".csv.qc.json").read_text()) + cls.labels = {row["AlleleSequence"]: row for row in read_dicts(cls.output.with_suffix(".csv.genome_labels.csv"))} + cls.markers = {row["CloneID"]: row for row in read_dicts(cls.output.with_suffix(".csv.genome_markers.csv"))} + + @classmethod + def tearDownClass(cls): + cls.temporary.cleanup() + + def test_labels_from_the_genome(self): + expected = { + self.a_ref: "on_target", self.a_alt: "on_target", self.a_paralog: "paralog", self.a_match: "on_target", + self.b_ref: "on_target", self.b_alt: "on_target", self.b_tie: "ambiguous", + self.c_ref: "ambiguous", self.c_alt: "ambiguous", + } + self.assertEqual({sequence: row["Label"] for sequence, row in self.labels.items()}, expected) + self.assertTrue(self.labels[self.a_paralog]["OffTargetHit"].startswith("chr2:3001-3081")) + self.assertEqual(self.labels[self.a_paralog]["OffTargetScore"], "162") + + def test_marker_placement(self): + self.assertEqual(self.markers["chr1_001041"]["Sites"], "chr1:1001-1081(+)") + self.assertEqual(self.markers["chr1_001041"]["CloneIDCheck"], "agree") + multicopy = self.markers["chr1_005541"] + self.assertEqual((multicopy["Placement"], multicopy["RefTagSites"]), ("ref_tag+clone_id", "2")) + self.assertEqual(self.qc["markers"]["multicopy_ref_tags"], ["chr1_005541"]) + + def test_output_and_qc(self): + kept = [row[2] for row in read_rows(self.output)[1:]] + self.assertNotIn(self.a_paralog, kept) + self.assertNotIn(self.b_tie, kept) + self.assertIn(self.c_ref, kept) # canonical, protected despite its multi-copy tag + self.assertEqual(self.qc["rows"]["by_outcome"]["protected"]["rows"], 2) + self.assertEqual(self.qc["rows"]["by_outcome"]["dropped"], {"rows": 2, "reads": 27}) + self.assertEqual(self.qc["aligner"]["k"], 21) + + def test_saved_index_is_reused(self): + self.assertTrue(self.index.is_file()) + output = self.dir / "from_index_MADC.csv" + with contextlib.redirect_stderr(io.StringIO()) as log: + fp.run_filter_paralogs(mp.build_parser().parse_args( + ["filter-paralogs", "--madc", str(self.madc), "--genome-index", str(self.index), "--output", str(output)] + )) + self.assertIn("Loading index", log.getvalue()) + self.assertEqual(read_rows(output), read_rows(self.output)) + + def test_clone_positions_ignored(self): + # Marker C's Ref tag has equal-best copies on chr1 and chr3; only its CloneID picked chr1. + output = self.dir / "ignored_MADC.csv" + with contextlib.redirect_stderr(io.StringIO()): + fp.run_filter_paralogs(mp.build_parser().parse_args([ + "filter-paralogs", "--madc", str(self.madc), "--genome-index", str(self.index), + "--output", str(output), "--clone-positions", "ignore", + ])) + markers = {row["CloneID"]: row for row in read_dicts(output.with_suffix(".csv.genome_markers.csv"))} + self.assertEqual( + (markers["chr1_005541"]["Placement"], markers["chr1_005541"]["Sites"].count(";")), + ("ref_tag_multicopy", 1), + ) + self.assertEqual({row["CloneIDCheck"] for row in markers.values()}, {"ignored"}) + labels = {row["AlleleSequence"]: row["Label"] for row in read_dicts(output.with_suffix(".csv.genome_labels.csv"))} + self.assertEqual(labels, {sequence: row["Label"] for sequence, row in self.labels.items()}) + qc = json.loads(output.with_suffix(".csv.qc.json").read_text()) + self.assertEqual(qc["parameters"]["clone_positions"], "ignore") + + +@unittest.skipUnless(HAS_MAPPY, "mappy not installed (pip install 'madcats[genome]')") +class PlantedCopyTests(unittest.TestCase): + """Markers on one contig of a random genome, each with a diverged copy of its tag on another. + + The copies differ from the Ref tag at 1, 2, 4 or 8 bases, or at 1 base + and a 2-base deletion; four markers of each kind. Every marker has its + Ref and Alt rows, an allele of its own with one more substitution, and + two rows that come from the copy: the copy's tag, and that tag with one + more substitution. + """ + + KINDS = (("1", 1, 0), ("2", 2, 0), ("4", 4, 0), ("8", 8, 0), ("1_and_deletion", 1, 2)) + + @classmethod + def setUpClass(cls): + rng = random.Random(23) + home = [rng.choice("ACGT") for _ in range(30000)] + elsewhere = [rng.choice("ACGT") for _ in range(30000)] + + def other(base): + return rng.choice([choice for choice in "ACGT" if choice != base]) + + def substituted(sequence, positions): + for position in positions: + sequence = mutate(sequence, position, other(sequence[position])) + return sequence + + rows, cls.truth = [], {} + position = 1000 + for kind, substitutions, deleted in cls.KINDS: + for _ in range(4): + clone = f"home_{position + 41:06d}" + region = "".join(home[position - 100 : position + 181]) # the tag and 100 bases either side + ref = region[100:181] + changed_at = sorted(rng.sample(range(25, 70), substitutions)) + copy_region = substituted(region, [100 + offset for offset in changed_at]) + if deleted: + cut = rng.choice([offset for offset in range(30, 60) if offset not in changed_at]) + copy_region = copy_region[: 100 + cut] + copy_region[100 + cut + deleted :] + elsewhere[position : position + len(copy_region)] = copy_region + copy_tag = copy_region[100:181] + own_allele = substituted(ref, [72]) + copy_variant = substituted(copy_tag, [74]) + for category, sequence, origin in ( + ("Ref", ref, "home"), ("Alt", substituted(ref, [40]), "home"), ("RefMatch_1", own_allele, "home"), + ("RefMatch_2", copy_tag, "copy"), ("RefMatch_3", copy_variant, "copy"), + ): + rows.append(madc_row(clone, category, sequence, [8, 4, 0])) + cls.truth[(clone, sequence)] = (kind, origin) + position += 1400 + cls.temporary = tempfile.TemporaryDirectory() + cls.dir = Path(cls.temporary.name) + cls.genome = cls.dir / "genome.fa" + cls.genome.write_text(f">home\n{''.join(home)}\n>elsewhere\n{''.join(elsewhere)}\n") + cls.madc = cls.dir / "in_MADC.csv" + write_madc(cls.madc, rows) + + @classmethod + def tearDownClass(cls): + cls.temporary.cleanup() + + def filtered(self, name, madc=None, *options): + output = self.dir / f"{name}.csv" + with contextlib.redirect_stderr(io.StringIO()): + fp.run_filter_paralogs(mp.build_parser().parse_args([ + "filter-paralogs", "--madc", str(madc or self.madc), "--genome", str(self.genome), + "--genome-index", str(self.dir / "genome.mmi"), "--output", str(output), *options, + ])) + return output + + def test_copy_rows_are_paralogs_and_home_rows_are_on_target(self): + output = self.filtered("default") + labels = read_dicts(output.with_suffix(".csv.genome_labels.csv")) + self.assertEqual(len(labels), 100) + for row in labels: + kind, origin = self.truth[(row["CloneID"], row["AlleleSequence"])] + with self.subTest(kind=kind, allele=row["AlleleID"]): + self.assertEqual(row["Label"], "paralog" if origin == "copy" else "on_target") + self.assertEqual(row["Action"], "dropped" if origin == "copy" else "kept") + self.assertEqual(len(read_rows(output)), 1 + 60) + markers = read_dicts(output.with_suffix(".csv.genome_markers.csv")) + self.assertEqual(Counter(row["Placement"] for row in markers), {"ref_tag": 20}) + + def test_output_does_not_depend_on_the_number_of_threads(self): + one = self.filtered("one_thread", None, "--jobs", "1") + three = self.filtered("three_threads", None, "--jobs", "3") + for suffix in (".csv", ".csv.genome_labels.csv", ".csv.genome_markers.csv"): + self.assertEqual(one.with_suffix(suffix).read_bytes(), three.with_suffix(suffix).read_bytes()) + + def test_gzipped_madc_and_named_qc_file(self): + import gzip + + zipped = self.dir / "in_MADC.csv.gz" + zipped.write_bytes(gzip.compress(self.madc.read_bytes())) + qc_path = self.dir / "named.qc.json" + output = self.filtered("from_gzip", zipped, "--qc-output", str(qc_path)) + plain = self.filtered("from_plain") + self.assertEqual(output.read_bytes(), plain.read_bytes()) + self.assertEqual(json.loads(qc_path.read_text())["rows"]["by_label"]["paralog"]["rows"], 40) + self.assertFalse(output.with_suffix(".csv.qc.json").exists()) + + +@unittest.skipUnless(HAS_MAPPY, "mappy not installed (pip install 'madcats[genome]')") +class ReviewCaseTests(unittest.TestCase): + """Counterexamples from an external review (docs/DEVELOPMENT.md, "Review fixes"), each on a tiny genome with the real aligner.""" + + def setUp(self): + self.rng = random.Random(34) + self.temporary = tempfile.TemporaryDirectory() + self.dir = Path(self.temporary.name) + + def tearDown(self): + self.temporary.cleanup() + + def dna(self, length): + return "".join(self.rng.choice("ACGT") for _ in range(length)) + + def aligner(self, contigs, name="genome"): + fasta = self.dir / f"{name}.fa" + fasta.write_text("".join(f">{contig}\n{sequence}\n" for contig, sequence in contigs.items())) + with contextlib.redirect_stderr(io.StringIO()): + return fp.GenomeAligner.open(fasta, None, 1) + + def run_filter(self, aligner, rows): + madc = self.dir / "in.csv" + write_madc(madc, rows) + with contextlib.redirect_stderr(io.StringIO()): + fp.filter_paralogs(madc, self.dir / "out.csv", aligner, self.dir / "labels.csv", self.dir / "markers.csv") + labels = {row["AlleleSequence"]: row for row in read_dicts(self.dir / "labels.csv")} + return labels, read_dicts(self.dir / "markers.csv") + + def test_assembly_carrying_the_alt_allele_at_the_target(self): + # chr1 is the design site but carries the Alt allele; a paralog on chr2 carries Ref. The Ref tag + # aligns better on chr2, yet the marker is placed where its CloneID says. + ref = self.dna(81) + alt = changed(ref, 40) + allele = changed(alt, 65) # one SNP from the target, two from the paralog + aligner = self.aligner({"chr1": self.dna(500) + alt + self.dna(500), "chr2": self.dna(500) + ref + self.dna(500)}) + specs = [("Ref", ref, [5, 5, 5]), ("Alt", alt, [5, 5, 5]), ("AltMatch", allele, [3, 3, 3])] + labels, markers = self.run_filter(aligner, [madc_row("chr1_000540", *spec) for spec in specs]) + self.assertEqual( + (markers[0]["Placement"], markers[0]["CloneIDCheck"], markers[0]["Sites"]), + ("clone_id_site", "disagree", "chr1:501-581(+)"), + ) + found = labels[allele] + self.assertEqual( + (found["Label"], found["Action"], found["TargetScore"], found["OffTargetScore"]), ("on_target", "kept", "152", "142") + ) + # A CloneID position where the Ref tag does not align leaves the marker unplaced, keeping every row. + labels, markers = self.run_filter(aligner, [madc_row("chr1_000900", *spec) for spec in specs]) + self.assertEqual((markers[0]["Placement"], markers[0]["CloneIDCheck"]), ("unplaced", "disagree")) + self.assertEqual({(row["Label"], row["Action"]) for row in labels.values()}, {("unplaced", "kept")}) + + def test_markers_sharing_a_site_do_not_crowd_each_other_out(self): + # The allele has no 21-base seed at its own site, and its genome-wide hit is a 50-base partial copy. + ref = self.dna(81) + allele = changed(ref, 10, 30, 50, 70) + aligner = self.aligner({"target": self.dna(500) + ref + self.dna(500), "off": self.dna(500) + allele[:50] + self.dna(500)}) + for copies in (1, 30): # more markers on one site than the short-read preset returns hits + rows = [ + row for number in range(copies) + for row in (madc_row(f"m{number}", "Ref", ref, [9, 9, 9]), madc_row(f"m{number}", "RefMatch", allele, [3, 3, 3])) + ] + write_madc(self.dir / "in.csv", rows) + with contextlib.redirect_stderr(io.StringIO()): + fp.filter_paralogs( + self.dir / "in.csv", self.dir / "out.csv", aligner, self.dir / "labels.csv", self.dir / "markers.csv" + ) + found = Counter( + (row["Label"], row["TargetScore"], row["OffTargetScore"]) + for row in read_dicts(self.dir / "labels.csv") if row["AlleleID"].endswith("|RefMatch") + ) + self.assertEqual(found, {("on_target", "122", "100"): copies}) + + def test_adjacent_tandem_copies(self): + repeat = self.dna(80) + ref = repeat + repeat[0] + allele = changed(ref, 40) # one SNP from both copies + aligner = self.aligner({"chr1": self.dna(500) + repeat + repeat + repeat[0] + self.dna(500)}) + labels, markers = self.run_filter( + aligner, [madc_row("chr1_000540", "Ref", ref, [5, 5, 5]), madc_row("chr1_000540", "RefMatch", allele, [3, 3, 3])] + ) + self.assertEqual( + (markers[0]["Placement"], markers[0]["RefTagSites"], markers[0]["Sites"]), ("ref_tag+clone_id", "2", "chr1:501-581(+)") + ) + found = labels[allele] + self.assertEqual( + (found["Label"], found["TargetScore"], found["OffTargetScore"], found["OffTargetHit"]), + ("ambiguous", "152", "152", "chr1:581-661(+)"), + ) + + def test_multipart_index_is_refused(self): + import mappy + + parts = [] + for number in (1, 2): + fasta = self.dir / f"part{number}.fa" + fasta.write_text(f">chr{number}\n{self.dna(1000)}\n") + parts.append(self.dir / f"part{number}.mmi") + mappy.Aligner(str(fasta), preset="sr", fn_idx_out=str(parts[-1])) + joined = self.dir / "joined.mmi" # what minimap2 -I writes for a genome larger than its batch + joined.write_bytes(parts[0].read_bytes() + parts[1].read_bytes()) + self.assertEqual(fp._index_part_contigs(parts[0]), [["chr1"]]) + self.assertEqual(fp._index_part_contigs(joined), [["chr1"], ["chr2"]]) + self.assertIsNone(fp._index_part_contigs(self.dir / "part1.fa")) + truncated = self.dir / "truncated.mmi" + truncated.write_bytes(parts[0].read_bytes()[:-5]) + with self.assertRaises(mp.MadcError): + fp._index_part_contigs(truncated) + madc = self.dir / "in.csv" + write_madc(madc, [madc_row("chr1_000100", "Ref", self.dna(81), [1, 1, 1])]) + output = self.dir / "out.csv" + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(mp.MadcError) as caught: + fp.run_filter_paralogs(mp.build_parser().parse_args( + ["filter-paralogs", "--madc", str(madc), "--genome-index", str(joined), "--output", str(output)] + )) + self.assertIn("in 2 parts", str(caught.exception)) + self.assertFalse(output.exists()) + # The same file is refused when --genome names it. + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(mp.MadcError) as caught: + fp.run_filter_paralogs(mp.build_parser().parse_args( + ["filter-paralogs", "--madc", str(madc), "--genome", str(joined), "--output", str(output)] + )) + self.assertIn("in 2 parts", str(caught.exception)) + self.assertFalse(output.exists()) + # A one-part index given as --genome is loaded, and no copy is saved over --genome-index. + other = self.dir / "other.mmi" + with contextlib.redirect_stderr(io.StringIO()) as log: + fp.run_filter_paralogs(mp.build_parser().parse_args( + ["filter-paralogs", "--madc", str(madc), "--genome", str(parts[0]), "--genome-index", str(other), + "--output", str(output)] + )) + self.assertIn("Loading index", log.getvalue()) + self.assertFalse(other.exists()) + self.assertTrue(output.exists()) + + def test_label_change_between_assemblies_without_a_copy_change(self): + # Both assemblies have the target and one other copy; only the other copy's sequence differs. The + # label changes, so a label change alone does not show that a copy was gained or lost. + ref = self.dna(81) + allele = changed(ref, 40) + flanks = [self.dna(500) for _ in range(4)] + labels = [] + for name, copy in (("a", changed(ref, 20, 50)), ("b", allele)): + aligner = self.aligner({"target": flanks[0] + ref + flanks[1], "copy": flanks[2] + copy + flanks[3]}, name) + ref_placement = fp.distinct_sites(aligner.hits(ref, aligner.new_buffer()), max_sites=None) + marker = fp.place_marker("marker", ref_placement, aligner.contigs, 81, use_clone_position=False) + labels.append(fp.genome_label(fp.distinct_sites(aligner.hits(allele, aligner.new_buffer())), marker).label) + self.assertEqual(labels, ["on_target", "paralog"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_madcats.py b/tests/test_madcats.py index bc51093..4504bdf 100644 --- a/tests/test_madcats.py +++ b/tests/test_madcats.py @@ -1,5 +1,6 @@ import argparse import csv +import dataclasses import gzip import json import random @@ -2214,3 +2215,108 @@ def test_variants_column_and_sidecar(self): self.assertEqual(set(cells.values()), {"."}) self.assertFalse(sidecar.exists()) self.assertIsNone(qc["output_summary"]["variants"]["sidecar"]) + + +class TileFamilyTests(unittest.TestCase): + """Tiles: two target SNPs inside one amplicon, read through one shared tag window.""" + + # SNPs at 10 (C/T) and 20 (G/A); both tiles start at the same primer. + AMPLICON = "ACGTACGGTA" + "C" + "TTGACCAGT" + "G" + "ATCCGTAGC" + + def tiles(self, groups=("", ""), iupac=True): + """Tile A (SNP at 10) and tile B (SNP at 20); with ``iupac`` each tag carries + the other tile's SNP as an ambiguity code, as DArTag panels write them.""" + amplicon = self.AMPLICON + ref_a = amplicon[:20] + ("R" if iupac else "G") + amplicon[21:] + ref_b = amplicon[:10] + ("Y" if iupac else "C") + amplicon[11:] + primer = amplicon[:8] + return [ + mp.Locus(0, "tileA", primer, "", "C", "T", 10, ref_a, ref_a[:10] + "T" + ref_a[11:], locus_group=groups[0]), + mp.Locus(1, "tileB", primer, "", "G", "A", 20, ref_b, ref_b[:20] + "A" + ref_b[21:], locus_group=groups[1]), + ] + + def test_tied_read_is_credited_to_every_tile(self): + loci = self.tiles() + self.assertEqual(mp.tile_families(loci), ((0, 1),)) + family_of = {0: (0, 1), 1: (0, 1)} + for read in (self.AMPLICON, self.AMPLICON[:10] + "T" + self.AMPLICON[11:]): + self.assertEqual(mp.select_markers(read, [0, 1], loci, 3, family_of=family_of), ((0, 1), "assigned")) + # Without the family the read ties and is dropped, as before. + self.assertEqual(mp.select_marker(read, [0, 1], loci, 3), (None, "ambiguous")) + + def test_untied_read_reaches_every_tile_it_fits(self): + # Tiles with identical tags and no ambiguity codes: a read carrying + # tile A's Alt fits A exactly and B with one substitution. It no longer + # goes to A alone; B sees it too, as it would in a panel of its own. + loci = self.tiles(iupac=False) + read = self.AMPLICON[:10] + "T" + self.AMPLICON[11:] + self.assertEqual(mp.select_marker(read, [0, 1], loci, 3), (0, "assigned")) + families = mp.tile_families(loci) + family_of = {member: family for family in families for member in family} + self.assertEqual(mp.select_markers(read, [0, 1], loci, 3, family_of=family_of), ((0, 1), "assigned")) + + def test_tile_group_modes(self): + self.assertEqual(mp.tile_families(self.tiles(), "declared"), ()) + self.assertEqual(mp.tile_families(self.tiles(), "off"), ()) + self.assertEqual(mp.tile_families(self.tiles(("amp1", "amp1")), "declared"), ((0, 1),)) + self.assertEqual(mp.tile_families(self.tiles(("amp1", "")), "auto"), ((0, 1),)) + # Markers declared in different groups are never linked. + self.assertEqual(mp.tile_families(self.tiles(("amp1", "amp2")), "auto"), ()) + # Different windows of one LocusGroup (normalize-reads tiling) are not one tag window. + window_a, window_b = self.tiles(("amp1", "amp1")) + shifted = mp.Locus(1, "tileB", self.AMPLICON[5:13], "", "G", "A", 15, self.AMPLICON[5:] + "AAAAA", + self.AMPLICON[5:20] + "A" + self.AMPLICON[21:] + "AAAAA", locus_group="amp1") + self.assertEqual(mp.tile_families([window_a, shifted]), ()) + + def test_tie_across_families_stays_ambiguous(self): + tile_a, tile_b = self.tiles(("amp1", "amp1")) + elsewhere = dataclasses.replace(tile_a, index=2, marker="elsewhere", locus_group="amp2") + loci = [tile_a, tile_b, elsewhere] + families = mp.tile_families(loci) + self.assertEqual(families, ((0, 1),)) + family_of = {member: family for family in families for member in family} + self.assertEqual(mp.select_markers(self.AMPLICON, [0, 1, 2], loci, 3, family_of=family_of), ((), "ambiguous")) + + def test_generate_counts_tiles_in_one_panel(self): + amplicon = self.AMPLICON + haplotypes = [amplicon] * 12 + [amplicon[:10] + "T" + amplicon[11:]] * 6 + [amplicon[:20] + "A" + amplicon[21:]] * 4 + partials = [amplicon[:26]] * 2 + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + fastq_dir = root / "fastq" + fastq_dir.mkdir() + with (fastq_dir / "s1.fastq").open("w") as handle: + for number, sequence in enumerate(haplotypes + partials): + handle.write(f"@r{number}\n{sequence}\n+\n{'I' * len(sequence)}\n") + tags = root / "tags.csv" + mp.write_reference_tags(tags, self.tiles()) # no LocusGroup: the tiles are detected + outcomes = {} + for mode in ("auto", "off"): + output = root / f"{mode}.csv" + self.assertEqual(mp.main(["generate", "--fastq-dir", str(fastq_dir), "--reference-tags", str(tags), + "--tag-length", "30", "--jobs", "1", "--reproducible-min-samples", "0", + "--depth-min-reads", "1", "--tile-groups", mode, + "--cache-dir", str(root / "cache"), "--output", str(output)]), 0) + totals = mp.Counter() + for record in mp.read_madc(output)["records"]: + totals[record["clone_id"]] += record["counts"][0] + with Path(str(output) + ".qc.json").open() as handle: + outcomes[mode] = (totals, json.load(handle)) + totals, qc = outcomes["auto"] + self.assertEqual(totals, {"tileA": 24, "tileB": 24}) + self.assertEqual(qc["tile_families"], [["tileA", "tileB"]]) + accounting = qc["read_accounting"] + self.assertEqual(accounting["reads_assigned_full_length"], 22) # each read once + self.assertEqual(accounting["tile_extra_credits_full_length"], 22) + self.assertEqual(accounting["tile_extra_credits_partial"], 2) + self.assertEqual( + accounting["reads_in_final_count_table"], + accounting["reads_assigned_full_length"] + accounting["tile_extra_credits_full_length"] + - accounting["reads_n_dropped"] + accounting["partial_reads_added_to_counts"], + ) + totals, qc = outcomes["off"] + self.assertEqual(totals, {"tileA": 0, "tileB": 0}) + self.assertEqual(qc["tile_families"], []) + self.assertEqual(qc["sample_qc"][0]["reads_ambiguous"], 22) + # The families are part of the cache key: two runs, two cache entries. + self.assertEqual(len(list((root / "cache").glob("s1.*.json.gz"))), 2) diff --git a/tests/test_tiled_markers.py b/tests/test_tiled_markers.py new file mode 100644 index 0000000..840fc64 --- /dev/null +++ b/tests/test_tiled_markers.py @@ -0,0 +1,218 @@ +"""Tiled markers through ``generate``, ``filter-paralogs`` and ``call``. + +Two markers that read the same tag window (tiles) share every read of their +amplicon. ``generate`` credits each read to both. The other two commands +were written without tiles in mind, so this checks what they do with such a +table: the paralog filter must not take one tile's rows for a copy of the +other, and ``call`` must get each tile's own variant right. + +The panel is written both ways a tile's tag can carry the other tile's +variant: as an ambiguity code, the way DArTag panels write it, and as the +reference base. +""" +import contextlib +import csv +import importlib.util +import io +import json +import random +import tempfile +import unittest +from pathlib import Path + +import madcats as mp + +HAS_MAPPY = importlib.util.find_spec("mappy") is not None + +TAG_LENGTH = 81 +SNP_A, SNP_B = 30, 55 # 0-based offsets of the two tiles' variants in the tag +AMBIGUITY_CODE = { + frozenset("AG"): "R", frozenset("CT"): "Y", frozenset("AC"): "M", + frozenset("GT"): "K", frozenset("AT"): "W", frozenset("CG"): "S", +} +# Each sample's two copies of the amplicon, as (base at tile A's variant, base at tile B's): +# r is the reference base, a the alternate. +SAMPLES = { + "s01": ("rr", "rr"), "s02": ("rr", "ar"), "s03": ("ar", "ar"), "s04": ("rr", "ra"), "s05": ("ra", "ra"), + "s06": ("ar", "ra"), "s07": ("rr", "aa"), "s08": ("aa", "aa"), "s09": ("ar", "aa"), "s10": ("ra", "aa"), +} +READS_PER_COPY = 30 + + +def with_base(sequence, offset, base): + return sequence[:offset] + base + sequence[offset + 1:] + + +def another_base(base): + return {"A": "C", "C": "T", "G": "A", "T": "G"}[base] + + +def run_quietly(arguments): + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + return mp.main([str(argument) for argument in arguments]) + + +def madc_rows(path): + with open(path, newline="") as handle: + rows = list(csv.reader(handle)) + header = next(index for index, row in enumerate(rows) if row and row[0] == "AlleleID") + first_sample = rows[header].index(next(iter(SAMPLES))) + return [(row[0], row[2], [int(value) for value in row[first_sample:]]) for row in rows[header + 1:]] + + +def alternate_dosage(genotype): + """The number of alternate alleles in a GT value, or None for a genotype that was not called.""" + alleles = genotype.replace("|", "/").split("/") + return None if "." in alleles else alleles.count("1") + + +def alternate_dosages(vcf_path): + """Records of a VCF as {(marker, position): (is the target, alternate dosage per sample)}. + + A sample without a call has None, so it cannot pass for a homozygous reference call. + """ + records = {} + with open(vcf_path) as handle: + for line in handle: + if line.startswith("#"): + continue + fields = line.rstrip("\n").split("\t") + dosages = [alternate_dosage(column.split(":")[0]) for column in fields[9:]] + records[(fields[0], int(fields[1]))] = ("TARGET" in fields[7].split(";"), dosages) + return records + + +class TiledPanelTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.directory = tempfile.TemporaryDirectory() + cls.root = Path(cls.directory.name) + generator = random.Random(7) + genome = "".join(generator.choice("ACGT") for _ in range(20000)) + tag = genome[5000 : 5000 + TAG_LENGTH] + other_marker = genome[12000 : 12000 + TAG_LENGTH] + alternate_a, alternate_b = another_base(tag[SNP_A]), another_base(tag[SNP_B]) + (cls.root / "genome.fasta").write_text(">chr1\n" + genome + "\n") + cls.site = ("chr1", 5001, 5000 + TAG_LENGTH) + + copies = { + "rr": tag, + "ar": with_base(tag, SNP_A, alternate_a), + "ra": with_base(tag, SNP_B, alternate_b), + "aa": with_base(with_base(tag, SNP_A, alternate_a), SNP_B, alternate_b), + } + other_alternate = with_base(other_marker, 40, another_base(other_marker[40])) + fastq_dir = cls.root / "fastq" + fastq_dir.mkdir() + for sample, pair in SAMPLES.items(): + reads = [copies[copy] for copy in pair for _ in range(READS_PER_COPY)] + reads += [other_marker] * 20 + [other_alternate] * 20 + with (fastq_dir / f"{sample}.fastq").open("w") as handle: + for number, read in enumerate(reads): + handle.write(f"@{sample}_{number}\n{read}\n+\n{'I' * len(read)}\n") + cls.truth = { + "tileA": [sum(copy[0] == "a" for copy in pair) for pair in SAMPLES.values()], + "tileB": [sum(copy[1] == "a" for copy in pair) for pair in SAMPLES.values()], + } + + cls.madc = {} + for style in ("ambiguity codes", "reference bases"): + # Each tile's tags hold the other tile's variant as a code for both bases, or as the reference base. + at_b = AMBIGUITY_CODE[frozenset((tag[SNP_B], alternate_b))] if style == "ambiguity codes" else tag[SNP_B] + at_a = AMBIGUITY_CODE[frozenset((tag[SNP_A], alternate_a))] if style == "ambiguity codes" else tag[SNP_A] + ref_a, ref_b = with_base(tag, SNP_B, at_b), with_base(tag, SNP_A, at_a) + loci = [ + mp.Locus(0, "tileA", tag[:20], "", tag[SNP_A], alternate_a, SNP_A, ref_a, + with_base(ref_a, SNP_A, alternate_a)), + mp.Locus(1, "tileB", tag[:20], "", tag[SNP_B], alternate_b, SNP_B, ref_b, + with_base(ref_b, SNP_B, alternate_b)), + mp.Locus(2, "other", other_marker[:20], "", other_marker[40], other_alternate[40], 40, other_marker, + other_alternate), + ] + prefix = cls.root / style.replace(" ", "_") + mp.write_reference_tags(Path(str(prefix) + ".tags.csv"), loci) + cls.madc[style] = Path(str(prefix) + ".madc.csv") + exit_code = run_quietly([ + "generate", "--fastq-dir", fastq_dir, "--reference-tags", str(prefix) + ".tags.csv", + "--tag-length", TAG_LENGTH, "--jobs", 1, "--reproducible-min-samples", 0, "--depth-min-reads", 1, + "--cache-dir", str(prefix) + ".cache", "--output", cls.madc[style], + ]) + assert exit_code == 0 + + @classmethod + def tearDownClass(cls): + cls.directory.cleanup() + + def call(self, style, *options): + prefix = self.root / (style.replace(" ", "_") + ".calls") + self.assertEqual(run_quietly([ + "call", "--madc", self.madc[style], "--ploidy", 2, "--jobs", 1, "--min-dp", 0, "--min-gq", 0, + "--output-prefix", prefix, *options, + ]), 0) + return prefix + + def test_generate_gives_both_tiles_every_read_of_the_amplicon(self): + for style, madc in self.madc.items(): + with self.subTest(tags=style): + totals = {} + for allele_id, _, counts in madc_rows(madc): + marker = allele_id.split("|")[0] + totals[marker] = [total + count for total, count in zip(totals.get(marker, [0] * 10), counts)] + self.assertEqual(totals["tileA"], [2 * READS_PER_COPY] * len(SAMPLES)) + self.assertEqual(totals["tileB"], totals["tileA"]) + with open(str(madc) + ".qc.json") as handle: + self.assertEqual(json.load(handle)["tile_families"], [["tileA", "tileB"]]) + + def test_tags_with_ambiguity_codes_leave_the_ref_and_alt_rows_empty(self): + # No read can equal a tag that holds a code, so every read is on a RefMatch or + # AltMatch row. A genotype taken from the Ref and Alt rows alone would have no reads. + for allele_id, _, counts in madc_rows(self.madc["ambiguity codes"]): + if allele_id.startswith("tile") and allele_id.split("|")[1] in ("Ref", "Alt"): + self.assertEqual(sum(counts), 0, allele_id) + reads_on_match_rows = sum( + sum(counts) for allele_id, _, counts in madc_rows(self.madc["ambiguity codes"]) if "Match" in allele_id + ) + self.assertEqual(reads_on_match_rows, 2 * 2 * READS_PER_COPY * len(SAMPLES)) # both tiles, two copies each + + @unittest.skipUnless(HAS_MAPPY, "mappy not installed (pip install 'madcats[genome]')") + def test_paralog_filter_places_both_tiles_at_one_site_and_drops_nothing(self): + for style, madc in self.madc.items(): + with self.subTest(tags=style): + output = Path(str(madc) + ".filtered.csv") + self.assertEqual(run_quietly([ + "filter-paralogs", "--madc", madc, "--genome", self.root / "genome.fasta", "--output", output, + "--jobs", 1, + ]), 0) + with open(str(output) + ".genome_markers.csv", newline="") as handle: + sites = {row["CloneID"]: row["Sites"] for row in csv.DictReader(handle)} + contig, first, last = self.site + self.assertEqual(sites["tileA"], f"{contig}:{first}-{last}(+)") + self.assertEqual(sites["tileB"], sites["tileA"]) + with open(str(output) + ".genome_labels.csv", newline="") as handle: + labels = list(csv.DictReader(handle)) + self.assertEqual({row["Label"] for row in labels}, {"on_target"}) + self.assertEqual({row["Action"] for row in labels}, {"kept"}) + self.assertEqual(madc_rows(output), madc_rows(madc)) + + def test_call_gets_each_tile_its_own_variant(self): + for style in self.madc: + with self.subTest(tags=style): + targets = alternate_dosages(str(self.call(style)) + ".targets.vcf") + self.assertEqual(targets[("tileA", SNP_A + 1)], (True, self.truth["tileA"])) + self.assertEqual(targets[("tileB", SNP_B + 1)], (True, self.truth["tileB"])) + + def test_the_other_tiles_variant_is_an_off_target_record_only_where_the_tag_has_a_base(self): + # With the reference base in the tag, each tile also reports the other tile's variant, + # from the same reads and so with the same genotypes. + records = alternate_dosages(str(self.call("reference bases", "--vcf", "all")) + ".all.vcf") + self.assertEqual(records[("tileA", SNP_B + 1)], (False, self.truth["tileB"])) + self.assertEqual(records[("tileB", SNP_A + 1)], (False, self.truth["tileA"])) + self.assertEqual(len(records), 5) + # A position where the Ref tag holds an ambiguity code gets no off-target record: + # the variant is reported once, as the other tile's target. + records = alternate_dosages(str(self.call("ambiguity codes", "--vcf", "all")) + ".all.vcf") + self.assertEqual(sorted(records), [("other", 41), ("tileA", SNP_A + 1), ("tileB", SNP_B + 1)]) + + +if __name__ == "__main__": + unittest.main()