diff --git a/.Rbuildignore b/.Rbuildignore index 307d5d0..50c7896 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -10,3 +10,5 @@ ^package-code\.Rmd$ ^docs$ ^CRAN-SUBMISSION$ +^\.positai$ +^\.claude$ diff --git a/.gitignore b/.gitignore index d3ffaad..e056686 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ .Ruserdata revdep/ .DS_Store +.positai diff --git a/DESCRIPTION b/DESCRIPTION index 7b0280d..0a1c846 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,76 +1,69 @@ Package: BIGr -Title: Breeding Insight Genomics Functions for Polyploid and Diploid Species -Version: 0.7.2 -Authors@R: c(person(given='Alexander M.', - family='Sandercock', - email='sandercock.alex@gmail.com', - role=c('cre','aut')), - person(given='Cristiane', - family='Taniguti', - role = 'aut'), - person(given='Josue', - family='Chinchilla-Vargas', - role='aut'), - person(given='Shufen', - family='Chen', - role='ctb'), - person(given='Manoj', - family='Sapkota', - role='ctb'), - person(given='Meng', - family='Lin', - role='ctb'), - person(given='Dongyan', - family='Zhao', - role='ctb'), - person('University', "of Florida", - role=c('cph'), - comment = "Breeding Insight")) +Title: Breeding Insight Genomics Functions for Polyploid and Diploid + Species +Version: 0.10.0 +Authors@R: c( + person("Alexander M.", "Sandercock", , "sandercock.alex@gmail.com", role = c("cre", "aut")), + person("Cristiane", "Taniguti", role = "aut"), + person("Josue", "Chinchilla-Vargas", role = "aut"), + person("Shufen", "Chen", role = "ctb"), + person("Manoj", "Sapkota", role = "ctb"), + person("Meng", "Lin", role = "ctb"), + person("Dongyan", "Zhao", role = "ctb"), + person(family = "University of Florida", role = "cph", + comment = "Breeding Insight") + ) Maintainer: Alexander M. Sandercock Description: Functions developed within Breeding Insight to analyze diploid and polyploid breeding and genetic data. 'BIGr' provides the - ability to filter variant call format (VCF) files, extract single nucleotide polymorphisms (SNPs) - from diversity arrays technology missing allele discovery count (DArT MADC) files, - and manipulate genotype data for both diploid and polyploid species. It - also serves as the core dependency for the 'BIGapp' 'Shiny' app, which - provides a user-friendly interface for performing routine genotype - analysis tasks such as dosage calling, filtering, principal component analysis (PCA), - genome-wide association studies (GWAS), and - genomic prediction. For more details about the included 'breedTools' - functions, see Funkhouser et al. (2017) , and - the 'updog' output format, see Gerard et al. (2018) . + ability to filter variant call format (VCF) files, extract single + nucleotide polymorphisms (SNPs) from diversity arrays technology + missing allele discovery count (DArT MADC) files, and manipulate + genotype data for both diploid and polyploid species. It also serves + as the core dependency for the 'BIGapp' 'Shiny' app, which provides a + user-friendly interface for performing routine genotype analysis tasks + such as dosage calling, filtering, principal component analysis (PCA), + genome-wide association studies (GWAS), and genomic prediction. For + more details about the 'updog' output + format, see Gerard et al. (2018) . License: Apache License (>= 2) URL: https://github.com/Breeding-Insight/BIGr BugReports: https://github.com/Breeding-Insight/BIGr/issues -Encoding: UTF-8 -Roxygen: list(markdown = TRUE) -RoxygenNote: 7.3.3 -Depends: R (>= 4.4.0) -biocViews: -Imports: - parallel, +Depends: + R (>= 4.4.0) +Imports: + Biostrings, dplyr, - Rdpack (>= 0.7), + ggplot2, + grDevices, + graphics, + grid, + parallel, + pwalign, readr (>= 2.1.5), reshape2 (>= 1.4.4), rlang, - tidyr (>= 1.3.1), - vcfR (>= 1.15.0), Rsamtools, - Biostrings, - pwalign, - janitor, - quadprog, - tibble, + stats, stringr, - data.table -Suggests: + tibble, + tidyr (>= 1.3.1), + tools, + utils, + vcfR (>= 1.15.0) +Suggests: + testthat (>= 3.0.0), + circlize, covr, - ggplot2, spelling, + polyRAD, rmdformats, - knitr (>= 1.10), + knitr, rmarkdown, - polyRAD, - testthat (>= 3.0.0) -RdMacros: Rdpack + readxl, + writexl +biocViews: +Config/roxygen2/version: 8.0.0 +Encoding: UTF-8 +Roxygen: list(markdown = TRUE) +RoxygenNote: 7.3.3 diff --git a/NAMESPACE b/NAMESPACE index 4b5cb05..259a73a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,17 +1,14 @@ # Generated by roxygen2: do not edit by hand -export(allele_freq_poly) export(calculate_Het) export(calculate_MAF) export(check_homozygous_trios) export(check_madc_sanity) -export(check_ped) export(check_replicates) export(dosage2vcf) export(dosage_ratios) export(filterMADC) export(filterVCF) -export(find_parentage) export(flip_dosage) export(get_countsMADC) export(imputation_concordance) @@ -19,31 +16,20 @@ export(madc2gmat) export(madc2vcf_all) export(madc2vcf_multi) export(madc2vcf_targets) +export(madc_plot) +export(madc_summary) export(merge_MADCs) -export(solve_composition_poly) export(thinSNP) export(updog2vcf) -export(validate_pedigree) import(dplyr) -import(janitor) import(parallel) -import(quadprog) import(stringr) import(tibble) import(tidyr) import(vcfR) importFrom(Biostrings,DNAString) importFrom(Biostrings,reverseComplement) -importFrom(Rdpack,reprompt) importFrom(Rsamtools,bgzip) -importFrom(data.table,CJ) -importFrom(data.table,as.data.table) -importFrom(data.table,copy) -importFrom(data.table,data.table) -importFrom(data.table,fread) -importFrom(data.table,fwrite) -importFrom(data.table,rbindlist) -importFrom(data.table,set) importFrom(dplyr,"%>%") importFrom(dplyr,across) importFrom(dplyr,arrange) @@ -56,13 +42,19 @@ importFrom(dplyr,select) importFrom(dplyr,summarise) importFrom(dplyr,ungroup) importFrom(dplyr,where) +importFrom(ggplot2,aes) +importFrom(ggplot2,element_text) +importFrom(ggplot2,geom_bar) +importFrom(ggplot2,ggplot) +importFrom(ggplot2,labs) +importFrom(ggplot2,theme) +importFrom(ggplot2,theme_minimal) importFrom(pwalign,nucleotideSubstitutionMatrix) importFrom(pwalign,pairwiseAlignment) importFrom(readr,read_csv) importFrom(reshape2,dcast) importFrom(reshape2,melt) importFrom(rlang,sym) -importFrom(stats,cor) importFrom(stats,reorder) importFrom(stats,setNames) importFrom(tibble,as_tibble) diff --git a/NEWS.md b/NEWS.md index 309fc13..0effd09 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,46 @@ +# BIGr 0.10.0 + +* New `madc_summary()`: read-only summary tables for a fixed allele ID MADC - a per-marker microhaplotype frequency distribution, per-sample and per-marker total read depth (`depth`) and on-target `|Ref`/`|Alt` depth (`target_depth`) alongside missing rate, number of mHaps, and off-target read fraction, a missingness threshold sweep, an overall summary, and optional aggregation by a sample-metadata category (e.g. species). +* New `madc_plot()`: visualize a fixed allele ID MADC via a `plot.type` selector - a PCA of alternative read ratios (color by a metadata category, optional second variable by point `shape.col`, custom `palette`), a linear marker-distribution genome plot, a markers x samples read-depth/#mHaps heatmap (failed cells black; depth on a colorblind-safe diverging scale; samples sorted/labeled by category and optionally faceted by chromosome), a per-sample missing-data boxplot (optionally ordered by median missing rate via `miss.sort` and drawn horizontally via `horizontal`), and a publication-oriented `circos` circular plot. The `circos` figure carries four concentric tracks - dominant per-marker locus ticks with a distinct Mb coordinate axis ("Loci"), a marker-density heatmap ring ("Density", window set by `density.window`), log-scaled #mHaps two-tone bars - a neutral base with a red over-threshold segment above a dashed reference ring - flagging paralog-suspect loci ("mHaps", threshold `paralog.flag`), and a gapless depth ribbon flagging problematic and high-depth-outlier loci ("Depth") - identified by horizontal track titles in a 12 o'clock corridor (`label.gap`) with a single compact legend in the center; track colors are customizable (`mhap.col`, `depth.col`, `density.col`). The "Depth" ribbon flags a marker as problematic when its mean depth is below `min.depth` OR its missing rate exceeds `depth.qc.maxmiss` (default 0.5), so high-dropout markers (e.g. 50% zeros / 50% deep) that a mean-depth cut alone reads as normal are caught. Requesting several (non-circos) types returns a multi-panel figure; figures can be saved via `output.file`. Plotting uses `ggplot2` + base `grid`; the `circos` plot uses the suggested `circlize` package. +* `madc_plot()` also provides two QC-focused `plot.type`s: `"balance"`, a ploidy-aware allele read-ratio diagnostic (alt read ratio vs target depth as a 2D density), and `"depth"`, a log-scaled marker mean-depth (uniformity) distribution with a `min.depth` reference line and median/IQR. The `"balance"` plot shows every cell with a defined ratio (including below `min.depth`, marked by a dotted reference line) and, when `ploidy` is given, dashed guides at the expected `(0:ploidy)/ploidy` ratios plus a depth-dependent 95% binomial expectation band around each interior expected ratio (the spread of `A/D` under `Binomial(D, ratio)`), so allele-balance dispersion wider than sequencing-depth sampling noise stands out directly. `balance.density = TRUE` stacks a marginal histogram of the alt-read ratio above the heatmap (returning a composite grob). Heatmap sub-sampling (`max.loci`/`max.samples`) is now deterministic (evenly spaced) so repeated calls give identical figures. +* `madc_plot(titles = FALSE)` drops the title and subtitle from every requested plot (including the `circos` figure, the multi-panel, and the `balance.density` composite), leaving axis labels, legends, and the `"balance"` caption - for figures captioned externally (manuscript, report, slide). Default `TRUE` keeps the current behavior. +* `madc_summary()` additions: per-sample/per-marker depth is summarized with robust statistics (`median_depth`, `depth_mad`, `depth_q10`, `depth_q90`) alongside the mean, plus `total_reads`/`target_reads` and `call_rate`; off-target is reported both as the mean per-locus ratio (`offtarget_fraction`) and read-weighted `sum(offtarget)/sum(total)` (`offtarget_fraction_weighted`, the "% of reads off target" number); the ambiguous mHap count is split into `n_mhaps_defined` (allele rows in the panel) and `n_mhaps_observed` (alleles actually supported by reads). Two new tables: `depth_sweep` (a read-depth threshold sweep mirroring the missingness sweep) and `marker_qc` (advisory per-marker `PASS`/`FLAG`/`FAIL` status with a reason string, thresholds `min.locus.depth`/`max.locus.depth`/`max.missing`/`max.offtarget`/`max.mhaps.per.loci` reusing `filterMADC`'s vocabulary; it classifies without removing). Tables can be written as per-table CSVs or, with `output.format = "xlsx"`, a single workbook with one sheet per table (requires the suggested `writexl` package). +* `filterMADC` updates: + * Now only accepts fixed allele ID MADC files (processed through HapApp). The input is validated with `check_madc_sanity` and raw DArT MADC files (or any file without fixed AlleleIDs) are rejected with an error. This also fixes a crash on raw MADC files caused by non-unique AlleleIDs. + * Removed the `n.summary.columns` argument (breaking change). Fixed allele ID MADC has a fixed column layout, so summary columns no longer need to be specified or detected. + * Replaced the per-mhap `min.mean.reads`/`max.mean.reads` arguments with a per-locus depth window `min.locus.depth`/`max.locus.depth` (breaking change). Depth is the mean reads per sample at a locus (sum of all mhap reads at the locus divided by the number of samples), and loci outside the window are removed whole (Ref and Alt together) - e.g. `max.locus.depth` to exclude paralogous over-amplification. + * `max.mhaps.per.loci` and `target.only` now retain strictly the target `|Ref`/`|Alt` alleles at affected loci, also removing `|Other` alleles (previously only `|RefMatch`/`|AltMatch` were removed). + * The `min.ind.with.reads` presence filter now protects the target `|Ref`/`|Alt` alleles, only pruning non-target (`|RefMatch`/`|AltMatch`/`|Other`) mhaps, so a locus never loses its Ref/Alt pair. +* Fixed `madc2vcf_targets()` removing markers with no match in `markers_info` as invalid POS values, without the warning "markers removed: no matching entry found in markers_info". The POS correction now runs after these markers are removed and reported. +* Fixed `filterVCF()` deciding whether to apply `filter.DP` and `filter.MPP` from the FORMAT column of the first record only. The FORMAT fields are now read from every record, and the missing call written for a filtered genotype matches the FORMAT of its own record. Calls without a DP or MPP value are left unchanged. +* `filterVCF()` now gives an error when `filter.SAMPLE.miss` or `filter.SNP.miss` is outside 0 to 1, so a percentage such as `20` is no longer accepted as a threshold that keeps every sample or variant. +* `madc2vcf_all()` now records `markers_info`, `add_others`, `others_max_snps`, `others_rm_with_indels`, `others_min_dist`, and `others_max_close_snps` in the `##BIGrCommandLine` header line of the VCF. +* `updog2vcf()` now removes SNPs whose position cannot be read from the SNP ID as a positive integer (expected format CHR_POS), with a warning, instead of writing a POS of `0` or `NA` to the VCF. + + +# BIGr 0.9.0 + +* `check_madc_sanity` updates: distinguish presence of IUPAC codes on REF/ALT (return logical variable IUPACcodes) from RefMatch/AltMatch/Others (returned logical variable IUPACcodes_MatchAlleles) and from Identical IUPAC code in identical positions in REF/ALT (returned logical variable IUPACcodes_IdenticalRefAlt) +* Two new arguments to the `madc2vcf_all` function: + * `others_min_dist` (default 5bp) + * `others_max_close_snps` (default 3 SNPs) + +* By default, `Other` tags will be discarded if they have more than 3 SNPs with less than 5bp distance between them +* Adapt `madc2vcf_all` code to let pass identical IUPAC code in identical position in REF/ALT but ignore polymorphisms in Match or Other alleles at the same position +* `madc2vcf_targets` and `madc2vcf_multi` will only throw an error if different IUPAC are present in REF/ALT sequences (IUPACcodes = TRUE) + + +# BIGr 0.8.1 + +- Fixed `filterVCF()` discarding SNPs whose INFO values are written in scientific notation. The `OD`, `BIAS`, and `PMC` filters read INFO values with a pattern that accepted only digits and decimal points, so a value such as `PMC=4.78e-07` was read as `4.78` and failed a `filter.PMC = 0.05` threshold that it actually passed by five orders of magnitude. `updog2vcf()` writes these values with `paste0()`, so R's default formatting produces scientific notation for small numbers, and updog's `od` and `prop_mis` are routinely small for good markers - meaning the SNPs most likely to be dropped were the cleanest ones. INFO values are now parsed with `as.numeric()` on the whole field, accepting any valid numeric representation. Field names are also matched against a complete INFO entry rather than as a substring, so a field such as `XOD` is no longer mistaken for `OD`. +- Fixed `filterVCF()` writing corrupt all-`NA` variants when a filter value could not be read. A logical index containing `NA` does not drop a row in R, it inserts a row of `NA`s, so any variant with an unreadable `OD`, `BIAS` or `PMC` value, or an `NA` minor allele frequency, was written into the output with an empty `CHROM`, `POS`, `REF` and `ALT`. The `NA` frequency case was reachable through normal use, because `vcfR::maf()` returns `NA` for a variant left with no called genotypes and the `filter.DP` and `filter.MPP` masking can produce exactly that. These variants are now removed, and the number removed is reported in a warning so that they are never discarded silently. +- `filterVCF()` now removes variants when none of them have a readable value for a requested filter, instead of returning the data unfiltered. Previously the `OD`, `BIAS` and `PMC` filters skipped filtering entirely in that situation and reported only "No valid values found", so a run that filtered nothing could look like a run that succeeded. All of the filters now behave the same way, and the warning names the likely cause so it can be diagnosed. +- Fixed `filterVCF()` skipping a requested `OD`, `BIAS` or `PMC` filter when the first variant in the file did not carry that INFO field. The available INFO fields were read from the first record only, so a field that was absent from that one record disabled its filter for the entire file, with no warning and no indication in the output that filtering had not happened. A field may legally be absent from any individual record, so requested filters are now applied to every record, and records without a usable value are removed and reported. This also removes the last case where the order of records in the file could change the result of filtering. +- Corrected and expanded the documentation for `filterVCF()`. `filter.DP` was described as a total read depth filter applied to each SNP; it is applied to each genotype call using the `DP` value in the FORMAT column, setting calls below the threshold to missing rather than removing variants. `filter.MPP` works the same way and was previously undescribed. The remaining filters now state which field they read and which direction the comparison runs, `filter.SAMPLE.miss` and `filter.SNP.miss` state that they take a proportion rather than a percentage, and `filter.BIAS.min` and `filter.BIAS.max` state that neither has any effect unless both are supplied. The return value is documented as a vcfR object when `output.file` is omitted, rather than always a gzipped file. +- Updated madc2vcf_all and madc2vcf_targets. Before, it was possible for POS to be exported as scientific notation instead of integers, and for negative POS values to be present for off target SNPs. POS are corrected to be integers, and SNPs with a negative POS value are removed. +- Removed the breed composition and pedigree functions `allele_freq_poly()`, `solve_composition_poly()`, `check_ped()`, `find_parentage()`, and `validate_pedigree()` (breaking change). They are now part of the independent BIGpopA package: https://github.com/Breeding-Insight/BIGpopA +- Fixed `madc2vcf_all()` error "invalid substring arguments" that occurred with `add_others = TRUE` when an off-target ("Other") allele aligned to the reference with no mismatch positions remaining after the target SNP position was removed. The reference/alternate base lookups for off-target alleles are now guarded by the existing non-empty check, matching how the off-target Match alleles are already handled. + # BIGr 0.7.2 - Fixed manual text errors diff --git a/R/breedtools_functions.R b/R/breedtools_functions.R deleted file mode 100644 index 5f21836..0000000 --- a/R/breedtools_functions.R +++ /dev/null @@ -1,222 +0,0 @@ -#' Compute Allele Frequencies for Populations -#' -#' Computes allele frequencies for specified populations given SNP array data -#' -#' @param geno matrix of genotypes coded as the dosage of allele B \code{{0, 1, 2, ..., ploidy}} -#' with individuals in rows (named) and SNPs in columns (named) -#' @param populations list of named populations. Each population has a vector of IDs -#' that belong to the population. Allele frequencies will be derived from all animals -#' @param ploidy integer indicating the ploidy level (default is 2 for diploid) -#' @return data.frame consisting of allele_frequencies for populations (columns) for -#' each SNP (rows) -#' @references Funkhouser SA, Bates RO, Ernst CW, Newcom D, Steibel JP. Estimation of genome-wide and locus-specific -#' breed composition in pigs. Transl Anim Sci. 2017 Feb 1;1(1):36-44. -#' -#' @examples -#' # Example inputs -#' geno_matrix <- matrix( -#' c(4, 1, 4, 0, # S1 -#' 2, 2, 1, 3, # S2 -#' 0, 4, 0, 4, # S3 -#' 3, 3, 2, 2, # S4 -#' 1, 4, 2, 3),# S5 -#' nrow = 4, ncol = 5, byrow = FALSE, # individuals=rows, SNPs=cols -#' dimnames = list(paste0("Ind", 1:4), paste0("S", 1:5)) -#' ) -#' -#'pop_list <- list( -#' PopA = c("Ind1", "Ind2"), -#' PopB = c("Ind3", "Ind4") -#' ) -#' -#' allele_freqs <- allele_freq_poly(geno = geno_matrix, populations = pop_list, ploidy = 4) -#' print(allele_freqs) -#' -#' @export -allele_freq_poly <- function(geno, populations, ploidy = 2) { - - # Initialize returned df - X <- matrix(NA, nrow = ncol(geno), ncol = length(populations)) - - # Subset geno into different populations - for (i in 1:length(populations)) { - - # Get name of ith item in the list (population name) - pop_name <- names(populations[i]) - - # Subset geno to only include genotypes of IDs in pop - pop_geno <- geno[rownames(geno) %in% populations[[i]], ] - - # Calculate allele frequencies - al_freq <- colMeans(pop_geno, na.rm = TRUE) / ploidy - - # Add to X - X[, i] <- al_freq - } - - # Label X with populations and SNPs - colnames(X) <- names(populations) - rownames(X) <- colnames(geno) - - return(X) -} - - -#' -#' Performs whole genome breed composition prediction. -#' -#' @param Y numeric vector of genotypes (with names as SNPs) from a single animal. -#' coded as dosage of allele B \code{{0, 1, 2, ..., ploidy}} -#' @param X numeric matrix of allele frequencies from reference animals -#' @param p numeric indicating number of breeds represented in X -#' @param names character names of breeds -#' @return data.frame of breed composition estimates -#' @import quadprog -#' @importFrom stats cor -#' @references Funkhouser SA, Bates RO, Ernst CW, Newcom D, Steibel JP. Estimation of genome-wide and locus-specific -#' breed composition in pigs. Transl Anim Sci. 2017 Feb 1;1(1):36-44. -#' -#' @noRd -QPsolve <- function(Y, X) { - - # Remove NAs from Y and remove corresponding - # SNPs from X. Ensure Y is numeric - Ymod <- Y[!is.na(Y)] - Xmod <- X[names(Ymod), ] - - # Determine properties from X matrix - the number of parameters (breeds) p - # and the names of those parameters. - p <- ncol(X) - names <- colnames(X) - - # perfom steps needed to solve OLS by framing - # as a QP problem - # Rinv - matrix should be of dimensions px(p+1) where p is the number of variables in X - Rinv <- solve(chol(t(Xmod) %*% Xmod)) - - # C - the first column is a sum restriction (all equal to 1) and the rest of the columns an identity matrix - C <- cbind(rep(1, p), diag(p)) - - # b2 - This should be a vector of length p+1 the first element is the value of the sum (1) - # the other elements are the restriction of individual coefficients (>) - # so a value 0 produces positive coefficients - b2 <- c(1, rep(0, p)) - - # dd - this should be a matrix NOT a vector - dd <- (t(Ymod) %*% Xmod) - - qp <- solve.QP(Dmat = Rinv, factorized = TRUE, dvec = dd, Amat = C, bvec = b2, meq = 1) - beta <- qp$solution - rr <- cor(Ymod, Xmod %*% beta)^2 - result <- c(beta, rr) - names(result) <- c(names, "R2") - return(result) -} - - -#' Compute Genome-Wide Breed Composition -#' -#' Computes genome-wide breed/ancestry composition using quadratic programming on a -#' batch of animals. -#' -#' @param Y numeric matrix of genotypes (columns) from all animals (rows) in population -#' coded as dosage of allele B \code{{0, 1, 2, ..., ploidy}} -#' @param X numeric matrix of allele frequencies (rows) from each reference panel (columns). Frequencies are -#' relative to allele B. -#' @param ped data.frame giving pedigree information. Must be formatted "ID", "Sire", "Dam" -#' @param groups list of IDs categorized by breed/population. If specified, output will be a list -#' of results categorized by breed/population. -#' @param mia logical. Only applies if ped argument is supplied. If true, returns a data.frame -#' containing the inferred maternally inherited allele for each locus for each animal instead -#' of breed composition results. -#' @param sire logical. Only applies if ped argument is supplied. If true, returns a data.frame -#' containing sire genotypes for each locus for each animal instead of breed composition results. -#' @param dam logical. Only applies if ped argument is supplied. If true, returns a data.frame -#' containing dam genotypes for each locus for each animal instead of breed composition results. -#' @param ploidy integer. The ploidy level of the species (e.g., 2 for diploid, 3 for triploid, etc.). -#' @return A data.frame or list of data.frames (if groups is !NULL) with breed/ancestry composition -#' results -#' @import quadprog -#' @references Funkhouser SA, Bates RO, Ernst CW, Newcom D, Steibel JP. Estimation of genome-wide and locus-specific -#' breed composition in pigs. Transl Anim Sci. 2017 Feb 1;1(1):36-44. -#' -#' @examples -#' # Example inputs for solve_composition_poly (ploidy = 4) -#' -#' # (This would typically be the output from allele_freq_poly) -#' allele_freqs_matrix <- matrix( -#' c(0.625, 0.500, -#' 0.500, 0.500, -#' 0.500, 0.500, -#' 0.750, 0.500, -#' 0.625, 0.625), -#' nrow = 5, ncol = 2, byrow = TRUE, -#' dimnames = list(paste0("SNP", 1:5), c("VarA", "VarB")) -#' ) -#' -#' # Validation Genotypes (individuals x SNPs) -#' val_geno_matrix <- matrix( -#' c(2, 1, 2, 3, 4, # Test1 dosages for SNP1-5 -#' 3, 4, 2, 3, 0), # Test2 dosages for SNP1-5 -#' nrow = 2, ncol = 5, byrow = TRUE, -#' dimnames = list(paste0("Test", 1:2), paste0("SNP", 1:5)) -#' ) -#' -#' # Calculate Breed Composition -#' composition <- solve_composition_poly(Y = val_geno_matrix, -#' X = allele_freqs_matrix, -#' ploidy = 4) -#' print(composition) -#' -#' @export -solve_composition_poly <- function(Y, - X, - ped = NULL, - groups = NULL, - mia = FALSE, - sire = FALSE, - dam = FALSE, - ploidy = 2) { - - # Functions require Y to be animals x SNPs. Transpose - Y <- t(Y) - - # SNPs in Y should only be the ones present in X - Y <- Y[rownames(Y) %in% rownames(X), ] - - # If ped is supplied, use QPsolve_par to compute genomic composition using - # only animals who have genotyped parents (by incorporating Sire genotype). - if (!is.null(ped)) { - mat_results <- lapply(colnames(Y), - QPsolve_par, - Y, - X, - ped, - mia = mia, - sire = sire, - dam = dam) - - mat_results_tab <- do.call(rbind, mat_results) - return (mat_results_tab) - - # Else if groups supplied - perform regular genomic computation - # and list results by groups - } else if (!is.null(groups)) { - - # When using regular genomic computation - adjust dosage based on ploidy - Y <- Y / ploidy #(default is 2) - - grouped_results <- lapply(groups, QPseparate, Y, X) - return (grouped_results) - - # If neither using the ped or grouping option - just perform normal, unsegregated - # calculation - } else { - - # Adjust dosage based on ploidy (default is 2) - Y <- Y / ploidy - - results <- t(apply(Y, 2, QPsolve, X)) - return (results) - } -} diff --git a/R/check_madc_sanity.R b/R/check_madc_sanity.R index 88f564a..ee54d3c 100644 --- a/R/check_madc_sanity.R +++ b/R/check_madc_sanity.R @@ -1,20 +1,24 @@ #' Run basic sanity checks on a MADC-style allele report #' #' @description -#' Performs nine quick validations on an allele report: +#' Performs twelve quick validations on an allele report: #' 1) **Columns** - required columns are present (`CloneID`, `AlleleID`, `AlleleSequence`); #' 2) **FixAlleleIDs** - first column's first up-to-6 rows are not all blank or `"*"` #' *and* both `_0001` and `_0002` appear in `AlleleID`; -#' 3) **IUPACcodes** - presence of non-ATCG characters in `AlleleSequence`; -#' 4) **LowerCase** - presence of lowercase a/t/c/g in `AlleleSequence`; -#' 5) **Indels** - reference/alternate allele lengths differ for the same `CloneID`, +#' 3) **IUPACcodes** - presence of non-ATCG characters in Ref_/Alt_ `AlleleSequence` +#' that differ in position between Ref and Alt, or are present in only one; +#' 4) **IUPACcodes_IdenticalRefAlt** - presence of Ref_/Alt_ pairs that share IUPAC codes +#' at the same positions in their sequences; +#' 5) **LowerCase** - presence of lowercase a/t/c/g in `AlleleSequence`; +#' 6) **Indels** - reference/alternate allele lengths differ for the same `CloneID`, #' or a `"-"` character is present in `AlleleSequence`; -#' 6) **ChromPos** - all `CloneID` values follow the `Chr_Position` format +#' 7) **ChromPos** - all `CloneID` values follow the `Chr_Position` format #' (prefix matches `"chr"` case-insensitively, suffix is a positive integer); -#' 7) **allNAcol** - at least one column contains only `NA` or empty values; -#' 8) **allNArow** - at least one row contains only `NA` or empty values; -#' 9) **RefAltSeqs** - every `CloneID` has at least one `Ref` and one `Alt` allele row; -#' 10) **OtherAlleles** - presence of alleles where the target locus differs from both the Ref and Alt in `AlleleSequence`. +#' 8) **allNAcol** - at least one column contains only `NA` or empty values; +#' 9) **allNArow** - at least one row contains only `NA` or empty values; +#' 10) **RefAltSeqs** - every `CloneID` has at least one `Ref` and one `Alt` allele row; +#' 11) **OtherAlleles** - presence of alleles where the target locus differs from both the Ref and Alt in `AlleleSequence`; +#' 12) **IUPACcodes_MatchAlleles** - presence of non-ATCG characters in `AlleleSequence` for RefMatch_/AltMatch_/Other alleles. #' #' @param report A `data.frame` with at least the columns #' `CloneID`, `AlleleID`, and `AlleleSequence`. The first column is also @@ -30,6 +34,13 @@ #' `_0001`/`_0002` suffixes). #' - **IUPAC check:** Flags any character outside `A`, `T`, `C`, `G` and `"-"` #' (case-insensitive), which includes ambiguity codes (`N`, `R`, `Y`, etc.). +#' For Ref_/Alt_ alleles, two checks are performed: `IUPACcodes` is `TRUE` when +#' any CloneID has IUPAC codes at different positions between Ref and Alt, or +#' IUPAC is present in only one of them. `IUPACcodes_IdenticalRefAlt` is `TRUE` +#' when any CloneID has Ref and Alt with IUPAC codes at exactly the same positions +#' (non-IUPAC bases may still differ). Those CloneIDs are stored in +#' `iupac_identical_clone_ids`. RefMatch_/AltMatch_/Other alleles are checked +#' separately via `IUPACcodes_MatchAlleles`. #' - **Indels:** Rows are split by `AlleleID` containing `"Ref_0001"` vs #' `"Alt_0002"`, merged by `CloneID`, and flagged as indels if either (a) the #' lengths of `AlleleSequence` differ, (b) the sequences have the same length @@ -51,11 +62,12 @@ #' `FixAlleleIDs` are evaluated; all other checks remain `NA` and #' `indel_clone_ids`, `missRef`, and `missAlt` are returned as `NULL`. #' -#' @return A named list with five elements: +#' @return A named list with six elements: #' \describe{ -#' \item{checks}{Named logical vector with nine entries: -#' `Columns`, `FixAlleleIDs`, `IUPACcodes`, `LowerCase`, `Indels`, -#' `ChromPos`, `allNAcol`, `allNArow`, `RefAltSeqs`. +#' \item{checks}{Named logical vector with twelve entries: +#' `Columns`, `FixAlleleIDs`, `IUPACcodes`, `IUPACcodes_IdenticalRefAlt`, +#' `LowerCase`, `Indels`, `ChromPos`, `allNAcol`, `allNArow`, `RefAltSeqs`, +#' `OtherAlleles`, `IUPACcodes_MatchAlleles`. #' `TRUE` means the condition was detected (or passed for `Columns`, #' `FixAlleleIDs`, `ChromPos`, and `RefAltSeqs`); `NA` means the check #' was skipped.} @@ -65,6 +77,11 @@ #' \item{indel_clone_ids}{Character vector of `CloneID`s where ref/alt #' lengths differ. Returns `character(0)` if none are found, or `NULL` #' when required columns are missing.} +#' \item{iupac_identical_clone_ids}{Character vector of `CloneID`s where +#' Ref and Alt alleles have IUPAC codes at the same positions in their +#' sequences (the sequences may differ at standard ATCG positions). Returns +#' `character(0)` if none are found, or `NULL` when required columns are +#' missing.} #' \item{missRef}{Character vector of `CloneID`s that have no `Ref` allele #' row. Returns `character(0)` if all `CloneID`s have a `Ref` row, or #' `NULL` when required columns are missing.} @@ -77,10 +94,14 @@ check_madc_sanity <- function(report) { # Initialize - checks <- c(Columns = NA, FixAlleleIDs = NA, IUPACcodes = NA, LowerCase = NA, Indels = NA, - ChromPos = NA, allNAcol = NA, allNArow = NA, RefAltSeqs = NA, OtherAlleles = NA) - messages <- list(Columns = NA, FixAlleleIDs = NA, IUPACcodes = NA, LowerCase = NA, Indels = NA, - ChromPos = NA, allNAcol = NA, allNArow = NA, RefAltSeqs = NA, OtherAlleles = NA) + checks <- c(Columns = NA, FixAlleleIDs = NA, IUPACcodes = NA, IUPACcodes_IdenticalRefAlt = NA, + LowerCase = NA, Indels = NA, + ChromPos = NA, allNAcol = NA, allNArow = NA, RefAltSeqs = NA, OtherAlleles = NA, + IUPACcodes_MatchAlleles = NA) + messages <- list(Columns = NA, FixAlleleIDs = NA, IUPACcodes = NA, IUPACcodes_IdenticalRefAlt = NA, + LowerCase = NA, Indels = NA, + ChromPos = NA, allNAcol = NA, allNArow = NA, RefAltSeqs = NA, OtherAlleles = NA, + IUPACcodes_MatchAlleles = NA) # ---- FixAlleleIDs ---- # Check if first up-to-6 entries in the *first column* are all "" or "*" @@ -104,8 +125,59 @@ check_madc_sanity <- function(report) { if(checks[["Columns"]]){ # ---- IUPACcodes ---- - iu <- grepl("[^ATCG-]", report$AlleleSequence, ignore.case = TRUE) - checks["IUPACcodes"] <- any(iu, na.rm = TRUE) + # Check for IUPAC codes in Ref_ and Alt_ alleles + # Get Ref and Alt sequences + refs <- subset(report, grepl("Ref_", AlleleID), select = c(CloneID, AlleleID, AlleleSequence)) + alts <- subset(report, grepl("Alt_", AlleleID), select = c(CloneID, AlleleID, AlleleSequence)) + + # Identify sequences with IUPAC codes + refs_with_iupac <- refs[grepl("[^ATCG-]", refs$AlleleSequence, ignore.case = TRUE), ] + alts_with_iupac <- alts[grepl("[^ATCG-]", alts$AlleleSequence, ignore.case = TRUE), ] + + # Merge Ref and Alt by CloneID to compare sequences + merged_iupac <- merge(refs_with_iupac, alts_with_iupac, by = "CloneID", + suffixes = c("_ref", "_alt"), all = FALSE) + + # Check if IUPAC codes are at the same positions in both sequences + if (nrow(merged_iupac) > 0) { + identical_iupac_positions <- vapply(seq_len(nrow(merged_iupac)), function(i) { + ref_seq <- toupper(merged_iupac$AlleleSequence_ref[i]) + alt_seq <- toupper(merged_iupac$AlleleSequence_alt[i]) + + # Split sequences into characters + ref_chars <- strsplit(ref_seq, "")[[1]] + alt_chars <- strsplit(alt_seq, "")[[1]] + + # Check if same length + if (length(ref_chars) != length(alt_chars)) return(FALSE) + + # Check if IUPAC codes are at the same positions + ref_is_iupac <- !ref_chars %in% c("A", "T", "C", "G", "-") + alt_is_iupac <- !alt_chars %in% c("A", "T", "C", "G", "-") + + # IUPAC codes must be at exactly the same positions AND be the same codes + all(ref_is_iupac == alt_is_iupac) && + all(ref_chars[ref_is_iupac] == alt_chars[alt_is_iupac]) + }, logical(1)) + + iupac_identical_clone_ids <- unique(merged_iupac$CloneID[identical_iupac_positions]) + } else { + iupac_identical_clone_ids <- character(0) + } + + # Get all CloneIDs with IUPAC in either Ref or Alt + all_iupac_clones <- unique(c(refs_with_iupac$CloneID, alts_with_iupac$CloneID)) + # IUPAC codes that differ between Ref and Alt (or present in only one) + differing_iupac_clones <- setdiff(all_iupac_clones, iupac_identical_clone_ids) + + # Only flag as TRUE if there are IUPAC codes that differ + checks["IUPACcodes"] <- length(differing_iupac_clones) > 0 + checks["IUPACcodes_IdenticalRefAlt"] <- length(iupac_identical_clone_ids) > 0 + + # Check for IUPAC codes in RefMatch_, AltMatch_, and Other alleles + ref_alt_match_rows <- grepl("RefMatch_|AltMatch_|[|]Other", report$AlleleID) + iu_match <- grepl("[^ATCG-]", report$AlleleSequence[ref_alt_match_rows], ignore.case = TRUE) + checks["IUPACcodes_MatchAlleles"] <- any(iu_match, na.rm = TRUE) # ---- LowerCase ---- lc <- grepl("[atcg]", report$AlleleSequence) @@ -181,6 +253,7 @@ check_madc_sanity <- function(report) { } else { indels <- NULL + iupac_identical_clone_ids <- NULL missRef <- NULL missAlt <- NULL } @@ -189,8 +262,10 @@ check_madc_sanity <- function(report) { "One or more required columns missing. Verify if your file has columns: CloneID, AlleleID, AlleleSequence") messages[["FixAlleleIDs"]] <- c("Fixed Allele IDs look good", "MADC not processed by HapApp") - messages[["IUPACcodes"]] <- c("IUPAC (non-ATCG) codes found in AlleleSequence. This codes are not currently supported by BIGr/BIGapp. Run HapApp to replace them", - "No IUPAC (non-ATCG) codes found in AlleleSequence") + messages[["IUPACcodes"]] <- c("IUPAC (non-ATCG) codes found in Ref_/Alt_ AlleleSequence that differ between alleles or are present in only one. These codes are not currently supported by BIGr/BIGapp", + "No differing IUPAC (non-ATCG) codes found in Ref_/Alt_ AlleleSequence") + messages[["IUPACcodes_IdenticalRefAlt"]] <- c("Identical IUPAC (non-ATCG) codes found at identical positions in Ref and Alt AlleleSequence. These codes will be handled during VCF conversion", + "No IUPAC (non-ATCG) codes found at identical positions in Ref and Alt AlleleSequence") messages[["LowerCase"]] <- c("Lowercase bases found in AlleleSequence", "No lowercase bases found in AlleleSequence") messages[["Indels"]] <- c(paste("Indels found (ref/alt lengths differ or >1 mismatch between same-length sequences) for the CloneIDs:",paste(indels, collapse = " ")), @@ -207,8 +282,11 @@ check_madc_sanity <- function(report) { "Missing Alt: ", paste(missAlt, collapse = " "), ".")) messages[["OtherAlleles"]] <- c("Alleles other than Ref and Alt were found in AlleleID", "No alleles other than Ref and Alt found in AlleleID") + messages[["IUPACcodes_MatchAlleles"]] <- c("IUPAC (non-ATCG) codes found in RefMatch/AltMatch/Other AlleleSequence. These codes are not currently supported by BIGr/BIGapp they will be ignored in the conversion to VCF", + "No IUPAC (non-ATCG) codes found in RefMatch/AltMatch/Other AlleleSequence") list(checks = checks, messages = messages, indel_clone_ids = indels, + iupac_identical_clone_ids = iupac_identical_clone_ids, missRef = missRef, missAlt = missAlt) } diff --git a/R/check_ped.R b/R/check_ped.R deleted file mode 100644 index 1359eba..0000000 --- a/R/check_ped.R +++ /dev/null @@ -1,224 +0,0 @@ -#' Check a pedigree file for accuracy and report/correct common errors -#' -#' `check_ped` reads a 3-column pedigree file (tab-separated, columns labeled `id`, `sire`, `dam` in any order) -#' and performs quality checks, optionally correcting or flagging errors. -#' -#' The function checks for: -#' * Exact duplicate rows and removes them (keeping one copy) -#' * IDs that appear more than once with conflicting sire/dam assignments (sets sire/dam to "0") -#' * IDs that appear in both sire and dam columns -#' * Missing parents (IDs referenced as sire/dam but not in `id` column), adds them with sire/dam = "0" -#' * Direct and indirect pedigree dependencies (cycles), such as a parent being its own descendant -#' -#' After an initial run to clean exact duplicates and repeated IDs, you can run the function again to detect cycles more accurately. -#' -#' The function does **not** overwrite the input file or create objects in the -#' global environment. Instead, it returns the report and corrected pedigree in -#' a list. -#' -#' @param ped.file Path to the pedigree text file. -#' @param seed Optional seed for reproducibility. -#' @param verbose Logical. If TRUE (default), prints the report to the console. -#' -#' @return A list of data.frames containing detected issues: -#' * `exact_duplicates`: rows that were exact duplicates -#' * `repeated_ids_diff`: IDs appearing more than once with conflicting sire/dam -#' * `messy_parents`: IDs appearing as both sire and dam -#' * `missing_parents`: parents added to the pedigree with 0 as sire/dam -#' * `dependencies`: detected cycles in the pedigree -#' * `corrected_pedigree`: corrected pedigree table -#' -#' @examples -#' ped_file <- system.file("check_ped_test.txt", package = "BIGr") -#' ped_errors <- check_ped(ped.file = ped_file, seed = 101919, verbose = FALSE) -#' -#' # Access messy parents -#' ped_errors$messy_parents -#' -#' # Access corrected pedigree -#' ped_errors$corrected_pedigree -#' -#' # IDs with messy parents -#' messy_ids <- ped_errors$messy_parents$id -#' print(messy_ids) -#' -#' @import dplyr -#' @import janitor -#' @importFrom stats setNames -#' @importFrom utils read.table -#' @export -check_ped <- function(ped.file, seed = NULL, verbose = TRUE) { - - #### setup #### - if (!is.null(seed)) set.seed(seed) - - # Read and clean data - data <- utils::read.table(ped.file, header = TRUE) %>% - janitor::clean_names() %>% - mutate( - id = as.character(id), - sire = as.character(sire), - dam = as.character(dam) - ) - - original_data <- data - errors <- list() - missing_parents <- data.frame(id = character(), sire = character(), dam = character(), stringsAsFactors = FALSE) - - #### check 1: exact duplicates #### - exact_duplicates <- data[duplicated(data), ] - if (nrow(exact_duplicates) > 0) { - data <- distinct(data) # remove exact duplicates - } - - #### check 2: repeated IDs with conflicting sire/dam #### - repeated_ids <- data %>% - group_by(id) %>% - filter(n() > 1) %>% - ungroup() - - # Only IDs with actual conflicting sire/dam - conflicting_ids <- repeated_ids %>% - group_by(id) %>% - filter(n_distinct(sire) > 1 | n_distinct(dam) > 1) %>% - ungroup() - - if (nrow(conflicting_ids) > 0) { - # Keep one row per ID, set sire/dam to "0" - data <- data %>% - group_by(id) %>% - summarize( - sire = if(n_distinct(sire) > 1) "0" else first(sire), - dam = if(n_distinct(dam) > 1) "0" else first(dam), - .groups = "drop" - ) - } - - repeated_ids_report <- conflicting_ids - - #### check 3: missing parents #### - for (i in seq_len(nrow(data))) { - id <- data$id[i] - sire <- data$sire[i] - dam <- data$dam[i] - - if (sire != "0" && sire != id && !sire %in% data$id) { - missing_parents <- rbind(missing_parents, data.frame(id = sire, sire = "0", dam = "0", stringsAsFactors = FALSE)) - } - if (dam != "0" && dam != id && !dam %in% data$id) { - missing_parents <- rbind(missing_parents, data.frame(id = dam, sire = "0", dam = "0", stringsAsFactors = FALSE)) - } - - if (sire == id || dam == id) { - errors <- append(errors, paste("Dependency: Individual", id, "cannot be its own parent")) - } - } - - missing_parents <- distinct(missing_parents) - if (nrow(missing_parents) > 0) { - data <- bind_rows(data, missing_parents) - } - - #### check 4: messy parents #### - sire_ids <- unique(data$sire[data$sire != "0"]) - dam_ids <- unique(data$dam[data$dam != "0"]) - messy_ids <- intersect(sire_ids, dam_ids) - messy_parents <- data %>% filter(id %in% messy_ids) - - #### check 5: dependencies (cycles) #### - detect_all_cycles <- function(data) { - adj_list <- lapply(data$id, function(x) { - row <- data[data$id == x, ] - c(row$sire, row$dam) - }) - names(adj_list) <- data$id - - dfs <- function(node, visited, rec_stack, path) { - visited[node] <- TRUE - rec_stack[node] <- TRUE - path <- append(path, node) - cycles <- list() - - for (neighbor in adj_list[[node]]) { - if (neighbor %in% names(adj_list)) { - if (!visited[neighbor]) { - cycles <- append(cycles, dfs(neighbor, visited, rec_stack, path)) - } else if (rec_stack[neighbor]) { - cycle_start <- match(neighbor, path) - cycles <- append(cycles, list(path[cycle_start:length(path)])) - } - } - } - - rec_stack[node] <- FALSE - return(cycles) - } - - visited <- stats::setNames(rep(FALSE, length(adj_list)), names(adj_list)) - rec_stack <- stats::setNames(rep(FALSE, length(adj_list)), names(adj_list)) - all_cycles <- list() - - for (node in names(adj_list)) { - if (!visited[node]) { - node_cycles <- dfs(node, visited, rec_stack, character()) - if (length(node_cycles) > 0) { - all_cycles <- append(all_cycles, node_cycles) - } - } - } - return(all_cycles) - } - - cycles <- detect_all_cycles(data) - if (length(cycles) > 0) { - for (cycle_group in cycles) { - cycle_ids <- unique(unlist(cycle_group)) - errors <- append(errors, paste("Cycle detected involving IDs:", paste(cycle_ids, collapse = " -> "))) - } - } - - #### compile findings #### - input_ped_report <- list( - exact_duplicates = exact_duplicates, - repeated_ids_diff = repeated_ids_report, - messy_parents = messy_parents, - missing_parents = missing_parents, - dependencies = data.frame(Dependency = unique(unlist(errors))), - corrected_pedigree = data - ) - - #### output #### - if (verbose) { - cat("\n=== Pedigree Quality Check Report ===\n") - - if (nrow(exact_duplicates) > 0) { - cat("\n Exact duplicate trios detected (only one copy will be kept in corrected pedigree):\n") - print(exact_duplicates) - } else cat("\nNo exact duplicate trios found.\n") - - if (nrow(repeated_ids_report) > 0) { - cat("\nConflicting trios detected (sire/dam set to 0 in corrected pedigree):\n") - print(repeated_ids_report) - } else cat("\nNo conflicting repeated trios found.\n") - - if (nrow(missing_parents) > 0) { - cat("\n Parents missing as IDs found in the pedigree (will be added as founders in corrected pedigree):\n") - print(missing_parents) - } else cat("\nNo missing parents found.\n") - - if (nrow(messy_parents) > 0) { - cat("\n IDs found as both sire and dam (is selfing or hermaphrodytism possible?):\n") - print(messy_parents) - } else cat("\nNo IDs found as both sire and dam.\n") - - - if (nrow(input_ped_report$dependencies) > 0) { - cat("\nDependencies detected:\n") - print(input_ped_report$dependencies) - } else cat("\nNo dependencies detected.\n") - - cat("\nThe corrected pedigree is included in the returned list as `corrected_pedigree`.\n") - } - - invisible(input_ped_report) -} diff --git a/R/filterMADC.R b/R/filterMADC.R index 5c1056f..2a905ac 100644 --- a/R/filterMADC.R +++ b/R/filterMADC.R @@ -3,25 +3,32 @@ #' Filter and process MADC files to remove low quality microhaplotypes #' #' @details -#' This function can filter raw MADC files or pre-processed MADC files with fixed allele IDs. Additionally, -#' it can filter based on mean read depth, number of mhaps per target loci, and other criteria. Optionally, users -#' can plot summary statistics and save the filtered data to a file. +#' This function filters a fixed allele ID MADC file (one processed through HapApp so that +#' AlleleIDs carry unique \code{|Ref_0001}/\code{|Alt_0002}-style suffixes). The input is +#' validated with \code{\link{check_madc_sanity}}; raw DArT MADC files (or any file without +#' fixed AlleleIDs) are rejected with an error. +#' +#' Filtering is designed so that a locus never loses its target \code{|Ref}/\code{|Alt} pair: +#' the per-locus read-depth window (\code{min.locus.depth}/\code{max.locus.depth}) removes +#' whole loci (Ref and Alt together), and the other filters +#' (\code{max.mhaps.per.loci}, \code{min.ind.with.reads}, \code{target.only}) only prune +#' non-target \code{|RefMatch}/\code{|AltMatch}/\code{|Other} mhaps. The filtered data can +#' optionally be saved to a file. #' #'@import dplyr -#'@importFrom utils read.csv +#'@importFrom utils read.csv write.csv #' -#'@param madc_file Path to the MADC file to be filtered -#'@param min.mean.reads Minimum mean read depth for filtering -#'@param max.mean.reads Maximum mean read depth for filtering -#'@param max.mhaps.per.loci Maximum number of matching mhaps per target loci. Retains only the target Ref and Alt loci at the sites that exceeds the \code{max.mhaps.per.loci} threshold. -#'@param min.reads.per.site Minimum number of reads per site for \code{min.ind.with.reads}. Otherwise, this parameter is ignored -#'@param min.ind.with.reads Minimum number of individuals with \code{min.reads.per.site} reads for filtering -#'@param target.only Logical indicating whether to filter for target loci only -#'@param n.summary.columns (optional) Number of summary columns to remove from MADC file not including the first three. Otherwise, the columns will be automatically detected and removed. +#'@param madc_file Path to the fixed allele ID MADC file to be filtered +#'@param min.locus.depth Minimum mean read depth per sample at a locus. Depth is the sum of all mhap reads at the locus (\code{|Ref}, \code{|Alt}, \code{|RefMatch}, \code{|AltMatch}, and \code{|Other}) divided by the number of samples. Loci below this threshold are removed entirely (Ref and Alt together). If NULL, no minimum is applied. +#'@param max.locus.depth Maximum mean read depth per sample at a locus (computed as for \code{min.locus.depth}). Loci above this threshold are removed entirely, e.g. to exclude paralogous over-amplification. If NULL, no maximum is applied. +#'@param max.mhaps.per.loci Maximum number of mhaps per target loci. At loci whose total number of mhaps exceeds the \code{max.mhaps.per.loci} threshold, only the target \code{|Ref} and \code{|Alt} alleles are retained (the \code{|RefMatch}, \code{|AltMatch}, and \code{|Other} alleles are removed). +#'@param min.reads.per.site Minimum number of reads for a site (mhap in a sample) to count toward \code{min.ind.with.reads}. +#'@param min.ind.with.reads Minimum number of individuals with \code{min.reads.per.site} reads required to retain a mhap. Target \code{|Ref}/\code{|Alt} alleles are always retained; only non-target (\code{|RefMatch}/\code{|AltMatch}/\code{|Other}) mhaps are removed by this filter. +#'@param target.only Logical indicating whether to retain only the target \code{|Ref} and \code{|Alt} alleles (dropping \code{|RefMatch}, \code{|AltMatch}, and \code{|Other}) #@param plot.summary Logical indicating whether to plot summary statistics -#'@param output.file Path to save the filtered data (if NULL, data will not be saved) +#'@param output.file Path to save the filtered data; \code{.csv} is appended to the name (if NULL, data will not be saved) #' -#'@return data.frame or saved csv file +#'@return A data.frame of the filtered MADC when \code{output.file} is NULL. Otherwise the filtered data is saved to \code{paste0(output.file, ".csv")} and NULL is returned. #' #'@examples #' #Example @@ -31,123 +38,94 @@ #' #' #Remove mhaps exceeding 3 per target region including the ref and alt target mhaps #' filtered_df <- filterMADC(madc_file, -#' min.mean.reads = NULL, -#' max.mean.reads = NULL, +#' min.locus.depth = NULL, +#' max.locus.depth = NULL, #' max.mhaps.per.loci = 3, #' min.reads.per.site = 1, #' min.ind.with.reads = NULL, #' target.only = FALSE, -#' n.summary.columns = NULL, #' output.file = NULL) #' #' #' #'@export filterMADC <- function(madc_file, - min.mean.reads = NULL, - max.mean.reads = NULL, + min.locus.depth = NULL, + max.locus.depth = NULL, max.mhaps.per.loci = NULL, min.reads.per.site = 1, min.ind.with.reads = NULL, target.only = FALSE, - n.summary.columns = NULL, #plot.summary = FALSE, output.file = NULL) { - #Need to first inspect the first 7 rows of the MADC to see if it has been preprocessed or not - first_seven_rows <- read.csv(madc_file, header = FALSE, nrows = 7, colClasses = c(NA, "NULL")) - - #Check if all entries in the first column are either blank or "*" - check_entries <- all(first_seven_rows[, 1] %in% c("", "*")) - - #Check if the MADC file has the filler rows or is processed from updated fixed allele ID pipeline - if (check_entries) { - #Note: This assumes that the first 7 rows are placeholder info from DArT processing - - warning("The MADC file has not been pre-processed for Fixed Allele IDs. The first 7 rows are placeholder info from DArT processing.") + #Read and sanity-check (fixed allele ID MADC only) + report <- read.csv(madc_file, check.names = FALSE) + checks <- check_madc_sanity(report) - #Read the madc file - filtered_df <- read.csv(madc_file, sep = ',', skip = 7, check.names = FALSE) + #Surface all sanity checks as informational messages (non-blocking) + msgs <- mapply(function(check, message) if (isTRUE(check)) message[1] else message[2], + checks$checks, checks$messages) + for (i in seq_along(msgs)) message(msgs[i]) - #Remove extra text after Ref and Alt (_001 or _002) - #filtered_df$AlleleID <- sub("\\|Ref_.*", "|Ref", filtered_df$AlleleID) - #filtered_df$AlleleID <- sub("\\|Alt_.*", "|Alt", filtered_df$AlleleID) + #Hard stops: only accept fixed allele ID MADC files with the required columns + if (!isTRUE(checks$checks[["Columns"]])) + stop("The MADC file is missing required columns (CloneID, AlleleID, AlleleSequence)") - } else { + if (!isTRUE(checks$checks[["FixAlleleIDs"]])) + stop("The MADC file does not have fixed AlleleIDs. Please process the MADC file through HapApp before using this function.") - #Read the madc file - filtered_df <- read.csv(madc_file, sep = ',', check.names = FALSE) + #Fixed allele ID MADC has a standard layout: columns 1:3 are the ID columns + #(AlleleID, CloneID, AlleleSequence) and columns 4:ncol are numeric samples. + filtered_df <- report - #Remove extra text after Ref and Alt (_001 or _002) - #filtered_df$AlleleID <- sub("\\|Ref_.*", "|Ref", filtered_df$AlleleID) - #filtered_df$AlleleID <- sub("\\|Alt_.*", "|Alt", filtered_df$AlleleID) + ## Filtering + #Per-locus depth window (runs first, on the full mhap set so paralogous + #over-amplification is measured before any trimming). Depth is the mean reads + #per sample at a locus = (sum of all mhap reads at the locus) / number of samples. + #Loci outside [min.locus.depth, max.locus.depth] are removed whole (Ref and Alt + #together), so retained loci always keep their Ref/Alt pair intact. + if (!is.null(min.locus.depth) || !is.null(max.locus.depth)) { + n_samples <- ncol(filtered_df) - 3 + row_totals <- rowSums(filtered_df[, -c(1:3), drop = FALSE], na.rm = TRUE) + locus_depth <- tapply(row_totals, filtered_df$CloneID, sum) / n_samples + + keep_mask <- rep(TRUE, length(locus_depth)) + if (!is.null(min.locus.depth)) { + message("Filtering for minimum mean read depth per sample per locus") + keep_mask <- keep_mask & (locus_depth >= min.locus.depth) + } + if (!is.null(max.locus.depth)) { + message("Filtering for maximum mean read depth per sample per locus") + keep_mask <- keep_mask & (locus_depth <= max.locus.depth) + } + keep_loci <- names(locus_depth)[keep_mask] + filtered_df <- filtered_df[filtered_df$CloneID %in% keep_loci, ] } - #Check for extra columns - #Save the three columns for later adding to the output - saved_columns <- filtered_df[,1:3] - - if (!is.null(n.summary.columns)) { - #Remove the first n.summary.columns columns - filtered_df <- filtered_df[,-c(4:n.summary.columns)] - }else{ - rm.col <- c("ClusterConsensusSequence", - "CallRate", "OneRatioRef", "OneRatioSnp", "FreqHomRef", "FreqHomSnp", - "FreqHets", "PICRef", "PICSnp", "AvgPIC", "AvgCountRef", "AvgCountSnp","RatioAvgCountRefAvgCountSnp") - - filtered_df <- filtered_df[, !(colnames(filtered_df) %in% rm.col)] - } - - #Now add rownames - rownames(filtered_df) <- saved_columns[,1] - #Remove refmatch and altmatch if wanted + #Remove refmatch, altmatch, and other if wanted (retain only the target Ref and Alt haplotypes) if (target.only) { message("Retaining target markers only") - #Retain only the Ref and Alt haplotypes - filtered_df <- filtered_df[!grepl("\\|AltMatch|\\|RefMatch", filtered_df$AlleleID), ] - } - - ## Filtering - - #Min mean reads - if (!is.null(min.mean.reads)) { - message("Filtering for minimum mean reads across all samples") - #Get the mean value for each row, and remove the rows below that threshold - filtered_df$MeanReads <- rowMeans(filtered_df[, -c(1:3)], na.rm = TRUE) - filtered_df <- filtered_df[filtered_df$MeanReads >= min.mean.reads, ] - #Remove the MeanReads column - filtered_df <- filtered_df[, -which(colnames(filtered_df) == "MeanReads")] - } - - #Max mean reads - if (!is.null(max.mean.reads)) { - message("Filtering for maximum mean reads across all samples") - #Get the mean value for each row, and remove the rows above that threshold - filtered_df$MeanReads <- rowMeans(filtered_df[, -c(1:3)], na.rm = TRUE) - filtered_df <- filtered_df[filtered_df$MeanReads <= max.mean.reads, ] - #Remove the MeanReads column - filtered_df <- filtered_df[, -which(colnames(filtered_df) == "MeanReads")] + filtered_df <- filtered_df[grepl("\\|(Ref|Alt)_", filtered_df$AlleleID), ] } #Max mhaps per loci if (!is.null(max.mhaps.per.loci)) { message("Filtering for maximum number of matching mhaps per target loci") - #Get the number of matching mhaps for loci, and remove the mhaps at those loci that exceed the max number - mhap_counts <- filtered_df %>% + #Count ALL mhaps per locus; at loci exceeding the max, retain only the target Ref/Alt alleles + clone_ids_to_target <- filtered_df %>% group_by(CloneID) %>% summarise(Count = n(), .groups = 'drop') %>% - filter(Count > max.mhaps.per.loci) - - patterns_to_search <- "\\|AltMatch|\\|RefMatch" - clone_ids_to_target <- mhap_counts$CloneID + filter(Count > max.mhaps.per.loci) %>% + pull(CloneID) filtered_df <- filtered_df %>% filter( !( # "keep rows that DO NOT match both conditions" - CloneID %in% clone_ids_to_target & # Condition 1: CloneID is one of the targeted IDs - grepl(patterns_to_search, AlleleID) # Condition 2: AlleleID contains one of the patterns + CloneID %in% clone_ids_to_target & # Condition 1: locus is over the threshold + !grepl("\\|(Ref|Alt)_", AlleleID) # Condition 2: AlleleID is NOT a target Ref/Alt ) ) } @@ -171,8 +149,9 @@ filterMADC <- function(madc_file, ) ) %>% ungroup() %>% # Always ungroup after rowwise operations - # Filter rows where this count meets the 'min.ind.with.reads' threshold - filter(qualifying_sites_count >= min.ind.with.reads) %>% + # Always retain target Ref/Alt; only prune non-target (Match/Other) mhaps that + # fail the 'min.ind.with.reads' threshold, so a locus never loses its Ref/Alt pair + filter(grepl("\\|(Ref|Alt)_", AlleleID) | qualifying_sites_count >= min.ind.with.reads) %>% # Optionally, remove the temporary count column if it's no longer needed select(-qualifying_sites_count) } diff --git a/R/filterVCF.R b/R/filterVCF.R index a54e32e..0b71af6 100644 --- a/R/filterVCF.R +++ b/R/filterVCF.R @@ -6,19 +6,59 @@ #' The output file will be saved to the location and with the name that is specified. #' The VCF format is v4.3 #' +#' Filters are applied in a fixed order, which matters because each one acts on +#' what the previous ones left behind. `filter.DP` and `filter.MPP` run first and +#' set individual genotype calls to missing. `filter.OD`, `filter.BIAS.min` and +#' `filter.BIAS.max`, and `filter.PMC` then remove variants using the INFO column. +#' `filter.SNP.miss` removes variants and `filter.SAMPLE.miss` removes samples, +#' both counting the calls that were set to missing above. `filter.MAF` runs last, +#' so it is calculated from the samples that remain. +#' +#' A variant is removed if the value a requested filter needs cannot be read from +#' it, for instance because the INFO field is absent from that record or because +#' every genotype call for the variant is missing. The number of variants removed +#' for this reason is reported in a warning. +#' #' @param vcf.file vcfR object or path to VCF file. Can be unzipped (.vcf) or gzipped (.vcf.gz). -#' @param filter.OD Updog filter -#' @param filter.BIAS.min Updog filter (requires a value for both BIAS.min and BIAS.max) -#' @param filter.BIAS.max Updog filter (requires a value for both BIAS.min and BIAS.max) -#' @param filter.DP Total read depth at each SNP filter -#' @param filter.MPP Updog filter -#' @param filter.PMC Updog filter -#' @param filter.MAF Minor allele frequency filter -#' @param filter.SAMPLE.miss Sample missing data filter -#' @param filter.SNP.miss SNP missing data filter -#' @param ploidy The ploidy of the species being analyzed -#' @param output.file output file name (optional). If no output.file name provided, then a vcfR object will be returned. -#' @return A gzipped vcf file +#' @param filter.OD Maximum overdispersion, read from the `OD` field of the INFO +#' column as estimated by updog. Variants with an `OD` below this value are kept. +#' @param filter.BIAS.min Minimum allele bias, read from the `BIAS` field of the +#' INFO column as estimated by updog. Variants with a `BIAS` above this value are +#' kept. Has no effect unless `filter.BIAS.max` is also supplied. +#' @param filter.BIAS.max Maximum allele bias, read from the `BIAS` field of the +#' INFO column. Variants with a `BIAS` below this value are kept. Has no effect +#' unless `filter.BIAS.min` is also supplied. +#' @param filter.DP Minimum read depth for a genotype call. Calls whose FORMAT/DP +#' is below this value are set to missing. Calls without a DP value are left +#' unchanged. This does not remove variants, though the calls it sets to missing +#' then count towards `filter.SNP.miss`, `filter.SAMPLE.miss` and `filter.MAF`. +#' @param filter.MPP Minimum posterior probability for a genotype call, as +#' reported by updog. Calls whose FORMAT/MPP is below this value are set to +#' missing, on the same terms as `filter.DP`. +#' @param filter.PMC Maximum proportion of individuals misclassified, read from +#' the `PMC` field of the INFO column as estimated by updog. Variants with a +#' `PMC` below this value are kept. +#' @param filter.MAF Minimum minor allele frequency. Variants with a minor allele +#' frequency above this value are kept. Calculated after any sample removal, so +#' it reflects only the samples that survive `filter.SAMPLE.miss`. +#' @param filter.SAMPLE.miss Maximum proportion of missing genotype calls a sample +#' may have, between 0 and 1. Samples missing a smaller proportion than this are +#' kept. Note that this is a proportion and not a percentage; values outside 0 +#' to 1 give an error. +#' @param filter.SNP.miss Maximum proportion of missing genotype calls a variant +#' may have, between 0 and 1. Variants missing a smaller proportion than this are +#' kept. Note that this is a proportion and not a percentage; values outside 0 +#' to 1 give an error. +#' @param ploidy The ploidy of the species being analyzed. Required. Used to write +#' the missing genotype, so a diploid is recorded as `./.` and a tetraploid as +#' `./././.` when a call is filtered out. +#' @param output.file Output file name, without an extension (optional). When +#' supplied, the filtered VCF is written as a gzipped file and nothing is +#' returned. `.vcf.gz` is appended when `vcf.file` is a path, and +#' `_filtered.vcf.gz` when `vcf.file` is a vcfR object. When not supplied, a +#' vcfR object is returned instead and no file is written. +#' @return A vcfR object when `output.file` is not supplied. Otherwise the +#' filtered VCF is written to disk and nothing is returned. #' @importFrom vcfR read.vcfR #' @importFrom vcfR write.vcf #' @importFrom vcfR maf @@ -56,6 +96,37 @@ filterVCF <- function(vcf.file, #Should allow for any INFO field to be entered to be filtered + # A threshold that is not a single readable number turns the comparison it + # feeds into an NA index, which silently corrupts or skips the filter rather + # than erroring, so all thresholds are validated before anything else happens. + validate_filter_number <- function(x, name, proportion = FALSE) { + if (is.null(x)) return(NULL) + if (!(is.numeric(x) || is.character(x)) || length(x) != 1L) { + stop(name, " must be a single numeric value or NULL.", call. = FALSE) + } + value <- suppressWarnings(as.numeric(x)) + if (!is.finite(value)) { + stop(name, " must be a single finite numeric value or NULL, not ", + deparse(x), ".", call. = FALSE) + } + # A percentage such as 20 would otherwise keep every variant or sample + if (proportion && (value < 0 || value > 1)) { + stop(name, " is a proportion and must be between 0 and 1, not ", + value, ".", call. = FALSE) + } + value + } + + filter.OD <- validate_filter_number(filter.OD, "filter.OD") + filter.BIAS.min <- validate_filter_number(filter.BIAS.min, "filter.BIAS.min") + filter.BIAS.max <- validate_filter_number(filter.BIAS.max, "filter.BIAS.max") + filter.DP <- validate_filter_number(filter.DP, "filter.DP") + filter.MPP <- validate_filter_number(filter.MPP, "filter.MPP") + filter.PMC <- validate_filter_number(filter.PMC, "filter.PMC") + filter.MAF <- validate_filter_number(filter.MAF, "filter.MAF") + filter.SAMPLE.miss <- validate_filter_number(filter.SAMPLE.miss, "filter.SAMPLE.miss", proportion = TRUE) + filter.SNP.miss <- validate_filter_number(filter.SNP.miss, "filter.SNP.miss", proportion = TRUE) + # Import VCF (can be .vcf or .vcf.gz) if (!inherits(vcf.file, "vcfR")) { vcf <- read.vcfR(vcf.file, verbose = FALSE) @@ -89,43 +160,48 @@ filterVCF <- function(vcf.file, starting_snps <- nrow(vcf) starting_samples <- ncol(vcf@gt)-1 #subtract 1 to not include the FORMAT column - # Determine the number of items in the FORMAT field - format_string <- vcf@gt[1, "FORMAT"] - format_fields <- strsplit(format_string, ":")[[1]] - num_fields <- length(format_fields) - gt_pos <- which(format_fields == "GT") - # Create the NA format string, replacing only GT with "./." + # The FORMAT fields are read from every record rather than the first one only, + # because the fields present can differ between records + format_fields <- strsplit(vcf@gt[, "FORMAT"], ":", fixed = TRUE) + # Create the NA format string for each record, replacing only GT with "./." missing_gt <- paste(rep(".", ploidy), collapse = "/") - na_fields <- rep(".", num_fields) - na_fields[gt_pos] <- missing_gt - na_format <- paste(na_fields, collapse = ":") + na_format <- vapply(format_fields, function(fields) { + na_fields <- rep(".", length(fields)) + na_fields[fields == "GT"] <- missing_gt + paste(na_fields, collapse = ":") + }, character(1)) + format_has <- function(field) any(vapply(format_fields, function(fields) field %in% fields, logical(1))) # Extract the DP values - if ("DP" %in% format_fields && !is.null(filter.DP)) { + if (!is.null(filter.DP) && format_has("DP")) { message("Filtering by DP\n") dp <- extract.gt(vcf, element = "DP", as.numeric = TRUE) - # Identify cells to modify based on the DP threshold + # Identify cells to modify based on the DP threshold. Calls without a DP + # value are left unchanged threshold <- as.numeric(filter.DP) - to_modify <- dp < threshold + to_modify <- !is.na(dp) & dp < threshold # Replace cells in the vcf@gt matrix with NA format string where to_modify is TRUE - vcf@gt[, -1][to_modify] <- na_format + na_matrix <- matrix(na_format, nrow = nrow(dp), ncol = ncol(dp)) + vcf@gt[, -1][to_modify] <- na_matrix[to_modify] # Remove extra matrices - rm(to_modify) + rm(to_modify, na_matrix) rm(dp) } #Filter if the MPP field is present - if ("MPP" %in% format_fields && !is.null(filter.MPP)) { + if (!is.null(filter.MPP) && format_has("MPP")) { message("Filtering by MPP\n") # Extract the MPP values mpp <- extract.gt(vcf, element = "MPP", as.numeric = TRUE) - # Identify cells to modify based on the DP threshold - threshold <- as.numeric(filter.MPP) #Need to make a variable for user to enter - to_modify <- mpp < threshold + # Identify cells to modify based on the MPP threshold. Calls without an MPP + # value are left unchanged + threshold <- as.numeric(filter.MPP) + to_modify <- !is.na(mpp) & mpp < threshold # Replace cells in the vcf@gt matrix with NA format string where to_modify is TRUE - vcf@gt[, -1][to_modify] <- na_format + na_matrix <- matrix(na_format, nrow = nrow(mpp), ncol = ncol(mpp)) + vcf@gt[, -1][to_modify] <- na_matrix[to_modify] #remove extra matrices - rm(to_modify) + rm(to_modify, na_matrix) rm(mpp) } @@ -135,63 +211,79 @@ filterVCF <- function(vcf.file, info <- vcf@fix[, "INFO"] #Need to get after each filter.. # Function to extract a specific INFO field value + # The field is located by splitting on ';' so that the name is matched against a + # whole INFO entry, and the value is passed to as.numeric() unaltered so that any + # valid numeric representation is accepted. This includes scientific notation + # (e.g. PMC=4.78e-07), which updog2vcf() writes whenever a value is small enough + # for R's default formatting to use it. extract_info_value <- function(info, field) { - pattern <- paste0(".*", field, "=([0-9.]+).*") - values <- as.numeric(sub(pattern, "\\1", info)) + prefix <- paste0(field, "=") + values <- vapply(strsplit(info, ";", fixed = TRUE), function(parts) { + hit <- parts[startsWith(parts, prefix)] + if (length(hit) == 0L) { + NA_real_ + } else { + suppressWarnings(as.numeric(substring(hit[1L], nchar(prefix) + 1L))) + } + }, numeric(1)) return(values) } - # Function to extract INFO IDs from a single INFO string - extract_info_ids <- function(info_string) { - # Split the INFO string by ';' - info_parts <- strsplit(info_string, ";")[[1]] - # Extract the part before the '=' in each segment - info_ids <- gsub("=.*", "", info_parts) - return(info_ids) + # Build a row selection from a filter comparison. An NA in a logical index does + # not drop a variant, it inserts an all-NA row into the VCF, so variants whose + # value could not be read are removed explicitly here. The count is reported so + # that they are never discarded silently. + select_variants <- function(keep, values, label, hint) { + unusable <- is.na(values) + if (length(values) > 0 && all(unusable)) { + # Losing every variant is usually a problem with the input rather than a + # genuine result, so this case says what to look at instead of only + # reporting the count. + warning("No readable ", label, " values were found, so all ", length(values), + " variants were removed. ", hint, call. = FALSE) + } else if (any(unusable)) { + warning(sum(unusable), " of ", length(values), " variants had a missing or ", + "unreadable ", label, " value and were removed.", call. = FALSE) + } + return(!is.na(keep) & keep & !unusable) } - # Apply the function to the first INFO string - info_ids <- extract_info_ids(info[1]) + # A requested filter is applied to every record. Which INFO fields are present + # is not decided from the first record, because a field may legally be absent + # from any individual record; records without a usable value are removed by + # select_variants() and reported there. # Filtering by OD - if ("OD" %in% info_ids && !is.null(filter.OD)) { + if (!is.null(filter.OD)) { info <- vcf@fix[, "INFO"] #Need to get after each filter.. message("Filtering by OD\n") od_values <- extract_info_value(info, "OD") - # Ensure no NA values before filtering - if (!all(is.na(od_values))) { - vcf <- vcf[od_values < as.numeric(filter.OD), ] - } else { - warning("No valid OD values found.\n") - } + vcf <- vcf[select_variants(od_values < as.numeric(filter.OD), + od_values, "OD", + "Check that the INFO column of the VCF records OD values."), ] } info <- vcf@fix[, "INFO"] #Need to get after each filter.. # Filtering by BIAS - if ("BIAS" %in% info_ids && !is.null(filter.BIAS.min) && !is.null(filter.BIAS.max)) { + if (!is.null(filter.BIAS.min) && !is.null(filter.BIAS.max)) { info <- vcf@fix[, "INFO"] #Need to get after each filter.. message("Filtering by BIAS\n") bias_values <- extract_info_value(info, "BIAS") - # Ensure no NA values before filtering - if (!all(is.na(bias_values))) { - vcf <- vcf[bias_values > as.numeric(filter.BIAS.min) & bias_values < as.numeric(filter.BIAS.max), ] - } else { - warning("No valid BIAS values found.\n") - } + vcf <- vcf[select_variants(bias_values > as.numeric(filter.BIAS.min) & + bias_values < as.numeric(filter.BIAS.max), + bias_values, "BIAS", + "Check that the INFO column of the VCF records BIAS values."), ] } # Filtering by PMC - if ("PMC" %in% info_ids && !is.null(filter.PMC)) { + if (!is.null(filter.PMC)) { info <- vcf@fix[, "INFO"] #Need to get after each filter.. message("Filtering by PMC\n") pmc_values <- extract_info_value(info, "PMC") - # Ensure no NA values before filtering - if (!all(is.na(pmc_values))) { - vcf <- vcf[pmc_values < as.numeric(filter.PMC), ] - } else { - warning("No valid PMC values found.\n") - } + vcf <- vcf[select_variants(pmc_values < as.numeric(filter.PMC), + pmc_values, "PMC", + "Check that the INFO column of the VCF records PMC values."), ] } # Example: Filter based on missing data for samples and SNPs @@ -226,7 +318,11 @@ filterVCF <- function(vcf.file, if (!is.null(filter.MAF)) { message("Filtering by MAF\n") maf_df <- data.frame(vcfR::maf(vcf, element = 2)) - vcf <- vcf[maf_df$Frequency > as.numeric(filter.MAF), ] + # maf() returns NA for a variant with no called genotypes, which the DP and + # MPP masking above can produce. + vcf <- vcf[select_variants(maf_df$Frequency > as.numeric(filter.MAF), + maf_df$Frequency, "MAF", + "This happens when no variant has any called genotypes, which the filter.DP and filter.MPP masking can cause."), ] } ### Export the modified VCF file (this exports as a .vcf.gz, so make sure to have the name end in .vcf.gz) message("Exporting VCF\n") diff --git a/R/find_parentage.R b/R/find_parentage.R deleted file mode 100644 index b220a46..0000000 --- a/R/find_parentage.R +++ /dev/null @@ -1,423 +0,0 @@ -#' Find Parentage Assignments for Progeny -#' -#' Assigns the most likely parent(s) to each progeny individual based on -#' genotypic data using Mendelian error rates or homozygous mismatch rates. -#' -#' @param genotypes_file Path to a TSV/CSV/TXT file containing genotype data. -#' Must include an 'ID' column followed by marker columns coded as 0, 1, 2 -#' (allele dosage). -#' @param parents_file Path to a TSV/CSV/TXT file listing candidate parent IDs. -#' Must include an 'ID' column. An optional 'Sex' column with values -#' 'M' (male parent), 'F' (female parent), or 'A' (ambiguous) determines -#' which parents are tested for each role. If absent, all parents are treated -#' as ambiguous. -#' @param progeny_file Path to a TSV/CSV/TXT file listing progeny IDs to assign. -#' Must include an 'ID' column. -#' @param method Character. Parentage assignment method. One of: -#' \itemize{ -#' \item \code{"best_male_parent"} - finds the best male parent for each -#' progeny using homozygous mismatch rate. -#' \item \code{"best_female_parent"} - finds the best female parent for each -#' progeny using homozygous mismatch rate. -#' \item \code{"best_match"} - finds the single best parent (either sex) -#' using homozygous mismatch rate. -#' \item \code{"best_pair"} - finds the best male-female parent pair for -#' each progeny using full Mendelian error rate (default). -#' } -#' @param min_markers Integer. Minimum number of non-missing markers required -#' to report a parentage assignment. Progeny-parent comparisons with fewer -#' markers are flagged as \code{LOW_MARKERS} and no assignment is made -#' (default: \code{10}). -#' @param error_threshold Numeric. Maximum mismatch percentage to report a -#' parentage assignment as confident. Assignments above this threshold are -#' flagged as \code{HIGH_ERROR} in the \code{Assignment_Status} column -#' (default: \code{5.0}). Must be between 0 and 100. -#' @param show_ties Logical. If \code{TRUE}, all tied best pairs (after -#' tie-breaking by maximum markers tested) are reported as additional columns -#' (\code{Male_Parent_1}, \code{Male_Parent_2}, etc.) when -#' \code{method = "best_pair"}. The base columns (\code{Male_Parent}, -#' \code{Female_Parent}, etc.) are always populated with the top result. -#' If \code{FALSE}, only one tied pair is reported with a warning. -#' Default is \code{TRUE}. -#' @param allow_selfing Logical. If \code{FALSE}, male-female parent pairs where -#' both IDs are identical are excluded when \code{method = "best_pair"}. -#' Default is \code{TRUE}. -#' @param verbose Logical. If \code{TRUE}, prints progress messages, summary -#' statistics, and the results table to the console. Default is \code{TRUE}. -#' @param write_txt Logical. If \code{TRUE}, writes results to -#' \code{parentage_results_dt.txt} in the working directory. Default is -#' \code{TRUE}. -#' -#' @return A \code{data.table} with one row per progeny. Columns depend on the -#' method used: -#' \itemize{ -#' \item \code{best_male_parent} / \code{best_female_parent} / \code{best_match}: -#' \code{Progeny}, \code{Best_Match}, \code{Mendelian_Error_Pct}, -#' \code{Markers_Tested}, \code{Assignment_Status}. -#' \item \code{best_pair} (no ties after tie-breaking): \code{Progeny}, -#' \code{Male_Parent}, \code{Female_Parent}, \code{Mendelian_Error_Pct}, -#' \code{Markers_Tested}, \code{Assignment_Status}. -#' \item \code{best_pair} (ties remain after tie-breaking, -#' \code{show_ties = TRUE}): base columns are always populated with the -#' top result, plus suffix columns \code{Male_Parent_1}, -#' \code{Female_Parent_1}, etc. for each tied pair. -#' } -#' \code{Assignment_Status} is one of \code{PASS}, \code{HIGH_ERROR}, or -#' \code{LOW_MARKERS}. Returned invisibly when \code{verbose = TRUE}. -#' -#' @details -#' A homozygous-only genotype matrix is pre-computed once at startup and shared -#' across all methods that require it, avoiding redundant computation. -#' -#' For \code{"best_male_parent"}, \code{"best_female_parent"}, and -#' \code{"best_match"}, only homozygous markers (coded 0 or 2) are used for -#' comparison; heterozygous markers (coded 1) are set to \code{NA}. This -#' reduces false mismatches caused by phase ambiguity. -#' -#' For \code{"best_pair"}, all markers are used and full Mendelian inheritance -#' rules are applied. Mismatch rates and comparison counts are computed across -#' all progeny simultaneously using vectorised \code{vapply} calls, producing -#' \code{n_pairs x n_progeny} matrices and giving substantial speed gains for -#' large datasets. Both matrices are explicitly coerced to matrix form to -#' handle the edge case of a single parent pair correctly. -#' -#' When multiple pairs share the lowest Mendelian error rate, ties are broken -#' by selecting the pair(s) with the greatest number of markers tested. If ties -#' still remain after this step, all remaining tied pairs are reported when -#' \code{show_ties = TRUE}. -#' -#' The base columns (\code{Male_Parent}, \code{Female_Parent}, -#' \code{Mendelian_Error_Pct}, \code{Markers_Tested}) are always populated with -#' the top result, ensuring no missing values in these columns regardless of -#' tie status. -#' -#' Output rows are pre-allocated as a \code{data.table} and filled by reference -#' using \code{data.table::set()}, avoiding repeated memory allocation during -#' the results-building step. -#' -#' Individuals in \code{parents_file} or \code{progeny_file} that are absent -#' from \code{genotypes_file} are removed with a warning. -#' -#' Progeny with fewer non-missing markers than \code{min_markers} are flagged -#' \code{LOW_MARKERS} and no parent assignment is reported. Progeny whose best -#' match exceeds \code{error_threshold} are flagged \code{HIGH_ERROR}. -#' -#' @examples -#' \dontrun{ -#' # Assign best male-female parent pair to each progeny -#' results <- find_parentage( -#' genotypes_file = "genotypes.txt", -#' parents_file = "parents.txt", -#' progeny_file = "progeny.txt", -#' method = "best_pair", -#' min_markers = 50, -#' error_threshold = 5.0, -#' show_ties = TRUE, -#' allow_selfing = FALSE -#' ) -#' -#' # Find best individual parent match (ignoring sex) -#' results <- find_parentage( -#' genotypes_file = "genotypes.txt", -#' parents_file = "parents.txt", -#' progeny_file = "progeny.txt", -#' method = "best_match", -#' min_markers = 30, -#' error_threshold = 3.0 -#' ) -#' } -#' -#' @importFrom data.table fread fwrite copy CJ rbindlist set data.table as.data.table -#' @export -find_parentage <- function(genotypes_file, parents_file, progeny_file, - method = "best_pair", - min_markers = 10, - error_threshold = 5.0, - show_ties = TRUE, - allow_selfing = TRUE, - verbose = TRUE, - write_txt = TRUE) { - - #### Input Validation and Data Loading #### - allowed_methods <- c("best_male_parent", "best_female_parent", "best_match", "best_pair") - if (!method %in% allowed_methods) - stop("Method must be one of: ", paste(allowed_methods, collapse = ", ")) - if (min_markers < 1) - stop("min_markers must be a positive integer.") - if (error_threshold < 0 || error_threshold > 100) - stop("error_threshold must be between 0 and 100.") - - tryCatch({ - genos <- data.table::fread(genotypes_file, sep = "auto") - all_parents <- data.table::fread(parents_file, sep = "auto") - progeny_candidates <- data.table::fread(progeny_file, sep = "auto") - }, error = function(e) { - stop("Error reading input files. Ensure paths are correct and files are TXT/TSV/CSV.") - }) - - valid_ids <- genos$ID - removed_parents <- base::setdiff(all_parents$ID, valid_ids) - if (base::length(removed_parents) > 0) { - warning("The following parent IDs were not in the genotype file and will not be analyzed: ", - paste(removed_parents, collapse = ", "), call. = FALSE) - all_parents <- all_parents[ID %in% valid_ids] - } - - removed_progeny <- base::setdiff(progeny_candidates$ID, valid_ids) - if (base::length(removed_progeny) > 0) { - warning("The following progeny IDs were not in the genotype file and will not be analyzed: ", - paste(removed_progeny, collapse = ", "), call. = FALSE) - progeny_candidates <- progeny_candidates[ID %in% valid_ids] - } - - if (!"Sex" %in% base::colnames(all_parents)) { - warning("No 'Sex' column in parents file. All parents treated as ambiguous ('A').") - all_parents[, Sex := "A"] - } - - all_parents[, Sex := base::toupper(Sex)] - male_parent_candidates <- all_parents[Sex %in% c("M", "A", "NA"), .SD] - female_parent_candidates <- all_parents[Sex %in% c("F", "A", "NA")] - - if (base::nrow(male_parent_candidates) == 0 && method %in% c("best_male_parent", "best_pair")) - warning("No valid male parent candidates remain after filtering.", call. = FALSE) - if (base::nrow(female_parent_candidates) == 0 && method %in% c("best_female_parent", "best_pair")) - warning("No valid female parent candidates remain after filtering.", call. = FALSE) - if (base::nrow(progeny_candidates) == 0) - stop("No valid progeny candidates remain after filtering.") - - #### Pre-compute genotype matrices once (shared across all methods) #### - # Full genotype matrix used by best_pair - genos_mat <- base::as.matrix(genos, rownames = "ID") - - # Homozygous-only matrix (het markers set to NA) used by hom methods - genos_hom <- data.table::copy(genos) - marker_cols <- base::setdiff(base::names(genos_hom), "ID") - for (col in marker_cols) - genos_hom[base::get(col) == 1, (col) := NA_integer_] - genos_hom_mat <- base::as.matrix(genos_hom, rownames = "ID") - - #### Assignment_Status from markers and error rate #### - # Returns LOW_MARKERS, HIGH_ERROR, or PASS - assign_status <- function(markers, error_pct) { - base::ifelse(markers < min_markers, "LOW_MARKERS", - base::ifelse(error_pct > error_threshold, "HIGH_ERROR", "PASS")) - } - - #### Logic for Homozygous Matching Methods #### - if (method %in% c("best_male_parent", "best_female_parent", "best_match")) { - parent_ids <- base::switch(method, - "best_male_parent" = male_parent_candidates$ID, - "best_female_parent" = female_parent_candidates$ID, - "best_match" = base::union(male_parent_candidates$ID, - female_parent_candidates$ID)) - - # Subset pre-computed homozygous matrix for relevant parents and progeny - parent_genos <- genos_hom_mat[base::rownames(genos_hom_mat) %in% parent_ids, , drop = FALSE] - progeny_genos <- genos_hom_mat[base::rownames(genos_hom_mat) %in% progeny_candidates$ID, , drop = FALSE] - - # Pre-allocate results data.table; fill by reference with set() - n_progeny <- base::nrow(progeny_genos) - results_dt <- data.table::data.table( - Progeny = base::rownames(progeny_genos), - Best_Match = NA_character_, - Mendelian_Error_Pct = NA_real_, - Markers_Tested = NA_integer_, - Assignment_Status = NA_character_ - ) - - for (i in base::seq_len(n_progeny)) { - progeny_vec <- progeny_genos[i, ] - mismatches <- base::rowSums(parent_genos != progeny_vec, na.rm = TRUE) - comparisons <- base::rowSums(!base::is.na(parent_genos) & !base::is.na(progeny_vec)) - percent_mismatch <- (mismatches / comparisons) * 100 - percent_mismatch[base::is.nan(percent_mismatch)] <- NA - - best_idx <- base::which.min(percent_mismatch) - - # No candidate found - flag LOW_MARKERS and continue - if (base::length(best_idx) == 0) { - data.table::set(results_dt, i, "Markers_Tested", 0L) - data.table::set(results_dt, i, "Assignment_Status", "LOW_MARKERS") - next - } - - best_markers <- comparisons[best_idx] - best_error <- base::round(percent_mismatch[best_idx], 2) - - data.table::set(results_dt, i, "Best_Match", base::rownames(parent_genos)[best_idx]) - data.table::set(results_dt, i, "Mendelian_Error_Pct", best_error) - data.table::set(results_dt, i, "Markers_Tested", base::as.integer(best_markers)) - data.table::set(results_dt, i, "Assignment_Status", assign_status(best_markers, best_error)) - } - final_df <- results_dt - } - - #### Logic for Best Pair Method #### - if (method == "best_pair") { - parent_pairs <- data.table::CJ(Male_Parent = male_parent_candidates$ID, - Female_Parent = female_parent_candidates$ID) - if (!allow_selfing) { - parent_pairs <- parent_pairs[Male_Parent != Female_Parent] - if (verbose) base::cat("Selfing is disallowed. Pairs with identical parents are removed.\n") - } - if (base::nrow(parent_pairs) == 0) stop("No valid parent pairs to test.") - - # Pre-extract parent genotype row blocks for vectorised operations - male_parent_genos_mat <- genos_mat[parent_pairs$Male_Parent, , drop = FALSE] - female_parent_genos_mat <- genos_mat[parent_pairs$Female_Parent, , drop = FALSE] - - # Subset full genotype matrix to progeny only - progeny_ids <- progeny_candidates$ID - progeny_mat <- genos_mat[progeny_ids, , drop = FALSE] - n_progeny <- base::nrow(progeny_mat) - n_pairs <- base::nrow(parent_pairs) - - # Vectorised mismatch computation across ALL progeny at once - # Wrapped in matrix() to handle the n_pairs = 1 edge case - mismatch_mat <- base::matrix( - base::vapply(base::seq_len(n_progeny), function(j) { - progeny_vec <- progeny_mat[j, ] - base::rowSums( - (male_parent_genos_mat == 0 & female_parent_genos_mat == 0 & progeny_vec > 0) | - (male_parent_genos_mat == 2 & female_parent_genos_mat == 2 & progeny_vec < 2) | - ((male_parent_genos_mat == 0 & female_parent_genos_mat == 1) | - (male_parent_genos_mat == 1 & female_parent_genos_mat == 0)) & (progeny_vec == 2) | - ((male_parent_genos_mat == 2 & female_parent_genos_mat == 1) | - (male_parent_genos_mat == 1 & female_parent_genos_mat == 2)) & (progeny_vec == 0) | - ((male_parent_genos_mat == 0 & female_parent_genos_mat == 2) | - (male_parent_genos_mat == 2 & female_parent_genos_mat == 0)) & (progeny_vec != 1), - na.rm = TRUE - ) - }, numeric(n_pairs)), - nrow = n_pairs, ncol = n_progeny - ) - - # Vectorised comparison count across ALL progeny at once - # Wrapped in matrix() to handle the n_pairs = 1 edge case - comparison_mat <- base::matrix( - base::vapply(base::seq_len(n_progeny), function(j) { - progeny_vec <- progeny_mat[j, ] - base::rowSums(!base::is.na(male_parent_genos_mat) & - !base::is.na(female_parent_genos_mat) & - !base::is.na(progeny_vec)) - }, numeric(n_pairs)), - nrow = n_pairs, ncol = n_progeny - ) - - # Percent mismatch matrix: n_pairs x n_progeny - pct_mismatch_mat <- (mismatch_mat / comparison_mat) * 100 - pct_mismatch_mat[base::is.nan(pct_mismatch_mat)] <- NA - - # Pre-allocate base results data.table; tie columns added dynamically - results_dt <- data.table::data.table( - Progeny = progeny_ids, - Male_Parent = NA_character_, - Female_Parent = NA_character_, - Mendelian_Error_Pct = NA_character_, - Markers_Tested = NA_integer_, - Assignment_Status = NA_character_ - ) - - # Per-progeny result extraction from pre-computed matrices - results_list <- base::vector("list", n_progeny) - for (j in base::seq_len(n_progeny)) { - prog_id <- progeny_ids[j] - percent_mismatch <- pct_mismatch_mat[, j] - comparisons <- comparison_mat[, j] - min_mismatch_val <- base::min(percent_mismatch, na.rm = TRUE) - - # No markers overlap at all - flag LOW_MARKERS - if (base::is.infinite(min_mismatch_val)) { - data.table::set(results_dt, j, "Markers_Tested", 0L) - data.table::set(results_dt, j, "Assignment_Status", "LOW_MARKERS") - next - } - - best_indices <- base::which(percent_mismatch == min_mismatch_val) - - # Tie-break: prefer pair(s) with the most markers tested - if (base::length(best_indices) > 1) { - best_markers_per_pair <- comparisons[best_indices] - max_markers <- base::max(best_markers_per_pair) - best_indices <- best_indices[best_markers_per_pair == max_markers] - } - - best_pairs <- parent_pairs[best_indices] - best_markers <- comparisons[best_indices[1]] - best_error <- base::round(min_mismatch_val, 2) - a_status <- assign_status(best_markers, best_error) - - if (!show_ties && base::nrow(best_pairs) > 1) { - warning("Progeny '", prog_id, "' has ", base::nrow(best_pairs), - " tied best pairs. Only one is reported as show_ties=FALSE.", call. = FALSE) - } - - num_to_report <- base::min(base::nrow(best_pairs), - if (show_ties) base::nrow(best_pairs) else 1) - - # Always populate base columns with the top result - data.table::set(results_dt, j, "Male_Parent", best_pairs$Male_Parent[1]) - data.table::set(results_dt, j, "Female_Parent", best_pairs$Female_Parent[1]) - data.table::set(results_dt, j, "Mendelian_Error_Pct", base::sprintf("%.2f", min_mismatch_val)) - data.table::set(results_dt, j, "Markers_Tested", base::as.integer(best_markers)) - data.table::set(results_dt, j, "Assignment_Status", a_status) - - # If ties remain after tie-breaking and show_ties is TRUE, - # store tie details for later column binding - if (show_ties && num_to_report > 1) { - tie_row <- base::list(Progeny = prog_id) - for (k in base::seq_len(num_to_report)) { - tie_row[[base::paste0("Male_Parent_", k)]] <- best_pairs$Male_Parent[k] - tie_row[[base::paste0("Female_Parent_", k)]] <- best_pairs$Female_Parent[k] - tie_row[[base::paste0("Mendelian_Error_Pct_", k)]] <- min_mismatch_val - tie_row[[base::paste0("Markers_Tested_", k)]] <- comparisons[best_indices[k]] - } - results_list[[j]] <- data.table::as.data.table(tie_row) - } - } - - # Merge any tie suffix columns onto the pre-allocated base table - tie_rows <- data.table::rbindlist( - base::Filter(Negate(base::is.null), results_list), - fill = TRUE - ) - if (base::nrow(tie_rows) > 0) { - final_df <- merge(results_dt, tie_rows, by = "Progeny", all.x = TRUE) - } else { - final_df <- results_dt - } - } - - #### Summary #### - if (verbose) { - total <- base::nrow(final_df) - a_counts <- base::table(final_df$Assignment_Status) - base::cat("\n--- Parentage Assignment Summary ---\n") - base::cat("Total progeny evaluated:", total, "\n") - for (s in base::names(a_counts)) - base::cat(base::sprintf(" %-14s: %d (%.1f%%)\n", s, - a_counts[s], (a_counts[s] / total) * 100)) - base::cat("Min markers threshold :", min_markers, "\n") - base::cat("Error threshold :", error_threshold, "%\n\n") - } - - #### Output #### - if (write_txt) { - output_filename <- "parentage_results_dt.txt" - tryCatch({ - data.table::fwrite(final_df, file = output_filename, sep = "\t", quote = FALSE) - if (verbose) base::cat("Results successfully written to:", output_filename, "\n") - }, error = function(e) { - warning("Could not write results to file. Error: ", e$message, call. = FALSE) - }) - } - - if (verbose) { - base::cat("\n--- Parentage Assignment Results ---\n") - base::print(final_df) - return(base::invisible(final_df)) - } else { - return(final_df) - } -} diff --git a/R/imputation_concordance.R b/R/imputation_concordance.R index 918071a..7ebbdcc 100644 --- a/R/imputation_concordance.R +++ b/R/imputation_concordance.R @@ -1,79 +1,60 @@ -#' Calculate Concordance between Imputed and Reference Genotypes -#' -#' This function calculates the concordance between imputed and reference -#' genotypes. It assumes that samples are rows and markers are columns. -#' Allele dosages (0, 1, 2) are recommended but other numeric formats are supported. -#' Missing data in either dataset can be excluded from the concordance calculation -#' using the `missing_code` argument. Specific markers can be excluded using -#' the `snps_2_exclude` argument. -#' -#' @param reference_genos A data frame containing reference genotype data, -#' with rows as samples and columns as markers. Must include a column named `ID`. -#' -#' @param imputed_genos A data frame containing imputed genotype data, -#' with rows as samples and columns as markers. Must include a column named `ID`. -#' -#' @param missing_code Optional value specifying missing data. If provided, -#' loci with this value in either dataset will be excluded from the concordance calculation. -#' -#' @param snps_2_exclude Optional vector of marker IDs to exclude from the concordance calculation. -#' -#' @param verbose Logical. If `TRUE`, prints summary statistics (minimum, quartiles, -#' median, mean, maximum) of concordance percentages. -#' -#' @param plot Logical. If `TRUE`, produces a bar plot of concordance percentage -#' by sample. -#' -#' @param print_result Logical. If `TRUE` (default), prints the concordance -#' results data frame to the console. If `FALSE`, results are returned invisibly. -#' -#' @return A data frame with: -#' \itemize{ -#' \item \code{ID}: Sample identifiers shared between the datasets. -#' \item \code{Concordance}: Percentage of matching genotypes per sample. +#' Calculate concordance between imputed and reference genotypes +#' +#' Compares imputed and reference genotype datasets sample by sample, returning +#' the percentage of matching genotypes per sample. Expects samples as rows and +#' markers as columns, with allele dosages (0, 1, 2) recommended. Other numeric +#' formats are supported. +#' +#' @param reference_genos A data frame of reference genotypes (samples × markers) +#' with a column named `ID`. +#' @param imputed_genos A data frame of imputed genotypes (samples × markers) +#' with a column named `ID`. +#' @param missing_code Optional value indicating missing data. Loci carrying this +#' value in either dataset are excluded from the concordance calculation. +#' @param snps_2_exclude Optional character vector of marker names to exclude. +#' @param verbose Logical. Print a five-number summary of concordance? Default `FALSE`. +#' @param plot Logical. Produce a bar plot of concordance by sample? Default `FALSE`. +#' @param print_result Logical. Print the results data frame to the console? +#' Default `TRUE`. If `FALSE`, results are returned invisibly. +#' +#' @return A data frame with columns: +#' \describe{ +#' \item{ID}{Sample identifiers shared between the two datasets.} +#' \item{Concordance}{Percentage of matching genotypes per sample.} #' } -#' If \code{print_result = FALSE}, the data frame is returned invisibly. #' #' @details -#' The function: -#' \enumerate{ -#' \item Identifies common samples and markers between the datasets. -#' \item Optionally excludes specified SNPs. -#' \item Removes loci with missing data (if \code{missing_code} is provided). -#' \item Computes per-sample concordance as the percentage of matching genotypes. -#' } -#' -#' When \code{plot = TRUE}, a bar plot showing concordance percentage per sample -#' is generated using \pkg{ggplot2}. -#' -#' @import dplyr +#' The function identifies common samples and markers between the two datasets, +#' optionally removes specified SNPs and loci with missing data, then computes +#' per-sample concordance as the percentage of matching genotypes over valid loci. +#' When `plot = TRUE`, a bar plot is produced with \pkg{ggplot2}. #' #' @examples #' ref <- data.frame( -#' ID = c("S1", "S2", "S3"), +#' ID = c("S1", "S2", "S3"), #' SNP1 = c(0, 1, 2), #' SNP2 = c(1, 1, 0), #' SNP3 = c(2, 5, 1) #' ) -#' -#' test <- data.frame( -#' ID = c("S1", "S2", "S3"), +#' imp <- data.frame( +#' ID = c("S1", "S2", "S3"), #' SNP1 = c(0, 0, 2), #' SNP2 = c(1, 1, 1), #' SNP3 = c(2, 5, 0) #' ) -#' -#' result <- imputation_concordance( +#' imputation_concordance( #' reference_genos = ref, -#' imputed_genos = test, -#' snps_2_exclude = "SNP2", -#' missing_code = 5, -#' print_result = FALSE +#' imputed_genos = imp, +#' snps_2_exclude = "SNP2", +#' missing_code = 5, +#' print_result = FALSE #' ) #' -#' result +#' @author Josué Chinchilla-Vargas #' +#' @importFrom dplyr %>% filter arrange #' @importFrom stats reorder +#' @importFrom ggplot2 ggplot aes geom_bar labs theme_minimal theme element_text #' @export imputation_concordance <- function(reference_genos, imputed_genos, diff --git a/R/madc2vcf_all.R b/R/madc2vcf_all.R index e223977..2000139 100644 --- a/R/madc2vcf_all.R +++ b/R/madc2vcf_all.R @@ -11,10 +11,12 @@ #' @param alignment_score_thr A numeric value specifying the minimum alignment score threshold. Default is 40. #' @param n.cores An integer specifying the number of cores to use for parallel processing. Default is 1. #' @param out_vcf A string specifying the name of the output VCF file. If the file extension is not `.vcf`, it will be appended automatically. -#' @param markers_info A string specifying the path to a CSV file with marker information (CloneID/BI_markerID, Chr, Pos, Ref, Alt, Type, Indel_pos columns as needed). +#' @param markers_info A string specifying the path to a CSV file with marker information (CloneID/Marker_ID/BI_markerID, Chr, Pos, Ref, Alt, Type, Indel_pos columns as needed). #' @param add_others A logical value. If TRUE, alleles labeled "Other" in the MADC file are included in off-target SNP extraction. Default is TRUE. -#' @param others_max_snps An integer or NULL. If not NULL, Other alleles with more than this many SNP differences versus the Ref sequence (as detected by pairwise alignment) are discarded. Default is NULL (no limit). +#' @param others_max_snps An integer or NULL. If not NULL, Other alleles with more than this many SNP differences versus the Ref sequence (as detected by pairwise alignment) are discarded. Default is 5. #' @param others_rm_with_indels A logical value. If TRUE, Other alleles that contain insertions or deletions relative to the Ref sequence (as detected by pairwise alignment) are discarded. Default is TRUE. +#' @param others_min_dist An integer or NULL. Distance in base pairs below which two SNPs in an Other allele are considered close to each other. Used together with `others_max_close_snps`. Default is 5. +#' @param others_max_close_snps An integer or NULL. If not NULL, Other alleles with more than this many SNPs that are closer than `others_min_dist` base pairs to each other are discarded. The SNPs are counted before the target position and positions with N or IUPAC codes are removed. The filter is not applied if either `others_min_dist` or `others_max_close_snps` is NULL. Default is 3. #' @param verbose A logical value indicating whether to print metrics and progress to the console. Default is TRUE. #' #' @return This function does not return an R object. It writes the processed VCF file v4.3 to the specified `out_vcf` path. @@ -80,8 +82,16 @@ madc2vcf_all <- function(madc, add_others = TRUE, others_max_snps = 5, others_rm_with_indels = TRUE, + others_min_dist = 5, + others_max_close_snps = 3, verbose = TRUE){ + + # Check that file path arguments are strings before any processing + if(!is.character(madc) || length(madc) != 1L) stop("madc should be a single string specifying the path or URL to the MADC file.") + if(!is.character(botloci_file) || length(botloci_file) != 1L) stop("botloci_file should be a single string specifying the path or URL to the botloci file.") + if(!is.null(hap_seq_file) && (!is.character(hap_seq_file) || length(hap_seq_file) != 1L)) stop("hap_seq_file should be a single string specifying the path to the haplotype database fasta file.") + vmsg("Running BIGr madc2vcf_all", verbose = verbose, level = 0, type = ">>") vmsg("madc : %s", verbose = verbose, level = 1, type = ">>", madc) vmsg("botloci_file : %s", verbose = verbose, level = 1, type = ">>", if (is.null(botloci_file)) "NULL" else botloci_file) @@ -95,6 +105,8 @@ madc2vcf_all <- function(madc, vmsg("add_others : %s", verbose = verbose, level = 1, type = ">>", add_others) vmsg("others_max_snps : %s", verbose = verbose, level = 1, type = ">>", if (is.null(others_max_snps)) "NULL" else others_max_snps) vmsg("others_rm_with_indels : %s", verbose = verbose, level = 1, type = ">>", others_rm_with_indels) + vmsg("others_min_dist : %s", verbose = verbose, level = 1, type = ">>", if (is.null(others_min_dist)) "NULL" else others_min_dist) + vmsg("others_max_close_snps : %s", verbose = verbose, level = 1, type = ">>", if (is.null(others_max_close_snps)) "NULL" else others_max_close_snps) vmsg("out_vcf : %s", verbose = verbose, level = 1, type = ">>", if (is.null(out_vcf)) "NULL" else out_vcf) vmsg("Checking inputs", verbose = verbose, level = 0, type = ">>") @@ -119,6 +131,8 @@ madc2vcf_all <- function(madc, if(!is.logical(add_others)) stop("add_others should be logical.") if(!is.null(others_max_snps) && (!is.numeric(others_max_snps) || others_max_snps < 1)) stop("others_max_snps should be a positive integer or NULL.") if(!is.logical(others_rm_with_indels)) stop("others_rm_with_indels should be logical.") + if(!is.null(others_min_dist) && (!is.numeric(others_min_dist) || others_min_dist < 1)) stop("others_min_dist should be a positive integer or NULL.") + if(!is.null(others_max_close_snps) && (!is.numeric(others_max_close_snps) || others_max_close_snps < 1)) stop("others_max_close_snps should be a positive integer or NULL.") if(!is.logical(verbose)) stop("verbose should be logical.") bigr_meta <- paste0('##BIGrCommandLine.madc2vcf_all=') @@ -158,21 +178,21 @@ madc2vcf_all <- function(madc, # Check whether markers_info is present and contains Ref + Alt columns if(!is.null(markers_info)) { mi_df <- read.csv(markers_info) - id_cols <- intersect(c("CloneID", "BI_markerID"), colnames(mi_df)) + id_cols <- intersect(c("CloneID", "BI_markerID","Marker_ID"), colnames(mi_df)) if(!length(id_cols)) { - stop("markers_info must contain a marker ID column named either 'CloneID' or 'BI_markerID'.") + stop("markers_info must contain a marker ID column named either 'CloneID', 'Marker_ID', or 'BI_markerID'.") } match_n <- vapply(id_cols, function(col) { sum(unique(report$CloneID) %in% unique(stats::na.omit(mi_df[[col]]))) }, integer(1)) if(!any(match_n)) { - stop("None of the markers_info CloneID or BI_markerID values match the MADC CloneID column. Please make sure they use the same marker IDs.") + stop("None of the markers_info CloneID, Marker_ID, or BI_markerID values match the MADC CloneID column. Please make sure they use the same marker IDs.") } id_col <- id_cols[which.max(match_n)] if(id_col != "CloneID" || !"CloneID" %in% colnames(mi_df)) { mi_df$CloneID <- mi_df[[id_col]] - if(id_col == "BI_markerID") { - vmsg("markers_info: 'BI_markerID' column copied to 'CloneID' for internal use", verbose = verbose, level = 1) + if(id_col == "BI_markerID" || id_col == "Marker_ID") { + vmsg("markers_info: 'BI_markerID' or 'Marker_ID' column copied to 'CloneID' for internal use", verbose = verbose, level = 1) } } # Validate CloneID values @@ -257,6 +277,8 @@ madc2vcf_all <- function(madc, add_others = add_others, others_max_snps = others_max_snps, others_rm_with_indels = others_rm_with_indels, + others_min_dist = others_min_dist, + others_max_close_snps = others_max_close_snps, verbose = verbose) vmsg("All information gathered!", verbose = verbose, level = 0, type = ">>") @@ -305,9 +327,19 @@ madc2vcf_all <- function(madc, #' @import parallel #' #' @noRd -loop_though_dartag_report <- function(report, botloci, hap_seq, n.cores=1, alignment_score_thr=40, - checks = NULL, mi_df = NULL, pad_width = NULL, - add_others = TRUE, others_max_snps = NULL, others_rm_with_indels = TRUE, +loop_though_dartag_report <- function(report, + botloci, + hap_seq, + n.cores=1, + alignment_score_thr=40, + checks = NULL, + mi_df = NULL, + pad_width = NULL, + add_others = TRUE, + others_max_snps = 5, + others_rm_with_indels = TRUE, + others_min_dist = 5, + others_max_close_snps = 3, verbose = TRUE){ if(!is.null(hap_seq) & (is.null(checks) | !isTRUE(checks$checks["RefAltSeqs"]))){ @@ -351,10 +383,17 @@ loop_though_dartag_report <- function(report, botloci, hap_seq, n.cores=1, align vmsg("Pairwise alignments of sequences to recover SNP position, reference and alternative bases...", verbose = verbose, level = 0) clust <- makeCluster(n.cores) #clusterExport(clust, c("botloci", "compare", "nucleotideSubstitutionMatrix", "pairwiseAlignment", "DNAString", "reverseComplement")) - #clusterExport(clust, c("botloci", "alignment_score_thr", "mi_df", "add_others", "others_max_snps", "others_rm_with_indels")) - compare_results <- parLapply(clust, updated_by_cloneID, function(x) compare(x, botloci, alignment_score_thr, mi_df, - add_others = add_others, others_max_snps = others_max_snps, - others_rm_with_indels = others_rm_with_indels, verbose = FALSE)) + #clusterExport(clust, c("botloci", "alignment_score_thr", "mi_df", "add_others", "others_max_snps", "others_rm_with_indels", "others_min_dist", "others_max_close_snps")) + compare_results <- parLapply(clust, updated_by_cloneID, function(x) compare(x, + botloci, + alignment_score_thr, + mi_df, + add_others = add_others, + others_max_snps = others_max_snps, + others_rm_with_indels = others_rm_with_indels, + others_min_dist = others_min_dist, + others_max_close_snps = others_max_close_snps, + verbose = FALSE)) stopCluster(clust) my_results_csv <- lapply(compare_results, "[[", 1) @@ -368,6 +407,7 @@ loop_though_dartag_report <- function(report, botloci, hap_seq, n.cores=1, align rm_indels <- unlist(rm_indels) n_rm_others_indels <- sum(sapply(compare_results, "[[", 5)) n_rm_others_maxsnps <- sum(sapply(compare_results, "[[", 6)) + n_rm_others_close_snps <- sum(sapply(compare_results, "[[", 7)) vmsg("Number of tags removed because of low alignment score (threshold = %s): %s tags", verbose = verbose, level = 2, type = ">>", alignment_score_thr, length(rm_score)) vmsg("Number of tags removed because of N in the alternative sequence: %s tags", verbose = verbose, level = 2, type = ">>", length(rm_N)) @@ -380,15 +420,18 @@ loop_though_dartag_report <- function(report, botloci, hap_seq, n.cores=1, align } else { vmsg("Number of tags removed because of indels as targets: 0 tags", verbose = verbose, level = 2, type = ">>") } - n_others_total <- sum(sapply(compare_results, "[[", 7)) - n_others_kept <- n_others_total - n_rm_others_indels - n_rm_others_maxsnps - others_added_info <- unlist(lapply(compare_results, "[[", 8)) + n_others_total <- sum(sapply(compare_results, "[[", 8)) + n_others_kept <- n_others_total - n_rm_others_indels - n_rm_others_maxsnps - n_rm_others_close_snps + others_added_info <- unlist(lapply(compare_results, "[[", 9)) if(add_others) { vmsg("Number of Other alleles found: %s (%s kept after filters, %s discarded)", verbose = verbose, level = 2, type = ">>", n_others_total, n_others_kept, n_others_total - n_others_kept) if(others_rm_with_indels) vmsg("Number of Other alleles discarded due to indels vs Ref: %s", verbose = verbose, level = 2, type = ">>", n_rm_others_indels) if(!is.null(others_max_snps)) vmsg("Number of Other alleles discarded due to exceeding max SNPs (%s): %s", verbose = verbose, level = 2, type = ">>", others_max_snps, n_rm_others_maxsnps) + if(!is.null(others_max_close_snps)) + vmsg("Number of Other alleles discarded due to more than %s SNPs with less than %s bp distant to each other: %s", verbose = verbose, level = 2, type = ">>", others_max_close_snps, others_min_dist, n_rm_others_close_snps) + # if(length(others_added_info) > 0) { # vmsg("Others tags added:", verbose = verbose, level = 3, type = ">>") # for(msg in others_added_info) vmsg(" %s", verbose = verbose, level = 3, type = ">>", msg) @@ -492,10 +535,20 @@ add_ref_alt <- function(one_tag, hap_seq, nsamples, verbose = TRUE) { #' @importFrom pwalign pairwiseAlignment nucleotideSubstitutionMatrix #' #' @noRd -compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, add_others = TRUE, others_max_snps = NULL, others_rm_with_indels = TRUE, verbose = FALSE){ - - #idx <- which(names(updated_by_cloneID) == "Ra01_020534029") +compare <- function(one_tag, + botloci, + alignment_score_thr = 40, + mi_df = NULL, + add_others = TRUE, + others_max_snps = 5, + others_rm_with_indels = TRUE, + others_min_dist = 5, + others_max_close_snps = 3, + verbose = FALSE){ + + #idx <- which(names(updated_by_cloneID) == "TA_1_7807815_1002_VariantMasked_000000166") #one_tag <- updated_by_cloneID[[idx]] + cloneID <- one_tag$CloneID[1] isBotLoci <- cloneID %in% botloci[,1] @@ -529,6 +582,7 @@ compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, ad rm_indels = NULL, n_rm_others_indels = 0L, n_rm_others_maxsnps = 0L, + n_rm_others_close_snps = 0L, n_others_total = 0L, others_added_info = character(0))) } @@ -553,7 +607,7 @@ compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, ad type = "global") # The score is a bit different from the python script despite same weights - if(align@score > alignment_score_thr){ # if score for the target sequence is smaller than the threshold, the tag will be discarted + if(align@score > alignment_score_thr){ # if score for the target sequence is smaller than the threshold, the tag will be discarded pos_target_idx <- align@pattern@mismatch@unlistData if(length(align@pattern@mismatch@unlistData) == 0){ #No polymorphisms found between ref and alt sequences - or just indels @@ -563,6 +617,7 @@ compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, ad rm_indels = cloneID, n_rm_others_indels = 0L, n_rm_others_maxsnps = 0L, + n_rm_others_close_snps = 0L, n_others_total = 0L, others_added_info = character(0))) } @@ -588,21 +643,26 @@ compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, ad pos_ref_idx <- align@pattern@mismatch@unlistData rm_target <- which(pos_ref_idx == pos_target_idx) # remove target position when is AltMatch if(length(rm_target) >0) pos_ref_idx <- pos_ref_idx[-rm_target] - # Cases found where the AltMatch is another alternative for the target SNP - they are discarted + # Cases found where the AltMatch is another alternative for the target SNP - they are discarded, they should have been listed as Other if(length(pos_ref_idx) >0){ ref_base_match <- substring(ref_seq, pos_ref_idx, pos_ref_idx) pos_alt_idx <- align@subject@mismatch@unlistData # If there are indels, the position in the alternative is not the same as the reference if(length(rm_target) >0) pos_alt_idx <- pos_alt_idx[-rm_target] # remove target position when is AltMatch - but the order in the sequence is the same alt_base_match <- substring(Match_seq[j,]$AlleleSequence, pos_alt_idx, pos_alt_idx) - # If Match sequences have N, do not consider as polymorphism - if(any(!alt_base_match %in% c("A", "T", "C", "G"))) { - ref_base_match <- ref_base_match[-which(!alt_base_match %in% c("A", "T", "C", "G"))] - pos_ref_idx <- pos_ref_idx[-which(!alt_base_match %in% c("A", "T", "C", "G"))] - alt_base_match <- alt_base_match[-which(!alt_base_match %in% c("A", "T", "C", "G"))] + # If either base at a mismatch position is non-ATCG (IUPAC/N), do not + # consider it as a polymorphism. Filter positions where either the + # reference or the Match allele base is non-ATCG so that identical + # IUPAC codes allowed between REF/ALT don't surface as false SNPs. + rm_pos <- which(!alt_base_match %in% c("A", "T", "C", "G") | + !ref_base_match %in% c("A", "T", "C", "G")) + if(length(rm_pos) > 0) { + ref_base_match <- ref_base_match[-rm_pos] + pos_ref_idx <- pos_ref_idx[-rm_pos] + alt_base_match <- alt_base_match[-rm_pos] } - if(length(alt_base_match) >0){ # If the N is the only polymorphis found, the Match tag will be discarted + if(length(alt_base_match) >0){ # If the N is the only polymorphism found, the Match tag will be discarded # The reported position is always on reference pos <- pos_target - (pos_target_idx - pos_ref_idx) @@ -623,6 +683,7 @@ compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, ad n_others_total <- nrow(others_seq) n_rm_others_indels <- 0L n_rm_others_maxsnps <- 0L + n_rm_others_close_snps <- 0L others_added_info <- character(0) if(add_others && nrow(others_seq) > 0){ @@ -644,6 +705,21 @@ compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, ad n_rm_others_maxsnps <- n_rm_others_maxsnps + 1L next } + + # Discard Others with SNPs too close to each other + ## Defined others_min_dist (default 5) + ## Defined others_max_close_snps (default 3) + if(!is.null(others_min_dist) && + !is.null(others_max_close_snps) && + any(diff(pos_ref_idx) < others_min_dist)) { + gaps_close <- diff(pos_ref_idx) < others_min_dist + snp_is_close <- c(gaps_close, FALSE) | c(FALSE, gaps_close) + if(sum(snp_is_close) > others_max_close_snps) { + n_rm_others_close_snps <- n_rm_others_close_snps + 1L + next + } + } + rm_target_other <- which(pos_ref_idx == pos_target_idx) # remove target position if base is the same as Ref or Alt if(length(rm_target_other) > 0) { other_tag_base <- substring(others_seq[j,]$AlleleSequence, pos_target_idx, pos_target_idx) @@ -652,18 +728,24 @@ compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, ad pos_alt_idx <- pos_alt_idx[-rm_target_other] } } - other_ref_base <- substring(ref_seq, pos_ref_idx, pos_ref_idx) - other_alt_base <- substring(others_seq[j,]$AlleleSequence, pos_alt_idx, pos_alt_idx) - # Cases found where the AltMatch is another alternative for the target SNP - they are discarted + # Cases found where the AltMatch is another alternative for the target SNP - they are discarded if(length(pos_ref_idx) >0){ - # If Match sequences have N, do not consider as polymorphism - if(any(!other_alt_base %in% c("A", "T", "C", "G"))) { - other_ref_base <- other_ref_base[-which(!other_alt_base %in% c("A", "T", "C", "G"))] - pos_ref_idx <- pos_ref_idx[-which(!other_alt_base %in% c("A", "T", "C", "G"))] - other_alt_base <- other_alt_base[-which(!other_alt_base %in% c("A", "T", "C", "G"))] + # Compute bases only when mismatch positions remain; substring() errors on integer(0) indices + other_ref_base <- substring(ref_seq, pos_ref_idx, pos_ref_idx) + other_alt_base <- substring(others_seq[j,]$AlleleSequence, pos_alt_idx, pos_alt_idx) + # If either base at a mismatch position is non-ATCG (IUPAC/N), do not + # consider it as a polymorphism. Filter positions where either the + # reference or the Other allele base is non-ATCG so that identical + # IUPAC codes allowed between REF/ALT don't surface as false SNPs. + rm_pos <- which(!other_alt_base %in% c("A", "T", "C", "G") | + !other_ref_base %in% c("A", "T", "C", "G")) + if(length(rm_pos) > 0) { + other_ref_base <- other_ref_base[-rm_pos] + pos_ref_idx <- pos_ref_idx[-rm_pos] + other_alt_base <- other_alt_base[-rm_pos] } - if(length(other_alt_base) >0){ # If the N is the only polymorphis found, the Match tag will be discarted + if(length(other_alt_base) >0){ # If the N or IUPAC code is the only polymorphism found, the Other tag will be discarded # The reported position is always on reference pos <- pos_target - (pos_target_idx - pos_ref_idx) @@ -683,12 +765,13 @@ compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, ad } } - return(list(update_tag = update_tag, # updated data.frame, NULL if discarted + return(list(update_tag = update_tag, # updated data.frame, NULL if discarded rm_score = NULL, # cloneID if removed because of low alignment score, NULL if kept rm_N = NULL, rm_indels = NULL, n_rm_others_indels = n_rm_others_indels, n_rm_others_maxsnps = n_rm_others_maxsnps, + n_rm_others_close_snps = n_rm_others_close_snps, n_others_total = n_others_total, others_added_info = others_added_info)) } else { @@ -698,6 +781,7 @@ compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, ad rm_indels = NULL, n_rm_others_indels = 0L, n_rm_others_maxsnps = 0L, + n_rm_others_close_snps = 0L, n_others_total = 0L, others_added_info = character(0))) } @@ -708,6 +792,7 @@ compare <- function(one_tag, botloci, alignment_score_thr = 40, mi_df = NULL, ad rm_indels = NULL, n_rm_others_indels = 0L, n_rm_others_maxsnps = 0L, + n_rm_others_close_snps = 0L, n_others_total = 0L, others_added_info = character(0))) } @@ -855,7 +940,20 @@ create_VCF_body <- function(csv, vcf_body_new <- vcf_body_new[,-1] colnames(vcf_body_new) <- c("#CHROM", "POS", "ID", "REF", "ALT","QUAL", "FILTER", "INFO","FORMAT", colnames(csv)[-c(1:7)]) - vcf_body_new <- vcf_body_new[order(vcf_body_new[,1], vcf_body_new[,2]),] + + #Correct scientific notation POS and remove rows with negative POS information + pos_num <- suppressWarnings(as.numeric(vcf_body_new$POS)) + bad_pos <- is.na(pos_num) | !is.finite(pos_num) | pos_num < 1 | pos_num != floor(pos_num) + + if (any(bad_pos)) { + vmsg("Removed %s VCF records with invalid POS values.", verbose = verbose, level = 1, type = ">>", sum(bad_pos)) + vcf_body_new <- vcf_body_new[!bad_pos, , drop = FALSE] + pos_num <- pos_num[!bad_pos] + } + + ord <- order(vcf_body_new[[1]], pos_num) + vcf_body_new <- vcf_body_new[ord, , drop = FALSE] + vcf_body_new$POS <- sprintf("%.0f", pos_num[ord]) return(vcf_body_new) } diff --git a/R/madc2vcf_multi.R b/R/madc2vcf_multi.R index e71cf0f..04858c8 100644 --- a/R/madc2vcf_multi.R +++ b/R/madc2vcf_multi.R @@ -89,7 +89,7 @@ madc2vcf_multi <- function(madc_file, stop("The MADC file is missing required columns (CloneID, AlleleID, AlleleSequence)") if (checks$checks["IUPACcodes"]) - stop("MADC Allele Sequences contain IUPAC (non-ATCG) codes. Please run HapApp to clean MADC file before using this function.") + stop("MADC Allele Sequences contain different IUPAC (non-ATCG) codes in REF and ALT sequences. Please run HapApp to clean MADC file before using this function.") if (!isTRUE(checks$checks["RefAltSeqs"])) stop("Not all Ref sequences have a corresponding Alt or vice versa. Please provide a complete MADC file before using this function.") @@ -100,7 +100,7 @@ madc2vcf_multi <- function(madc_file, if (!isTRUE(checks$checks["ChromPos"])) { if (is.null(markers_info)) stop("CloneID column does not follow the 'Chr_Pos' format. ", - "Please provide a markers_info file with at least 'CloneID'/'BI_markerID', ", + "Please provide a markers_info file with at least 'CloneID'/'Marker_ID'/'BI_markerID', ", "'Chr', and 'Pos' columns.") if (!all(c("Chr", "Pos") %in% colnames(mi_df))) stop("CloneID column does not follow the 'Chr_Pos' format. ", diff --git a/R/madc2vcf_targets.R b/R/madc2vcf_targets.R index 81e0643..32d4750 100644 --- a/R/madc2vcf_targets.R +++ b/R/madc2vcf_targets.R @@ -128,6 +128,7 @@ #' @import dplyr #' @import tidyr #' @import tibble +#' @importFrom stats setNames #' @importFrom reshape2 melt dcast #' @importFrom utils write.table #' @importFrom Biostrings DNAString reverseComplement @@ -521,10 +522,9 @@ madc2vcf_targets <- function(madc_file, # Add # to the CHROM column name colnames(vcf_df)[1] <- "#CHROM" - # Sort - vcf_df <- vcf_df[order(vcf_df[,1],as.numeric(as.character(vcf_df[,2]))),] - # Remove markers with NA CHROM/POS (unmatched in markers_info, Case 3) + # This runs before the POS correction below, which would otherwise remove these + # markers as invalid POS values without the markers_info warning na_coord <- is.na(vcf_df[, 1]) | is.na(vcf_df$POS) if(any(na_coord)) { vmsg(paste(sum(na_coord), "markers removed: no matching entry found in markers_info."), verbose = verbose, level = 1, type = ">>") @@ -532,6 +532,20 @@ madc2vcf_targets <- function(madc_file, vcf_df <- vcf_df[!na_coord, ] } + # Sort and correct any potential scientific notation + pos_num <- suppressWarnings(as.numeric(vcf_df$POS)) + bad_pos <- is.na(pos_num) | !is.finite(pos_num) | pos_num < 1 | pos_num != floor(pos_num) + + if (any(bad_pos)) { + vmsg("Removed %s VCF records with invalid POS values.", verbose = verbose, level = 1, type = ">>", sum(bad_pos)) + vcf_df <- vcf_df[!bad_pos, , drop = FALSE] + pos_num <- pos_num[!bad_pos] + } + + ord <- order(vcf_df[[1]], pos_num) + vcf_df <- vcf_df[ord, , drop = FALSE] + vcf_df$POS <- sprintf("%.0f", pos_num[ord]) + if(sum(is.na(vcf_df$REF)) > 0) { vmsg( paste( diff --git a/R/madc_plot.R b/R/madc_plot.R new file mode 100644 index 0000000..c715884 --- /dev/null +++ b/R/madc_plot.R @@ -0,0 +1,842 @@ +#' Plot a fixed allele ID MADC +#' +#' Visualize a fixed allele ID MADC file (one processed through HapApp) with a +#' choice of plots selected via `plot.type`: a PCA of alternative read ratios, a +#' linear marker-distribution genome plot, a markers x samples read-depth/#mHaps +#' heatmap, a per-sample missing-data boxplot, an allele read-ratio balance +#' diagnostic, a marker mean-depth (uniformity) distribution, and a `circos` +#' circular plot with concentric marker-position, marker-density, #mHaps, and +#' read-depth QC tracks. Requesting one type returns that plot; requesting +#' several (non-circos) returns a multi-panel figure. +#' +#' @details +#' The input is validated with [check_madc_sanity()]; raw DArT MADC files are +#' rejected with an error. With `target.only = TRUE`, only the target +#' `|Ref`/`|Alt` alleles are used. With `target.only = FALSE` (default), read +#' depth and mHap counts include all alleles, while the alternative read ratios +#' used by the `"pca"` and `"balance"` plots add the `|RefMatch`/`|AltMatch` +#' reads to their `|Ref`/`|Alt` alleles (as in [get_countsMADC()] with +#' `collapse_matches_counts = TRUE`) and do not use the `|Other` reads. +#' +#' `min.depth` (locus x sample cells below this depth are treated as missing) is +#' applied in the `"missing"` plot and the `circos` "Depth" ring, and is drawn as +#' a reference line in the `"balance"` and `"depth"` plots. It is not applied to +#' the `"pca"`, `"marker"`, or `"heatmap"` plots. +#' +#' Plotting uses `ggplot2` and base `grid`; the `circos` plot additionally +#' requires the suggested `circlize` package and, being a base graphics figure, +#' must be requested on its own (it cannot be embedded in a multi-panel). The +#' `"pca"` plot is a sample/data-interpretation diagnostic (a ratio is missing +#' only when the marker has no `|Ref`/`|Alt` reads in the sample, missing ratios +#' are mean-imputed, loci with no variation are dropped, and the PCA is +#' unscaled), so strong biological structure can dominate it - read it for +#' sample swaps / population separation, not as evidence the sequencing run +#' itself was good or bad. The `"balance"` plot is the direct read-out of whether +#' the assay yields interpretable dosage signal; it shows every cell with a +#' defined ratio (including below `min.depth`, marked by a dotted line) and, with +#' `ploidy`, a depth-dependent binomial expectation band. +#' +#' @param madc Path to a fixed allele ID MADC file, or an already-read data.frame. +#' @param plot.type One or more of `"pca"`, `"marker"`, `"heatmap"`, `"missing"`, +#' `"balance"`, `"depth"`, `"circos"`. A single value returns that plot; several +#' (excluding `"circos"`) return a multi-panel figure. `"circos"` must be +#' requested on its own. Default `c("pca", "marker", "heatmap", "missing")`. +#' @param target.only Logical; use only the target `|Ref`/`|Alt` alleles (see +#' Details). Default `FALSE`. +#' @param min.depth Minimum locus x sample read depth for a cell to be counted as +#' present (see Details for the plots that use it). Default `10`. +#' @param metadata Optional sample-metadata data.frame (see [madc_summary()]) used +#' to color the PCA, group the missing-data boxplot, and sort the heatmap +#' samples by category. +#' @param group.col Category column in `metadata` (e.g. species/population). +#' If `NULL` (default), samples are not grouped. In the heatmap, samples are +#' sorted by category, with one x-axis label per category and white lines +#' between categories. Samples not found in `metadata` are left out of the +#' heatmap and the missing-data boxplot. +#' @param shape.col Optional second `metadata` column mapped to point shape in +#' the PCA (e.g. plate or location), so two variables can be shown at once. +#' Only used when `metadata` is supplied. +#' @param palette Optional vector of colors for the categorical scales (PCA +#' point color, missing-boxplot fill); interpolated to the number of +#' categories. `NULL` uses the ggplot2 defaults. Not used by the other plots. +#' @param ploidy Optional integer species ploidy for the `"balance"` plot. When +#' supplied, dashed guides are drawn at the expected alt-read ratios +#' `(0:ploidy)/ploidy` (e.g. 0, 0.5, 1 for a diploid) and a depth-dependent 95% +#' binomial interval band is drawn around each interior expected ratio (the +#' expected spread of `A/D` under `Binomial(D, ratio)`), so observed dispersion +#' wider than sampling noise stands out. `NULL` (default) draws the ratio x +#' depth density with no guides/band, which also suits mixed/unknown ploidy. +#' @param balance.density Logical; for a lone `plot.type = "balance"`, stack a +#' histogram of the alt-read ratio above the heatmap (aligned x-axis). This +#' returns a composite grob instead of a `ggplot`, so it is ignored when +#' `"balance"` is combined with other plot types. Default `FALSE`. +#' @param markers_info Optional marker lookup (id + `Chr` + `Pos`) used when the +#' CloneIDs are not in `Chr_Pos` format. It sets the marker positions in the +#' marker and circos plots and the marker order in the heatmap. +#' @param pc.x,pc.y Principal components to plot (PCA). Defaults `1`, `2`. +#' @param loci.miss.max,sample.miss.max Drop loci, then samples, whose fraction +#' of missing alternative read ratios (no `|Ref`/`|Alt` reads) is above these +#' values before PCA. Defaults `0.5`. +#' @param miss.sort Ordering of the category boxes in the `"missing"` plot by their +#' median missing rate: `"none"` (default, keeps category order), `"desc"` +#' (highest to lowest), or `"asc"` (lowest to highest). Only applies when a +#' grouping category is supplied. +#' @param horizontal Logical; draw the `"missing"` boxplots horizontally +#' (`coord_flip`). Default `FALSE`. With `miss.sort`, the order then reads +#' top-to-bottom. +#' @param fill Heatmap fill metric: `"depth"` (default; total read depth per +#' locus x sample) or `"mhaps"` (number of mHaps with at least +#' `mhap.min.reads` reads). Cells with a value of 0 are drawn black. +#' @param facet.chrom Logical; facet the heatmap by chromosome. Default `FALSE`. +#' @param max.loci,max.samples Optional caps; larger heatmaps are subsampled to +#' evenly spaced loci/samples (with a message when `verbose = TRUE`). +#' @param density.window Window size in bp for the circos marker-density track. +#' Default `1e6` (1 Mb). +#' @param density.col Gradient colors (low -> high) for the circos marker-density +#' heatmap ring; interpolated across the range. Default `c("grey92", "#9e9ac8", "#3f007d")`. +#' @param depth.col Length-3 vector of colors for the circos "Depth" ribbon: +#' markers that are problematic (mean depth below `min.depth` or missing rate +#' above `depth.qc.maxmiss`), `normal`, or a high-depth `outlier` (see +#' `depth.outlier`). Only problematic and outlier markers are highlighted; +#' normal markers take the neutral middle color. Default +#' `c("#2166ac","#e0e0e0","#b2182b")`. +#' @param depth.outlier Multiplier `k` for the robust high-depth outlier cutoff +#' `median + k * MAD` of the marker mean depths (not mean +/- SD, which a few +#' extreme loci would drag up). Default `3`. +#' @param depth.qc.maxmiss Circos "Depth" ring: a marker is flagged problematic +#' when its mean depth is below `min.depth` OR its missing rate exceeds this +#' fraction - catching high-dropout markers (e.g. 50% zeros / 50% deep) that a +#' mean-depth cut alone reads as normal. Default `0.5`. +#' @param mhap.col Fill color for the circos "mHaps" log-scaled radial bars. +#' Default `"#35978f"`. +#' @param paralog.flag Number of mHaps (allele rows for the marker) above which a +#' circos "mHaps" locus is flagged as paralog-suspect: its bar's over-threshold +#' excess is drawn as a red segment above a dashed reference ring. Default `10`. +#' @param tick.height,density.height,mhap.height Circos track heights (fraction of +#' the radius) for the per-marker locus tick rug, the density heatmap ring, and +#' the mHap bar track. Defaults `0.03`, `0.08`, and `0.18`. +#' @param label.gap Width in degrees of the label corridor opened at 12 o'clock +#' for the circos track titles. Default `12`. With a single chromosome, at least +#' `30` is used. +#' @param mhap.min.reads Minimum reads for a mHap to count as present in a +#' locus x sample cell (heatmap with `fill = "mhaps"`). Default `1`. +#' @param titles Logical; draw the plot titles and subtitles. Default `TRUE`. +#' `FALSE` drops both from every requested plot (including the `"circos"` +#' figure and the multi-panel), leaving the axis labels, legends and the +#' `"balance"` caption - useful when the figure is captioned externally +#' (manuscript, report, slide). +#' @param output.file If not `NULL`, the figure is also saved here. The format is +#' set by the extension: `.png`, `.pdf`, `.jpg`/`.jpeg`, `.tif`/`.tiff`, or +#' `.svg`; other extensions give an error. +#' @param width,height,dpi Saved-figure dimensions (inches) and resolution (`dpi` +#' applies to PNG, JPEG, and TIFF only). +#' @param verbose Logical; print progress/validation messages. Default `TRUE`. +#' +#' @return For a single `plot.type`, a `ggplot`, returned invisibly when +#' `output.file` is given. With `plot.type = "balance"` and +#' `balance.density = TRUE` this is a composite grob; use +#' `grid::grid.draw()` to display it. For several types, an invisible list with +#' `plots` (named list) and `panel` (the assembled multi-panel grob); when +#' `output.file` is `NULL`, the panel is also drawn to the current graphics +#' device (in an interactive session, or when a device is already open). `"circos"` +#' returns invisibly the path of the saved figure (`output.file`, or a +#' temporary PNG when no `output.file` is given and no graphics device is open), +#' or `NULL` when it is drawn to an already open device. +#' +#' @examples +#' madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") +#' p <- madc_plot(madc_file, plot.type = "missing") +#' \donttest{ +#' madc_plot(madc_file, plot.type = c("pca", "heatmap")) +#' madc_plot(madc_file, plot.type = "balance", ploidy = 2) +#' } +#' +#' @seealso [madc_summary()], [filterMADC()] +#' @export +madc_plot <- function(madc, + plot.type = c("pca", "marker", "heatmap", "missing"), + target.only = FALSE, + min.depth = 10, + metadata = NULL, + group.col = NULL, + shape.col = NULL, + palette = NULL, + ploidy = NULL, + balance.density = FALSE, + markers_info = NULL, + pc.x = 1, + pc.y = 2, + loci.miss.max = 0.5, + sample.miss.max = 0.5, + miss.sort = c("none", "desc", "asc"), + horizontal = FALSE, + fill = c("depth", "mhaps"), + facet.chrom = FALSE, + max.loci = NULL, + max.samples = NULL, + density.window = 1e6, + density.col = c("grey92", "#9e9ac8", "#3f007d"), + depth.col = c("#2166ac", "#e0e0e0", "#b2182b"), + depth.outlier = 3, + depth.qc.maxmiss = 0.5, + mhap.col = "#35978f", + paralog.flag = 10, + tick.height = 0.03, + density.height = 0.08, + mhap.height = 0.18, + label.gap = 12, + mhap.min.reads = 1, + titles = TRUE, + output.file = NULL, + width = 8, height = 6, dpi = 300, + verbose = TRUE) { + + plot.type <- match.arg(plot.type, c("pca", "marker", "heatmap", "missing", + "balance", "depth", "circos"), + several.ok = TRUE) + fill <- match.arg(fill) + miss.sort <- match.arg(miss.sort) + + if (!is.null(output.file)) .madc_check_ext(output.file) + + report <- .read_and_check_madc(madc, verbose = verbose) + m <- .madc_metrics(report, min.depth = min.depth, target.only = target.only, + mhap.min.reads = mhap.min.reads, markers_info = markers_info, + verbose = FALSE) + grp <- .madc_group(metadata, group.col, m$samples) + shp <- .madc_group(metadata, shape.col, m$samples, arg = "shape.col") + + # circos is a standalone base-graphics (circlize) figure - it cannot be embedded + # in a grid multi-panel, so it must be requested on its own. + if ("circos" %in% plot.type) { + if (length(plot.type) > 1) + stop("plot.type = 'circos' must be requested on its own (it renders as a standalone circular figure).") + save_to <- output.file + # no output.file and no graphics device open: capture to a temp file rather + # than letting base graphics leave a stray Rplots.pdf. When a device is already + # open (interactive session, knitr/R Markdown, or a user-opened device) draw + # straight to it so the circos renders in place. + if (is.null(save_to) && grDevices::dev.cur() == 1L) { + save_to <- tempfile(fileext = ".png") + if (verbose) message("No graphics device / output.file; circos written to ", save_to) + } + if (!is.null(save_to)) { + .madc_open_device(save_to, width, height, dpi) + on.exit(grDevices::dev.off(), add = TRUE) + } + .madc_plot_circos(m, paralog.flag = paralog.flag, density.window = density.window, + density.col = density.col, depth.col = depth.col, + depth.outlier = depth.outlier, depth.qc.maxmiss = depth.qc.maxmiss, + mhap.col = mhap.col, + tick.height = tick.height, density.height = density.height, + mhap.height = mhap.height, label.gap = label.gap, + titles = titles) + return(invisible(save_to)) + } + + # the marginal-density composite only applies to a lone "balance" plot (it + # returns a grob, which can't embed in the ggplot multi-panel) + bal_density <- isTRUE(balance.density) && length(plot.type) == 1L + + build <- list( + pca = function() .madc_plot_pca(m, pc.x, pc.y, loci.miss.max, sample.miss.max, grp, shp, palette), + marker = function() .madc_plot_marker(m), + heatmap = function() .madc_plot_heatmap(m, fill, facet.chrom, max.loci, max.samples, verbose, grp), + missing = function() .madc_plot_missing(m, grp, palette, sort = miss.sort, + horizontal = horizontal), + balance = function() .madc_plot_balance(m, ploidy = ploidy, density = bal_density, + palette = palette, titles = titles), + depth = function() .madc_plot_depth(m) + ) + + plots <- lapply(plot.type, function(pt) build[[pt]]()) + names(plots) <- plot.type + # `balance` handles `titles` itself (its density form returns a grob); the rest + # are plain ggplots, so the title/subtitle come off after they are built. + if (!isTRUE(titles)) plots <- lapply(plots, .madc_drop_titles) + + if (length(plots) == 1L) { + result <- plots[[1]] + if (!is.null(output.file)) { + .madc_save(result, output.file, width, height, dpi) + return(invisible(result)) # saved to file: don't auto-print the object + } + return(result) + } + + n <- length(plots) + ncol <- 2L + nrow <- ceiling(n / ncol) + panel <- .arrange_grobs(plots, nrow = nrow, ncol = ncol) + if (!is.null(output.file)) { + .madc_save(panel, output.file, width, height, dpi) + } else if (interactive() || grDevices::dev.cur() != 1L) { + # draw the panel like a single ggplot would print; skipped when no device is + # open in a non-interactive session so no stray Rplots.pdf is left behind + grid::grid.newpage() + grid::grid.draw(panel) + } + invisible(list(plots = plots, panel = panel)) +} + +# drop the title/subtitle of a built ggplot (grobs pass through untouched) +#' @keywords internal +#' @noRd +.madc_drop_titles <- function(p) { + if (inherits(p, "ggplot")) p + ggplot2::labs(title = NULL, subtitle = NULL) else p +} + +# ---- PCA ------------------------------------------------------------------- +#' @keywords internal +#' @noRd +.madc_plot_pca <- function(m, pc.x = 1, pc.y = 2, + loci.miss.max = 0.5, sample.miss.max = 0.5, + grp = NULL, shp = NULL, palette = NULL) { + mat <- t(m$alt_ratio) # samples x loci + mat <- mat[, colMeans(is.na(mat)) <= loci.miss.max, drop = FALSE] + mat <- mat[rowMeans(is.na(mat)) <= sample.miss.max, , drop = FALSE] + if (nrow(mat) < 2 || ncol(mat) < 2) + stop("Too few samples/loci pass the missingness filters for PCA.") + + # per-locus mean imputation + col_means <- colMeans(mat, na.rm = TRUE) + na_idx <- which(is.na(mat), arr.ind = TRUE) + if (nrow(na_idx)) mat[na_idx] <- col_means[na_idx[, "col"]] + # drop zero-variance loci + keep <- apply(mat, 2, function(x) stats::sd(x) > 0) + mat <- mat[, keep, drop = FALSE] + if (ncol(mat) < 2) stop("No informative (non-constant) loci remain for PCA.") + + pc <- stats::prcomp(mat, center = TRUE, scale. = FALSE) + n_pc <- ncol(pc$x) + bad_pc <- function(v) length(v) != 1L || !is.numeric(v) || is.na(v) || + v != round(v) || v < 1 || v > n_pc + if (bad_pc(pc.x) || bad_pc(pc.y)) + stop(sprintf("`pc.x` and `pc.y` must be whole numbers between 1 and %d (the number of principal components available).", + n_pc)) + ve <- pc$sdev^2 / sum(pc$sdev^2) + df <- data.frame(sample = rownames(mat), + PCx = pc$x[, pc.x], PCy = pc$x[, pc.y], + stringsAsFactors = FALSE) + has_grp <- !is.null(grp) + has_shp <- !is.null(shp) + if (has_grp) df$group <- grp[match(df$sample, m$samples)] + if (has_shp) df$shape <- shp[match(df$sample, m$samples)] + + aes_args <- list(x = quote(PCx), y = quote(PCy)) + if (has_grp) aes_args$colour <- quote(group) + if (has_shp) aes_args$shape <- quote(shape) + + p <- ggplot2::ggplot(df, do.call(ggplot2::aes, aes_args)) + + ggplot2::geom_point(size = 2.6, alpha = 0.9) + + ggplot2::labs(title = "PCA of alternative read ratios", + x = sprintf("PC%d (%.1f%%)", pc.x, 100 * ve[pc.x]), + y = sprintf("PC%d (%.1f%%)", pc.y, 100 * ve[pc.y])) + + .madc_theme() + + if (has_grp && !is.null(palette)) { + n <- length(unique(stats::na.omit(df$group))) + p <- p + ggplot2::scale_colour_manual(values = grDevices::colorRampPalette(palette)(n)) + } + if (has_shp) { + shapes <- c(16, 17, 15, 18, 3, 4, 8, 1, 2, 0, 5, 6, 7, 9, 10, 11, 12, 13, 14) + nsh <- length(unique(stats::na.omit(df$shape))) + p <- p + ggplot2::scale_shape_manual(values = rep(shapes, length.out = nsh)) + } + p +} + +# ---- Marker distribution (linear, positions only) -------------------------- +#' @keywords internal +#' @noRd +.madc_plot_marker <- function(m) { + df <- .madc_marker_df(m) + chr_levels <- rev(sort(unique(df$chr))) + df$chr <- factor(df$chr, levels = chr_levels) + backbone <- data.frame(chr = factor(chr_levels, levels = chr_levels), + len = vapply(chr_levels, function(c) max(df$pos[df$chr == c]), + numeric(1)), stringsAsFactors = FALSE) + mb <- function(b) format(round(b / 1e6, 2), big.mark = ",", trim = TRUE) + + ggplot2::ggplot(df) + + ggplot2::geom_segment(data = backbone, + ggplot2::aes(x = 0, xend = len, y = chr, yend = chr), + color = "grey75", linewidth = 3) + + ggplot2::geom_point(ggplot2::aes(x = pos, y = chr), shape = 124, size = 4, + color = "steelblue") + + ggplot2::scale_x_continuous("Position (Mb)", labels = mb, + expand = ggplot2::expansion(mult = c(0.01, 0.03))) + + ggplot2::labs(title = "Marker distribution", y = NULL) + .madc_theme() +} + +# ---- Circos (circlize, three concentric tracks) ---------------------------- +#' @keywords internal +#' @noRd +.madc_plot_circos <- function(m, paralog.flag = 10, density.window = 1e6, + density.col = c("grey92", "#9e9ac8", "#3f007d"), + depth.col = c("#2166ac", "#e0e0e0", "#b2182b"), + depth.outlier = 3, depth.qc.maxmiss = 0.5, + mhap.col = "#35978f", + tick.height = 0.03, density.height = 0.08, + mhap.height = 0.18, label.gap = 12, + titles = TRUE) { + if (!requireNamespace("circlize", quietly = TRUE)) + stop("plot.type = 'circos' requires the 'circlize' package. Install it with install.packages('circlize').") + + df <- .madc_marker_df(m) + df$chr <- factor(df$chr, levels = sort(unique(df$chr))) + df <- df[order(df$chr, df$pos), , drop = FALSE] + + # per-window marker density (markers per `density.window` bp) per chromosome + dens <- lapply(levels(df$chr), function(cc) { + p <- df$pos[df$chr == cc] + br <- seq(0, max(p) + density.window, by = density.window) + h <- graphics::hist(p, breaks = br, plot = FALSE) + data.frame(start = utils::head(br, -1), end = br[-1], count = h$counts) + }) + names(dens) <- levels(df$chr) + dmax <- max(1, vapply(dens, function(d) max(d$count), numeric(1))) + col_dens <- circlize::colorRamp2(seq(0, dmax, length.out = length(density.col)), density.col) + + # mHap bar half-width scaled to marker spacing (visible but not overlapping) + bar_hw <- max(vapply(split(df$pos, df$chr), function(p) + if (length(p) > 1) stats::median(diff(sort(p))) * 0.35 else diff(range(df$pos)) * 0.01, + numeric(1))) + + # Combined depth QC. The outlier cut is the robust median + k*MAD (not mean +/- + # SD, which a few extreme loci would drag up). A marker is "problematic" if its + # mean depth is below min.depth OR it is missing (below min.depth) in more than + # depth.qc.maxmiss of samples - the high-dropout case (e.g. 50% zeros / 50% deep) + # that a mean-depth cut alone reads as "normal". Colors are precomputed per + # marker and looked up inside the track by a chr_pos key. + depth_thr <- stats::median(df$depth, na.rm = TRUE) + + depth.outlier * stats::mad(df$depth, na.rm = TRUE) + if (!is.finite(depth_thr) || depth_thr <= 0) depth_thr <- max(df$depth, na.rm = TRUE) + miss_thr <- m$min.depth + qc_problem <- (df$depth < miss_thr) | ((1 - df$call_rate) > depth.qc.maxmiss) + qc_col <- rep(depth.col[2], nrow(df)) # normal + qc_col[qc_problem] <- depth.col[1] # low depth / high missing + qc_col[df$depth >= depth_thr & !qc_problem] <- depth.col[3] # high-depth outlier + names(qc_col) <- paste(df$chr, df$pos, sep = "_") + + op <- graphics::par(mar = c(1, 1, if (isTRUE(titles)) 2 else 1, 1)) + on.exit(graphics::par(op), add = TRUE) + circlize::circos.clear() + on.exit(circlize::circos.clear(), add = TRUE) + N <- nlevels(df$chr) + top_gap <- if (N == 1) max(label.gap, 30) else label.gap # roomier corridor when one sector wraps + gaps <- if (N == 1) top_gap else c(rep(3, N - 1), top_gap) + circlize::circos.par(gap.after = gaps, start.degree = 90 - top_gap / 2, # center gap at 12 o'clock + cell.padding = c(0, 0, 0, 0), track.margin = c(0.006, 0.006), + canvas.xlim = c(-1.35, 1.35), canvas.ylim = c(-1.35, 1.35), + points.overflow.warning = FALSE) + circlize::circos.initialize(sectors = df$chr, x = df$pos) + + chr_lab_y <- 1 + 0.18 / tick.height + + # Track 1 (Loci): per-marker position ticks (kept dominant) + Mb coordinate + # axis (longer/heavier/darker so it stays distinct) + chromosome label + circlize::circos.track( + sectors = df$chr, x = df$pos, ylim = c(0, 1), track.height = tick.height, bg.border = NA, + panel.fun = function(x, y) { + xl <- circlize::CELL_META$xlim + circlize::circos.segments(x, rep(0, length(x)), x, rep(1, length(x)), + col = "grey25", lwd = 0.4) + circlize::circos.text(circlize::CELL_META$xcenter, chr_lab_y, + circlize::CELL_META$sector.index, + facing = "bending.inside", niceFacing = TRUE, cex = 0.8) + at <- pretty(xl, n = 5); at <- at[at >= xl[1] & at <= xl[2]] + circlize::circos.axis(h = "top", major.at = at, labels = paste0(round(at / 1e6, 1)), + labels.cex = 0.45, major.tick.length = circlize::mm_y(1.5), + minor.ticks = 0, lwd = 0.9, col = "black") + }) + + # Track 2 (Density): marker density heatmap over a grey background + circlize::circos.track( + sectors = df$chr, ylim = c(0, 1), track.height = density.height, + bg.col = "grey92", bg.border = "grey80", + panel.fun = function(x, y) { + d <- dens[[circlize::CELL_META$sector.index]] + circlize::circos.rect(d$start, 0, d$end, 1, col = col_dens(d$count), border = NA) + }) + + # Track 3 (mHaps): log2 bars (real-count axis) as two-tone bars - a neutral base + # up to the paralog threshold and a red segment for the over-threshold excess - + # plus a dashed reference ring at the threshold + ymax_l <- log2(max(df$n_mhaps, na.rm = TRUE) + 1) + ref_l <- log2(paralog.flag + 1) + circlize::circos.track( + sectors = df$chr, x = df$pos, y = log2(df$n_mhaps + 1), ylim = c(0, ymax_l), + track.height = mhap.height, bg.border = "grey90", + panel.fun = function(x, y) { + base <- pmin(y, ref_l) + circlize::circos.rect(x - bar_hw, 0, x + bar_hw, base, col = mhap.col, border = NA) + hi <- y > ref_l + if (any(hi)) + circlize::circos.rect(x[hi] - bar_hw, ref_l, x[hi] + bar_hw, y[hi], + col = "#b2182b", border = NA) + if (ref_l < ymax_l) + circlize::circos.lines(circlize::CELL_META$xlim, c(ref_l, ref_l), + col = "grey45", lty = 2, lwd = 0.6) + }) + # radial scale on a sector near the bottom, clear of the 12 o'clock corridor + mlabs <- c(0, 2, 5, 10, 25, 50, 100); mk <- log2(mlabs + 1) <= ymax_l + ax_sector <- levels(df$chr)[1L + (N %/% 2L)] + circlize::circos.yaxis("left", at = log2(mlabs[mk] + 1), labels = mlabs[mk], + sector.index = ax_sector, track.index = 3, + labels.cex = 0.4, lwd = 0.5) + + # Track 4 (Depth): contiguous categorical tile ribbon (gapless midpoint edges) + circlize::circos.track( + sectors = df$chr, x = df$pos, ylim = c(0, 1), + track.height = 0.055, bg.border = "grey90", + panel.fun = function(x, y) { + xs <- sort(x) + mid <- (utils::head(xs, -1) + xs[-1]) / 2 + key <- paste(circlize::CELL_META$sector.index, xs, sep = "_") + circlize::circos.rect(c(circlize::CELL_META$xlim[1], mid), 0, + c(mid, circlize::CELL_META$xlim[2]), 1, + col = qc_col[key], border = NA) + }) + + # horizontal track titles in the 12 o'clock corridor, aligned to each ring + titles <- c("Loci", "Density", "mHaps", "Depth") + s1 <- levels(df$chr)[1] + for (ti in seq_along(titles)) { + rt <- circlize::get.cell.meta.data("cell.top.radius", s1, track.index = ti) + rb <- circlize::get.cell.meta.data("cell.bottom.radius", s1, track.index = ti) + graphics::text(0, (rt + rb) / 2, titles[ti], adj = c(0.5, 0.5), + cex = 0.5, font = 2, xpd = NA) + } + + # compact color/scale keys in the empty center + .circos_center_legend(col_dens, dmax, density.window, mhap.col, paralog.flag, + depth.col, miss_thr, depth_thr, depth.qc.maxmiss) + + if (isTRUE(titles)) graphics::title("Marker distribution", cex.main = 1.1) + invisible(NULL) +} + +# compact legend drawn in the center hole of the circos (base-graphics, circos coords) +#' @keywords internal +#' @noRd +.circos_center_legend <- function(col_dens, dmax, density.window, mhap.col, paralog.flag, + depth.col, miss_thr, depth_thr, depth.qc.maxmiss = 0.5) { + # density: small horizontal continuous colour bar + n <- 64; bx <- seq(-0.17, 0.17, length.out = n + 1); yb <- 0.15; yt <- 0.19 + vals <- seq(0, dmax, length.out = n) + for (i in seq_len(n)) + graphics::rect(bx[i], yb, bx[i + 1], yt, col = col_dens(vals[i]), border = NA, xpd = NA) + graphics::rect(-0.17, yb, 0.17, yt, border = "grey55", lwd = 0.5, xpd = NA) + at <- pretty(c(0, dmax), n = 4); at <- at[at >= 0 & at <= dmax] + graphics::text(-0.17 + 0.34 * at / dmax, yb - 0.028, labels = at, cex = 0.42, xpd = NA) + graphics::text(0, yt + 0.035, sprintf("Markers/%g Mb", density.window / 1e6), + cex = 0.5, font = 2, xpd = NA) + + # mHaps key + graphics::text(0, 0.075, "mHaps", cex = 0.5, font = 2, xpd = NA) + graphics::rect(-0.16, 0.03, -0.13, 0.055, col = mhap.col, border = NA, xpd = NA) + graphics::text(-0.115, 0.0425, "count (log2)", cex = 0.42, adj = 0, xpd = NA) + graphics::rect(0.03, 0.03, 0.06, 0.055, col = "#b2182b", border = NA, xpd = NA) + graphics::text(0.07, 0.0425, sprintf("> %d (paralog)", paralog.flag), cex = 0.42, adj = 0, xpd = NA) + + # depth key + graphics::text(0, -0.03, "Depth", cex = 0.5, font = 2, xpd = NA) + dl <- c(sprintf("low <%g / miss >%g", miss_thr, depth.qc.maxmiss), "normal", + sprintf("outlier (>= %g)", round(depth_thr))) + yk <- -0.065 - c(0, 0.035, 0.07) + for (j in 1:3) { + graphics::rect(-0.16, yk[j] - 0.011, -0.13, yk[j] + 0.011, + col = depth.col[j], border = "grey70", xpd = NA) + graphics::text(-0.115, yk[j], dl[j], cex = 0.42, adj = 0, xpd = NA) + } +} + +# shared marker data.frame (positions + per-marker #mHaps and mean depth) +#' @keywords internal +#' @noRd +.madc_marker_df <- function(m) { + if (all(is.na(m$pos))) + stop("CloneIDs are not in Chr_Pos format; supply `markers_info` with Chr/Pos for the marker/circos plot.") + df <- data.frame(CloneID = m$markers, chr = m$chr, pos = m$pos, + n_mhaps = as.integer(m$n_mhaps_marker), + depth = rowMeans(m$depth_total), + call_rate = 1 - rowMeans(m$missing_mask), + stringsAsFactors = FALSE) + df[!is.na(df$pos), , drop = FALSE] +} + +# ---- Heatmap --------------------------------------------------------------- +#' @keywords internal +#' @noRd +.madc_plot_heatmap <- function(m, fill = "depth", facet.chrom = FALSE, + max.loci = NULL, max.samples = NULL, verbose = TRUE, + grp = NULL) { + mat <- if (fill == "mhaps") m$n_mhaps_present else m$depth_total + fill_lab <- if (fill == "mhaps") "# mHaps" else "Read depth" + + # order loci by chr + pos + ord <- order(m$chr, m$pos, na.last = TRUE) + mat <- mat[ord, , drop = FALSE] + chr_ord <- m$chr[ord] + + # optional subsample for very large panels - deterministic evenly-spaced pick so + # repeated calls give identical figures and genomic/sample spread is preserved + if (!is.null(max.loci) && nrow(mat) > max.loci) { + idx <- unique(round(seq(1, nrow(mat), length.out = max.loci))) + mat <- mat[idx, , drop = FALSE]; chr_ord <- chr_ord[idx] + if (verbose) message("Heatmap: subsampled to ", length(idx), " of ", + length(ord), " loci (evenly spaced).") + } + if (!is.null(max.samples) && ncol(mat) > max.samples) { + idx <- unique(round(seq(1, ncol(mat), length.out = max.samples))) + mat <- mat[, idx, drop = FALSE] + if (verbose) message("Heatmap: subsampled to ", length(idx), " samples (evenly spaced).") + } + + long <- reshape2::melt(mat, varnames = c("marker", "sample"), value.name = "value") + long$marker <- factor(long$marker, levels = rownames(mat)) + if (facet.chrom) long$chr <- chr_ord[match(long$marker, rownames(mat))] + + # when a category is given, sort samples by category and add ONE x-axis label + # per category (at its centre) so category labels never overlap, with white + # separators between categories. + has_grp <- !is.null(grp) && !all(is.na(grp)) + cat_breaks <- cat_labels <- boundaries <- NULL + if (has_grp) { + sgrp <- grp[match(as.character(long$sample), m$samples)] + keep <- !is.na(sgrp) + long <- long[keep, , drop = FALSE]; sgrp <- sgrp[keep] + samp_order <- unique(long$sample[order(sgrp, as.character(long$sample))]) + long$sample <- factor(long$sample, levels = samp_order) + og <- grp[match(as.character(samp_order), m$samples)] # group per ordered sample + runs <- rle(og) + ends <- cumsum(runs$lengths); starts <- ends - runs$lengths + 1L + cat_breaks <- as.character(samp_order)[round((starts + ends) / 2)] + cat_labels <- runs$values + boundaries <- utils::head(ends, -1) + 0.5 + } + + # 0 (failed / no data) -> black; positive values -> a colorblind-safe diverging + # scale (RdBu) centered at the median so both low- and high-depth markers stand out. + pos_mid <- stats::median(long$value[long$value > 0], na.rm = TRUE) + if (!is.finite(pos_mid)) pos_mid <- 1 + long$value[long$value == 0] <- NA + + p <- ggplot2::ggplot(long, ggplot2::aes(x = sample, y = marker, fill = value)) + + ggplot2::geom_raster() + + ggplot2::scale_fill_gradient2(fill_lab, low = "#2166ac", mid = "#f7f7f7", + high = "#b2182b", midpoint = pos_mid, + na.value = "black") + + ggplot2::labs(title = paste0("MADC heatmap (", fill_lab, ")"), x = NULL, y = NULL) + + .madc_theme() + if (nrow(mat) > 60) p <- p + ggplot2::theme(axis.text.y = ggplot2::element_blank()) + + if (has_grp) { + p <- p + + ggplot2::geom_vline(xintercept = boundaries, color = "white", linewidth = 0.4) + + ggplot2::scale_x_discrete(breaks = cat_breaks, labels = cat_labels) + + ggplot2::theme(axis.text.x = ggplot2::element_text(angle = 45, hjust = 1, size = 8), + axis.ticks.x = ggplot2::element_blank()) + } else if (ncol(mat) > 60) { + p <- p + ggplot2::theme(axis.text.x = ggplot2::element_blank(), + axis.ticks.x = ggplot2::element_blank()) + } else { + p <- p + ggplot2::theme(axis.text.x = ggplot2::element_text(angle = 90, hjust = 1, vjust = 0.5)) + } + + # facet by chromosome (rows), keeping the chromosome labels visible on the left + if (facet.chrom) + p <- p + ggplot2::facet_grid(chr ~ ., scales = "free_y", space = "free_y", switch = "y") + + ggplot2::theme(strip.text.y.left = ggplot2::element_text(angle = 0), + strip.placement = "outside", panel.spacing.y = grid::unit(2, "pt")) + p +} + +# ---- Missing-data boxplot -------------------------------------------------- +#' @keywords internal +#' @noRd +.madc_plot_missing <- function(m, grp = NULL, palette = NULL, + sort = "none", horizontal = FALSE) { + df <- data.frame(sample = m$samples, + missing_rate = colMeans(m$missing_mask), + stringsAsFactors = FALSE) + if (!is.null(grp)) df$group <- grp[match(df$sample, m$samples)] + + grouped <- !is.null(grp) && !all(is.na(df$group)) + if (grouped) { + df <- df[!is.na(df$group), , drop = FALSE] + # order the category boxes by their median missing rate when requested; in + # horizontal mode reverse so the sort reads top-to-bottom (coord_flip stacks + # the first factor level at the bottom) + if (sort != "none") { + med <- tapply(df$missing_rate, df$group, stats::median, na.rm = TRUE) + ord <- names(base::sort(med, decreasing = (sort == "desc"))) + if (horizontal) ord <- rev(ord) + df$group <- factor(df$group, levels = ord) + } + p <- ggplot2::ggplot(df, ggplot2::aes(x = group, y = missing_rate, fill = group)) + + ggplot2::geom_boxplot(outlier.shape = NA, alpha = 0.7) + + ggplot2::geom_jitter(width = 0.15, size = 1.2, alpha = 0.7) + + ggplot2::labs(x = NULL) + + ggplot2::theme(legend.position = "none") + # angled tick labels only help the vertical layout; horizontal reads flat + if (!horizontal) + p <- p + ggplot2::theme(axis.text.x = ggplot2::element_text(angle = 45, hjust = 1)) + if (!is.null(palette)) { + n <- length(unique(df$group)) + p <- p + ggplot2::scale_fill_manual(values = grDevices::colorRampPalette(palette)(n)) + } + } else { + p <- ggplot2::ggplot(df, ggplot2::aes(x = "all samples", y = missing_rate)) + + ggplot2::geom_boxplot(outlier.shape = NA, alpha = 0.7, fill = "grey80") + + ggplot2::geom_jitter(width = 0.15, size = 1.5, alpha = 0.8) + + ggplot2::labs(x = NULL) + } + p <- p + ggplot2::labs(title = "Per-sample missing rate", y = "Missing rate") + + ggplot2::ylim(0, NA) + .madc_theme() + if (horizontal) p <- p + ggplot2::coord_flip() + p +} + +# ---- Allele read-ratio balance -------------------------------------------- +#' @keywords internal +#' @noRd +.madc_plot_balance <- function(m, ploidy = NULL, density = FALSE, palette = NULL, + titles = TRUE) { + # One point-density cell per (marker x sample) with a defined allele ratio: alt + # read ratio on x, target depth (ref + alt) on a log y. Shows whether the + # chemistry yields interpretable dosage signal (bands), whether balance depends + # on depth, and - with `ploidy` - whether observed spread exceeds binomial + # sampling. `palette` is accepted for signature symmetry but unused (the fill is + # a continuous cell count). + ratio <- m$alt_ratio + size <- m$size_matrix + keep <- is.finite(ratio) & is.finite(size) & size >= 1 # keep sub-min.depth cells too + if (!any(keep)) + stop("No (marker x sample) cells have a defined allele ratio for the balance plot.") + df <- data.frame(alt_ratio = as.numeric(ratio[keep]), + depth = as.numeric(size[keep])) + + gx <- NULL + if (!is.null(ploidy)) { + ploidy <- as.integer(ploidy) + if (is.na(ploidy) || ploidy < 1L) stop("`ploidy` must be a positive integer.") + gx <- (0:ploidy) / ploidy + } + + # shared x scale so the optional marginal panel lines up with the heatmap + x_scale <- ggplot2::scale_x_continuous(limits = c(-0.02, 1.02), + breaks = seq(0, 1, 0.25), expand = c(0, 0)) + guides <- if (!is.null(gx)) + ggplot2::geom_vline(xintercept = gx, linetype = "dashed", + colour = "grey50", linewidth = 0.4, alpha = 0.6) else NULL + comma <- function(x) format(x, big.mark = ",", scientific = FALSE, trim = TRUE) + + # depth-dependent 95% binomial interval of A/D around each interior expected + # ratio (funnels toward the ratio as depth rises); homozygous 0/1 are degenerate + env <- if (!is.null(gx)) .balance_binom_envelope(gx, max(df$depth)) else NULL + + # --- heatmap: count in BOTH fill and alpha (log) so sparse noise recedes --- + p <- ggplot2::ggplot(df, ggplot2::aes(x = alt_ratio, y = depth)) + + ggplot2::geom_bin2d(ggplot2::aes(alpha = ggplot2::after_stat(count)), bins = 60) + + ggplot2::scale_fill_viridis_c("Observations per bin", trans = "log10", + breaks = c(1, 10, 100, 1000, 10000), labels = comma) + + ggplot2::scale_alpha_continuous(trans = "log10", range = c(0.3, 1), guide = "none") + + x_scale + ggplot2::scale_y_log10() + + ggplot2::geom_hline(yintercept = m$min.depth, linetype = "dotted", + linewidth = 0.5, colour = "grey30") + if (!is.null(env)) + p <- p + + ggplot2::geom_ribbon(data = env, inherit.aes = FALSE, + ggplot2::aes(y = depth, xmin = ratio_lo, xmax = ratio_hi, group = grp), + orientation = "y", fill = "grey30", alpha = 0.10) + + ggplot2::geom_path(data = env, inherit.aes = FALSE, + ggplot2::aes(x = ratio_lo, y = depth, group = grp), + linetype = "dotted", linewidth = 0.35, colour = "grey20") + + ggplot2::geom_path(data = env, inherit.aes = FALSE, + ggplot2::aes(x = ratio_hi, y = depth, group = grp), + linetype = "dotted", linewidth = 0.35, colour = "grey20") + if (!is.null(guides)) p <- p + guides + + sub <- if (!is.null(gx)) + paste0(sprintf("Expected ratios (ploidy %d): %s", ploidy, + paste(round(gx, 3), collapse = ", ")), + if (!is.null(env)) "; band = 95% binomial interval" else "") else NULL + if (!isTRUE(titles)) sub <- NULL + p <- p + + ggplot2::labs(title = if (isTRUE(titles)) "Allele balance vs. target depth", + subtitle = sub, + x = "ALT read ratio (ALT / [REF + ALT])", + y = "Target depth (REF + ALT reads, log scale)", + caption = sprintf("Dotted line = min.depth (%g); cells below would be dropped at that threshold", + m$min.depth)) + + .madc_theme() + + if (!isTRUE(density)) return(p) + + # --- optional marginal observed-ratio density stacked above (aligned x) --- + # histogram (not KDE) because allele ratios are discrete at low depth. Use a + # data-range scale + coord_cartesian (zoom, not hard limits) so the boundary + # bins aren't clipped/warned; the -0.02..1.02 view matches the heatmap panel. + top <- ggplot2::ggplot(df, ggplot2::aes(x = alt_ratio)) + + ggplot2::geom_histogram(bins = 80, fill = "grey35", colour = NA) + + ggplot2::scale_x_continuous(breaks = seq(0, 1, 0.25), expand = c(0, 0)) + + ggplot2::coord_cartesian(xlim = c(-0.02, 1.02)) + + ggplot2::labs(title = if (isTRUE(titles)) "Allele balance vs. target depth", + subtitle = sub, y = "Obs.") + + .madc_theme() + + ggplot2::theme(axis.title.x = ggplot2::element_blank(), + axis.text.x = ggplot2::element_blank(), + axis.ticks.x = ggplot2::element_blank()) + if (!is.null(guides)) top <- top + guides + p <- p + ggplot2::labs(title = NULL, subtitle = NULL) # one title, on the top panel + + .arrange_grobs(list(top, p), nrow = 2, ncol = 1, + heights = grid::unit(c(0.25, 0.75), "null"), align_widths = TRUE) +} + +# depth-stratified 95% binomial interval of A/D around each interior expected ratio +#' @keywords internal +#' @noRd +.balance_binom_envelope <- function(gx, max_depth, n = 200, lo = 0.025, hi = 0.975) { + interior <- gx[gx > 0 & gx < 1] + if (!length(interior)) return(NULL) + Dgrid <- unique(round(10^seq(0, log10(max(max_depth, 2)), length.out = n))) + Dgrid <- Dgrid[Dgrid >= 1] + do.call(rbind, lapply(seq_along(interior), function(i) { + p <- interior[i] + data.frame(depth = Dgrid, + ratio_lo = stats::qbinom(lo, Dgrid, p) / Dgrid, + ratio_hi = stats::qbinom(hi, Dgrid, p) / Dgrid, + grp = paste0("p", i), + stringsAsFactors = FALSE) + })) +} + +# ---- Marker depth (uniformity) distribution -------------------------------- +#' @keywords internal +#' @noRd +.madc_plot_depth <- function(m) { + d <- rowMeans(m$depth_total) + d <- d[is.finite(d)] + med <- stats::median(d) + q1 <- stats::quantile(d, 0.25, names = FALSE) + q3 <- stats::quantile(d, 0.75, names = FALSE) + nz <- sum(d <= 0) + df <- data.frame(depth = d[d > 0]) + if (!nrow(df)) stop("All markers have zero mean depth; nothing to plot.") + + sub <- sprintf("median %.1f (IQR %.1f-%.1f); dashed line = min.depth %g", + med, q1, q3, m$min.depth) + if (nz > 0) sub <- paste0(sub, sprintf("; %d zero-depth marker(s) omitted", nz)) + + ggplot2::ggplot(df, ggplot2::aes(x = depth)) + + ggplot2::geom_histogram(bins = 40, fill = "#4292c6", colour = "white", linewidth = 0.2) + + ggplot2::scale_x_log10() + + ggplot2::geom_vline(xintercept = m$min.depth, linetype = 2, + colour = "#b2182b", linewidth = 0.5) + + ggplot2::labs(title = "Marker mean-depth distribution", subtitle = sub, + x = "Mean read depth per marker (log10)", y = "Markers") + + .madc_theme() +} diff --git a/R/madc_summary.R b/R/madc_summary.R new file mode 100644 index 0000000..265efa8 --- /dev/null +++ b/R/madc_summary.R @@ -0,0 +1,338 @@ +#' Summarize a fixed allele ID MADC +#' +#' Computes read-only summary tables for a fixed allele ID MADC file (one +#' processed through HapApp): a per-marker microhaplotype frequency +#' distribution, per-sample and per-marker metric tables (read depth, missing +#' rate, number of mHaps, off-target read fraction), missingness and read-depth +#' threshold sweeps, an advisory per-marker QC table, an overall summary, and - +#' when a sample metadata table is supplied - the per-sample metrics aggregated +#' by category. +#' +#' @details +#' The input is validated with [check_madc_sanity()]; raw DArT MADC files (or +#' any file without fixed AlleleIDs) are rejected with an error. The first three +#' columns are taken as the ID columns (`AlleleID`, `CloneID`, `AlleleSequence`) +#' and all remaining columns as samples. A locus x sample cell is treated as +#' missing when its total read depth is below `min.depth`. +#' +#' The per-sample, per-marker, and by-category tables report both `depth` (total +#' reads at the locus - how well the region amplifies) and `target_depth` (reads +#' on the designed `|Ref`/`|Alt` alleles - how well the target markers work), +#' together with `offtarget_fraction`, the share of reads coming from the +#' `|RefMatch`/`|AltMatch`/`|Other` alleles (i.e. the additional information +#' contributed by non-target haplotypes). Per-sample values are averaged over +#' all markers, including markers that are missing in that sample. +#' +#' When `target.only = TRUE`, every metric is restricted to the target +#' `|Ref`/`|Alt` alleles (`|RefMatch`/`|AltMatch`/`|Other` are ignored), so +#' `depth` and `target_depth` are the same, the off-target columns are 0, and no +#' marker has more than two mHaps. +#' +#' @param madc Path to a fixed allele ID MADC file, or an already-read +#' data.frame. +#' @param target.only Logical; restrict all metrics to the target `|Ref`/`|Alt` +#' alleles. Default `FALSE`. +#' @param min.depth Minimum total read depth for a locus x sample cell to be +#' considered present (below this it is missing). Default `10`. +#' @param metadata Optional data.frame mapping samples to a grouping category. +#' The sample-id column is auto-detected (`sample`/`Sample`/`ID`/`SampleID`/ +#' `sample_id`/`Sample_ID`) or taken as the first column; ids are matched to +#' the MADC sample column names. +#' @param group.col Name of the category column in `metadata` (e.g. species or +#' population), used to produce the `by_category` table. If `NULL` (default), +#' no `by_category` table is returned. +#' @param markers_info Optional marker lookup table with an id column +#' (`CloneID`/`Marker_ID`/`BI_markerID`) plus `Chr` and `Pos`, used when the +#' CloneIDs are not in `Chr_Pos` format. +#' @param mhap.cap Largest explicit row in the microhaplotype-frequency table; +#' markers with more mHaps are pooled into a `">mhap.cap"` bucket. Markers with +#' fewer than two mHaps are not included in the table. Default `10`. +#' @param miss.thresholds Numeric vector of missingness fractions for the +#' threshold sweep. Default `c(0, 5, 10, 20, ..., 100) / 100`. +#' @param depth.thresholds Numeric vector of read-depth cutoffs for the +#' `depth_sweep` table. Default `c(1, 5, 10, 20, 50, 100)`. +#' @param mhap.min.reads Minimum reads for a mHap to count as present in a +#' sample. Used for the per-sample `n_mhaps_observed` (mean mHaps present per +#' marker) and the per-marker `n_mhaps_observed` (mHaps present in at least one +#' sample). Default `1`. +#' @param min.locus.depth,max.locus.depth Marker-QC read-depth window (mean reads +#' per sample at the locus). A marker below `min.locus.depth` is `FAIL`ed (low +#' depth); one above `max.locus.depth` is `FLAG`ged (over-amplified). Names/units +#' match [filterMADC()]: with `target.only = FALSE` the locus depth is computed +#' over all mHaps, as in `filterMADC`, so a `FAIL` on `min.locus.depth` is the +#' marker `filterMADC` would drop at the same value. With `target.only = TRUE` +#' only the `|Ref`/`|Alt` reads are counted. `min.locus.depth` defaults to +#' `min.depth`; `max.locus.depth` is off (`NULL`) by default. Set either to +#' `NULL` to disable that criterion. +#' @param max.missing Marker-QC maximum per-marker missing rate; markers above it +#' are `FAIL`ed (catches the high-dropout case a mean-depth cut alone misses). +#' `NULL` (default) disables it. +#' @param max.offtarget Marker-QC maximum read-weighted off-target fraction; +#' markers above it are `FLAG`ged. `NULL` (default) disables it. +#' @param max.mhaps.per.loci Marker-QC maximum number of defined mHaps; markers +#' above it are `FLAG`ged (paralog-suspect). `NULL` (default) disables it. +#' @param output.file If not `NULL`, the tables are written here: as +#' `paste0(output.file, "_.csv")` when `output.format = "csv"`, or as a +#' single `.xlsx` workbook (one sheet per table) when `output.format = "xlsx"`. +#' The `.xlsx` extension is added if it is not already present. +#' @param output.format One of `"csv"` (default; one CSV per table) or `"xlsx"` +#' (a single multi-sheet workbook, requires the `writexl` package). +#' @param verbose Logical; print progress/validation messages. Default `TRUE`. +#' +#' @return A named list of data.frames: +#' \describe{ +#' \item{`mhap_freq`}{Number of markers (`n_markers`) with 2, 3, ..., +#' `mhap.cap`, and more than `mhap.cap` defined mHaps (`n_mhaps`). Markers +#' with fewer than two allele rows are not counted, so the total can be lower +#' than the number of markers in the file.} +#' \item{`per_sample`}{One row per sample: mean locus read depth (`depth`), +#' `median_depth`, `depth_mad`, `depth_q10`, `depth_q90`, `target_depth`, +#' `total_reads`, `target_reads`, `missing_rate`, `call_rate`, +#' `n_mhaps_observed` (mean number of mHaps with at least `mhap.min.reads` +#' reads per marker, over all markers), `offtarget_fraction` (mean of the +#' per-locus ratios), and `offtarget_fraction_weighted` +#' (`sum(offtarget) / sum(total)`).} +#' \item{`per_marker`}{One row per marker (`CloneID`, `Chr`, `Pos`): mean +#' read depth across samples (`depth`), `median_depth`, `depth_mad`, +#' `depth_q10`, `depth_q90`, `target_depth`, `missing_rate`, `call_rate`, +#' `n_mhaps_defined` (allele rows for the marker in the MADC), +#' `n_mhaps_observed` (alleles with at least `mhap.min.reads` reads in at +#' least one sample), `offtarget_fraction`, and +#' `offtarget_fraction_weighted`. `Chr` and `Pos` are dropped when they cannot be +#' obtained from the CloneIDs or `markers_info`.} +#' \item{`missingness`}{For each value in `miss.thresholds`, the number of loci +#' and samples removed (missing rate above the threshold) and retained.} +#' \item{`depth_sweep`}{For each value in `depth.thresholds`, the fraction of +#' locus x sample cells at or above it (`cells_passing`), the number of loci +#' passing in at least 90% and 95% of samples, and the number of samples +#' passing in at least 90% of loci.} +#' \item{`marker_qc`}{Advisory per-marker `qc_status` (`PASS`, `FLAG`, or +#' `FAIL`; `FAIL` takes precedence) with a `qc_reason` string listing every +#' criterion that was triggered. Markers are not removed.} +#' \item{`overall`}{A one-row summary: number of samples, markers and mHaps, +#' mean and median depth, overall missing and call rate, mean and +#' read-weighted off-target fraction, and the median number of mHaps per +#' marker.} +#' \item{`by_category`}{Only when `metadata`/`group.col` are given and at +#' least one sample matches: per-category number of samples and the means of +#' the per-sample `depth`, `target_depth`, `missing_rate`, +#' `n_mhaps_observed` (`n_mhaps_mean`), and `offtarget_fraction`. Samples +#' not found in `metadata` are left out.} +#' } +#' +#' @examples +#' madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") +#' tabs <- madc_summary(madc_file, min.depth = 10) +#' tabs$mhap_freq +#' head(tabs$per_sample) +#' +#' @seealso [madc_plot()], [filterMADC()], [check_madc_sanity()] +#' @export +madc_summary <- function(madc, + target.only = FALSE, + min.depth = 10, + metadata = NULL, + group.col = NULL, + markers_info = NULL, + mhap.cap = 10, + miss.thresholds = c(0, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100) / 100, + depth.thresholds = c(1, 5, 10, 20, 50, 100), + mhap.min.reads = 1, + min.locus.depth = min.depth, + max.locus.depth = NULL, + max.missing = NULL, + max.offtarget = NULL, + max.mhaps.per.loci = NULL, + output.file = NULL, + output.format = c("csv", "xlsx"), + verbose = TRUE) { + + output.format <- match.arg(output.format) + report <- .read_and_check_madc(madc, verbose = verbose) + m <- .madc_metrics(report, min.depth = min.depth, target.only = target.only, + mhap.min.reads = mhap.min.reads, markers_info = markers_info, + verbose = FALSE) + + ## --- per-marker microhaplotype frequency distribution (paralog-flag table) --- + cnt <- m$n_mhaps_marker[m$n_mhaps_marker >= 2] + labels <- c(as.character(2:mhap.cap), paste0(">", mhap.cap)) + n_markers <- c(vapply(2:mhap.cap, function(k) sum(cnt == k), integer(1)), + sum(cnt > mhap.cap)) + mhap_freq <- data.frame(n_mhaps = factor(labels, levels = labels), + n_markers = n_markers, + stringsAsFactors = FALSE) + + ## --- shared helpers ------------------------------------------------------- + # depth = total reads at the locus (how well the region amplifies); + # target_depth= reads on the designed |Ref/|Alt alleles (how well the targets + # work); offtarget_fraction is the mean of the per-locus off-target + # ratios, offtarget_fraction_weighted is the read-weighted share + # sum(offtarget)/sum(total) (the "% of reads off target" number). + q10 <- function(v) stats::quantile(v, 0.10, names = FALSE, na.rm = TRUE) + q90 <- function(v) stats::quantile(v, 0.90, names = FALSE, na.rm = TRUE) + safe_ratio <- function(a, b) { r <- a / b; r[!is.finite(r)] <- NA; r } + + ## --- per-sample metrics --- + per_sample <- data.frame( + sample = m$samples, + depth = colMeans(m$depth_total), + median_depth = apply(m$depth_total, 2, stats::median), + depth_mad = apply(m$depth_total, 2, stats::mad), + depth_q10 = apply(m$depth_total, 2, q10), + depth_q90 = apply(m$depth_total, 2, q90), + target_depth = colMeans(m$ontarget_depth), + total_reads = colSums(m$depth_total), + target_reads = colSums(m$ontarget_depth), + missing_rate = colMeans(m$missing_mask), + call_rate = 1 - colMeans(m$missing_mask), + n_mhaps_observed = colMeans(m$n_mhaps_present), + offtarget_fraction = colMeans(m$offtarget_frac, na.rm = TRUE), + offtarget_fraction_weighted = safe_ratio(colSums(m$offtarget), colSums(m$depth_total)), + stringsAsFactors = FALSE) + rownames(per_sample) <- NULL + + ## --- per-marker metrics --- + per_marker <- data.frame( + CloneID = m$markers, + Chr = m$chr, + Pos = m$pos, + depth = rowMeans(m$depth_total), + median_depth = apply(m$depth_total, 1, stats::median), + depth_mad = apply(m$depth_total, 1, stats::mad), + depth_q10 = apply(m$depth_total, 1, q10), + depth_q90 = apply(m$depth_total, 1, q90), + target_depth = rowMeans(m$ontarget_depth), + missing_rate = rowMeans(m$missing_mask), + call_rate = 1 - rowMeans(m$missing_mask), + n_mhaps_defined = as.integer(m$n_mhaps_defined), + n_mhaps_observed = as.integer(m$n_mhaps_observed), + offtarget_fraction = rowMeans(m$offtarget_frac, na.rm = TRUE), + offtarget_fraction_weighted = safe_ratio(rowSums(m$offtarget), rowSums(m$depth_total)), + stringsAsFactors = FALSE) + rownames(per_marker) <- NULL + if (all(is.na(per_marker$Chr)) && all(is.na(per_marker$Pos))) { + per_marker$Chr <- NULL + per_marker$Pos <- NULL + } + + ## --- missingness threshold sweep --- + locus_miss <- rowMeans(m$missing_mask) + samp_miss <- colMeans(m$missing_mask) + missingness <- data.frame( + threshold_pct = miss.thresholds * 100, + loci_removed = vapply(miss.thresholds, function(t) sum(locus_miss > t), integer(1)), + loci_retained = vapply(miss.thresholds, function(t) sum(locus_miss <= t), integer(1)), + samples_removed = vapply(miss.thresholds, function(t) sum(samp_miss > t), integer(1)), + samples_retained = vapply(miss.thresholds, function(t) sum(samp_miss <= t), integer(1)), + stringsAsFactors = FALSE) + + ## --- depth threshold sweep (mirrors the missingness sweep) --- + cell_depth <- m$depth_total + depth_sweep <- data.frame( + depth_threshold = depth.thresholds, + cells_passing = vapply(depth.thresholds, function(t) mean(cell_depth >= t), numeric(1)), + loci_passing_90pct_samples = vapply(depth.thresholds, function(t) + sum(rowMeans(cell_depth >= t) >= 0.90), integer(1)), + loci_passing_95pct_samples = vapply(depth.thresholds, function(t) + sum(rowMeans(cell_depth >= t) >= 0.95), integer(1)), + samples_passing_90pct_loci = vapply(depth.thresholds, function(t) + sum(colMeans(cell_depth >= t) >= 0.90), integer(1)), + stringsAsFactors = FALSE) + + ## --- per-marker QC status (advisory; does NOT filter) --- + # Thresholds reuse filterMADC's vocabulary so a marker FAILing `min.locus.depth` + # here is the same marker filterMADC drops at that threshold (both key off the + # mean per-sample locus depth). Each threshold set to NULL disables its criterion. + n_mk <- length(m$markers) + qc_reason <- character(n_mk) + fail <- logical(n_mk); flag <- logical(n_mk) + add_qc <- function(cond, label, is_fail) { + cond[is.na(cond)] <- FALSE + if (is_fail) fail <<- fail | cond else flag <<- flag | cond + qc_reason[cond] <<- ifelse(nzchar(qc_reason[cond]), + paste0(qc_reason[cond], "; ", label), label) + } + if (!is.null(min.locus.depth)) + add_qc(per_marker$depth < min.locus.depth, sprintf("low depth (<%g)", min.locus.depth), TRUE) + if (!is.null(max.missing)) + add_qc(per_marker$missing_rate > max.missing, sprintf("high missing (>%g)", max.missing), TRUE) + if (!is.null(max.locus.depth)) + add_qc(per_marker$depth > max.locus.depth, sprintf("over-amplified (>%g)", max.locus.depth), FALSE) + if (!is.null(max.offtarget)) + add_qc(per_marker$offtarget_fraction_weighted > max.offtarget, + sprintf("high off-target (>%g)", max.offtarget), FALSE) + if (!is.null(max.mhaps.per.loci)) + add_qc(per_marker$n_mhaps_defined > max.mhaps.per.loci, + sprintf("high mHaps (>%g, paralog-suspect)", max.mhaps.per.loci), FALSE) + marker_qc <- data.frame( + CloneID = m$markers, + Chr = m$chr, + Pos = m$pos, + mean_depth = per_marker$depth, + call_rate = 1 - rowMeans(m$missing_mask), + offtarget_fraction_weighted = per_marker$offtarget_fraction_weighted, + n_mhaps_defined = as.integer(m$n_mhaps_defined), + n_mhaps_observed = as.integer(m$n_mhaps_observed), + qc_status = ifelse(fail, "FAIL", ifelse(flag, "FLAG", "PASS")), + qc_reason = qc_reason, + stringsAsFactors = FALSE) + rownames(marker_qc) <- NULL + if (all(is.na(marker_qc$Chr)) && all(is.na(marker_qc$Pos))) { + marker_qc$Chr <- NULL; marker_qc$Pos <- NULL + } + + ## --- overall one-row summary --- + overall <- data.frame( + n_samples = length(m$samples), + n_markers = length(m$markers), + n_mhaps_total = sum(m$n_mhaps_marker), + mean_depth = mean(m$depth_total), + median_depth = stats::median(m$depth_total), + overall_missing_rate = mean(m$missing_mask), + overall_call_rate = 1 - mean(m$missing_mask), + mean_offtarget_fraction = mean(m$offtarget_frac, na.rm = TRUE), + read_weighted_offtarget_fraction = safe_ratio(sum(m$offtarget), sum(m$depth_total)), + median_mhaps_per_marker = stats::median(m$n_mhaps_marker), + stringsAsFactors = FALSE) + + out <- list(mhap_freq = mhap_freq, per_sample = per_sample, + per_marker = per_marker, missingness = missingness, + depth_sweep = depth_sweep, marker_qc = marker_qc, overall = overall) + + ## --- optional by-category aggregation --- + grp <- .madc_group(metadata, group.col, m$samples) + if (!is.null(grp) && !all(is.na(grp))) { + keep <- !is.na(grp) + g <- grp[keep] + cats <- sort(unique(g)) + agg <- function(v, f) vapply(cats, function(cc) f(v[keep][g == cc]), numeric(1)) + out$by_category <- data.frame( + group = cats, + n_samples = as.integer(table(g)[cats]), + depth_mean = agg(per_sample$depth, mean), + target_depth_mean = agg(per_sample$target_depth, mean), + missing_rate_mean = agg(per_sample$missing_rate, mean), + n_mhaps_mean = agg(per_sample$n_mhaps, mean), + offtarget_fraction_mean = agg(per_sample$offtarget_fraction, + function(x) mean(x, na.rm = TRUE)), + stringsAsFactors = FALSE) + } + + ## --- optional output: per-table CSVs or a single multi-sheet workbook --- + if (!is.null(output.file)) { + if (output.format == "xlsx") { + if (!requireNamespace("writexl", quietly = TRUE)) + stop("output.format = 'xlsx' requires the 'writexl' package. Install it with install.packages('writexl').") + path <- if (grepl("\\.xlsx$", output.file, ignore.case = TRUE)) output.file else paste0(output.file, ".xlsx") + writexl::write_xlsx(out, path) + if (verbose) message("Wrote ", length(out), " summary tables as sheets to ", path) + } else { + for (tab in names(out)) + utils::write.csv(out[[tab]], paste0(output.file, "_", tab, ".csv"), row.names = FALSE) + if (verbose) message("Wrote ", length(out), " summary tables to ", + output.file, "_*.csv") + } + } + + out +} diff --git a/R/madc_utils.R b/R/madc_utils.R new file mode 100644 index 0000000..24b1624 --- /dev/null +++ b/R/madc_utils.R @@ -0,0 +1,283 @@ +# Internal shared helpers for madc_plot() and madc_summary(). +# Not exported. External packages are fully qualified (ggplot2::, grid::, ...); +# NSE symbols used in ggplot2 aes() are registered in utils::globalVariables() +# in R/utils.R. + +#' Read (if needed) and validate a fixed allele ID MADC +#' +#' Accepts a file path or an already-read data.frame, runs check_madc_sanity(), +#' surfaces the checks as messages, and hard-stops unless the file has the +#' required columns and fixed AlleleIDs. +#' +#' @keywords internal +#' @noRd +.read_and_check_madc <- function(madc, verbose = TRUE) { + if (is.data.frame(madc)) { + report <- madc + } else if (is.character(madc) && length(madc) == 1L) { + if (!file.exists(madc)) stop("MADC file not found: ", madc) + report <- utils::read.csv(madc, check.names = FALSE) + } else { + stop("`madc` must be a single file path or a data.frame.") + } + + checks <- check_madc_sanity(report) + + msgs <- mapply(function(check, message) if (isTRUE(check)) message[1] else message[2], + checks$checks, checks$messages) + if (verbose) for (i in seq_along(msgs)) message(msgs[i]) + + if (!isTRUE(checks$checks[["Columns"]])) + stop("The MADC file is missing required columns (CloneID, AlleleID, AlleleSequence)") + if (!isTRUE(checks$checks[["FixAlleleIDs"]])) + stop("The MADC file does not have fixed AlleleIDs. Please process the MADC file through HapApp before using this function.") + + report +} + +#' Compute the shared per-locus / per-sample MADC metric set once +#' +#' Everything downstream (plots and tables) reads from this so visuals and +#' tables never disagree. All matrices are returned on a master CloneID index +#' (file order of unique CloneIDs); markers that get_countsMADC drops (no Ref or +#' no Alt) still appear, with NA in the ratio matrices. +#' +#' @keywords internal +#' @noRd +.madc_metrics <- function(report, min.depth = 10, target.only = FALSE, + mhap.min.reads = 1, markers_info = NULL, verbose = FALSE) { + + samp_cols <- names(report)[-(1:3)] + if (length(samp_cols) < 1L) stop("MADC has no sample columns.") + + X <- data.matrix(report[samp_cols]) + storage.mode(X) <- "double" + X[is.na(X)] <- 0 + + clone <- as.character(report$CloneID) + master <- unique(clone) + + is_target <- grepl("\\|(Ref|Alt)_", report$AlleleID) + is_off <- grepl("RefMatch_|AltMatch_|[|]Other", report$AlleleID) + use_rows <- if (target.only) is_target else rep(TRUE, nrow(report)) + + # --- reindex helpers onto the master locus set --- + reidx <- function(m, fill = NA_real_) { + out <- matrix(fill, length(master), length(samp_cols), + dimnames = list(master, samp_cols)) + if (!is.null(m) && nrow(m) > 0) { + common <- intersect(rownames(m), master) + if (length(common)) out[common, colnames(m)] <- m[common, , drop = FALSE] + } + out + } + + # --- Ref/Alt/size/ratio from the existing count engine (paired markers only) --- + cm <- suppressWarnings(get_countsMADC(madc_object = report, + collapse_matches_counts = !target.only, + verbose = FALSE)) + size_p <- cm$size_matrix + ref_p <- cm$ref_matrix + alt_p <- size_p - ref_p + ratio_p <- alt_p / size_p + ratio_p[!is.finite(ratio_p)] <- NA + + size_matrix <- reidx(size_p) + ref_matrix <- reidx(ref_p) + alt_matrix <- reidx(alt_p) + alt_ratio <- reidx(ratio_p) + + # --- total read depth per (locus x sample), over the used rows --- + depth_total <- reidx(rowsum(X[use_rows, , drop = FALSE], clone[use_rows]), fill = 0) + + # --- missing mask (TRUE = missing) --- + missing_mask <- depth_total < min.depth + + # --- number of mHaps present per (locus x sample) --- + present <- (X[use_rows, , drop = FALSE] >= mhap.min.reads) * 1 + n_mhaps_present <- reidx(rowsum(present, clone[use_rows]), fill = 0) + + # --- panel-level number of mHaps per marker --- + # n_mhaps_defined = number of AlleleID rows defined for the marker in the panel + # (independent of read counts). + # n_mhaps_observed = number of those alleles actually supported (>= mhap.min.reads + # in at least one sample) - empirical run evidence. + tab <- table(clone[use_rows]) + n_mhaps_marker <- stats::setNames(as.integer(tab[master]), master) + n_mhaps_marker[is.na(n_mhaps_marker)] <- 0L + + allele_obs <- rowSums(present) >= 1 + obs_tab <- table(clone[use_rows][allele_obs]) + n_mhaps_observed <- stats::setNames(as.integer(obs_tab[master]), master) + n_mhaps_observed[is.na(n_mhaps_observed)] <- 0L + + # --- off-target reads and fraction --- + if (!target.only && any(is_off)) { + offtarget <- reidx(rowsum(X[is_off, , drop = FALSE], clone[is_off]), fill = 0) + } else { + offtarget <- matrix(0, length(master), length(samp_cols), + dimnames = list(master, samp_cols)) + } + offtarget_frac <- offtarget / depth_total + offtarget_frac[!is.finite(offtarget_frac)] <- NA + # on-target reads = total minus off-target (i.e. the target |Ref/|Alt reads) + ontarget_depth <- depth_total - offtarget + + # --- chromosome / position --- + cp <- .madc_chrpos(master, markers_info = markers_info) + + list(ref_matrix = ref_matrix, size_matrix = size_matrix, alt_matrix = alt_matrix, + alt_ratio = alt_ratio, depth_total = depth_total, ontarget_depth = ontarget_depth, + missing_mask = missing_mask, + n_mhaps_present = n_mhaps_present, n_mhaps_marker = n_mhaps_marker, + n_mhaps_defined = n_mhaps_marker, n_mhaps_observed = n_mhaps_observed, + offtarget = offtarget, offtarget_frac = offtarget_frac, + markers = master, chr = cp$chr, pos = cp$pos, samples = samp_cols, + mhap.min.reads = mhap.min.reads, + target.only = target.only, min.depth = min.depth) +} + +#' Extract Chr/Pos from CloneIDs (furthest-right underscore = Pos) or a lookup +#' +#' @keywords internal +#' @noRd +.madc_chrpos <- function(cloneids, markers_info = NULL) { + if (!is.null(markers_info)) { + idcol <- pick_markers_info_id_col(markers_info, cloneids) + if (!all(c("Chr", "Pos") %in% names(markers_info))) + stop("`markers_info` must contain 'Chr' and 'Pos' columns.") + mm <- match(cloneids, markers_info[[idcol]]) + chr <- as.character(markers_info$Chr[mm]) + pos <- suppressWarnings(as.numeric(markers_info$Pos[mm])) + return(list(chr = chr, pos = pos)) + } + parts <- strsplit(cloneids, "_") + chr <- vapply(parts, function(p) + if (length(p) >= 2) paste(p[-length(p)], collapse = "_") else NA_character_, + character(1)) + pos <- suppressWarnings(as.numeric(vapply(parts, function(p) + if (length(p) >= 2) p[length(p)] else NA_character_, character(1)))) + list(chr = chr, pos = pos) +} + +#' Resolve a per-sample grouping vector from a metadata table +#' +#' The sample-id column is auto-detected (sample/Sample/ID/SampleID/...) or the +#' first column; values are matched to the MADC sample column names. Returns +#' NULL when `metadata` or the requested column is NULL. `arg` is the argument +#' name used in the error message (e.g. "group.col" or "shape.col"). +#' +#' @keywords internal +#' @noRd +.madc_group <- function(metadata, group.col, samples, arg = "group.col") { + if (is.null(metadata)) return(NULL) + if (!is.data.frame(metadata)) stop("`metadata` must be a data.frame.") + if (is.null(group.col)) return(NULL) + if (!is.character(group.col) || length(group.col) != 1L || !group.col %in% names(metadata)) + stop(sprintf("`%s` must be a column name in `metadata`.", arg)) + id_candidates <- intersect(c("sample", "Sample", "ID", "SampleID", "sample_id", + "Sample_ID"), names(metadata)) + id_col <- if (length(id_candidates)) id_candidates[1] else names(metadata)[1] + grp <- as.character(metadata[[group.col]][match(samples, metadata[[id_col]])]) + n_match <- sum(!is.na(grp)) + if (all(is.na(grp))) { + warning("No `metadata` sample IDs matched the MADC sample columns; grouping skipped.") + } else if (n_match < length(samples)) { + message(sprintf("Metadata matched %d of %d MADC samples; %d unmatched sample(s) grouped as NA.", + n_match, length(samples), length(samples) - n_match)) + } + grp +} + +#' Shared minimal ggplot theme for MADC plots +#' +#' @keywords internal +#' @noRd +.madc_theme <- function() { + ggplot2::theme_minimal(base_size = 11) + + ggplot2::theme(panel.grid.minor = ggplot2::element_blank(), + legend.position = "right") +} + +#' Arrange ggplots/grobs into a layout using base grid (no gridExtra/patchwork) +#' +#' Returns a drawable gTree (does not draw to a device itself). +#' +#' @keywords internal +#' @noRd +.arrange_grobs <- function(grobs, nrow, ncol, heights = NULL, widths = NULL, + align_widths = FALSE) { + hts <- if (is.null(heights)) grid::unit(rep(1, nrow), "null") else heights + wds <- if (is.null(widths)) grid::unit(rep(1, ncol), "null") else widths + + # Draw on an explicit throwaway device so neither ggplotGrob() text-measurement + # nor the grid drawing leaves a stray Rplots.pdf. The layout is captured with + # grid.grab(wrap.grobs = TRUE) - wrapping avoids gtables' shared internal grob + # names colliding when several panels are combined. + tmp <- tempfile(fileext = ".png") + grDevices::png(tmp, width = 7, height = 7, units = "in", res = 100) + on.exit({ grDevices::dev.off(); unlink(tmp) }, add = TRUE) + + gl <- lapply(grobs, function(g) if (grid::is.grob(g)) g else ggplot2::ggplotGrob(g)) + if (align_widths) { # line up stacked panels on their x-axes + has_w <- vapply(gl, function(g) !is.null(g$widths), logical(1)) + if (all(has_w)) { + maxw <- do.call(grid::unit.pmax, lapply(gl, function(g) g$widths)) + gl <- lapply(gl, function(g) { g$widths <- maxw; g }) + } + } + + grid::grid.newpage() + grid::pushViewport(grid::viewport( + layout = grid::grid.layout(nrow, ncol, heights = hts, widths = wds))) + for (i in seq_along(gl)) { + r <- ((i - 1) %/% ncol) + 1 + cc <- ((i - 1) %% ncol) + 1 + grid::pushViewport(grid::viewport(layout.pos.row = r, layout.pos.col = cc)) + grid::grid.draw(gl[[i]]) + grid::upViewport() + } + grid::upViewport() + grid::grid.grab(wrap.grobs = TRUE) +} + +#' Check that an output file has a supported image extension +#' +#' @keywords internal +#' @noRd +.madc_check_ext <- function(file) { + ext <- tolower(tools::file_ext(file)) + if (!ext %in% c("png", "pdf", "jpg", "jpeg", "tif", "tiff", "svg")) + stop("`output.file` must end in .png, .pdf, .jpg/.jpeg, .tif/.tiff, or .svg.") + ext +} + +#' Open a graphics device matching the output file extension +#' +#' @keywords internal +#' @noRd +.madc_open_device <- function(file, width = 8, height = 6, dpi = 300) { + ext <- .madc_check_ext(file) + switch(ext, + pdf = grDevices::pdf(file, width = width, height = height), + png = grDevices::png(file, width = width, height = height, units = "in", res = dpi), + jpg = , + jpeg = grDevices::jpeg(file, width = width, height = height, units = "in", res = dpi, + quality = 95), + tif = , + tiff = grDevices::tiff(file, width = width, height = height, units = "in", res = dpi, + compression = "lzw"), + svg = grDevices::svg(file, width = width, height = height)) + invisible(file) +} + +#' Save a ggplot or an assembled grob to an image file +#' +#' @keywords internal +#' @noRd +.madc_save <- function(x, file, width = 8, height = 6, dpi = 300) { + .madc_open_device(file, width, height, dpi) + on.exit(grDevices::dev.off(), add = TRUE) + if (inherits(x, "ggplot")) print(x) else { grid::grid.newpage(); grid::grid.draw(x) } + invisible(file) +} diff --git a/R/updog2vcf.R b/R/updog2vcf.R index 11c6635..3a8862b 100644 --- a/R/updog2vcf.R +++ b/R/updog2vcf.R @@ -14,7 +14,6 @@ #' @return A vcf file #' @import dplyr #' @import tidyr -#' @importFrom Rdpack reprompt #' @importFrom utils write.table #' @importFrom Rsamtools bgzip #' @references @@ -126,9 +125,16 @@ updog2vcf <- function(multidog.object, output.file, updog_version = NULL, RefAlt #CHROM and POS from SNP ID new_df <- mout$snpdf %>% - separate(snp, into = c("CHROM", "POS"), sep = "_") %>% + extract( + snp, + into = c("CHROM", "POS"), + regex = "^(.*)_([^_]*)$" + ) %>% + mutate( + POS = sub("^0+", "", POS), + POS = if_else(POS == "", "0", POS) + ) %>% select(CHROM, POS) - new_df$POS <- sub("^0+", "", new_df$POS) #Make the VCF df vcf_df <- data.frame( @@ -220,6 +226,14 @@ updog2vcf <- function(multidog.object, output.file, updog_version = NULL, RefAlt #Add samples to vcf df vcf_df <- cbind(vcf_df,format_wide[,-1]) + #Remove SNPs without a valid position in the SNP ID (POS must be a positive integer) + bad_pos <- is.na(vcf_df$POS) | !grepl("^[1-9][0-9]*$", vcf_df$POS) + if (any(bad_pos)) { + warning(sum(bad_pos), " SNPs removed because their position could not be read from the SNP ID. ", + "SNP IDs are expected in the CHR_POS format, with POS a positive integer.", call. = FALSE) + vcf_df <- vcf_df[!bad_pos, , drop = FALSE] + } + # Add # to the CHROM column name colnames(vcf_df)[1] <- "#CHROM" diff --git a/R/utils.R b/R/utils.R index 41acdbc..37cb127 100644 --- a/R/utils.R +++ b/R/utils.R @@ -3,12 +3,17 @@ utils::globalVariables(c( "ALT", "AlleleID", "AlleleSequence", "CHROM", "Concordance", "Data", "ID", "MarkerName", "POS", - "QPseparate", "QPsolve_par", "REF", "Type", "Var1", "Variant", "geno", + "REF", "Type", "Var1", "Variant", "geno", "ind", "ref", "row_name", "size", "snp", "CloneID", "Count", "qualifying_sites_count", "MarkerID", "SampleID", "Dosage", "pos", "alt", "match_key", - ":=", ".SD", "Sex", "Male_Parent", "Female_Parent", "chr" + "chr", + # madc_plot() aes symbols + "n_mhaps", "depth", "len", "marker", "sample", "value", + "missing_rate", "group", "PCx", "PCy", "Var2", + "gx", "x0", "x1", "shape", "alt_ratio", "count", + "ratio_lo", "ratio_hi", "grp" )) #' Convert GT format to numeric dosage diff --git a/R/validate_pedigree.R b/R/validate_pedigree.R deleted file mode 100644 index dfcaf17..0000000 --- a/R/validate_pedigree.R +++ /dev/null @@ -1,400 +0,0 @@ -#' Validate Pedigree Trios Using Mendelian Error Analysis -#' -#' Validates parent-offspring trios by calculating Mendelian error rates from -#' SNP genotype data. Identifies incorrect parentage assignments and suggests -#' best-matching replacements. If a list of founders is supplied, trios that -#' are declared founders (both parents coded as 0) are preserved unchanged -#' with no recommendations. Trios removed due to missing genotype data are -#' retained in the output with a NO_GENOTYPE_DATA status. -#' -#' @param pedigree_file Character. Path to the pedigree file (TSV/CSV/TXT) -#' with columns: ID, Male_Parent, Female_Parent. -#' @param genotypes_file Character. Path to the genotypes file (TSV/CSV/TXT) -#' with an ID column followed by marker columns coded as 0, 1, 2. -#' @param founders_file Character, optional. Path to a one-column file -#' listing the IDs of founder individuals. Founders with both parents -#' coded as 0 are left unchanged with no recommendations. Defaults to NULL. -#' @param trio_error_threshold Numeric. Maximum Mendelian error percentage -#' to classify a trio as PASS (default: 5.0). Must be between 0 and 100. -#' @param min_markers Integer. Minimum number of non-missing markers -#' required to evaluate a trio (default: 10). -#' @param single_parent_error_threshold Numeric. Maximum homozygous-marker -#' mismatch percentage for a parent to be considered acceptable during -#' parent-level evaluation (default: 2.0). Must be between 0 and 100. -#' @param verbose Logical. If TRUE, prints progress messages, a summary -#' table, and results to the console (default: TRUE). -#' @param write_txt Logical. If TRUE, writes validation results to -#' output_filename (default: TRUE). -#' @param output_filename Character. Path/name of the output file -#' (default: "pedigree_validation_results.txt"). -#' -#' @return A data.table (returned invisibly) with one row per trio and -#' the following columns: -#' \describe{ -#' \item{ID}{Individual ID.} -#' \item{Male_Parent}{Declared male parent ID.} -#' \item{Female_Parent}{Declared female parent ID.} -#' \item{Mendelian_Error_Pct}{Trio-level Mendelian error percentage.} -#' \item{Markers_Tested}{Number of markers with non-missing genotypes.} -#' \item{Status}{One of PASS, FAIL, LOW_MARKERS, NO_DATA, FOUNDERS, -#' MISSING_MALE_PARENT, MISSING_FEMALE_PARENT, MISSING_BOTH_PARENTS, -#' or NO_GENOTYPE_DATA.} -#' \item{Correction_Decision}{One of NONE, KEEP_BOTH, -#' REMOVE_MALE_PARENT, REMOVE_FEMALE_PARENT, REMOVE_BOTH, -#' LOW_MARKERS_KEEP_BOTH, LOW_MARKERS_REMOVE_MALE_PARENT, -#' LOW_MARKERS_REMOVE_FEMALE_PARENT, LOW_MARKERS_REMOVE_BOTH.} -#' \item{Male_Parent_Hom_Error_Pct}{Male parent homozygous-marker mismatch percentage.} -#' \item{Female_Parent_Hom_Error_Pct}{Female parent homozygous-marker mismatch percentage.} -#' \item{Best_Male_Parent}{Best-matching male parent candidate ID.} -#' \item{Best_Male_Parent_Error_Pct}{Homozygous mismatch percentage for the best male parent candidate.} -#' \item{Best_Female_Parent}{Best-matching female parent candidate ID.} -#' \item{Best_Female_Parent_Error_Pct}{Homozygous mismatch percentage for the best female parent candidate.} -#' } -#' @export -validate_pedigree <- function(pedigree_file, genotypes_file, - founders_file = NULL, - trio_error_threshold = 5.0, - min_markers = 10, - single_parent_error_threshold = 2.0, - verbose = TRUE, - write_txt = TRUE, - output_filename = "pedigree_validation_results.txt") { - - #### Input validation #### - if (trio_error_threshold < 0 || trio_error_threshold > 100) - stop("trio_error_threshold must be between 0 and 100") - if (single_parent_error_threshold < 0 || single_parent_error_threshold > 100) - stop("single_parent_error_threshold must be between 0 and 100") - - tryCatch({ - pedigree <- data.table::fread(pedigree_file, sep = "auto", colClasses = "character") - genos <- data.table::fread(genotypes_file, sep = "auto") - }, error = function(e) { - stop("Error reading input files. Ensure paths are correct and files are TXT/TSV/CSV.") - }) - - #### Check required columns #### - required_ped_cols <- c("ID", "Male_Parent", "Female_Parent") - missing_cols <- base::setdiff(required_ped_cols, base::names(pedigree)) - if (base::length(missing_cols) > 0) - stop("Pedigree file missing required columns: ", - base::paste(missing_cols, collapse = ", ")) - if (!"ID" %in% base::names(genos)) - stop("Genotypes file must have an 'ID' column") - - # Ensure parent columns are character for consistent "0" comparisons - pedigree[, Male_Parent := as.character(Male_Parent)] - pedigree[, Female_Parent := as.character(Female_Parent)] - original_pedigree <- data.table::copy(pedigree) - - #### Read founders list #### - if (!is.null(founders_file)) { - founders_raw <- tryCatch({ - data.table::fread(founders_file, header = FALSE, colClasses = "character") - }, error = function(e) { - stop("Could not read founders list. Ensure it is a plain text or CSV/TSV file.") - }) - founder_ids <- unique(founders_raw[[1]]) - } else { - founder_ids <- character(0) - } - - #### Identify trios missing from the genotype file #### - valid_ids <- as.character(genos$ID) - has_geno <- pedigree[ID %in% valid_ids & - (Male_Parent %in% valid_ids | Male_Parent == "0") & - (Female_Parent %in% valid_ids | Female_Parent == "0")] - - no_geno_rows <- pedigree[!(ID %in% valid_ids) | - (!(Male_Parent %in% valid_ids) & Male_Parent != "0") | - (!(Female_Parent %in% valid_ids) & Female_Parent != "0")] - - if (base::nrow(no_geno_rows) > 0 && verbose) - base::cat("Found", base::nrow(no_geno_rows), - "trios with missing genotype data; flagged as NO_GENOTYPE_DATA.\n") - - pedigree <- has_geno - if (base::nrow(pedigree) == 0) - stop("No valid trios remain after filtering for genotype availability.") - - #### Build genotype matrices #### - genos_mat <- base::as.matrix(genos, rownames = "ID") - - # Homozygous-only matrix (het markers set to NA) - genos_hom <- data.table::copy(genos) - marker_cols <- base::setdiff(base::names(genos_hom), "ID") - for (col in marker_cols) - genos_hom[base::get(col) == 1, (col) := NA_integer_] - genos_hom_mat <- base::as.matrix(genos_hom, rownames = "ID") - - #### Helper: find best matching parent via homozygous mismatch #### - find_best_parent <- function(prog_id, exclude_ids = base::character(0)) { - candidates <- base::setdiff(base::rownames(genos_hom_mat), - c(prog_id, exclude_ids)) - if (base::length(candidates) == 0) - return(base::list(id = NA_character_, error_pct = NA_real_)) - - prog_hom <- genos_hom_mat[prog_id, ] - errors <- base::sapply(candidates, function(cand_id) { - cand_hom <- genos_hom_mat[cand_id, ] - comparisons <- base::sum(!base::is.na(cand_hom) & !base::is.na(prog_hom)) - if (comparisons == 0) return(NA_real_) - (base::sum(cand_hom != prog_hom, na.rm = TRUE) / comparisons) * 100 - }) - - best_idx <- base::which.min(errors) - base::list(id = candidates[best_idx], - error_pct = base::round(errors[best_idx], 2)) - } - - #### Main trio evaluation loop #### - results_list <- base::lapply(base::seq_len(base::nrow(pedigree)), function(i) { - - prog_id <- pedigree$ID[i] - male_parent_id <- pedigree$Male_Parent[i] - female_parent_id <- pedigree$Female_Parent[i] - - # Default values - correction_decision <- "NONE" - error_pct <- NA_real_ - status <- "NO_DATA" - markers_tested <- 0L - male_parent_error_pct <- NA_real_ - female_parent_error_pct <- NA_real_ - best_male_parent <- NA_character_ - best_male_parent_pct <- NA_real_ - best_female_parent <- NA_character_ - best_female_parent_pct <- NA_real_ - - ## Founder check - both parents "0" and ID in founders list - if (male_parent_id == "0" && female_parent_id == "0" && - prog_id %in% founder_ids) { - status <- "FOUNDERS" - correction_decision <- "NONE" - - } else { - - ## Missing parent(s) - recommendations only, "0"s preserved in pedigree - if (male_parent_id == "0" && female_parent_id == "0") { - status <- "MISSING_BOTH_PARENTS" - correction_decision <- "NONE" - - best_m <- find_best_parent(prog_id, exclude_ids = character(0)) - best_male_parent <- best_m$id - best_male_parent_pct <- best_m$error_pct - - best_f <- find_best_parent(prog_id, exclude_ids = c(best_m$id)) - best_female_parent <- best_f$id - best_female_parent_pct <- best_f$error_pct - - } else if (male_parent_id == "0" && female_parent_id != "0") { - status <- "MISSING_MALE_PARENT" - correction_decision <- "NONE" - - best_m <- find_best_parent(prog_id, exclude_ids = c(female_parent_id)) - best_male_parent <- best_m$id - best_male_parent_pct <- best_m$error_pct - - } else if (male_parent_id != "0" && female_parent_id == "0") { - status <- "MISSING_FEMALE_PARENT" - correction_decision <- "NONE" - - best_f <- find_best_parent(prog_id, exclude_ids = c(male_parent_id)) - best_female_parent <- best_f$id - best_female_parent_pct <- best_f$error_pct - - } else { - - ## Both parents present - Mendelian error calculation - progeny_vec <- genos_mat[prog_id, ] - male_parent_vec <- genos_mat[male_parent_id, ] - female_parent_vec <- genos_mat[female_parent_id, ] - - mismatches <- base::sum( - (male_parent_vec == 0 & female_parent_vec == 0 & progeny_vec > 0) | - (male_parent_vec == 2 & female_parent_vec == 2 & progeny_vec < 2) | - ((male_parent_vec == 0 & female_parent_vec == 1) | - (male_parent_vec == 1 & female_parent_vec == 0)) & (progeny_vec == 2) | - ((male_parent_vec == 2 & female_parent_vec == 1) | - (male_parent_vec == 1 & female_parent_vec == 2)) & (progeny_vec == 0) | - ((male_parent_vec == 0 & female_parent_vec == 2) | - (male_parent_vec == 2 & female_parent_vec == 0)) & (progeny_vec != 1), - na.rm = TRUE - ) - - markers_tested <- base::sum(!base::is.na(male_parent_vec) & - !base::is.na(female_parent_vec) & - !base::is.na(progeny_vec)) - - if (markers_tested == 0) { - status <- "NO_DATA" - correction_decision <- "NONE" - - } else { - error_pct <- (mismatches / markers_tested) * 100 - - # LOW_MARKERS still computes parent mismatch/recommendations - if (markers_tested < min_markers) { - status <- "LOW_MARKERS" - } else if (error_pct <= trio_error_threshold) { - status <- "PASS" - correction_decision <- "NONE" - } else { - status <- "FAIL" - } - - # Run parent-level evaluation for both FAIL and LOW_MARKERS - if (status %in% c("FAIL", "LOW_MARKERS")) { - - # Homozygous mismatch per parent - progeny_hom <- genos_hom_mat[prog_id, ] - male_parent_hom <- genos_hom_mat[male_parent_id, ] - female_parent_hom <- genos_hom_mat[female_parent_id, ] - - male_comparisons <- base::sum(!base::is.na(male_parent_hom) & - !base::is.na(progeny_hom)) - male_parent_error_pct <- if (male_comparisons == 0) NA_real_ else - base::round((base::sum(male_parent_hom != progeny_hom, na.rm = TRUE) / - male_comparisons) * 100, 2) - - female_comparisons <- base::sum(!base::is.na(female_parent_hom) & - !base::is.na(progeny_hom)) - female_parent_error_pct <- if (female_comparisons == 0) NA_real_ else - base::round((base::sum(female_parent_hom != progeny_hom, na.rm = TRUE) / - female_comparisons) * 100, 2) - - male_acceptable <- !is.na(male_parent_error_pct) && - male_parent_error_pct <= single_parent_error_threshold - female_acceptable <- !is.na(female_parent_error_pct) && - female_parent_error_pct <= single_parent_error_threshold - - if (male_acceptable && female_acceptable) { - correction_decision <- "KEEP_BOTH" - } else if (male_acceptable && !female_acceptable) { - correction_decision <- "REMOVE_FEMALE_PARENT" - best_f <- find_best_parent(prog_id, exclude_ids = c(male_parent_id)) - best_female_parent <- best_f$id - best_female_parent_pct <- best_f$error_pct - } else if (!male_acceptable && female_acceptable) { - correction_decision <- "REMOVE_MALE_PARENT" - best_m <- find_best_parent(prog_id, exclude_ids = c(female_parent_id)) - best_male_parent <- best_m$id - best_male_parent_pct <- best_m$error_pct - } else { - correction_decision <- "REMOVE_BOTH" - best_m <- find_best_parent(prog_id, exclude_ids = character(0)) - best_male_parent <- best_m$id - best_male_parent_pct <- best_m$error_pct - best_f <- find_best_parent(prog_id, exclude_ids = c(best_m$id)) - best_female_parent <- best_f$id - best_female_parent_pct <- best_f$error_pct - } - - # Do not alter corrected pedigree for LOW_MARKERS rows - if (status == "LOW_MARKERS") - correction_decision <- paste0("LOW_MARKERS_", correction_decision) - } - } - } - } - - data.table::data.table( - ID = prog_id, - Male_Parent = male_parent_id, - Female_Parent = female_parent_id, - Mendelian_Error_Pct = base::round(error_pct, 2), - Markers_Tested = markers_tested, - Status = status, - Correction_Decision = correction_decision, - Male_Parent_Hom_Error_Pct = male_parent_error_pct, - Female_Parent_Hom_Error_Pct = female_parent_error_pct, - Best_Male_Parent = best_male_parent, - Best_Male_Parent_Error_Pct = best_male_parent_pct, - Best_Female_Parent = best_female_parent, - Best_Female_Parent_Error_Pct = best_female_parent_pct - ) - }) - - final_df <- data.table::rbindlist(results_list) - - #### Append NO_GENOTYPE_DATA rows to the final report #### - if (base::nrow(no_geno_rows) > 0) { - no_geno_df <- data.table::data.table( - ID = no_geno_rows$ID, - Male_Parent = no_geno_rows$Male_Parent, - Female_Parent = no_geno_rows$Female_Parent, - Mendelian_Error_Pct = NA_real_, - Markers_Tested = 0L, - Status = "NO_GENOTYPE_DATA", - Correction_Decision = "NONE", - Male_Parent_Hom_Error_Pct = NA_real_, - Female_Parent_Hom_Error_Pct = NA_real_, - Best_Male_Parent = NA_character_, - Best_Male_Parent_Error_Pct = NA_real_, - Best_Female_Parent = NA_character_, - Best_Female_Parent_Error_Pct = NA_real_ - ) - final_df <- data.table::rbindlist(list(final_df, no_geno_df)) - } - - #### Write corrected pedigree #### - corrected_pedigree <- data.table::copy(original_pedigree) - for (i in base::seq_len(base::nrow(final_df))) { - prog_id <- final_df$ID[i] - decision <- final_df$Correction_Decision[i] - row_idx <- base::which(corrected_pedigree$ID == prog_id) - - if (decision == "REMOVE_MALE_PARENT") { - data.table::set(corrected_pedigree, row_idx, "Male_Parent", "0") - } else if (decision == "REMOVE_FEMALE_PARENT") { - data.table::set(corrected_pedigree, row_idx, "Female_Parent", "0") - } else if (decision == "REMOVE_BOTH") { - data.table::set(corrected_pedigree, row_idx, "Male_Parent", "0") - data.table::set(corrected_pedigree, row_idx, "Female_Parent", "0") - } - } - - tryCatch({ - data.table::fwrite(corrected_pedigree, file = "corrected_pedigree.txt", - sep = "\t", quote = FALSE) - if (verbose) base::cat("Corrected pedigree written to: corrected_pedigree.txt\n") - }, error = function(e) { - warning("Could not write corrected pedigree. Error: ", e$message, call. = FALSE) - }) - - #### Summary output #### - if (verbose) { - total_trios <- base::nrow(final_df) - status_counts <- base::table(final_df$Status) - base::cat("\n--- Trio Validation Summary ---\n") - base::cat("Total trios in pedigree:", total_trios, "\n") - for (s in base::names(status_counts)) - base::cat(base::sprintf("%-24s: %d (%.1f%%)\n", s, - status_counts[s], - (status_counts[s] / total_trios) * 100)) - base::cat("Error threshold:", trio_error_threshold, "%\n") - base::cat("Homozygous threshold:", single_parent_error_threshold, "%\n") - base::cat("Minimum markers required:", min_markers, "\n\n") - - corrections <- base::table(final_df$Correction_Decision) - base::cat("Correction summary:\n") - for (decision in base::names(corrections)) - if (decision != "NONE") - base::cat(" ", decision, ":", corrections[decision], "\n") - base::cat("\n") - base::print(final_df) - } - - #### Write results #### - if (write_txt) { - tryCatch({ - data.table::fwrite(final_df, file = output_filename, - sep = "\t", quote = FALSE) - if (verbose) base::cat("Results written to:", output_filename, "\n") - }, error = function(e) { - warning("Could not write results. Error: ", e$message, call. = FALSE) - }) - } - - return(base::invisible(final_df)) -} diff --git a/README.md b/README.md index a7c1735..82d2655 100644 --- a/README.md +++ b/README.md @@ -46,5 +46,5 @@ BIGr development is supported by [Breeding Insight](https://breedinginsight.org/ If you use BIGr in your research, please cite as: -Sandercock, Alexander M., Cristiane H. Taniguti, Josue Chinchilla-Vargas, Dongyan Zhao, Shufen Chen, Meng Lin, Manoj Sapkota, and Breeding Insight Team. 2025. “Breeding Insight Genomics Functions for Polypoid and Diploid Species.” https://github.com/Breeding-Insight/BIGr. +Sandercock A.M., Peel M.D., Taniguti C.H., Chinchilla-Vargas J., Chen S., Sapkota M., Lin M., Zhao D., Ackerman A.J., Basnet B.R., Beil C.T., Sheehan M.J. (2025). BIGapp: A User-Friendly Genomic Tool Kit Identified Quantitative Trait Loci for Creeping Rootedness in Alfalfa (Medicago sativa L.). The Plant Genome, 18(3), e70067. https://doi.org/10.1002/tpg2.70067 diff --git a/dev/dev_history.R b/dev/dev_history.R index 34e9324..22be3d4 100644 --- a/dev/dev_history.R +++ b/dev/dev_history.R @@ -2,7 +2,7 @@ # Update dependencies in DESCRIPTION # install.packages('attachment', repos = 'https://thinkr-open.r-universe.dev') -#attachment::att_amend_desc() +if (requireNamespace("attachment", quietly = TRUE)) attachment::att_amend_desc() # Check package coverage covr::package_coverage() diff --git a/inst/CITATION b/inst/CITATION index e1caeb4..9adc87a 100644 --- a/inst/CITATION +++ b/inst/CITATION @@ -23,6 +23,10 @@ bibentry( year = "2025" ) +# Year of the CRAN release, or the current year for a development version +year <- sub("-.*", "", meta$`Date/Publication`) +if (!length(year)) year <- format(Sys.Date(), "%Y") + bibentry( bibtype = "Manual", title = "BIGr: Breeding Insight Genomics functions for polyploid and diploid species", @@ -48,10 +52,10 @@ bibentry( person(given='Dongyan', family='Zhao', role='ctb'), - person('Cornell', 'University', + person(family='University of Florida', role=c('cph'), comment = "Breeding Insight")), - year = "2025", - note = "R package version 0.6.2", + year = year, + note = paste("R package version", meta$Version), url = "https://github.com/Breeding-Insight/BIGr" ) diff --git a/inst/REFERENCES.bib b/inst/REFERENCES.bib deleted file mode 100644 index c39cf5c..0000000 --- a/inst/REFERENCES.bib +++ /dev/null @@ -1,18 +0,0 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2761 -\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fmodern\fcharset0 Courier;} -{\colortbl;\red255\green255\blue255;\red0\green0\blue0;} -{\*\expandedcolortbl;;\cssrgb\c0\c0\c0;} -\margl1440\margr1440\vieww17580\viewh9740\viewkind0 -\deftab720 -\pard\pardeftab720\partightenfactor0 - -\f0\fs26 \cf0 \expnd0\expndtw0\kerning0 -\outl0\strokewidth0 \strokec2 @article\{sandercock2023road,\ - title=\{The road to restoration: Identifying and conserving the adaptive legacy of American chestnut\},\ - author=\{Sandercock, Alexander M and Westbrook, Jared W and Zhang, Qian and Holliday, Jason A\},\ - journal=\{bioRxiv\},\ - pages=\{2023--05\},\ - year=\{2023\},\ - publisher=\{Cold Spring Harbor Laboratory\}\ -\}\ -} \ No newline at end of file diff --git a/inst/WORDLIST b/inst/WORDLIST index 4ba9f3d..54ff0c3 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -1,5 +1,7 @@ +AlleleIDs BIGapp CMD +CloneIDs DArT DArTag Dongyan @@ -9,14 +11,26 @@ fasta et al Funkhouser GWAS +HapApp Heterozygosity +Mb +PCA +circos +ggplot +ideogram madc MADC MADCs MAF +mHap +mHaps +microhaplotype +microhaplotypes Newcom OffTargets +offtarget Polypoid +prcomp RO Steibel Updog diff --git a/inst/check_ped_test.txt b/inst/check_ped_test.txt deleted file mode 100644 index ae7bc08..0000000 --- a/inst/check_ped_test.txt +++ /dev/null @@ -1,16 +0,0 @@ -id sire dam -off1 sire1 dam1 -off2 sire2 dam2 -off3 sire3 dam3 -off4 sire4 dam4 -off5 sire5 dam5 -sire1 off1 grandmother1 -sire2 grandfather2 grandfather2 -sire3 grandfather3 grandfather3 -sire4 grandfather4 grandmother4 -sire5 grandfather5 grandmother5 -dam1 grandfather6 grandmother6 -dam2 grandfather7 grandmother7 -dam3 grandfather8 grandmother8 -dam4 0 0 -dam5 0 0 \ No newline at end of file diff --git a/man/allele_freq_poly.Rd b/man/allele_freq_poly.Rd deleted file mode 100644 index 407d158..0000000 --- a/man/allele_freq_poly.Rd +++ /dev/null @@ -1,49 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/breedtools_functions.R -\name{allele_freq_poly} -\alias{allele_freq_poly} -\title{Compute Allele Frequencies for Populations} -\usage{ -allele_freq_poly(geno, populations, ploidy = 2) -} -\arguments{ -\item{geno}{matrix of genotypes coded as the dosage of allele B \code{{0, 1, 2, ..., ploidy}} -with individuals in rows (named) and SNPs in columns (named)} - -\item{populations}{list of named populations. Each population has a vector of IDs -that belong to the population. Allele frequencies will be derived from all animals} - -\item{ploidy}{integer indicating the ploidy level (default is 2 for diploid)} -} -\value{ -data.frame consisting of allele_frequencies for populations (columns) for -each SNP (rows) -} -\description{ -Computes allele frequencies for specified populations given SNP array data -} -\examples{ -# Example inputs -geno_matrix <- matrix( -c(4, 1, 4, 0, # S1 - 2, 2, 1, 3, # S2 - 0, 4, 0, 4, # S3 - 3, 3, 2, 2, # S4 - 1, 4, 2, 3),# S5 -nrow = 4, ncol = 5, byrow = FALSE, # individuals=rows, SNPs=cols -dimnames = list(paste0("Ind", 1:4), paste0("S", 1:5)) -) - -pop_list <- list( -PopA = c("Ind1", "Ind2"), -PopB = c("Ind3", "Ind4") -) - -allele_freqs <- allele_freq_poly(geno = geno_matrix, populations = pop_list, ploidy = 4) -print(allele_freqs) - -} -\references{ -Funkhouser SA, Bates RO, Ernst CW, Newcom D, Steibel JP. Estimation of genome-wide and locus-specific -breed composition in pigs. Transl Anim Sci. 2017 Feb 1;1(1):36-44. -} diff --git a/man/check_madc_sanity.Rd b/man/check_madc_sanity.Rd index 1d7eebb..21d2809 100644 --- a/man/check_madc_sanity.Rd +++ b/man/check_madc_sanity.Rd @@ -14,11 +14,12 @@ If \code{FixAlleleIDs} is \code{FALSE} (raw DArT format), the first 7 rows are treated as header filler and skipped before further checks are run.} } \value{ -A named list with five elements: +A named list with six elements: \describe{ -\item{checks}{Named logical vector with nine entries: -\code{Columns}, \code{FixAlleleIDs}, \code{IUPACcodes}, \code{LowerCase}, \code{Indels}, -\code{ChromPos}, \code{allNAcol}, \code{allNArow}, \code{RefAltSeqs}. +\item{checks}{Named logical vector with twelve entries: +\code{Columns}, \code{FixAlleleIDs}, \code{IUPACcodes}, \code{IUPACcodes_IdenticalRefAlt}, +\code{LowerCase}, \code{Indels}, \code{ChromPos}, \code{allNAcol}, \code{allNArow}, \code{RefAltSeqs}, +\code{OtherAlleles}, \code{IUPACcodes_MatchAlleles}. \code{TRUE} means the condition was detected (or passed for \code{Columns}, \code{FixAlleleIDs}, \code{ChromPos}, and \code{RefAltSeqs}); \code{NA} means the check was skipped.} @@ -28,6 +29,11 @@ when it is \code{FALSE}. Indexed by the same names as \code{checks}.} \item{indel_clone_ids}{Character vector of \code{CloneID}s where ref/alt lengths differ. Returns \code{character(0)} if none are found, or \code{NULL} when required columns are missing.} +\item{iupac_identical_clone_ids}{Character vector of \code{CloneID}s where +Ref and Alt alleles have IUPAC codes at the same positions in their +sequences (the sequences may differ at standard ATCG positions). Returns +\code{character(0)} if none are found, or \code{NULL} when required columns are +missing.} \item{missRef}{Character vector of \code{CloneID}s that have no \code{Ref} allele row. Returns \code{character(0)} if all \code{CloneID}s have a \code{Ref} row, or \code{NULL} when required columns are missing.} @@ -37,12 +43,15 @@ row. Returns \code{character(0)} if all \code{CloneID}s have an \code{Alt} row, } } \description{ -Performs nine quick validations on an allele report: +Performs twelve quick validations on an allele report: \enumerate{ \item \strong{Columns} - required columns are present (\code{CloneID}, \code{AlleleID}, \code{AlleleSequence}); \item \strong{FixAlleleIDs} - first column's first up-to-6 rows are not all blank or \code{"*"} \emph{and} both \verb{_0001} and \verb{_0002} appear in \code{AlleleID}; -\item \strong{IUPACcodes} - presence of non-ATCG characters in \code{AlleleSequence}; +\item \strong{IUPACcodes} - presence of non-ATCG characters in Ref_/Alt_ \code{AlleleSequence} +that differ in position between Ref and Alt, or are present in only one; +\item \strong{IUPACcodes_IdenticalRefAlt} - presence of Ref_/Alt_ pairs that share IUPAC codes +at the same positions in their sequences; \item \strong{LowerCase} - presence of lowercase a/t/c/g in \code{AlleleSequence}; \item \strong{Indels} - reference/alternate allele lengths differ for the same \code{CloneID}, or a \code{"-"} character is present in \code{AlleleSequence}; @@ -51,7 +60,8 @@ or a \code{"-"} character is present in \code{AlleleSequence}; \item \strong{allNAcol} - at least one column contains only \code{NA} or empty values; \item \strong{allNArow} - at least one row contains only \code{NA} or empty values; \item \strong{RefAltSeqs} - every \code{CloneID} has at least one \code{Ref} and one \code{Alt} allele row; -\item \strong{OtherAlleles} - presence of alleles where the target locus differs from both the Ref and Alt in \code{AlleleSequence}. +\item \strong{OtherAlleles} - presence of alleles where the target locus differs from both the Ref and Alt in \code{AlleleSequence}; +\item \strong{IUPACcodes_MatchAlleles} - presence of non-ATCG characters in \code{AlleleSequence} for RefMatch_/AltMatch_/Other alleles. } } \details{ @@ -63,6 +73,13 @@ first 7 rows are dropped before subsequent checks are run. The check is \verb{_0001}/\verb{_0002} suffixes). \item \strong{IUPAC check:} Flags any character outside \code{A}, \code{T}, \code{C}, \code{G} and \code{"-"} (case-insensitive), which includes ambiguity codes (\code{N}, \code{R}, \code{Y}, etc.). +For Ref_/Alt_ alleles, two checks are performed: \code{IUPACcodes} is \code{TRUE} when +any CloneID has IUPAC codes at different positions between Ref and Alt, or +IUPAC is present in only one of them. \code{IUPACcodes_IdenticalRefAlt} is \code{TRUE} +when any CloneID has Ref and Alt with IUPAC codes at exactly the same positions +(non-IUPAC bases may still differ). Those CloneIDs are stored in +\code{iupac_identical_clone_ids}. RefMatch_/AltMatch_/Other alleles are checked +separately via \code{IUPACcodes_MatchAlleles}. \item \strong{Indels:} Rows are split by \code{AlleleID} containing \code{"Ref_0001"} vs \code{"Alt_0002"}, merged by \code{CloneID}, and flagged as indels if either (a) the lengths of \code{AlleleSequence} differ, (b) the sequences have the same length diff --git a/man/check_ped.Rd b/man/check_ped.Rd deleted file mode 100644 index 9a12a07..0000000 --- a/man/check_ped.Rd +++ /dev/null @@ -1,61 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/check_ped.R -\name{check_ped} -\alias{check_ped} -\title{Check a pedigree file for accuracy and report/correct common errors} -\usage{ -check_ped(ped.file, seed = NULL, verbose = TRUE) -} -\arguments{ -\item{ped.file}{Path to the pedigree text file.} - -\item{seed}{Optional seed for reproducibility.} - -\item{verbose}{Logical. If TRUE (default), prints the report to the console.} -} -\value{ -A list of data.frames containing detected issues: -\itemize{ -\item \code{exact_duplicates}: rows that were exact duplicates -\item \code{repeated_ids_diff}: IDs appearing more than once with conflicting sire/dam -\item \code{messy_parents}: IDs appearing as both sire and dam -\item \code{missing_parents}: parents added to the pedigree with 0 as sire/dam -\item \code{dependencies}: detected cycles in the pedigree -\item \code{corrected_pedigree}: corrected pedigree table -} -} -\description{ -\code{check_ped} reads a 3-column pedigree file (tab-separated, columns labeled \code{id}, \code{sire}, \code{dam} in any order) -and performs quality checks, optionally correcting or flagging errors. -} -\details{ -The function checks for: -\itemize{ -\item Exact duplicate rows and removes them (keeping one copy) -\item IDs that appear more than once with conflicting sire/dam assignments (sets sire/dam to "0") -\item IDs that appear in both sire and dam columns -\item Missing parents (IDs referenced as sire/dam but not in \code{id} column), adds them with sire/dam = "0" -\item Direct and indirect pedigree dependencies (cycles), such as a parent being its own descendant -} - -After an initial run to clean exact duplicates and repeated IDs, you can run the function again to detect cycles more accurately. - -The function does \strong{not} overwrite the input file or create objects in the -global environment. Instead, it returns the report and corrected pedigree in -a list. -} -\examples{ -ped_file <- system.file("check_ped_test.txt", package = "BIGr") -ped_errors <- check_ped(ped.file = ped_file, seed = 101919, verbose = FALSE) - -# Access messy parents -ped_errors$messy_parents - -# Access corrected pedigree -ped_errors$corrected_pedigree - -# IDs with messy parents -messy_ids <- ped_errors$messy_parents$id -print(messy_ids) - -} diff --git a/man/filterMADC.Rd b/man/filterMADC.Rd index 671a599..f5aabb0 100644 --- a/man/filterMADC.Rd +++ b/man/filterMADC.Rd @@ -6,45 +6,50 @@ \usage{ filterMADC( madc_file, - min.mean.reads = NULL, - max.mean.reads = NULL, + min.locus.depth = NULL, + max.locus.depth = NULL, max.mhaps.per.loci = NULL, min.reads.per.site = 1, min.ind.with.reads = NULL, target.only = FALSE, - n.summary.columns = NULL, output.file = NULL ) } \arguments{ -\item{madc_file}{Path to the MADC file to be filtered} +\item{madc_file}{Path to the fixed allele ID MADC file to be filtered} -\item{min.mean.reads}{Minimum mean read depth for filtering} +\item{min.locus.depth}{Minimum mean read depth per sample at a locus. Depth is the sum of all mhap reads at the locus (\code{|Ref}, \code{|Alt}, \code{|RefMatch}, \code{|AltMatch}, and \code{|Other}) divided by the number of samples. Loci below this threshold are removed entirely (Ref and Alt together). If NULL, no minimum is applied.} -\item{max.mean.reads}{Maximum mean read depth for filtering} +\item{max.locus.depth}{Maximum mean read depth per sample at a locus (computed as for \code{min.locus.depth}). Loci above this threshold are removed entirely, e.g. to exclude paralogous over-amplification. If NULL, no maximum is applied.} -\item{max.mhaps.per.loci}{Maximum number of matching mhaps per target loci. Retains only the target Ref and Alt loci at the sites that exceeds the \code{max.mhaps.per.loci} threshold.} +\item{max.mhaps.per.loci}{Maximum number of mhaps per target loci. At loci whose total number of mhaps exceeds the \code{max.mhaps.per.loci} threshold, only the target \code{|Ref} and \code{|Alt} alleles are retained (the \code{|RefMatch}, \code{|AltMatch}, and \code{|Other} alleles are removed).} -\item{min.reads.per.site}{Minimum number of reads per site for \code{min.ind.with.reads}. Otherwise, this parameter is ignored} +\item{min.reads.per.site}{Minimum number of reads for a site (mhap in a sample) to count toward \code{min.ind.with.reads}.} -\item{min.ind.with.reads}{Minimum number of individuals with \code{min.reads.per.site} reads for filtering} +\item{min.ind.with.reads}{Minimum number of individuals with \code{min.reads.per.site} reads required to retain a mhap. Target \code{|Ref}/\code{|Alt} alleles are always retained; only non-target (\code{|RefMatch}/\code{|AltMatch}/\code{|Other}) mhaps are removed by this filter.} -\item{target.only}{Logical indicating whether to filter for target loci only} +\item{target.only}{Logical indicating whether to retain only the target \code{|Ref} and \code{|Alt} alleles (dropping \code{|RefMatch}, \code{|AltMatch}, and \code{|Other})} -\item{n.summary.columns}{(optional) Number of summary columns to remove from MADC file not including the first three. Otherwise, the columns will be automatically detected and removed.} - -\item{output.file}{Path to save the filtered data (if NULL, data will not be saved)} +\item{output.file}{Path to save the filtered data; \code{.csv} is appended to the name (if NULL, data will not be saved)} } \value{ -data.frame or saved csv file +A data.frame of the filtered MADC when \code{output.file} is NULL. Otherwise the filtered data is saved to \code{paste0(output.file, ".csv")} and NULL is returned. } \description{ Filter and process MADC files to remove low quality microhaplotypes } \details{ -This function can filter raw MADC files or pre-processed MADC files with fixed allele IDs. Additionally, -it can filter based on mean read depth, number of mhaps per target loci, and other criteria. Optionally, users -can plot summary statistics and save the filtered data to a file. +This function filters a fixed allele ID MADC file (one processed through HapApp so that +AlleleIDs carry unique \code{|Ref_0001}/\code{|Alt_0002}-style suffixes). The input is +validated with \code{\link{check_madc_sanity}}; raw DArT MADC files (or any file without +fixed AlleleIDs) are rejected with an error. + +Filtering is designed so that a locus never loses its target \code{|Ref}/\code{|Alt} pair: +the per-locus read-depth window (\code{min.locus.depth}/\code{max.locus.depth}) removes +whole loci (Ref and Alt together), and the other filters +(\code{max.mhaps.per.loci}, \code{min.ind.with.reads}, \code{target.only}) only prune +non-target \code{|RefMatch}/\code{|AltMatch}/\code{|Other} mhaps. The filtered data can +optionally be saved to a file. } \examples{ #Example @@ -54,13 +59,12 @@ madc_file <- system.file("example_MADC_FixedAlleleID.csv", package="BIGr") #Remove mhaps exceeding 3 per target region including the ref and alt target mhaps filtered_df <- filterMADC(madc_file, - min.mean.reads = NULL, - max.mean.reads = NULL, + min.locus.depth = NULL, + max.locus.depth = NULL, max.mhaps.per.loci = 3, min.reads.per.site = 1, min.ind.with.reads = NULL, target.only = FALSE, - n.summary.columns = NULL, output.file = NULL) diff --git a/man/filterVCF.Rd b/man/filterVCF.Rd index 676ef7f..39af09e 100644 --- a/man/filterVCF.Rd +++ b/man/filterVCF.Rd @@ -22,30 +22,57 @@ filterVCF( \arguments{ \item{vcf.file}{vcfR object or path to VCF file. Can be unzipped (.vcf) or gzipped (.vcf.gz).} -\item{filter.OD}{Updog filter} - -\item{filter.BIAS.min}{Updog filter (requires a value for both BIAS.min and BIAS.max)} - -\item{filter.BIAS.max}{Updog filter (requires a value for both BIAS.min and BIAS.max)} - -\item{filter.DP}{Total read depth at each SNP filter} - -\item{filter.MPP}{Updog filter} - -\item{filter.PMC}{Updog filter} - -\item{filter.MAF}{Minor allele frequency filter} - -\item{filter.SAMPLE.miss}{Sample missing data filter} - -\item{filter.SNP.miss}{SNP missing data filter} - -\item{ploidy}{The ploidy of the species being analyzed} - -\item{output.file}{output file name (optional). If no output.file name provided, then a vcfR object will be returned.} +\item{filter.OD}{Maximum overdispersion, read from the \code{OD} field of the INFO +column as estimated by updog. Variants with an \code{OD} below this value are kept.} + +\item{filter.BIAS.min}{Minimum allele bias, read from the \code{BIAS} field of the +INFO column as estimated by updog. Variants with a \code{BIAS} above this value are +kept. Has no effect unless \code{filter.BIAS.max} is also supplied.} + +\item{filter.BIAS.max}{Maximum allele bias, read from the \code{BIAS} field of the +INFO column. Variants with a \code{BIAS} below this value are kept. Has no effect +unless \code{filter.BIAS.min} is also supplied.} + +\item{filter.DP}{Minimum read depth for a genotype call. Calls whose FORMAT/DP +is below this value are set to missing. Calls without a DP value are left +unchanged. This does not remove variants, though the calls it sets to missing +then count towards \code{filter.SNP.miss}, \code{filter.SAMPLE.miss} and \code{filter.MAF}.} + +\item{filter.MPP}{Minimum posterior probability for a genotype call, as +reported by updog. Calls whose FORMAT/MPP is below this value are set to +missing, on the same terms as \code{filter.DP}.} + +\item{filter.PMC}{Maximum proportion of individuals misclassified, read from +the \code{PMC} field of the INFO column as estimated by updog. Variants with a +\code{PMC} below this value are kept.} + +\item{filter.MAF}{Minimum minor allele frequency. Variants with a minor allele +frequency above this value are kept. Calculated after any sample removal, so +it reflects only the samples that survive \code{filter.SAMPLE.miss}.} + +\item{filter.SAMPLE.miss}{Maximum proportion of missing genotype calls a sample +may have, between 0 and 1. Samples missing a smaller proportion than this are +kept. Note that this is a proportion and not a percentage; values outside 0 +to 1 give an error.} + +\item{filter.SNP.miss}{Maximum proportion of missing genotype calls a variant +may have, between 0 and 1. Variants missing a smaller proportion than this are +kept. Note that this is a proportion and not a percentage; values outside 0 +to 1 give an error.} + +\item{ploidy}{The ploidy of the species being analyzed. Required. Used to write +the missing genotype, so a diploid is recorded as \code{./.} and a tetraploid as +\code{./././.} when a call is filtered out.} + +\item{output.file}{Output file name, without an extension (optional). When +supplied, the filtered VCF is written as a gzipped file and nothing is +returned. \code{.vcf.gz} is appended when \code{vcf.file} is a path, and +\verb{_filtered.vcf.gz} when \code{vcf.file} is a vcfR object. When not supplied, a +vcfR object is returned instead and no file is written.} } \value{ -A gzipped vcf file +A vcfR object when \code{output.file} is not supplied. Otherwise the +filtered VCF is written to disk and nothing is returned. } \description{ This function will filter a VCF file or vcfR object and export the updated version @@ -54,6 +81,19 @@ This function will filter a VCF file or vcfR object and export the updated versi This function will input a VCF file or vcfR object and filter based on the user defined options. The output file will be saved to the location and with the name that is specified. The VCF format is v4.3 + +Filters are applied in a fixed order, which matters because each one acts on +what the previous ones left behind. \code{filter.DP} and \code{filter.MPP} run first and +set individual genotype calls to missing. \code{filter.OD}, \code{filter.BIAS.min} and +\code{filter.BIAS.max}, and \code{filter.PMC} then remove variants using the INFO column. +\code{filter.SNP.miss} removes variants and \code{filter.SAMPLE.miss} removes samples, +both counting the calls that were set to missing above. \code{filter.MAF} runs last, +so it is calculated from the samples that remain. + +A variant is removed if the value a requested filter needs cannot be read from +it, for instance because the INFO field is absent from that record or because +every genotype call for the variant is missing. The number of variants removed +for this reason is reported in a warning. } \examples{ ## Use file paths for each file on the local system diff --git a/man/find_parentage.Rd b/man/find_parentage.Rd deleted file mode 100644 index 77b9c3c..0000000 --- a/man/find_parentage.Rd +++ /dev/null @@ -1,159 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/find_parentage.R -\name{find_parentage} -\alias{find_parentage} -\title{Find Parentage Assignments for Progeny} -\usage{ -find_parentage( - genotypes_file, - parents_file, - progeny_file, - method = "best_pair", - min_markers = 10, - error_threshold = 5, - show_ties = TRUE, - allow_selfing = TRUE, - verbose = TRUE, - write_txt = TRUE -) -} -\arguments{ -\item{genotypes_file}{Path to a TSV/CSV/TXT file containing genotype data. -Must include an 'ID' column followed by marker columns coded as 0, 1, 2 -(allele dosage).} - -\item{parents_file}{Path to a TSV/CSV/TXT file listing candidate parent IDs. -Must include an 'ID' column. An optional 'Sex' column with values -'M' (male parent), 'F' (female parent), or 'A' (ambiguous) determines -which parents are tested for each role. If absent, all parents are treated -as ambiguous.} - -\item{progeny_file}{Path to a TSV/CSV/TXT file listing progeny IDs to assign. -Must include an 'ID' column.} - -\item{method}{Character. Parentage assignment method. One of: -\itemize{ -\item \code{"best_male_parent"} - finds the best male parent for each -progeny using homozygous mismatch rate. -\item \code{"best_female_parent"} - finds the best female parent for each -progeny using homozygous mismatch rate. -\item \code{"best_match"} - finds the single best parent (either sex) -using homozygous mismatch rate. -\item \code{"best_pair"} - finds the best male-female parent pair for -each progeny using full Mendelian error rate (default). -}} - -\item{min_markers}{Integer. Minimum number of non-missing markers required -to report a parentage assignment. Progeny-parent comparisons with fewer -markers are flagged as \code{LOW_MARKERS} and no assignment is made -(default: \code{10}).} - -\item{error_threshold}{Numeric. Maximum mismatch percentage to report a -parentage assignment as confident. Assignments above this threshold are -flagged as \code{HIGH_ERROR} in the \code{Assignment_Status} column -(default: \code{5.0}). Must be between 0 and 100.} - -\item{show_ties}{Logical. If \code{TRUE}, all tied best pairs (after -tie-breaking by maximum markers tested) are reported as additional columns -(\code{Male_Parent_1}, \code{Male_Parent_2}, etc.) when -\code{method = "best_pair"}. The base columns (\code{Male_Parent}, -\code{Female_Parent}, etc.) are always populated with the top result. -If \code{FALSE}, only one tied pair is reported with a warning. -Default is \code{TRUE}.} - -\item{allow_selfing}{Logical. If \code{FALSE}, male-female parent pairs where -both IDs are identical are excluded when \code{method = "best_pair"}. -Default is \code{TRUE}.} - -\item{verbose}{Logical. If \code{TRUE}, prints progress messages, summary -statistics, and the results table to the console. Default is \code{TRUE}.} - -\item{write_txt}{Logical. If \code{TRUE}, writes results to -\code{parentage_results_dt.txt} in the working directory. Default is -\code{TRUE}.} -} -\value{ -A \code{data.table} with one row per progeny. Columns depend on the -method used: -\itemize{ -\item \code{best_male_parent} / \code{best_female_parent} / \code{best_match}: -\code{Progeny}, \code{Best_Match}, \code{Mendelian_Error_Pct}, -\code{Markers_Tested}, \code{Assignment_Status}. -\item \code{best_pair} (no ties after tie-breaking): \code{Progeny}, -\code{Male_Parent}, \code{Female_Parent}, \code{Mendelian_Error_Pct}, -\code{Markers_Tested}, \code{Assignment_Status}. -\item \code{best_pair} (ties remain after tie-breaking, -\code{show_ties = TRUE}): base columns are always populated with the -top result, plus suffix columns \code{Male_Parent_1}, -\code{Female_Parent_1}, etc. for each tied pair. -} -\code{Assignment_Status} is one of \code{PASS}, \code{HIGH_ERROR}, or -\code{LOW_MARKERS}. Returned invisibly when \code{verbose = TRUE}. -} -\description{ -Assigns the most likely parent(s) to each progeny individual based on -genotypic data using Mendelian error rates or homozygous mismatch rates. -} -\details{ -A homozygous-only genotype matrix is pre-computed once at startup and shared -across all methods that require it, avoiding redundant computation. - -For \code{"best_male_parent"}, \code{"best_female_parent"}, and -\code{"best_match"}, only homozygous markers (coded 0 or 2) are used for -comparison; heterozygous markers (coded 1) are set to \code{NA}. This -reduces false mismatches caused by phase ambiguity. - -For \code{"best_pair"}, all markers are used and full Mendelian inheritance -rules are applied. Mismatch rates and comparison counts are computed across -all progeny simultaneously using vectorised \code{vapply} calls, producing -\code{n_pairs x n_progeny} matrices and giving substantial speed gains for -large datasets. Both matrices are explicitly coerced to matrix form to -handle the edge case of a single parent pair correctly. - -When multiple pairs share the lowest Mendelian error rate, ties are broken -by selecting the pair(s) with the greatest number of markers tested. If ties -still remain after this step, all remaining tied pairs are reported when -\code{show_ties = TRUE}. - -The base columns (\code{Male_Parent}, \code{Female_Parent}, -\code{Mendelian_Error_Pct}, \code{Markers_Tested}) are always populated with -the top result, ensuring no missing values in these columns regardless of -tie status. - -Output rows are pre-allocated as a \code{data.table} and filled by reference -using \code{data.table::set()}, avoiding repeated memory allocation during -the results-building step. - -Individuals in \code{parents_file} or \code{progeny_file} that are absent -from \code{genotypes_file} are removed with a warning. - -Progeny with fewer non-missing markers than \code{min_markers} are flagged -\code{LOW_MARKERS} and no parent assignment is reported. Progeny whose best -match exceeds \code{error_threshold} are flagged \code{HIGH_ERROR}. -} -\examples{ -\dontrun{ -# Assign best male-female parent pair to each progeny -results <- find_parentage( - genotypes_file = "genotypes.txt", - parents_file = "parents.txt", - progeny_file = "progeny.txt", - method = "best_pair", - min_markers = 50, - error_threshold = 5.0, - show_ties = TRUE, - allow_selfing = FALSE -) - -# Find best individual parent match (ignoring sex) -results <- find_parentage( - genotypes_file = "genotypes.txt", - parents_file = "parents.txt", - progeny_file = "progeny.txt", - method = "best_match", - min_markers = 30, - error_threshold = 3.0 -) -} - -} diff --git a/man/imputation_concordance.Rd b/man/imputation_concordance.Rd index 22e9462..f96493f 100644 --- a/man/imputation_concordance.Rd +++ b/man/imputation_concordance.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/imputation_concordance.R \name{imputation_concordance} \alias{imputation_concordance} -\title{Calculate Concordance between Imputed and Reference Genotypes} +\title{Calculate concordance between imputed and reference genotypes} \usage{ imputation_concordance( reference_genos, @@ -15,77 +15,65 @@ imputation_concordance( ) } \arguments{ -\item{reference_genos}{A data frame containing reference genotype data, -with rows as samples and columns as markers. Must include a column named \code{ID}.} +\item{reference_genos}{A data frame of reference genotypes (samples × markers) +with a column named \code{ID}.} -\item{imputed_genos}{A data frame containing imputed genotype data, -with rows as samples and columns as markers. Must include a column named \code{ID}.} +\item{imputed_genos}{A data frame of imputed genotypes (samples × markers) +with a column named \code{ID}.} -\item{missing_code}{Optional value specifying missing data. If provided, -loci with this value in either dataset will be excluded from the concordance calculation.} +\item{missing_code}{Optional value indicating missing data. Loci carrying this +value in either dataset are excluded from the concordance calculation.} -\item{snps_2_exclude}{Optional vector of marker IDs to exclude from the concordance calculation.} +\item{snps_2_exclude}{Optional character vector of marker names to exclude.} -\item{verbose}{Logical. If \code{TRUE}, prints summary statistics (minimum, quartiles, -median, mean, maximum) of concordance percentages.} +\item{verbose}{Logical. Print a five-number summary of concordance? Default \code{FALSE}.} -\item{plot}{Logical. If \code{TRUE}, produces a bar plot of concordance percentage -by sample.} +\item{plot}{Logical. Produce a bar plot of concordance by sample? Default \code{FALSE}.} -\item{print_result}{Logical. If \code{TRUE} (default), prints the concordance -results data frame to the console. If \code{FALSE}, results are returned invisibly.} +\item{print_result}{Logical. Print the results data frame to the console? +Default \code{TRUE}. If \code{FALSE}, results are returned invisibly.} } \value{ -A data frame with: -\itemize{ -\item \code{ID}: Sample identifiers shared between the datasets. -\item \code{Concordance}: Percentage of matching genotypes per sample. +A data frame with columns: +\describe{ +\item{ID}{Sample identifiers shared between the two datasets.} +\item{Concordance}{Percentage of matching genotypes per sample.} } -If \code{print_result = FALSE}, the data frame is returned invisibly. } \description{ -This function calculates the concordance between imputed and reference -genotypes. It assumes that samples are rows and markers are columns. -Allele dosages (0, 1, 2) are recommended but other numeric formats are supported. -Missing data in either dataset can be excluded from the concordance calculation -using the \code{missing_code} argument. Specific markers can be excluded using -the \code{snps_2_exclude} argument. +Compares imputed and reference genotype datasets sample by sample, returning +the percentage of matching genotypes per sample. Expects samples as rows and +markers as columns, with allele dosages (0, 1, 2) recommended. Other numeric +formats are supported. } \details{ -The function: -\enumerate{ -\item Identifies common samples and markers between the datasets. -\item Optionally excludes specified SNPs. -\item Removes loci with missing data (if \code{missing_code} is provided). -\item Computes per-sample concordance as the percentage of matching genotypes. -} - -When \code{plot = TRUE}, a bar plot showing concordance percentage per sample -is generated using \pkg{ggplot2}. +The function identifies common samples and markers between the two datasets, +optionally removes specified SNPs and loci with missing data, then computes +per-sample concordance as the percentage of matching genotypes over valid loci. +When \code{plot = TRUE}, a bar plot is produced with \pkg{ggplot2}. } \examples{ ref <- data.frame( - ID = c("S1", "S2", "S3"), + ID = c("S1", "S2", "S3"), SNP1 = c(0, 1, 2), SNP2 = c(1, 1, 0), SNP3 = c(2, 5, 1) ) - -test <- data.frame( - ID = c("S1", "S2", "S3"), +imp <- data.frame( + ID = c("S1", "S2", "S3"), SNP1 = c(0, 0, 2), SNP2 = c(1, 1, 1), SNP3 = c(2, 5, 0) ) - -result <- imputation_concordance( +imputation_concordance( reference_genos = ref, - imputed_genos = test, - snps_2_exclude = "SNP2", - missing_code = 5, - print_result = FALSE + imputed_genos = imp, + snps_2_exclude = "SNP2", + missing_code = 5, + print_result = FALSE ) -result - +} +\author{ +Josué Chinchilla-Vargas } diff --git a/man/madc2vcf_all.Rd b/man/madc2vcf_all.Rd index c15e69b..990e500 100644 --- a/man/madc2vcf_all.Rd +++ b/man/madc2vcf_all.Rd @@ -18,6 +18,8 @@ madc2vcf_all( add_others = TRUE, others_max_snps = 5, others_rm_with_indels = TRUE, + others_min_dist = 5, + others_max_close_snps = 3, verbose = TRUE ) } @@ -40,14 +42,18 @@ madc2vcf_all( \item{out_vcf}{A string specifying the name of the output VCF file. If the file extension is not \code{.vcf}, it will be appended automatically.} -\item{markers_info}{A string specifying the path to a CSV file with marker information (CloneID/BI_markerID, Chr, Pos, Ref, Alt, Type, Indel_pos columns as needed).} +\item{markers_info}{A string specifying the path to a CSV file with marker information (CloneID/Marker_ID/BI_markerID, Chr, Pos, Ref, Alt, Type, Indel_pos columns as needed).} \item{add_others}{A logical value. If TRUE, alleles labeled "Other" in the MADC file are included in off-target SNP extraction. Default is TRUE.} -\item{others_max_snps}{An integer or NULL. If not NULL, Other alleles with more than this many SNP differences versus the Ref sequence (as detected by pairwise alignment) are discarded. Default is NULL (no limit).} +\item{others_max_snps}{An integer or NULL. If not NULL, Other alleles with more than this many SNP differences versus the Ref sequence (as detected by pairwise alignment) are discarded. Default is 5.} \item{others_rm_with_indels}{A logical value. If TRUE, Other alleles that contain insertions or deletions relative to the Ref sequence (as detected by pairwise alignment) are discarded. Default is TRUE.} +\item{others_min_dist}{An integer or NULL. Distance in base pairs below which two SNPs in an Other allele are considered close to each other. Used together with \code{others_max_close_snps}. Default is 5.} + +\item{others_max_close_snps}{An integer or NULL. If not NULL, Other alleles with more than this many SNPs that are closer than \code{others_min_dist} base pairs to each other are discarded. The SNPs are counted before the target position and positions with N or IUPAC codes are removed. The filter is not applied if either \code{others_min_dist} or \code{others_max_close_snps} is NULL. Default is 3.} + \item{verbose}{A logical value indicating whether to print metrics and progress to the console. Default is TRUE.} } \value{ diff --git a/man/madc_plot.Rd b/man/madc_plot.Rd new file mode 100644 index 0000000..d5659fa --- /dev/null +++ b/man/madc_plot.Rd @@ -0,0 +1,237 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/madc_plot.R +\name{madc_plot} +\alias{madc_plot} +\title{Plot a fixed allele ID MADC} +\usage{ +madc_plot( + madc, + plot.type = c("pca", "marker", "heatmap", "missing"), + target.only = FALSE, + min.depth = 10, + metadata = NULL, + group.col = NULL, + shape.col = NULL, + palette = NULL, + ploidy = NULL, + balance.density = FALSE, + markers_info = NULL, + pc.x = 1, + pc.y = 2, + loci.miss.max = 0.5, + sample.miss.max = 0.5, + miss.sort = c("none", "desc", "asc"), + horizontal = FALSE, + fill = c("depth", "mhaps"), + facet.chrom = FALSE, + max.loci = NULL, + max.samples = NULL, + density.window = 1e+06, + density.col = c("grey92", "#9e9ac8", "#3f007d"), + depth.col = c("#2166ac", "#e0e0e0", "#b2182b"), + depth.outlier = 3, + depth.qc.maxmiss = 0.5, + mhap.col = "#35978f", + paralog.flag = 10, + tick.height = 0.03, + density.height = 0.08, + mhap.height = 0.18, + label.gap = 12, + mhap.min.reads = 1, + titles = TRUE, + output.file = NULL, + width = 8, + height = 6, + dpi = 300, + verbose = TRUE +) +} +\arguments{ +\item{madc}{Path to a fixed allele ID MADC file, or an already-read data.frame.} + +\item{plot.type}{One or more of \code{"pca"}, \code{"marker"}, \code{"heatmap"}, \code{"missing"}, +\code{"balance"}, \code{"depth"}, \code{"circos"}. A single value returns that plot; several +(excluding \code{"circos"}) return a multi-panel figure. \code{"circos"} must be +requested on its own. Default \code{c("pca", "marker", "heatmap", "missing")}.} + +\item{target.only}{Logical; use only the target \verb{|Ref}/\verb{|Alt} alleles (see +Details). Default \code{FALSE}.} + +\item{min.depth}{Minimum locus x sample read depth for a cell to be counted as +present (see Details for the plots that use it). Default \code{10}.} + +\item{metadata}{Optional sample-metadata data.frame (see \code{\link[=madc_summary]{madc_summary()}}) used +to color the PCA, group the missing-data boxplot, and sort the heatmap +samples by category.} + +\item{group.col}{Category column in \code{metadata} (e.g. species/population). +If \code{NULL} (default), samples are not grouped. In the heatmap, samples are +sorted by category, with one x-axis label per category and white lines +between categories. Samples not found in \code{metadata} are left out of the +heatmap and the missing-data boxplot.} + +\item{shape.col}{Optional second \code{metadata} column mapped to point shape in +the PCA (e.g. plate or location), so two variables can be shown at once. +Only used when \code{metadata} is supplied.} + +\item{palette}{Optional vector of colors for the categorical scales (PCA +point color, missing-boxplot fill); interpolated to the number of +categories. \code{NULL} uses the ggplot2 defaults. Not used by the other plots.} + +\item{ploidy}{Optional integer species ploidy for the \code{"balance"} plot. When +supplied, dashed guides are drawn at the expected alt-read ratios +\code{(0:ploidy)/ploidy} (e.g. 0, 0.5, 1 for a diploid) and a depth-dependent 95\% +binomial interval band is drawn around each interior expected ratio (the +expected spread of \code{A/D} under \code{Binomial(D, ratio)}), so observed dispersion +wider than sampling noise stands out. \code{NULL} (default) draws the ratio x +depth density with no guides/band, which also suits mixed/unknown ploidy.} + +\item{balance.density}{Logical; for a lone \code{plot.type = "balance"}, stack a +histogram of the alt-read ratio above the heatmap (aligned x-axis). This +returns a composite grob instead of a \code{ggplot}, so it is ignored when +\code{"balance"} is combined with other plot types. Default \code{FALSE}.} + +\item{markers_info}{Optional marker lookup (id + \code{Chr} + \code{Pos}) used when the +CloneIDs are not in \code{Chr_Pos} format. It sets the marker positions in the +marker and circos plots and the marker order in the heatmap.} + +\item{pc.x, pc.y}{Principal components to plot (PCA). Defaults \code{1}, \code{2}.} + +\item{loci.miss.max, sample.miss.max}{Drop loci, then samples, whose fraction +of missing alternative read ratios (no \verb{|Ref}/\verb{|Alt} reads) is above these +values before PCA. Defaults \code{0.5}.} + +\item{miss.sort}{Ordering of the category boxes in the \code{"missing"} plot by their +median missing rate: \code{"none"} (default, keeps category order), \code{"desc"} +(highest to lowest), or \code{"asc"} (lowest to highest). Only applies when a +grouping category is supplied.} + +\item{horizontal}{Logical; draw the \code{"missing"} boxplots horizontally +(\code{coord_flip}). Default \code{FALSE}. With \code{miss.sort}, the order then reads +top-to-bottom.} + +\item{fill}{Heatmap fill metric: \code{"depth"} (default; total read depth per +locus x sample) or \code{"mhaps"} (number of mHaps with at least +\code{mhap.min.reads} reads). Cells with a value of 0 are drawn black.} + +\item{facet.chrom}{Logical; facet the heatmap by chromosome. Default \code{FALSE}.} + +\item{max.loci, max.samples}{Optional caps; larger heatmaps are subsampled to +evenly spaced loci/samples (with a message when \code{verbose = TRUE}).} + +\item{density.window}{Window size in bp for the circos marker-density track. +Default \code{1e6} (1 Mb).} + +\item{density.col}{Gradient colors (low -> high) for the circos marker-density +heatmap ring; interpolated across the range. Default \code{c("grey92", "#9e9ac8", "#3f007d")}.} + +\item{depth.col}{Length-3 vector of colors for the circos "Depth" ribbon: +markers that are problematic (mean depth below \code{min.depth} or missing rate +above \code{depth.qc.maxmiss}), \code{normal}, or a high-depth \code{outlier} (see +\code{depth.outlier}). Only problematic and outlier markers are highlighted; +normal markers take the neutral middle color. Default +\code{c("#2166ac","#e0e0e0","#b2182b")}.} + +\item{depth.outlier}{Multiplier \code{k} for the robust high-depth outlier cutoff +\code{median + k * MAD} of the marker mean depths (not mean +/- SD, which a few +extreme loci would drag up). Default \code{3}.} + +\item{depth.qc.maxmiss}{Circos "Depth" ring: a marker is flagged problematic +when its mean depth is below \code{min.depth} OR its missing rate exceeds this +fraction - catching high-dropout markers (e.g. 50\% zeros / 50\% deep) that a +mean-depth cut alone reads as normal. Default \code{0.5}.} + +\item{mhap.col}{Fill color for the circos "mHaps" log-scaled radial bars. +Default \code{"#35978f"}.} + +\item{paralog.flag}{Number of mHaps (allele rows for the marker) above which a +circos "mHaps" locus is flagged as paralog-suspect: its bar's over-threshold +excess is drawn as a red segment above a dashed reference ring. Default \code{10}.} + +\item{tick.height, density.height, mhap.height}{Circos track heights (fraction of +the radius) for the per-marker locus tick rug, the density heatmap ring, and +the mHap bar track. Defaults \code{0.03}, \code{0.08}, and \code{0.18}.} + +\item{label.gap}{Width in degrees of the label corridor opened at 12 o'clock +for the circos track titles. Default \code{12}. With a single chromosome, at least +\code{30} is used.} + +\item{mhap.min.reads}{Minimum reads for a mHap to count as present in a +locus x sample cell (heatmap with \code{fill = "mhaps"}). Default \code{1}.} + +\item{titles}{Logical; draw the plot titles and subtitles. Default \code{TRUE}. +\code{FALSE} drops both from every requested plot (including the \code{"circos"} +figure and the multi-panel), leaving the axis labels, legends and the +\code{"balance"} caption - useful when the figure is captioned externally +(manuscript, report, slide).} + +\item{output.file}{If not \code{NULL}, the figure is also saved here. The format is +set by the extension: \code{.png}, \code{.pdf}, \code{.jpg}/\code{.jpeg}, \code{.tif}/\code{.tiff}, or +\code{.svg}; other extensions give an error.} + +\item{width, height, dpi}{Saved-figure dimensions (inches) and resolution (\code{dpi} +applies to PNG, JPEG, and TIFF only).} + +\item{verbose}{Logical; print progress/validation messages. Default \code{TRUE}.} +} +\value{ +For a single \code{plot.type}, a \code{ggplot}, returned invisibly when +\code{output.file} is given. With \code{plot.type = "balance"} and +\code{balance.density = TRUE} this is a composite grob; use +\code{grid::grid.draw()} to display it. For several types, an invisible list with +\code{plots} (named list) and \code{panel} (the assembled multi-panel grob); when +\code{output.file} is \code{NULL}, the panel is also drawn to the current graphics +device (in an interactive session, or when a device is already open). \code{"circos"} +returns invisibly the path of the saved figure (\code{output.file}, or a +temporary PNG when no \code{output.file} is given and no graphics device is open), +or \code{NULL} when it is drawn to an already open device. +} +\description{ +Visualize a fixed allele ID MADC file (one processed through HapApp) with a +choice of plots selected via \code{plot.type}: a PCA of alternative read ratios, a +linear marker-distribution genome plot, a markers x samples read-depth/#mHaps +heatmap, a per-sample missing-data boxplot, an allele read-ratio balance +diagnostic, a marker mean-depth (uniformity) distribution, and a \code{circos} +circular plot with concentric marker-position, marker-density, #mHaps, and +read-depth QC tracks. Requesting one type returns that plot; requesting +several (non-circos) returns a multi-panel figure. +} +\details{ +The input is validated with \code{\link[=check_madc_sanity]{check_madc_sanity()}}; raw DArT MADC files are +rejected with an error. With \code{target.only = TRUE}, only the target +\verb{|Ref}/\verb{|Alt} alleles are used. With \code{target.only = FALSE} (default), read +depth and mHap counts include all alleles, while the alternative read ratios +used by the \code{"pca"} and \code{"balance"} plots add the \verb{|RefMatch}/\verb{|AltMatch} +reads to their \verb{|Ref}/\verb{|Alt} alleles (as in \code{\link[=get_countsMADC]{get_countsMADC()}} with +\code{collapse_matches_counts = TRUE}) and do not use the \verb{|Other} reads. + +\code{min.depth} (locus x sample cells below this depth are treated as missing) is +applied in the \code{"missing"} plot and the \code{circos} "Depth" ring, and is drawn as +a reference line in the \code{"balance"} and \code{"depth"} plots. It is not applied to +the \code{"pca"}, \code{"marker"}, or \code{"heatmap"} plots. + +Plotting uses \code{ggplot2} and base \code{grid}; the \code{circos} plot additionally +requires the suggested \code{circlize} package and, being a base graphics figure, +must be requested on its own (it cannot be embedded in a multi-panel). The +\code{"pca"} plot is a sample/data-interpretation diagnostic (a ratio is missing +only when the marker has no \verb{|Ref}/\verb{|Alt} reads in the sample, missing ratios +are mean-imputed, loci with no variation are dropped, and the PCA is +unscaled), so strong biological structure can dominate it - read it for +sample swaps / population separation, not as evidence the sequencing run +itself was good or bad. The \code{"balance"} plot is the direct read-out of whether +the assay yields interpretable dosage signal; it shows every cell with a +defined ratio (including below \code{min.depth}, marked by a dotted line) and, with +\code{ploidy}, a depth-dependent binomial expectation band. +} +\examples{ +madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") +p <- madc_plot(madc_file, plot.type = "missing") +\donttest{ +madc_plot(madc_file, plot.type = c("pca", "heatmap")) +madc_plot(madc_file, plot.type = "balance", ploidy = 2) +} + +} +\seealso{ +\code{\link[=madc_summary]{madc_summary()}}, \code{\link[=filterMADC]{filterMADC()}} +} diff --git a/man/madc_summary.Rd b/man/madc_summary.Rd new file mode 100644 index 0000000..f1dcaf3 --- /dev/null +++ b/man/madc_summary.Rd @@ -0,0 +1,176 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/madc_summary.R +\name{madc_summary} +\alias{madc_summary} +\title{Summarize a fixed allele ID MADC} +\usage{ +madc_summary( + madc, + target.only = FALSE, + min.depth = 10, + metadata = NULL, + group.col = NULL, + markers_info = NULL, + mhap.cap = 10, + miss.thresholds = c(0, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100)/100, + depth.thresholds = c(1, 5, 10, 20, 50, 100), + mhap.min.reads = 1, + min.locus.depth = min.depth, + max.locus.depth = NULL, + max.missing = NULL, + max.offtarget = NULL, + max.mhaps.per.loci = NULL, + output.file = NULL, + output.format = c("csv", "xlsx"), + verbose = TRUE +) +} +\arguments{ +\item{madc}{Path to a fixed allele ID MADC file, or an already-read +data.frame.} + +\item{target.only}{Logical; restrict all metrics to the target \verb{|Ref}/\verb{|Alt} +alleles. Default \code{FALSE}.} + +\item{min.depth}{Minimum total read depth for a locus x sample cell to be +considered present (below this it is missing). Default \code{10}.} + +\item{metadata}{Optional data.frame mapping samples to a grouping category. +The sample-id column is auto-detected (\code{sample}/\code{Sample}/\code{ID}/\code{SampleID}/ +\code{sample_id}/\code{Sample_ID}) or taken as the first column; ids are matched to +the MADC sample column names.} + +\item{group.col}{Name of the category column in \code{metadata} (e.g. species or +population), used to produce the \code{by_category} table. If \code{NULL} (default), +no \code{by_category} table is returned.} + +\item{markers_info}{Optional marker lookup table with an id column +(\code{CloneID}/\code{Marker_ID}/\code{BI_markerID}) plus \code{Chr} and \code{Pos}, used when the +CloneIDs are not in \code{Chr_Pos} format.} + +\item{mhap.cap}{Largest explicit row in the microhaplotype-frequency table; +markers with more mHaps are pooled into a \code{">mhap.cap"} bucket. Markers with +fewer than two mHaps are not included in the table. Default \code{10}.} + +\item{miss.thresholds}{Numeric vector of missingness fractions for the +threshold sweep. Default \code{c(0, 5, 10, 20, ..., 100) / 100}.} + +\item{depth.thresholds}{Numeric vector of read-depth cutoffs for the +\code{depth_sweep} table. Default \code{c(1, 5, 10, 20, 50, 100)}.} + +\item{mhap.min.reads}{Minimum reads for a mHap to count as present in a +sample. Used for the per-sample \code{n_mhaps_observed} (mean mHaps present per +marker) and the per-marker \code{n_mhaps_observed} (mHaps present in at least one +sample). Default \code{1}.} + +\item{min.locus.depth, max.locus.depth}{Marker-QC read-depth window (mean reads +per sample at the locus). A marker below \code{min.locus.depth} is \code{FAIL}ed (low +depth); one above \code{max.locus.depth} is \code{FLAG}ged (over-amplified). Names/units +match \code{\link[=filterMADC]{filterMADC()}}: with \code{target.only = FALSE} the locus depth is computed +over all mHaps, as in \code{filterMADC}, so a \code{FAIL} on \code{min.locus.depth} is the +marker \code{filterMADC} would drop at the same value. With \code{target.only = TRUE} +only the \verb{|Ref}/\verb{|Alt} reads are counted. \code{min.locus.depth} defaults to +\code{min.depth}; \code{max.locus.depth} is off (\code{NULL}) by default. Set either to +\code{NULL} to disable that criterion.} + +\item{max.missing}{Marker-QC maximum per-marker missing rate; markers above it +are \code{FAIL}ed (catches the high-dropout case a mean-depth cut alone misses). +\code{NULL} (default) disables it.} + +\item{max.offtarget}{Marker-QC maximum read-weighted off-target fraction; +markers above it are \code{FLAG}ged. \code{NULL} (default) disables it.} + +\item{max.mhaps.per.loci}{Marker-QC maximum number of defined mHaps; markers +above it are \code{FLAG}ged (paralog-suspect). \code{NULL} (default) disables it.} + +\item{output.file}{If not \code{NULL}, the tables are written here: as +\code{paste0(output.file, "_
.csv")} when \code{output.format = "csv"}, or as a +single \code{.xlsx} workbook (one sheet per table) when \code{output.format = "xlsx"}. +The \code{.xlsx} extension is added if it is not already present.} + +\item{output.format}{One of \code{"csv"} (default; one CSV per table) or \code{"xlsx"} +(a single multi-sheet workbook, requires the \code{writexl} package).} + +\item{verbose}{Logical; print progress/validation messages. Default \code{TRUE}.} +} +\value{ +A named list of data.frames: +\describe{ +\item{\code{mhap_freq}}{Number of markers (\code{n_markers}) with 2, 3, ..., +\code{mhap.cap}, and more than \code{mhap.cap} defined mHaps (\code{n_mhaps}). Markers +with fewer than two allele rows are not counted, so the total can be lower +than the number of markers in the file.} +\item{\code{per_sample}}{One row per sample: mean locus read depth (\code{depth}), +\code{median_depth}, \code{depth_mad}, \code{depth_q10}, \code{depth_q90}, \code{target_depth}, +\code{total_reads}, \code{target_reads}, \code{missing_rate}, \code{call_rate}, +\code{n_mhaps_observed} (mean number of mHaps with at least \code{mhap.min.reads} +reads per marker, over all markers), \code{offtarget_fraction} (mean of the +per-locus ratios), and \code{offtarget_fraction_weighted} +(\code{sum(offtarget) / sum(total)}).} +\item{\code{per_marker}}{One row per marker (\code{CloneID}, \code{Chr}, \code{Pos}): mean +read depth across samples (\code{depth}), \code{median_depth}, \code{depth_mad}, +\code{depth_q10}, \code{depth_q90}, \code{target_depth}, \code{missing_rate}, \code{call_rate}, +\code{n_mhaps_defined} (allele rows for the marker in the MADC), +\code{n_mhaps_observed} (alleles with at least \code{mhap.min.reads} reads in at +least one sample), \code{offtarget_fraction}, and +\code{offtarget_fraction_weighted}. \code{Chr} and \code{Pos} are dropped when they cannot be +obtained from the CloneIDs or \code{markers_info}.} +\item{\code{missingness}}{For each value in \code{miss.thresholds}, the number of loci +and samples removed (missing rate above the threshold) and retained.} +\item{\code{depth_sweep}}{For each value in \code{depth.thresholds}, the fraction of +locus x sample cells at or above it (\code{cells_passing}), the number of loci +passing in at least 90\% and 95\% of samples, and the number of samples +passing in at least 90\% of loci.} +\item{\code{marker_qc}}{Advisory per-marker \code{qc_status} (\code{PASS}, \code{FLAG}, or +\code{FAIL}; \code{FAIL} takes precedence) with a \code{qc_reason} string listing every +criterion that was triggered. Markers are not removed.} +\item{\code{overall}}{A one-row summary: number of samples, markers and mHaps, +mean and median depth, overall missing and call rate, mean and +read-weighted off-target fraction, and the median number of mHaps per +marker.} +\item{\code{by_category}}{Only when \code{metadata}/\code{group.col} are given and at +least one sample matches: per-category number of samples and the means of +the per-sample \code{depth}, \code{target_depth}, \code{missing_rate}, +\code{n_mhaps_observed} (\code{n_mhaps_mean}), and \code{offtarget_fraction}. Samples +not found in \code{metadata} are left out.} +} +} +\description{ +Computes read-only summary tables for a fixed allele ID MADC file (one +processed through HapApp): a per-marker microhaplotype frequency +distribution, per-sample and per-marker metric tables (read depth, missing +rate, number of mHaps, off-target read fraction), missingness and read-depth +threshold sweeps, an advisory per-marker QC table, an overall summary, and - +when a sample metadata table is supplied - the per-sample metrics aggregated +by category. +} +\details{ +The input is validated with \code{\link[=check_madc_sanity]{check_madc_sanity()}}; raw DArT MADC files (or +any file without fixed AlleleIDs) are rejected with an error. The first three +columns are taken as the ID columns (\code{AlleleID}, \code{CloneID}, \code{AlleleSequence}) +and all remaining columns as samples. A locus x sample cell is treated as +missing when its total read depth is below \code{min.depth}. + +The per-sample, per-marker, and by-category tables report both \code{depth} (total +reads at the locus - how well the region amplifies) and \code{target_depth} (reads +on the designed \verb{|Ref}/\verb{|Alt} alleles - how well the target markers work), +together with \code{offtarget_fraction}, the share of reads coming from the +\verb{|RefMatch}/\verb{|AltMatch}/\verb{|Other} alleles (i.e. the additional information +contributed by non-target haplotypes). Per-sample values are averaged over +all markers, including markers that are missing in that sample. + +When \code{target.only = TRUE}, every metric is restricted to the target +\verb{|Ref}/\verb{|Alt} alleles (\verb{|RefMatch}/\verb{|AltMatch}/\verb{|Other} are ignored), so +\code{depth} and \code{target_depth} are the same, the off-target columns are 0, and no +marker has more than two mHaps. +} +\examples{ +madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") +tabs <- madc_summary(madc_file, min.depth = 10) +tabs$mhap_freq +head(tabs$per_sample) + +} +\seealso{ +\code{\link[=madc_plot]{madc_plot()}}, \code{\link[=filterMADC]{filterMADC()}}, \code{\link[=check_madc_sanity]{check_madc_sanity()}} +} diff --git a/man/solve_composition_poly.Rd b/man/solve_composition_poly.Rd deleted file mode 100644 index bd92739..0000000 --- a/man/solve_composition_poly.Rd +++ /dev/null @@ -1,82 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/breedtools_functions.R -\name{solve_composition_poly} -\alias{solve_composition_poly} -\title{Compute Genome-Wide Breed Composition} -\usage{ -solve_composition_poly( - Y, - X, - ped = NULL, - groups = NULL, - mia = FALSE, - sire = FALSE, - dam = FALSE, - ploidy = 2 -) -} -\arguments{ -\item{Y}{numeric matrix of genotypes (columns) from all animals (rows) in population -coded as dosage of allele B \code{{0, 1, 2, ..., ploidy}}} - -\item{X}{numeric matrix of allele frequencies (rows) from each reference panel (columns). Frequencies are -relative to allele B.} - -\item{ped}{data.frame giving pedigree information. Must be formatted "ID", "Sire", "Dam"} - -\item{groups}{list of IDs categorized by breed/population. If specified, output will be a list -of results categorized by breed/population.} - -\item{mia}{logical. Only applies if ped argument is supplied. If true, returns a data.frame -containing the inferred maternally inherited allele for each locus for each animal instead -of breed composition results.} - -\item{sire}{logical. Only applies if ped argument is supplied. If true, returns a data.frame -containing sire genotypes for each locus for each animal instead of breed composition results.} - -\item{dam}{logical. Only applies if ped argument is supplied. If true, returns a data.frame -containing dam genotypes for each locus for each animal instead of breed composition results.} - -\item{ploidy}{integer. The ploidy level of the species (e.g., 2 for diploid, 3 for triploid, etc.).} -} -\value{ -A data.frame or list of data.frames (if groups is !NULL) with breed/ancestry composition -results -} -\description{ -Computes genome-wide breed/ancestry composition using quadratic programming on a -batch of animals. -} -\examples{ -# Example inputs for solve_composition_poly (ploidy = 4) - -# (This would typically be the output from allele_freq_poly) -allele_freqs_matrix <- matrix( - c(0.625, 0.500, - 0.500, 0.500, - 0.500, 0.500, - 0.750, 0.500, - 0.625, 0.625), - nrow = 5, ncol = 2, byrow = TRUE, - dimnames = list(paste0("SNP", 1:5), c("VarA", "VarB")) -) - -# Validation Genotypes (individuals x SNPs) -val_geno_matrix <- matrix( - c(2, 1, 2, 3, 4, # Test1 dosages for SNP1-5 - 3, 4, 2, 3, 0), # Test2 dosages for SNP1-5 - nrow = 2, ncol = 5, byrow = TRUE, - dimnames = list(paste0("Test", 1:2), paste0("SNP", 1:5)) -) - -# Calculate Breed Composition -composition <- solve_composition_poly(Y = val_geno_matrix, - X = allele_freqs_matrix, - ploidy = 4) -print(composition) - -} -\references{ -Funkhouser SA, Bates RO, Ernst CW, Newcom D, Steibel JP. Estimation of genome-wide and locus-specific -breed composition in pigs. Transl Anim Sci. 2017 Feb 1;1(1):36-44. -} diff --git a/man/validate_pedigree.Rd b/man/validate_pedigree.Rd deleted file mode 100644 index 8f47a7d..0000000 --- a/man/validate_pedigree.Rd +++ /dev/null @@ -1,80 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/validate_pedigree.R -\name{validate_pedigree} -\alias{validate_pedigree} -\title{Validate Pedigree Trios Using Mendelian Error Analysis} -\usage{ -validate_pedigree( - pedigree_file, - genotypes_file, - founders_file = NULL, - trio_error_threshold = 5, - min_markers = 10, - single_parent_error_threshold = 2, - verbose = TRUE, - write_txt = TRUE, - output_filename = "pedigree_validation_results.txt" -) -} -\arguments{ -\item{pedigree_file}{Character. Path to the pedigree file (TSV/CSV/TXT) -with columns: ID, Male_Parent, Female_Parent.} - -\item{genotypes_file}{Character. Path to the genotypes file (TSV/CSV/TXT) -with an ID column followed by marker columns coded as 0, 1, 2.} - -\item{founders_file}{Character, optional. Path to a one-column file -listing the IDs of founder individuals. Founders with both parents -coded as 0 are left unchanged with no recommendations. Defaults to NULL.} - -\item{trio_error_threshold}{Numeric. Maximum Mendelian error percentage -to classify a trio as PASS (default: 5.0). Must be between 0 and 100.} - -\item{min_markers}{Integer. Minimum number of non-missing markers -required to evaluate a trio (default: 10).} - -\item{single_parent_error_threshold}{Numeric. Maximum homozygous-marker -mismatch percentage for a parent to be considered acceptable during -parent-level evaluation (default: 2.0). Must be between 0 and 100.} - -\item{verbose}{Logical. If TRUE, prints progress messages, a summary -table, and results to the console (default: TRUE).} - -\item{write_txt}{Logical. If TRUE, writes validation results to -output_filename (default: TRUE).} - -\item{output_filename}{Character. Path/name of the output file -(default: "pedigree_validation_results.txt").} -} -\value{ -A data.table (returned invisibly) with one row per trio and -the following columns: -\describe{ -\item{ID}{Individual ID.} -\item{Male_Parent}{Declared male parent ID.} -\item{Female_Parent}{Declared female parent ID.} -\item{Mendelian_Error_Pct}{Trio-level Mendelian error percentage.} -\item{Markers_Tested}{Number of markers with non-missing genotypes.} -\item{Status}{One of PASS, FAIL, LOW_MARKERS, NO_DATA, FOUNDERS, -MISSING_MALE_PARENT, MISSING_FEMALE_PARENT, MISSING_BOTH_PARENTS, -or NO_GENOTYPE_DATA.} -\item{Correction_Decision}{One of NONE, KEEP_BOTH, -REMOVE_MALE_PARENT, REMOVE_FEMALE_PARENT, REMOVE_BOTH, -LOW_MARKERS_KEEP_BOTH, LOW_MARKERS_REMOVE_MALE_PARENT, -LOW_MARKERS_REMOVE_FEMALE_PARENT, LOW_MARKERS_REMOVE_BOTH.} -\item{Male_Parent_Hom_Error_Pct}{Male parent homozygous-marker mismatch percentage.} -\item{Female_Parent_Hom_Error_Pct}{Female parent homozygous-marker mismatch percentage.} -\item{Best_Male_Parent}{Best-matching male parent candidate ID.} -\item{Best_Male_Parent_Error_Pct}{Homozygous mismatch percentage for the best male parent candidate.} -\item{Best_Female_Parent}{Best-matching female parent candidate ID.} -\item{Best_Female_Parent_Error_Pct}{Homozygous mismatch percentage for the best female parent candidate.} -} -} -\description{ -Validates parent-offspring trios by calculating Mendelian error rates from -SNP genotype data. Identifies incorrect parentage assignments and suggests -best-matching replacements. If a list of founders is supplied, trios that -are declared founders (both parents coded as 0) are preserved unchanged -with no recommendations. Trios removed due to missing genotype data are -retained in the output with a NO_GENOTYPE_DATA status. -} diff --git a/tests/testthat/corrected_pedigree.txt b/tests/testthat/corrected_pedigree.txt deleted file mode 100644 index 0434b0c..0000000 --- a/tests/testthat/corrected_pedigree.txt +++ /dev/null @@ -1,4 +0,0 @@ -ID Male_Parent Female_Parent -IND_C IND_A IND_B -IND_D 0 IND_A -GHOST IND_A IND_B diff --git a/tests/testthat/test-breedtools_poly.R b/tests/testthat/test-breedtools_poly.R deleted file mode 100644 index 91ed356..0000000 --- a/tests/testthat/test-breedtools_poly.R +++ /dev/null @@ -1,31 +0,0 @@ -context("BreedTools") - - -test_that("test breedtools poly",{ - #Input variables - ref_file <- system.file("test_ref.txt", package="BIGr") - val_file <- system.file("test_test.txt", package="BIGr") - ref_ids <- system.file("ref_ids.txt", package="BIGr") - - #import files - reference = read.table(ref_file, header = T, row.names = 1, sep = "\t") - validation = read.table(val_file, header = T, row.names = 1, sep = "\t") - reference_ids = read.table(ref_ids, header = T, sep = "\t") - - #Calculations - ref_ids = lapply(as.list(reference_ids),as.character) - - freq = allele_freq_poly(reference, ref_ids, ploidy = 4) - - prediction = as.data.frame(solve_composition_poly(validation,freq, ploidy = 4)) - - #Check - freq_mean <- round(mean(as.numeric(freq)),6) - pred_mean <- round(mean(as.numeric(prediction$R2)),6) - - - expect_equal(freq_mean, 0.888889, tolerance = 0.01) - expect_equal(pred_mean, 0.841454, tolerance = 0.01) - expect_true(nrow(prediction) == 175) - -}) diff --git a/tests/testthat/test-check_madc_sanity.R b/tests/testthat/test-check_madc_sanity.R index 4b60cfe..ab3f07b 100644 --- a/tests/testthat/test-check_madc_sanity.R +++ b/tests/testthat/test-check_madc_sanity.R @@ -2,13 +2,15 @@ test_that("check madc",{ skip_if_offline("raw.githubusercontent.com") github_path <- "https://raw.githubusercontent.com/Breeding-Insight/BIGapp-PanelHub/refs/heads/long_seq/test_madcs/" - names <- c("Columns", "FixAlleleIDs", "IUPACcodes", "LowerCase", "Indels", "ChromPos", "allNAcol", "allNArow", "RefAltSeqs", "OtherAlleles") + names <- c("Columns", "FixAlleleIDs", "IUPACcodes", "IUPACcodes_IdenticalRefAlt", + "LowerCase", "Indels", "ChromPos", "allNAcol", "allNArow", + "RefAltSeqs", "OtherAlleles", "IUPACcodes_MatchAlleles") # raw madc report <- read.csv(paste0(github_path,"/alfalfa_madc_raw.csv")) res <- check_madc_sanity(report) - exp <- c(TRUE, FALSE, FALSE, FALSE, FALSE, TRUE, TRUE, FALSE, TRUE, FALSE) + exp <- c(TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, TRUE, TRUE, FALSE, TRUE, FALSE, FALSE) names(exp) <- names expect_equal(res$checks, exp) @@ -16,7 +18,7 @@ test_that("check madc",{ report <- read.csv(paste0(github_path,"/alfalfa_lowercase.csv")) res <- check_madc_sanity(report) - exp <- c(TRUE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE) + exp <- c(TRUE, TRUE, FALSE, FALSE, TRUE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE) names(exp) <- names expect_equal(res$checks, exp) @@ -24,7 +26,7 @@ test_that("check madc",{ report <- read.csv(paste0(github_path,"/alfalfa_IUPAC.csv")) res <- check_madc_sanity(report) - exp <- c(TRUE, TRUE, TRUE, FALSE, FALSE, TRUE, FALSE, FALSE,TRUE, FALSE) + exp <- c(TRUE, TRUE, TRUE, FALSE,FALSE, FALSE, TRUE, FALSE, FALSE,TRUE, FALSE,TRUE) names(exp) <- names expect_equal(res$checks, exp) @@ -32,7 +34,7 @@ test_that("check madc",{ report <- read.csv(paste0(github_path,"/alfalfa_madc.csv")) res <- check_madc_sanity(report) - exp <- c(TRUE, TRUE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE,TRUE, FALSE) + exp <- c(TRUE, TRUE, FALSE, FALSE,FALSE, FALSE, TRUE, FALSE, FALSE,TRUE, FALSE, FALSE) names(exp) <- names expect_equal(res$checks, exp) @@ -40,7 +42,7 @@ test_that("check madc",{ report <- read.csv(paste0(github_path,"/potato_indel_madc.csv")) res <- check_madc_sanity(report) - exp <- c(TRUE, TRUE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE,TRUE, FALSE) + exp <- c(TRUE, TRUE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE,TRUE, FALSE, FALSE) names(exp) <- names expect_equal(res$checks, exp) @@ -48,7 +50,7 @@ test_that("check madc",{ report <- read.csv(paste0(github_path,"/potato_indel_IUPAC.csv")) res <- check_madc_sanity(report) - exp <- c(TRUE, TRUE, TRUE, FALSE, TRUE, FALSE, FALSE, FALSE,TRUE, FALSE) + exp <- c(TRUE, TRUE, TRUE, FALSE,FALSE, TRUE, FALSE, FALSE, FALSE,TRUE, FALSE,TRUE) names(exp) <- names expect_equal(res$checks, exp) @@ -56,7 +58,7 @@ test_that("check madc",{ report <- read.csv(paste0(github_path,"/potato_indel_lowercase.csv")) res <- check_madc_sanity(report) - exp <- c(TRUE, TRUE, FALSE, TRUE, TRUE, FALSE, FALSE, FALSE,TRUE, FALSE) + exp <- c(TRUE, TRUE, FALSE, FALSE, TRUE, TRUE, FALSE, FALSE, FALSE,TRUE, FALSE, FALSE) names(exp) <- names expect_equal(res$checks, exp) @@ -64,7 +66,7 @@ test_that("check madc",{ report <- read.csv(paste0(github_path,"/potato_more_indels_madc_ChromPosFALSE.csv")) res <- check_madc_sanity(report) - exp <- c(TRUE, TRUE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE,TRUE, FALSE) + exp <- c(TRUE, TRUE, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE,TRUE, FALSE, FALSE) names(exp) <- names expect_equal(res$checks, exp) }) @@ -167,3 +169,27 @@ test_that("check_botloci keeps AlleleID synchronized after CloneID remap", { expect_equal(res[[2]]$CloneID, "1_0001") expect_equal(res[[2]]$AlleleID, "1_0001|Ref_0001") }) + +test_that("identical IUPAC codes in Ref and Alt are reported separately", { + + clone <- "Chr01_000000100" + ref <- "ACGTTGCAAGCTTGACCTGAAGTCCATGGATCCGTACGATCGGATNCAAGTTCGACTGAT" + alt <- "ACGTTGCAAGCTTGACCTGAAGTCCATGGGTCCGTACGATCGGATNCAAGTTCGACTGAT" + report <- data.frame(AlleleID = paste0(clone, c("|Ref_0001", "|Alt_0002")), + CloneID = clone, + AlleleSequence = c(ref, alt), + S1 = c(10, 5), S2 = c(3, 8)) + + # Same IUPAC code at the same position in Ref and Alt + res <- check_madc_sanity(report) + expect_false(res$checks[["IUPACcodes"]]) + expect_true(res$checks[["IUPACcodes_IdenticalRefAlt"]]) + expect_equal(res$iupac_identical_clone_ids, clone) + + # IUPAC code present in only one of them + report$AlleleSequence[2] <- sub("N", "C", alt) + res <- check_madc_sanity(report) + expect_true(res$checks[["IUPACcodes"]]) + expect_false(res$checks[["IUPACcodes_IdenticalRefAlt"]]) + expect_equal(res$iupac_identical_clone_ids, character(0)) +}) diff --git a/tests/testthat/test-check_ped.R b/tests/testthat/test-check_ped.R deleted file mode 100644 index 887eb75..0000000 --- a/tests/testthat/test-check_ped.R +++ /dev/null @@ -1,30 +0,0 @@ -context("Imputation Concordance") - - -test_that("test imputation",{ - #Input variables - ped_file <- system.file("check_ped_test.txt", package="BIGr") - temp_ped <- tempfile("check_ped_test_", fileext = ".txt") - file.copy(ped_file, temp_ped) - - #Calculations - output.list <- check_ped(temp_ped, - seed = 101919, - verbose = FALSE) - - #Check - df_length <- length(output.list) - messy_parents <- output.list$messy_parents - missing_parents <- output.list$missing_parents - corrected_pedigree <- output.list$corrected_pedigree - file_base <- tools::file_path_sans_ext(basename(temp_ped)) - - expect_true(df_length == 6) - expect_true(all(messy_parents$id == c("grandfather2","grandfather3"))) - expect_true(nrow(missing_parents) == 13) - expect_s3_class(corrected_pedigree, "data.frame") - expect_true(all(missing_parents$id %in% corrected_pedigree$id)) - expect_false(exists(paste0(file_base, "_corrected"), envir = .GlobalEnv, inherits = FALSE)) - expect_false(exists(paste0(file_base, "_report"), envir = .GlobalEnv, inherits = FALSE)) - -}) diff --git a/tests/testthat/test-filterMADC.R b/tests/testthat/test-filterMADC.R index b7cf687..b7dea34 100644 --- a/tests/testthat/test-filterMADC.R +++ b/tests/testthat/test-filterMADC.R @@ -8,16 +8,20 @@ test_that("test filter madc",{ #Calculations temp <- tempfile() + #Reference values from the raw input, used to check filter invariants + raw_in <- read.csv(madc_file, check.names = FALSE) + in_counts <- table(raw_in$CloneID) + n_refalt_in <- sum(grepl("\\|(Ref|Alt)_", raw_in$AlleleID)) + # Filtering (target only) - filtered_df <- filterMADC(madc_file, - min.mean.reads = NULL, - max.mean.reads = NULL, + filtered_df <- suppressMessages(filterMADC(madc_file, + min.locus.depth = NULL, + max.locus.depth = NULL, max.mhaps.per.loci = NULL, min.reads.per.site = 1, min.ind.with.reads = NULL, target.only = TRUE, - n.summary.columns = NULL, - output.file = NULL) + output.file = NULL)) #Test that a valid output was provided @@ -26,119 +30,133 @@ test_that("test filter madc",{ expect_true(is.data.frame(filtered_df)) # Checking for no filtering - filtered_df <- filterMADC(madc_file, - min.mean.reads = NULL, - max.mean.reads = NULL, + filtered_df <- suppressMessages(filterMADC(madc_file, + min.locus.depth = NULL, + max.locus.depth = NULL, max.mhaps.per.loci = NULL, min.reads.per.site = 1, min.ind.with.reads = NULL, target.only = FALSE, - n.summary.columns = NULL, - output.file = NULL) + output.file = NULL)) expect_equal(nrow(filtered_df), 51) expect_equal(sum(filtered_df[,-c(1:3)]), 53952) expect_true(all(names(filtered_df[1:3]) == c("AlleleID", "CloneID", "AlleleSequence"))) - #Checking for min.mean.reads filtering - filtered_df <- filterMADC(madc_file, - min.mean.reads = 10, - max.mean.reads = NULL, + #Checking for min.locus.depth filtering (drops whole low-depth loci) + filtered_df <- suppressMessages(filterMADC(madc_file, + min.locus.depth = 50, + max.locus.depth = NULL, max.mhaps.per.loci = NULL, min.reads.per.site = 1, min.ind.with.reads = NULL, target.only = FALSE, - n.summary.columns = NULL, - output.file = NULL) + output.file = NULL)) - expect_equal(nrow(filtered_df), 36) + expect_equal(nrow(filtered_df), 37) expect_equal(ncol(filtered_df), 13) - - #Checking for max.mean.reads filtering - filtered_df <- filterMADC(madc_file, - min.mean.reads = NULL, - max.mean.reads = 10, + #Whole-locus removal: every retained locus keeps ALL of its original mhap rows (no un-pairing) + out_counts <- table(filtered_df$CloneID) + expect_true(all(out_counts == in_counts[names(out_counts)])) + + #Checking for max.locus.depth filtering (drops whole high-depth loci) + filtered_df <- suppressMessages(filterMADC(madc_file, + min.locus.depth = NULL, + max.locus.depth = 50, max.mhaps.per.loci = NULL, min.reads.per.site = 1, min.ind.with.reads = NULL, target.only = FALSE, - n.summary.columns = NULL, - output.file = NULL) + output.file = NULL)) - expect_equal(nrow(filtered_df), 15) + expect_equal(nrow(filtered_df), 14) expect_equal(ncol(filtered_df), 13) + expect_equal(sum(filtered_df[,-c(1:3)]), 1122) #Remove max mhaps - filtered_df <- filterMADC(madc_file, - min.mean.reads = NULL, - max.mean.reads = NULL, + filtered_df <- suppressMessages(filterMADC(madc_file, + min.locus.depth = NULL, + max.locus.depth = NULL, max.mhaps.per.loci = 3, min.reads.per.site = 1, min.ind.with.reads = NULL, target.only = FALSE, - n.summary.columns = NULL, - output.file = NULL) + output.file = NULL)) expect_equal(nrow(filtered_df), 44) expect_equal(ncol(filtered_df), 13) - #Remove min ind with reads - filtered_df <- filterMADC(madc_file, - min.mean.reads = NULL, - max.mean.reads = NULL, + #Presence filter (min ind with reads) - target Ref/Alt are protected + filtered_df <- suppressMessages(filterMADC(madc_file, + min.locus.depth = NULL, + max.locus.depth = NULL, max.mhaps.per.loci = NULL, min.reads.per.site = 10, min.ind.with.reads = 10, target.only = FALSE, - n.summary.columns = NULL, - output.file = NULL) + output.file = NULL)) - expect_equal(nrow(filtered_df), 9) + expect_equal(nrow(filtered_df), 41) expect_equal(ncol(filtered_df), 13) - expect_equal(sum(filtered_df[,-c(1:3)]), 31642) + expect_equal(sum(filtered_df[,-c(1:3)]), 47985) + #Every input target Ref/Alt row is retained (only non-target mhaps can be pruned) + expect_equal(sum(grepl("\\|(Ref|Alt)_", filtered_df$AlleleID)), n_refalt_in) #Check that the output file is created - filterMADC(madc_file, - min.mean.reads = NULL, - max.mean.reads = NULL, + suppressMessages(filterMADC(madc_file, + min.locus.depth = NULL, + max.locus.depth = NULL, max.mhaps.per.loci = NULL, min.reads.per.site = 1, min.ind.with.reads = NULL, target.only = FALSE, - n.summary.columns = NULL, - output.file = temp) + output.file = temp)) expect_true(file.exists(paste0(temp,".csv"))) - #Check that the plots are created in the console - #filtered_df <- filterMADC(madc_file, - # min.mean.reads = NULL, - # max.mean.reads = NULL, - # max.mhaps.per.loci = 3, - # min.reads.per.site = 1, - # min.ind.with.reads = NULL, - # target.only = FALSE, - # n.summary.columns = NULL, - # plot.summary = TRUE, - # output.file = NULL) - - #expect_true(is.numeric(dev.cur())) - #expect_true(dev.cur() > 1) - #Now checking that all paramaters can work together - filtered_df <- filterMADC(madc_file, - min.mean.reads =1, - max.mean.reads = 150, + filtered_df <- suppressMessages(filterMADC(madc_file, + min.locus.depth = 10, + max.locus.depth = 300, max.mhaps.per.loci = 3, min.reads.per.site = 10, - min.ind.with.reads = 10, + min.ind.with.reads = 5, target.only = FALSE, - n.summary.columns = NULL, - output.file = NULL) + output.file = NULL)) - expect_equal(nrow(filtered_df), 3) + expect_equal(nrow(filtered_df), 29) expect_equal(ncol(filtered_df), 13) - expect_equal(sum(filtered_df[,-c(1:3)]), 3960) - + expect_equal(sum(filtered_df[,-c(1:3)]), 15375) + + + #Reject a raw (non fixed allele ID) MADC file + raw_file <- system.file("iris_DArT_MADC.csv", package="BIGr") + expect_error(suppressMessages(filterMADC(raw_file)), "HapApp") + + + #Fixed allele ID fixture containing |Other alleles to exercise the new + #Ref/Alt-only retention in max.mhaps.per.loci and target.only + tmp_other <- tempfile(fileext = ".csv") + other_df <- data.frame( + AlleleID = c("Chr1_001|Ref_0001", "Chr1_001|Alt_0002", + "Chr1_001|RefMatch_0001", "Chr1_001|AltMatch_0001", + "Chr1_001|Other_0001"), + CloneID = "Chr1_001", + AlleleSequence = c("ACGT", "ACGA", "ACGT", "ACGA", "TTTT"), + Sample_1 = c(10, 8, 5, 4, 3), + Sample_2 = c(12, 9, 6, 5, 2), + check.names = FALSE + ) + write.csv(other_df, tmp_other, row.names = FALSE) + + # max.mhaps.per.loci: 5 mhaps at the locus > 3 -> keep only |Ref/|Alt (drop Match AND Other) + out_mhap <- suppressMessages(filterMADC(tmp_other, max.mhaps.per.loci = 3)) + expect_equal(nrow(out_mhap), 2) + expect_true(all(grepl("\\|(Ref|Alt)_", out_mhap$AlleleID))) + + # target.only: retains strictly |Ref/|Alt, dropping Match AND Other + out_target <- suppressMessages(filterMADC(tmp_other, target.only = TRUE)) + expect_equal(nrow(out_target), 2) + expect_false(any(grepl("\\|Other", out_target$AlleleID))) }) diff --git a/tests/testthat/test-filterVCF.R b/tests/testthat/test-filterVCF.R index 3cc8ecb..e777efb 100644 --- a/tests/testthat/test-filterVCF.R +++ b/tests/testthat/test-filterVCF.R @@ -61,7 +61,10 @@ test_that("Filtering with Updog metrics",{ filtering_files$snp_miss_df <- rowMeans(is.na(gt_matrix)) #SNP missing values filtering_files$sample_miss_df <- as.numeric(colMeans(is.na(gt_matrix))) #Sample missing values - expect_true(all(table(gt_matrix[,10]) == c(20,13,8))) + #These counts increased when the INFO parser was fixed to read scientific + #notation (see the final_snps note below) - more SNPs are retained, so each + #sample has more genotype calls. + expect_true(all(table(gt_matrix[,10]) == c(37,36,18))) rm(gt_matrix) #Remove gt matrix @@ -69,7 +72,177 @@ test_that("Filtering with Updog metrics",{ write.vcf(vcf, file = temp_file) #Get final_snps + #Was 43 while filterVCF()'s INFO parser could not read scientific notation: + #110 of the 499 SNPs in the test file have a PMC written as e.g. 4.78e-07, + #which parsed as 4.78 and so failed filter.PMC = 0.05 despite being well + #under it. Those SNPs are now retained, hence 93 rather than 43. final_snps <- nrow(vcf) - expect_equal(final_snps, 43) + expect_equal(final_snps, 93) + +}) + +test_that("INFO filters read scientific notation",{ + + #updog2vcf() writes INFO values with paste0(), so R's default formatting emits + #scientific notation for small values. filterVCF() must read those as written. + vcf <- read.vcfR(system.file("iris_DArT_VCF.vcf.gz", package = "BIGr"), verbose = FALSE) + vcf <- vcf[1:4, ] + + #Two SNPs written in fixed notation, two in scientific notation. All four have + #an OD and a PMC that pass the filters below. + vcf@fix[,"INFO"] <- c("DP=100;ADS=60,40;BIAS=1.0;OD=0.001;PMC=0.001", + "DP=100;ADS=60,40;BIAS=1.0;OD=1.2e-05;PMC=4.78e-07", + "DP=100;ADS=60,40;BIAS=1.0;OD=0.002;PMC=0.002", + "DP=100;ADS=60,40;BIAS=1.0;OD=3.4e-08;PMC=9.10e-09") + + expect_equal(nrow(filterVCF(vcf, filter.PMC = 0.05, ploidy = 2)), 4) + expect_equal(nrow(filterVCF(vcf, filter.OD = 0.05, ploidy = 2)), 4) + + #A value that genuinely exceeds the threshold is still removed, so the parser + #is reading the exponent rather than ignoring it. + vcf@fix[2,"INFO"] <- "DP=100;ADS=60,40;BIAS=1.0;OD=1.2e+01;PMC=5.0e-01" + expect_equal(nrow(filterVCF(vcf, filter.PMC = 0.05, ploidy = 2)), 3) + expect_equal(nrow(filterVCF(vcf, filter.OD = 0.05, ploidy = 2)), 3) + + #The field name is matched against a whole INFO entry, not as a substring, so + #a longer field ending in the same name is not picked up by mistake. The + #look-alike is placed after the real OD on purpose: the pattern this replaced + #searched greedily and so took the rightmost match, meaning a look-alike placed + #before the real field would be read correctly by accident. + vcf@fix[,"INFO"] <- sub("(OD=[^;]*)", "\\1;MOD=77", vcf@fix[,"INFO"]) + expect_equal(nrow(filterVCF(vcf, filter.OD = 0.05, ploidy = 2)), 3) + +}) + +test_that("Variants with unreadable filter values are removed, not inserted as NA",{ + + #An NA in a logical index does not drop a row, it inserts an all-NA row, so a + #variant whose filter value cannot be read must be removed explicitly. + vcf <- read.vcfR(system.file("iris_DArT_VCF.vcf.gz", package = "BIGr"), verbose = FALSE) + vcf <- vcf[1:4, ] + starting_snps <- nrow(vcf) + + #Record 3 carries no PMC at all. + vcf@fix[,"INFO"] <- "DP=100;ADS=60,40;BIAS=1.0;OD=0.001;PMC=0.001" + vcf@fix[3,"INFO"] <- "DP=100;ADS=60,40;BIAS=1.0;OD=0.001" + + expect_warning(filtered <- filterVCF(vcf, filter.PMC = 0.05, ploidy = 2), + "1 of 4 variants had a missing or unreadable PMC value") + expect_equal(nrow(filtered), starting_snps - 1) + expect_false(any(is.na(filtered@fix[,"CHROM"]))) + expect_false(any(is.na(filtered@fix[,"POS"]))) + + #The same applies to MAF, which vcfR::maf() reports as NA for a variant left + #with no called genotypes - something the DP and MPP masking can produce. Only + #the variants without calls should go, not the whole file. + vcf@fix[,"INFO"] <- "DP=100;ADS=60,40;BIAS=1.0;OD=0.001;PMC=0.001" + vcf@gt[2:3, -1] <- "./.:.:.:.:.:." + expect_warning(masked <- filterVCF(vcf, filter.MAF = 0.05, ploidy = 2), + "2 of 4 variants had a missing or unreadable MAF value") + expect_lte(nrow(masked), 2) + expect_false(any(is.na(masked@fix[,"CHROM"]))) + + #A file with nothing unreadable must not warn (progress messages are expected). + expect_no_warning(suppressMessages(filterVCF(vcf, filter.PMC = 0.05, ploidy = 2))) + +}) + +test_that("A requested INFO filter runs whatever the first record contains",{ + + #Which INFO fields exist used to be read from the first record alone, so a + #field missing from that one record disabled its filter for the whole file, + #silently. A field may legally be absent from any individual record. + vcf <- read.vcfR(system.file("iris_DArT_VCF.vcf.gz", package = "BIGr"), verbose = FALSE) + vcf <- vcf[1:4, ] + vcf@fix[,"INFO"] <- "DP=100;ADS=60,40;BIAS=1.0;OD=0.99;PMC=0.001" + vcf@fix[1,"INFO"] <- "DP=100;ADS=60,40;BIAS=1.0;PMC=0.001" + + #Every record either fails OD < 0.05 or has no OD at all, so none survive. + expect_warning(filtered <- filterVCF(vcf, filter.OD = 0.05, ploidy = 2), + "1 of 4 variants had a missing or unreadable OD value") + expect_equal(nrow(filtered), 0) + + #Reordering the records must not change the result. + reordered <- vcf + reordered@fix[,"INFO"] <- reordered@fix[c(2,3,4,1),"INFO"] + expect_warning(also <- filterVCF(reordered, filter.OD = 0.05, ploidy = 2), + "1 of 4 variants had a missing or unreadable OD value") + expect_equal(nrow(also), nrow(filtered)) + + #A filter that was never requested stays inactive. + expect_no_warning(suppressMessages(filterVCF(vcf, filter.MAF = 0.05, ploidy = 2))) + +}) + +test_that("A filter whose values are all unreadable removes every variant",{ + + #Every filter behaves the same way here. Previously OD, BIAS and PMC returned + #the data unfiltered in this situation, which reported success while applying + #no filter at all. + vcf <- read.vcfR(system.file("iris_DArT_VCF.vcf.gz", package = "BIGr"), verbose = FALSE) + vcf <- vcf[1:4, ] + + vcf@fix[,"INFO"] <- "DP=100;ADS=60,40;BIAS=1.0;OD=.;PMC=0.001" + expect_warning(dropped <- filterVCF(vcf, filter.OD = 0.05, ploidy = 2), + "No readable OD values were found") + expect_equal(nrow(dropped), 0) + + #The warning names the INFO column, so the cause is identifiable. + expect_warning(filterVCF(vcf, filter.OD = 0.05, ploidy = 2), + "INFO column") + + #MAF reports the cause that actually applies to it, rather than an INFO field. + vcf@fix[,"INFO"] <- "DP=100;ADS=60,40;BIAS=1.0;OD=0.001;PMC=0.001" + expect_warning(masked <- filterVCF(vcf, filter.DP = 1e6, filter.MAF = 0.05, ploidy = 2), + "no variant has any called genotypes") + expect_equal(nrow(masked), 0) + +}) + +test_that("Filter thresholds must be a single finite number or NULL",{ + + vcf <- read.vcfR(system.file("iris_DArT_VCF.vcf.gz", package = "BIGr"), verbose = FALSE) + + expect_error(filterVCF(vcf, filter.OD = NA, ploidy = 2), "single numeric value") + expect_error(filterVCF(vcf, filter.OD = c(0.5, 0.6), ploidy = 2), "single numeric value") + expect_error(filterVCF(vcf, filter.SNP.miss = numeric(0), ploidy = 2), "single numeric value") + expect_error(filterVCF(vcf, filter.MAF = "abc", ploidy = 2), "single finite numeric value") + expect_error(filterVCF(vcf, filter.SAMPLE.miss = NA_real_, ploidy = 2), "single finite numeric value") + +}) + +test_that("Missing data thresholds must be proportions",{ + + vcf <- read.vcfR(system.file("iris_DArT_VCF.vcf.gz", package = "BIGr"), verbose = FALSE) + + #A percentage such as 20 would otherwise keep every sample or variant. + expect_error(filterVCF(vcf, filter.SAMPLE.miss = 20, ploidy = 2), "between 0 and 1") + expect_error(filterVCF(vcf, filter.SNP.miss = 20, ploidy = 2), "between 0 and 1") + expect_error(filterVCF(vcf, filter.SNP.miss = -0.1, ploidy = 2), "between 0 and 1") + +}) + +test_that("DP and MPP filters run whatever the first record's FORMAT contains",{ + + #The FORMAT fields used to be read from the first record alone, so a DP field + #missing from that one record disabled filter.DP for the whole file. + vcf <- read.vcfR(system.file("iris_DArT_VCF.vcf.gz", package = "BIGr"), verbose = FALSE) + vcf <- vcf[1:4, ] + + #Remove DP from the first record only. + vcf@gt[1, "FORMAT"] <- "GT:UD:RA:AD:MPP" + vcf@gt[1, -1] <- sub("^([^:]*:[^:]*):[^:]*", "\\1", vcf@gt[1, -1]) + + filtered <- suppressMessages(filterVCF(vcf, filter.DP = 1e6, ploidy = 2)) + gt <- extract.gt(filtered, element = "GT") + + #Every call with a DP value fails the threshold and is set to missing, while the + #first record has no DP value and keeps its calls. + expect_true(all(is.na(gt[2:4, ]))) + expect_false(any(is.na(gt[1, ]))) + + #The missing call matches the FORMAT of its own record. + expect_true(all(filtered@gt[2:4, -1] == "./.:.:.:.:.:.")) + expect_equal(filtered@gt[1, -1], vcf@gt[1, -1]) }) diff --git a/tests/testthat/test-find_parentage.R b/tests/testthat/test-find_parentage.R deleted file mode 100644 index 55e6913..0000000 --- a/tests/testthat/test-find_parentage.R +++ /dev/null @@ -1,435 +0,0 @@ -# tests/testthat/test-find_parentage.R -# Run with: testthat::test_file("tests/testthat/test-find_parentage.R") -library(testthat) -library(data.table) - -# ============================================================================== -# Helpers -# ============================================================================== - -make_files <- function(genos, parents, progeny, dir = tempdir()) { - geno_file <- file.path(dir, paste0("genos_", sample(1e6,1), ".txt")) - parent_file <- file.path(dir, paste0("parents_", sample(1e6,1), ".txt")) - progeny_file <- file.path(dir, paste0("progeny_", sample(1e6,1), ".txt")) - data.table::fwrite(genos, geno_file, sep = "\t") - data.table::fwrite(parents, parent_file, sep = "\t") - data.table::fwrite(progeny, progeny_file, sep = "\t") - list(g = geno_file, p = parent_file, pr = progeny_file) -} - -# ------------------------------------------------------------------------------ -# Base toy data -# S1 / D1: all 0 → child1 (all 0) is a perfect Mendelian child of S1 x D1 -# S2 / D2: all 2 → child2 (all 2) is a perfect Mendelian child of S2 x D2 -# ------------------------------------------------------------------------------ -base_genos <- data.table::data.table( - ID = c("S1","S2","D1","D2","child1","child2"), - M1 = c(0L, 2L, 0L, 2L, 0L, 2L), - M2 = c(0L, 2L, 0L, 2L, 0L, 2L), - M3 = c(0L, 2L, 0L, 2L, 0L, 2L), - M4 = c(0L, 2L, 0L, 2L, 0L, 2L), - M5 = c(0L, 2L, 0L, 2L, 0L, 2L) -) -base_parents <- data.table::data.table(ID = c("S1","S2","D1","D2"), - Sex = c("M","M","F","F")) -child1_progeny <- data.table::data.table(ID = "child1") -child2_progeny <- data.table::data.table(ID = "child2") -base_progeny <- data.table::data.table(ID = c("child1","child2")) - -# All-zero genotypes — every pair ties at 0% error -tied_genos <- data.table::data.table( - ID = c("S1","S2","D1","D2","child_tie"), - M1 = c(0L, 0L, 0L, 0L, 0L), - M2 = c(0L, 0L, 0L, 0L, 0L) -) -tied_parents <- data.table::data.table(ID = c("S1","S2","D1","D2"), - Sex = c("M","M","F","F")) -tied_progeny <- data.table::data.table(ID = "child_tie") - -# ============================================================================== -# 1. Input validation -# ============================================================================== - -test_that("invalid method throws an error", { - f <- make_files(base_genos, base_parents, child1_progeny) - expect_error( - find_parentage(f$g, f$p, f$pr, method = "bad_method", - verbose = FALSE, write_txt = FALSE), - regexp = "Method must be one of" - ) -}) - -test_that("min_markers < 1 throws an error", { - f <- make_files(base_genos, base_parents, child1_progeny) - expect_error( - find_parentage(f$g, f$p, f$pr, min_markers = 0, - verbose = FALSE, write_txt = FALSE), - regexp = "min_markers" - ) -}) - -test_that("error_threshold out of range throws an error", { - f <- make_files(base_genos, base_parents, child1_progeny) - expect_error( - find_parentage(f$g, f$p, f$pr, error_threshold = 150, - verbose = FALSE, write_txt = FALSE), - regexp = "error_threshold" - ) - expect_error( - find_parentage(f$g, f$p, f$pr, error_threshold = -1, - verbose = FALSE, write_txt = FALSE), - regexp = "error_threshold" - ) -}) - -test_that("missing genotype file throws an error", { - f <- make_files(base_genos, base_parents, child1_progeny) - expect_error( - find_parentage("nonexistent.txt", f$p, f$pr, - verbose = FALSE, write_txt = FALSE) - ) -}) - -test_that("parent IDs absent from genotype file raise a warning and are dropped", { - extra_parents <- rbind(base_parents, - data.table::data.table(ID = "GHOST", Sex = "M")) - f <- make_files(base_genos, extra_parents, child1_progeny) - expect_warning( - find_parentage(f$g, f$p, f$pr, method = "best_pair", - verbose = FALSE, write_txt = FALSE), - regexp = "GHOST" - ) -}) - -test_that("progeny IDs absent from genotype file raise a warning and are dropped", { - extra_progeny <- rbind(child1_progeny, - data.table::data.table(ID = "GHOST_KID")) - f <- make_files(base_genos, base_parents, extra_progeny) - expect_warning( - find_parentage(f$g, f$p, f$pr, method = "best_pair", - verbose = FALSE, write_txt = FALSE), - regexp = "GHOST_KID" - ) -}) - -test_that("no valid progeny candidates after filtering stops with an error", { - ghost_progeny <- data.table::data.table(ID = "NOBODY") - f <- make_files(base_genos, base_parents, ghost_progeny) - expect_warning( - expect_error( - find_parentage(f$g, f$p, f$pr, method = "best_pair", - verbose = FALSE, write_txt = FALSE), - regexp = "No valid progeny" - ) - ) -}) - -test_that("missing Sex column raises a warning and defaults to ambiguous", { - parents_no_sex <- data.table::data.table(ID = c("S1","D1")) - f <- make_files(base_genos, parents_no_sex, child1_progeny) - expect_warning( - find_parentage(f$g, f$p, f$pr, method = "best_match", - verbose = FALSE, write_txt = FALSE), - regexp = "Sex" - ) -}) - -# ============================================================================== -# 2. Return structure -# ============================================================================== - -test_that("best_pair returns a data.table with expected columns (no ties)", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_s3_class(res, "data.table") - expect_true(all(c("Progeny","Male_Parent","Female_Parent", - "Mendelian_Error_Pct","Markers_Tested", - "Assignment_Status") %in% names(res))) - expect_equal(nrow(res), 1L) -}) - -test_that("best_male_parent returns expected columns", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_male_parent", - verbose = FALSE, write_txt = FALSE) - expect_s3_class(res, "data.table") - expect_true(all(c("Progeny","Best_Match","Mendelian_Error_Pct", - "Markers_Tested","Assignment_Status") %in% names(res))) - expect_equal(nrow(res), 1L) -}) - -test_that("best_female_parent returns expected columns", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_female_parent", - verbose = FALSE, write_txt = FALSE) - expect_s3_class(res, "data.table") - expect_true(all(c("Progeny","Best_Match","Mendelian_Error_Pct", - "Markers_Tested","Assignment_Status") %in% names(res))) - expect_equal(nrow(res), 1L) -}) - -test_that("best_match returns expected columns", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_match", - verbose = FALSE, write_txt = FALSE) - expect_s3_class(res, "data.table") - expect_true(all(c("Progeny","Best_Match","Mendelian_Error_Pct", - "Markers_Tested","Assignment_Status") %in% names(res))) - expect_equal(nrow(res), 1L) -}) - -test_that("one row returned per progeny for single-parent methods", { - f <- make_files(base_genos, base_parents, child1_progeny) - for (m in c("best_male_parent","best_female_parent","best_match")) { - res <- find_parentage(f$g, f$p, f$pr, method = m, - verbose = FALSE, write_txt = FALSE) - expect_equal(nrow(res), 1L, label = paste("row count for method", m)) - } -}) - -test_that("Markers_Tested equals the number of non-NA marker columns", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_equal(res$Markers_Tested, ncol(base_genos) - 1L) -}) - -test_that("Mendelian_Error_Pct is between 0 and 100", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - pct <- as.numeric(res$Mendelian_Error_Pct) - expect_true(all(pct >= 0 & pct <= 100, na.rm = TRUE)) -}) - -# ============================================================================== -# 3. Biological correctness -# ============================================================================== - -test_that("best_pair correctly identifies S1 x D1 for child1 with 0% error", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_equal(res$Male_Parent, "S1") - expect_equal(res$Female_Parent, "D1") - expect_equal(as.numeric(res$Mendelian_Error_Pct), 0) -}) - -test_that("best_pair correctly identifies S2 x D2 for child2 with 0% error", { - f <- make_files(base_genos, base_parents, child2_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_equal(res$Male_Parent, "S2") - expect_equal(res$Female_Parent, "D2") - expect_equal(as.numeric(res$Mendelian_Error_Pct), 0) -}) - -test_that("best_male_parent identifies S1 as best male for child1", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_male_parent", - verbose = FALSE, write_txt = FALSE) - expect_equal(res$Best_Match, "S1") -}) - -test_that("best_female_parent identifies D1 as best female for child1", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_female_parent", - verbose = FALSE, write_txt = FALSE) - expect_equal(res$Best_Match, "D1") -}) - -test_that("Mendelian_Error_Pct is 0 for a perfect parent-progeny trio", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_equal(as.numeric(res$Mendelian_Error_Pct), 0) -}) - -# ============================================================================== -# 4. Assignment_Status — min_markers and error_threshold -# ============================================================================== - -test_that("Assignment_Status = PASS for perfect trio within thresholds", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - min_markers = 3, error_threshold = 5.0, - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_equal(res$Assignment_Status, "PASS") -}) - -test_that("Assignment_Status = LOW_MARKERS when min_markers exceeds available markers", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - min_markers = 99999, error_threshold = 5.0, - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_equal(res$Assignment_Status, "LOW_MARKERS") -}) - -test_that("Assignment_Status = HIGH_ERROR when error rate exceeds threshold", { - # Use wrong parents so error rate is high - high_error_genos <- data.table::data.table( - ID = c("S1","D1","bad_child"), - M1 = c(0L, 0L, 2L), - M2 = c(0L, 0L, 2L), - M3 = c(0L, 0L, 2L), - M4 = c(0L, 0L, 2L), - M5 = c(0L, 0L, 2L) - ) - parents <- data.table::data.table(ID = c("S1","D1"), Sex = c("M","F")) - progeny <- data.table::data.table(ID = "bad_child") - f <- make_files(high_error_genos, parents, progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - min_markers = 3, error_threshold = 5.0, - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_equal(res$Assignment_Status, "HIGH_ERROR") -}) - -test_that("Assignment_Status column is present in all methods", { - f <- make_files(base_genos, base_parents, child1_progeny) - for (m in c("best_pair","best_male_parent","best_female_parent","best_match")) { - res <- find_parentage(f$g, f$p, f$pr, method = m, - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_true("Assignment_Status" %in% names(res), - label = paste("Assignment_Status present for method", m)) - } -}) - -# ============================================================================== -# 5. allow_selfing -# ============================================================================== - -test_that("allow_selfing = FALSE removes self-pairs from candidates", { - ambig_parents <- data.table::data.table(ID = c("S1","D1"), Sex = c("A","A")) - f <- make_files(base_genos, ambig_parents, child1_progeny) - expect_warning( - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - allow_selfing = FALSE, show_ties = FALSE, - verbose = FALSE, write_txt = FALSE), - regexp = "tied" - ) - if (!is.na(res$Male_Parent) && !is.na(res$Female_Parent)) - expect_false(res$Male_Parent == res$Female_Parent) -}) - -# ============================================================================== -# 6. show_ties -# ============================================================================== - -test_that("show_ties = TRUE produces _1/_2 suffixed columns when ties exist", { - f <- make_files(tied_genos, tied_parents, tied_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = TRUE, verbose = FALSE, write_txt = FALSE) - expect_true(any(grepl("^Male_Parent_", names(res)))) - expect_true(any(grepl("^Female_Parent_", names(res)))) -}) - -test_that("show_ties = FALSE warns about ties and returns single-result columns", { - f <- make_files(tied_genos, tied_parents, tied_progeny) - expect_warning( - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = FALSE, verbose = FALSE, write_txt = FALSE), - regexp = "tied" - ) - expect_true("Male_Parent" %in% names(res)) - expect_true("Female_Parent" %in% names(res)) - expect_false(any(grepl("^Male_Parent_\\d", names(res)))) - expect_false(any(grepl("^Female_Parent_\\d", names(res)))) -}) - -# ============================================================================== -# 7. verbose / write_txt -# ============================================================================== - -test_that("verbose = TRUE returns the result invisibly as data.table", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - verbose = TRUE, write_txt = FALSE) - expect_s3_class(res, "data.table") -}) - -test_that("verbose = FALSE returns the result as data.table", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - verbose = FALSE, write_txt = FALSE) - expect_s3_class(res, "data.table") -}) - -test_that("write_txt = TRUE creates the output file", { - old_wd <- getwd() - tmp <- tempdir() - setwd(tmp) - on.exit(setwd(old_wd), add = TRUE) - f <- make_files(base_genos, base_parents, child1_progeny, dir = tmp) - find_parentage(f$g, f$p, f$pr, method = "best_pair", - verbose = FALSE, write_txt = TRUE) - expect_true(file.exists(file.path(tmp, "parentage_results_dt.txt"))) -}) - -test_that("write_txt = FALSE does not create the output file", { - old_wd <- getwd() - tmp <- tempdir() - setwd(tmp) - on.exit(setwd(old_wd), add = TRUE) - out_file <- file.path(tmp, "parentage_results_dt.txt") - if (file.exists(out_file)) file.remove(out_file) - f <- make_files(base_genos, base_parents, child1_progeny, dir = tmp) - find_parentage(f$g, f$p, f$pr, method = "best_pair", - verbose = FALSE, write_txt = FALSE) - expect_false(file.exists(out_file)) -}) - -# ============================================================================== -# 8. Sex-based candidate filtering -# ============================================================================== - -test_that("best_male_parent only assigns M or A parents", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_male_parent", - verbose = FALSE, write_txt = FALSE) - valid_male_parents <- base_parents[Sex %in% c("M","A")]$ID - expect_true(res$Best_Match %in% valid_male_parents) -}) - -test_that("best_female_parent only assigns F or A parents", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_female_parent", - verbose = FALSE, write_txt = FALSE) - valid_female_parents <- base_parents[Sex %in% c("F","A")]$ID - expect_true(res$Best_Match %in% valid_female_parents) -}) - -# ============================================================================== -# 9. Edge cases -# ============================================================================== - -test_that("single progeny individual is handled correctly", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_equal(nrow(res), 1L) -}) - -test_that("all-NA marker column does not cause an error", { - na_genos <- data.table::copy(base_genos) - na_genos[, M1 := NA_integer_] - f <- make_files(na_genos, base_parents, child1_progeny) - expect_no_error( - find_parentage(f$g, f$p, f$pr, method = "best_pair", - verbose = FALSE, write_txt = FALSE) - ) -}) - -test_that("Progeny column contains the correct progeny IDs", { - f <- make_files(base_genos, base_parents, child1_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_setequal(res$Progeny, child1_progeny$ID) -}) - -test_that("multiple progeny are all represented in output", { - f <- make_files(base_genos, base_parents, base_progeny) - res <- find_parentage(f$g, f$p, f$pr, method = "best_pair", - show_ties = FALSE, verbose = FALSE, write_txt = FALSE) - expect_setequal(res$Progeny, base_progeny$ID) -}) diff --git a/tests/testthat/test-madc2vcf_all.R b/tests/testthat/test-madc2vcf_all.R index 9694f75..2c0a07b 100644 --- a/tests/testthat/test-madc2vcf_all.R +++ b/tests/testthat/test-madc2vcf_all.R @@ -48,15 +48,15 @@ test_that("test madc offtargets",{ # Without hap_seq provided set.seed(123) - madc2vcf_all(madc = madc_file, - botloci_file = bot_file, - hap_seq_file = NULL, - n.cores = 2, - rm_multiallelic_SNP = FALSE, - multiallelic_SNP_dp_thr = 0, - multiallelic_SNP_sample_thr = 0, - out_vcf = temp, - verbose = FALSE) + expect_warning(madc2vcf_all(madc = madc_file, + botloci_file = bot_file, + hap_seq_file = NULL, + n.cores = 2, + rm_multiallelic_SNP = FALSE, + multiallelic_SNP_dp_thr = 0, + multiallelic_SNP_sample_thr = 0, + out_vcf = temp, + verbose = FALSE)) vcf <- read.vcfR(temp) @@ -285,7 +285,7 @@ test_that("simu alfalfa",{ markers_info = alfalfa_markers_info, out_vcf = out, verbose = FALSE), - regexp = "None of the markers_info CloneID( or BI_markerID)? values match the MADC CloneID column. Please make sure they use the same marker IDs." + regexp = "None of the markers_info CloneID, Marker_ID, or BI_markerID values match the MADC CloneID column. Please make sure they use the same marker IDs." ) # Test error when markers_info_ChromPos is provided but IDs still don't match botloci @@ -336,12 +336,12 @@ test_that("simu alfalfa",{ expect_equal(sum(dp[1,]), 4534) expect_equal(sum(dp[,5]), 233719) - madc2vcf_all(madc = alfalfa_lowercase, - botloci_file = alfalfa_botloci, - hap_seq_file = NULL, - n.cores = 1, - out_vcf = out, - verbose = FALSE) + expect_warning(madc2vcf_all(madc = alfalfa_lowercase, + botloci_file = alfalfa_botloci, + hap_seq_file = NULL, + n.cores = 1, + out_vcf = out, + verbose = FALSE)) vcf <- read.vcfR(out, verbose = FALSE) lut <- read.csv(alfalfa_markers_info) @@ -356,13 +356,13 @@ test_that("simu alfalfa",{ expect_equal(sum(dp[1,]), 4534) expect_equal(sum(dp[,5]), 230415) - madc2vcf_all(madc = alfalfa_lowercase, - botloci_file = alfalfa_botloci, - hap_seq_file = NULL, - n.cores = 1, - markers_info = alfalfa_markers_info, - out_vcf = out, - verbose = FALSE) + expect_warning(madc2vcf_all(madc = alfalfa_lowercase, + botloci_file = alfalfa_botloci, + hap_seq_file = NULL, + n.cores = 1, + markers_info = alfalfa_markers_info, + out_vcf = out, + verbose = FALSE)) vcf <- read.vcfR(out, verbose = FALSE) lut <- read.csv(alfalfa_markers_info) @@ -389,7 +389,7 @@ test_that("simu alfalfa",{ n.cores = 1, out_vcf = out, verbose = FALSE), - regexp = "IUPAC \\(non-ATCG\\) codes found in AlleleSequence\\. This codes are not currently supported by BIGr/BIGapp\\. Run HapApp to replace them" + regexp = "IUPAC \\(non-ATCG\\) codes found in Ref_/Alt_ AlleleSequence that differ between alleles or are present in only one. These codes are not currently supported by BIGr/BIGapp" ) }) @@ -576,7 +576,7 @@ test_that("simu alfalfa",{ markers_info = potato_markers_info, out_vcf = out, verbose = FALSE), - regexp = "IUPAC \\(non-ATCG\\) codes found in AlleleSequence. This codes are not currently supported by BIGr/BIGapp. Run HapApp to replace them" + regexp = "IUPAC \\(non-ATCG\\) codes found in Ref_/Alt_ AlleleSequence that differ between alleles or are present in only one. These codes are not currently supported by BIGr/BIGapp" ) }) @@ -593,3 +593,71 @@ test_that("simu alfalfa",{ ) }) }) + +# ======================================================================= +# Other allele filters and IUPAC codes (internal compare function) +# ======================================================================= + +test_that("Other alleles with too many close SNPs are discarded", { + + clone <- "Chr01_000000100" + make_tag <- function(seqs, ids) { + data.frame(AlleleID = paste0(clone, "|", ids), Chromosome = NA, SNP_position_in_Genome = NA, + Ref = NA, Alt = NA, CloneID = clone, AlleleSequence = seqs, + S1 = 10, S2 = 5, check.names = FALSE) + } + sub_bases <- function(seq, pos, bases) { + x <- strsplit(seq, "")[[1]] + x[pos] <- bases + paste(x, collapse = "") + } + + # Target SNP at position 30 of the sequence (position 100 in the genome) + ref <- "ACGTTGCAAGCTTGACCTGAAGTCCATGGATCCGTACGATCGGATCCAAGTTCGACTGAT" + alt <- sub_bases(ref, 30, "G") + # Other_0003 has four SNPs 2bp apart, Other_0004 has three + other4 <- sub_bases(ref, c(10, 12, 14, 16), c("C", "A", "A", "T")) + other3 <- sub_bases(ref, c(10, 12, 14), c("C", "A", "A")) + tag <- make_tag(c(ref, alt, other4, other3), c("Ref_0001", "Alt_0002", "Other_0003", "Other_0004")) + botloci <- data.frame(V1 = "Chr02_000000500") + + # Defaults (others_min_dist = 5, others_max_close_snps = 3): only Other_0003 is discarded + res <- compare(tag, botloci) + expect_equal(res$n_rm_others_close_snps, 1L) + expect_false(any(grepl("Other_0003", res$update_tag$AlleleID))) + expect_equal(sum(grepl("Other_0004", res$update_tag$AlleleID)), 3) + expect_equal(res$update_tag$SNP_position_in_Genome[grepl("Other_0004", res$update_tag$AlleleID)], + c("80", "82", "84")) + + # The filter is not applied when others_max_close_snps is NULL + res <- compare(tag, botloci, others_max_close_snps = NULL) + expect_equal(res$n_rm_others_close_snps, 0L) + expect_equal(sum(grepl("Other_0003", res$update_tag$AlleleID)), 4) + + # SNPs 2bp apart are not close when others_min_dist = 2 + res <- compare(tag, botloci, others_min_dist = 2) + expect_equal(res$n_rm_others_close_snps, 0L) + expect_equal(sum(grepl("Other_0003", res$update_tag$AlleleID)), 4) +}) + +test_that("identical IUPAC codes in Ref and Alt are not reported as SNPs", { + + clone <- "Chr01_000000100" + ref <- "ACGTTGCAAGCTTGACCTGAAGTCCATGGATCCGTACGATCGGATNCAAGTTCGACTGAT" + alt <- "ACGTTGCAAGCTTGACCTGAAGTCCATGGGTCCGTACGATCGGATNCAAGTTCGACTGAT" + # RefMatch with one SNP at position 10 and the same N at position 46 + ref_match <- "ACGTTGCAACCTTGACCTGAAGTCCATGGATCCGTACGATCGGATNCAAGTTCGACTGAT" + tag <- data.frame(AlleleID = paste0(clone, c("|Ref_0001", "|Alt_0002", "|RefMatch_0003")), + Chromosome = NA, SNP_position_in_Genome = NA, Ref = NA, Alt = NA, + CloneID = clone, AlleleSequence = c(ref, alt, ref_match), + S1 = 10, S2 = 5, check.names = FALSE) + botloci <- data.frame(V1 = "Chr02_000000500") + + res <- compare(tag, botloci) + # The tag is kept, with the target SNP and the RefMatch SNP only + expect_null(res$rm_N) + expect_null(res$rm_indels) + expect_equal(nrow(res$update_tag), 3) + expect_equal(res$update_tag$SNP_position_in_Genome, c("100", "100", "80")) + expect_false(any(res$update_tag$Ref == "N" | res$update_tag$Alt == "N")) +}) diff --git a/tests/testthat/test-madc2vcf_multi.R b/tests/testthat/test-madc2vcf_multi.R index adca090..58450dd 100644 --- a/tests/testthat/test-madc2vcf_multi.R +++ b/tests/testthat/test-madc2vcf_multi.R @@ -114,7 +114,7 @@ test_that("madc2vcf_multi — alfalfa (BIGapp-PanelHub)", { outfile = out, ploidy = 4L, verbose = TRUE - ), regexp = "MADC Allele Sequences contain IUPAC \\(non-ATCG\\) codes. Please run HapApp to clean MADC file before using this function." + ), regexp = "MADC Allele Sequences contain different IUPAC \\(non-ATCG\\) codes in REF and ALT sequences. Please run HapApp to clean MADC file before using this function." ) out <- tempfile(fileext = ".vcf") diff --git a/tests/testthat/test-madc2vcf_targets.R b/tests/testthat/test-madc2vcf_targets.R index 9ffc72e..9d04042 100644 --- a/tests/testthat/test-madc2vcf_targets.R +++ b/tests/testthat/test-madc2vcf_targets.R @@ -760,3 +760,33 @@ test_that("simu alfalfa",{ expect_equal(sum(DP[,5]), 56547) }) }) + +test_that("markers missing from markers_info are reported before the POS correction", { + madc_file <- system.file("example_MADC_FixedAlleleID.csv", package="BIGr") + report <- read.csv(madc_file) + + # markers_info with Chr and Pos for every marker except chr1.1_000194324 + ids <- setdiff(unique(report$CloneID), "chr1.1_000194324") + mi <- data.frame(CloneID = ids, + Chr = sub("_.*", "", ids), + Pos = as.numeric(sub(".*_", "", ids))) + mi_file <- tempfile(fileext = ".csv") + write.csv(mi, mi_file, row.names = FALSE) + + temp <- tempfile(fileext = ".vcf") + warns <- character(0) + withCallingHandlers( + madc2vcf_targets(madc_file = madc_file, output.file = temp, markers_info = mi_file, + get_REF_ALT = FALSE, verbose = FALSE), + warning = function(w) { + warns <<- c(warns, conditionMessage(w)) + invokeRestart("muffleWarning") + } + ) + + expect_true(any(grepl("1 markers removed: no matching entry found in markers_info.", warns, fixed = TRUE))) + + vcf <- read.vcfR(temp, verbose = FALSE) + expect_equal(nrow(vcf), 19) + expect_false("chr1.1_000194324" %in% vcf@fix[, "ID"]) +}) diff --git a/tests/testthat/test-madc_plot.R b/tests/testthat/test-madc_plot.R new file mode 100644 index 0000000..f064573 --- /dev/null +++ b/tests/testthat/test-madc_plot.R @@ -0,0 +1,229 @@ +context("MADC plot") + +madc_file <- function() system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") + +test_that("single plot.type returns a ggplot", { + p <- suppressWarnings(suppressMessages(madc_plot(madc_file(), plot.type = "missing"))) + expect_s3_class(p, "ggplot") +}) + +test_that("pca, heatmap, marker build without error", { + expect_no_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "pca")))) + expect_no_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "heatmap", fill = "depth")))) + expect_no_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "heatmap", fill = "mhaps")))) + # example CloneIDs are Chr_Pos -> marker plot works from CloneID directly + expect_no_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "marker")))) +}) + +test_that("balance and depth plots build", { + expect_s3_class(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "balance"))), "ggplot") + # ploidy adds expected-ratio guides + a binomial envelope (diploid and higher) + expect_s3_class(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "balance", ploidy = 2))), "ggplot") + expect_s3_class(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "balance", ploidy = 4))), "ggplot") + expect_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "balance", ploidy = 0))), "positive") + expect_s3_class(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "depth"))), "ggplot") + # both embed in a multi-panel figure + res <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = c("depth", "balance")))) + expect_true(grid::is.grob(res$panel)) +}) + +test_that("balance.density stacks a marginal panel (composite grob)", { + # a lone balance plot with the marginal returns a composite grob + g <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "balance", ploidy = 2, balance.density = TRUE))) + expect_true(grid::is.grob(g)) + # combined with another type, the density is ignored and balance stays a ggplot + res <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = c("balance", "depth"), balance.density = TRUE))) + expect_s3_class(res$plots$balance, "ggplot") +}) + +test_that("circos plot builds without error", { + skip_if_not_installed("circlize") + tmp <- tempfile(fileext = ".png") + expect_no_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "circos", output.file = tmp)))) + expect_true(file.exists(tmp)) +}) + +test_that("circos must be requested on its own", { + expect_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = c("circos", "pca")))), "on its own") +}) + +test_that("circos honors a custom density.window", { + skip_if_not_installed("circlize") + tmp <- tempfile(fileext = ".png") + expect_no_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "circos", density.window = 5e5, output.file = tmp)))) +}) + +test_that("circos honors depth.qc.maxmiss for the combined Depth ring", { + skip_if_not_installed("circlize") + tmp <- tempfile(fileext = ".png") + expect_no_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "circos", depth.qc.maxmiss = 0.3, output.file = tmp)))) + expect_true(file.exists(tmp)) +}) + +test_that("circos accepts custom label.gap, paralog.flag, and track colors", { + skip_if_not_installed("circlize") + tmp <- tempfile(fileext = ".png") + expect_no_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "circos", label.gap = 20, paralog.flag = 5, + mhap.col = "#1f78b4", depth.col = c("navy", "grey90", "orange"), + output.file = tmp)))) + expect_true(file.exists(tmp)) +}) + +test_that("heatmap groups by category and facets by chromosome", { + meta <- data.frame(sample = paste0("Sample_", 1:10), + species = rep(c("A", "B"), 5), stringsAsFactors = FALSE) + p <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "heatmap", + metadata = meta, group.col = "species", facet.chrom = TRUE))) + expect_s3_class(p, "ggplot") +}) + +test_that("marker plot works with a markers_info lookup", { + cids <- unique(read.csv(madc_file(), check.names = FALSE)$CloneID) + mi <- data.frame(CloneID = cids, Chr = "chr1.1", + Pos = as.numeric(sub(".*_", "", cids)), + stringsAsFactors = FALSE) + expect_no_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "marker", markers_info = mi)))) +}) + +test_that("multiple plot.types return list + panel", { + res <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = c("pca", "missing")))) + expect_type(res, "list") + expect_s3_class(res$plots$pca, "ggplot") + expect_true(grid::is.grob(res$panel)) +}) + +test_that("bare default call builds the 4-panel figure (no circos)", { + res <- suppressWarnings(suppressMessages(madc_plot(madc_file()))) + expect_type(res, "list") + expect_equal(names(res$plots), c("pca", "marker", "heatmap", "missing")) + expect_true(grid::is.grob(res$panel)) +}) + +test_that("metadata colors PCA and groups the missing boxplot", { + meta <- data.frame(sample = paste0("Sample_", 1:10), + species = rep(c("A", "B"), 5), stringsAsFactors = FALSE) + p1 <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "pca", metadata = meta, group.col = "species"))) + p2 <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "missing", metadata = meta, group.col = "species"))) + expect_s3_class(p1, "ggplot") + expect_s3_class(p2, "ggplot") +}) + +test_that("PCA supports shape.col and a custom palette", { + meta <- data.frame(sample = paste0("Sample_", 1:10), + species = rep(c("A", "B"), 5), + plate = rep(c("P1", "P2"), each = 5), + stringsAsFactors = FALSE) + p <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "pca", metadata = meta, + group.col = "species", shape.col = "plate", + palette = c("#440154", "#21918c", "#fde725")))) + expect_s3_class(p, "ggplot") +}) + +test_that("missing boxplot supports miss.sort and horizontal", { + meta <- data.frame(sample = paste0("Sample_", 1:10), + species = rep(c("A", "B"), 5), stringsAsFactors = FALSE) + for (s in c("none", "asc", "desc")) for (h in c(TRUE, FALSE)) { + p <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "missing", metadata = meta, + group.col = "species", miss.sort = s, horizontal = h))) + expect_s3_class(p, "ggplot") + } + # horizontal also works without a grouping category + expect_s3_class(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "missing", horizontal = TRUE))), "ggplot") + expect_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "missing", miss.sort = "bogus")))) +}) + +test_that("titles = FALSE drops the title and subtitle", { + for (pt in c("missing", "depth", "balance", "marker", "heatmap")) { + p <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = pt, ploidy = 2, titles = FALSE))) + expect_null(p$labels$title) + expect_null(p$labels$subtitle) + } + # titles are on by default + p <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "depth"))) + expect_false(is.null(p$labels$title)) + expect_false(is.null(p$labels$subtitle)) + # the balance caption is kept (it documents the min.depth line) + p <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "balance", titles = FALSE))) + expect_false(is.null(p$labels$caption)) + # multi-panel and the balance density composite still build + res <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = c("pca", "missing"), titles = FALSE))) + expect_null(res$plots$pca$labels$title) + expect_true(grid::is.grob(res$panel)) + expect_true(grid::is.grob(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "balance", ploidy = 2, + balance.density = TRUE, titles = FALSE))))) +}) + +test_that("circos builds with titles = FALSE", { + skip_if_not_installed("circlize") + tmp <- tempfile(fileext = ".png") + expect_no_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "circos", titles = FALSE, output.file = tmp)))) + expect_true(file.exists(tmp)) +}) + +test_that("output.file writes an image", { + out <- tempfile(fileext = ".png") + suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "missing", output.file = out))) + expect_true(file.exists(out)) +}) + +test_that("raw MADC is rejected", { + raw <- system.file("iris_DArT_MADC.csv", package = "BIGr") + expect_error(suppressMessages(madc_plot(raw, plot.type = "pca")), "HapApp") +}) + +test_that("metadata without group.col is accepted; bad columns name the argument", { + meta <- data.frame(sample = paste0("Sample_", 1:10), + plate = rep(c("P1", "P2"), each = 5), + stringsAsFactors = FALSE) + p <- suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "pca", metadata = meta, shape.col = "plate"))) + expect_s3_class(p, "ggplot") + expect_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "pca", metadata = meta, shape.col = "nope"))), + "shape.col", fixed = TRUE) +}) + +test_that("pc.x/pc.y outside the available components give a clear error", { + expect_error(suppressWarnings(suppressMessages( + madc_plot(madc_file(), plot.type = "pca", pc.y = 99))), + "`pc.x` and `pc.y`", fixed = TRUE) +}) + +test_that("unsupported output.file extensions are rejected", { + expect_error(madc_plot(madc_file(), plot.type = "missing", + output.file = tempfile(fileext = ".bmp")), + "output.file", fixed = TRUE) +}) diff --git a/tests/testthat/test-madc_summary.R b/tests/testthat/test-madc_summary.R new file mode 100644 index 0000000..c84be58 --- /dev/null +++ b/tests/testthat/test-madc_summary.R @@ -0,0 +1,120 @@ +context("MADC summary") + +test_that("madc_summary returns the expected tables", { + madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") + res <- suppressWarnings(suppressMessages(madc_summary(madc_file, min.depth = 10))) + + expect_type(res, "list") + for (nm in c("mhap_freq", "per_sample", "per_marker", "missingness", + "depth_sweep", "marker_qc", "overall")) + expect_s3_class(res[[nm]], "data.frame") + + # mhap_freq: rows 2..10 + ">10" = 10 rows; example distribution 15/3/3 at 2/3/4 + expect_equal(nrow(res$mhap_freq), 10) + expect_equal(res$mhap_freq$n_markers[res$mhap_freq$n_mhaps == "2"], 15) + expect_equal(res$mhap_freq$n_markers[res$mhap_freq$n_mhaps == "3"], 3) + expect_equal(res$mhap_freq$n_markers[res$mhap_freq$n_mhaps == "4"], 3) + expect_equal(sum(res$mhap_freq$n_markers), 21) + + expect_equal(nrow(res$per_sample), 10) # 10 samples + expect_equal(nrow(res$per_marker), 21) # 21 CloneIDs + expect_equal(nrow(res$missingness), 12) # 12 thresholds + expect_equal(nrow(res$overall), 1) + + # per_sample columns and value ranges (robust stats + read-weighted off-target) + expect_true(all(c("sample", "depth", "median_depth", "depth_mad", "total_reads", + "missing_rate", "call_rate", "n_mhaps_observed", + "offtarget_fraction", "offtarget_fraction_weighted") + %in% names(res$per_sample))) + expect_true(all(res$per_sample$missing_rate >= 0 & res$per_sample$missing_rate <= 1)) + expect_equal(res$per_sample$call_rate, 1 - res$per_sample$missing_rate) + + # per_marker splits defined vs observed mHap counts, defined >= observed + expect_true(all(c("n_mhaps_defined", "n_mhaps_observed") %in% names(res$per_marker))) + expect_true(all(res$per_marker$n_mhaps_defined >= res$per_marker$n_mhaps_observed)) + + # depth_sweep: one row per threshold, monotone non-increasing cells_passing + expect_equal(nrow(res$depth_sweep), 6) + expect_true(all(diff(res$depth_sweep$cells_passing) <= 0)) + + # marker_qc: one row per marker, valid status vocabulary + expect_equal(nrow(res$marker_qc), 21) + expect_true(all(res$marker_qc$qc_status %in% c("PASS", "FLAG", "FAIL"))) + # overall carries both the mean-locus and read-weighted off-target + expect_true(all(c("mean_offtarget_fraction", "read_weighted_offtarget_fraction", + "median_depth", "overall_call_rate") %in% names(res$overall))) +}) + +test_that("marker_qc criteria flag/fail as configured", { + madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") + # disable the depth FAIL, enable an mHap FLAG that the fixture triggers + res <- suppressWarnings(suppressMessages( + madc_summary(madc_file, min.locus.depth = NULL, max.mhaps.per.loci = 3))) + flagged <- res$marker_qc$qc_status == "FLAG" + expect_true(any(flagged)) + expect_true(all(grepl("paralog", res$marker_qc$qc_reason[flagged]))) + # with all criteria off, every marker passes + res0 <- suppressWarnings(suppressMessages( + madc_summary(madc_file, min.locus.depth = NULL))) + expect_true(all(res0$marker_qc$qc_status == "PASS")) +}) + +test_that("target.only counts only Ref/Alt", { + madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") + res <- suppressWarnings(suppressMessages(madc_summary(madc_file, target.only = TRUE))) + # 20 markers keep both Ref and Alt (one marker has only a Ref target row) + expect_equal(res$mhap_freq$n_markers[res$mhap_freq$n_mhaps == "2"], 20) + expect_equal(res$overall$mean_offtarget_fraction, 0) +}) + +test_that("metadata adds a by_category table", { + madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") + meta <- data.frame(sample = paste0("Sample_", 1:10), + species = rep(c("A", "B"), 5), + stringsAsFactors = FALSE) + res <- suppressWarnings(suppressMessages( + madc_summary(madc_file, metadata = meta, group.col = "species"))) + expect_s3_class(res$by_category, "data.frame") + expect_equal(nrow(res$by_category), 2) + expect_equal(sort(res$by_category$group), c("A", "B")) + expect_equal(sum(res$by_category$n_samples), 10) +}) + +test_that("output.file writes CSVs", { + madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") + out <- tempfile() + suppressWarnings(suppressMessages(madc_summary(madc_file, output.file = out))) + expect_true(file.exists(paste0(out, "_mhap_freq.csv"))) + expect_true(file.exists(paste0(out, "_per_sample.csv"))) + expect_true(file.exists(paste0(out, "_missingness.csv"))) + expect_true(file.exists(paste0(out, "_depth_sweep.csv"))) + expect_true(file.exists(paste0(out, "_marker_qc.csv"))) +}) + +test_that("output.format = 'xlsx' writes one workbook with a sheet per table", { + skip_if_not_installed("writexl") + skip_if_not_installed("readxl") + madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") + out <- tempfile() + suppressWarnings(suppressMessages( + madc_summary(madc_file, output.file = out, output.format = "xlsx"))) + xl <- paste0(out, ".xlsx") + expect_true(file.exists(xl)) + expect_true(all(c("mhap_freq", "per_sample", "per_marker", "missingness", + "depth_sweep", "marker_qc", "overall") %in% + readxl::excel_sheets(xl))) +}) + +test_that("raw MADC is rejected", { + raw <- system.file("iris_DArT_MADC.csv", package = "BIGr") + expect_error(suppressMessages(madc_summary(raw)), "HapApp") +}) + +test_that("metadata without group.col returns no by_category table", { + madc_file <- system.file("example_MADC_FixedAlleleID.csv", package = "BIGr") + meta <- data.frame(sample = paste0("Sample_", 1:10), + species = rep(c("A", "B"), 5), + stringsAsFactors = FALSE) + res <- suppressWarnings(suppressMessages(madc_summary(madc_file, metadata = meta))) + expect_null(res$by_category) +}) diff --git a/tests/testthat/test-validate_pedigree.R b/tests/testthat/test-validate_pedigree.R deleted file mode 100644 index ef4e493..0000000 --- a/tests/testthat/test-validate_pedigree.R +++ /dev/null @@ -1,471 +0,0 @@ -# tests/testthat/test-validate_pedigree.R -# Run with: testthat::test_file("tests/testthat/test-validate_pedigree.R") -library(testthat) -library(data.table) - -# ============================================================================== -# Helpers -# ============================================================================== - -make_genos <- function() { - # IND_A: all 0, IND_B: all 2, IND_C: all 1 (het), IND_D: all 0, IND_E: all 0 - n_markers <- 20 - marker_names <- paste0("M", seq_len(n_markers)) - dt <- data.table( - ID = c("IND_A", "IND_B", "IND_C", "IND_D", "IND_E"), - rbind( - rep(0L, n_markers), # IND_A — all ref homozygous - rep(2L, n_markers), # IND_B — all alt homozygous - rep(1L, n_markers), # IND_C — all het (valid child of IND_A x IND_B) - rep(0L, n_markers), # IND_D — all ref (impossible child of IND_B x IND_A) - rep(0L, n_markers) # IND_E — all ref - ) - ) - setnames(dt, c("ID", marker_names)) - dt -} - -make_pedigree <- function() { - # IND_C: perfect Mendelian child of IND_A x IND_B -> PASS - # IND_D: declared parents swapped -> FAIL - data.table( - ID = c("IND_C", "IND_D"), - Male_Parent = c("IND_A", "IND_B"), - Female_Parent = c("IND_B", "IND_A") - ) -} - -write_temp_files <- function(genos = make_genos(), ped = make_pedigree()) { - ped_file <- tempfile(fileext = ".txt") - genos_file <- tempfile(fileext = ".txt") - fwrite(ped, ped_file, sep = "\t") - fwrite(genos, genos_file, sep = "\t") - list(ped = ped_file, genos = genos_file) -} - -# ============================================================================== -# 1. Input validation -# ============================================================================== - -test_that("trio_error_threshold out of range raises an error", { - f <- write_temp_files() - expect_error(validate_pedigree(f$ped, f$genos, - trio_error_threshold = 150, - verbose = FALSE, write_txt = FALSE)) - expect_error(validate_pedigree(f$ped, f$genos, - trio_error_threshold = -1, - verbose = FALSE, write_txt = FALSE)) -}) - -test_that("single_parent_error_threshold out of range raises an error", { - f <- write_temp_files() - expect_error(validate_pedigree(f$ped, f$genos, - single_parent_error_threshold = 101, - verbose = FALSE, write_txt = FALSE)) - expect_error(validate_pedigree(f$ped, f$genos, - single_parent_error_threshold = -5, - verbose = FALSE, write_txt = FALSE)) -}) - -test_that("missing required pedigree column raises an error", { - bad_ped <- data.table(ID = "IND_C", Parent1 = "IND_A", Female_Parent = "IND_B") - f <- write_temp_files(ped = bad_ped) - expect_error( - validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE), - regexp = "missing required columns" - ) -}) - -test_that("missing ID column in genotypes raises an error", { - bad_genos <- copy(make_genos()) - setnames(bad_genos, "ID", "SampleID") - f <- write_temp_files(genos = bad_genos) - expect_error( - validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE), - regexp = "ID" - ) -}) - -test_that("all trios with no genotype data stop with an error", { - ped <- data.table(ID = "GHOST", Male_Parent = "IND_A", Female_Parent = "IND_B") - f <- write_temp_files(ped = ped) - expect_error( - validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE), - regexp = "No valid trios remain" - ) -}) - -# ============================================================================== -# 2. Return structure -# ============================================================================== - -test_that("returns a data.table", { - f <- write_temp_files() - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - expect_s3_class(res, "data.table") -}) - -test_that("result has one row per pedigree entry", { - f <- write_temp_files() - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - expect_equal(nrow(res), 2L) -}) - -test_that("result has all expected columns", { - f <- write_temp_files() - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - expected_cols <- c( - "ID", "Male_Parent", "Female_Parent", - "Mendelian_Error_Pct", "Markers_Tested", "Status", - "Correction_Decision", - "Male_Parent_Hom_Error_Pct", "Female_Parent_Hom_Error_Pct", - "Best_Male_Parent", "Best_Male_Parent_Error_Pct", - "Best_Female_Parent", "Best_Female_Parent_Error_Pct" - ) - expect_true(all(expected_cols %in% names(res))) -}) - -# ============================================================================== -# 3. PASS / FAIL / LOW_MARKERS / NO_DATA statuses -# ============================================================================== - -test_that("PASS trio is correctly identified", { - f <- write_temp_files() - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - pass_row <- res[ID == "IND_C"] - expect_equal(nrow(pass_row), 1L) - expect_equal(pass_row$Status, "PASS") - expect_equal(pass_row$Mendelian_Error_Pct, 0) - expect_equal(pass_row$Correction_Decision, "NONE") -}) - -test_that("FAIL trio is correctly identified", { - f <- write_temp_files() - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - fail_row <- res[ID == "IND_D"] - expect_equal(nrow(fail_row), 1L) - expect_equal(fail_row$Status, "FAIL") - expect_gt(fail_row$Mendelian_Error_Pct, 5.0) -}) - -test_that("FAIL trio has REMOVE_MALE_PARENT decision with best match populated", { - f <- write_temp_files() - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - fail_row <- res[ID == "IND_D"] - expect_equal(fail_row$Correction_Decision, "REMOVE_MALE_PARENT") - expect_false(is.na(fail_row$Best_Male_Parent)) - expect_true(is.na(fail_row$Best_Female_Parent)) -}) - -test_that("Mendelian_Error_Pct is 0 for a perfect trio", { - f <- write_temp_files() - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - expect_equal(res[ID == "IND_C"]$Mendelian_Error_Pct, 0) -}) - -test_that("Markers_Tested equals number of markers for complete data", { - f <- write_temp_files() - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - expect_equal(res[ID == "IND_C"]$Markers_Tested, 20L) -}) - -test_that("LOW_MARKERS status assigned when markers_tested < min_markers", { - f <- write_temp_files() - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, - write_txt = FALSE, min_markers = 25L) - expect_true(all(res$Status == "LOW_MARKERS")) - expect_true(all(grepl("^LOW_MARKERS_", res$Correction_Decision))) -}) - -test_that("NA markers reduce Markers_Tested and do not cause errors", { - genos <- make_genos() - genos[ID == "IND_C", M1 := NA_integer_] - genos[ID == "IND_C", M2 := NA_integer_] - f <- write_temp_files(genos = genos) - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - expect_equal(res[ID == "IND_C"]$Markers_Tested, 18L) - expect_equal(res[ID == "IND_C"]$Status, "PASS") -}) - -# ============================================================================== -# 4. Missing parent statuses (MISSING_MALE_PARENT / MISSING_FEMALE_PARENT / -# MISSING_BOTH_PARENTS) -# Note: each test includes make_pedigree() rows so has_geno is never empty, -# and filters res by the specific ID under test [3][4] -# ============================================================================== - -test_that("MISSING_MALE_PARENT status and recommendation are correct", { - ped <- rbind( - make_pedigree(), - data.table(ID = "IND_E", Male_Parent = "0", Female_Parent = "IND_B") - ) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - r <- res[ID == "IND_E"] - expect_equal(r$Status, "MISSING_MALE_PARENT") - expect_equal(r$Correction_Decision, "NONE") - expect_false(is.na(r$Best_Male_Parent)) - expect_true(is.na(r$Best_Female_Parent)) -}) - -test_that("MISSING_FEMALE_PARENT status and recommendation are correct", { - ped <- rbind( - make_pedigree(), - data.table(ID = "IND_E", Male_Parent = "IND_A", Female_Parent = "0") - ) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - r <- res[ID == "IND_E"] - expect_equal(r$Status, "MISSING_FEMALE_PARENT") - expect_equal(r$Correction_Decision, "NONE") - expect_true(is.na(r$Best_Male_Parent)) - expect_false(is.na(r$Best_Female_Parent)) -}) - -test_that("MISSING_BOTH_PARENTS status and recommendations are correct", { - ped <- rbind( - make_pedigree(), - data.table(ID = "IND_E", Male_Parent = "0", Female_Parent = "0") - ) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - r <- res[ID == "IND_E"] - expect_equal(r$Status, "MISSING_BOTH_PARENTS") - expect_equal(r$Correction_Decision, "NONE") - expect_false(is.na(r$Best_Male_Parent)) - expect_false(is.na(r$Best_Female_Parent)) -}) - -test_that("MISSING_* rows preserve 0s in corrected pedigree", { - ped <- rbind( - make_pedigree(), - data.table(ID = "IND_E", Male_Parent = "0", Female_Parent = "IND_B") - ) - f <- write_temp_files(ped = ped) - tmp_dir <- tempfile() - dir.create(tmp_dir) - old_wd <- getwd() - setwd(tmp_dir) - on.exit({ setwd(old_wd); unlink(tmp_dir, recursive = TRUE) }, add = TRUE) - validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - corr <- fread(file.path(tmp_dir, "corrected_pedigree.txt"), - colClasses = "character") - expect_equal(corr[ID == "IND_E"]$Male_Parent, "0") -}) - -test_that("Best_Male_Parent for MISSING_MALE_PARENT is excluded from being the known female", { - ped <- rbind( - make_pedigree(), - data.table(ID = "IND_E", Male_Parent = "0", Female_Parent = "IND_B") - ) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - r <- res[ID == "IND_E"] - # The known female parent should not be suggested as the best male parent - expect_false(r$Best_Male_Parent == "IND_B") -}) - -# ============================================================================== -# 5. FOUNDERS status -# ============================================================================== - -test_that("FOUNDERS status is assigned when ID in founders list with 0 0 parents", { - ped <- rbind( - make_pedigree(), - data.table(ID = "IND_A", Male_Parent = "0", Female_Parent = "0") - ) - founders_file <- tempfile(fileext = ".txt") - fwrite(data.table(ID = "IND_A"), founders_file, - sep = "\t", quote = FALSE, col.names = FALSE) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, - founders_file = founders_file, - verbose = FALSE, - write_txt = FALSE) - founder_row <- res[ID == "IND_A"] - expect_equal(founder_row$Status, "FOUNDERS") - expect_equal(founder_row$Correction_Decision, "NONE") - expect_true(is.na(founder_row$Best_Male_Parent)) - expect_true(is.na(founder_row$Best_Female_Parent)) -}) - -test_that("non-founder rows are still evaluated normally when founders file is supplied", { - ped <- rbind( - make_pedigree(), - data.table(ID = "IND_A", Male_Parent = "0", Female_Parent = "0") - ) - founders_file <- tempfile(fileext = ".txt") - fwrite(data.table(ID = "IND_A"), founders_file, - sep = "\t", quote = FALSE, col.names = FALSE) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, - founders_file = founders_file, - verbose = FALSE, - write_txt = FALSE) - # IND_C has real parents — should still get PASS - expect_equal(res[ID == "IND_C"]$Status, "PASS") -}) - -test_that("0 0 parents NOT in founders list get MISSING_BOTH_PARENTS", { - ped <- rbind( - make_pedigree(), - data.table(ID = "IND_E", Male_Parent = "0", Female_Parent = "0") - ) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - expect_equal(res[ID == "IND_E"]$Status, "MISSING_BOTH_PARENTS") -}) - -test_that("0 0 parents with no founders file gets MISSING_BOTH_PARENTS not FOUNDERS", { - ped <- rbind( - make_pedigree(), - data.table(ID = "IND_E", Male_Parent = "0", Female_Parent = "0") - ) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, - founders_file = NULL, - verbose = FALSE, - write_txt = FALSE) - expect_equal(res[ID == "IND_E"]$Status, "MISSING_BOTH_PARENTS") -}) - -# ============================================================================== -# 6. NO_GENOTYPE_DATA status -# ============================================================================== - -test_that("NO_GENOTYPE_DATA is flagged for progeny absent from genotype file", { - ped <- rbind( - make_pedigree(), - data.table(ID = "GHOST", Male_Parent = "IND_A", Female_Parent = "IND_B") - ) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - ghost_row <- res[ID == "GHOST"] - expect_equal(nrow(ghost_row), 1L) - expect_equal(ghost_row$Status, "NO_GENOTYPE_DATA") - expect_equal(ghost_row$Correction_Decision, "NONE") -}) - -test_that("NO_GENOTYPE_DATA rows have NA for all analysis columns", { - ped <- rbind( - make_pedigree(), - data.table(ID = "GHOST", Male_Parent = "IND_A", Female_Parent = "IND_B") - ) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - ghost_row <- res[ID == "GHOST"] - expect_true(is.na(ghost_row$Mendelian_Error_Pct)) - expect_equal(ghost_row$Markers_Tested, 0L) - expect_true(is.na(ghost_row$Best_Male_Parent)) - expect_true(is.na(ghost_row$Best_Female_Parent)) -}) - -test_that("NO_GENOTYPE_DATA flagged when declared parent is absent from genotype file", { - ped <- rbind( - make_pedigree(), # ensures has_geno is not empty - data.table(ID = "IND_C_GHOST", Male_Parent = "GHOST_DAD", Female_Parent = "IND_B") - ) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - expect_equal(res[ID == "IND_C_GHOST"]$Status, "NO_GENOTYPE_DATA") -}) - -test_that("valid trios still evaluated correctly when ghost rows are present", { - ped <- rbind( - make_pedigree(), - data.table(ID = "GHOST", Male_Parent = "IND_A", Female_Parent = "IND_B") - ) - f <- write_temp_files(ped = ped) - res <- validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - expect_equal(res[ID == "IND_C"]$Status, "PASS") - expect_equal(res[ID == "IND_D"]$Status, "FAIL") -}) - -# ============================================================================== -# 7. Corrected pedigree output -# ============================================================================== - -test_that("corrected_pedigree.txt is written and PASS parents are unchanged", { - f <- write_temp_files() - tmp_dir <- tempfile() - dir.create(tmp_dir) - old_wd <- getwd() - setwd(tmp_dir) - on.exit({ setwd(old_wd); unlink(tmp_dir, recursive = TRUE) }, add = TRUE) - validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - expect_true(file.exists(file.path(tmp_dir, "corrected_pedigree.txt"))) - corr <- fread(file.path(tmp_dir, "corrected_pedigree.txt")) - # IND_C passes — parents must be unchanged - expect_equal(as.character(corr[ID == "IND_C"]$Male_Parent), "IND_A") - expect_equal(as.character(corr[ID == "IND_C"]$Female_Parent), "IND_B") -}) - -test_that("corrected_pedigree.txt sets bad parent to 0 for FAIL trio", { - f <- write_temp_files() - tmp_dir <- tempfile() - dir.create(tmp_dir) - old_wd <- getwd() - setwd(tmp_dir) - on.exit({ setwd(old_wd); unlink(tmp_dir, recursive = TRUE) }, add = TRUE) - validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - corr <- fread(file.path(tmp_dir, "corrected_pedigree.txt"), - colClasses = "character") - # IND_D fails with REMOVE_MALE_PARENT — male should become "0" - expect_equal(corr[ID == "IND_D"]$Male_Parent, "0") - expect_equal(corr[ID == "IND_D"]$Female_Parent, "IND_A") -}) - -test_that("corrected_pedigree.txt preserves 0s for MISSING_* rows", { - ped <- rbind( - make_pedigree(), - data.table(ID = "IND_E", Male_Parent = "0", Female_Parent = "IND_B") - ) - f <- write_temp_files(ped = ped) - tmp_dir <- tempfile() - dir.create(tmp_dir) - old_wd <- getwd() - setwd(tmp_dir) - on.exit({ setwd(old_wd); unlink(tmp_dir, recursive = TRUE) }, add = TRUE) - validate_pedigree(f$ped, f$genos, verbose = FALSE, write_txt = FALSE) - corr <- fread(file.path(tmp_dir, "corrected_pedigree.txt"), - colClasses = "character") - # MISSING_MALE_PARENT — male stays "0", female unchanged - expect_equal(corr[ID == "IND_E"]$Male_Parent, "0") - expect_equal(corr[ID == "IND_E"]$Female_Parent, "IND_B") -}) - -# ============================================================================== -# 8. write_txt / output file -# ============================================================================== - -test_that("write_txt = TRUE writes output file with correct row count", { - f <- write_temp_files() - out_file <- tempfile(fileext = ".txt") - validate_pedigree(f$ped, f$genos, verbose = FALSE, - write_txt = TRUE, output_filename = out_file) - expect_true(file.exists(out_file)) - written <- fread(out_file) - expect_equal(nrow(written), 2L) -}) - -test_that("write_txt = FALSE does not create output file", { - f <- write_temp_files() - out_file <- tempfile(fileext = ".txt") - validate_pedigree(f$ped, f$genos, verbose = FALSE, - write_txt = FALSE, output_filename = out_file) - expect_false(file.exists(out_file)) -}) - -test_that("output file contains correct number of rows when ghost rows present", { - ped <- rbind( - make_pedigree(), - data.table(ID = "GHOST", Male_Parent = "IND_A", Female_Parent = "IND_B") - ) - f <- write_temp_files(ped = ped) - out_file <- tempfile(fileext = ".txt") - validate_pedigree(f$ped, f$genos, verbose = FALSE, - write_txt = TRUE, output_filename = out_file) - written <- fread(out_file) - # 2 valid trios + 1 ghost = 3 rows total - expect_equal(nrow(written), 3L) -})