Source code for gnomad.utils.constraint.metrics

"""
Functions to compute constraint metrics.

Covers pLI, the o/e confidence interval, z-scores, flags, and GENCODE annotations.
"""

import copy
from typing import Dict, List, Optional, Tuple, Union

import hail as hl
from hail.utils.misc import divide_null, new_temp_file

from gnomad.utils.constraint.constants import DEFAULT_GENCODE_ANNOTATIONS


[docs]def compute_pli( ht: hl.Table, obs_expr: hl.expr.Int64Expression, exp_expr: hl.expr.Float64Expression, expected_values: Optional[Dict[str, float]] = None, min_diff_convergence: float = 0.001, ) -> hl.StructExpression: """ Compute the pLI score using the observed and expected variant counts. Full details on pLI can be found in the ExAC paper: Lek, M., Karczewski, K., Minikel, E. et al. Analysis of protein-coding genetic variation in 60,706 humans. Nature 536, 285–291 (2016). pLI is the probability of being loss-of-function intolerant, and this function computes that probability using the expectation-maximization (EM) algorithm. We assume a 3 state model, where each gene fits into one of three categories with respect loss-of-function variation sensitivity: - Null: where protein truncating variation is completely tolerated by natural selection. - Recessive (Rec): where heterozygous pLoFs are tolerated but homozygous pLoFs are not. - Haploinsufficient (LI): where heterozygous pLoFs are not tolerated. The function requires the expected amount of loss-of-function depletion for each of these states. The default provided is based on the observed depletion of protein-truncating variation in the Blekhman autosomal recessive and ClinGen dosage sensitivity gene sets (Supplementary Information Table 12 of the above reference): - Null: 1.0, assume tolerant genes have the expected amount of truncating variation. - Rec: 0.463, derived from the empirical mean observed/expected rate of truncating variation for recessive disease genes (0.463). - LI: 0.089, derived from the empirical mean observed/expected rate of truncating variation for severe haploinsufficient genes. The output StructExpression will include the following annotations: - pLI: Probability of loss-of-function intolerance; probability that transcript falls into distribution of haploinsufficient genes. - pNull: Probability that transcript falls into distribution of unconstrained genes. - pRec: Probability that transcript falls into distribution of recessive genes. :param ht: Input Table containing `obs_expr` and `exp_expr`. :param obs_expr: Expression for the number of observed variants on each gene or transcript in `ht`. :param exp_expr: Expression for the number of expected variants on each gene or transcript in `ht`. :param expected_values: Dictionary containing the expected values for 'Null', 'Rec', and 'LI' to use as starting values. :param min_diff_convergence: Minimum iteration change in LI to consider the EM model convergence criteria as met. Default is 0.001. :return: StructExpression for pLI scores. """ if expected_values is None: expected_values = {"Null": 1.0, "Rec": 0.463, "LI": 0.089} # Set up initial values. last_pi = {k: 0 for k in expected_values.keys()} pi = {k: 1 / len(expected_values.keys()) for k in expected_values.keys()} dpois_expr = { k: hl.or_missing( exp_expr > 0, hl.dpois(obs_expr, exp_expr * expected_values[k]) ) for k in pi } _ht = ht.select(dpois=dpois_expr) # Checkpoint the temp HT because it will need to be aggregated several times. _ht = _ht.checkpoint(new_temp_file(prefix="compute_pli", extension="ht")) # Calculate pLI scores. while abs(pi["LI"] - last_pi["LI"]) > min_diff_convergence: last_pi = copy.deepcopy(pi) pi_expr = {k: v * _ht.dpois[k] for k, v in pi.items()} row_sum_expr = hl.sum([pi_expr[k] for k in pi]) pi_expr = {k: pi_expr[k] / row_sum_expr for k, v in pi.items()} pi = _ht.aggregate({k: hl.agg.mean(pi_expr[k]) for k in pi.keys()}) # Get expression for pLI scores. pli_expr = {k: v * dpois_expr[k] for k, v in pi.items()} row_sum_expr = hl.sum([pli_expr[k] for k in pi]) return hl.struct(**{f"p{k}": pli_expr[k] / row_sum_expr for k in pi.keys()})
def _oe_ci_discretized_poisson( obs_expr: hl.expr.Int64Expression, exp_expr: hl.expr.Float64Expression, alpha: float = 0.05, ) -> hl.expr.StructExpression: """ Compute OE confidence interval via discretized Poisson CDF. Sweeps the OE ratio parameter over [0, 2) in steps of 0.001, evaluates ``dpois(obs, exp * x)`` at each point, normalises the cumulative sum, and reads off the bounds at ``alpha`` and ``1 - alpha``. :param obs_expr: Observed variant count expression. :param exp_expr: Expected variant count expression. :param alpha: Significance level. Default is 0.05. :return: Struct with ``lower`` and ``upper`` bounds. """ # Set up range between 0 and 2. range_expr = hl.range(0, 2000).map(lambda x: hl.float64(x) / 1000) range_dpois_expr = range_expr.map(lambda x: hl.dpois(obs_expr, exp_expr * x)) # Compute cumulative density function of the Poisson distribution density. cumulative_dpois_expr = hl.cumulative_sum(range_dpois_expr) max_cumulative_dpois_expr = cumulative_dpois_expr[-1] norm_dpois_expr = cumulative_dpois_expr.map(lambda x: x / max_cumulative_dpois_expr) # Extract the value of the varying parameter within specified range. lower_idx_expr = hl.argmax( norm_dpois_expr.map(lambda x: hl.or_missing(x < alpha, x)) ) upper_idx_expr = hl.argmin( norm_dpois_expr.map(lambda x: hl.or_missing(x > 1 - alpha, x)) ) return hl.struct( lower=hl.if_else(obs_expr > 0, range_expr[lower_idx_expr], 0), upper=range_expr[upper_idx_expr], ) def _oe_ci_gamma( obs_expr: hl.expr.Int64Expression, exp_expr: hl.expr.Float64Expression, alpha: float = 0.05, ) -> hl.expr.StructExpression: """ Compute OE confidence interval using the Gamma distribution. Uses Hail's ``hl.qgamma`` quantile function. :param obs_expr: Observed variant count expression. :param exp_expr: Expected variant count expression. :param alpha: Significance level. Default is 0.05. :return: Struct with ``lower`` and ``upper`` bounds. """ shape = obs_expr + hl.literal(1.0) scale = divide_null(hl.literal(1.0), exp_expr) return hl.struct( lower=hl.qgamma(hl.literal(alpha), shape, scale), upper=hl.qgamma(hl.literal(1.0 - alpha), shape, scale), )
[docs]def oe_confidence_interval( obs_expr: hl.expr.Int64Expression, exp_expr: hl.expr.Float64Expression, alpha: float = 0.05, method: str = "gamma", ) -> hl.expr.StructExpression: """ Compute a confidence interval around the observed/expected ratio. Two methods are available: - ``"gamma"`` (default): uses ``hl.qgamma`` to compute exact quantiles of the Gamma posterior. Fast and precise. - ``"poisson"``: sweeps a discretized Poisson likelihood over the OE parameter space [0, 2). Retained for backwards compatibility. .. note:: The default changed from the discretized Poisson computation to ``"gamma"``, so calls that do not pass ``method`` return different bounds than before. ``"gamma"`` requires Hail >= 0.2.137, which added ``hl.qgamma``; on older Hail versions pass ``method="poisson"``. :param obs_expr: Observed variant count expression. :param exp_expr: Expected variant count expression. :param alpha: Significance level for the confidence interval. Default is 0.05 (90% CI). :param method: CI method — ``"gamma"`` or ``"poisson"``. Default is ``"gamma"``. :return: Struct with ``lower`` and ``upper`` bounds. """ if method == "gamma": if not hasattr(hl, "qgamma"): raise RuntimeError( "method='gamma' requires hl.qgamma, available in Hail >= 0.2.137. " "Use method='poisson' or upgrade Hail." ) return _oe_ci_gamma(obs_expr, exp_expr, alpha) elif method == "poisson": return _oe_ci_discretized_poisson(obs_expr, exp_expr, alpha) else: raise ValueError(f"Unknown CI method: {method!r}. Use 'gamma' or 'poisson'.")
[docs]def calculate_raw_z_score( obs_expr: hl.expr.Int64Expression, exp_expr: hl.expr.Float64Expression, ) -> hl.expr.Float64Expression: """ Compute the signed raw z-score using observed and expected variant counts. The raw z-scores are positive when the transcript had fewer variants than expected, and are negative when transcripts had more variants than expected. :param obs_expr: Observed variant count expression. :param exp_expr: Expected variant count expression. :return: Raw z-score expression. """ chisq_expr = divide_null((obs_expr - exp_expr) ** 2, exp_expr) return hl.sqrt(chisq_expr) * hl.if_else(obs_expr > exp_expr, -1, 1)
[docs]def get_constraint_flags( exp_expr: hl.expr.Float64Expression, raw_z_expr: hl.expr.Float64Expression, raw_z_lower_threshold: Optional[Union[float, hl.expr.Float64Expression]] = -5.0, raw_z_upper_threshold: Optional[Union[float, hl.expr.Float64Expression]] = 5.0, flag_postfix: str = "", ) -> Dict[str, hl.expr.Expression]: """ Determine the constraint flags that define why constraint will not be calculated. Flags which are added: - "no_exp_{flag_postfix}" - for genes that have missing or zero expected variants. - "outlier_{flag_postfix}" - for genes that are raw z-score outliers: (`raw_z_expr` < `raw_z_lower_threshold`) or (`raw_z_expr` > `raw_z_upper_threshold`). :param exp_expr: Expression for the expected variant counts of pLoF, missense, or synonymous variants. :param raw_z_expr: Expression for the signed raw z-score of pLoF, missense, or synonymous variants. :param raw_z_lower_threshold: Lower threshold for the raw z-score. When `raw_z_expr` is less than this threshold it is considered an 'outlier'. Default is -5.0. :param raw_z_upper_threshold: Upper threshold for the raw z-score. When `raw_z_expr` is greater than this threshold it is considered an 'outlier'. Default is 5.0. :param flag_postfix: Postfix to add to the end of the constraint flag names. :return: Dictionary containing expressions for constraint flags. """ outlier_expr = False if raw_z_lower_threshold is not None: outlier_expr |= hl.or_else(raw_z_expr < raw_z_lower_threshold, False) if raw_z_upper_threshold is not None: outlier_expr |= hl.or_else(raw_z_expr > raw_z_upper_threshold, False) if flag_postfix: flag_postfix = f"_{flag_postfix}" constraint_flags = { f"no_exp{flag_postfix}": hl.or_else(exp_expr <= 0, True), f"outlier{flag_postfix}": hl.or_else(outlier_expr, False), } return constraint_flags
[docs]def calculate_raw_z_score_sd( raw_z_expr: hl.expr.Float64Expression, flag_expr: hl.expr.StringExpression, mirror_neg_raw_z: bool = True, ) -> hl.expr.Expression: """ Calculate the standard deviation of the raw z-score. When using `mirror_neg_raw_z` is True, all the negative raw z-scores (defined by `raw_z_expr`) are combined with those same z-scores multiplied by -1 (to create a mirrored distribution). :param raw_z_expr: Expression for the raw z-score. :param flag_expr: Expression for the constraint flags. z-score will not be calculated if flags are present. :param mirror_neg_raw_z: Whether the standard deviation should be computed using a mirrored distribution of negative `raw_z_expr`. :return: StructExpression containing standard deviation of the raw z-score and the z-score. """ filter_expr = (hl.len(flag_expr) == 0) & hl.is_defined(raw_z_expr) if mirror_neg_raw_z: filter_expr &= raw_z_expr < 0 sd_expr = hl.agg.explode( lambda x: hl.agg.stats(x), [raw_z_expr, -raw_z_expr] ).stdev else: sd_expr = hl.agg.stats(raw_z_expr).stdev return hl.agg.filter(filter_expr, sd_expr)
[docs]def add_gencode_transcript_annotations( ht: hl.Table, gencode_ht: hl.Table, annotations: Union[List[str], Tuple[str, ...]] = DEFAULT_GENCODE_ANNOTATIONS, remove_y_par: bool = True, ) -> hl.Table: """ Add GENCODE annotations to Table based on transcript id. In addition to the annotations specified by ``annotations``, the following computed annotations are always added: - chromosome - cds_length - num_coding_exons :param ht: Input Table. :param gencode_ht: Table with GENCODE annotations. :param annotations: GENCODE annotations to add. Default is ``DEFAULT_GENCODE_ANNOTATIONS``. :param remove_y_par: Whether to remove features for the Y chromosome PAR regions. Default is True because the Y chromosome PAR regions are typically not included in the constraint calculations and both chrX and chrY will have the same 'transcript_id' field for these regions. This parameter can only be True if ``gencode_ht`` includes a 'transcript_id_version' field because Y_PAR (or PAR_Y, depending on the GENCODE release) is included in the version of the transcript, which has been stripped from the 'transcript_id' field. :return: Table with transcript annotations from GENCODE added. """ annotations = list(annotations) if remove_y_par and "transcript_id_version" not in gencode_ht.row: raise ValueError( "remove_y_par is True but 'transcript_id_version' is not in gencode_ht" ) if remove_y_par: gencode_ht = gencode_ht.filter( ~gencode_ht.transcript_id_version.endswith("Y_PAR") & ~gencode_ht.transcript_id_version.endswith("PAR_Y") ) gencode_ht = gencode_ht.annotate( length=gencode_ht.interval.end.position - gencode_ht.interval.start.position + 1, chromosome=gencode_ht.interval.start.contig, start_position=gencode_ht.interval.start.position, end_position=gencode_ht.interval.end.position, ) # Add transcript annotations to input Table. annotations_to_add = set(annotations + ["chromosome", "transcript_id"]) gencode_transcript_ht = ( gencode_ht.filter(gencode_ht.feature == "transcript") .select(*annotations_to_add) .key_by("transcript_id") .drop("interval") ) # Obtain CDS annotations from GENCODE file and calculate CDS length and # number of exons. annotations_to_add = set(annotations + ["chromosome", "transcript_id", "length"]) gencode_cds_ht = ( gencode_ht.filter(gencode_ht.feature == "CDS") .select(*annotations_to_add) .key_by("transcript_id") .drop("interval") ) annotations_to_add.remove("length") gencode_cds_ht = gencode_cds_ht.group_by("transcript_id").aggregate( cds_length=hl.agg.sum(gencode_cds_ht.length), num_coding_exons=hl.agg.count(), ) # Join the transcript and CDS annotations. gencode_transcript_ht = gencode_transcript_ht.join(gencode_cds_ht).checkpoint( new_temp_file(prefix="gencode", extension="ht") ) # Add GENCODE annotations to input Table. ht = ht.annotate(**gencode_transcript_ht[ht.transcript]) return ht