Source code for gnomad.utils.constraint.counts

"""Functions to count observed and possible variants and sum them by group."""

import logging
from typing import Dict, List, Optional, Tuple, Union

import hail as hl

from gnomad.utils.constraint.constants import DEFAULT_FIELDS_TO_SUM

logging.basicConfig(
    format="%(asctime)s (%(name)s %(lineno)s): %(message)s",
    datefmt="%m/%d/%Y %I:%M:%S %p",
)
logger = logging.getLogger("constraint_utils")
logger.setLevel(logging.INFO)


def _resolve_row_annotation_expr(
    t: Optional[Union[hl.Table, hl.MatrixTable]] = None,
    annotation_name: Optional[str] = None,
    expr: Optional[hl.expr.Expression] = None,
    expr_param_name: Optional[str] = None,
) -> hl.expr.Expression:
    """
    Get a row annotation from a Table/MatrixTable, or return a provided expression.

    Provides a consistent pattern for functions that accept either an explicit
    Hail expression or fall back to a well-known field on a Table. This avoids
    duplicating "resolve expr or look it up on ht" logic across callers.

    If ``expr`` is provided it is returned directly. Otherwise, ``t`` and
    ``annotation_name`` must both be supplied and the named row field is looked up
    on ``t``.

    Example usage inside a public function::

        def my_function(
            ht: hl.Table,
            freq_expr: Optional[hl.expr.StructExpression] = None,
        ) -> ...:
            freq_expr = _resolve_row_annotation_expr(
                t=ht,
                annotation_name="freq",
                expr=freq_expr,
                expr_param_name="freq_expr",
            )
            # freq_expr is now guaranteed to be a valid expression —
            # either the caller's explicit value or ht.freq.

    :param t: Input Table or MatrixTable to look up ``annotation_name`` on. Required
        when ``expr`` is None.
    :param annotation_name: Name of the field to retrieve from ``t``. Required when
        ``expr`` is None.
    :param expr: Expression to return directly. When provided, ``t`` and
        ``annotation_name`` are ignored.
    :param expr_param_name: Human-readable name for ``expr``, used in log/error
        messages. Defaults to "expr".
    :return: The resolved Hail expression.
    """
    if expr is None and (t is None or annotation_name is None):
        raise ValueError("Either 't' and 'annotation_name' or 'expr' must be provided.")

    expr_param_name = expr_param_name or "expr"
    if expr is None and annotation_name in t.row:
        logger.info(
            "%s was not provided, using '%s'.", expr_param_name, annotation_name
        )
        expr = t[annotation_name]
    elif expr is None:
        raise ValueError(
            f"{expr_param_name} was not provided and '{annotation_name}' is "
            f"not present in the input Table or MatrixTable."
        )

    return expr


[docs]def variant_observed_expr( freq_expr: Optional[ Union[hl.expr.StructExpression, hl.expr.ArrayExpression] ] = None, ht: Optional[hl.Table] = None, singleton: bool = False, max_af: Optional[float] = None, count_missing: bool = False, ) -> hl.expr.Int32Expression: """ Return 1 if a variant meets frequency criteria, 0 otherwise. One of ``ht`` or ``freq_expr`` must be provided. If ``freq_expr`` is not provided, ``ht.freq`` is used. When ``freq_expr`` is an ArrayExpression, the first element is used. The returned count is 1 when: - ``singleton`` is True and ``freq_expr.AC == 1``. - ``max_af`` is not None and ``freq_expr.AC > 0`` and ``freq_expr.AF <= max_af``. - Neither ``singleton`` nor ``max_af`` is set but ``freq_expr`` is available: ``freq_expr.AC > 0``. - Neither ``singleton`` nor ``max_af`` is set and no ``freq_expr`` is available: unconditionally 1. :param ht: Input Table. Used to look up ``freq`` when ``freq_expr`` is None. :param freq_expr: StructExpression (or ArrayExpression of structs) with ``AC`` and ``AF`` fields. When an array, ``freq_expr[0]`` is used. :param singleton: Count only singletons (AC == 1). When True, ``max_af`` is ignored. Default is False. :param max_af: Maximum allele frequency threshold. Default is None (no cutoff). :param count_missing: Whether to count a variant whose count expression is missing (e.g. frequency is None). True counts it as 1, False as 0. Default is False. :return: Int32Expression equal to 0 or 1. """ if ht is None and freq_expr is None: raise ValueError("Either ht or freq_expr must be provided.") if freq_expr is None and (max_af is not None or singleton or "freq" in ht.row): freq_expr = _resolve_row_annotation_expr(ht, "freq", freq_expr, "freq_expr") if isinstance(freq_expr, hl.expr.ArrayExpression): freq_expr = freq_expr[0] if singleton: count_var = freq_expr.AC == 1 elif max_af is not None: count_var = (freq_expr.AC > 0) & (freq_expr.AF <= max_af) elif freq_expr is not None: count_var = freq_expr.AC > 0 else: count_var = True return hl.int(hl.or_else(count_var, count_missing))
[docs]def variant_observed_and_possible_expr( freq_expr: hl.ArrayExpression, max_af: Optional[float] = None, use_possible_adj: bool = True, ) -> hl.expr.StructExpression: """ Return per-variant observed and possible count expressions. For each element of the frequency array, ``observed_variants`` is 1 when the variant meets the frequency criteria (AC > 0, and AF <= ``max_af`` if set) and 0 otherwise. ``possible_variants`` uses the same criteria, except that a missing frequency counts as possible: a variant absent from the dataset could still have been observed. A variant with a defined frequency that fails the criteria (AC == 0, or AF > ``max_af``) is therefore 0 in both, so it is excluded from the denominator as well as the numerator. With ``max_af`` set, this removes common variants from the possible counts. When ``freq_expr`` itself is missing (e.g. the variant is not in the dataset), ``observed_variants`` is missing and ``possible_variants`` is 1, or missing when ``use_possible_adj`` is False. When ``use_possible_adj`` is True (default), the possible count is a scalar derived from the first (adj) element of the frequency array. When False, it is an array with one entry per array element, matching the shape of ``observed_variants``. :param freq_expr: Array of frequency structs with ``AC`` and ``AF`` fields. :param max_af: Maximum allele frequency threshold. Variants above it are excluded from both counts. Default is None (no cutoff). :param use_possible_adj: If True, compute possible count from only the adj (first) frequency element. If False, compute per-downsampling. Default is True. :return: Struct with ``observed_variants`` (array of int) and ``possible_variants`` (int if ``use_possible_adj``, else array of int). """ pos_expr = hl.array([freq_expr[0]]) if use_possible_adj else freq_expr pos_expr = pos_expr.map( lambda x: variant_observed_expr(freq_expr=x, max_af=max_af, count_missing=True) ) pos_expr = pos_expr[0] if use_possible_adj else pos_expr return hl.struct( observed_variants=freq_expr.map( lambda x: variant_observed_expr(freq_expr=x, max_af=max_af) ), possible_variants=pos_expr, )
[docs]def weighted_sum_agg_expr( expr: Union[hl.expr.ArrayNumericExpression, hl.expr.NumericExpression], weight_expr: Union[hl.expr.ArrayNumericExpression, hl.expr.NumericExpression], ) -> Union[hl.expr.ArrayExpression, hl.expr.NumericExpression]: """ Return the weighted aggregate sum of ``expr`` weighted by ``weight_expr``. Both parameters may be scalar or array numeric expressions: - Both arrays: elements are multiplied pairwise and summed per-element across rows (``hl.agg.array_sum``). - Both scalars: standard ``hl.agg.sum(expr * weight_expr)``. - Mixed: the scalar is broadcast across the array elements and summed per-element across rows. :param expr: Numeric expression (scalar or array) to be weighted. :param weight_expr: Numeric expression (scalar or array) to weight by. :return: Weighted aggregate sum expression. """ expr_is_array = isinstance(expr, hl.expr.ArrayNumericExpression) weight_is_array = isinstance(weight_expr, hl.expr.ArrayNumericExpression) if expr_is_array and weight_is_array: return hl.agg.array_sum(hl.zip(expr, weight_expr).map(lambda x: x[0] * x[1])) elif not expr_is_array and not weight_is_array: return hl.agg.sum(expr * weight_expr) else: return hl.agg.array_sum(expr * weight_expr)
[docs]def counts_agg_expr( freq_expr: Optional[ Union[hl.expr.ArrayExpression, hl.expr.StructExpression] ] = None, ht: Optional[hl.Table] = None, count_singletons: bool = False, max_af: Optional[float] = None, count_missing: bool = False, ) -> hl.expr.StructExpression: """ Return an aggregation expression for variant and singleton counts. Aggregates per-variant counts (via ``variant_observed_expr``) across rows. Each variant contributes 0 or 1 to the count based on its frequency metadata (AC, AF). The result is a struct with ``variant_count`` and, when ``count_singletons`` is True, ``singleton_count``. One of ``freq_expr`` or ``ht`` must be provided. If ``freq_expr`` is not provided, ``ht.freq`` is used as the fallback. The shape of ``freq_expr`` controls whether the output counts are scalars or arrays: - **StructExpression** (single frequency entry with ``AC`` / ``AF``): returns scalar ``variant_count`` and ``singleton_count``. - **ArrayExpression** (array of frequency structs, e.g. one per downsampling): returns array-valued counts where each element corresponds to a position in the input array. Internally the array is mapped through ``variant_observed_expr`` and summed element-wise with ``hl.agg.array_sum``. When ``max_af`` is set, only variants with ``AF <= max_af`` and ``AC > 0`` are counted. Singleton counting (``count_singletons=True``) counts only variants with ``AC == 1``, independent of the ``max_af`` filter. :param freq_expr: Frequency expression — an ArrayExpression of structs or a single StructExpression with ``AC`` and ``AF`` fields. If None, falls back to ``ht.freq``. :param ht: Input Table. Used to look up ``freq`` when ``freq_expr`` is None. :param count_singletons: If True, include a ``singleton_count`` field in the result. Default is False. :param max_af: Maximum allele frequency threshold. Variants with ``AF > max_af`` are excluded from ``variant_count``. Does not affect ``singleton_count``. Default is None (no cutoff). :param count_missing: Whether to count a variant whose frequency is missing. True counts it as 1, False as 0. Default is False. :return: Aggregation StructExpression with ``variant_count`` (and optionally ``singleton_count``). Values are scalars when ``freq_expr`` is a StructExpression, or arrays when it is an ArrayExpression. """ if ht is None and freq_expr is None: raise ValueError("Either ht or freq_expr must be provided.") freq_expr = _resolve_row_annotation_expr(ht, "freq", freq_expr, "freq_expr") is_struct = isinstance(freq_expr, hl.expr.StructExpression) if is_struct: freq_expr = hl.array([freq_expr]) params = {"variant_count": {"singleton": False, "max_af": max_af}} if count_singletons: params["singleton_count"] = {"singleton": True} count_expr = { k: freq_expr.map( lambda x: variant_observed_expr( freq_expr=x, **p, count_missing=count_missing ) ) for k, p in params.items() } agg_expr = {k: hl.agg.array_sum(v) for k, v in count_expr.items()} agg_expr = {k: agg_expr[k][0] for k in agg_expr} if is_struct else agg_expr return hl.struct(**agg_expr)
[docs]def count_observed_and_possible_by_group( ht: hl.Table, possible_expr: hl.expr.Int32Expression, observed_expr: hl.expr.ArrayExpression, additional_grouping: Union[List[str], Tuple[str, ...]] = ("methylation_level",), partition_hint: int = 100, weight_exprs: Optional[ Union[ List[str], Dict[str, Union[hl.expr.ArrayExpression, hl.expr.NumericExpression]], ] ] = None, additional_agg_sum_exprs: Optional[ Union[ List[str], Dict[str, Union[hl.expr.ArrayExpression, hl.expr.NumericExpression]], ] ] = None, ) -> hl.Table: """ Aggregate observed and possible variant counts by substitution context group. Groups rows by ``context``, ``ref``, ``alt``, and any fields in ``additional_grouping``, then sums observed and possible counts within each group. :param ht: Input Table with ``context``, ``ref``, ``alt`` fields and any fields named in ``additional_grouping``. :param possible_expr: Per-variant possible count (scalar). :param observed_expr: Per-variant observed count (array, one element per array element of the frequency array). :param additional_grouping: Field names to append to the base ``(context, ref, alt)`` grouping. Default is ``("methylation_level",)``. :param partition_hint: Target number of partitions for the ``group_by``. Default is 100. :param weight_exprs: Weighted sums of ``possible_expr`` to include. Pass field names (looked up on ``ht``) or a dict mapping output names to weight expressions. Each produces ``weighted_sum_agg_expr(possible_expr, weight)``. :param additional_agg_sum_exprs: Extra fields to sum alongside observed/possible. Pass field names (looked up on ``ht``) or a dict mapping output names to expressions. Arrays use ``hl.agg.array_sum``; scalars use ``hl.agg.sum``. :return: Grouped Table with ``observed_variants``, ``possible_variants``, and any weighted/additional sum fields. """ if isinstance(weight_exprs, list): weight_exprs = {k: ht[k] for k in weight_exprs} if isinstance(additional_agg_sum_exprs, list): additional_agg_sum_exprs = {k: ht[k] for k in additional_agg_sum_exprs} weight_exprs = weight_exprs or {} additional_agg_sum_exprs = additional_agg_sum_exprs or {} # Build the grouping struct for the variant count aggregation. grouping = hl.struct(context=ht.context, ref=ht.ref, alt=ht.alt) grouping = grouping.annotate( **{g: ht[g] for g in additional_grouping if g not in grouping} ) logger.info( "The following annotations will be used to group the input Table rows when" " counting variants: %s.", ", ".join(grouping.keys()), ) agg_expr = { "observed_variants": hl.agg.array_sum(observed_expr), "possible_variants": hl.agg.sum(possible_expr), } # Update the possible variant count aggregation expression to include weighted sums # of possible variant counts. agg_expr.update( {k: weighted_sum_agg_expr(possible_expr, v) for k, v in weight_exprs.items()} ) # Get sum aggregation expressions for requested fields. agg_expr.update(_build_sum_agg_struct(exprs_to_sum=additional_agg_sum_exprs)) # Apply each variant count aggregation in `agg_expr` to get counts for all # combinations of `grouping`. ht = ht.group_by(**grouping).partition_hint(partition_hint).aggregate(**agg_expr) return ht
def _build_sum_agg_struct( fields_to_sum: Optional[List[str]] = None, exprs_to_sum: Optional[ Union[hl.expr.StructExpression, Dict[str, hl.expr.NumericExpression]] ] = None, t: Optional[Union[hl.Table, hl.expr.StructExpression]] = None, ) -> hl.expr.StructExpression: """ Return an aggregation expression to sum fields or expressions in a Table/StructExpression. The aggregation expression is a struct with the sum or array_sum of the fields or expressions provided in `fields_to_sum` or `exprs_to_sum`. :param fields_to_sum: List of fields to sum. Must provide ``t`` if using this. Default is None. :param exprs_to_sum: Dictionary of expressions to sum. Default is None. :param t: Optional Table or StructExpression to get `fields_to_sum` from. Default is None. :return: Aggregation expression to sum fields or expressions in the Table. """ if fields_to_sum is None and exprs_to_sum is None: raise ValueError("Either 'fields_to_sum' or 'exprs_to_sum' must be provided.") if fields_to_sum is not None and t is None: raise ValueError("t must be provided if 'fields_to_sum' is provided.") exprs_to_sum = exprs_to_sum or {} exprs_to_sum = hl.struct(**exprs_to_sum, **{f: t[f] for f in fields_to_sum or []}) return hl.struct( **{ k: ( hl.agg.array_sum(v) if isinstance(v, hl.ArrayExpression) else hl.agg.sum(v) ) for k, v in exprs_to_sum.items() } )
[docs]def aggregate_constraint_metrics_expr( t: Union[hl.Table, hl.StructExpression], fields_to_sum: Union[List[str], Tuple[str, ...]] = DEFAULT_FIELDS_TO_SUM, additional_exprs_to_sum: Optional[Dict[str, hl.expr.Expression]] = None, ) -> hl.expr.StructExpression: """ Get an aggregation expression for the sum of specified fields and expressions. An aggregate sum or array sum is created for each field in ``fields_to_sum``. :param t: Input Table or StructExpression. :param fields_to_sum: Fields in ``t`` to get an aggregate sum expression for. Default is ``DEFAULT_FIELDS_TO_SUM``. :param additional_exprs_to_sum: Dictionary of additional expressions to get an aggregate sum expression for. Field names are the keys and expressions are the values. Default is None. :return: StructExpression with the sum of each field and expression. """ return _build_sum_agg_struct( fields_to_sum=list(fields_to_sum), exprs_to_sum=additional_exprs_to_sum, t=t, )