import math
from collections import Counter
from typing import Dict, Optional
from .feature_decorators import FeatureType, FeatureDomain
[docs]
class NGramCounter:
"""A stateful n-gram counter that accumulates counts across multiple sequences."""
[docs]
def __init__(self):
"""Initialize an empty n-gram counter."""
self.ngram_counts = {}
self.ngram_counts_by_order = {}
self._total_tokens = None
self._freq_spec = None
self._count_values = None
[docs]
def count_ngrams(self, tokens: list, max_order: int = 5) -> None:
"""Count n-grams in the token sequence up to max_order length.
Parameters
----------
tokens : list
List of tokens to count n-grams from
max_order : int, optional
Maximum n-gram length to count (default: 5)
"""
# Clear previous counts and caches
self.ngram_counts = {}
self.ngram_counts_by_order = {}
self._total_tokens = None
self._freq_spec = None
self._count_values = None
# Count n-grams for each possible length up to max_order
max_length = min(max_order, len(tokens))
for length in range(1, max_length + 1):
counts_for_length = self.ngram_counts_by_order.setdefault(length, {})
for i in range(len(tokens) - length + 1):
ngram = tuple(tokens[i : i + length])
self.ngram_counts[ngram] = self.ngram_counts.get(ngram, 0) + 1
counts_for_length[ngram] = counts_for_length.get(ngram, 0) + 1
[docs]
def reset(self) -> None:
"""Reset the n-gram counter to empty."""
self.ngram_counts = {}
self.ngram_counts_by_order = {}
self._total_tokens = None
self._freq_spec = None
self._count_values = None
[docs]
def get_counts(self, n: Optional[int] = None) -> Dict:
"""Get the current n-gram counts.
Parameters
----------
n : int, optional
If provided, only return counts for n-grams of this length.
If None, return counts for all n-gram lengths.
Returns
-------
dict
Dictionary mapping each n-gram to its count
"""
if n is None:
return self.ngram_counts.copy()
return {k: v for k, v in self.ngram_counts.items() if len(k) == n}
@property
def total_tokens(self) -> int:
"""Underlying unigram token count of the sequence."""
if self._total_tokens is None:
unigram_counts = self.ngram_counts_by_order.get(1, {})
self._total_tokens = sum(unigram_counts.values())
return self._total_tokens
@property
def freq_spec(self) -> dict:
"""Frequency spectrum of n-gram counts."""
if self._freq_spec is None:
self._freq_spec = Counter(self.ngram_counts.values())
return self._freq_spec
@property
def count_values(self) -> list:
"""List of all n-gram counts."""
if self._count_values is None:
self._count_values = list(self.ngram_counts.values())
return self._count_values
@property
def yules_k(self) -> float:
"""Yule's K measure of m-type repetitiveness.
This lexical-diversity feature is calculated from the frequency spectrum
of m-types in the melody. Higher values indicate that a smaller set of
m-types is repeated more often, whereas lower values indicate a more even
or varied m-type vocabulary.
Citation
--------
Yule (1944)
"""
try:
if len(self.count_values) <= 1:
import warnings
warnings.warn("Cannot calculate Yule's K when distinct n-gram types <= 1")
return float("nan")
n = self.total_tokens
if n == 0:
return float("nan")
s1 = sum(self.count_values)
s2 = sum(x * x for x in self.count_values)
if s1 == 0:
return float("nan")
return (10000 * (s2 - s1)) / (s1 * s1)
except Exception as e:
import warnings
warnings.warn(f"Error calculating Yule's K: {str(e)}")
return float("nan")
@property
def simpsons_d(self) -> float:
"""Simpson's D measure of m-type concentration.
Simpson's D is calculated from squared m-type frequencies. Higher values
indicate a greater probability that two sampled tokens belong to the same
m-type, and therefore a more concentrated or repetitive m-type vocabulary.
Citation
--------
Simpson (1949)
"""
try:
if len(self.count_values) <= 1:
import warnings
warnings.warn(
"Cannot calculate Simpson's D when distinct n-gram types <= 1"
)
return float("nan")
n = self.total_tokens
if n == 0:
return float("nan")
s2 = sum(x * x for x in self.count_values)
return s2 / (n * n)
except Exception as e:
import warnings
warnings.warn(f"Error calculating Simpson's D: {str(e)}")
return float("nan")
@property
def sichels_s(self) -> float:
"""The proportion of m-types that occur exactly twice.
Sichel's S is the number of distinct m-types with frequency two divided
by the total number of distinct m-types. Higher values indicate that more
of the melody's m-type vocabulary consists of types that recur once.
Citation
--------
Sichel (1975)
"""
try:
if len(self.count_values) <= 1:
import warnings
warnings.warn("Cannot calculate Sichel's S when distinct n-gram types <= 1")
return float("nan")
v = len(self.ngram_counts)
if v == 0:
return float("nan")
v2 = self.freq_spec.get(2, 0)
return v2 / v if v > 0 else float("nan")
except Exception as e:
import warnings
warnings.warn(f"Error calculating Sichel's S: {str(e)}")
return float("nan")
@property
def honores_h(self) -> float:
"""Honoré's H measure of m-type lexical richness.
Honoré's H relates the total number of m-type tokens to the proportion of
distinct m-types that occur exactly once (hapax legomena). It increases
when a sequence contains many single-occurrence m-types relative to its
overall m-type vocabulary.
Citation
--------
Honoré (1979)
"""
try:
if len(self.count_values) <= 1:
import warnings
warnings.warn("Cannot calculate Honoré's H when distinct n-gram types <= 1")
return float("nan")
n = self.total_tokens
v = len(self.ngram_counts)
v1 = self.freq_spec.get(1, 0)
if n == 0 or v == 0:
return float("nan")
return 100 * math.log(n) / (1 - v1 / v) if v1 != v else float("nan")
except Exception as e:
import warnings
warnings.warn(f"Error calculating Honoré's H: {str(e)}")
return float("nan")
@property
def mean_entropy(self) -> float:
"""Mean zeroth-order m-type entropy across counted n-gram orders.
For each n-gram order, this feature treats the m-type counts as a discrete
distribution and computes zeroth-order entropy. The returned value is the
mean of those entropy values across the counted orders. Higher values
indicate more even m-type distributions."""
try:
if len(self.count_values) <= 1:
import warnings
warnings.warn(
"Cannot calculate mean entropy when distinct n-gram types <= 1"
)
return float("nan")
entropies = []
for counts_by_ngram in self.ngram_counts_by_order.values():
n = sum(counts_by_ngram.values())
if n <= 0:
continue
probs = [count / n for count in counts_by_ngram.values() if count > 0]
if probs:
entropies.append(-sum(p * math.log2(p) for p in probs))
if not entropies:
return float("nan")
return float(sum(entropies) / len(entropies))
except Exception as e:
import warnings
warnings.warn(f"Error calculating mean entropy: {str(e)}")
return float("nan")
@property
def mean_productivity(self) -> float:
"""The proportion of distinct m-types that occur only once.
M-types that occur only once are hapax legomena. This feature divides the
number of hapax m-types by the total number of distinct m-types, so higher
values indicate that more of the melody's m-type vocabulary is used only
once.
"""
try:
if len(self.count_values) <= 1:
import warnings
warnings.warn(
"Cannot calculate mean productivity when distinct n-gram types <= 1"
)
return float("nan")
productivities = []
for counts_by_ngram in self.ngram_counts_by_order.values():
n = sum(counts_by_ngram.values())
if n <= 0:
continue
v1 = sum(1 for count in counts_by_ngram.values() if count == 1)
productivities.append(v1 / n)
if not productivities:
return float("nan")
return float(sum(productivities) / len(productivities))
except Exception as e:
import warnings
warnings.warn(f"Error calculating mean productivity: {str(e)}")
return float("nan")
# add decorator attributes here
mtype_properties = [
NGramCounter.yules_k,
NGramCounter.simpsons_d,
NGramCounter.sichels_s,
NGramCounter.honores_h,
NGramCounter.mean_entropy,
NGramCounter.mean_productivity,
]
for prop in mtype_properties:
fget = prop.fget
if fget is not None:
if not hasattr(fget, '_feature_types'):
fget._feature_types = []
if FeatureType.COMPLEXITY not in fget._feature_types:
fget._feature_types.append(FeatureType.COMPLEXITY)
fget._feature_domain = FeatureDomain.BOTH