feat(metrics): Rényi entropy spectrum & Hill numbers (#46)

lib/codeqa/engine/analyzer.ex

@@ -17,6 +17,7 @@ defmodule CodeQA.Engine.Analyzer do
             |> Registry.register_file_metric(Metrics.Heaps)
             |> Registry.register_file_metric(Metrics.Vocabulary)
             |> Registry.register_file_metric(Metrics.LexicalConcentration)
+            |> Registry.register_file_metric(Metrics.RenyiEntropy)
             |> Registry.register_file_metric(Metrics.Ngram)
             |> Registry.register_file_metric(Metrics.Halstead)
             |> Registry.register_file_metric(Metrics.Readability)

lib/codeqa/metrics/file/renyi_entropy.ex

@@ -0,0 +1,103 @@
+defmodule CodeQA.Metrics.File.RenyiEntropy do
+  @moduledoc """
+  Computes the Rényi entropy spectrum and Hill numbers over the token
+  distribution.
+
+  Rényi entropy `H_α` generalizes Shannon entropy across an order parameter
+  `α`: low orders weight rare tokens, high orders only the dominant ones. The
+  spectrum `H₀ ≥ H₁ ≥ H₂ ≥ H_∞` therefore characterizes the *shape* of the
+  distribution rather than a single scalar. `spectrum_slope = H₀ − H₂` is a
+  concentration indicator: near zero for a flat (uniform) vocabulary, large
+  when a few tokens dominate.
+
+  Hill numbers `D_α = 2^(H_α)` express the *effective vocabulary size* — how
+  many tokens really count — which is more interpretable than raw `vocab_size`.
+
+  See [Rényi entropy](https://en.wikipedia.org/wiki/R%C3%A9nyi_entropy) and
+  [Hill numbers / diversity](https://en.wikipedia.org/wiki/Diversity_index#Effective_number_of_species).
+  """
+
+  @behaviour CodeQA.Metrics.File.FileMetric
+
+  @impl true
+  def name, do: "renyi_entropy"
+
+  @impl true
+  def keys,
+    do: [
+      "renyi_0",
+      "renyi_1",
+      "renyi_2",
+      "renyi_inf",
+      "hill_1",
+      "hill_2",
+      "spectrum_slope"
+    ]
+
+  @impl true
+  def description,
+    do:
+      "Rényi entropy spectrum (α ∈ {0,1,2,∞}) and Hill numbers (effective vocabulary size) over the token distribution."
+
+  @spec analyze(map()) :: map()
+  @impl true
+  def analyze(%{tokens: []}), do: zero_result()
+
+  def analyze(%{token_counts: token_counts, tokens: tokens}) do
+    total = length(tokens)
+    probs = token_counts |> Map.values() |> Enum.map(&(&1 / total))
+    spectrum(probs)
+  end
+
+  defp spectrum(probs) do
+    h0 = renyi_0(probs)
+    h1 = renyi_1(probs)
+    h2 = renyi_2(probs)
+    h_inf = renyi_inf(probs)
+
+    %{
+      "renyi_0" => round4(h0),
+      "renyi_1" => round4(h1),
+      "renyi_2" => round4(h2),
+      "renyi_inf" => round4(h_inf),
+      "hill_1" => round4(hill(h1)),
+      "hill_2" => round4(hill(h2)),
+      "spectrum_slope" => round4(h0 - h2)
+    }
+  end
+
+  # H₀ = log2(support size) — Hartley entropy, counts only distinct tokens.
+  defp renyi_0(probs), do: log2(length(probs))
+
+  # H₁ = -Σ pᵢ log2 pᵢ — Shannon entropy, the α→1 limit.
+  defp renyi_1(probs),
+    do: -Enum.reduce(probs, 0.0, fn p, acc -> acc + p * :math.log2(p) end)
+
+  # H₂ = -log2(Σ pᵢ²) — collision entropy.
+  defp renyi_2(probs) do
+    sum_sq = Enum.reduce(probs, 0.0, fn p, acc -> acc + p * p end)
+    -:math.log2(sum_sq)
+  end
+
+  # H_∞ = -log2(max pᵢ) — min-entropy, governed by the most frequent token.
+  defp renyi_inf(probs), do: -:math.log2(Enum.max(probs))
+
+  # Hill number D_α = 2^(H_α), the effective vocabulary size.
+  defp hill(h), do: :math.pow(2, h)
+
+  defp log2(n) when n > 0, do: :math.log2(n)
+  defp log2(_), do: 0.0
+
+  defp round4(x), do: Float.round(x * 1.0, 4)
+
+  defp zero_result,
+    do: %{
+      "renyi_0" => 0.0,
+      "renyi_1" => 0.0,
+      "renyi_2" => 0.0,
+      "renyi_inf" => 0.0,
+      "hill_1" => 0.0,
+      "hill_2" => 0.0,
+      "spectrum_slope" => 0.0
+    }
+end

priv/combined_metrics/code_smells.yml

@@ -2,7 +2,7 @@ consistent_string_quote_style:
   _doc: "Files should use a single, consistent string quoting style throughout."
   _fix_hint: "Use one quote style consistently for string literals in this {{type}}."
   _languages: [elixir]
-  _log_baseline: -19.2276
+  _log_baseline: -20.1399
   branching:
     mean_branching_density: 0.0243
     mean_non_blank_count: -0.0248
@@ -90,6 +90,14 @@ consistent_string_quote_style:
     mean_flesch_adapted: 0.0046
     mean_fog_adapted: -0.0301
     mean_total_lines: -0.0248
+  renyi_entropy:
+    mean_hill_1: -0.0705
+    mean_hill_2: -0.1119
+    mean_renyi_0: -0.0141
+    mean_renyi_1: -0.0178
+    mean_renyi_2: -0.0322
+    mean_renyi_inf: -0.1217
+    mean_spectrum_slope: 0.0390
   separator_counts:
     mean_dot_count: -0.1743
     mean_underscore_count: -0.0644
@@ -112,7 +120,7 @@ no_dead_code_after_return:
   _doc: "There should be no unreachable statements after a return or early exit."
   _excludes_languages: [elixir]
   _fix_hint: "Remove the unreachable code after the return/raise in this {{type}}."
-  _log_baseline: -70.7283
+  _log_baseline: -71.9927
   branching:
     mean_branch_count: -0.9417
     mean_branching_density: -0.1794
@@ -211,6 +219,14 @@ no_dead_code_after_return:
     mean_flesch_adapted: 0.0346
     mean_fog_adapted: 0.0141
     mean_total_lines: -0.7334
+  renyi_entropy:
+    mean_hill_1: -0.1341
+    mean_hill_2: -0.1381
+    mean_renyi_0: -0.0501
+    mean_renyi_1: -0.0360
+    mean_renyi_2: -0.0409
+    mean_renyi_inf: -0.0720
+    mean_spectrum_slope: -0.0737
   separator_counts:
     mean_dot_count: -1.2574
     mean_hyphen_count: -1.8488
@@ -236,7 +252,7 @@ no_debug_print_statements:
   _doc: "Debug output (`console.log`, `IO.inspect`, `fmt.Println`) must not be left in committed code."
   _fix_hint: "Remove the debug print (IO.puts/IO.inspect/console.log) or replace it with a proper logger."
   _languages: [elixir, go, javascript, python, ruby]
-  _log_baseline: -29.5797
+  _log_baseline: -29.0583
   branching:
     mean_branch_count: 0.1865
     mean_branching_density: 0.4692
@@ -341,6 +357,14 @@ no_debug_print_statements:
     mean_flesch_adapted: -0.0375
     mean_fog_adapted: 0.0931
     mean_total_lines: -0.1329
+  renyi_entropy:
+    mean_hill_1: 0.0824
+    mean_hill_2: 0.0412
+    mean_renyi_0: 0.0188
+    mean_renyi_1: 0.0224
+    mean_renyi_2: 0.0114
+    mean_renyi_inf: -0.0190
+    mean_spectrum_slope: 0.0415
   separator_counts:
     mean_dot_count: -0.1479
     mean_hyphen_count: 0.0827
@@ -366,7 +390,7 @@ no_fixme_comments:
   _doc: "FIXME, XXX, and HACK comments indicate known problems that should be resolved before merging."
   _fix_hint: "Resolve the FIXME and delete the comment, or convert it into a tracked issue."
   _languages: [elixir, go, javascript, python, ruby]
-  _log_baseline: 12.0359
+  _log_baseline: 10.9439
   branching:
     mean_branch_count: 0.1713
     mean_branching_density: 0.1042
@@ -475,6 +499,14 @@ no_fixme_comments:
     mean_flesch_adapted: -0.0152
     mean_fog_adapted: -0.0037
     mean_total_lines: 0.0927
+  renyi_entropy:
+    mean_hill_1: -0.1284
+    mean_hill_2: -0.1032
+    mean_renyi_0: -0.0222
+    mean_renyi_1: -0.0313
+    mean_renyi_2: -0.0296
+    mean_renyi_inf: -0.0276
+    mean_spectrum_slope: -0.0024
   separator_counts:
     mean_dot_count: 0.3139
     mean_hyphen_count: -0.0251
@@ -499,7 +531,7 @@ no_nested_ternary:
   _doc: "Nested conditional expressions (ternary-within-ternary) are harder to read than a plain if-else."
   _excludes_languages: [elixir]
   _fix_hint: "Flatten the nested ternary into pattern matching or an if/cond expression."
-  _log_baseline: 8.9473
+  _log_baseline: 8.7697
   branching:
     mean_branch_count: -0.5662
     mean_branching_density: -0.3441
@@ -599,6 +631,14 @@ no_nested_ternary:
     mean_flesch_adapted: -0.0367
     mean_fog_adapted: 0.3545
     mean_total_lines: -0.2221
+  renyi_entropy:
+    mean_hill_1: -0.0050
+    mean_hill_2: -0.0440
+    mean_renyi_0: 0.0110
+    mean_renyi_1: -0.0014
+    mean_renyi_2: -0.0142
+    mean_renyi_inf: -0.0561
+    mean_spectrum_slope: 0.0714
   separator_counts:
     mean_dot_count: -0.1824
     mean_hyphen_count: -0.1067

priv/combined_metrics/consistency.yml

@@ -2,7 +2,7 @@ consistent_casing_within_file:
   _doc: "A file should use one naming convention throughout — no mixing of camelCase and snake_case for the same kind of identifier."
   _fix_hint: "Align identifier casing in this {{type}} with the rest of the file."
   _languages: [elixir]
-  _log_baseline: -0.6528
+  _log_baseline: -0.8912
   brevity:
     mean_sample_size: -0.0471
   casing_entropy:
@@ -49,6 +49,13 @@ consistent_casing_within_file:
     mean_trigram_unique: -0.0172
   readability:
     mean_avg_line_length: 0.0221
+  renyi_entropy:
+    mean_hill_1: -0.0362
+    mean_hill_2: -0.0104
+    mean_renyi_0: -0.0101
+    mean_renyi_1: -0.0090
+    mean_renyi_2: -0.0030
+    mean_spectrum_slope: -0.0302
   separator_counts:
     mean_underscore_count: 0.4311
   symbol_density:
@@ -67,7 +74,7 @@ consistent_error_return_shape:
   _doc: "All functions in a module should return errors in the same shape — mixed `nil`, `false`, and `{:error, _}` returns are confusing."
   _fix_hint: "Return errors in one shape everywhere (e.g. {:error, reason})."
   _languages: [elixir]
-  _log_baseline: -2.8073
+  _log_baseline: -3.3807
   branching:
     mean_branch_count: -0.2800
     mean_branching_density: 0.5563
@@ -165,6 +172,13 @@ consistent_error_return_shape:
     mean_flesch_adapted: -0.0433
     mean_fog_adapted: 0.1454
     mean_total_lines: -0.3834
+  renyi_entropy:
+    mean_hill_1: -0.0658
+    mean_hill_2: -0.0443
+    mean_renyi_0: -0.0167
+    mean_renyi_1: -0.0195
+    mean_renyi_2: -0.0216
+    mean_renyi_inf: -0.0655
   separator_counts:
     mean_dot_count: 0.6069
     mean_hyphen_count: -1.1258
@@ -186,7 +200,7 @@ consistent_function_style:
   _doc: "A module should not mix one-liner and multi-clause function definitions for the same concern."
   _fix_hint: "Match the surrounding function-definition style (def vs anonymous, single vs multi-clause)."
   _languages: [elixir]
-  _log_baseline: 57.4562
+  _log_baseline: 60.0235
   branching:
     mean_branch_count: 0.3619
     mean_branching_density: 0.2247
@@ -296,6 +310,14 @@ consistent_function_style:
     mean_flesch_adapted: 0.0263
     mean_fog_adapted: -0.1088
     mean_total_lines: 0.3305
+  renyi_entropy:
+    mean_hill_1: 0.3154
+    mean_hill_2: 0.2457
+    mean_renyi_0: 0.0888
+    mean_renyi_1: 0.0864
+    mean_renyi_2: 0.0741
+    mean_renyi_inf: 0.0286
+    mean_spectrum_slope: 0.1323
   separator_counts:
     mean_dot_count: 0.3510
     mean_hyphen_count: 0.4708
@@ -322,7 +344,7 @@ same_concept_same_name:
   _doc: "The same domain concept should use the same name throughout a file — mixing `user`, `usr`, and `u` for the same thing harms readability."
   _fix_hint: "Use one name for this concept across the file instead of synonyms."
   _languages: [elixir]
-  _log_baseline: -13.5397
+  _log_baseline: -22.1414
   brevity:
     mean_sample_size: -1.3457
   compression:
@@ -370,6 +392,13 @@ same_concept_same_name:
     mean_avg_line_length: 0.1837
     mean_avg_sub_words_per_id: -0.1771
     mean_flesch_adapted: 0.2348
+  renyi_entropy:
+    mean_hill_1: -1.3316
+    mean_hill_2: -0.5598
+    mean_renyi_0: -0.2899
+    mean_renyi_1: -0.3546
+    mean_renyi_2: -0.1759
+    mean_spectrum_slope: -0.5362
   separator_counts:
     mean_underscore_count: -0.5711
   symbol_density:

priv/combined_metrics/dependencies.yml

@@ -2,7 +2,7 @@ import_count_under_10:
   _doc: "Files should import fewer than 10 modules; high import counts signal excessive coupling."
   _fix_hint: "This {{type}} pulls in too many dependencies — split it or narrow the imports."
   _languages: [elixir]
-  _log_baseline: 7.1957
+  _log_baseline: 7.0143
   branching:
     mean_branch_count: 0.2110
     mean_branching_density: -1.0683
@@ -101,6 +101,14 @@ import_count_under_10:
     mean_flesch_adapted: -0.0204
     mean_fog_adapted: 0.2028
     mean_total_lines: -0.0265
+  renyi_entropy:
+    mean_hill_1: -0.0153
+    mean_hill_2: -0.0444
+    mean_renyi_0: 0.0026
+    mean_renyi_1: -0.0040
+    mean_renyi_2: -0.0139
+    mean_renyi_inf: 0.0125
+    mean_spectrum_slope: 0.0397
   separator_counts:
     mean_dot_count: -0.1234
     mean_underscore_count: 0.1368
@@ -124,7 +132,7 @@ low_coupling:
   _doc: "Modules should depend on few external symbols — a low unique-operand count relative to total is a proxy for tight coupling."
   _fix_hint: "Reduce this {{type}}'s outbound references; depend on fewer modules or invert the dependency."
   _languages: [elixir]
-  _log_baseline: -45.2561
+  _log_baseline: -45.6308
   branching:
     mean_branch_count: 0.0398
     mean_branching_density: 0.2038
@@ -225,6 +233,13 @@ low_coupling:
     mean_flesch_adapted: 0.0138
     mean_fog_adapted: -0.0557
     mean_total_lines: -0.1476
+  renyi_entropy:
+    mean_hill_1: -0.0660
+    mean_hill_2: -0.0031
+    mean_renyi_0: -0.0276
+    mean_renyi_1: -0.0174
+    mean_renyi_inf: 0.0132
+    mean_spectrum_slope: -0.1079
   separator_counts:
     mean_dot_count: -0.3285
     mean_underscore_count: -0.1206
@@ -248,7 +263,7 @@ no_wildcard_imports:
   _doc: "Wildcard imports (`import *`, `using Module`) pollute the local namespace and hide dependencies."
   _fix_hint: "Replace the wildcard import with explicit named imports."
   _languages: [elixir]
-  _log_baseline: -8.8366
+  _log_baseline: -8.7732
   branching:
     mean_branching_density: 0.0249
     mean_non_blank_count: -0.0268
@@ -331,6 +346,11 @@ no_wildcard_imports:
     mean_flesch_adapted: -0.0142
     mean_fog_adapted: 0.0290
     mean_total_lines: -0.0268
+  renyi_entropy:
+    mean_hill_1: 0.0090
+    mean_hill_2: 0.0059
+    mean_renyi_inf: 0.0160
+    mean_spectrum_slope: -0.0086
   separator_counts:
     mean_dot_count: -0.0137
     mean_underscore_count: 0.0302