Reliability Diagram and Expected Calibration Error

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314

/** Reference implementations for D40-F05 classification and score validation. */

type AnyRecord = Record<string, any>;

function numberValue(value: unknown, name: string): number {
  if (typeof value !== "number" || !Number.isFinite(value)) throw new TypeError(`${name} must be finite numeric`);
  return value;
}

function integerValue(value: unknown, name: string): number {
  const result = numberValue(value, name);
  if (!Number.isInteger(result)) throw new RangeError(`${name} must be an integer`);
  return result;
}

function records(inputs: AnyRecord, options: { score?: boolean; probability?: boolean } = {}): AnyRecord[] {
  if (!Array.isArray(inputs.records) || inputs.records.length === 0) throw new RangeError("records must be a nonempty array");
  const cutoff = inputs.evaluation_cutoff;
  if (cutoff !== undefined && (typeof cutoff !== "string" || cutoff.length === 0)) throw new TypeError("evaluation_cutoff must be a nonempty ISO-8601 string");
  const seen = new Set<string>();
  return inputs.records.map((raw: unknown, index: number) => {
    if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new TypeError(`records[${index}] must be an object`);
    const item = raw as AnyRecord;
    if (typeof item.id !== "string" || item.id.length === 0 || seen.has(item.id)) throw new RangeError("record ids must be unique nonempty strings");
    seen.add(item.id);
    if (item.label !== 0 && item.label !== 1) throw new RangeError("labels must be numeric 0 or 1");
    const weight = numberValue(item.weight ?? 1, `records[${index}].weight`);
    if (weight <= 0) throw new RangeError("weights must be positive");
    const row: AnyRecord = { ...item, label: item.label, weight };
    if (options.score) row.score = numberValue(item.score, `records[${index}].score`);
    if (options.probability) {
      row.probability = numberValue(item.probability, `records[${index}].probability`);
      if (row.probability < 0 || row.probability > 1) throw new RangeError("probabilities must be in [0,1]");
    }
    if (cutoff !== undefined) {
      for (const key of ["score_available_at", "label_available_at"]) {
        const timestamp = item[key];
        if (timestamp !== undefined) {
          if (typeof timestamp !== "string" || timestamp.length === 0) throw new TypeError(`${key} must be a nonempty ISO-8601 string`);
          if (timestamp > cutoff) throw new RangeError(`${key} exceeds evaluation_cutoff`);
        }
      }
    }
    return row;
  });
}

function classWeights(rows: AnyRecord[]): [number, number] {
  let positive = 0, negative = 0;
  for (const row of rows) row.label === 1 ? positive += row.weight : negative += row.weight;
  return [positive, negative];
}

function scoreGroups(rows: AnyRecord[]): Array<[number, AnyRecord[]]> {
  const ordered = [...rows].sort((a, b) => b.score - a.score || String(a.id).localeCompare(String(b.id)));
  const groups: Array<[number, AnyRecord[]]> = [];
  for (const row of ordered) {
    const last = groups.at(-1);
    if (!last || last[0] !== row.score) groups.push([row.score, [row]]);
    else last[1].push(row);
  }
  return groups;
}

export function rocCurveAuc(inputs: AnyRecord): AnyRecord {
  const rows = records(inputs, { score: true });
  const [positive, negative] = classWeights(rows);
  if (positive <= 0 || negative <= 0) throw new RangeError("ROC requires positive and negative weight");
  let tp = 0, fp = 0;
  const points: AnyRecord[] = [{ threshold: null, true_positive: 0, false_positive: 0, true_negative: negative, false_negative: positive, tpr: 0, fpr: 0 }];
  for (const [threshold, group] of scoreGroups(rows)) {
    for (const row of group) row.label === 1 ? tp += row.weight : fp += row.weight;
    points.push({ threshold, true_positive: tp, false_positive: fp, true_negative: negative - fp, false_negative: positive - tp, tpr: tp / positive, fpr: fp / negative });
  }
  let auc = 0;
  for (let i = 1; i < points.length; i++) auc += (points[i].fpr - points[i - 1].fpr) * (points[i].tpr + points[i - 1].tpr) / 2;
  return { points, roc_auc: auc, positive_weight: positive, negative_weight: negative, tie_group_count: points.length - 1, state: "ranking-evaluated" };
}

export function precisionRecallAuc(inputs: AnyRecord): AnyRecord {
  const rows = records(inputs, { score: true });
  const [positive, negative] = classWeights(rows);
  if (positive <= 0) throw new RangeError("precision-recall requires positive weight");
  let tp = 0, fp = 0, previousRecall = 0, averagePrecision = 0;
  const points: AnyRecord[] = [{ threshold: null, true_positive: 0, false_positive: 0, precision: 1, recall: 0 }];
  for (const [threshold, group] of scoreGroups(rows)) {
    for (const row of group) row.label === 1 ? tp += row.weight : fp += row.weight;
    const precision = tp / (tp + fp), recall = tp / positive;
    averagePrecision += (recall - previousRecall) * precision;
    previousRecall = recall;
    points.push({ threshold, true_positive: tp, false_positive: fp, precision, recall });
  }
  return { points, pr_auc_average_precision: averagePrecision, baseline_prevalence: positive / (positive + negative), positive_weight: positive, negative_weight: negative, tie_group_count: points.length - 1, integration_rule: "average-precision-right-step", state: "ranking-evaluated" };
}

export function brierScore(inputs: AnyRecord): AnyRecord {
  const rows = records(inputs, { probability: true });
  const total = rows.reduce((sum, row) => sum + row.weight, 0);
  const eventWeight = rows.reduce((sum, row) => sum + row.weight * row.label, 0);
  const eventRate = eventWeight / total;
  const score = rows.reduce((sum, row) => sum + row.weight * (row.probability - row.label) ** 2, 0) / total;
  const baseline = eventRate * (1 - eventRate);
  return { brier_score: score, event_rate: eventRate, baseline_brier: baseline, brier_skill: baseline === 0 ? null : 1 - score / baseline, weight_sum: total, record_count: rows.length, state: "probability-evaluated" };
}

export function binaryLogLoss(inputs: AnyRecord): AnyRecord {
  const rows = records(inputs, { probability: true });
  const epsilon = numberValue(inputs.epsilon ?? 1e-15, "epsilon");
  if (epsilon <= 0 || epsilon >= 0.5) throw new RangeError("epsilon must be in (0,0.5)");
  const total = rows.reduce((sum, row) => sum + row.weight, 0);
  const eventWeight = rows.reduce((sum, row) => sum + row.weight * row.label, 0);
  let loss = 0, clippedCount = 0;
  for (const row of rows) {
    const q = Math.min(Math.max(row.probability, epsilon), 1 - epsilon);
    if (q !== row.probability) clippedCount++;
    loss += row.weight * -Math.log(row.label === 1 ? q : 1 - q);
  }
  return { log_loss: loss / total, clipped_count: clippedCount, epsilon, weight_sum: total, event_rate: eventWeight / total, state: "probability-evaluated" };
}

export function reliabilityEce(inputs: AnyRecord): AnyRecord {
  const rows = records(inputs, { probability: true });
  const binCount = integerValue(inputs.bins ?? 5, "bins");
  if (binCount < 2 || binCount > 20) throw new RangeError("bins must be between 2 and 20");
  if ((inputs.bin_strategy ?? "uniform") !== "uniform") throw new RangeError("only uniform bin_strategy is supported");
  const total = rows.reduce((sum, row) => sum + row.weight, 0);
  const ledgers: AnyRecord[][] = Array.from({ length: binCount }, () => []);
  for (const row of rows) ledgers[Math.min(Math.floor(row.probability * binCount), binCount - 1)].push(row);
  const bins: AnyRecord[] = [];
  let ece = 0, mce = 0, signed = 0;
  for (let index = 0; index < binCount; index++) {
    const members = ledgers[index], lower = index / binCount, upper = (index + 1) / binCount;
    if (members.length === 0) {
      bins.push({ index: index + 1, lower, upper, right_inclusive: index === binCount - 1, record_count: 0, weight_sum: 0, mean_probability: null, event_rate: null, signed_gap: null, absolute_gap: null });
      continue;
    }
    const weight = members.reduce((sum, row) => sum + row.weight, 0);
    const meanProbability = members.reduce((sum, row) => sum + row.weight * row.probability, 0) / weight;
    const eventRate = members.reduce((sum, row) => sum + row.weight * row.label, 0) / weight;
    const signedGap = eventRate - meanProbability, absoluteGap = Math.abs(signedGap);
    ece += weight / total * absoluteGap;
    signed += weight / total * signedGap;
    mce = Math.max(mce, absoluteGap);
    bins.push({ index: index + 1, lower, upper, right_inclusive: index === binCount - 1, record_count: members.length, weight_sum: weight, mean_probability: meanProbability, event_rate: eventRate, signed_gap: signedGap, absolute_gap: absoluteGap });
  }
  return { bins, expected_calibration_error: ece, maximum_calibration_error: mce, signed_calibration_error: signed, bin_count: binCount, weight_sum: total, state: "calibration-evaluated" };
}

export function gainsLift(inputs: AnyRecord): AnyRecord {
  const rows = records(inputs, { score: true });
  const bucketCount = integerValue(inputs.buckets ?? 10, "buckets");
  if (bucketCount < 2 || bucketCount > rows.length) throw new RangeError("buckets must be between 2 and record count");
  const ordered = [...rows].sort((a, b) => b.score - a.score || String(a.id).localeCompare(String(b.id)));
  const totalWeight = ordered.reduce((sum, row) => sum + row.weight, 0);
  const totalEvents = ordered.reduce((sum, row) => sum + row.weight * row.label, 0);
  if (totalEvents <= 0) throw new RangeError("gains and lift require positive event weight");
  const prevalence = totalEvents / totalWeight;
  const members: Array<Array<[number, AnyRecord]>> = Array.from({ length: bucketCount }, () => []);
  ordered.forEach((row, rank) => members[Math.min(Math.floor(rank * bucketCount / ordered.length), bucketCount - 1)].push([rank + 1, row]));
  let cumulativeWeight = 0, cumulativeEvents = 0;
  const buckets = members.map((bucket, index) => {
    const weight = bucket.reduce((sum, [, row]) => sum + row.weight, 0);
    const events = bucket.reduce((sum, [, row]) => sum + row.weight * row.label, 0);
    cumulativeWeight += weight; cumulativeEvents += events;
    const populationShare = weight / totalWeight, eventCapture = events / totalEvents;
    const cumulativePopulation = cumulativeWeight / totalWeight, cumulativeGain = cumulativeEvents / totalEvents;
    return { bucket: index + 1, rank_start: bucket[0][0], rank_end: bucket.at(-1)![0], record_count: bucket.length, population_weight: weight, event_weight: events, population_share: populationShare, event_capture: eventCapture, bucket_lift: (events / weight) / prevalence, cumulative_population: cumulativePopulation, cumulative_gain: cumulativeGain, cumulative_lift: cumulativeGain / cumulativePopulation };
  });
  return { buckets, top_decile_capture: buckets[0].event_capture, overall_prevalence: prevalence, total_events: totalEvents, total_weight: totalWeight, bucket_count: bucketCount, tie_break: "score-descending-id-ascending", state: "ranking-evaluated" };
}

export function costSensitiveThreshold(inputs: AnyRecord): AnyRecord {
  const rows = records(inputs, { score: true });
  if (!inputs.costs || typeof inputs.costs !== "object" || Array.isArray(inputs.costs)) throw new TypeError("costs must be an object");
  const names = ["false_positive", "false_negative", "true_positive", "true_negative"];
  const costs: AnyRecord = {};
  for (const name of names) costs[name] = numberValue(inputs.costs[name], `costs.${name}`);
  if (names.some(name => costs[name] < 0) || (costs.false_positive === 0 && costs.false_negative === 0)) throw new RangeError("costs must be nonnegative with a positive misclassification cost");
  const total = rows.reduce((sum, row) => sum + row.weight, 0);
  const thresholds: Array<number | null> = [null, ...[...new Set(rows.map(row => row.score))].sort((a, b) => b - a)];
  const candidates = thresholds.map(threshold => {
    let tp = 0, fp = 0, tn = 0, fn = 0, selected = 0;
    for (const row of rows) {
      const predicted = threshold !== null && row.score >= threshold;
      if (predicted) selected += row.weight;
      if (predicted && row.label === 1) tp += row.weight;
      else if (predicted && row.label === 0) fp += row.weight;
      else if (!predicted && row.label === 0) tn += row.weight;
      else fn += row.weight;
    }
    const expectedCost = (costs.false_positive * fp + costs.false_negative * fn + costs.true_positive * tp + costs.true_negative * tn) / total;
    return { threshold, true_positive: tp, false_positive: fp, true_negative: tn, false_negative: fn, selected_weight: selected, selected_rate: selected / total, expected_cost: expectedCost };
  });
  const optimal = [...candidates].sort((a, b) => a.expected_cost - b.expected_cost || a.selected_weight - b.selected_weight || -((a.threshold ?? Infinity) - (b.threshold ?? Infinity)))[0];
  return { candidates, optimal_threshold: optimal.threshold, optimal_expected_cost: optimal.expected_cost, optimal_selected_rate: optimal.selected_rate, optimal_confusion: { true_positive: optimal.true_positive, false_positive: optimal.false_positive, true_negative: optimal.true_negative, false_negative: optimal.false_negative }, costs, tie_break: "minimum-cost-then-lower-selected-weight-then-higher-threshold", state: "threshold-selected" };
}

function band(score: number, edges: number[]): number {
  let result = 0;
  while (result + 1 < edges.length && score >= edges[result + 1]) result++;
  return Math.min(result, edges.length - 2);
}

export function scoreMigration(inputs: AnyRecord): AnyRecord {
  if (!Array.isArray(inputs.baseline) || inputs.baseline.length === 0 || !Array.isArray(inputs.current) || inputs.current.length === 0) throw new RangeError("baseline and current must be nonempty arrays");
  if (inputs.higher_score_higher_risk !== true) throw new RangeError("canonical orientation requires higher_score_higher_risk=true");
  if (typeof inputs.baseline_observed_at !== "string" || typeof inputs.current_observed_at !== "string" || inputs.baseline_observed_at >= inputs.current_observed_at) throw new RangeError("baseline_observed_at must be before current_observed_at");
  if (!Array.isArray(inputs.band_edges) || inputs.band_edges.length < 3) throw new RangeError("band_edges must contain at least three values");
  const edges = inputs.band_edges.map((value: unknown) => numberValue(value, "band_edges"));
  if (edges[0] !== 0 || edges.at(-1) !== 1 || edges.slice(1).some((value: number, index: number) => edges[index] >= value)) throw new RangeError("band_edges must increase strictly from 0 to 1");
  function snapshot(rawRows: unknown[], name: string): Map<string, number> {
    const result = new Map<string, number>();
    rawRows.forEach((raw, index) => {
      if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new TypeError(`${name}[${index}] must be an object`);
      const row = raw as AnyRecord;
      if (typeof row.id !== "string" || row.id.length === 0 || result.has(row.id)) throw new RangeError(`${name} ids must be unique nonempty strings`);
      const score = numberValue(row.score, `${name}[${index}].score`);
      if (score < 0 || score > 1) throw new RangeError("migration scores must be in [0,1]");
      result.set(row.id, score);
    });
    return result;
  }
  const baseline = snapshot(inputs.baseline, "baseline"), current = snapshot(inputs.current, "current");
  if (baseline.size !== current.size || [...baseline.keys()].some(id => !current.has(id))) throw new RangeError("baseline and current must contain identical ids");
  const size = edges.length - 1, matrix = Array.from({ length: size }, () => Array(size).fill(0));
  const migrations: AnyRecord[] = [];
  let stable = 0, improved = 0, worsened = 0, absoluteMove = 0, scoreChange = 0, absoluteScoreChange = 0;
  for (const id of [...baseline.keys()].sort()) {
    const before = baseline.get(id)!, after = current.get(id)!;
    const a = band(before, edges), b = band(after, edges), move = b - a, delta = after - before;
    matrix[a][b]++;
    move === 0 ? stable++ : move < 0 ? improved++ : worsened++;
    absoluteMove += Math.abs(move); scoreChange += delta; absoluteScoreChange += Math.abs(delta);
    migrations.push({ id, baseline_score: before, current_score: after, baseline_band: a + 1, current_band: b + 1, band_move: move, score_change: delta });
  }
  const rowRates = matrix.map(row => { const total = row.reduce((sum, value) => sum + value, 0); return row.map(value => total === 0 ? null : value / total); });
  const baselineCounts = matrix.map(row => row.reduce((sum, value) => sum + value, 0));
  const currentCounts = Array.from({ length: size }, (_, column) => matrix.reduce((sum, row) => sum + row[column], 0));
  const n = baseline.size, baselineShares = baselineCounts.map(value => value / n), currentShares = currentCounts.map(value => value / n);
  const tv = 0.5 * baselineShares.reduce((sum, value, index) => sum + Math.abs(value - currentShares[index]), 0);
  return { band_edges: edges, matrix, row_rates: rowRates, baseline_band_shares: baselineShares, current_band_shares: currentShares, migrations, matched_count: n, stable_count: stable, improved_count: improved, worsened_count: worsened, stable_rate: stable / n, mean_band_move: migrations.reduce((sum, row) => sum + row.band_move, 0) / n, mean_absolute_band_move: absoluteMove / n, mean_score_change: scoreChange / n, mean_absolute_score_change: absoluteScoreChange / n, band_distribution_total_variation: tv, state: "migration-evaluated" };
}

function basicMetrics(rows: AnyRecord[], epsilon: number): AnyRecord {
  const total = rows.reduce((sum, row) => sum + row.weight, 0), [positive, negative] = classWeights(rows);
  const eventRate = positive / total;
  const brier = rows.reduce((sum, row) => sum + row.weight * (row.probability - row.label) ** 2, 0) / total;
  const logloss = rows.reduce((sum, row) => { const q = Math.min(Math.max(row.probability, epsilon), 1 - epsilon); return sum + row.weight * -Math.log(row.label === 1 ? q : 1 - q); }, 0) / total;
  const auc = positive > 0 && negative > 0 ? rocCurveAuc({ records: rows }).roc_auc : null;
  return { record_count: rows.length, weight_sum: total, positive_weight: positive, negative_weight: negative, event_rate: eventRate, brier_score: brier, log_loss: logloss, roc_auc: auc };
}

export function sliceValidation(inputs: AnyRecord): AnyRecord {
  const rows = records(inputs, { score: true, probability: true });
  if (!Array.isArray(inputs.slice_fields) || inputs.slice_fields.length === 0 || new Set(inputs.slice_fields).size !== inputs.slice_fields.length || inputs.slice_fields.some((field: unknown) => typeof field !== "string" || field.length === 0)) throw new RangeError("slice_fields must be unique nonempty strings");
  const fields = inputs.slice_fields as string[], minimum = integerValue(inputs.minimum_support ?? 4, "minimum_support");
  if (minimum < 2) throw new RangeError("minimum_support must be at least 2");
  const epsilon = numberValue(inputs.epsilon ?? 1e-15, "epsilon");
  if (epsilon <= 0 || epsilon >= 0.5) throw new RangeError("epsilon must be in (0,0.5)");
  for (const field of fields) if (rows.some(row => typeof row[field] !== "string" || row[field].length === 0)) throw new RangeError(`slice field ${field} is missing or invalid`);
  const overall = basicMetrics(rows, epsilon), slices: AnyRecord[] = [], eligibleBrier: number[] = [];
  let eligible = 0, flagged = 0;
  for (const field of fields) {
    const values = [...new Set(rows.map(row => row[field] as string))].sort();
    for (const value of values) {
      const members = rows.filter(row => row[field] === value), metrics = basicMetrics(members, epsilon);
      let status = "ok";
      if (members.length < minimum) status = "insufficient-support";
      else if (metrics.positive_weight <= 0 || metrics.negative_weight <= 0) status = "single-class";
      if (status === "ok") { eligible++; eligibleBrier.push(metrics.brier_score); } else flagged++;
      slices.push({ field, value, status, ...metrics, brier_gap_from_overall: metrics.brier_score - overall.brier_score, log_loss_gap_from_overall: metrics.log_loss - overall.log_loss, roc_auc_gap_from_overall: metrics.roc_auc === null ? null : metrics.roc_auc - overall.roc_auc });
    }
  }
  if (eligibleBrier.length === 0) throw new RangeError("no slice meets the canonical support and class requirements");
  return { overall, slices, slice_fields: fields, minimum_support: minimum, worst_slice_brier: Math.max(...eligibleBrier), eligible_slice_count: eligible, flagged_slice_count: flagged, state: "slices-evaluated" };
}

export function rareEventBounds(inputs: AnyRecord): AnyRecord {
  const events = integerValue(inputs.observed_events, "observed_events"), trials = integerValue(inputs.trials, "trials");
  if (trials <= 0 || events < 0 || events > trials) throw new RangeError("require 0 <= observed_events <= trials and trials > 0");
  const expected = numberValue(inputs.expected_probability, "expected_probability");
  if (expected < 0 || expected > 1) throw new RangeError("expected_probability must be in [0,1]");
  const confidence = numberValue(inputs.confidence_level, "confidence_level");
  const zMap = new Map([[0.90, 1.6448536269514722], [0.95, 1.959963984540054], [0.99, 2.5758293035489004]]);
  const z = zMap.get(confidence);
  if (z === undefined) throw new RangeError("confidence_level must be 0.90, 0.95, or 0.99");
  if (inputs.sampling_assumption !== "independent-bernoulli-approximation") throw new RangeError("unsupported sampling_assumption");
  const observed = events / trials, denominator = 1 + z * z / trials;
  const center = (observed + z * z / (2 * trials)) / denominator;
  const half = z * Math.sqrt(observed * (1 - observed) / trials + z * z / (4 * trials * trials)) / denominator;
  const lower = events === 0 ? 0 : Math.max(0, center - half), upper = events === trials ? 1 : Math.min(1, center + half);
  const consistency = expected < lower ? "expected-below-interval" : expected > upper ? "expected-above-interval" : "inside-interval";
  return { observed_events: events, trials, observed_rate: observed, expected_probability: expected, expected_count: trials * expected, confidence_level: confidence, z_value: z, wilson_center: center, wilson_half_width: half, wilson_lower: lower, wilson_upper: upper, consistency, sampling_assumption: "independent-bernoulli-approximation", state: "rare-event-evaluated" };
}

export function calculate(topicId: string, inputs: AnyRecord): AnyRecord {
  if (!inputs || typeof inputs !== "object" || Array.isArray(inputs)) throw new TypeError("inputs must be an object");
  const dispatch: Record<string, (value: AnyRecord) => AnyRecord> = {
    "D40-F05-A01": rocCurveAuc,
    "D40-F05-A02": precisionRecallAuc,
    "D40-F05-A03": brierScore,
    "D40-F05-A04": binaryLogLoss,
    "D40-F05-A05": reliabilityEce,
    "D40-F05-A06": gainsLift,
    "D40-F05-A07": costSensitiveThreshold,
    "D40-F05-A08": scoreMigration,
    "D40-F05-A09": sliceValidation,
    "D40-F05-A10": rareEventBounds,
  };
  const implementation = dispatch[topicId];
  if (!implementation) throw new RangeError(`unsupported topic_id: ${topicId}`);
  return implementation(inputs);
}