LLM Judge Calibration & Bias

// Closed-world fact fixtures only. Equality of structured facts is NOT an NLI judge.
const object = value => value !== null && typeof value === 'object' && !Array.isArray(value);
export function authorizedSources(sources, scope, now) {
  if (!object(scope) || typeof scope.tenant !== 'string' || typeof scope.user !== 'string' || !Number.isFinite(now)) throw new Error('Invalid trusted scope/time');
  return sources.filter(source => source.tenant === scope.tenant && source.approved === true && !source.deleted
    && source.expiresAt > now && source.readers.includes(scope.user));
}
export function verifyClaims(claims, sources, scope, now) {
  if (!Array.isArray(claims) || claims.length > 20) throw new Error('Invalid claims');
  const evidence = authorizedSources(sources, scope, now);
  return claims.map(claim => {
    if (!object(claim) || Object.keys(claim).sort().join(',') !== 'key,sourceId,value'
      || ![claim.key,claim.sourceId,claim.value].every(value => typeof value === 'string' && value.length > 0 && value.length <= 200)) throw new Error('Invalid structured claim');
    const source = evidence.find(item => item.id === claim.sourceId);
    const value = source?.facts[claim.key];
    return { ...claim, label: value === undefined ? 'INSUFFICIENT_EVIDENCE' : value === claim.value ? 'SUPPORTED' : 'CONTRADICTED' };
  });
}
export function answerPolicy(claims, sources, scope, now) {
  const labels = verifyClaims(claims,sources,scope,now);
  return { status:labels.length > 0 && labels.every(item => item.label === 'SUPPORTED') ? 'ANSWER' : 'ABSTAIN', claims:labels };
}
export function retrievalRecall(relevantIds, retrievedIds) {
  if (!relevantIds.length) return null;
  return [...new Set(relevantIds)].filter(id => retrievedIds.includes(id)).length / new Set(relevantIds).size;
}
export function judgeAgreement(human, judge) {
  if (human.length !== judge.length) throw new Error('Unpaired annotations');
  return human.length ? human.filter((label,index) => label === judge[index]).length / human.length : null;
}