## ----include = FALSE----------------------------------------------------------
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)
set.seed(2026)

## ----setup--------------------------------------------------------------------
library(aiEvalR)

## ----reliability--------------------------------------------------------------
# 12 prompts, each answered on 3 separate occasions (e.g. repeated
# sampling). Scores are stable across occasions -> high reliability.
latent <- rnorm(12, mean = 5)
responses <- sapply(1:3, function(occasion) latent + rnorm(12, sd = 0.3))

rel <- ai_reliability(responses)
rel$test_retest$icc

## ----reliability-boot---------------------------------------------------------
boot <- ai_bootstrap_reliability(responses, n_boot = 200, seed = 1)
c(estimate = boot$estimate, lower = boot$ci_lower, upper = boot$ci_upper)

## ----robustness---------------------------------------------------------------
baseline  <- rnorm(100, mean = 5, sd = 1)
# a mild perturbation and a severe one, applied to the same prompts
mild   <- baseline + rnorm(100, mean = 0.1, sd = 0.05)
severe <- baseline + rnorm(100, mean = 1.0, sd = 0.05)

st <- stress_test(baseline, list(mild = mild, severe = severe))
st$robustness_index

## ----fairness-----------------------------------------------------------------
outcome <- rbinom(200, 1, 0.4)
group   <- sample(c("A", "B"), 200, replace = TRUE)

disp <- ai_group_disparity(outcome, group)
disp$demographic_parity_diff

## ----calibration--------------------------------------------------------------
# Well-calibrated: predicted probabilities match empirical frequencies
confidence <- runif(500)
outcome    <- rbinom(500, 1, confidence)

cal <- ai_calibration(outcome, confidence, n_bins = 10)
c(ECE = cal$ece, Brier = cal$brier)

## ----hallucination------------------------------------------------------------
# externally adjudicated: TRUE = claim unsupported
verdicts <- c(FALSE, FALSE, TRUE, FALSE, TRUE)
hallucination_rate(verdicts)

# lexical overlap is NOT factual consistency -- note the name
lexical_overlap("the treatment reduces mortality",
                "the treatment does not reduce mortality")

## ----dashboard----------------------------------------------------------------
ai_dashboard(
  reliability   = 0.94,
  fairness      = 0.82,
  robustness    = 0.70,
  calibration   = 0.88,
  hallucination = 0.81
)