## ----include = FALSE----------------------------------------------------------
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)

## ----setup--------------------------------------------------------------------
library(probcal)
library(dplyr)

## ----data---------------------------------------------------------------------
set.seed(2026)
n <- 800
predictions <- data.frame(x = rnorm(n)) |>
  mutate(
    true_p = inv_logit(-0.5 + 1.2 * x),
    y = rbinom(n(), 1, true_p),
    raw_logits = 1.7 * (-0.5 + 1.2 * x),
    raw_p = inv_logit(raw_logits),
    split = sample(rep(c("calibration", "test"), each = n / 2))
  )

calibration <- predictions |>
  filter(split == "calibration")

test <- predictions |>
  filter(split == "test")

## ----beta---------------------------------------------------------------------
beta_fit <- cal_beta(calibration$raw_p, calibration$y)

test <- test |>
  mutate(beta = predict(beta_fit, raw_p))

test |>
  summarise(
    raw_ece = ece(raw_p, y, bins = 10),
    beta_ece = ece(beta, y, bins = 10)
  )

## ----methods------------------------------------------------------------------
platt_fit <- cal_platt(calibration$raw_p, calibration$y)
iso_fit <- cal_isotonic(calibration$raw_p, calibration$y)
hist_fit <- cal_histogram(calibration$raw_p, calibration$y, bins = 10)
temp_fit <- cal_temperature(calibration$raw_logits, calibration$y)

test <- test |>
  mutate(
    platt = predict(platt_fit, raw_p),
    isotonic = predict(iso_fit, raw_p),
    histogram = predict(hist_fit, raw_p),
    temperature = predict(temp_fit, raw_logits)
  )

bind_rows(
  test |> summarise(method = "raw", ece = ece(raw_p, y, bins = 10)),
  test |> summarise(method = "platt", ece = ece(platt, y, bins = 10)),
  test |> summarise(method = "beta", ece = ece(beta, y, bins = 10)),
  test |> summarise(method = "isotonic", ece = ece(isotonic, y, bins = 10)),
  test |> summarise(method = "histogram", ece = ece(histogram, y, bins = 10)),
  test |> summarise(method = "temperature", ece = ece(temperature, y, bins = 10))
) |>
  arrange(ece)

## ----diagram, fig.width = 6, fig.height = 5, fig.alt = "Reliability diagram with points near the diagonal, comparing predicted probability and observed event frequency by bin."----
reliability_diagram(test$beta, test$y, bins = 10)

## ----cv-----------------------------------------------------------------------
cv_fit <- cal_cv(
  predictions$raw_p,
  predictions$y,
  method = "histogram",
  folds = 5,
  bins = 10,
  seed = 1
)

predictions |>
  mutate(oof = cv_fit$oof_predictions) |>
  summarise(oof_ece = ece(oof, y, bins = 10))