example_bayes_probit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare Bayesian Probit model to classical Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([0, 1]), array([4137, 5863]))\n",
      "Iter. 10 Lower Bound -2189.6795 - Delta LB 34.0317\n",
      "Iter. 20 Lower Bound -2065.0435 - Delta LB 4.3064\n",
      "Iter. 30 Lower Bound -2047.5047 - Delta LB 0.6552\n",
      "Iter. 40 Lower Bound -2044.7978 - Delta LB 0.1022\n",
      "Iter. 50 Lower Bound -2044.3744 - Delta LB 0.016\n",
      "Iter. 60 Lower Bound -2044.3081 - Delta LB 0.0025\n",
      "Iter. 70 Lower Bound -2044.2977 - Delta LB 0.0004\n",
      "Converged!\n",
      "\n",
      "Classification Report for Bayesian Probit Model:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.84      0.78      1239\n",
      "           1       0.87      0.77      0.82      1761\n",
      "\n",
      "    accuracy                           0.80      3000\n",
      "   macro avg       0.80      0.81      0.80      3000\n",
      "weighted avg       0.81      0.80      0.80      3000\n",
      "\n",
      "Classification Report for Logistic Regression Model:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.76      0.77      1239\n",
      "           1       0.83      0.84      0.84      1761\n",
      "\n",
      "    accuracy                           0.81      3000\n",
      "   macro avg       0.80      0.80      0.80      3000\n",
      "weighted avg       0.81      0.81      0.81      3000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "from bayes_linear.bprobit import BayesProbit, BayesProbit_MCMC, BayesProbit_VI\n",
    "\n",
    "# Generate the data using the BayesProbit class\n",
    "bayes_probit = BayesProbit(verbose=True)\n",
    "N = 10000  # Number of data points\n",
    "D = 10     # Number of features\n",
    "y, X, true_theta = bayes_probit.simulate_DGP(N=N, D=D, mu_theta=0, set_seed=42)\n",
    "\n",
    "print(np.unique(y, return_counts=True))\n",
    "\n",
    "# Split the data into training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "\n",
    "# Fit the Bayesian Probit model\n",
    "#bayes_model = BayesProbit_MCMC(N_sim=5000, burn_in=1000, verbose=True, pred_mode=['full'])\n",
    "bayes_model = BayesProbit_VI(max_iter=1000, epsilon_conv=1e-4, verbose=True)\n",
    "\n",
    "bayes_model.fit(X_train, y_train)\n",
    "\n",
    "# Predict with the Bayesian Probit model\n",
    "y_pred_bayes = bayes_model.predict(X_test)\n",
    "\n",
    "# Fit a logistic regression model\n",
    "logistic_model = LogisticRegression(random_state=42)\n",
    "logistic_model.fit(X_train, y_train)\n",
    "\n",
    "# Predict with the logistic regression model\n",
    "y_pred_logistic = logistic_model.predict(X_test)\n",
    "\n",
    "# Print classification reports\n",
    "print(\"Classification Report for Bayesian Probit Model:\")\n",
    "print(classification_report(y_test, y_pred_bayes))\n",
    "\n",
    "print(\"Classification Report for Logistic Regression Model:\")\n",
    "print(classification_report(y_test, y_pred_logistic))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optional: Transform X space according to polynomial basis transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter. 10 Lower Bound -2173.0405 - Delta LB 34.5207\n",
      "Iter. 20 Lower Bound -2045.7585 - Delta LB 4.4453\n",
      "Iter. 30 Lower Bound -2027.5063 - Delta LB 0.6905\n",
      "Iter. 40 Lower Bound -2024.6283 - Delta LB 0.1101\n",
      "Iter. 50 Lower Bound -2024.1681 - Delta LB 0.0176\n",
      "Iter. 60 Lower Bound -2024.0944 - Delta LB 0.0028\n",
      "Iter. 70 Lower Bound -2024.0825 - Delta LB 0.0005\n",
      "Converged!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from bayes_linear.bprobit import design_matrix\n",
    "\n",
    "# Split the data into training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "\n",
    "dmat = design_matrix(n_features=3, add_bias=False)\n",
    "\n",
    "X_train = dmat.fit_transform(X_train)\n",
    "X_test = dmat.transform(X_test)\n",
    "\n",
    "# Fit the Bayesian Probit model\n",
    "#bayes_model = BayesProbit_MCMC(N_sim=5000, burn_in=1000, verbose=True, pred_mode=['full'])\n",
    "bayes_model = BayesProbit_VI(max_iter=1000, epsilon_conv=1e-4, verbose=True)\n",
    "\n",
    "fitted = bayes_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report for Bayesian Probit Model:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.82      0.77      1239\n",
      "           1       0.86      0.79      0.82      1761\n",
      "\n",
      "    accuracy                           0.80      3000\n",
      "   macro avg       0.80      0.80      0.80      3000\n",
      "weighted avg       0.81      0.80      0.80      3000\n",
      "\n",
      "Classification Report for Logistic Regression Model:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.76      0.76      1239\n",
      "           1       0.83      0.84      0.84      1761\n",
      "\n",
      "    accuracy                           0.81      3000\n",
      "   macro avg       0.80      0.80      0.80      3000\n",
      "weighted avg       0.81      0.81      0.81      3000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Predict with the Bayesian Probit MCMC model\n",
    "y_pred_bayes = bayes_model.predict(X_test)\n",
    "\n",
    "# Fit a logistic regression model\n",
    "logistic_model = LogisticRegression(random_state=42)\n",
    "logistic_model.fit(X_train, y_train)\n",
    "\n",
    "# Predict with the logistic regression model\n",
    "y_pred_logistic = logistic_model.predict(X_test)\n",
    "\n",
    "# Print classification reports\n",
    "print(\"Classification Report for Bayesian Probit Model:\")\n",
    "print(classification_report(y_test, y_pred_bayes))\n",
    "\n",
    "print(\"Classification Report for Logistic Regression Model:\")\n",
    "print(classification_report(y_test, y_pred_logistic))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sandbox",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}