{
  "schema_version": "1.0",
  "study_id": "S04",
  "run_id": "S04-36e06a94-2716e1bf",
  "code_version": "36e06a9402b561bcf051e186c11ae9aeabf8aa42",
  "source_sha256": {
    "research_program/__init__.py": "5abb4829b477fd3068ee4c11dad2b59cd33e90aad06a17a5e9fb2f840a75abf1",
    "research_program/evaluation.py": "09a5d060c34b8cd1e5a3330d90f5ff3e12adbca8cfe1ebe3297da2b91d227e45",
    "research_program/io.py": "16de123c16d80bce5eb8d6766224dec1729d7c00449c2528f2348d86ad68d35f",
    "research_program/validate.py": "3e6a591e2b77799bdf73683db438b2998e2e457340626c98b7ceed510ab3329e",
    "studies/S04/config.json": "6fd142c850fa1c1b3b60b83640f52371c1b4f7be7bb6c8802cdd3bec043d9e4a",
    "studies/S04/effects.py": "69283d7daf8ffd89145615448acffe0a180d7edb3f37ed92901f5ed0034fca7c",
    "studies/S04/ingest.py": "a7e0c9ec87561df73dbe4133696f6bc0709ef67b2032326c745185109e9c666f",
    "studies/S04/study.py": "3fea8664792268e3953d34aba6e469f34e2986f003a8afecbea87edd7381a281",
    "uv.lock": "15f381c7c88897e1df049af6297905625b6869a6c9ff110ab1647ec2d0f305f4"
  },
  "evaluated_on": "2026-09-29",
  "data": {
    "source_url": "https://ailab.criteo.com/criteo-uplift-prediction-dataset/",
    "release": "Corrected Criteo v2.1; downloaded 2026-09-29",
    "sha256": "2716e1bf0fd157a93b5bf86924d9088419dfbac2022c6cd90030220634f616dc",
    "license": "CC BY-NC-SA 4.0; https://creativecommons.org/licenses/by-nc-sa/4.0/; aggregate research transformations; no warranty.",
    "citation": "Diemert, Betlei, Renaudin & Amini (2018), A Large Scale Benchmark for Uplift Modeling, AdKDD/TargetAd Workshop.",
    "coverage": "13,979,592 source records; anonymous benchmark with no released dates or campaign IDs."
  },
  "target": {
    "label": "Recorded conversion under assigned advertising eligibility",
    "estimand": "Released-benchmark policy treatment contrast conditional on sampling and exchangeability assumptions",
    "prediction_cutoff": "Anonymized baseline features only; realized exposure and both outcomes excluded",
    "horizon": "Publisher conversion label; exact attribution window and per-record dates unavailable",
    "unit": "Released anonymous record, clustered by matching feature profile"
  },
  "cohort": "Outcome-independent profile-hash sample; disjoint development/validation/evaluation profiles; bounded fitting subset.",
  "samples": {
    "total": 1995142,
    "train": 596910,
    "validation": 399506,
    "test": 398506,
    "test_profiles": 352598,
    "events": {
      "train": 1777,
      "validation": 1132,
      "test": 1243
    }
  },
  "models": [
    {
      "id": "dr_l2_10",
      "label": "Doubly robust learner · L2 10",
      "specification": {
        "family": "cross-fitted doubly robust learner",
        "l2": 10,
        "leaves": 7,
        "iterations": 120
      }
    },
    {
      "id": "dr_l2_100",
      "label": "Doubly robust learner · L2 100",
      "specification": {
        "family": "cross-fitted doubly robust learner",
        "l2": 100,
        "leaves": 7,
        "iterations": 120
      }
    },
    {
      "id": "honest_forest_100",
      "label": "Honest causal forest · leaf 100",
      "specification": {
        "family": "EconML honest causal forest DML",
        "min_leaf": 100,
        "trees": 128,
        "depth": 10,
        "max_samples": 0.45,
        "honest": true,
        "inference": "independent profile-cluster policy intervals"
      }
    },
    {
      "id": "honest_forest_500",
      "label": "Honest causal forest · leaf 500",
      "specification": {
        "family": "EconML honest causal forest DML",
        "min_leaf": 500,
        "trees": 128,
        "depth": 10,
        "max_samples": 0.45,
        "honest": true,
        "inference": "independent profile-cluster policy intervals"
      }
    },
    {
      "id": "response",
      "label": "Response-probability targeting",
      "specification": {
        "family": "development-fitted treated conditional mean"
      }
    },
    {
      "id": "random",
      "label": "Expected random allocation",
      "specification": {
        "family": "expected random allocation"
      }
    }
  ],
  "metrics": [
    {
      "name": "incremental_benchmark_conversions_per_10000",
      "model": "dr_l2_10",
      "unit": "incremental benchmark conversions per 10000 eligible records",
      "split": "untouched profile-separated final cohort",
      "estimate": 9.5433928,
      "lower": 5.87596261,
      "upper": 13.21082299,
      "confidence": 0.95,
      "standard_error": 1.87117224,
      "clusters": 352598
    },
    {
      "name": "incremental_benchmark_conversions_per_10000",
      "model": "dr_l2_100",
      "unit": "incremental benchmark conversions per 10000 eligible records",
      "split": "untouched profile-separated final cohort",
      "estimate": 9.66355226,
      "lower": 6.02288456,
      "upper": 13.30421996,
      "confidence": 0.95,
      "standard_error": 1.85751765,
      "clusters": 352598
    },
    {
      "name": "incremental_benchmark_conversions_per_10000",
      "model": "honest_forest_100",
      "unit": "incremental benchmark conversions per 10000 eligible records",
      "split": "untouched profile-separated final cohort",
      "estimate": 9.49525489,
      "lower": 5.69709872,
      "upper": 13.29341106,
      "confidence": 0.95,
      "standard_error": 1.93787039,
      "clusters": 352598
    },
    {
      "name": "incremental_benchmark_conversions_per_10000",
      "model": "honest_forest_500",
      "unit": "incremental benchmark conversions per 10000 eligible records",
      "split": "untouched profile-separated final cohort",
      "estimate": 9.77926724,
      "lower": 5.96002988,
      "upper": 13.59850459,
      "confidence": 0.95,
      "standard_error": 1.94862629,
      "clusters": 352598
    },
    {
      "name": "incremental_benchmark_conversions_per_10000",
      "model": "response",
      "unit": "incremental benchmark conversions per 10000 eligible records",
      "split": "untouched profile-separated final cohort",
      "estimate": 9.86157129,
      "lower": 6.05611669,
      "upper": 13.66702589,
      "confidence": 0.95,
      "standard_error": 1.94159415,
      "clusters": 352598
    },
    {
      "name": "incremental_benchmark_conversions_per_10000",
      "model": "random",
      "unit": "incremental benchmark conversions per 10000 eligible records",
      "split": "untouched profile-separated final cohort",
      "estimate": 2.04057666,
      "lower": 1.24385683,
      "upper": 2.83729648,
      "confidence": 0.95,
      "standard_error": 0.40649718,
      "clusters": 352598
    }
  ],
  "uncertainty": "Paired 95% profile-cluster normal intervals from independent held-out score contributions. Conditional on fitted models and rankings; unobserved campaign dependence and privacy-selection bias are not identified.",
  "assumptions": [
    "Consistency, overlap and benchmark-sample exchangeability are needed for causal interpretation.",
    "Non-uniform privacy subsampling prevents recovery of original advertiser effects or economics.",
    "Matching profiles are grouped conservatively; they are not asserted to identify people.",
    "Assumed conversion value and contact cost are scenario inputs only."
  ],
  "limitations": [
    "No dates, campaign IDs, demographic meanings or original experimental strata are released.",
    "Rare conversion and control events can make capacity-specific contrasts imprecise.",
    "Cluster intervals omit fitting uncertainty and are not simultaneous frontier guarantees.",
    "Profile-hash sampling and bounded fitting do not establish full-source or current-market performance.",
    "Exposure and visit are excluded as post-assignment variables.",
    "No advertiser ROI, realized profit or fairness conclusion is identified.",
    "Independent technical review is pending."
  ],
  "tables": {
    "audit": {
      "source_rows": 13979592,
      "source_label_counts": {
        "treatment": 11882655,
        "conversion": 40774,
        "visit": 656929,
        "exposure": 428212
      },
      "missing": {
        "f0": 0,
        "f1": 0,
        "f2": 0,
        "f3": 0,
        "f4": 0,
        "f5": 0,
        "f6": 0,
        "f7": 0,
        "f8": 0,
        "f9": 0,
        "f10": 0,
        "f11": 0,
        "treatment": 0,
        "conversion": 0,
        "visit": 0,
        "exposure": 0
      },
      "bounded_rows": 1995142,
      "bounded_unique_profiles": 1763767,
      "repeated_profile_rows": 231375,
      "partitions": {
        "development": {
          "rows": 596910,
          "profiles": 527686,
          "cells": [
            {
              "treatment": 0,
              "conversion": 0,
              "n": 89440
            },
            {
              "treatment": 0,
              "conversion": 1,
              "n": 170
            },
            {
              "treatment": 1,
              "conversion": 0,
              "n": 505693
            },
            {
              "treatment": 1,
              "conversion": 1,
              "n": 1607
            }
          ],
          "exposure_cells": [
            {
              "treatment": 0,
              "exposure": 0,
              "n": 89610
            },
            {
              "treatment": 1,
              "exposure": 0,
              "n": 489082
            },
            {
              "treatment": 1,
              "exposure": 1,
              "n": 18218
            }
          ]
        },
        "validation": {
          "rows": 399506,
          "profiles": 353275,
          "cells": [
            {
              "treatment": 0,
              "conversion": 0,
              "n": 59695
            },
            {
              "treatment": 0,
              "conversion": 1,
              "n": 113
            },
            {
              "treatment": 1,
              "conversion": 0,
              "n": 338679
            },
            {
              "treatment": 1,
              "conversion": 1,
              "n": 1019
            }
          ],
          "exposure_cells": [
            {
              "treatment": 0,
              "exposure": 0,
              "n": 59808
            },
            {
              "treatment": 1,
              "exposure": 0,
              "n": 327572
            },
            {
              "treatment": 1,
              "exposure": 1,
              "n": 12126
            }
          ]
        },
        "test": {
          "rows": 398506,
          "profiles": 352598,
          "cells": [
            {
              "treatment": 0,
              "conversion": 0,
              "n": 59891
            },
            {
              "treatment": 0,
              "conversion": 1,
              "n": 127
            },
            {
              "treatment": 1,
              "conversion": 0,
              "n": 337372
            },
            {
              "treatment": 1,
              "conversion": 1,
              "n": 1116
            }
          ],
          "exposure_cells": [
            {
              "treatment": 0,
              "exposure": 0,
              "n": 60018
            },
            {
              "treatment": 1,
              "exposure": 0,
              "n": 326267
            },
            {
              "treatment": 1,
              "exposure": 1,
              "n": 12221
            }
          ]
        }
      }
    },
    "crossfit": [
      {
        "fold": 0,
        "fit_rows": 397845,
        "held_rows": 199065,
        "held_conversions": 550,
        "held_control": 30057
      },
      {
        "fold": 1,
        "fit_rows": 397331,
        "held_rows": 199579,
        "held_conversions": 635,
        "held_control": 29911
      },
      {
        "fold": 2,
        "fit_rows": 398644,
        "held_rows": 198266,
        "held_conversions": 592,
        "held_control": 29642
      }
    ],
    "tuning": [
      {
        "model": "dr_l2_10",
        "validation_contrast_per_10000": 7.69757602,
        "selected": 79901
      },
      {
        "model": "dr_l2_100",
        "validation_contrast_per_10000": 7.15341574,
        "selected": 79901
      },
      {
        "model": "honest_forest_100",
        "validation_contrast_per_10000": 8.27337988,
        "selected": 79901
      },
      {
        "model": "honest_forest_500",
        "validation_contrast_per_10000": 8.31766578,
        "selected": 79901
      }
    ],
    "selected_model": "honest_forest_500",
    "primary_difference": {
      "estimate": -0.08230405,
      "lower": -0.89144603,
      "upper": 0.72683793,
      "confidence": 0.95,
      "standard_error": 0.41283513,
      "clusters": 352598,
      "comparison": "honest_forest_500 minus response targeting",
      "capacity": 0.2
    },
    "capacity_frontier": [
      {
        "model": "dr_l2_10",
        "capacity": 0,
        "selected": 0.0,
        "selected_fraction": 0.0,
        "kind": "frozen ranked selection",
        "estimate": 0.0,
        "lower": 0.0,
        "upper": 0.0,
        "confidence": 0.95,
        "standard_error": 0.0,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 0.0,
        "selected_control": 0.0,
        "observed_conversions_in_selection": 0.0
      },
      {
        "model": "dr_l2_10",
        "capacity": 0.01,
        "selected": 3985.0,
        "selected_fraction": 0.00999985,
        "kind": "frozen ranked selection",
        "estimate": 4.36858487,
        "lower": 1.77910972,
        "upper": 6.95806002,
        "confidence": 0.95,
        "standard_error": 1.32118507,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 3462.0,
        "selected_control": 523.0,
        "observed_conversions_in_selection": 544.0
      },
      {
        "model": "dr_l2_10",
        "capacity": 0.05,
        "selected": 19925.0,
        "selected_fraction": 0.04999925,
        "kind": "frozen ranked selection",
        "estimate": 8.41535538,
        "lower": 5.16258806,
        "upper": 11.6681227,
        "confidence": 0.95,
        "standard_error": 1.65960566,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 17323.0,
        "selected_control": 2602.0,
        "observed_conversions_in_selection": 867.0
      },
      {
        "model": "dr_l2_10",
        "capacity": 0.1,
        "selected": 39850.0,
        "selected_fraction": 0.09999849,
        "kind": "frozen ranked selection",
        "estimate": 8.24877485,
        "lower": 4.71185016,
        "upper": 11.78569955,
        "confidence": 0.95,
        "standard_error": 1.80458658,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 34401.0,
        "selected_control": 5449.0,
        "observed_conversions_in_selection": 961.0
      },
      {
        "model": "dr_l2_10",
        "capacity": 0.2,
        "selected": 79701.0,
        "selected_fraction": 0.1999995,
        "kind": "frozen ranked selection",
        "estimate": 9.5433928,
        "lower": 5.87596261,
        "upper": 13.21082299,
        "confidence": 0.95,
        "standard_error": 1.87117224,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 68147.0,
        "selected_control": 11554.0,
        "observed_conversions_in_selection": 1042.0
      },
      {
        "model": "dr_l2_10",
        "capacity": 0.3,
        "selected": 119551.0,
        "selected_fraction": 0.29999799,
        "kind": "frozen ranked selection",
        "estimate": 9.42122675,
        "lower": 5.74739747,
        "upper": 13.09505602,
        "confidence": 0.95,
        "standard_error": 1.87443713,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 107766.0,
        "selected_control": 11785.0,
        "observed_conversions_in_selection": 1056.0
      },
      {
        "model": "dr_l2_10",
        "capacity": 0.5,
        "selected": 199253.0,
        "selected_fraction": 0.5,
        "kind": "frozen ranked selection",
        "estimate": 9.51940191,
        "lower": 5.82872215,
        "upper": 13.21008167,
        "confidence": 0.95,
        "standard_error": 1.88303448,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 187468.0,
        "selected_control": 11785.0,
        "observed_conversions_in_selection": 1093.0
      },
      {
        "model": "dr_l2_10",
        "capacity": 0.75,
        "selected": 298879.0,
        "selected_fraction": 0.74999875,
        "kind": "frozen ranked selection",
        "estimate": 10.18155906,
        "lower": 6.34439707,
        "upper": 14.01872104,
        "confidence": 0.95,
        "standard_error": 1.95777168,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 250882.0,
        "selected_control": 47997.0,
        "observed_conversions_in_selection": 1121.0
      },
      {
        "model": "dr_l2_10",
        "capacity": 1,
        "selected": 398506.0,
        "selected_fraction": 1.0,
        "kind": "frozen ranked selection",
        "estimate": 10.20288328,
        "lower": 6.21928413,
        "upper": 14.18648242,
        "confidence": 0.95,
        "standard_error": 2.03248589,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 338488.0,
        "selected_control": 60018.0,
        "observed_conversions_in_selection": 1243.0
      },
      {
        "model": "dr_l2_100",
        "capacity": 0,
        "selected": 0.0,
        "selected_fraction": 0.0,
        "kind": "frozen ranked selection",
        "estimate": 0.0,
        "lower": 0.0,
        "upper": 0.0,
        "confidence": 0.95,
        "standard_error": 0.0,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 0.0,
        "selected_control": 0.0,
        "observed_conversions_in_selection": 0.0
      },
      {
        "model": "dr_l2_100",
        "capacity": 0.01,
        "selected": 3985.0,
        "selected_fraction": 0.00999985,
        "kind": "frozen ranked selection",
        "estimate": 4.37955701,
        "lower": 1.73991395,
        "upper": 7.01920008,
        "confidence": 0.95,
        "standard_error": 1.34678142,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 3449.0,
        "selected_control": 536.0,
        "observed_conversions_in_selection": 557.0
      },
      {
        "model": "dr_l2_100",
        "capacity": 0.05,
        "selected": 19925.0,
        "selected_fraction": 0.04999925,
        "kind": "frozen ranked selection",
        "estimate": 8.30757641,
        "lower": 5.01937763,
        "upper": 11.59577518,
        "confidence": 0.95,
        "standard_error": 1.67768326,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 17261.0,
        "selected_control": 2664.0,
        "observed_conversions_in_selection": 875.0
      },
      {
        "model": "dr_l2_100",
        "capacity": 0.1,
        "selected": 39850.0,
        "selected_fraction": 0.09999849,
        "kind": "frozen ranked selection",
        "estimate": 8.42253127,
        "lower": 4.91101374,
        "upper": 11.93404881,
        "confidence": 0.95,
        "standard_error": 1.7916235,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 34427.0,
        "selected_control": 5423.0,
        "observed_conversions_in_selection": 958.0
      },
      {
        "model": "dr_l2_100",
        "capacity": 0.2,
        "selected": 79701.0,
        "selected_fraction": 0.1999995,
        "kind": "frozen ranked selection",
        "estimate": 9.66355226,
        "lower": 6.02288456,
        "upper": 13.30421996,
        "confidence": 0.95,
        "standard_error": 1.85751765,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 68153.0,
        "selected_control": 11548.0,
        "observed_conversions_in_selection": 1038.0
      },
      {
        "model": "dr_l2_100",
        "capacity": 0.3,
        "selected": 119551.0,
        "selected_fraction": 0.29999799,
        "kind": "frozen ranked selection",
        "estimate": 9.54289619,
        "lower": 5.89578245,
        "upper": 13.19000994,
        "confidence": 0.95,
        "standard_error": 1.86080651,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 107772.0,
        "selected_control": 11779.0,
        "observed_conversions_in_selection": 1052.0
      },
      {
        "model": "dr_l2_100",
        "capacity": 0.5,
        "selected": 199253.0,
        "selected_fraction": 0.5,
        "kind": "frozen ranked selection",
        "estimate": 9.64106381,
        "lower": 5.97697659,
        "upper": 13.30515103,
        "confidence": 0.95,
        "standard_error": 1.86946661,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 187474.0,
        "selected_control": 11779.0,
        "observed_conversions_in_selection": 1089.0
      },
      {
        "model": "dr_l2_100",
        "capacity": 0.75,
        "selected": 298879.0,
        "selected_fraction": 0.74999875,
        "kind": "frozen ranked selection",
        "estimate": 10.30319139,
        "lower": 6.49160011,
        "upper": 14.11478266,
        "confidence": 0.95,
        "standard_error": 1.94472516,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 250888.0,
        "selected_control": 47991.0,
        "observed_conversions_in_selection": 1117.0
      },
      {
        "model": "dr_l2_100",
        "capacity": 1,
        "selected": 398506.0,
        "selected_fraction": 1.0,
        "kind": "frozen ranked selection",
        "estimate": 10.20288328,
        "lower": 6.21928413,
        "upper": 14.18648242,
        "confidence": 0.95,
        "standard_error": 2.03248589,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 338488.0,
        "selected_control": 60018.0,
        "observed_conversions_in_selection": 1243.0
      },
      {
        "model": "honest_forest_100",
        "capacity": 0,
        "selected": 0.0,
        "selected_fraction": 0.0,
        "kind": "frozen ranked selection",
        "estimate": 0.0,
        "lower": 0.0,
        "upper": 0.0,
        "confidence": 0.95,
        "standard_error": 0.0,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 0.0,
        "selected_control": 0.0,
        "observed_conversions_in_selection": 0.0
      },
      {
        "model": "honest_forest_100",
        "capacity": 0.01,
        "selected": 3985.0,
        "selected_fraction": 0.00999985,
        "kind": "frozen ranked selection",
        "estimate": 3.75428945,
        "lower": 1.0048918,
        "upper": 6.5036871,
        "confidence": 0.95,
        "standard_error": 1.40277968,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 3477.0,
        "selected_control": 508.0,
        "observed_conversions_in_selection": 640.0
      },
      {
        "model": "honest_forest_100",
        "capacity": 0.05,
        "selected": 19925.0,
        "selected_fraction": 0.04999925,
        "kind": "frozen ranked selection",
        "estimate": 8.1143818,
        "lower": 4.71518467,
        "upper": 11.51357894,
        "confidence": 0.95,
        "standard_error": 1.73431612,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 17324.0,
        "selected_control": 2601.0,
        "observed_conversions_in_selection": 960.0
      },
      {
        "model": "honest_forest_100",
        "capacity": 0.1,
        "selected": 39850.0,
        "selected_fraction": 0.09999849,
        "kind": "frozen ranked selection",
        "estimate": 9.34473123,
        "lower": 5.75427596,
        "upper": 12.93518651,
        "confidence": 0.95,
        "standard_error": 1.8318986,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 34410.0,
        "selected_control": 5440.0,
        "observed_conversions_in_selection": 1066.0
      },
      {
        "model": "honest_forest_100",
        "capacity": 0.2,
        "selected": 79701.0,
        "selected_fraction": 0.1999995,
        "kind": "frozen ranked selection",
        "estimate": 9.49525489,
        "lower": 5.69709872,
        "upper": 13.29341106,
        "confidence": 0.95,
        "standard_error": 1.93787039,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 68359.0,
        "selected_control": 11342.0,
        "observed_conversions_in_selection": 1147.0
      },
      {
        "model": "honest_forest_100",
        "capacity": 0.3,
        "selected": 119551.0,
        "selected_fraction": 0.29999799,
        "kind": "frozen ranked selection",
        "estimate": 9.4777745,
        "lower": 5.61363453,
        "upper": 13.34191448,
        "confidence": 0.95,
        "standard_error": 1.97153621,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 102162.0,
        "selected_control": 17389.0,
        "observed_conversions_in_selection": 1173.0
      },
      {
        "model": "honest_forest_100",
        "capacity": 0.5,
        "selected": 199253.0,
        "selected_fraction": 0.5,
        "kind": "frozen ranked selection",
        "estimate": 9.31394784,
        "lower": 5.39091677,
        "upper": 13.23697891,
        "confidence": 0.95,
        "standard_error": 2.00158324,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 169645.0,
        "selected_control": 29608.0,
        "observed_conversions_in_selection": 1193.0
      },
      {
        "model": "honest_forest_100",
        "capacity": 0.75,
        "selected": 298879.0,
        "selected_fraction": 0.74999875,
        "kind": "frozen ranked selection",
        "estimate": 9.67023121,
        "lower": 5.74203198,
        "upper": 13.59843044,
        "confidence": 0.95,
        "standard_error": 2.00422011,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 254162.0,
        "selected_control": 44717.0,
        "observed_conversions_in_selection": 1205.0
      },
      {
        "model": "honest_forest_100",
        "capacity": 1,
        "selected": 398506.0,
        "selected_fraction": 1.0,
        "kind": "frozen ranked selection",
        "estimate": 10.20288328,
        "lower": 6.21928413,
        "upper": 14.18648242,
        "confidence": 0.95,
        "standard_error": 2.03248589,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 338488.0,
        "selected_control": 60018.0,
        "observed_conversions_in_selection": 1243.0
      },
      {
        "model": "honest_forest_500",
        "capacity": 0,
        "selected": 0.0,
        "selected_fraction": 0.0,
        "kind": "frozen ranked selection",
        "estimate": 0.0,
        "lower": 0.0,
        "upper": 0.0,
        "confidence": 0.95,
        "standard_error": 0.0,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 0.0,
        "selected_control": 0.0,
        "observed_conversions_in_selection": 0.0
      },
      {
        "model": "honest_forest_500",
        "capacity": 0.01,
        "selected": 3985.0,
        "selected_fraction": 0.00999985,
        "kind": "frozen ranked selection",
        "estimate": 3.63228626,
        "lower": 0.8651646,
        "upper": 6.39940792,
        "confidence": 0.95,
        "standard_error": 1.41182271,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 3462.0,
        "selected_control": 523.0,
        "observed_conversions_in_selection": 644.0
      },
      {
        "model": "honest_forest_500",
        "capacity": 0.05,
        "selected": 19925.0,
        "selected_fraction": 0.04999925,
        "kind": "frozen ranked selection",
        "estimate": 8.19451046,
        "lower": 4.79003359,
        "upper": 11.59898733,
        "confidence": 0.95,
        "standard_error": 1.73700991,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 17329.0,
        "selected_control": 2596.0,
        "observed_conversions_in_selection": 968.0
      },
      {
        "model": "honest_forest_500",
        "capacity": 0.1,
        "selected": 39850.0,
        "selected_fraction": 0.09999849,
        "kind": "frozen ranked selection",
        "estimate": 9.23275694,
        "lower": 5.58553955,
        "upper": 12.87997434,
        "confidence": 0.95,
        "standard_error": 1.8608594,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 34421.0,
        "selected_control": 5429.0,
        "observed_conversions_in_selection": 1084.0
      },
      {
        "model": "honest_forest_500",
        "capacity": 0.2,
        "selected": 79701.0,
        "selected_fraction": 0.1999995,
        "kind": "frozen ranked selection",
        "estimate": 9.77926724,
        "lower": 5.96002988,
        "upper": 13.59850459,
        "confidence": 0.95,
        "standard_error": 1.94862629,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 68376.0,
        "selected_control": 11325.0,
        "observed_conversions_in_selection": 1164.0
      },
      {
        "model": "honest_forest_500",
        "capacity": 0.3,
        "selected": 119551.0,
        "selected_fraction": 0.29999799,
        "kind": "frozen ranked selection",
        "estimate": 9.97646667,
        "lower": 6.12160454,
        "upper": 13.83132881,
        "confidence": 0.95,
        "standard_error": 1.96680253,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 102205.0,
        "selected_control": 17346.0,
        "observed_conversions_in_selection": 1184.0
      },
      {
        "model": "honest_forest_500",
        "capacity": 0.5,
        "selected": 199253.0,
        "selected_fraction": 0.5,
        "kind": "frozen ranked selection",
        "estimate": 9.97602306,
        "lower": 6.05802247,
        "upper": 13.89402365,
        "confidence": 0.95,
        "standard_error": 1.99901662,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 169714.0,
        "selected_control": 29539.0,
        "observed_conversions_in_selection": 1210.0
      },
      {
        "model": "honest_forest_500",
        "capacity": 0.75,
        "selected": 298879.0,
        "selected_fraction": 0.74999875,
        "kind": "frozen ranked selection",
        "estimate": 10.21057844,
        "lower": 6.27401511,
        "upper": 14.14714178,
        "confidence": 0.95,
        "standard_error": 2.00848759,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 254164.0,
        "selected_control": 44715.0,
        "observed_conversions_in_selection": 1224.0
      },
      {
        "model": "honest_forest_500",
        "capacity": 1,
        "selected": 398506.0,
        "selected_fraction": 1.0,
        "kind": "frozen ranked selection",
        "estimate": 10.20288328,
        "lower": 6.21928413,
        "upper": 14.18648242,
        "confidence": 0.95,
        "standard_error": 2.03248589,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 338488.0,
        "selected_control": 60018.0,
        "observed_conversions_in_selection": 1243.0
      },
      {
        "model": "response",
        "capacity": 0,
        "selected": 0.0,
        "selected_fraction": 0.0,
        "kind": "frozen ranked selection",
        "estimate": 0.0,
        "lower": 0.0,
        "upper": 0.0,
        "confidence": 0.95,
        "standard_error": 0.0,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 0.0,
        "selected_control": 0.0,
        "observed_conversions_in_selection": 0.0
      },
      {
        "model": "response",
        "capacity": 0.01,
        "selected": 3985.0,
        "selected_fraction": 0.00999985,
        "kind": "frozen ranked selection",
        "estimate": 4.06202472,
        "lower": 1.24142264,
        "upper": 6.8826268,
        "confidence": 0.95,
        "standard_error": 1.43910914,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 3464.0,
        "selected_control": 521.0,
        "observed_conversions_in_selection": 665.0
      },
      {
        "model": "response",
        "capacity": 0.05,
        "selected": 19925.0,
        "selected_fraction": 0.04999925,
        "kind": "frozen ranked selection",
        "estimate": 8.43221807,
        "lower": 5.01305534,
        "upper": 11.8513808,
        "confidence": 0.95,
        "standard_error": 1.74450284,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 17312.0,
        "selected_control": 2613.0,
        "observed_conversions_in_selection": 979.0
      },
      {
        "model": "response",
        "capacity": 0.1,
        "selected": 39850.0,
        "selected_fraction": 0.09999849,
        "kind": "frozen ranked selection",
        "estimate": 8.7549347,
        "lower": 5.07949581,
        "upper": 12.43037358,
        "confidence": 0.95,
        "standard_error": 1.87525838,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 34352.0,
        "selected_control": 5498.0,
        "observed_conversions_in_selection": 1080.0
      },
      {
        "model": "response",
        "capacity": 0.2,
        "selected": 79701.0,
        "selected_fraction": 0.1999995,
        "kind": "frozen ranked selection",
        "estimate": 9.86157129,
        "lower": 6.05611669,
        "upper": 13.66702589,
        "confidence": 0.95,
        "standard_error": 1.94159415,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 68436.0,
        "selected_control": 11265.0,
        "observed_conversions_in_selection": 1160.0
      },
      {
        "model": "response",
        "capacity": 0.3,
        "selected": 119551.0,
        "selected_fraction": 0.29999799,
        "kind": "frozen ranked selection",
        "estimate": 10.46551176,
        "lower": 6.61845653,
        "upper": 14.31256699,
        "confidence": 0.95,
        "standard_error": 1.96281935,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 101908.0,
        "selected_control": 17643.0,
        "observed_conversions_in_selection": 1192.0
      },
      {
        "model": "response",
        "capacity": 0.5,
        "selected": 199253.0,
        "selected_fraction": 0.5,
        "kind": "frozen ranked selection",
        "estimate": 10.43208508,
        "lower": 6.52045663,
        "upper": 14.34371352,
        "confidence": 0.95,
        "standard_error": 1.99576547,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 170462.0,
        "selected_control": 28791.0,
        "observed_conversions_in_selection": 1220.0
      },
      {
        "model": "response",
        "capacity": 0.75,
        "selected": 298879.0,
        "selected_fraction": 0.74999875,
        "kind": "frozen ranked selection",
        "estimate": 9.98443421,
        "lower": 6.05277194,
        "upper": 13.91609647,
        "confidence": 0.95,
        "standard_error": 2.005987,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 264677.0,
        "selected_control": 34202.0,
        "observed_conversions_in_selection": 1237.0
      },
      {
        "model": "response",
        "capacity": 1,
        "selected": 398506.0,
        "selected_fraction": 1.0,
        "kind": "frozen ranked selection",
        "estimate": 10.20288328,
        "lower": 6.21928413,
        "upper": 14.18648242,
        "confidence": 0.95,
        "standard_error": 2.03248589,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 338488.0,
        "selected_control": 60018.0,
        "observed_conversions_in_selection": 1243.0
      },
      {
        "model": "random",
        "capacity": 0,
        "selected": 0.0,
        "selected_fraction": 0.0,
        "kind": "analytical expected selection",
        "estimate": 0.0,
        "lower": 0.0,
        "upper": 0.0,
        "confidence": 0.95,
        "standard_error": 0.0,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 0.0,
        "selected_control": 0.0,
        "observed_conversions_in_selection": 0.0
      },
      {
        "model": "random",
        "capacity": 0.01,
        "selected": 3985.06,
        "selected_fraction": 0.01,
        "kind": "analytical expected selection",
        "estimate": 0.10202883,
        "lower": 0.06219284,
        "upper": 0.14186482,
        "confidence": 0.95,
        "standard_error": 0.02032486,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 3384.88,
        "selected_control": 600.18,
        "observed_conversions_in_selection": 12.43
      },
      {
        "model": "random",
        "capacity": 0.05,
        "selected": 19925.3,
        "selected_fraction": 0.05,
        "kind": "analytical expected selection",
        "estimate": 0.51014416,
        "lower": 0.31096421,
        "upper": 0.70932412,
        "confidence": 0.95,
        "standard_error": 0.10162429,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 16924.4,
        "selected_control": 3000.9,
        "observed_conversions_in_selection": 62.15
      },
      {
        "model": "random",
        "capacity": 0.1,
        "selected": 39850.6,
        "selected_fraction": 0.1,
        "kind": "analytical expected selection",
        "estimate": 1.02028833,
        "lower": 0.62192841,
        "upper": 1.41864824,
        "confidence": 0.95,
        "standard_error": 0.20324859,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 33848.8,
        "selected_control": 6001.8,
        "observed_conversions_in_selection": 124.3
      },
      {
        "model": "random",
        "capacity": 0.2,
        "selected": 79701.2,
        "selected_fraction": 0.2,
        "kind": "analytical expected selection",
        "estimate": 2.04057666,
        "lower": 1.24385683,
        "upper": 2.83729648,
        "confidence": 0.95,
        "standard_error": 0.40649718,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 67697.6,
        "selected_control": 12003.6,
        "observed_conversions_in_selection": 248.6
      },
      {
        "model": "random",
        "capacity": 0.3,
        "selected": 119551.8,
        "selected_fraction": 0.3,
        "kind": "analytical expected selection",
        "estimate": 3.06086498,
        "lower": 1.86578524,
        "upper": 4.25594473,
        "confidence": 0.95,
        "standard_error": 0.60974577,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 101546.4,
        "selected_control": 18005.4,
        "observed_conversions_in_selection": 372.9
      },
      {
        "model": "random",
        "capacity": 0.5,
        "selected": 199253.0,
        "selected_fraction": 0.5,
        "kind": "analytical expected selection",
        "estimate": 5.10144164,
        "lower": 3.10964206,
        "upper": 7.09324121,
        "confidence": 0.95,
        "standard_error": 1.01624295,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 169244.0,
        "selected_control": 30009.0,
        "observed_conversions_in_selection": 621.5
      },
      {
        "model": "random",
        "capacity": 0.75,
        "selected": 298879.5,
        "selected_fraction": 0.75,
        "kind": "analytical expected selection",
        "estimate": 7.65216246,
        "lower": 4.6644631,
        "upper": 10.63986182,
        "confidence": 0.95,
        "standard_error": 1.52436442,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 253866.0,
        "selected_control": 45013.5,
        "observed_conversions_in_selection": 932.25
      },
      {
        "model": "random",
        "capacity": 1,
        "selected": 398506.0,
        "selected_fraction": 1.0,
        "kind": "analytical expected selection",
        "estimate": 10.20288328,
        "lower": 6.21928413,
        "upper": 14.18648242,
        "confidence": 0.95,
        "standard_error": 2.03248589,
        "clusters": 352598,
        "unit": "incremental benchmark conversions per 10000 eligible records",
        "selected_treated": 338488.0,
        "selected_control": 60018.0,
        "observed_conversions_in_selection": 1243.0
      }
    ],
    "effect_calibration": [
      {
        "model": "dr_l2_10",
        "decile": 1,
        "n": 39851,
        "events": 110,
        "treated": 39797,
        "predicted_effect_per_10000": -11.54155802,
        "estimate": -3.25200667,
        "lower": -13.27679188,
        "upper": 6.77277854,
        "confidence": 0.95,
        "standard_error": 5.11478032,
        "clusters": 38041
      },
      {
        "model": "dr_l2_10",
        "decile": 2,
        "n": 39851,
        "events": 24,
        "treated": 39851,
        "predicted_effect_per_10000": 2.16689848,
        "estimate": 2.11795406,
        "lower": -0.71716802,
        "upper": 4.95307615,
        "confidence": 0.95,
        "standard_error": 1.44651744,
        "clusters": 37957
      },
      {
        "model": "dr_l2_10",
        "decile": 3,
        "n": 39851,
        "events": 14,
        "treated": 39851,
        "predicted_effect_per_10000": 2.16689848,
        "estimate": -0.84341694,
        "lower": -3.00992921,
        "upper": 1.32309534,
        "confidence": 0.95,
        "standard_error": 1.10538372,
        "clusters": 37996
      },
      {
        "model": "dr_l2_10",
        "decile": 4,
        "n": 39851,
        "events": 15,
        "treated": 19353,
        "predicted_effect_per_10000": 2.16689848,
        "estimate": 4.60213384,
        "lower": -3.52389502,
        "upper": 12.72816271,
        "confidence": 0.95,
        "standard_error": 4.14600928,
        "clusters": 37955
      },
      {
        "model": "dr_l2_10",
        "decile": 5,
        "n": 39851,
        "events": 8,
        "treated": 29309,
        "predicted_effect_per_10000": 2.16689848,
        "estimate": 0.42673399,
        "lower": -5.23376066,
        "upper": 6.08722863,
        "confidence": 0.95,
        "standard_error": 2.88806054,
        "clusters": 39183
      },
      {
        "model": "dr_l2_10",
        "decile": 6,
        "n": 39851,
        "events": 10,
        "treated": 33228,
        "predicted_effect_per_10000": 2.16689848,
        "estimate": -0.4790888,
        "lower": -5.36720649,
        "upper": 4.40902889,
        "confidence": 0.95,
        "standard_error": 2.49398342,
        "clusters": 39120
      },
      {
        "model": "dr_l2_10",
        "decile": 7,
        "n": 39850,
        "events": 11,
        "treated": 35258,
        "predicted_effect_per_10000": 2.16689848,
        "estimate": 1.84214765,
        "lower": -0.07631815,
        "upper": 3.76061344,
        "confidence": 0.95,
        "standard_error": 0.97882707,
        "clusters": 39106
      },
      {
        "model": "dr_l2_10",
        "decile": 8,
        "n": 39850,
        "events": 30,
        "treated": 33046,
        "predicted_effect_per_10000": 2.2083163,
        "estimate": 4.91133592,
        "lower": -1.57792438,
        "upper": 11.40059621,
        "confidence": 0.95,
        "standard_error": 3.31090793,
        "clusters": 38837
      },
      {
        "model": "dr_l2_10",
        "decile": 9,
        "n": 39850,
        "events": 66,
        "treated": 34513,
        "predicted_effect_per_10000": 5.22317826,
        "estimate": 11.58528301,
        "lower": 4.03923147,
        "upper": 19.13133455,
        "confidence": 0.95,
        "standard_error": 3.85009704,
        "clusters": 39506
      },
      {
        "model": "dr_l2_10",
        "decile": 10,
        "n": 39850,
        "events": 955,
        "treated": 34282,
        "predicted_effect_per_10000": 109.66472504,
        "estimate": 81.11922836,
        "lower": 45.78548738,
        "upper": 116.45296934,
        "confidence": 0.95,
        "standard_error": 18.02775013,
        "clusters": 39708
      },
      {
        "model": "dr_l2_100",
        "decile": 1,
        "n": 39851,
        "events": 114,
        "treated": 39791,
        "predicted_effect_per_10000": -8.50919336,
        "estimate": -4.46861656,
        "lower": -15.42545618,
        "upper": 6.48822305,
        "confidence": 0.95,
        "standard_error": 5.59032702,
        "clusters": 38043
      },
      {
        "model": "dr_l2_100",
        "decile": 2,
        "n": 39851,
        "events": 24,
        "treated": 39851,
        "predicted_effect_per_10000": 2.33087376,
        "estimate": 2.1178661,
        "lower": -0.71725606,
        "upper": 4.95298826,
        "confidence": 0.95,
        "standard_error": 1.44651748,
        "clusters": 37956
      },
      {
        "model": "dr_l2_100",
        "decile": 3,
        "n": 39851,
        "events": 14,
        "treated": 39851,
        "predicted_effect_per_10000": 2.33087376,
        "estimate": -0.84332648,
        "lower": -3.0098388,
        "upper": 1.32318585,
        "confidence": 0.95,
        "standard_error": 1.10538374,
        "clusters": 37992
      },
      {
        "model": "dr_l2_100",
        "decile": 4,
        "n": 39851,
        "events": 15,
        "treated": 19388,
        "predicted_effect_per_10000": 2.33087376,
        "estimate": 4.57382015,
        "lower": -3.55220582,
        "upper": 12.69984613,
        "confidence": 0.95,
        "standard_error": 4.1460078,
        "clusters": 37955
      },
      {
        "model": "dr_l2_100",
        "decile": 5,
        "n": 39851,
        "events": 8,
        "treated": 29309,
        "predicted_effect_per_10000": 2.33087376,
        "estimate": 0.41852693,
        "lower": -5.24195718,
        "upper": 6.07901103,
        "confidence": 0.95,
        "standard_error": 2.88805516,
        "clusters": 39182
      },
      {
        "model": "dr_l2_100",
        "decile": 6,
        "n": 39851,
        "events": 10,
        "treated": 33228,
        "predicted_effect_per_10000": 2.33087376,
        "estimate": -0.47574822,
        "lower": -5.36387482,
        "upper": 4.41237838,
        "confidence": 0.95,
        "standard_error": 2.49398797,
        "clusters": 39121
      },
      {
        "model": "dr_l2_100",
        "decile": 7,
        "n": 39850,
        "events": 11,
        "treated": 35223,
        "predicted_effect_per_10000": 2.33087376,
        "estimate": 1.8753079,
        "lower": -0.0431812,
        "upper": 3.79379701,
        "confidence": 0.95,
        "standard_error": 0.97883896,
        "clusters": 39107
      },
      {
        "model": "dr_l2_100",
        "decile": 8,
        "n": 39850,
        "events": 30,
        "treated": 33046,
        "predicted_effect_per_10000": 2.46929909,
        "estimate": 4.93386377,
        "lower": -1.55538683,
        "upper": 11.42311437,
        "confidence": 0.95,
        "standard_error": 3.31090298,
        "clusters": 38834
      },
      {
        "model": "dr_l2_100",
        "decile": 9,
        "n": 39850,
        "events": 60,
        "treated": 34312,
        "predicted_effect_per_10000": 5.47062234,
        "estimate": 8.25286027,
        "lower": 0.19785173,
        "upper": 16.30786882,
        "confidence": 0.95,
        "standard_error": 4.10977375,
        "clusters": 39505
      },
      {
        "model": "dr_l2_100",
        "decile": 10,
        "n": 39850,
        "events": 957,
        "treated": 34489,
        "predicted_effect_per_10000": 106.75808611,
        "estimate": 85.64578191,
        "lower": 50.70426676,
        "upper": 120.58729706,
        "confidence": 0.95,
        "standard_error": 17.82763123,
        "clusters": 39708
      },
      {
        "model": "honest_forest_100",
        "decile": 1,
        "n": 39851,
        "events": 30,
        "treated": 33769,
        "predicted_effect_per_10000": -4.88660565,
        "estimate": 4.80590169,
        "lower": -0.77930704,
        "upper": 10.39111042,
        "confidence": 0.95,
        "standard_error": 2.84964865,
        "clusters": 33989
      },
      {
        "model": "honest_forest_100",
        "decile": 2,
        "n": 39851,
        "events": 5,
        "treated": 33754,
        "predicted_effect_per_10000": -1.22276129,
        "estimate": -0.42164909,
        "lower": -3.83095051,
        "upper": 2.98765233,
        "confidence": 0.95,
        "standard_error": 1.73947146,
        "clusters": 28258
      },
      {
        "model": "honest_forest_100",
        "decile": 3,
        "n": 39851,
        "events": 4,
        "treated": 33706,
        "predicted_effect_per_10000": -0.76225937,
        "estimate": 1.24291041,
        "lower": 0.07763699,
        "upper": 2.40818382,
        "confidence": 0.95,
        "standard_error": 0.59453818,
        "clusters": 30538
      },
      {
        "model": "honest_forest_100",
        "decile": 4,
        "n": 39851,
        "events": 4,
        "treated": 33800,
        "predicted_effect_per_10000": -0.45855776,
        "estimate": 1.1642759,
        "lower": -0.00017562,
        "upper": 2.32872742,
        "confidence": 0.95,
        "standard_error": 0.59411884,
        "clusters": 33023
      },
      {
        "model": "honest_forest_100",
        "decile": 5,
        "n": 39851,
        "events": 7,
        "treated": 33818,
        "predicted_effect_per_10000": -0.10591847,
        "estimate": 2.09598501,
        "lower": 0.55962072,
        "upper": 3.63234929,
        "confidence": 0.95,
        "standard_error": 0.78387373,
        "clusters": 35288
      },
      {
        "model": "honest_forest_100",
        "decile": 6,
        "n": 39851,
        "events": 9,
        "treated": 33786,
        "predicted_effect_per_10000": 0.50620937,
        "estimate": -4.98423115,
        "lower": -11.47568363,
        "upper": 1.50722133,
        "confidence": 0.95,
        "standard_error": 3.31202641,
        "clusters": 36617
      },
      {
        "model": "honest_forest_100",
        "decile": 7,
        "n": 39850,
        "events": 11,
        "treated": 33694,
        "predicted_effect_per_10000": 1.20395859,
        "estimate": 3.34778674,
        "lower": 1.42005265,
        "upper": 5.27552082,
        "confidence": 0.95,
        "standard_error": 0.98355587,
        "clusters": 36931
      },
      {
        "model": "honest_forest_100",
        "decile": 8,
        "n": 39850,
        "events": 26,
        "treated": 33803,
        "predicted_effect_per_10000": 2.81641084,
        "estimate": -0.174828,
        "lower": -7.2854724,
        "upper": 6.93581641,
        "confidence": 0.95,
        "standard_error": 3.62794646,
        "clusters": 38686
      },
      {
        "model": "honest_forest_100",
        "decile": 9,
        "n": 39850,
        "events": 81,
        "treated": 33948,
        "predicted_effect_per_10000": 10.77998219,
        "estimate": 1.50540018,
        "lower": -10.88297347,
        "upper": 13.89377382,
        "confidence": 0.95,
        "standard_error": 6.32071494,
        "clusters": 39476
      },
      {
        "model": "honest_forest_100",
        "decile": 10,
        "n": 39850,
        "events": 1066,
        "treated": 34410,
        "predicted_effect_per_10000": 108.32959142,
        "estimate": 93.44871933,
        "lower": 57.55374787,
        "upper": 129.34369079,
        "confidence": 0.95,
        "standard_error": 18.31409747,
        "clusters": 39805
      },
      {
        "model": "honest_forest_500",
        "decile": 1,
        "n": 39851,
        "events": 15,
        "treated": 33735,
        "predicted_effect_per_10000": -4.05993848,
        "estimate": 0.63099818,
        "lower": -4.46423166,
        "upper": 5.72622803,
        "confidence": 0.95,
        "standard_error": 2.59965483,
        "clusters": 32882
      },
      {
        "model": "honest_forest_500",
        "decile": 2,
        "n": 39851,
        "events": 4,
        "treated": 33767,
        "predicted_effect_per_10000": -1.17468686,
        "estimate": -0.78187799,
        "lower": -4.14016312,
        "upper": 2.57640714,
        "confidence": 0.95,
        "standard_error": 1.71344226,
        "clusters": 28809
      },
      {
        "model": "honest_forest_500",
        "decile": 3,
        "n": 39851,
        "events": 2,
        "treated": 33756,
        "predicted_effect_per_10000": -0.73535724,
        "estimate": 0.67254673,
        "lower": -0.1587366,
        "upper": 1.50383006,
        "confidence": 0.95,
        "standard_error": 0.42413194,
        "clusters": 31322
      },
      {
        "model": "honest_forest_500",
        "decile": 4,
        "n": 39851,
        "events": 2,
        "treated": 33798,
        "predicted_effect_per_10000": -0.44561728,
        "estimate": 0.59267208,
        "lower": -0.23455331,
        "upper": 1.41989747,
        "confidence": 0.95,
        "standard_error": 0.42206152,
        "clusters": 32656
      },
      {
        "model": "honest_forest_500",
        "decile": 5,
        "n": 39851,
        "events": 10,
        "treated": 33720,
        "predicted_effect_per_10000": -0.10504099,
        "estimate": 1.15400433,
        "lower": -2.48127067,
        "upper": 4.78927933,
        "confidence": 0.95,
        "standard_error": 1.85476622,
        "clusters": 35444
      },
      {
        "model": "honest_forest_500",
        "decile": 6,
        "n": 39851,
        "events": 11,
        "treated": 33798,
        "predicted_effect_per_10000": 0.64773296,
        "estimate": -4.53943129,
        "lower": -11.17452966,
        "upper": 2.09566708,
        "confidence": 0.95,
        "standard_error": 3.38531648,
        "clusters": 36022
      },
      {
        "model": "honest_forest_500",
        "decile": 7,
        "n": 39850,
        "events": 15,
        "treated": 33703,
        "predicted_effect_per_10000": 1.36108469,
        "estimate": 4.54100219,
        "lower": 2.29394331,
        "upper": 6.78806108,
        "confidence": 0.95,
        "standard_error": 1.14647968,
        "clusters": 37443
      },
      {
        "model": "honest_forest_500",
        "decile": 8,
        "n": 39850,
        "events": 20,
        "treated": 33836,
        "predicted_effect_per_10000": 3.76533181,
        "estimate": 1.96621574,
        "lower": -3.26278434,
        "upper": 7.19521581,
        "confidence": 0.95,
        "standard_error": 2.6679062,
        "clusters": 38635
      },
      {
        "model": "honest_forest_500",
        "decile": 9,
        "n": 39850,
        "events": 80,
        "treated": 33954,
        "predicted_effect_per_10000": 14.99109709,
        "estimate": 5.46533645,
        "lower": -5.86817502,
        "upper": 16.79884792,
        "confidence": 0.95,
        "standard_error": 5.78251007,
        "clusters": 39668
      },
      {
        "model": "honest_forest_500",
        "decile": 10,
        "n": 39850,
        "events": 1084,
        "treated": 34421,
        "predicted_effect_per_10000": 100.88134977,
        "estimate": 92.32895953,
        "lower": 55.86594783,
        "upper": 128.79197123,
        "confidence": 0.95,
        "standard_error": 18.60391925,
        "clusters": 39823
      }
    ],
    "sensitivity": [
      {
        "model": "dr_l2_10",
        "analysis": "learned_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 9.5433928,
        "lower": 5.87596261,
        "upper": 13.21082299,
        "confidence": 0.95,
        "standard_error": 1.87117224,
        "clusters": 352598
      },
      {
        "model": "dr_l2_10",
        "analysis": "constant_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 9.96052847,
        "lower": 6.46151047,
        "upper": 13.45954647,
        "confidence": 0.95,
        "standard_error": 1.78524607,
        "clusters": 352598
      },
      {
        "model": "dr_l2_10",
        "analysis": "learned_propensity_IPW",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 9.86148965,
        "lower": 5.96448925,
        "upper": 13.75849005,
        "confidence": 0.95,
        "standard_error": 1.98830205,
        "clusters": 352598
      },
      {
        "model": "dr_l2_10",
        "analysis": "one_record_per_profile",
        "capacity": 0.2,
        "n": 352598,
        "selected_fraction": 0.22462124,
        "estimate": 10.96275561,
        "lower": 6.83458637,
        "upper": 15.09092484,
        "confidence": 0.95,
        "standard_error": 2.1062475,
        "clusters": 352598
      },
      {
        "model": "dr_l2_100",
        "analysis": "learned_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 9.66355226,
        "lower": 6.02288456,
        "upper": 13.30421996,
        "confidence": 0.95,
        "standard_error": 1.85751765,
        "clusters": 352598
      },
      {
        "model": "dr_l2_100",
        "analysis": "constant_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 10.06228166,
        "lower": 6.58413855,
        "upper": 13.54042477,
        "confidence": 0.95,
        "standard_error": 1.77459542,
        "clusters": 352598
      },
      {
        "model": "dr_l2_100",
        "analysis": "learned_propensity_IPW",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 9.98896567,
        "lower": 6.11524367,
        "upper": 13.86268767,
        "confidence": 0.95,
        "standard_error": 1.97642509,
        "clusters": 352598
      },
      {
        "model": "dr_l2_100",
        "analysis": "one_record_per_profile",
        "capacity": 0.2,
        "n": 352598,
        "selected_fraction": 0.22462124,
        "estimate": 11.09855974,
        "lower": 7.00076283,
        "upper": 15.19635666,
        "confidence": 0.95,
        "standard_error": 2.09075113,
        "clusters": 352598
      },
      {
        "model": "honest_forest_100",
        "analysis": "learned_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 9.49525489,
        "lower": 5.69709872,
        "upper": 13.29341106,
        "confidence": 0.95,
        "standard_error": 1.93787039,
        "clusters": 352598
      },
      {
        "model": "honest_forest_100",
        "analysis": "constant_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 9.92672543,
        "lower": 6.2873638,
        "upper": 13.56608707,
        "confidence": 0.95,
        "standard_error": 1.85685128,
        "clusters": 352598
      },
      {
        "model": "honest_forest_100",
        "analysis": "learned_propensity_IPW",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 9.77794363,
        "lower": 5.62038317,
        "upper": 13.93550408,
        "confidence": 0.95,
        "standard_error": 2.12124329,
        "clusters": 352598
      },
      {
        "model": "honest_forest_100",
        "analysis": "one_record_per_profile",
        "capacity": 0.2,
        "n": 352598,
        "selected_fraction": 0.22485096,
        "estimate": 10.92162779,
        "lower": 6.64512822,
        "upper": 15.19812736,
        "confidence": 0.95,
        "standard_error": 2.18192763,
        "clusters": 352598
      },
      {
        "model": "honest_forest_500",
        "analysis": "learned_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 9.77926724,
        "lower": 5.96002988,
        "upper": 13.59850459,
        "confidence": 0.95,
        "standard_error": 1.94862629,
        "clusters": 352598
      },
      {
        "model": "honest_forest_500",
        "analysis": "constant_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 10.20227998,
        "lower": 6.54055524,
        "upper": 13.86400472,
        "confidence": 0.95,
        "standard_error": 1.86826124,
        "clusters": 352598
      },
      {
        "model": "honest_forest_500",
        "analysis": "learned_propensity_IPW",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 10.08213704,
        "lower": 5.90560743,
        "upper": 14.25866665,
        "confidence": 0.95,
        "standard_error": 2.13092161,
        "clusters": 352598
      },
      {
        "model": "honest_forest_500",
        "analysis": "one_record_per_profile",
        "capacity": 0.2,
        "n": 352598,
        "selected_fraction": 0.22544654,
        "estimate": 11.23755767,
        "lower": 6.93714876,
        "upper": 15.53796659,
        "confidence": 0.95,
        "standard_error": 2.1941265,
        "clusters": 352598
      },
      {
        "model": "response",
        "analysis": "learned_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 9.86157129,
        "lower": 6.05611669,
        "upper": 13.66702589,
        "confidence": 0.95,
        "standard_error": 1.94159415,
        "clusters": 352598
      },
      {
        "model": "response",
        "analysis": "constant_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 10.28171189,
        "lower": 6.6361287,
        "upper": 13.92729508,
        "confidence": 0.95,
        "standard_error": 1.8600256,
        "clusters": 352598
      },
      {
        "model": "response",
        "analysis": "learned_propensity_IPW",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.1999995,
        "estimate": 10.14900514,
        "lower": 5.98499537,
        "upper": 14.31301492,
        "confidence": 0.95,
        "standard_error": 2.12453382,
        "clusters": 352598
      },
      {
        "model": "response",
        "analysis": "one_record_per_profile",
        "capacity": 0.2,
        "n": 352598,
        "selected_fraction": 0.22527638,
        "estimate": 11.32483694,
        "lower": 7.04006673,
        "upper": 15.60960715,
        "confidence": 0.95,
        "standard_error": 2.18614742,
        "clusters": 352598
      },
      {
        "model": "random",
        "analysis": "learned_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.2,
        "estimate": 2.04057666,
        "lower": 1.24385683,
        "upper": 2.83729648,
        "confidence": 0.95,
        "standard_error": 0.40649718,
        "clusters": 352598
      },
      {
        "model": "random",
        "analysis": "constant_propensity_DR",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.2,
        "estimate": 2.13035632,
        "lower": 1.36324046,
        "upper": 2.89747217,
        "confidence": 0.95,
        "standard_error": 0.39139283,
        "clusters": 352598
      },
      {
        "model": "random",
        "analysis": "learned_propensity_IPW",
        "capacity": 0.2,
        "n": 398506,
        "selected_fraction": 0.2,
        "estimate": 2.0929704,
        "lower": 1.22747756,
        "upper": 2.95846324,
        "confidence": 0.95,
        "standard_error": 0.4415861,
        "clusters": 352598
      },
      {
        "model": "random",
        "analysis": "one_record_per_profile",
        "capacity": 0.2,
        "n": 352598,
        "selected_fraction": 0.2,
        "estimate": 2.29794431,
        "lower": 1.40067541,
        "upper": 3.19521322,
        "confidence": 0.95,
        "standard_error": 0.45779867,
        "clusters": 352598
      }
    ],
    "feature_balance": [
      {
        "feature": "f0",
        "standardized_treatment_difference": -0.00785732
      },
      {
        "feature": "f1",
        "standardized_treatment_difference": 0.01778791
      },
      {
        "feature": "f2",
        "standardized_treatment_difference": -6.18e-06
      },
      {
        "feature": "f3",
        "standardized_treatment_difference": -0.05023528
      },
      {
        "feature": "f4",
        "standardized_treatment_difference": 0.01421085
      },
      {
        "feature": "f5",
        "standardized_treatment_difference": -0.02694526
      },
      {
        "feature": "f6",
        "standardized_treatment_difference": -0.04582442
      },
      {
        "feature": "f7",
        "standardized_treatment_difference": 0.02242284
      },
      {
        "feature": "f8",
        "standardized_treatment_difference": -0.02866414
      },
      {
        "feature": "f9",
        "standardized_treatment_difference": 0.02558699
      },
      {
        "feature": "f10",
        "standardized_treatment_difference": 0.01976126
      },
      {
        "feature": "f11",
        "standardized_treatment_difference": -0.00451885
      }
    ],
    "propensity": {
      "min": 0.77338021,
      "p01": 0.84455221,
      "median": 0.8477864,
      "p99": 0.87473853,
      "max": 0.89547439,
      "assignment_auroc": 0.50920069,
      "assignment_brier": 0.12787855,
      "development_treatment_prevalence": 0.84987687,
      "clipped_fraction": 0.0
    },
    "all_vs_none": {
      "estimate": 10.20288328,
      "lower": 6.21928413,
      "upper": 14.18648242,
      "confidence": 0.95,
      "standard_error": 2.03248589,
      "clusters": 352598
    },
    "scenario_formula": "Incremental benchmark conversions per 10000 × assumed conversion value − selected fraction × 10000 × assumed contact cost.",
    "economic_ranges": {
      "conversion_value_USD": [
        0,
        1000
      ],
      "contact_cost_USD": [
        0,
        10
      ]
    }
  },
  "software": {
    "python": "3.11.16",
    "numpy": "2.3.5",
    "pandas": "2.3.3",
    "scipy": "1.17.1",
    "scikit-learn": "1.7.2",
    "econml": "0.17.0"
  }
}