{
  "slug": "ab-test",
  "title": "A/B Test Significance Calculator",
  "heading": "A/B Test Significance Calculator",
  "category": "math",
  "url": "https://www.revenuelab.fyi/toolbox/ab-test",
  "summary": "Is your test result real, or is it noise?",
  "description": "Runs a two-proportion z-test on your control and variant. Stopping a test the moment it looks good is how teams ship changes that do nothing — wait for significance and a pre-planned sample size.",
  "formula": "z = (p₂ − p₁) ÷ √(p(1−p)(1/n₁ + 1/n₂)) using the pooled conversion rate.",
  "dateModified": "2026-09-30",
  "run_url": "https://www.revenuelab.fyi/api/public/calc?tool=ab-test",
  "inputs": [
    {
      "id": "visitorsA",
      "label": "Control visitors",
      "kind": "number",
      "hint": null,
      "default": 5000,
      "unit": null,
      "min": 1,
      "max": null
    },
    {
      "id": "conversionsA",
      "label": "Control conversions",
      "kind": "number",
      "hint": null,
      "default": 250,
      "unit": null,
      "min": 0,
      "max": null
    },
    {
      "id": "visitorsB",
      "label": "Variant visitors",
      "kind": "number",
      "hint": null,
      "default": 5000,
      "unit": null,
      "min": 1,
      "max": null
    },
    {
      "id": "conversionsB",
      "label": "Variant conversions",
      "kind": "number",
      "hint": null,
      "default": 305,
      "unit": null,
      "min": 0,
      "max": null
    }
  ],
  "outputs": [
    {
      "id": "significant",
      "label": "Verdict",
      "format": "raw",
      "hint": null,
      "primary": true
    },
    {
      "id": "lift",
      "label": "Relative lift",
      "format": "percent",
      "hint": null,
      "primary": false
    },
    {
      "id": "rateA",
      "label": "Control conversion rate",
      "format": "percent",
      "hint": null,
      "primary": false
    },
    {
      "id": "rateB",
      "label": "Variant conversion rate",
      "format": "percent",
      "hint": null,
      "primary": false
    },
    {
      "id": "confidence",
      "label": "Confidence level",
      "format": "percent",
      "hint": null,
      "primary": false
    },
    {
      "id": "z",
      "label": "Z-statistic",
      "format": "decimal",
      "hint": null,
      "primary": false
    }
  ],
  "worked_example": {
    "inputs": [
      "Control visitors: 5000",
      "Control conversions: 250",
      "Variant visitors: 5000",
      "Variant conversions: 305"
    ],
    "outputs": [
      "Verdict: Significant at 95% (p = 0.0163)",
      "Relative lift: 22.00%",
      "Control conversion rate: 5.000%",
      "Variant conversion rate: 6.100%",
      "Confidence level: 98.37%",
      "Z-statistic: 2.4022"
    ]
  },
  "how_to": {
    "title": "How to use this",
    "steps": [
      "Enter control visitors.",
      "Enter control conversions.",
      "Enter variant visitors.",
      "Enter variant conversions.",
      "Read your verdict on the right — it updates as you type.",
      "Hit Share to keep the scenario or send it to someone."
    ]
  },
  "scenarios": [
    {
      "name": "Conservative",
      "description": "Lower-end numbers — what if things land soft?",
      "values": {
        "visitorsA": 3000,
        "conversionsA": 150,
        "visitorsB": 3000,
        "conversionsB": 183
      }
    },
    {
      "name": "Typical",
      "description": "Defaults — the most common real-world setup.",
      "values": {
        "visitorsA": 5000,
        "conversionsA": 250,
        "visitorsB": 5000,
        "conversionsB": 305
      }
    },
    {
      "name": "Ambitious",
      "description": "Higher-end numbers — what if things really pop?",
      "values": {
        "visitorsA": 8000,
        "conversionsA": 400,
        "visitorsB": 8000,
        "conversionsB": 488
      }
    }
  ],
  "limitations": [
    "Results are rounded for display; the underlying calculation keeps full precision.",
    "Very large or very small inputs may hit floating-point limits in the browser.",
    "Inputs outside the accepted range are clamped rather than rejected."
  ],
  "faq": [
    {
      "q": "How does the a/b test significance calculator work?",
      "a": "Runs a two-proportion z-test on your control and variant. Stopping a test the moment it looks good is how teams ship changes that do nothing — wait for significance and a pre-planned sample size. The underlying maths is: z = (p₂ − p₁) ÷ √(p(1−p)(1/n₁ + 1/n₂)) using the pooled conversion rate."
    },
    {
      "q": "What do I need to enter?",
      "a": "4 values: control visitors, control conversions, variant visitors, and variant conversions. Each field starts with a sensible default, so you can change one number at a time and watch the result move."
    },
    {
      "q": "What does the verdict result mean?",
      "a": "It is returned as a plain value and updates live as you edit the inputs, so you can compare two or three versions of a scenario in a few seconds."
    },
    {
      "q": "Is this calculator free, and do I need an account?",
      "a": "Yes, it's free, and no account is required. Nothing you type is stored on our servers — the maths runs entirely in your browser."
    },
    {
      "q": "How accurate is the result?",
      "a": "It applies the standard formula exactly, so the arithmetic is precise. Results are rounded for display; the underlying calculation keeps full precision."
    },
    {
      "q": "Who is this tool for?",
      "a": "It's built for students, developers, and anyone who needs the answer right now without setting up a spreadsheet."
    }
  ],
  "related": [
    "https://www.revenuelab.fyi/toolbox/conversion-rate",
    "https://www.revenuelab.fyi/toolbox/z-score",
    "https://www.revenuelab.fyi/toolbox/probability"
  ],
  "license": "CC-BY-4.0",
  "citation": "RevenueLab — A/B Test Significance Calculator (https://www.revenuelab.fyi/toolbox/ab-test)"
}