A/B Test Sample Size Calculator
Calculate minimum sample size for statistically significant A/B tests
Source Code
const [baselineRate, mde, significance, power, dailyTraffic] = process.argv.slice(2);
const p1 = parseFloat(baselineRate);
const relativeEffect = parseFloat(mde);
const alpha = 1 - parseFloat(significance || "0.95");
const beta = 1 - parseFloat(power || "0.8");
const traffic = parseInt(dailyTraffic || "0", 10);
// Validate inputs
if (isNaN(p1) || p1 <= 0 || p1 >= 1) {
console.log(JSON.stringify({ error: "baselineRate must be between 0 and 1" }));
process.exit(1);
}
if (isNaN(relativeEffect) || relativeEffect <= 0) {
console.log(JSON.stringify({ error: "mde must be a positive number" }));
process.exit(1);
}
// Calculate target rate (baseline + relative lift)
const p2 = p1 * (1 + relativeEffect);
// Z-scores for common alpha and beta values
// Using approximations for standard values
function getZScore(probability) {
// Common z-scores lookup
const zScores = {
0.005: 2.576, // 99.5%
0.01: 2.326, // 99%
0.025: 1.96, // 97.5% (two-tailed 95%)
0.05: 1.645, // 95%
0.1: 1.282, // 90%
0.2: 0.842, // 80%
};
// Find closest match or interpolate
const keys = Object.keys(zScores).map(Number).sort((a, b) => a - b);
for (const key of keys) {
if (Math.abs(key - probability) < 0.001) return zScores[key];
}
// Fallback approximation using inverse error function approximation
const t = Math.sqrt(-2 * Math.log(probability));
return t - (2.515517 + 0.802853 * t + 0.010328 * t * t) /
(1 + 1.432788 * t + 0.189269 * t * t + 0.001308 * t * t * t);
}
const zAlpha = getZScore(alpha / 2); // Two-tailed test
const zBeta = getZScore(beta);
// Sample size formula for two proportions
// n = (zα/2 + zβ)² × (p1(1-p1) + p2(1-p2)) / (p2 - p1)²
const pooledVariance = p1 * (1 - p1) + p2 * (1 - p2);
const effectSize = Math.abs(p2 - p1);
const sampleSizePerVariant = Math.ceil(
Math.pow(zAlpha + zBeta, 2) * pooledVariance / Math.pow(effectSize, 2)
);
const totalSampleSize = sampleSizePerVariant * 2;
// Duration estimate
let durationDays = null;
let durationWeeks = null;
if (traffic > 0) {
// Assuming 50/50 split, we need totalSampleSize users
durationDays = Math.ceil(totalSampleSize / traffic);
durationWeeks = Math.ceil(durationDays / 7);
}
const result = {
inputs: {
baselineRate: p1,
targetRate: Math.round(p2 * 10000) / 10000,
relativeEffect: `${Math.round(relativeEffect * 100)}%`,
significance: `${Math.round((1 - alpha) * 100)}%`,
power: `${Math.round((1 - beta) * 100)}%`,
},
sampleSize: {
perVariant: sampleSizePerVariant,
total: totalSampleSize,
},
...(traffic > 0 && {
duration: {
dailyTraffic: traffic,
estimatedDays: durationDays,
estimatedWeeks: durationWeeks,
note: "Assumes 50/50 traffic split between control and variant",
},
}),
methodology: {
test: "Two-proportion z-test",
tails: "Two-tailed",
formula: "n = (zα/2 + zβ)² × (p1(1-p1) + p2(1-p2)) / (p2 - p1)²",
},
};
console.log(JSON.stringify(result, null, 2));