Split JSON Array
Split a large JSON array into smaller chunk files
Source Code
import fs from "fs";
import path from "path";
const [inputPath, chunkSizeArg, outputDir] = process.argv.slice(2);
if (!inputPath || !chunkSizeArg || !outputDir) {
console.error("Usage: inputPath chunkSize outputDir");
process.exit(1);
}
const chunkSize = parseInt(chunkSizeArg);
if (isNaN(chunkSize) || chunkSize < 1) {
console.error("chunkSize must be a positive number");
process.exit(1);
}
try {
console.log(`Reading ${inputPath}...`);
const raw = fs.readFileSync(inputPath, "utf-8");
const data = JSON.parse(raw);
const items = Array.isArray(data)
? data
: data.items || data.results || data.messages || [];
if (!Array.isArray(items)) {
console.error("Input must be a JSON array or object with array property");
process.exit(1);
}
const totalChunks = Math.ceil(items.length / chunkSize);
console.log(
`Splitting ${items.length} items into ${totalChunks} chunks of ${chunkSize}...`
);
// Ensure output directory exists
fs.mkdirSync(outputDir, { recursive: true });
const chunkFiles = [];
for (let i = 0; i < totalChunks; i++) {
const start = i * chunkSize;
const end = Math.min(start + chunkSize, items.length);
const chunk = items.slice(start, end);
// Zero-pad chunk number for proper sorting
const padLength = String(totalChunks).length;
const chunkNum = String(i + 1).padStart(padLength, "0");
const chunkPath = path.join(outputDir, `chunk-${chunkNum}.json`);
fs.writeFileSync(chunkPath, JSON.stringify(chunk, null, 2));
chunkFiles.push(chunkPath);
console.log(` Written chunk ${i + 1}/${totalChunks}: ${chunk.length} items`);
}
console.log(`\nā Split ${items.length} items into ${totalChunks} chunks`);
console.log(` Chunk size: ${chunkSize}`);
console.log(` Output directory: ${outputDir}`);
console.log(
JSON.stringify({
success: true,
outputDir,
totalItems: items.length,
chunkSize,
totalChunks,
chunkFiles,
})
);
} catch (error) {
console.error("Error:", error.message);
process.exit(1);
}