code icon Code

Split JSON Array

Split a large JSON array into smaller chunk files

Source Code

import fs from "fs";
import path from "path";

const [inputPath, chunkSizeArg, outputDir] = process.argv.slice(2);

if (!inputPath || !chunkSizeArg || !outputDir) {
  console.error("Usage: inputPath chunkSize outputDir");
  process.exit(1);
}

const chunkSize = parseInt(chunkSizeArg);
if (isNaN(chunkSize) || chunkSize < 1) {
  console.error("chunkSize must be a positive number");
  process.exit(1);
}

try {
  console.log(`Reading ${inputPath}...`);
  const raw = fs.readFileSync(inputPath, "utf-8");
  const data = JSON.parse(raw);

  const items = Array.isArray(data)
    ? data
    : data.items || data.results || data.messages || [];

  if (!Array.isArray(items)) {
    console.error("Input must be a JSON array or object with array property");
    process.exit(1);
  }

  const totalChunks = Math.ceil(items.length / chunkSize);
  console.log(
    `Splitting ${items.length} items into ${totalChunks} chunks of ${chunkSize}...`
  );

  // Ensure output directory exists
  fs.mkdirSync(outputDir, { recursive: true });

  const chunkFiles = [];

  for (let i = 0; i < totalChunks; i++) {
    const start = i * chunkSize;
    const end = Math.min(start + chunkSize, items.length);
    const chunk = items.slice(start, end);

    // Zero-pad chunk number for proper sorting
    const padLength = String(totalChunks).length;
    const chunkNum = String(i + 1).padStart(padLength, "0");
    const chunkPath = path.join(outputDir, `chunk-${chunkNum}.json`);

    fs.writeFileSync(chunkPath, JSON.stringify(chunk, null, 2));
    chunkFiles.push(chunkPath);

    console.log(`  Written chunk ${i + 1}/${totalChunks}: ${chunk.length} items`);
  }

  console.log(`\nāœ“ Split ${items.length} items into ${totalChunks} chunks`);
  console.log(`  Chunk size: ${chunkSize}`);
  console.log(`  Output directory: ${outputDir}`);

  console.log(
    JSON.stringify({
      success: true,
      outputDir,
      totalItems: items.length,
      chunkSize,
      totalChunks,
      chunkFiles,
    })
  );
} catch (error) {
  console.error("Error:", error.message);
  process.exit(1);
}