Firecrawl Scraper

# Firecrawl API Key Configuration
# Get your API key from: https://www.firecrawl.dev/app/api-keys

FIRECRAWL_API_KEY=your_api_key_here


.env
node_modules/


#!/usr/bin/env node

/**
 * Firecrawl API Helper Script
 * Provides a CLI wrapper around Firecrawl endpoints for skill integration.
 *
 * Usage:
 *   node firecrawl-api.js <scrape|crawl|map|batch-scrape|crawl-status> [<json-string>]
 *   cat payload.json | node firecrawl-api.js scrape
 *   node firecrawl-api.js scrape --file ./payload.json
 *   node firecrawl-api.js crawl --wait < payload.json
 *   node firecrawl-api.js crawl-status <crawl-id> [--wait]
 */

const https = require('https');
const fs = require('fs');
const path = require('path');

const API_BASE = 'https://api.firecrawl.dev';

function loadApiKey() {
  if (process.env.FIRECRAWL_API_KEY) {
    return process.env.FIRECRAWL_API_KEY;
  }

  const envPath = path.join(__dirname, '.env');
  if (!fs.existsSync(envPath)) {
    return null;
  }

  const envContent = fs.readFileSync(envPath, 'utf8');
  const match = envContent.match(/FIRECRAWL_API_KEY\s*=\s*(.+)/);
  if (!match) {
    return null;
  }

  return match[1].trim().replace(/^[\"']|[\"']$/g, '');
}

function usage() {
  const cmd = path.basename(process.argv[1] || 'firecrawl-api.js');
  console.error(
    [
      'Usage:',
      `  node ${cmd} <scrape|crawl|map|batch-scrape|crawl-status> [<json-string>]`,
      `  cat payload.json | node ${cmd} scrape`,
      `  node ${cmd} scrape --file ./payload.json`,
      `  node ${cmd} crawl --wait < payload.json`,
      `  node ${cmd} crawl-status <crawl-id> [--wait]`,
      '',
      'Options:',
      '  --wait  Wait for crawl job completion (crawl / crawl-status only)',
      '  --id    Crawl job id (crawl-status only)',
      '',
      'Env:',
      '  FIRECRAWL_API_KEY (env var) or .env file next to this script',
    ].join('\n'),
  );
}

function readStdin() {
  return new Promise((resolve, reject) => {
    let data = '';
    process.stdin.setEncoding('utf8');
    process.stdin.on('data', (chunk) => {
      data += chunk;
    });
    process.stdin.on('end', () => resolve(data));
    process.stdin.on('error', reject);
  });
}

async function readPayload(args) {
  const fileFlagIndex = args.findIndex((arg) => arg === '--file');
  if (fileFlagIndex !== -1) {
    const filePath = args[fileFlagIndex + 1];
    if (!filePath) {
      throw new Error('Missing value for --file');
    }
    const content = fs.readFileSync(filePath, 'utf8');
    return JSON.parse(content);
  }

  const dataFlagIndex = args.findIndex((arg) => arg === '--data');
  if (dataFlagIndex !== -1) {
    const json = args[dataFlagIndex + 1];
    if (!json) {
      throw new Error('Missing value for --data');
    }
    return JSON.parse(json);
  }

  if (args[0] && !args[0].startsWith('-')) {
    return JSON.parse(args[0]);
  }

  if (process.stdin.isTTY) {
    throw new Error('No payload provided (pass JSON arg, --data, --file, or pipe via stdin)');
  }

  const stdin = await readStdin();
  if (!stdin.trim()) {
    throw new Error('Empty stdin payload');
  }
  return JSON.parse(stdin);
}

function requestJson(method, endpointPath, apiKey, payload) {
  return new Promise((resolve, reject) => {
    const body = payload === undefined ? null : JSON.stringify(payload);
    const url = new URL(endpointPath, API_BASE);

    const req = https.request(
      url,
      {
        method,
        headers: {
          'Authorization': `Bearer ${apiKey}`,
          ...(body === null
            ? {}
            : {
                'Content-Type': 'application/json',
                'Content-Length': Buffer.byteLength(body),
              }),
          'User-Agent': 'Firecrawl-Skill/1.0',
        },
        timeout: 60_000,
      },
      (res) => {
        let data = '';
        res.setEncoding('utf8');
        res.on('data', (chunk) => {
          data += chunk;
        });
        res.on('end', () => {
          const ok = res.statusCode && res.statusCode >= 200