Scrape Webpage

#!/usr/bin/env node

/*
 * Copyright 2025 Adobe. All rights reserved.
 * This file is licensed to you under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under
 * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
 * OF ANY KIND, either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */

/**
 * Analyze Webpage for Migration
 *
 * Uses npm playwright to analyze a webpage and prepare it for content migration.
 *
 * Features:
 * - Scrolls to trigger lazy-loaded content
 * - Captures images and converts to web-friendly formats
 * - Takes full-page screenshot
 * - Extracts cleaned HTML with preserved attributes
 * - Extracts metadata (SEO, Open Graph, etc.)
 *
 * Usage:
 *   node analyze-webpage.js "https://example.com/page" --output ./analysis
 *
 * Requirements:
 *   npm install playwright
 *   npx playwright install chromium
 */

import { chromium } from 'playwright';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { generateDocumentPathInfo } from './generate-path.js';
import { setupImageCapture, waitForPendingImages, replaceImageUrls } from './image-capture.js';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

/**
 * Scroll through the entire page to trigger lazy-loaded images
 */
async function scrollToTriggerLazyLoad(page) {
  await page.evaluate(async () => {
    await new Promise((resolve) => {
      let totalHeight = 0;
      const distance = 100;
      const timer = setInterval(() => {
        const scrollHeight = document.body.scrollHeight;
        window.scrollBy(0, distance);
        totalHeight += distance;
        if (totalHeight >= scrollHeight) {
          clearInterval(timer);
          resolve();
        }
      }, 100);
    });
  });
}


/**
 * Fix images in the DOM to ensure none are missed during extraction
 * Adapted from site-transfer-agent-importscript/resources/transformers/images.js
 */
async function fixImagesInDom(page, url) {
  await page.evaluate((sourceUrl) => {
    // Helper: Extract URL from background-image CSS property
    function extractUrlFromBackgroundImage(backgroundImage) {
      if (!backgroundImage || backgroundImage.toLowerCase() === 'none') {
        return null;
      }
      const urlMatch = backgroundImage.match(/url\(['"]?([^'")\s]+)['"]?\)/);
      return urlMatch ? urlMatch[1] : null;
    }

    // Helper: Get background image from inline style or data attribute
    function getBackgroundImageFromElement(element) {
      const inlineStyle = element.getAttribute('style');
      if (inlineStyle) {
        const styleParts = inlineStyle.split(';');
        for (const style of styleParts) {
          const [prop, ...valueParts] = style.split(':');
          if (prop?.trim() === 'background-image') {
            return valueParts.join(':').trim();
          }
        }
      }
      const bgImage = window.getComputedStyle(element)?.getPropertyValue('background-image');
      if (bgImage && bgImage !== 'none' && bgImage.includes('url(')) {
        return bgImage;
      }

      return null;
    }

    // Helper: Extract picture source URL (largest viewport)
    function extractPictureSource(pictureElement) {
      const sources = pictureElement.querySelectorAll('source');
      if (sources.length === 0) return null;

      let largestSource = null;
      let largestMaxWidth = -1;

      for (const source of sources) {
        const mediaQuery = source.getAttribute('media');
        if (!mediaQuery) {
          largestSource = source;
          break;
        }

        const match = mediaQuery.match(/max-width:\s*(\d+)px/);
        if (match) {
          const maxWidth = parseInt(match[1], 10);
          if (maxWidth > largestMaxWidth) {
            largestMaxWidth = maxWidth;
            largestSource = source;
          }
        }
      }

      if (!largestSource) {
        largestSource = sources[sources.length - 1];
      }

      if (largestSource) {
        const srcset = largestSource.getAttribute('srcset');
        if (srcset) {
          return srcset.split(',')[0].trim().split(/\s+/)[0];
        }
      }
      return null;
    }

    // 1. Transform background images to actual img elements
    // Check common elements that often have background images from CSS
    const elementsToCheck = document.body.querySelectorAll('div, section, article, header, footer, aside, main, figure');
    elementsToCheck.forEach((element) => {
      const backgroundImage = getBackgroundImageFromElement(element);
      const src = extractUrlFromBackgroundImage(backgroundImage);
      if (src) {
        const img = document.createElement('img');
        img.src = src;
        element.prepend(img);
        element.style.backgroundImage = 'none';
      }
    });

    // 2. Ensure picture elements have img with src
    const pictures = document.body.querySelectorAll('picture');
    pictures.forEach((picture) => {
      const img = picture.querySelector('img');
      if (!img || !img.src) {
        const newImg = document.createElement('img');
        const src = extractPictureSource(picture);
        if (src) {
          newImg.src = src;
          if (img) {
            img.replaceWith(newImg);
          } else {
            picture.appendChild(newImg);
          }
        }
      }
    });

    // 3. Fix images with srcset but no src
    document.body.querySelectorAll('img').forEach((img) => {
      let src = img.getAttribute('src');
      const srcset = img.getAttribute('srcset')?.split(' ')[0];
      if (!src && srcset) {
        img.setAttribute('src', srcset);
      }
      src = img.getAttribute('src');

      // 4. Convert relative URLs to absolute
      if (src) {
        try {
          new URL(src);
          // Already absolute, leave it
        } catch (e) {
          // Relative URL - convert to absolute
          if (!src.startsWith('/')) {
            src = `./${src}`;
          }
          try {
            const absoluteUrl = new URL(src, sourceUrl);
            img.src = absoluteUrl.toString();
          } catch (err) {
            console.warn(`Unable to adjust image URL ${src}`);
          }
        }
      }
    });

    // 5. Transform inline SVG elements to img with data URLs
    const svgs = document.body.querySelectorAll('svg');
    svgs.forEach((svg) => {
      let svgString = '<svg';
      for (const attr of svg.attributes) {
        svgString += ` ${attr.name}="${attr.value}"`;
      }
      svgString += '>';
      svgString += svg.innerHTML;
      svgString += '</svg>';

      const svgDataUrl = `data:image/svg+xml;base64,${btoa(unescape(encodeURIComponent(svgString)))}`;
      const img = document.createElement('img');
      img.src = svgDataUrl;
      svg.replaceWith(img);
    });
  }, url);
}

/**
 * Format HTML with proper indentation for readability
 */
function formatHtml(html) {
  let formatted = '';
  let indent = 0;
  const indentSize = 2;

  // Add newlines after tags
  html = html.replace(/></g, '>\n<');

  const lines = html.split('\n');

  lines.forEach(line => {
    const trimmed = line.trim();
    if (!trimmed) return;

    // Decrease indent for closing tags
    if (trimmed.startsWith('</')) {
      indent = Math.max(0, indent - 1);
    }

    // Add indentation
    formatted += ' '.repeat(indent * indentSize) + trimmed + '\n';

    // Increase indent for opening tags (but not self-closing or immediately closed)
    if (trimmed.startsWith('<') && !trimmed.startsWith('</') && !trimmed.endsWith('/>') && !trimmed.match(/<[^>]+>.*<\/[^>]+>$/)) {
      indent++;
    }
  });

  return formatted.trim();
}

/**
 * Extract cleaned HTML with preserved attributes
 */
async function extractCleanedHTML(page) {
  const html = await page.evaluate(() => {
    // 1. Remove non-content elements
    const selectorsToRemove = [
      'script', 'style', 'noscript'
    ];

    selectorsToRemove.forEach(selector => {
      document.querySelectorAll(selector).forEach(el => el.remove());
    });

    // 2. CRITICAL: Preserve essential attributes, strip all others
    const keepAttributes = ['src', 'href', 'alt', 'title', 'class', 'id'];
    document.body.querySelectorAll('*').forEach(el => {
      const attrs = Array.from(el.attributes);
      attrs.forEach(attr => {
        if (!keepAttributes.includes(attr.name)) {
          el.removeAttribute(attr.name);
        }
      });
    });

    // 3. Return full body HTML (will be formatted on Node.js side)
    return document.body.outerHTML;
  });

  // Format HTML for readability
  return formatHtml(html);
}

/**
 * Extract metadata from page
 */
async function extractMetadata(page) {
  const metadata = await page.evaluate(() => {
    const meta = {};

    // Extract title
    const titleTag = document.querySelector('title');
    if (titleTag) meta.title = titleTag.textContent.trim();

    // Extract all meta tags
    document.querySelectorAll('meta').forEach(tag => {
      const name = tag.getAttribute('name') || tag.getAttribute('property');
      const content = tag.getAttribute('content');
      if (name && content) meta[name] = content;
    });

    // Extract canonical link
    const canonical = document.querySelector('link[rel="canonical"]');
    if (canonical) meta.canonical = canonical.getAttribute('href');

    // Extract JSON-LD
    const jsonLd = document.querySelector('script[type="application/ld+json"]');
    if (jsonLd) {
      try {
        meta.jsonLd = JSON.parse(jsonLd.textContent);
      } catch (e) {
        meta.jsonLd = jsonLd.textContent;
      }
    }

    return meta;
  });

  return metadata;
}

/**
 * Main analysis function
 */
async function analyzeWebpage(url, outputDir) {
  // Ensure output directory exists
  if (!fs.existsSync(outputDir)) {
    fs.mkdirSync(outputDir, { recursive: true });
  }

  console.error(`Analyzing: ${url}`);
  console.error(`Output directory: ${outputDir}`);

  // Launch browser
  const browser = await chromium.launch();
  const page = await browser.newPage();

  try {
    // Set up image capture BEFORE navigation
    console.error('Setting up image capture...');
    const captureState = setupImageCapture(page, outputDir);

    // Navigate to page
    console.error('Navigating to page...');
    try {
      // Try networkidle first (most reliable when it works)
      await page.goto(url);
    } catch (error) {
      // Fall back to domcontentloaded if networkidle times out
      console.error('⚠️  networkidle timeout, falling back to domcontentloaded...');
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
      await page.waitForTimeout(3000); // Give page extra time to settle
    }

    // Scroll to trigger lazy loading
    console.error('Scrolling to trigger lazy-loaded content...');
    await scrollToTriggerLazyLoad(page);
    await page.waitForTimeout(1000); // Give lazy-loaded images time to populate

    // Wait for all pending images to complete
    console.error(`Waiting for ${captureState.pendingImages.size} pending images...`);
    await waitForPendingImages(captureState, 5000);
    console.error(`✅ Image capture complete: ${captureState.stats.total} total, ${captureState.stats.converted} converted, ${captureState.stats.failed} failed`);

    // Take screenshot
    console.error('Capturing screenshot...');
    const screenshot = path.join(outputDir, 'screenshot.png');
    await page.screenshot({ path: screenshot, fullPage: true });

    // Extract metadata
    console.error('Extracting metadata...');
    const metadata = await extractMetadata(page);

    // Disable image capture (images already captured)
    captureState.disable();

    // Fix images in DOM (background images, picture elements, relative URLs, inline SVGs)
    console.error('Fixing images in DOM...');
    await fixImagesInDom(page, url);

    // Extract cleaned HTML
    console.error('Extracting cleaned HTML...');
    let html = await extractCleanedHTML(page);

    // Replace image URLs with local paths
    console.error('Replacing image URLs with local paths...');
    html = replaceImageUrls(html, captureState.imageMap);

    const htmlPath = path.join(outputDir, 'cleaned.html');
    fs.writeFileSync(htmlPath, html, 'utf-8');

    // Generate document paths
    console.error('Generating document paths...');
    const paths = generateDocumentPathInfo(url);

    // Build result object
    const result = {
      url,
      timestamp: new Date().toISOString(),
      paths: {
        documentPath: paths.documentPath,
        htmlFilePath: paths.htmlFilePath,
        mdFilePath: paths.mdFilePath,
        dirPath: paths.dirPath,
        filename: paths.filename
      },
      screenshot,
      html: {
        filePath: htmlPath,
        size: html.length
      },
      metadata,
      images: {
        count: captureState.imageMap.size,
        mapping: Object.fromEntries(captureState.imageMap),
        stats: captureState.stats
      }
    };

    // Save metadata.json file
    const metadataPath = path.join(outputDir, 'metadata.json');
    fs.writeFileSync(metadataPath, JSON.stringify(result, null, 2), 'utf-8');
    console.error(`Saved metadata to: ${metadataPath}`);

    console.error('Analysis complete!');

    return result;
  } finally {
    await browser.close();
  }
}

/**
 * CLI entry point
 */
async function main() {
  const args = process.argv.slice(2);

  if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {
    console.error(`
Usage: node analyze-webpage.js <url> [--output <dir>]

Analyze a webpage and prepare it for content migration.

Arguments:
  <url>              URL of the webpage to analyze (required)
  --output <dir>     Output directory for artifacts (default: ./page-analysis)

Examples:
  node analyze-webpage.js "https://example.com/page"
  node analyze-webpage.js "https://example.com/page" --output ./my-analysis

Output:
  - screenshot.png            Screenshot of the page
  - cleaned.html              Extracted HTML with preserved attributes
  - metadata.json             Complete analysis results
  - images/                   Downloaded images

Requirements:
  npm install playwright
  npx playwright install chromium
`);
    process.exit(args.length === 0 ? 1 : 0);
  }

  const url = args[0];
  let outputDir = './page-analysis';

  // Parse --output flag
  const outputIndex = args.indexOf('--output');
  if (outputIndex !== -1 && args[outputIndex + 1]) {
    outputDir = args[outputIndex + 1];
  }

  try {
    const result = await analyzeWebpage(url, outputDir);

    // Output JSON to stdout (stderr used for progress messages above)
    console.log(JSON.stringify(result, null, 2));
  } catch (error) {
    console.error(`Error analyzing webpage: ${error.message}`);
    console.error(error.stack);
    process.exit(1);
  }
}

// Run if executed directly
if (import.meta.url === `file://${process.argv[1]}`) {
  main();
}

export { analyzeWebpage, scrollToTriggerLazyLoad, extractCleanedHTML, extractMetadata };

#!/usr/bin/env node

/*
 * Copyright 2025 Adobe. All rights reserved.
 * This file is licensed to you under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under
 * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
 * OF ANY KIND, either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */

/**
 * Path generation script for page migration
 * Generates document paths from source URLs
 *
 * This uses the exact same algorithm as the EXCAT MCP tool's generate_document_path
 * to ensure consistent path generation across all migration workflows.
 *
 * Usage:
 *   node generate-path.js "https://example.com/us/en/about.html"
 *
 * Output (JSON):
 *   {
 *     "success": true,
 *     "url": "https://example.com/us/en/about.html",
 *     "documentPath": "/us/en/about",
 *     "mdFilePath": "us/en/about.md",
 *     "htmlFilePath": "us/en/about.plain.html",
 *     "dirPath": "us/en",
 *     "filename": "about",
 *     "directoryCreated": true,
 *     "timestamp": "2025-11-02T10:30:45.123Z"
 *   }
 */

import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

/**
 * Sanitize a filename according to standard naming conventions
 * @param {string} name - The filename to sanitize
 * @returns {string} - Sanitized filename
 */
function sanitizeFilename(name) {
  if (!name) return '';
  return decodeURIComponent(name)
    .toLowerCase()
    .normalize('NFD')
    .replace(/[\u0300-\u036f]/g, '')
    .replace(/[^a-z0-9]+/g, '-')
    .replace(/^-|-$/g, '');
}

/**
 * Sanitize a path according to standard naming conventions
 * @param {string} pathStr - The path to sanitize
 * @returns {string} - Sanitized path
 */
function sanitizePath(pathStr) {
  if (!pathStr) return '';
  const extension = pathStr.split('.').pop();
  const pathname = extension !== pathStr ? pathStr.substring(0, pathStr.lastIndexOf('.')) : pathStr;
  let sanitizedPath = '';
  pathname.split('/').forEach((p) => {
    if (p !== '') {
      sanitizedPath += `/${sanitizeFilename(p)}`;
    }
  });
  if (extension !== pathStr) {
    sanitizedPath += `.${extension}`;
  }
  return sanitizedPath;
}

/**
 * Generate document path from URL according to standard conventions
 * @param {string} url - Source URL
 * @returns {string} - Document path (without extension)
 */
function generateDocumentPath({ url }) {
  let p = new URL(url).pathname;
  if (p.endsWith('/')) {
    p = `${p}index`;
  }
  p = decodeURIComponent(p)
    .toLowerCase()
    .replace(/\.html$/, '')
    .replace(/[^a-z0-9/]/gm, '-');
  return sanitizePath(p);
}

/**
 * Generate document path with full file information
 * @param {string} url - Source URL to process
 * @returns {Object} - Complete path information
 */
function generateDocumentPathInfo(url) {
  try {
    // Generate the document path (no extension)
    const documentPath = generateDocumentPath({ url });

    // Build file paths
    const mdFilePath = `${documentPath}.md`;
    const htmlFilePath = `${documentPath}.plain.html`;

    // Extract directory path (parent directory)
    const dirPath = `${documentPath.substring(0, documentPath.lastIndexOf('/'))}`;

    // Extract just the filename (without extension)
    const filename = documentPath.substring(documentPath.lastIndexOf('/') + 1);

    // Automatically create directory structure if needed
    let directoryCreated = false;
    if (dirPath && dirPath !== 'content') {
      // Resolve to absolute path relative to workspace
      const workspaceRoot = process.env.WORKSPACE_ROOT || process.cwd();
      const absoluteDirPath = path.join(workspaceRoot, dirPath);

      try {
        fs.mkdirSync(absoluteDirPath, { recursive: true });
        directoryCreated = true;
      } catch (error) {
        // Don't fail - directory might already exist
        directoryCreated = false;
      }
    }

    return {
      success: true,
      url,
      documentPath,
      mdFilePath,
      htmlFilePath,
      dirPath,
      filename,
      directoryCreated,
      timestamp: new Date().toISOString(),
    };
  } catch (error) {
    throw new Error(`Failed to generate document path: ${error.message}`);
  }
}

// CLI wrapper
async function main() {
  try {
    // Get URL from command line arguments
    const urlArg = process.argv[2];

    if (!urlArg) {
      console.error(JSON.stringify({
        success: false,
        error: 'Missing URL argument. Usage: node generate-path.js "https://example.com/page"',
        timestamp: new Date().toISOString(),
      }, null, 2));
      process.exit(1);
    }

    // Validate URL
    try {
      new URL(urlArg);
    } catch (e) {
      console.error(JSON.stringify({
        success: false,
        error: `Invalid URL: ${urlArg}`,
        timestamp: new Date().toISOString(),
      }, null, 2));
      process.exit(1);
    }

    // Generate path info
    const result = generateDocumentPathInfo(urlArg);

    // Output as JSON
    console.log(JSON.stringify(result, null, 2));

  } catch (error) {
    console.error(JSON.stringify({
      success: false,
      error: error.message,
      url: process.argv[2],
      timestamp: new Date().toISOString(),
    }, null, 2));
    process.exit(1);
  }
}

// Run if called directly
if (import.meta.url === `file://${process.argv[1]}`) {
  main();
}

// Export functions for use by other scripts
export { sanitizeFilename, sanitizePath, generateDocumentPath, generateDocumentPathInfo };

#!/usr/bin/env node

/*
 * Copyright 2025 Adobe. All rights reserved.
 * This file is licensed to you under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under
 * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
 * OF ANY KIND, either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */

/**
 * Image Capture Utilities for Page Migration
 *
 * Adapted from excat-ops MCP network-image-capture.js
 * Intercepts images during page load and saves them locally
 */

import { mkdir, writeFile } from 'fs/promises';
import { join } from 'path';
import { createHash } from 'crypto';
import sharp from 'sharp';

// Image format mapping (Content-Type → format string)
const IMAGE_FORMAT_MAP = {
  'image/webp': 'webp',
  'image/avif': 'avif',
  'image/svg+xml': 'svg',
  'image/jpeg': 'jpeg',
  'image/jpg': 'jpeg',
  'image/png': 'png',
  'image/gif': 'gif',
  'image/bmp': 'bmp',
  'image/tiff': 'tiff',
  'image/heic': 'heic',
  'image/heif': 'heif',
};

// Supported formats (no conversion needed)
const SUPPORTED_FORMATS = ['jpeg', 'png'];

/**
 * Convert image buffer to PNG using Sharp
 */
async function convertToPng(buffer, originalFormat) {
  try {
    return await sharp(buffer)
      .png({
        compressionLevel: 9,
        palette: true,
      })
      .toBuffer();
  } catch (error) {
    throw new Error(`Failed to convert ${originalFormat} to PNG: ${error.message}`);
  }
}

/**
 * Check if format needs conversion
 */
function needsConversion(format) {
  return format !== 'unknown' && !SUPPORTED_FORMATS.includes(format);
}

/**
 * Set up network event listeners to capture images during page load
 * Must be called BEFORE page.goto()
 *
 * Uses proper request lifecycle:
 * 1. 'request' event - Track image requests as they are issued
 * 2. 'requestfinished' event - Process once body is fully downloaded
 *
 * @param {import('playwright').Page} page - Playwright page object
 * @param {string} outputDir - Directory to save images
 * @param {Object} options - Optional configuration
 * @param {number} options.maxImageSize - Max image size in bytes (default: 10MB)
 * @param {number} options.maxImages - Max number of images to capture (default: 100)
 * @returns {Promise<Object>} Capture state object
 */
export function setupImageCapture(page, outputDir, options = {}) {
  const {
    maxImageSize = 10 * 1024 * 1024, // 10MB
    maxImages = 1000,
  } = options;

  const imageMap = new Map();
  const pendingImages = new Set();
  const trackedRequests = new Set(); // Track image requests
  const stats = {
    total: 0,
    converted: 0,
    skipped: 0,
    failed: 0,
    tooLarge: 0,
    limitReached: 0
  };

  // Handler for request event - track image requests as they start
  const requestHandler = (request) => {
    // Check if this is an image request
    if (request.resourceType() !== 'image') {
      return;
    }

    const url = request.url();

    // Check if already tracked (avoid duplicates)
    if (trackedRequests.has(url)) {
      return;
    }

    // Check if max images limit reached
    if (stats.total >= maxImages) {
      if (stats.limitReached === 0) {
        console.error(`⚠️  Max images limit (${maxImages}) reached, skipping remaining images`);
      }
      stats.limitReached++;
      return;
    }

    // Track this request
    trackedRequests.add(url);
    pendingImages.add(url);
    stats.total++;
  };

  // Handler for requestfinished event - process images once fully downloaded
  const requestFinishedHandler = async (request) => {
    // Only process tracked image requests
    if (request.resourceType() !== 'image') {
      return;
    }

    const url = request.url();

    // Only process if we tracked this request
    if (!trackedRequests.has(url)) {
      return;
    }

    // Check if already captured (should not happen, but safeguard)
    if (imageMap.has(url)) {
      pendingImages.delete(url);
      return;
    }

    try {
      // Get the response object
      const response = await request.response();

      if (!response) {
        console.error(`⚠️  No response for image request: ${url.slice(0, 60)}...`);
        stats.failed++;
        pendingImages.delete(url);
        return;
      }

      // Get binary data from response body (now fully downloaded)
      const buffer = await response.body();

      // Check image size
      if (buffer.length > maxImageSize) {
        console.error(`⚠️  Image too large (${Math.round(buffer.length / 1024 / 1024)}MB): ${url.slice(0, 60)}...`);
        stats.tooLarge++;
        stats.failed++;
        pendingImages.delete(url);
        return;
      }

      // Get content type and determine format
      const contentType = response.headers()['content-type'] || '';
      const format = IMAGE_FORMAT_MAP[contentType.split(';')[0]];

      if (!format) {
        console.error(`⚠️  Unknown image format (${contentType}): ${url.slice(0, 60)}...`);
        stats.failed++;
        pendingImages.delete(url);
        return;
      }

      // Generate filename using MD5 hash
      const hash = createHash('md5').update(url).digest('hex');
      const imagesDir = join(outputDir, 'images');
      await mkdir(imagesDir, { recursive: true });

      // Convert if needed, save as PNG
      if (needsConversion(format)) {
        const pngBuffer = await convertToPng(buffer, format);
        const filename = `${hash}.png`;
        const filePath = join(imagesDir, filename);
        await writeFile(filePath, pngBuffer);

        imageMap.set(url, `./images/${filename}`);
        stats.converted++;
        console.error(`✓ Captured & converted (${format}→PNG): ${url.slice(0, 50)}... → ./images/${filename}`);
      } else {
        // Save original format (JPEG, PNG)
        const ext = format === 'jpeg' ? 'jpg' : format;
        const filename = `${hash}.${ext}`;
        const filePath = join(imagesDir, filename);
        await writeFile(filePath, buffer);

        imageMap.set(url, `./images/${filename}`);
        stats.skipped++;
        console.error(`✓ Captured (${format}): ${url.slice(0, 50)}... → ./images/${filename}`);
      }

    } catch (error) {
      console.error(`✗ Failed to capture ${url.slice(0, 50)}...: ${error.message}`);
      stats.failed++;
    } finally {
      pendingImages.delete(url);
    }
  };

  // Set up both event listeners
  page.on('request', requestHandler);
  page.on('requestfinished', requestFinishedHandler);

  return {
    imageMap,
    pendingImages,
    stats,
    disable: () => {
      page.off('request', requestHandler);
      page.off('requestfinished', requestFinishedHandler);
      console.error('🛑 Image capture disabled');
    },
  };
}

/**
 * Wait for all pending images to complete processing
 *
 * @param {Object} captureState - State object returned from setupImageCapture
 * @param {number} timeout - Max time to wait in milliseconds (default: 5000)
 * @returns {Promise<void>}
 */
export async function waitForPendingImages(captureState, timeout = 5000) {
  const startTime = Date.now();
  const { pendingImages } = captureState;

  while (pendingImages.size > 0) {
    if (Date.now() - startTime > timeout) {
      console.error(`⚠️  Timeout waiting for ${pendingImages.size} pending images after ${timeout}ms`);
      break;
    }
    await new Promise(resolve => setTimeout(resolve, 100));
  }
}

/**
 * Replace image URLs in HTML with local paths
 *
 * @param {string} html - HTML content
 * @param {Map} imageMap - Map of original URL to local path
 * @returns {string} - HTML with replaced image URLs
 */
export function replaceImageUrls(html, imageMap) {
  let modifiedHtml = html;
  let replacements = 0;

  for (const [originalUrl, localPath] of imageMap.entries()) {
    // Replace in src attributes
    const srcRegex = new RegExp(`src="${originalUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}"`, 'g');
    const srcMatch = modifiedHtml.match(srcRegex);
    if (srcMatch) {
      modifiedHtml = modifiedHtml.replace(srcRegex, `src="${localPath}"`);
      replacements += srcMatch.length;
    }

    // Also try without quotes (less common but possible)
    const srcRegex2 = new RegExp(`src=${originalUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`, 'g');
    const srcMatch2 = modifiedHtml.match(srcRegex2);
    if (srcMatch2) {
      modifiedHtml = modifiedHtml.replace(srcRegex2, `src="${localPath}"`);
      replacements += srcMatch2.length;
    }
  }

  console.error(`🔄 Replaced ${replacements} image URL references in HTML`);
  return modifiedHtml;
}

{
  "type": "module",
  "dependencies": {
    "sharp": "^0.34.0"
  }
}