How to Build a Document Analysis System with Claude

┌─────────────┐    ┌──────────────┐    ┌─────────────┐    ┌──────────────┐    ┌────────────┐
│  Ingestion   │───▶│  Extraction  │───▶│  Chunking   │───▶│  Analysis    │───▶│  Output    │
│  (Upload)    │    │  (Text/OCR)  │    │  (Smart)    │    │  (Claude)    │    │  (Typed)   │
└─────────────┘    └──────────────┘    └─────────────┘    └──────────────┘    └────────────┘
     PDF/IMG         Raw text or          Semantic          Structured          Validated
     bytes           base64 image         sections          extraction          JSON

interface DocumentPipeline {
  id: string;
  fileName: string;
  mimeType: string;
  status: "uploading" | "extracting" | "chunking" | "analyzing" | "complete" | "failed";
  documentType: DocumentType;
  rawText: string | null;
  chunks: DocumentChunk[];
  extractedData: Record<string, unknown>;
  confidence: number;
  errors: PipelineError[];
  createdAt: Date;
  completedAt: Date | null;
}

type DocumentType = "invoice" | "contract" | "receipt" | "report" | "unknown";

interface DocumentChunk {
  index: number;
  content: string;
  tokenCount: number;
  sectionHeader: string | null;
  pageRange: { start: number; end: number };
}

interface PipelineError {
  stage: string;
  message: string;
  recoverable: boolean;
}

import pdf from "pdf-parse";

interface ExtractionResult {
  text: string;
  pageCount: number;
  pageTexts: string[];
  isScanned: boolean;
  metadata: {
    author: string | null;
    title: string | null;
    creationDate: string | null;
  };
}

async function extractPdfText(
  buffer: Buffer
): Promise<ExtractionResult> {
  const pageTexts: string[] = [];

  const data = await pdf(buffer, {
    pagerender: (pageData: { getTextContent: () => Promise<{ items: Array<{ str: string }> }> }) => {
      return pageData.getTextContent().then((textContent) => {
        const strings = textContent.items.map((item) => item.str);
        const pageText = strings.join(" ").trim();
        pageTexts.push(pageText);
        return pageText;
      });
    },
  });

  const isScanned = detectScannedDocument(data.text, data.numpages);

  return {
    text: data.text,
    pageCount: data.numpages,
    pageTexts,
    isScanned,
    metadata: {
      author: data.info?.Author ?? null,
      title: data.info?.Title ?? null,
      creationDate: data.info?.CreationDate ?? null,
    },
  };
}

function detectScannedDocument(
  text: string,
  pageCount: number
): boolean {
  const charCount = text.replace(/\s/g, "").length;
  const charsPerPage = charCount / Math.max(pageCount, 1);

  // Digital PDFs typically have 200+ characters per page
  // Scanned PDFs have very little extractable text
  return charsPerPage < 50;
}

import Anthropic from "@anthropic-ai/sdk";
import { fromPath } from "pdf2pic";
import { readFileSync } from "fs";
import path from "path";

const anthropic = new Anthropic();

interface VisionExtractionOptions {
  filePath: string;
  pageNumbers: number[];
  prompt: string;
}

async function extractWithVision({
  filePath,
  pageNumbers,
  prompt,
}: VisionExtractionOptions): Promise<string[]> {
  const converter = fromPath(filePath, {
    density: 300,
    format: "png",
    width: 2048,
    height: 2048,
    saveFilename: "page",
    savePath: path.join("/tmp", "doc-analysis"),
  });

  const results: string[] = [];

  for (const pageNum of pageNumbers) {
    const converted = await converter(pageNum);

    if (!converted.path) {
      throw new Error(`Failed to convert page ${pageNum}`);
    }

    const imageBuffer = readFileSync(converted.path);
    const base64Image = imageBuffer.toString("base64");

    const response = await anthropic.messages.create({
      model: "claude-sonnet-4-20250514",
      max_tokens: 4096,
      messages: [
        {
          role: "user",
          content: [
            {
              type: "image",
              source: {
                type: "base64",
                media_type: "image/png",
                data: base64Image,
              },
            },
            {
              type: "text",
              text: prompt,
            },
          ],
        },
      ],
    });

    const textBlock = response.content.find(
      (block) => block.type === "text"
    );
    if (textBlock && textBlock.type === "text") {
      results.push(textBlock.text);
    }
  }

  return results;
}

interface ExtractionSchema {
  fields: Array<{
    name: string;
    type: string;
    description: string;
    required: boolean;
    examples?: string[];
  }>;
}

function buildExtractionPrompt(
  schema: ExtractionSchema,
  documentType: string,
  context: string
): string {
  const fieldDescriptions = schema.fields
    .map((field) => {
      const required = field.required ? "REQUIRED" : "OPTIONAL";
      const examples = field.examples
        ? ` Examples: ${field.examples.join(", ")}`
        : "";
      return `- "${field.name}" (${field.type}, ${required}): ${field.description}${examples}`;
    })
    .join("\n");

  return `You are a document analysis system extracting structured data from a ${documentType}.

DOCUMENT CONTENT:
${context}

EXTRACTION INSTRUCTIONS:
Extract the following fields from the document. Return ONLY a valid JSON object with the specified fields. Do not include any explanatory text before or after the JSON.

FIELDS:
${fieldDescriptions}

RULES:
1. If a required field cannot be found, set its value to null and add it to the "missingFields" array.
2. If a value is ambiguous, include both possible values in the "ambiguities" array.
3. For monetary values, extract the number without currency symbols. Include the currency in a separate "currency" field.
4. For dates, use ISO 8601 format (YYYY-MM-DD).
5. Include a "confidence" field (0.0 to 1.0) indicating overall extraction confidence.

Return the JSON object now.`;
}

import Anthropic from "@anthropic-ai/sdk";

const anthropic = new Anthropic();

interface InvoiceData {
  vendorName: string | null;
  vendorAddress: string | null;
  invoiceNumber: string | null;
  invoiceDate: string | null;
  dueDate: string | null;
  subtotal: number | null;
  taxAmount: number | null;
  totalAmount: number | null;
  currency: string;
  lineItems: LineItem[];
  paymentTerms: string | null;
  purchaseOrderNumber: string | null;
  missingFields: string[];
  ambiguities: string[];
  confidence: number;
}

interface LineItem {
  description: string;
  quantity: number | null;
  unitPrice: number | null;
  totalPrice: number | null;
}

const INVOICE_SCHEMA: ExtractionSchema = {
  fields: [
    {
      name: "vendorName",
      type: "string",
      description: "The company or person issuing the invoice",
      required: true,
      examples: ["Acme Corp", "John Smith Consulting"],
    },
    {
      name: "vendorAddress",
      type: "string",
      description: "Full address of the vendor",
      required: false,
    },
    {
      name: "invoiceNumber",
      type: "string",
      description: "Unique invoice identifier",
      required: true,
      examples: ["INV-2024-001", "12345"],
    },
    {
      name: "invoiceDate",
      type: "string (ISO 8601)",
      description: "Date the invoice was issued",
      required: true,
    },
    {
      name: "dueDate",
      type: "string (ISO 8601)",
      description: "Payment due date",
      required: false,
    },
    {
      name: "lineItems",
      type: "array of objects",
      description:
        "Each item with description, quantity, unitPrice, totalPrice",
      required: true,
    },
    {
      name: "subtotal",
      type: "number",
      description: "Sum before tax",
      required: false,
    },
    {
      name: "taxAmount",
      type: "number",
      description: "Tax amount",
      required: false,
    },
    {
      name: "totalAmount",
      type: "number",
      description: "Final total including tax",
      required: true,
    },
    {
      name: "currency",
      type: "string",
      description: "Three-letter currency code",
      required: true,
      examples: ["USD", "GBP", "LKR", "EUR"],
    },
    {
      name: "paymentTerms",
      type: "string",
      description: "Payment terms if stated",
      required: false,
      examples: ["Net 30", "Due on receipt"],
    },
    {
      name: "purchaseOrderNumber",
      type: "string",
      description: "PO number if referenced",
      required: false,
    },
  ],
};

async function processInvoice(
  fileBuffer: Buffer,
  fileName: string
): Promise<InvoiceData> {
  // Stage 1: Extract text
  const extraction = await extractPdfText(fileBuffer);

  let documentContent: string;

  if (extraction.isScanned) {
    // Stage 2a: Vision-based extraction for scanned docs
    const tempPath = `/tmp/doc-analysis/${fileName}`;
    const { writeFileSync } = await import("fs");
    writeFileSync(tempPath, fileBuffer);

    const pageResults = await extractWithVision({
      filePath: tempPath,
      pageNumbers: Array.from(
        { length: extraction.pageCount },
        (_, i) => i + 1
      ),
      prompt:
        "Extract all visible text from this invoice page. Preserve the layout structure, especially tables and columns. Include all numbers, dates, and amounts exactly as shown.",
    });

    documentContent = pageResults.join("\n\n--- PAGE BREAK ---\n\n");
  } else {
    // Stage 2b: Use extracted text directly
    documentContent = extraction.text;
  }

  // Stage 3: Build prompt and extract
  const prompt = buildExtractionPrompt(
    INVOICE_SCHEMA,
    "invoice",
    documentContent
  );

  const response = await anthropic.messages.create({
    model: "claude-sonnet-4-20250514",
    max_tokens: 4096,
    messages: [{ role: "user", content: prompt }],
  });

  const textBlock = response.content.find(
    (block) => block.type === "text"
  );
  if (!textBlock || textBlock.type !== "text") {
    throw new Error("No text response from Claude");
  }

  // Stage 4: Parse and validate
  const parsed = parseJsonResponse<InvoiceData>(textBlock.text);
  const validated = validateInvoiceData(parsed);

  return validated;
}

function parseJsonResponse<T>(raw: string): T {
  // Claude sometimes wraps JSON in markdown code blocks
  const cleaned = raw
    .replace(/^```json?\s*\n?/m, "")
    .replace(/\n?```\s*$/m, "")
    .trim();

  try {
    return JSON.parse(cleaned) as T;
  } catch {
    throw new Error(
      `Failed to parse extraction response as JSON: ${cleaned.substring(0, 200)}`
    );
  }
}

function validateInvoiceData(data: InvoiceData): InvoiceData {
  // Cross-validate: line item totals should sum to subtotal
  if (data.lineItems.length > 0 && data.subtotal !== null) {
    const calculatedSubtotal = data.lineItems.reduce(
      (sum, item) => sum + (item.totalPrice ?? 0),
      0
    );
    const discrepancy = Math.abs(
      calculatedSubtotal - data.subtotal
    );

    if (discrepancy > 0.01) {
      data.ambiguities = data.ambiguities ?? [];
      data.ambiguities.push(
        `Line item sum (${calculatedSubtotal.toFixed(2)}) differs from stated subtotal (${data.subtotal.toFixed(2)}) by ${discrepancy.toFixed(2)}`
      );
      data.confidence = Math.min(data.confidence, 0.7);
    }
  }

  // Validate: total should equal subtotal + tax
  if (
    data.subtotal !== null &&
    data.taxAmount !== null &&
    data.totalAmount !== null
  ) {
    const expectedTotal = data.subtotal + data.taxAmount;
    const totalDiscrepancy = Math.abs(
      expectedTotal - data.totalAmount
    );

    if (totalDiscrepancy > 0.01) {
      data.ambiguities = data.ambiguities ?? [];
      data.ambiguities.push(
        `Subtotal (${data.subtotal}) + tax (${data.taxAmount}) = ${expectedTotal.toFixed(2)}, but stated total is ${data.totalAmount}`
      );
      data.confidence = Math.min(data.confidence, 0.6);
    }
  }

  return data;
}

interface ChunkOptions {
  maxTokens: number;
  overlapTokens: number;
}

function chunkDocument(
  pageTexts: string[],
  options: ChunkOptions = { maxTokens: 8000, overlapTokens: 500 }
): DocumentChunk[] {
  const chunks: DocumentChunk[] = [];
  let currentChunk = "";
  let currentStartPage = 1;
  let chunkIndex = 0;

  for (let i = 0; i < pageTexts.length; i++) {
    const pageText = pageTexts[i];
    const combinedTokens = estimateTokens(
      currentChunk + "\n\n" + pageText
    );

    if (
      combinedTokens > options.maxTokens &&
      currentChunk.length > 0
    ) {
      // Find a natural break point in the current chunk
      const breakPoint = findNaturalBreak(currentChunk);
      const chunkContent = currentChunk.substring(0, breakPoint);
      const overflow = currentChunk.substring(breakPoint);

      chunks.push({
        index: chunkIndex,
        content: chunkContent.trim(),
        tokenCount: estimateTokens(chunkContent),
        sectionHeader: extractSectionHeader(chunkContent),
        pageRange: { start: currentStartPage, end: i },
      });

      chunkIndex++;
      currentChunk = overflow + "\n\n" + pageText;
      currentStartPage = i; // Overlap: include context from prev
    } else {
      currentChunk +=
        (currentChunk.length > 0 ? "\n\n" : "") + pageText;
    }
  }

  // Final chunk
  if (currentChunk.trim().length > 0) {
    chunks.push({
      index: chunkIndex,
      content: currentChunk.trim(),
      tokenCount: estimateTokens(currentChunk),
      sectionHeader: extractSectionHeader(currentChunk),
      pageRange: {
        start: currentStartPage,
        end: pageTexts.length,
      },
    });
  }

  return chunks;
}

function findNaturalBreak(text: string): number {
  // Priority: section headers > paragraph breaks > sentence ends
  const sectionPattern = /\n(?=#{1,3}\s|\d+\.\s|[A-Z][A-Z\s]{3,}:?\n)/g;
  const paragraphPattern = /\n\s*\n/g;
  const sentencePattern = /[.!?]\s+/g;

  const targetPosition = Math.floor(text.length * 0.8);

  // Try section break near the end
  let lastMatch: RegExpExecArray | null = null;
  let match: RegExpExecArray | null;

  match = sectionPattern.exec(text);
  while (match !== null) {
    if (match.index <= targetPosition) {
      lastMatch = match;
    }
    match = sectionPattern.exec(text);
  }
  if (lastMatch && lastMatch.index > text.length * 0.5) {
    return lastMatch.index;
  }

  // Try paragraph break
  lastMatch = null;
  match = paragraphPattern.exec(text);
  while (match !== null) {
    if (match.index <= targetPosition) {
      lastMatch = match;
    }
    match = paragraphPattern.exec(text);
  }
  if (lastMatch && lastMatch.index > text.length * 0.5) {
    return lastMatch.index;
  }

  // Fall back to sentence break
  lastMatch = null;
  match = sentencePattern.exec(text);
  while (match !== null) {
    if (match.index <= targetPosition) {
      lastMatch = match;
    }
    match = sentencePattern.exec(text);
  }
  if (lastMatch) {
    return lastMatch.index + lastMatch[0].length;
  }

  return targetPosition;
}

function extractSectionHeader(text: string): string | null {
  const headerMatch = text.match(
    /^(?:#{1,3}\s+(.+)|(\d+\.?\s+[A-Z].+)|([A-Z][A-Z\s]{3,}):?\s*$)/m
  );
  return headerMatch
    ? (headerMatch[1] ?? headerMatch[2] ?? headerMatch[3])?.trim() ??
        null
    : null;
}

function estimateTokens(text: string): number {
  // Rough estimate: 1 token ≈ 4 characters for English text
  return Math.ceil(text.length / 4);
}

interface AnalysisResult<T> {
  data: T | null;
  confidence: number;
  errors: PipelineError[];
  processingTimeMs: number;
  fallbackUsed: boolean;
}

async function analyzeWithRetry<T>(
  content: string,
  prompt: string,
  validator: (data: unknown) => data is T,
  maxRetries: number = 2
): Promise<AnalysisResult<T>> {
  const startTime = Date.now();
  const errors: PipelineError[] = [];
  let fallbackUsed = false;

  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      const response = await anthropic.messages.create({
        model:
          attempt === 0
            ? "claude-sonnet-4-20250514"
            : "claude-sonnet-4-20250514",
        max_tokens: 4096,
        messages: [{ role: "user", content: prompt }],
      });

      const textBlock = response.content.find(
        (block) => block.type === "text"
      );
      if (!textBlock || textBlock.type !== "text") {
        throw new Error("Empty response from Claude");
      }

      const parsed = parseJsonResponse<T>(textBlock.text);

      if (!validator(parsed)) {
        throw new Error("Response failed schema validation");
      }

      return {
        data: parsed,
        confidence: calculateConfidence(parsed, errors),
        errors,
        processingTimeMs: Date.now() - startTime,
        fallbackUsed,
      };
    } catch (error) {
      const message =
        error instanceof Error ? error.message : String(error);
      errors.push({
        stage: "analysis",
        message: `Attempt ${attempt + 1}: ${message}`,
        recoverable: attempt < maxRetries,
      });

      if (attempt < maxRetries) {
        fallbackUsed = true;
        // Modify prompt to be more explicit on retry
        prompt = addRetryContext(prompt, message);
      }
    }
  }

  return {
    data: null,
    confidence: 0,
    errors,
    processingTimeMs: Date.now() - startTime,
    fallbackUsed,
  };
}

function addRetryContext(
  originalPrompt: string,
  previousError: string
): string {
  return `${originalPrompt}

IMPORTANT: A previous extraction attempt failed with this error: "${previousError}"
Please ensure your response is ONLY a valid JSON object with no additional text, markdown formatting, or code block markers. Double-check that all required fields are present.`;
}

function calculateConfidence(
  data: unknown,
  errors: PipelineError[]
): number {
  let confidence = 1.0;

  // Reduce confidence for each retry needed
  confidence -= errors.length * 0.1;

  // Check for null required fields
  if (data && typeof data === "object") {
    const record = data as Record<string, unknown>;
    const missingFields = record["missingFields"];
    if (Array.isArray(missingFields) && missingFields.length > 0) {
      confidence -= missingFields.length * 0.1;
    }

    const ambiguities = record["ambiguities"];
    if (Array.isArray(ambiguities) && ambiguities.length > 0) {
      confidence -= ambiguities.length * 0.05;
    }
  }

  return Math.max(0, Math.min(1, confidence));
}

import { NextRequest, NextResponse } from "next/server";

const MAX_FILE_SIZE = 20 * 1024 * 1024; // 20MB
const ALLOWED_TYPES = [
  "application/pdf",
  "image/png",
  "image/jpeg",
  "image/webp",
];

export async function POST(request: NextRequest) {
  try {
    const formData = await request.formData();
    const file = formData.get("file") as File | null;
    const documentType = formData.get("type") as DocumentType | null;

    // Input validation
    if (!file) {
      return NextResponse.json(
        { error: { code: "MISSING_FILE", message: "No file provided" } },
        { status: 400 }
      );
    }

    if (!ALLOWED_TYPES.includes(file.type)) {
      return NextResponse.json(
        {
          error: {
            code: "INVALID_TYPE",
            message: `File type ${file.type} is not supported. Accepted: ${ALLOWED_TYPES.join(", ")}`,
          },
        },
        { status: 400 }
      );
    }

    if (file.size > MAX_FILE_SIZE) {
      return NextResponse.json(
        {
          error: {
            code: "FILE_TOO_LARGE",
            message: `File size ${(file.size / 1024 / 1024).toFixed(1)}MB exceeds limit of ${MAX_FILE_SIZE / 1024 / 1024}MB`,
          },
        },
        { status: 400 }
      );
    }

    const buffer = Buffer.from(await file.arrayBuffer());

    // Process based on document type
    const result = await processDocument(
      buffer,
      file.name,
      file.type,
      documentType ?? "unknown"
    );

    return NextResponse.json({
      success: true,
      data: result.data,
      metadata: {
        confidence: result.confidence,
        processingTimeMs: result.processingTimeMs,
        fallbackUsed: result.fallbackUsed,
        errors: result.errors.filter((e) => !e.recoverable),
      },
    });
  } catch (error) {
    console.error("Document analysis failed:", error);

    return NextResponse.json(
      {
        error: {
          code: "PROCESSING_FAILED",
          message: "Document analysis failed. Please try again.",
        },
      },
      { status: 500 }
    );
  }
}

async function processDocument(
  buffer: Buffer,
  fileName: string,
  mimeType: string,
  documentType: DocumentType
): Promise<AnalysisResult<Record<string, unknown>>> {
  if (mimeType.startsWith("image/")) {
    // Direct image: send to Claude Vision
    const base64 = buffer.toString("base64");
    const mediaType = mimeType as
      | "image/png"
      | "image/jpeg"
      | "image/webp";

    return analyzeImage(base64, mediaType, documentType);
  }

  // PDF processing pipeline
  if (documentType === "invoice") {
    const invoiceData = await processInvoice(buffer, fileName);
    return {
      data: invoiceData as unknown as Record<string, unknown>,
      confidence: invoiceData.confidence,
      errors: [],
      processingTimeMs: 0,
      fallbackUsed: false,
    };
  }

  // Generic document processing
  const extraction = await extractPdfText(buffer);
  const chunks = chunkDocument(
    extraction.pageTexts,
    { maxTokens: 8000, overlapTokens: 500 }
  );

  const chunkResults = await Promise.all(
    chunks.map((chunk) =>
      analyzeChunk(chunk, documentType)
    )
  );

  return mergeChunkResults(chunkResults);
}