The DIY way vs Bluefield

import { chromium } from 'playwright'
import { Readability } from '@mozilla/readability'
import { JSDOM } from 'jsdom'
import TurndownService from 'turndown'

// 1. Render with a real browser so client-side pages actually load
async function render(url: string): Promise<string> {
  const browser = await chromium.launch({ args: ['--no-sandbox'] })
  const ctx = await browser.newContext({
    userAgent: nextAgent(),     // 2. rotate identities to avoid blocks
    proxy: pickProxy(),         //    and rotate egress IPs
  })
  const page = await ctx.newPage()

  // 3. Retry past bot walls, challenges, and rate limits
  let html = ''
  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
    const res = await page.goto(url, { waitUntil: 'networkidle', timeout: 30_000 })
    if (res?.status() === 429) {
      await sleep(backoff(attempt))
      await ctx.setExtraHTTPHeaders({ 'user-agent': nextAgent() })
      continue
    }
    if (await looksLikeChallenge(page)) {
      await solveOrWait(page)
      continue
    }
    html = await page.content()
    break
  }
  await browser.close()
  if (!html) throw new Error(`could not fetch ${url}`)
  return html
}

// 4. Strip nav, ads, cookie banners, and other boilerplate
function extractMain(html: string, url: string): string {
  const dom = new JSDOM(html, { url })
  stripKnownNoise(dom.window.document)
  const article = new Readability(dom.window.document).parse()
  if (!article?.content) throw new Error('no readable content')
  return article.content
}

// 5. Convert the cleaned HTML to markdown for your model
const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' })

// 6. Chunk the markdown to fit your embedding window
function chunk(markdown: string, maxTokens = 512): string[] {
  const out: string[] = []
  let buf = ''
  for (const block of markdown.split(/\n{2,}/)) {
    if (estimateTokens(buf + block) > maxTokens) { out.push(buf.trim()); buf = '' }
    buf += `${block}\n\n`
  }
  if (buf.trim()) out.push(buf.trim())
  return out
}

// Wire it together for a single URL
const html = await render(url)
const markdown = turndown.turndown(extractMain(html, url))
const chunks = chunk(markdown)
// ...and you still own every proxy bill, browser crash, and site change

import { BlueFieldsClient } from '@bluefields/sdk'

const bf = new BlueFieldsClient({ apiKey: process.env.BF_KEY })
const { markdown } = await bf.scrape({ url: 'https://example.com', formats: ['markdown'] })