const response = await fetch('https://news.ycombinator.com/');
const rawHtml = await response.json();

// Run the scraper
const pages = await scraper.runFiles([rawHtml], {
  model: 'gpt-4-turbo',
  schema,
  mode: 'html',
  closeOnFinish: true,
})

// Grab today's HN front page to run the example
await fetch('https://news.ycombinator.com/')
  .then((res) => res.text())
  .then((html) => writeFileSync(HNRawHtmlPath, html, { encoding: 'utf-8', flag: 'w' }))
  .catch((e) => {
    console.error("Failed to fetch content from Hackernews", e)
  })

// Local file paths to scrape - will be loaded from local filepaths.
const filePaths = [HNRawHtmlPath]

// Run the scraper
const pages = await scraper.runFiles(filePaths, {
  model: 'gpt-4-turbo',
  schema,
  mode: 'html',
  closeOnFinish: true,
})