import got from 'got';
import type { Observations } from '../core/types.js';

export async function coletarRobots(obs: Observations): Promise<void> {
  let base: URL;
  try {
    base = new URL(obs.urlFinal || obs.urlSolicitada);
  } catch {
    return;
  }

  obs.robotsTxt = await buscarTexto(new URL('/robots.txt', base).toString());

  const declarado = obs.robotsTxt?.match(/^\s*sitemap:\s*(\S+)/im)?.[1];
  const urlSitemap = declarado ?? new URL('/sitemap.xml', base).toString();
  obs.sitemapXml = await buscarTexto(urlSitemap, 200_000);
}

async function buscarTexto(url: string, limite = 100_000): Promise<string | undefined> {
  try {
    const resp = await got(url, {
      throwHttpErrors: false,
      timeout: { request: 8000 },
      headers: { 'user-agent': 'Red' },
    });
    if (resp.statusCode >= 200 && resp.statusCode < 300 && typeof resp.body === 'string') {
      return resp.body.slice(0, limite);
    }
  } catch {
    return undefined;
  }
  return undefined;
}