IndexNow 429 on Cloudflare Workers? Move It to GitHub Actions

IndexNow API status: 429

Same JSON body, same key, same host:
  from the Cloudflare Worker  → 429, 429, 429  (Bing, Seznam, api.indexnow.org)
  from a random clean machine → 202 on the first try

// submit changed URLs ...
await env.SITEMAP_KV.put(KV_KEY, JSON.stringify(currentUrls));  // always!

.github/workflows/indexnow-daily.yml   ← the schedule
scripts/indexnow-submit.mjs            ← the logic
scripts/indexnow-state.json            ← the state (written by the workflow)

name: IndexNow daily submission

on:
  schedule:
    - cron: "15 12 * * *"   # daily, 12:15 UTC — pick your own time
  workflow_dispatch:         # manual "Run workflow" button

permissions:
  contents: write            # to commit the updated state file

concurrency:
  group: indexnow
  cancel-in-progress: false

jobs:
  submit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v5

      - name: Submit new/removed URLs
        env:
          INDEXNOW_KEY: ${{ secrets.INDEXNOW_KEY }}
          BING_WMT_API_KEY: ${{ secrets.BING_WMT_API_KEY }}
          RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
          TO_EMAIL: you@yoursite.com
          FROM_EMAIL: indexnow@updates.yoursite.com
        run: node scripts/indexnow-submit.mjs

      - name: Commit updated state
        run: |
          if ! git diff --quiet scripts/indexnow-state.json; then
            git config user.name "indexnow-bot"
            git config user.email "actions@users.noreply.github.com"
            git add scripts/indexnow-state.json
            git commit -m "indexnow: update submission state"
            git push
          else
            echo "State unchanged."
          fi

// IndexNow + Bing URL Submission — GitHub Actions edition.
//
// Per host:
//   1. Fetches the sitemap and diffs it against scripts/indexnow-state.json.
//   2. Merges today's new/removed URLs with any still-pending URLs from
//      previously failed submissions — nothing is ever lost.
//   3. Submits to IndexNow endpoints in order (Bing first), retrying on 429/5xx.
//      One accepted submission propagates to all IndexNow engines.
//   4. Additionally pushes NEW URLs through the Bing Webmaster URL Submission
//      API (authenticated, per-site quota — immune to IP reputation).
//   5. Only accepted URLs leave the pending queue; failures retry next run.

import { readFile, writeFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";

const STATE_PATH = fileURLToPath(new URL("./indexnow-state.json", import.meta.url));

export const SETTINGS = {
  retryDelayMs: 5000,
  attemptsPerEndpoint: 2,
  maxBatch: 10000,          // IndexNow max urlList per request
  bingApiMaxPerRun: 100     // stay within the Bing Submission API daily quota
};

export const ENDPOINTS = [
  "https://www.bing.com/indexnow",
  "https://search.seznam.cz/indexnow",
  "https://api.indexnow.org/indexnow"
];

// One entry per host. To add a subdomain: ensure its sitemap exists, make sure
// the key file (INDEXNOW_KEY) is served at https://<host>/<key>.txt, and add
// an entry. bingSiteUrls: the site URL(s) exactly as verified in Bing
// Webmaster Tools (empty array = skip the Bing Submission API for this host).
export const HOSTS = [
  {
    host: "yoursite.com",
    sitemap: "https://yoursite.com/sitemap.xml",
    bingSiteUrls: ["http://yoursite.com", "https://yoursite.com"]
  }
];

const env = process.env;
const DRY_RUN = env.DRY_RUN === "1";

async function main() {
  if (!env.INDEXNOW_KEY) {
    console.error("FATAL: INDEXNOW_KEY is not set");
    process.exit(1);
  }

  let state = {};
  try { state = JSON.parse(await readFile(STATE_PATH, "utf8")); }
  catch { console.log("No state file yet — first run submits the full sitemap."); }

  const reports = [];
  for (const h of HOSTS) {
    try {
      reports.push(await runHost(h, state));
    } catch (e) {
      reports.push({ host: h.host, error: e.message });
    }
  }

  if (!DRY_RUN) {
    await writeFile(STATE_PATH, JSON.stringify(state, null, 1) + "\n");
  }

  const noteworthy = reports.some(r => r.error || r.submission);
  if (noteworthy && !DRY_RUN) {
    try { await sendSummaryNotification(reports); }
    catch (err) { console.error("Email notification failed:", err.message); }
  }

  for (const r of reports) {
    console.log(JSON.stringify(r, null, 2));
  }

  // Fatal-config errors should fail the workflow so GitHub notifies too;
  // ordinary submission failures do not (the queue retries them tomorrow).
  if (reports.every(r => r.error)) process.exit(1);
}

// Node fetch has no default timeout — without this, one hanging endpoint
// would stall the workflow until GitHub's 6-hour job kill.
function fetchWithTimeout(url, opts = {}) {
  return fetch(url, { ...opts, signal: AbortSignal.timeout(30_000) });
}

async function runHost(config, state) {
  const sitemapRes = await fetchWithTimeout(config.sitemap);
  if (!sitemapRes.ok) {
    return { host: config.host, error: `Sitemap fetch failed: ${sitemapRes.status}` };
  }
  const xml = await sitemapRes.text();

  const currentUrls = [...xml.matchAll(/<loc>(.*?)<\/loc>/g)]
    .map(m => m[1].trim())
    .filter(u => {
      try { return new URL(u).hostname === config.host; }
      catch { return false; }
    });

  if (currentUrls.length === 0) {
    return { host: config.host, error: "No matching-host URLs found in sitemap" };
  }

  const hostState = state[config.host] ?? { last_urls: [], pending_urls: [] };
  const previousSet = new Set(hostState.last_urls);
  const currentSet = new Set(currentUrls);
  const newUrls = currentUrls.filter(u => !previousSet.has(u));
  const removedUrls = hostState.last_urls.filter(u => !currentSet.has(u));

  const pending = [...new Set([...hostState.pending_urls, ...newUrls, ...removedUrls])];

  // "Seen" is tracked separately from "successfully submitted". The FULL merged
  // pending list goes into state BEFORE submitting — so even if this process
  // dies mid-submission, no URL is ever marked seen without being queued.
  state[config.host] = { last_urls: currentUrls, pending_urls: pending };

  if (pending.length === 0) {
    return { host: config.host, totalUrls: currentUrls.length, newUrls: [], removedUrls: [], retriedCount: 0, submission: null };
  }

  if (DRY_RUN) {
    return { host: config.host, totalUrls: currentUrls.length, newUrls, removedUrls,
      retriedCount: hostState.pending_urls.length, submission: { dryRun: true, wouldSubmit: pending.length } };
  }

  const submission = await submitToIndexNow(config.host, env.INDEXNOW_KEY, pending);
  state[config.host].pending_urls = submission.failedUrls;

  // Second channel: Bing URL Submission API for NEW urls (quota-capped).
  let bingApi = null;
  if (env.BING_WMT_API_KEY && config.bingSiteUrls.length > 0 && newUrls.length > 0) {
    bingApi = await submitToBingApi(config.bingSiteUrls, newUrls.slice(0, SETTINGS.bingApiMaxPerRun));
    if (newUrls.length > SETTINGS.bingApiMaxPerRun) {
      bingApi.note = `capped at ${SETTINGS.bingApiMaxPerRun} of ${newUrls.length} new URLs (daily quota)`;
    }
  }

  return {
    host: config.host,
    totalUrls: currentUrls.length,
    newUrls,
    removedUrls,
    retriedCount: hostState.pending_urls.length,
    submission,
    bingApi
  };
}

export async function submitToIndexNow(host, key, urls) {
  const attempts = [];
  const failedUrls = [];
  let acceptedCount = 0;

  const chunks = [];
  for (let i = 0; i < urls.length; i += SETTINGS.maxBatch) {
    chunks.push(urls.slice(i, i + SETTINGS.maxBatch));
  }

  for (let i = 0; i < chunks.length; i++) {
    const label = chunks.length > 1 ? ` [batch ${i + 1}/${chunks.length}]` : "";
    const ok = await submitChunk(host, key, chunks[i], attempts, label);
    if (ok) acceptedCount += chunks[i].length;
    else failedUrls.push(...chunks[i]);
  }

  return { ok: failedUrls.length === 0, acceptedCount, failedUrls, attempts };
}

async function submitChunk(host, key, chunk, attempts, label) {
  const body = JSON.stringify({
    host,
    key,
    keyLocation: `https://${host}/${key}.txt`,
    urlList: chunk
  });

  for (const endpoint of ENDPOINTS) {
    for (let attempt = 1; attempt <= SETTINGS.attemptsPerEndpoint; attempt++) {
      let status;
      try {
        const res = await fetchWithTimeout(endpoint, {
          method: "POST",
          headers: { "Content-Type": "application/json; charset=utf-8" },
          body
        });
        status = res.status;
      } catch (err) {
        status = `network error: ${err.message}`;
      }
      attempts.push(`${endpoint}${label} (attempt ${attempt}): ${status}`);

      if (status === 200 || status === 202) return true;

      // 429/5xx/network errors are worth one retry; 400/403/422 mean a config
      // problem on our side — retrying won't help, try the next endpoint.
      const retryable = status === 429 || status === 500 || status === 503 || typeof status === "string";
      if (!retryable) break;
      if (attempt < SETTINGS.attemptsPerEndpoint) await sleep(SETTINGS.retryDelayMs);
    }
  }
  return false;
}

async function submitToBingApi(siteUrls, urls) {
  const attempts = [];
  // Bing registers sites under an exact URL (often http://); try known variants.
  for (const siteUrl of siteUrls) {
    let status, detail = "";
    try {
      const res = await fetchWithTimeout(`https://ssl.bing.com/webmaster/api.svc/json/SubmitUrlBatch?apikey=${env.BING_WMT_API_KEY}`, {
        method: "POST",
        headers: { "Content-Type": "application/json; charset=utf-8" },
        body: JSON.stringify({ siteUrl, urlList: urls })
      });
      status = res.status;
      if (!res.ok) detail = (await res.text()).slice(0, 200);
    } catch (err) {
      status = `network error: ${err.message}`;
    }
    attempts.push(`SubmitUrlBatch ${siteUrl}: ${status} ${detail}`.trim());
    if (status === 200) return { ok: true, submitted: urls.length, attempts };
  }
  return { ok: false, submitted: 0, attempts };
}

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

async function sendSummaryNotification(reports) {
  if (!env.RESEND_API_KEY || !env.TO_EMAIL) return;
  const fromEmail = env.FROM_EMAIL || "onboarding@resend.dev";

  const failedHosts = reports.filter(r => r.error || (r.submission && !r.submission.ok));
  const totalNew = reports.reduce((n, r) => n + (r.newUrls?.length ?? 0), 0);
  const totalRemoved = reports.reduce((n, r) => n + (r.removedUrls?.length ?? 0), 0);

  const subject = failedHosts.length > 0
    ? `IndexNow: FAILED (${failedHosts.map(r => r.host).join(", ")}) — queued for retry`
    : `IndexNow: OK — ${totalNew} new, ${totalRemoved} removed`;

  const lines = [];
  lines.push(`IndexNow daily run (GitHub Actions) — ${reports.length} host(s)`);
  lines.push(`Timestamp: ${new Date().toISOString()}`);
  lines.push(``);

  for (const r of reports) {
    lines.push(`=== ${r.host} ===`);
    if (r.error) { lines.push(`  ERROR: ${r.error}`, ``); continue; }
    if (!r.submission) { lines.push(`  Nothing changed (${r.totalUrls} URLs in sitemap).`, ``); continue; }
    const s = r.submission;
    lines.push(`  IndexNow: ${s.ok ? "OK" : `FAILED — ${s.failedUrls.length} URLs queued for retry`}`);
    lines.push(`  Accepted: ${s.acceptedCount} URLs`);
    if (r.retriedCount > 0) lines.push(`  Retried from previous failed runs: ${r.retriedCount} URLs`);
    lines.push(`  Attempts:`);
    s.attempts.forEach(a => lines.push(`    ${a}`));
    if (r.bingApi) {
      lines.push(`  Bing Submission API: ${r.bingApi.ok ? `OK — ${r.bingApi.submitted} URLs` : "FAILED"}${r.bingApi.note ? ` (${r.bingApi.note})` : ""}`);
      r.bingApi.attempts.forEach(a => lines.push(`    ${a}`));
    }
    if (r.newUrls.length > 0) {
      lines.push(`  New URLs (${r.newUrls.length}):`);
      r.newUrls.forEach(u => lines.push(`    ${u}`));
    }
    if (r.removedUrls.length > 0) {
      lines.push(`  Removed URLs (${r.removedUrls.length}):`);
      r.removedUrls.forEach(u => lines.push(`    ${u}`));
    }
    lines.push(``);
  }

  const res = await fetchWithTimeout("https://api.resend.com/emails", {
    method: "POST",
    headers: {
      "Authorization": `Bearer ${env.RESEND_API_KEY}`,
      "Content-Type": "application/json"
    },
    body: JSON.stringify({
      from: fromEmail,
      to: env.TO_EMAIL,
      subject,
      text: lines.join("\n")
    })
  });
  if (!res.ok) throw new Error(`Resend API ${res.status}: ${await res.text()}`);
}

main();

{
  "host": "yoursite.com",
  "totalUrls": 196,
  "newUrls": [],
  "removedUrls": [],
  "retriedCount": 0,
  "submission": null
}