Pipe library / sitemap-new-pages

sitemap-new-pages

New URLs in a public XML sitemap since the last successful run.

Utilities

blog.rust-lang.org

CLI

pipes run examples/sitemap-new-pages.pipe.yamlCopy CLI

MCP call

{
  "name": "pipe__sitemap-new-pages",
  "arguments": {}
}Copy MCP call

Pipe file

examples/sitemap-new-pages.pipe.yaml

pipes: 0
name: sitemap-new-pages
version: 0.1.0
description: New URLs in a public XML sitemap since the last successful run.
permissions:
  network: [blog.rust-lang.org]
inputs:
  type: object
  properties:
    url:
      type: string
      format: uri
      default: https://blog.rust-lang.org/sitemap.xml
    n:
      type: integer
      minimum: 1
      maximum: 30
      default: 15
  required: [url]
outputs:
  type: object
  properties:
    text: { type: string }
    count: { type: integer }
  required: [text, count]
output:
  text: "${steps.format.text}"
  count: "${steps.format.count}"
steps:
  - id: fetch
    block: fetch_http
    with:
      url: "${inputs.url}"
      max_bytes: 2000000
    retry: { max: 2, backoff_ms: 500 }
    timeout_ms: 20000
  - id: locs
    block: regex_extract
    in: "${steps.fetch}"
    with:
      field: body
      pattern: "<loc>([^<]+)</loc>"
      all: true
      into: { url: 1 }
  - id: unique
    block: dedupe
    in: "${steps.locs}"
    with: { key: [url], keep: first }
  - id: capped
    block: limit
    in: "${steps.unique}"
    with: { n: "${inputs.n}" }
  - id: new
    block: diff_since_last
    in: "${steps.capped}"
    with:
      key: [url]
      compare: [url]
      emit: [added]
      first_run: all
  - id: format
    block: format
    in: "${steps.new}"
    with:
      header: "Sitemap:\n"
      item_template: "- ${item.url}"
      separator: "\n"
  - id: print
    block: emit_stdout
    in: "${steps.format.text}"
    with: { format: text }
Copy