Skip to content

Commit 126d27d

Browse files
author
stlc-bot
committed
feat(sitemap): add subdomain discovery
Stainless-Generated-From: a90de429784938edd4d439daafa6d613b3448932
1 parent 9cb38d9 commit 126d27d

2 files changed

Lines changed: 8 additions & 1 deletion

File tree

pkg/cmd/web.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -948,7 +948,7 @@ var webWebScrapeMd = requestflag.WithInnerFlags(cli.Command{
948948

949949
var webWebScrapeSitemap = cli.Command{
950950
Name: "web-scrape-sitemap",
951-
Usage: "Crawl an entire website's sitemap and return all discovered page URLs. Pass\n`search` to have the crawled sitemap filtered down to the pages about a phrase\n(for example `pricing and plans` or `api authentication docs`), most relevant\nfirst — a searched crawl scans the whole sitemap and costs 2 credits instead\nof 1.",
951+
Usage: "Crawl an entire website's sitemap and return all discovered page URLs. Set\n`includeSubdomains=true` to also discover public pages and sitemaps on child\nhosts such as `docs.example.com` or `brand.example.com`. Pass `search` to have\nthe discovered URLs filtered down to the pages about a phrase (for example\n`pricing and plans` or `api authentication docs`), most relevant first — a\nsearched crawl scans the whole sitemap and costs 2 credits instead of 1.",
952952
Suggest: true,
953953
Flags: []cli.Flag{
954954
&requestflag.Flag[string]{
@@ -962,6 +962,12 @@ var webWebScrapeSitemap = cli.Command{
962962
Usage: "Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache.",
963963
QueryPath: "headers",
964964
},
965+
&requestflag.Flag[bool]{
966+
Name: "include-subdomains",
967+
Usage: "When true, discover and include public pages and sitemaps on subdomains of the requested domain. Defaults to false.",
968+
Default: false,
969+
QueryPath: "includeSubdomains",
970+
},
965971
&requestflag.Flag[int64]{
966972
Name: "max-links",
967973
Usage: "Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Minimum is 1, maximum is 100,000.",

pkg/cmd/web_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -591,6 +591,7 @@ func TestWebWebScrapeSitemap(t *testing.T) {
591591
"web", "web-scrape-sitemap",
592592
"--domain", "xxx",
593593
"--headers", "{foo: J!}",
594+
"--include-subdomains=true",
594595
"--max-links", "1",
595596
"--search", "help center and troubleshooting articles",
596597
"--sitemap-url", "https://example.com",

0 commit comments

Comments
 (0)