{
  "$schema": "https://datapackage.org/profiles/2.0/datapackage.json",
  "name": "sitemaps-robots-txt-polish-web-2026-09-24",
  "title": "Sitemap lines in the robots.txt of Polish sites",
  "description": "Every .pl site in a pinned Tranco list whose homepage answered with HTML and whose robots.txt is a text file: its Sitemap lines, what each points at (own origin, another host, another protocol, the www twin, a relative path, a development host, a template), the software behind the site, and, for the top 1,000, whether the named sitemap or /sitemap.xml answered.",
  "version": "2026-09-24",
  "created": "2026-10-12T04:26:57.202Z",
  "homepage": "https://www.outofplace.space/blog/sitemaps-robots-txt-polish-web",
  "keywords": [
    "sitemap",
    "robots.txt",
    "RFC 9309",
    "sitemaps.org",
    "XML sitemap",
    "Poland"
  ],
  "licenses": [
    {
      "name": "CC-BY-4.0",
      "path": "https://creativecommons.org/licenses/by/4.0/",
      "title": "Creative Commons Attribution 4.0"
    }
  ],
  "contributors": [
    {
      "title": "Outofplace Poland sp. z o.o.",
      "roles": [
        "creator"
      ],
      "path": "https://www.outofplace.space"
    }
  ],
  "sources": [
    {
      "title": "Tranco list Y8YYG (https://doi.org/10.14722/ndss.2019.23386)",
      "path": "https://tranco-list.eu/list/Y8YYG/1000000"
    }
  ],
  "resources": [
    {
      "name": "sites",
      "path": "sites.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 1033006,
      "hash": "sha256:6c649c068fb454ea84d8b0264acc5830710bee4f755aa803fdb3a7cdad03a8da",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "rank_tranco",
            "type": "integer",
            "description": "Rank in the pinned Tranco list (the highest-ranked domain per origin)."
          },
          {
            "name": "rank_sample",
            "type": "integer",
            "description": "Position among the census's .pl origins with an HTML homepage, by Tranco rank."
          },
          {
            "name": "domain",
            "type": "string",
            "description": "Registrable .pl domain."
          },
          {
            "name": "origin",
            "type": "string",
            "description": "Scheme and host of the homepage after redirects."
          },
          {
            "name": "robots_state",
            "type": "string",
            "description": "robots.txt of that origin by the AI-bots study's rules: ok (a text file), html, 4xx, unreachable, waf."
          },
          {
            "name": "robots_bytes",
            "type": "integer",
            "description": "Size of the robots.txt body, when ok."
          },
          {
            "name": "is_shop",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "A shop by the e-commerce study's classifier (scripts/research/cwv-ecommerce/fingerprints.mjs)."
          },
          {
            "name": "platform",
            "type": "string",
            "description": "Shop platform by the same classifier: none when no platform was recognised."
          },
          {
            "name": "wordpress",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "WordPress, by its generator tag or its asset folders."
          },
          {
            "name": "seo_plugin",
            "type": "string",
            "description": "SEO plugin by its marker in the HTML: yoast, rank_math, aioseo, seopress, or empty."
          },
          {
            "name": "sitemap_lines",
            "type": "integer",
            "description": "Sitemap lines with a value, as the RFC 9309 parser reads them; empty when robots.txt isn't ok."
          },
          {
            "name": "sitemap_unique",
            "type": "integer",
            "description": "Distinct Sitemap values."
          },
          {
            "name": "n_same",
            "type": "integer",
            "description": "Sitemap lines pointing at the homepage's own origin."
          },
          {
            "name": "n_subdomain",
            "type": "integer",
            "description": "Sitemap lines pointing at another host of the same site."
          },
          {
            "name": "n_other_site",
            "type": "integer",
            "description": "Sitemap lines pointing at another site."
          },
          {
            "name": "n_scheme",
            "type": "integer",
            "description": "Sitemap lines pointing at the homepage's host with another protocol or port."
          },
          {
            "name": "n_host_twin",
            "type": "integer",
            "description": "Sitemap lines pointing at the www or bare twin of the homepage's host."
          },
          {
            "name": "n_dev_host",
            "type": "integer",
            "description": "Sitemap lines pointing at localhost, an IP, a reserved or development host."
          },
          {
            "name": "n_relative",
            "type": "integer",
            "description": "Sitemap lines pointing at a path with no host."
          },
          {
            "name": "n_no_scheme",
            "type": "integer",
            "description": "Sitemap lines pointing at a host and path with no http(s)://."
          },
          {
            "name": "n_template",
            "type": "integer",
            "description": "Sitemap lines pointing at an unfilled template placeholder."
          },
          {
            "name": "n_bad_url",
            "type": "integer",
            "description": "Sitemap lines pointing at any other value that isn't an http(s) URL."
          },
          {
            "name": "verdict",
            "type": "string",
            "description": "The worst pointer: none, broken, mismatch, elsewhere (every pointer on another host) or ok."
          },
          {
            "name": "files_own",
            "type": "list",
            "delimiter": "|",
            "itemType": "string",
            "description": "File names the pointers to the site's own host name (same origin, other protocol or twin), lowercased."
          },
          {
            "name": "probed_at",
            "type": "datetime",
            "description": "UTC time of the census fetch."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "verdicts",
      "path": "verdicts.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 208,
      "hash": "sha256:e86974d42fbaae0c11cc20f7a1d24900cdb7f35eb60d9cc4e4deeb3a25c0d5ff",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "verdict",
            "type": "string",
            "description": "none, ok, elsewhere, mismatch or broken (scripts/research/sitemaps/pointers.mjs)."
          },
          {
            "name": "sites",
            "type": "integer",
            "description": "Sites with a robots.txt text file and this verdict."
          },
          {
            "name": "share",
            "type": "number",
            "description": "sites / sites with a robots.txt text file."
          },
          {
            "name": "share_ci_low",
            "type": "number",
            "description": "Wilson 95% lower bound."
          },
          {
            "name": "share_ci_high",
            "type": "number",
            "description": "Wilson 95% upper bound."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "pointers",
      "path": "pointers.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 534,
      "hash": "sha256:132b71963399da20e8fb451c2aa418697c901aaa3fde58a1c420866f6359ec06",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "class",
            "type": "string",
            "description": "Pointer class (scripts/research/sitemaps/pointers.mjs)."
          },
          {
            "name": "group",
            "type": "string",
            "description": "ok, elsewhere (allowed), mismatch (works only through a redirect) or broken (unusable as written)."
          },
          {
            "name": "pointers",
            "type": "integer",
            "description": "Sitemap lines of this class."
          },
          {
            "name": "sites",
            "type": "integer",
            "description": "Sites naming a sitemap with at least one line of this class."
          },
          {
            "name": "share",
            "type": "number",
            "description": "sites / sites naming a sitemap."
          },
          {
            "name": "share_ci_low",
            "type": "number",
            "description": "Wilson 95% lower bound."
          },
          {
            "name": "share_ci_high",
            "type": "number",
            "description": "Wilson 95% upper bound."
          },
          {
            "name": "sites_only",
            "type": "integer",
            "description": "Sites whose every Sitemap line is of this class."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "software",
      "path": "software.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 1203,
      "hash": "sha256:468742c40271ea8b66d55b495f916627ba3766a0d70c1771d302aaf525c3e594",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "group",
            "type": "string",
            "description": "Shop platform, else SEO plugin, else other WordPress, else unrecognised; other = groups under 30 sites."
          },
          {
            "name": "sites",
            "type": "integer",
            "description": "Sites with a robots.txt text file in the group."
          },
          {
            "name": "names_sitemap",
            "type": "integer",
            "description": "Of those, sites with at least one Sitemap line."
          },
          {
            "name": "share",
            "type": "number",
            "description": "sites / sites in the group."
          },
          {
            "name": "share_ci_low",
            "type": "number",
            "description": "Wilson 95% lower bound."
          },
          {
            "name": "share_ci_high",
            "type": "number",
            "description": "Wilson 95% upper bound."
          },
          {
            "name": "wrong",
            "type": "integer",
            "description": "Sites naming a sitemap with a broken or mismatched pointer."
          },
          {
            "name": "wrong_share_of_namers",
            "type": "number",
            "description": "wrong / names_sitemap."
          },
          {
            "name": "wrong_ci_low",
            "type": "number",
            "description": "Wilson 95% lower bound."
          },
          {
            "name": "wrong_ci_high",
            "type": "number",
            "description": "Wilson 95% upper bound."
          },
          {
            "name": "folded",
            "type": "list",
            "delimiter": "|",
            "itemType": "string",
            "description": "Groups folded into other."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "lines",
      "path": "lines.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 189,
      "hash": "sha256:96bf103d97c352a2b6a84fc624bbe8d70b261756586423960b50adee8f0c2af5",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "bucket",
            "type": "string",
            "description": "Number of Sitemap lines in the file."
          },
          {
            "name": "sites",
            "type": "integer",
            "description": "Sites naming a sitemap with that many lines."
          },
          {
            "name": "share",
            "type": "number",
            "description": "sites / sites naming a sitemap."
          },
          {
            "name": "share_ci_low",
            "type": "number",
            "description": "Wilson 95% lower bound."
          },
          {
            "name": "share_ci_high",
            "type": "number",
            "description": "Wilson 95% upper bound."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "files",
      "path": "files.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 537,
      "hash": "sha256:c6a59a488babf2eee27faf99142a2dc8dfa0ab797e4ea6d643b0be7b8d67567c",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "file",
            "type": "string",
            "description": "File name a pointer to the site's own host names, lowercased."
          },
          {
            "name": "sites",
            "type": "integer",
            "description": "Sites naming it."
          },
          {
            "name": "share",
            "type": "number",
            "description": "sites / sites with a pointer to their own host."
          },
          {
            "name": "share_ci_low",
            "type": "number",
            "description": "Wilson 95% lower bound."
          },
          {
            "name": "share_ci_high",
            "type": "number",
            "description": "Wilson 95% upper bound."
          },
          {
            "name": "top_group",
            "type": "string",
            "description": "The software group (as in software.csv, before folding) most of these sites run."
          },
          {
            "name": "top_group_share",
            "type": "number",
            "description": "Share of the sites naming this file that run top_group."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "fetch-top1000",
      "path": "fetch_top1000.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 212,
      "hash": "sha256:082a406f16be95b440daa021b314f14e3d54f3bea9a810a6cb062296b96cfca6",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "group",
            "type": "string",
            "description": "named: robots.txt names a sitemap; not_named: it names none."
          },
          {
            "name": "url_fetched",
            "type": "string",
            "description": "What the AI-bots crawl fetched."
          },
          {
            "name": "sites",
            "type": "integer",
            "description": "Top-1,000 .pl domains with a robots.txt text file in the group whose sitemap request our crawler was allowed to make."
          },
          {
            "name": "xml_sitemap",
            "type": "integer",
            "description": "The answer was an XML sitemap or sitemap index (root element urlset or sitemapindex)."
          },
          {
            "name": "share",
            "type": "number",
            "description": "sites / sites."
          },
          {
            "name": "share_ci_low",
            "type": "number",
            "description": "Wilson 95% lower bound."
          },
          {
            "name": "share_ci_high",
            "type": "number",
            "description": "Wilson 95% upper bound."
          },
          {
            "name": "not_found",
            "type": "integer",
            "description": "The request's status was not 200."
          },
          {
            "name": "html",
            "type": "integer",
            "description": "200 with an HTML page."
          },
          {
            "name": "other",
            "type": "integer",
            "description": "200 with anything else (text sitemaps included)."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "summary",
      "path": "summary.json",
      "description": "Headline numbers and chart-ready rows derived from the CSVs.",
      "format": "json",
      "mediatype": "application/json",
      "encoding": "utf-8",
      "bytes": 31328,
      "hash": "sha256:262da5b6eedfe00fa10d785125813172ff656a353171f36d2e77e2db2080dcb0"
    }
  ]
}