{
  "$schema": "https://datapackage.org/profiles/2.0/datapackage.json",
  "name": "snippet-controls-polish-web-2026-09-24",
  "title": "Snippet and AI controls on Polish homepages",
  "description": "Every .pl homepage in a pinned Tranco list that answered with HTML: its robots meta tags and X-Robots-Tag header, what Google and Bing would apply (snippet length, image and video previews, noarchive, nocache), the unofficial noai and noimageai, data-nosnippet, and what its robots.txt says to AI crawlers.",
  "version": "2026-09-24",
  "created": "2026-10-09T17:29:24.623Z",
  "homepage": "https://www.outofplace.space/blog/snippet-controls-polish-web",
  "keywords": [
    "nosnippet",
    "max-snippet",
    "max-image-preview",
    "noai",
    "X-Robots-Tag",
    "AI Overviews",
    "Poland"
  ],
  "licenses": [
    {
      "name": "CC-BY-4.0",
      "path": "https://creativecommons.org/licenses/by/4.0/",
      "title": "Creative Commons Attribution 4.0"
    }
  ],
  "contributors": [
    {
      "title": "Outofplace Poland sp. z o.o.",
      "roles": [
        "creator"
      ],
      "path": "https://www.outofplace.space"
    }
  ],
  "sources": [
    {
      "title": "Tranco list Y8YYG (https://doi.org/10.14722/ndss.2019.23386)",
      "path": "https://tranco-list.eu/list/Y8YYG/1000000"
    }
  ],
  "resources": [
    {
      "name": "sites",
      "path": "sites.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 1504121,
      "hash": "sha256:8cc84248702d0e10669f223ace5e7ca3064ed1984d21bf794e3ad26e4b1d006e",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "rank_tranco",
            "type": "integer",
            "description": "Rank in the pinned Tranco list (the highest-ranked domain per origin)."
          },
          {
            "name": "domain",
            "type": "string",
            "description": "Registrable .pl domain."
          },
          {
            "name": "origin",
            "type": "string",
            "description": "Scheme and host of the homepage after redirects."
          },
          {
            "name": "html_truncated",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "The homepage was longer than the census's 1 MiB cap; markup after it is unseen."
          },
          {
            "name": "is_shop",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "A shop by the e-commerce study's classifier (scripts/research/cwv-ecommerce/fingerprints.mjs)."
          },
          {
            "name": "platform",
            "type": "string",
            "description": "Shop platform by the same classifier, empty when none was recognised."
          },
          {
            "name": "wordpress",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "WordPress, by its generator tag or its asset folders."
          },
          {
            "name": "seo_plugin",
            "type": "string",
            "description": "SEO plugin by the marker it leaves in the HTML: yoast, rank_math, aioseo, seopress, or empty."
          },
          {
            "name": "meta_names",
            "type": "list",
            "delimiter": "|",
            "itemType": "string",
            "description": "Names of the robots meta tags on the page (robots, googlebot, bingbot…)."
          },
          {
            "name": "meta_robots",
            "type": "list",
            "delimiter": "|",
            "itemType": "string",
            "description": "Each robots meta tag as name=content, whitespace collapsed, cut at 300 characters."
          },
          {
            "name": "x_robots_tag",
            "type": "string",
            "description": "The X-Robots-Tag header of the final response, cut at 300 characters."
          },
          {
            "name": "rules",
            "type": "list",
            "delimiter": "|",
            "itemType": "string",
            "description": "Distinct rules stated anywhere, with value classes (max-snippet:-1, max-snippet:n, max-image-preview:large)."
          },
          {
            "name": "google_indexed",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "No noindex or none rule for Google (robots and googlebot tags, header for all or googlebot)."
          },
          {
            "name": "google_snippet",
            "type": "string",
            "description": "Google's snippet, the stricter rule winning: default, unlimited (-1), limited (a length) or none."
          },
          {
            "name": "google_snippet_chars",
            "type": "integer",
            "description": "The smallest positive max-snippet, when limited."
          },
          {
            "name": "google_image",
            "type": "string",
            "description": "Google's image preview: default, large, standard or none."
          },
          {
            "name": "google_video",
            "type": "string",
            "description": "Google's video preview: default, unlimited, limited or none."
          },
          {
            "name": "bing_ai",
            "type": "string",
            "description": "Bing's AI-answer control: noarchive, nocache (also when both) or none."
          },
          {
            "name": "noai",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "The unofficial noai rule appears in any robots meta tag or the header."
          },
          {
            "name": "noimageai",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "The unofficial noimageai rule appears in any robots meta tag or the header."
          },
          {
            "name": "noai_meta",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "noai or noimageai in a meta tag."
          },
          {
            "name": "noai_header",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "noai or noimageai in the X-Robots-Tag header."
          },
          {
            "name": "dn_elements",
            "type": "integer",
            "description": "Elements carrying data-nosnippet in the markup (scripts, styles, templates and comments removed)."
          },
          {
            "name": "dn_consent",
            "type": "integer",
            "description": "Of those, on a cookie or consent banner or inside one."
          },
          {
            "name": "dn_content",
            "type": "integer",
            "description": "Of those, anywhere else."
          },
          {
            "name": "dn_unsupported",
            "type": "integer",
            "description": "Of those, on an element other than span, div or section, which Google doesn't read."
          },
          {
            "name": "dn_banners",
            "type": "list",
            "delimiter": "|",
            "itemType": "string",
            "description": "Consent tools whose banner carries it: complianz, cookie_law_info, cookieyes, cookiefirst, wpconsent, other."
          },
          {
            "name": "robots_state",
            "type": "string",
            "description": "robots.txt state by the AI-bots study's rules: ok, html, 4xx, unreachable, waf; empty if not fetched."
          },
          {
            "name": "blocks_gptbot",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "robots.txt blocks GPTBot from the homepage."
          },
          {
            "name": "blocks_google_extended",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "robots.txt blocks Google-Extended from the homepage."
          },
          {
            "name": "blocks_ai_training",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "robots.txt blocks at least one current tier-1 AI training crawler (lib/robots/bots.ts)."
          },
          {
            "name": "blocks_googlebot",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "robots.txt blocks Googlebot from the homepage."
          },
          {
            "name": "probed_at",
            "type": "datetime",
            "description": "UTC time of the census fetch."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "rules",
      "path": "rules.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 1311,
      "hash": "sha256:c3d387f15ee2c81a7f6f3c1f6724f66eae1e2eca342fcf4f29372b2470d6e836",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "rule",
            "type": "string",
            "description": "Rule, with its value class."
          },
          {
            "name": "limits",
            "type": "boolean",
            "trueValues": [
              "1"
            ],
            "falseValues": [
              "0"
            ],
            "description": "The rule keeps text or images out of results or AI answers."
          },
          {
            "name": "sites",
            "type": "integer",
            "description": "Homepages stating it."
          },
          {
            "name": "share",
            "type": "number",
            "description": "sites / all origins."
          },
          {
            "name": "share_ci_low",
            "type": "number",
            "description": "Wilson 95% lower bound."
          },
          {
            "name": "share_ci_high",
            "type": "number",
            "description": "Wilson 95% upper bound."
          },
          {
            "name": "meaning",
            "type": "string",
            "description": "What the rule asks, by Google's and Bing's documentation."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "software",
      "path": "software.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 326,
      "hash": "sha256:2a35ecbfcd43bb0e5bc69684e8fcc025e8f561162025c5102c2a43024984c868",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "group",
            "type": "string",
            "description": "SEO plugin, other WordPress, not WordPress, or other (under 30 homepages)."
          },
          {
            "name": "sites",
            "type": "integer",
            "description": "Homepages in the group."
          },
          {
            "name": "image_large",
            "type": "integer",
            "description": "Google image preview large."
          },
          {
            "name": "image_default",
            "type": "integer",
            "description": "No image preview rule."
          },
          {
            "name": "image_limited",
            "type": "integer",
            "description": "Image preview standard or none."
          },
          {
            "name": "snippet_unlimited",
            "type": "integer",
            "description": "max-snippet:-1 and nothing stricter."
          },
          {
            "name": "snippet_default",
            "type": "integer",
            "description": "No snippet rule."
          },
          {
            "name": "snippet_limited",
            "type": "integer",
            "description": "nosnippet or a max-snippet length."
          },
          {
            "name": "widens",
            "type": "integer",
            "description": "Asks for a large image, unlimited snippet or unlimited video preview."
          },
          {
            "name": "limits_preview",
            "type": "integer",
            "description": "Limits the snippet, the image or the video preview."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "robots-txt",
      "path": "robots_txt.csv",
      "type": "table",
      "format": "csv",
      "mediatype": "text/csv",
      "encoding": "utf-8",
      "bytes": 418,
      "hash": "sha256:e53701479eea2346a33fab66f78061d3508dba979e89cfece92e84707e5d8986",
      "dialect": {
        "header": true,
        "delimiter": ",",
        "lineTerminator": "\r\n",
        "quoteChar": "\"",
        "doubleQuote": true
      },
      "schema": {
        "fields": [
          {
            "name": "group",
            "type": "string",
            "description": "Homepages by what their robots.txt says to AI crawlers."
          },
          {
            "name": "sites",
            "type": "integer",
            "description": "Homepages in the group."
          },
          {
            "name": "ai_control",
            "type": "integer",
            "description": "With any on-page AI control: a snippet limit, noarchive or nocache, noai or noimageai, data-nosnippet on content, or no image preview."
          },
          {
            "name": "ai_control_share",
            "type": "number",
            "description": "ai_control / sites."
          },
          {
            "name": "ai_control_ci_low",
            "type": "number",
            "description": "Wilson 95% lower bound."
          },
          {
            "name": "ai_control_ci_high",
            "type": "number",
            "description": "Wilson 95% upper bound."
          },
          {
            "name": "limits_text",
            "type": "integer",
            "description": "nosnippet or a max-snippet length."
          },
          {
            "name": "bing_ai",
            "type": "integer",
            "description": "noarchive or nocache."
          },
          {
            "name": "noai",
            "type": "integer",
            "description": "noai or noimageai."
          },
          {
            "name": "dn_content",
            "type": "integer",
            "description": "data-nosnippet on content."
          },
          {
            "name": "noindex",
            "type": "integer",
            "description": "noindex or none for Google."
          }
        ],
        "missingValues": [
          ""
        ]
      }
    },
    {
      "name": "summary",
      "path": "summary.json",
      "description": "Headline numbers and chart-ready rows derived from the CSVs.",
      "format": "json",
      "mediatype": "application/json",
      "encoding": "utf-8",
      "bytes": 25678,
      "hash": "sha256:7237adc6447524148345b454cb42cafa3878979caee06f4becbc4ccdb18ae573"
    }
  ]
}