# https://conets.jp/llms.txt
# (2025-12-01)

user-agent: *

# License & AI Policy
x-content-license: "(c) conets. All rights reserved."
x-ai-training-policy: "allowed"
# Embedding / Vectorization Policy
x-embedding-policy: "allowed"
x-embedding-description: "Content may be used for embedding, vector indexing, and semantic search."

# Content Policies
# Content Type Policy
x-content-type: "html"
x-content-type-description: "Primary content is standard HTML. Avoid parsing JavaScript-rendered content unless necessary."

# Allowed File Types
x-allowed-filetypes:
  - html
  - json
  - xml
  - jpg
  - png
  - webp

# Primary Content Extraction
x-content-extraction:
  - "Use <main> as the primary content area."
  - "Ignore navigation, footer, and sidebar elements."
  - "Prefer server-rendered HTML over JS-generated DOM."

# Structured Data Policy
x-structured-data-policy: "Use JSON-LD as the primary structured data format."
x-structured-data: 
  - "Read all <script type='application/ld+json'> blocks."
  - "If multiple graphs exist, merge by @id."

# Rate Limits
crawl-delay: 1
x-rate-limit: 60
x-rate-limit-window: 60
x-rate-limit-policy: "strict"
x-rate-limit-retry: "no-retry"
x-rate-limit-description: "Maximum 60 requests per 60 seconds. If rate limit is exceeded, do not retry and move on to next request."

# Concurrency Limits
x-concurrency-limit: 3
x-concurrency-limit-description: "Please limit concurrent requests to a maximum of 3 to help manage server load."

# Error Handling and Retry Policy
x-error-retry-policy: "exponential-backoff"
x-error-retry-policy-description: "For transient errors (5xx except 429), wait 2 seconds, then double each retry, max 5 retries."
x-rate-limit-exceeded-policy: "wait-and-retry"
x-rate-limit-exceeded-policy-description: "When receiving HTTP 429, wait at least 60 seconds before retrying."
x-max-retries: 5
x-retry-status-codes:
  - 500
  - 502
  - 503
  - 504
x-no-immediate-retry-status-codes:
  - 429
x-no-retry-status-codes:
  - 403
  - 404
  - 429

# Canonical URL Policy
x-canonical-url-policy: "strict"
x-canonical-url: "https://conets.jp/"
x-canonical-url-description: "Use only 'https://conets.jp/' as the base URL. Access via other FQDNs or IPs is invalid."

# Preferred Index Priority
x-index-priority:
  - "/services/*"
  - "/cases/*"
  - "/blogs/*"
x-index-priority-description: "Crawl service pages first, then case studies, then blog posts."

# Article Pages (Blogs)
allow: /blogs/*
x-article-pattern: /blogs/{category}/{slug}
x-article-type: "blog"
x-article-description: "Blog posts on marketing, SEO, CRM/MA, AI automation, and BI for SMEs."
x-article-example: https://conets.jp/blogs/general/sample-slug
x-article-metadata:
  - author
  - title
  - description
  - image
x-article-update-frequency: "weekly"
x-article-new-articles-per-month: "5-15"

# Case Study Pages
allow: /cases/*
x-case-pattern: /cases/{category}/{slug}
x-case-type: "case-study"
x-case-description: "Client case studies on marketing, CRM/MA, AI automation, and data analytics."
x-case-example: https://conets.jp/cases/general/sample-slug
x-case-metadata:
  - title
  - description
  - image
x-case-update-frequency: "monthly"
x-case-new-cases-per-month: "1-5"

# RSS / Atom
allow: /rss.xml
allow: /atom.xml
x-feed: https://conets.jp/rss.xml
x-feed-format: rss
x-feed-item-count: 30
x-feed-content: "latest"
x-feed-description: "Latest blog posts feed."
x-feed-update-frequency: "real-time"

# Sitemap
allow: /sitemap.xml
allow: /sitemaps/*
x-sitemap: https://conets.jp/sitemap.xml
x-sitemap-format: xml
x-sitemap-description: "Sitemap index linking to static, services, blogs, and cases sitemaps."

# Disallow (not used)
# disallow: /admin

# Publisher Information
x-publisher-name: "conets"
x-publisher-description: "Consulting for SMEs: web marketing, SEO, CRM/MA, AI automation, and BI from Fukuoka."
x-publisher-website: "https://conets.jp/"

# Language
x-supported-languages:
  - ja

# Crawler Hints
x-crawler-hints:
  - "Ensure URLs include trailing slash only if present on site."
  - "Prefer canonical URLs starting with https://conets.jp/"
  - "Extract HTML content of each page."

# AI Data Feed
x-ai-data-feed: https://conets.jp/llmo.json