Design a Web Crawler

3 collapsed lines
1
// Headers
2
Authorization: Bearer {internal_token}
3
Content-Type: application/json
4

5
// Request body
6
{
7
  "urls": [
8
    {
9
      "url": "https://example.com/page1",
10
      "priority": 0.85,
11
      "source_url": "https://example.com/",
12
      "depth": 1,
13
      "discovered_at": "2024-01-15T10:30:00Z"
14
    },
15
    {
16
      "url": "https://example.com/page2",
17
      "priority": 0.72,
18
      "source_url": "https://example.com/",
19
      "depth": 1,
5 collapsed lines
20
      "discovered_at": "2024-01-15T10:30:00Z"
21
    }
22
  ],
23
  "crawl_id": "crawl-2024-01-15"
24
}

1
{
2
  "accepted": 2,
3
  "rejected": 0,
4
  "duplicates": 0
5
}

3 collapsed lines
1
{
2
  "urls": [
3
    {
4
      "url": "https://example.com/page1",
5
      "priority": 0.85,
6
      "host": "example.com",
7
      "ip": "93.184.216.34",
8
      "robots_rules": {
9
        "allowed": true,
10
        "crawl_delay": null
11
      }
12
    },
13
    {
14
      "url": "https://other.com/page2",
15
      "priority": 0.72,
16
      "host": "other.com",
17
      "ip": "203.0.113.50",
18
      "robots_rules": {
19
        "allowed": true,
20
        "crawl_delay": 2
21
      }
22
    }
23
  ],
24
  "lease_id": "lease-abc123",
2 collapsed lines
25
  "lease_expires_at": "2024-01-15T10:35:00Z"
26
}

3 collapsed lines
1
// Headers
2
Authorization: Bearer {internal_token}
3
Content-Type: application/json
4

5
// Request body
6
{
7
  "lease_id": "lease-abc123",
8
  "results": [
9
    {
10
      "url": "https://example.com/page1",
11
      "status": "success",
12
      "http_status": 200,
13
      "content_type": "text/html",
14
      "content_length": 45230,
15
      "content_hash": "a1b2c3d4e5f6...",
16
      "simhash": "0x1234567890abcdef",
17
      "fetch_time_ms": 234,
18
      "links_found": 47,
19
      "is_duplicate": false
20
    },
21
    {
22
      "url": "https://other.com/page2",
23
      "status": "error",
24
      "http_status": 503,
25
      "error": "Service Unavailable",
26
      "retry_after": 300
27
    }
28
  ]
29
}

1
{
2
  "processed": 2,
3
  "retries_scheduled": 1
4
}

5 collapsed lines
1
{
2
  "crawl_id": "crawl-2024-01-15",
3
  "started_at": "2024-01-15T00:00:00Z",
4
  "runtime_hours": 10.5,
5
  "urls": {
6
    "discovered": 15234567,
7
    "queued": 8234123,
8
    "fetched": 7000444,
9
    "successful": 6543210,
10
    "failed": 457234,
11
    "duplicate_content": 234567,
12
    "blocked_robots": 123456
13
  },
14
  "throughput": {
15
    "current_pages_per_second": 185.3,
16
    "avg_pages_per_second": 173.2,
17
    "peak_pages_per_second": 312.5
18
  },
19
  "hosts": {
20
    "total_discovered": 1234567,
21
    "active_in_queue": 456789,
22
    "blocked_by_robots": 12345
23
  },
24
  "storage": {
25
    "content_stored_gb": 2345.6,
26
    "urls_stored_millions": 15.2
27
  },
28
  "workers": {
29
    "active": 12,
30
    "idle": 3,
31
    "failed": 0
32
  }
33
}

5 collapsed lines
1
-- URL state tracking
2
CREATE TABLE urls (
3
    id BIGSERIAL PRIMARY KEY,
4
    url_hash BYTEA NOT NULL,  -- SHA-256 of normalized URL (32 bytes)
5
    url TEXT NOT NULL,
6
    normalized_url TEXT NOT NULL,
7

8
    -- Discovery metadata
9
    host VARCHAR(255) NOT NULL,
10
    host_hash INT NOT NULL,  -- For partition routing
11
    depth SMALLINT DEFAULT 0,
12
    source_url_id BIGINT REFERENCES urls(id),
13
    discovered_at TIMESTAMPTZ DEFAULT NOW(),
14

15
    -- Priority and scheduling
16
    priority REAL DEFAULT 0.5,
17
    last_fetched_at TIMESTAMPTZ,
18
    next_fetch_at TIMESTAMPTZ,
19
    fetch_count INT DEFAULT 0,
20

21
    -- Fetch results
22
    last_status_code SMALLINT,
23
    last_content_hash BYTEA,  -- For change detection
24
    last_simhash BIGINT,      -- Content fingerprint
25
    content_length INT,
26
    content_type VARCHAR(100),
27

28
    -- State
29
    state VARCHAR(20) DEFAULT 'pending',
30
      -- pending, queued, fetching, completed, failed, blocked
31

32
    CONSTRAINT url_hash_unique UNIQUE (url_hash)
33
);
34

35
-- Indexes for common access patterns
36
CREATE INDEX idx_urls_host ON urls(host_hash, host);
37
CREATE INDEX idx_urls_state ON urls(state, next_fetch_at)
38
    WHERE state IN ('pending', 'queued');
39
CREATE INDEX idx_urls_refetch ON urls(next_fetch_at)
1 collapsed line
40
    WHERE state = 'completed' AND next_fetch_at IS NOT NULL;

3 collapsed lines
1
-- robots.txt cache
2
CREATE TABLE robots_cache (
3
    host VARCHAR(255) PRIMARY KEY,
4
    fetched_at TIMESTAMPTZ NOT NULL,
5
    expires_at TIMESTAMPTZ NOT NULL,
6
    status_code SMALLINT NOT NULL,
7
    content TEXT,  -- Raw robots.txt content
8
    parsed_rules JSONB,  -- Pre-parsed rules for fast lookup
9
    crawl_delay INT,  -- Crawl-delay directive (if present)
10
    sitemaps TEXT[],  -- Sitemap URLs discovered
11
    error_message TEXT
12
);
13

14
-- Example parsed_rules structure:
15
-- {
16
--   "user_agents": {
17
--     "*": {
18
--       "allow": ["/public/", "/api/"],
19
--       "disallow": ["/admin/", "/private/"]
20
--     },
21
--     "googlebot": {
22
--       "allow": ["/"],
23
--       "disallow": ["/no-google/"]
24
--     }
5 collapsed lines
25
--   }
26
-- }
27

28
CREATE INDEX idx_robots_expires ON robots_cache(expires_at)
29
    WHERE expires_at < NOW() + INTERVAL '1 hour';

5 collapsed lines
1
-- Page content archive (separate from URL metadata)
2
CREATE TABLE page_content (
3
    id BIGSERIAL PRIMARY KEY,
4
    url_id BIGINT NOT NULL REFERENCES urls(id),
5
    fetch_id BIGINT NOT NULL,  -- Links to fetch attempt
6

7
    -- Content
8
    raw_content BYTEA,  -- Compressed HTML/content
9
    compression VARCHAR(10) DEFAULT 'gzip',
10
    content_length INT NOT NULL,
11
    content_type VARCHAR(100),
12
    charset VARCHAR(50),
13

14
    -- Extracted data
15
    title TEXT,
16
    meta_description TEXT,
17
    canonical_url TEXT,
18
    language VARCHAR(10),
19

20
    -- Fingerprints
21
    content_hash BYTEA NOT NULL,  -- SHA-256 for exact match
22
    simhash BIGINT NOT NULL,      -- SimHash for near-duplicate
23

24
    -- Timestamps
25
    fetched_at TIMESTAMPTZ NOT NULL,
26
    http_date TIMESTAMPTZ,        -- From HTTP Date header
27
    last_modified TIMESTAMPTZ     -- From Last-Modified header
28
);
29

30
-- Content deduplication index
31
CREATE INDEX idx_content_simhash ON page_content(simhash);
32
CREATE INDEX idx_content_hash ON page_content(content_hash);
33

34
-- Partition by fetch date for efficient archival
2 collapsed lines
35
-- CREATE TABLE page_content_2024_01 PARTITION OF page_content
36
--     FOR VALUES FROM ('2024-01-01') TO ('2024-02-01');

15 collapsed lines
1
// Frontier configuration
2
interface FrontierConfig {
3
  numFrontQueues: number // F = number of priority levels (10-20)
4
  numBackQueues: number // B = 3 × crawler threads
5
  defaultPolitenessDelay: number // milliseconds between requests to same host
6
  maxQueueSize: number // per back queue
7
}
8

9
interface URLEntry {
10
  url: string
11
  normalizedUrl: string
12
  host: string
13
  priority: number // 0.0 to 1.0
14
  depth: number
15
  discoveredAt: Date
16
}
17

18
class URLFrontier {
19
  private frontQueues: PriorityQueue<URLEntry>[] // F queues
20
  private backQueues: Map<string, HostQueue> // Host → queue
21
  private hostToBackQueue: Map<string, number> // Host → back queue index
22
  private backQueueHeap: MinHeap<BackQueueEntry> // Sorted by next_allowed_time
23

24
  constructor(private config: FrontierConfig) {
25
    this.frontQueues = Array(config.numFrontQueues)
26
      .fill(null)
27
      .map(() => new PriorityQueue<URLEntry>((a, b) => b.priority - a.priority))
28

29
    this.backQueues = new Map()
30
    this.hostToBackQueue = new Map()
31
    this.backQueueHeap = new MinHeap((a, b) => a.nextAllowedTime - b.nextAllowedTime)
32
  }
33

34
  // Add URL to frontier
35
  async addURL(entry: URLEntry): Promise<void> {
36
    // Assign to front queue based on priority
37
    const frontQueueIndex = Math.floor(entry.priority * (this.config.numFrontQueues - 1))
38
    this.frontQueues[frontQueueIndex].enqueue(entry)
39

40
    // Ensure back queue exists for this host
41
    this.ensureBackQueue(entry.host)
42
  }
43

44
  // Get next URL to fetch (respects politeness)
45
  async getNext(): Promise<URLEntry | null> {
46
    // Find a back queue that's ready (next_allowed_time <= now)
47
    const now = Date.now()
48

49
    while (this.backQueueHeap.size() > 0) {
50
      const topBackQueue = this.backQueueHeap.peek()
51

52
      if (topBackQueue.nextAllowedTime > now) {
53
        // No queues ready yet, return null or wait
54
        return null
55
      }
56

57
      const backQueue = this.backQueues.get(topBackQueue.host)
58
      if (backQueue && backQueue.size() > 0) {
59
        // Pop URL from this back queue
60
        const url = backQueue.dequeue()
61

62
        // Update next_allowed_time
63
        this.backQueueHeap.extractMin()
64
        topBackQueue.nextAllowedTime = now + this.getPolitenessDelay(topBackQueue.host)
65
        this.backQueueHeap.insert(topBackQueue)
66

67
        // Refill back queue from front queues if needed
68
        this.refillBackQueue(topBackQueue.host)
69

21 collapsed lines
70
        return url
71
      }
72

73
      // Empty back queue, remove from heap
74
      this.backQueueHeap.extractMin()
75
    }
76

77
    return null
78
  }
79

80
  // Refill back queue from front queues (maintains priority)
81
  private refillBackQueue(host: string): void {
82
    const backQueue = this.backQueues.get(host)
83
    if (!backQueue || backQueue.size() >= 10) return // Keep 10 URLs buffered
84

85
    // Scan front queues for URLs matching this host
86
    for (const frontQueue of this.frontQueues) {
87
      const url = frontQueue.findAndRemove((e) => e.host === host)
88
      if (url) {
89
        backQueue.enqueue(url)
90
        if (backQueue.size() >= 10) break
91
      }
92
    }
93
  }
94

95
  private getPolitenessDelay(host: string): number {
96
    // Check robots.txt crawl-delay, fall back to default
97
    const robotsDelay = this.robotsCache.getCrawlDelay(host)
98
    return robotsDelay ?? this.config.defaultPolitenessDelay
99
  }
100
}

5 collapsed lines
1
interface PriorityFactors {
2
  pageRank: number // 0.0 to 1.0, from link analysis
3
  domainAuthority: number // 0.0 to 1.0, domain-level quality
4
  changeFrequency: number // 0.0 to 1.0, how often content changes
5
  depth: number // Distance from seed URL
6
  freshness: number // 0.0 to 1.0, based on last fetch age
7
}
8

9
function calculatePriority(factors: PriorityFactors): number {
10
  // Weighted combination of factors
11
  const weights = {
12
    pageRank: 0.3,
13
    domainAuthority: 0.25,
14
    changeFrequency: 0.2,
15
    depth: 0.15,
16
    freshness: 0.1,
17
  }
18

19
  // Depth penalty: deeper pages get lower priority
20
  const depthScore = Math.max(0, 1 - factors.depth * 0.1) // -10% per level
21

22
  const priority =
23
    weights.pageRank * factors.pageRank +
24
    weights.domainAuthority * factors.domainAuthority +
25
    weights.changeFrequency * factors.changeFrequency +
26
    weights.depth * depthScore +
27
    weights.freshness * factors.freshness
28

29
  return Math.max(0, Math.min(1, priority)) // Clamp to [0, 1]
30
}
31

32
// Example: High-value news site homepage
33
const priority = calculatePriority({
34
  pageRank: 0.9,
6 collapsed lines
35
  domainAuthority: 0.85,
36
  changeFrequency: 0.95, // Changes frequently
37
  depth: 0, // Seed URL
38
  freshness: 0.1, // Hasn't been fetched recently
39
})
40
// priority ≈ 0.77 → high priority front queue

10 collapsed lines
1
import { URL } from "url"
2

3
interface NormalizationOptions {
4
  removeFragment: boolean // Remove #hash
5
  removeDefaultPorts: boolean // Remove :80, :443
6
  removeTrailingSlash: boolean // Remove trailing /
7
  removeIndexFiles: boolean // Remove /index.html
8
  sortQueryParams: boolean // Alphabetize query params
9
  lowercaseHost: boolean // Lowercase hostname
10
  removeWWW: boolean // Remove www. prefix (controversial)
11
  decodeUnreserved: boolean // Decode safe percent-encoded chars
12
}
13

14
const DEFAULT_OPTIONS: NormalizationOptions = {
15
  removeFragment: true,
16
  removeDefaultPorts: true,
17
  removeTrailingSlash: true,
18
  removeIndexFiles: true,
19
  sortQueryParams: true,
20
  lowercaseHost: true,
21
  removeWWW: false, // www.example.com and example.com may be different
22
  decodeUnreserved: true,
23
}
24

25
function normalizeURL(urlString: string, options = DEFAULT_OPTIONS): string {
26
  const url = new URL(urlString)
27

28
  // Lowercase scheme and host
29
  url.protocol = url.protocol.toLowerCase()
30
  if (options.lowercaseHost) {
31
    url.hostname = url.hostname.toLowerCase()
32
  }
33

34
  // Remove default ports
35
  if (options.removeDefaultPorts) {
36
    if ((url.protocol === "http:" && url.port === "80") || (url.protocol === "https:" && url.port === "443")) {
37
      url.port = ""
38
    }
39
  }
40

41
  // Remove fragment
42
  if (options.removeFragment) {
43
    url.hash = ""
44
  }
45

46
  // Sort query parameters
47
  if (options.sortQueryParams && url.search) {
48
    const params = new URLSearchParams(url.search)
49
    const sorted = new URLSearchParams([...params.entries()].sort())
50
    url.search = sorted.toString()
51
  }
52

53
  // Remove trailing slash (except for root)
54
  if (options.removeTrailingSlash && url.pathname !== "/") {
16 collapsed lines
55
    url.pathname = url.pathname.replace(/\/+$/, "")
56
  }
57

58
  // Remove index files
59
  if (options.removeIndexFiles) {
60
    url.pathname = url.pathname.replace(/\/(index|default)\.(html?|php|asp)$/i, "/")
61
  }
62

63
  // Decode unreserved characters
64
  if (options.decodeUnreserved) {
65
    url.pathname = decodeURIComponent(url.pathname)
66
      .split("/")
67
      .map((segment) => encodeURIComponent(segment))
68
      .join("/")
69
  }
70

71
  return url.toString()
72
}
73

74
// Examples:
75
// normalizeURL('HTTP://Example.COM:80/Page/')
76
//   → 'http://example.com/page'
77
// normalizeURL('https://example.com/search?b=2&a=1')
78
//   → 'https://example.com/search?a=1&b=2'
79
// normalizeURL('https://example.com/index.html')
80
//   → 'https://example.com/'

10 collapsed lines
1
import { createHash } from "crypto"
2

3
class BloomFilter {
4
  private bitArray: Uint8Array
5
  private numHashFunctions: number
6
  private size: number
7

8
  constructor(expectedElements: number, falsePositiveRate: number = 0.01) {
9
    // Calculate optimal size and hash functions
10
    // m = -n * ln(p) / (ln(2)^2)
11
    // k = (m/n) * ln(2)
12
    this.size = Math.ceil((-expectedElements * Math.log(falsePositiveRate)) / Math.log(2) ** 2)
13
    this.numHashFunctions = Math.ceil((this.size / expectedElements) * Math.log(2))
14
    this.bitArray = new Uint8Array(Math.ceil(this.size / 8))
15

16
    console.log(`Bloom filter: ${this.size} bits, ${this.numHashFunctions} hash functions`)
17
    // For 1B URLs at 1% FP: ~1.2 GB, 7 hash functions
18
  }
19

20
  private getHashes(item: string): number[] {
21
    // Use double hashing: h(i) = h1 + i * h2
22
    const h1 = this.hash(item, 0)
23
    const h2 = this.hash(item, 1)
24

25
    return Array(this.numHashFunctions)
26
      .fill(0)
27
      .map((_, i) => Math.abs((h1 + i * h2) % this.size))
28
  }
29

30
  private hash(item: string, seed: number): number {
31
    const hash = createHash("md5").update(`${seed}:${item}`).digest()
32
    return hash.readUInt32LE(0)
33
  }
34

35
  add(item: string): void {
36
    for (const hash of this.getHashes(item)) {
37
      const byteIndex = Math.floor(hash / 8)
38
      const bitIndex = hash % 8
39
      this.bitArray[byteIndex] |= 1 << bitIndex
40
    }
41
  }
42

43
  mightContain(item: string): boolean {
44
    for (const hash of this.getHashes(item)) {
45
      const byteIndex = Math.floor(hash / 8)
46
      const bitIndex = hash % 8
47
      if ((this.bitArray[byteIndex] & (1 << bitIndex)) === 0) {
48
        return false // Definitely not in set
49
      }
16 collapsed lines
50
    }
51
    return true // Probably in set (may be false positive)
52
  }
53
}
54

55
// Usage
56
const seenURLs = new BloomFilter(1_000_000_000, 0.01)
57

58
function isURLSeen(url: string): boolean {
59
  const normalized = normalizeURL(url)
60

61
  if (!seenURLs.mightContain(normalized)) {
62
    return false // Definitely not seen
63
  }
64

65
  // Bloom filter says "maybe"—check persistent store
66
  return urlDatabase.exists(normalized)
67
}
68

69
function markURLSeen(url: string): void {
70
  const normalized = normalizeURL(url)
71
  seenURLs.add(normalized)
72
  urlDatabase.insert(normalized)
73
}

10 collapsed lines
1
import { createHash } from "crypto"
2

3
// SimHash generates a fingerprint where similar documents have similar hashes
4
// Hamming distance between fingerprints indicates similarity
5

6
function simhash(text: string, hashBits: number = 64): bigint {
7
  // Tokenize into shingles (word n-grams)
8
  const tokens = tokenize(text)
9
  const shingles = generateShingles(tokens, 3) // 3-word shingles
10

11
  // Initialize bit vector
12
  const v = new Array(hashBits).fill(0)
13

14
  for (const shingle of shingles) {
15
    // Hash each shingle to get bit positions
16
    const hash = hashShingle(shingle, hashBits)
17

18
    // Update vector: +1 if bit is 1, -1 if bit is 0
19
    for (let i = 0; i < hashBits; i++) {
20
      if ((hash >> BigInt(i)) & 1n) {
21
        v[i] += 1
22
      } else {
23
        v[i] -= 1
24
      }
25
    }
26
  }
27

28
  // Convert vector to fingerprint: 1 if positive, 0 if negative
29
  let fingerprint = 0n
30
  for (let i = 0; i < hashBits; i++) {
31
    if (v[i] > 0) {
32
      fingerprint |= 1n << BigInt(i)
33
    }
34
  }
35

36
  return fingerprint
37
}
38

39
function tokenize(text: string): string[] {
40
  return text
41
    .toLowerCase()
42
    .replace(/[^\w\s]/g, " ")
43
    .split(/\s+/)
44
    .filter((word) => word.length > 2)
45
}
46

47
function generateShingles(tokens: string[], n: number): string[] {
48
  const shingles: string[] = []
49
  for (let i = 0; i <= tokens.length - n; i++) {
50
    shingles.push(tokens.slice(i, i + n).join(" "))
51
  }
52
  return shingles
53
}
54

55
function hashShingle(shingle: string, bits: number): bigint {
56
  const hash = createHash("sha256").update(shingle).digest("hex")
57
  return BigInt("0x" + hash.slice(0, bits / 4))
58
}
59

21 collapsed lines
60
function hammingDistance(a: bigint, b: bigint): number {
61
  let xor = a ^ b
62
  let distance = 0
63
  while (xor > 0n) {
64
    distance += Number(xor & 1n)
65
    xor >>= 1n
66
  }
67
  return distance
68
}
69

70
// Usage
71
const DUPLICATE_THRESHOLD = 3 // Max 3 bits different
72

73
function isNearDuplicate(newFingerprint: bigint, existingFingerprints: bigint[]): boolean {
74
  for (const existing of existingFingerprints) {
75
    if (hammingDistance(newFingerprint, existing) <= DUPLICATE_THRESHOLD) {
76
      return true
77
    }
78
  }
79
  return false
80
}
81

82
// Example
83
const doc1 = "The quick brown fox jumps over the lazy dog"
84
const doc2 = "The quick brown fox leaps over the lazy dog" // One word changed
85
const doc3 = "Completely different content about web crawlers"
86

87
const hash1 = simhash(doc1)
88
const hash2 = simhash(doc2)
89
const hash3 = simhash(doc3)
90

91
console.log(hammingDistance(hash1, hash2)) // Small (< 3), near-duplicate
92
console.log(hammingDistance(hash1, hash3)) // Large (> 20), different content

10 collapsed lines
1
// RFC 9309 compliant robots.txt parser
2

3
interface RobotsRule {
4
  path: string
5
  allow: boolean
6
}
7

8
interface RobotsRules {
9
  rules: RobotsRule[]
10
  crawlDelay?: number
11
  sitemaps: string[]
12
}
13

14
function parseRobotsTxt(content: string, userAgent: string): RobotsRules {
15
  const lines = content.split("\n").map((l) => l.trim())
16
  const rules: RobotsRule[] = []
17
  let crawlDelay: number | undefined
18
  const sitemaps: string[] = []
19

20
  let currentUserAgents: string[] = []
21
  let inRelevantBlock = false
22

23
  for (const line of lines) {
24
    // Skip comments and empty lines
25
    if (line.startsWith("#") || line === "") continue
26

27
    const [directive, ...valueParts] = line.split(":")
28
    const value = valueParts.join(":").trim()
29

30
    switch (directive.toLowerCase()) {
31
      case "user-agent":
32
        // Check if this block applies to our user agent
33
        if (currentUserAgents.length > 0 && inRelevantBlock) {
34
          // We've finished a relevant block, stop processing
35
          // (RFC 9309: use first matching group)
36
          break
37
        }
38
        currentUserAgents.push(value.toLowerCase())
39
        inRelevantBlock = matchesUserAgent(value, userAgent)
40
        break
41

42
      case "allow":
43
        if (inRelevantBlock) {
44
          rules.push({ path: value, allow: true })
45
        }
46
        break
47

48
      case "disallow":
49
        if (inRelevantBlock) {
50
          rules.push({ path: value, allow: false })
51
        }
52
        break
53

54
      case "crawl-delay":
55
        // Note: crawl-delay is NOT in RFC 9309, but commonly used
56
        if (inRelevantBlock) {
57
          crawlDelay = parseInt(value, 10)
58
        }
59
        break
60

61
      case "sitemap":
62
        // Sitemaps apply globally
63
        sitemaps.push(value)
64
        break
65
    }
66
  }
67

68
  // Sort rules by path length (longest match wins per RFC 9309)
69
  rules.sort((a, b) => b.path.length - a.path.length)
21 collapsed lines
70

71
  return { rules, crawlDelay, sitemaps }
72
}
73

74
function matchesUserAgent(pattern: string, userAgent: string): boolean {
75
  const patternLower = pattern.toLowerCase()
76
  const uaLower = userAgent.toLowerCase()
77

78
  if (patternLower === "*") return true
79
  return uaLower.includes(patternLower)
80
}
81

82
function isAllowed(rules: RobotsRules, path: string): boolean {
83
  // RFC 9309: Longest matching path wins
84
  for (const rule of rules.rules) {
85
    if (pathMatches(rule.path, path)) {
86
      return rule.allow
87
    }
88
  }
89
  // Default: allowed if no rule matches
90
  return true
91
}
92

93
function pathMatches(pattern: string, path: string): boolean {
94
  // Handle wildcards (*) and end-of-path ($)
95
  let regex = pattern
96
    .replace(/[.+?^${}()|[\]\\]/g, "\\$&") // Escape special chars
97
    .replace(/\*/g, ".*") // * → .*
98
    .replace(/\$$/, "$") // Keep $ as end anchor
99

100
  if (!pattern.endsWith("$")) {
101
    regex = `^${regex}` // Match from start
102
  }
103

104
  return new RegExp(regex).test(path)
105
}
106

107
// Usage
108
const robotsTxt = `
109
User-agent: *
110
Disallow: /admin/
111
Disallow: /private/
112
Allow: /admin/public/
113

114
User-agent: Googlebot
115
Disallow: /no-google/
116

117
Sitemap: https://example.com/sitemap.xml
118
`
119

120
const rules = parseRobotsTxt(robotsTxt, "MyCrawler/1.0")
121
console.log(isAllowed(rules, "/admin/dashboard")) // false
122
console.log(isAllowed(rules, "/admin/public/")) // true (longest match wins)
123
console.log(isAllowed(rules, "/public/page")) // true (no matching rule)