Design a URL Shortener: IDs, Storage, and Scale

1
interface ClickEvent {
2
  shortCode: string
3
  timestamp: number
4

5
  // Client info
6
  ipHash: string // Anonymized for GDPR
7
  userAgent: string
8
  referer: string | null
9

10
  // Derived fields
11
  country: string
12
  city: string
13
  deviceType: "mobile" | "desktop" | "tablet"
14
  browser: string
15
  os: string
16

17
  // Bot detection
18
  isBot: boolean
19
  botType: string | null
20
}

1
{
2
  "url": "https://example.com/very/long/path?with=params",
3
  "customCode": "my-link",
4
  "expiresAt": "2025-12-31T23:59:59Z",
5
  "password": "optional-password",
6
  "maxClicks": 1000,
7
  "tags": ["campaign-2024", "social"]
8
}

1
{
2
  "id": "url_abc123def456",
3
  "shortCode": "my-link",
4
  "shortUrl": "https://suj.ee/my-link",
5
  "longUrl": "https://example.com/very/long/path?with=params",
6
  "createdAt": "2024-02-03T10:00:00Z",
7
  "expiresAt": "2025-12-31T23:59:59Z",
8
  "isPasswordProtected": true,
9
  "maxClicks": 1000,
10
  "clickCount": 0,
11
  "qrCode": "https://suj.ee/api/v1/urls/url_abc123def456/qr"
12
}

1
HTTP/1.1 302 Found
2
Location: https://example.com/very/long/path
3
Cache-Control: private, max-age=60
4
X-Robots-Tag: noindex

1
{
2
  "urlId": "url_abc123def456",
3
  "period": {
4
    "start": "2024-01-27T00:00:00Z",
5
    "end": "2024-02-03T23:59:59Z"
6
  },
7
  "summary": {
8
    "totalClicks": 15420,
9
    "uniqueClicks": 12350,
10
    "botClicks": 1230
11
  },
12
  "timeSeries": [
13
    { "date": "2024-01-27", "clicks": 2100, "unique": 1800 },
14
    { "date": "2024-01-28", "clicks": 2450, "unique": 2100 }
15
  ],
16
  "topReferrers": [
17
    { "referrer": "twitter.com", "clicks": 5200, "percentage": 33.7 },
18
    { "referrer": "facebook.com", "clicks": 3100, "percentage": 20.1 }
19
  ],
20
  "topCountries": [
21
    { "country": "US", "clicks": 6800, "percentage": 44.1 },
22
    { "country": "UK", "clicks": 2300, "percentage": 14.9 }
23
  ],
24
  "devices": {
25
    "mobile": { "clicks": 9200, "percentage": 59.7 },
26
    "desktop": { "clicks": 5800, "percentage": 37.6 },
27
    "tablet": { "clicks": 420, "percentage": 2.7 }
28
  }
29
}

1
{
2
  "urls": [{ "url": "https://example.com/page1" }, { "url": "https://example.com/page2", "customCode": "page2" }],
3
  "defaultExpiry": "2025-12-31T23:59:59Z",
4
  "tags": ["bulk-import"]
5
}

1
{
2
  "jobId": "job_xyz789",
3
  "status": "processing",
4
  "totalUrls": 2,
5
  "statusUrl": "/api/v1/jobs/job_xyz789"
6
}

1
{
2
  "urls": [
3
    {
4
      "id": "url_abc123",
5
      "shortCode": "abc123",
6
      "shortUrl": "https://suj.ee/abc123",
7
      "longUrl": "https://example.com/...",
8
      "clickCount": 1542,
9
      "createdAt": "2024-01-15T10:00:00Z",
10
      "status": "active"
11
    }
12
  ],
13
  "pagination": {
14
    "nextCursor": "cursor_def456",
15
    "hasMore": true,
16
    "total": 234
17
  }
18
}

1
CREATE TABLE url_mappings (
2
    short_code TEXT,
3
    long_url TEXT,
4
    user_id UUID,
5
    created_at TIMESTAMP,
6
    expires_at TIMESTAMP,
7
    is_active BOOLEAN,
8
    click_count COUNTER,
9
    password_hash TEXT,
10
    max_clicks INT,
11
    metadata MAP<TEXT, TEXT>,
12
    PRIMARY KEY (short_code)
13
) WITH default_time_to_live = 157680000  -- 5 years
14
  AND compaction = {'class': 'LeveledCompactionStrategy'}
15
  AND caching = {'keys': 'ALL', 'rows_per_partition': 'ALL'};

1
CREATE TABLE urls_by_user (
2
    user_id UUID,
3
    created_at TIMESTAMP,
4
    short_code TEXT,
5
    long_url TEXT,
6
    click_count BIGINT,
7
    is_active BOOLEAN,
8
    PRIMARY KEY ((user_id), created_at, short_code)
9
) WITH CLUSTERING ORDER BY (created_at DESC);

1
CREATE TABLE clicks (
2
    short_code String,
3
    clicked_at DateTime64(3),
4

5
    -- Client data
6
    ip_hash FixedString(16),
7
    country LowCardinality(String),
8
    city String,
9

10
    -- Device data
11
    device_type Enum8('mobile' = 1, 'desktop' = 2, 'tablet' = 3),
12
    browser LowCardinality(String),
13
    os LowCardinality(String),
14

15
    -- Referrer
16
    referrer_domain LowCardinality(String),
17
    referrer_path String,
18

19
    -- Bot detection
20
    is_bot UInt8,
21
    bot_type LowCardinality(String),
22

23
    -- Aggregation keys
24
    date Date MATERIALIZED toDate(clicked_at),
25
    hour UInt8 MATERIALIZED toHour(clicked_at)
26
)
27
ENGINE = MergeTree()
28
PARTITION BY toYYYYMM(clicked_at)
29
ORDER BY (short_code, clicked_at)
30
TTL clicked_at + INTERVAL 1 YEAR;
31

32
-- Materialized view for real-time aggregates
33
CREATE MATERIALIZED VIEW clicks_daily_mv
34
ENGINE = SummingMergeTree()
35
PARTITION BY toYYYYMM(date)
36
ORDER BY (short_code, date, country, device_type)
37
AS SELECT
38
    short_code,
39
    date,
40
    country,
41
    device_type,
42
    count() as clicks,
43
    uniqExact(ip_hash) as unique_clicks
44
FROM clicks
45
GROUP BY short_code, date, country, device_type;

1
CREATE TABLE users (
2
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
3
    email TEXT UNIQUE NOT NULL,
4
    password_hash TEXT NOT NULL,
5
    plan TEXT DEFAULT 'free',
6
    api_key_hash TEXT UNIQUE,
7
    created_at TIMESTAMPTZ DEFAULT NOW(),
8
    updated_at TIMESTAMPTZ DEFAULT NOW()
9
);
10

11
CREATE TABLE custom_domains (
12
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
13
    user_id UUID REFERENCES users(id),
14
    domain TEXT UNIQUE NOT NULL,
15
    is_verified BOOLEAN DEFAULT false,
16
    ssl_status TEXT DEFAULT 'pending',
17
    created_at TIMESTAMPTZ DEFAULT NOW()
18
);
19

20
CREATE TABLE api_keys (
21
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
22
    user_id UUID REFERENCES users(id),
23
    key_hash TEXT UNIQUE NOT NULL,
24
    name TEXT,
25
    permissions JSONB DEFAULT '["read", "write"]',
26
    last_used_at TIMESTAMPTZ,
27
    expires_at TIMESTAMPTZ,
28
    created_at TIMESTAMPTZ DEFAULT NOW()
29
);
30

31
CREATE INDEX idx_users_api_key ON users(api_key_hash);
32
CREATE INDEX idx_domains_user ON custom_domains(user_id);

1
CREATE TABLE keys_unused (
2
    short_code TEXT PRIMARY KEY,
3
    created_at TIMESTAMPTZ DEFAULT NOW()
4
);
5

6
CREATE TABLE keys_allocated (
7
    short_code TEXT PRIMARY KEY,
8
    allocated_to TEXT NOT NULL,  -- Server instance ID
9
    allocated_at TIMESTAMPTZ DEFAULT NOW()
10
);
11

12
CREATE TABLE keys_used (
13
    short_code TEXT PRIMARY KEY,
14
    used_at TIMESTAMPTZ DEFAULT NOW()
15
);
16

17
-- Batch allocation function
18
CREATE OR REPLACE FUNCTION allocate_keys(
19
    server_id TEXT,
20
    batch_size INT
21
) RETURNS TABLE(short_code TEXT) AS $$
22
BEGIN
23
    RETURN QUERY
24
    WITH allocated AS (
25
        DELETE FROM keys_unused
26
        WHERE short_code IN (
27
            SELECT ku.short_code
28
            FROM keys_unused ku
29
            LIMIT batch_size
30
            FOR UPDATE SKIP LOCKED
31
        )
32
        RETURNING keys_unused.short_code
33
    )
34
    INSERT INTO keys_allocated (short_code, allocated_to)
35
    SELECT a.short_code, server_id
36
    FROM allocated a
37
    RETURNING keys_allocated.short_code;
38
END;
39
$$ LANGUAGE plpgsql;

5 collapsed lines
1
const CHARSET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
2
const BASE = BigInt(62)
3

4
export function encodeBase62(num: bigint): string {
5
  if (num === 0n) return CHARSET[0]
6

7
  let result = ""
8
  while (num > 0n) {
9
    result = CHARSET[Number(num % BASE)] + result
10
    num = num / BASE
11
  }
12
  return result
13
}
14

15
export function decodeBase62(str: string): bigint {
16
  let result = 0n
17
  for (const char of str) {
18
    const index = CHARSET.indexOf(char)
19
    if (index === -1) throw new Error(`Invalid character: ${char}`)
20
    result = result * BASE + BigInt(index)
21
  }
22
  return result
23
}
24

25
// Pad to fixed length for consistent URLs
26
export function encodeBase62Padded(num: bigint, length: number): string {
27
  const encoded = encodeBase62(num)
28
  return encoded.padStart(length, "0")
29
}

15 collapsed lines
1
const EPOCH = 1609459200000n // 2021-01-01 00:00:00 UTC
2
const NODE_BITS = 10n
3
const SEQUENCE_BITS = 12n
4

5
const MAX_NODE_ID = (1n << NODE_BITS) - 1n
6
const MAX_SEQUENCE = (1n << SEQUENCE_BITS) - 1n
7

8
const NODE_SHIFT = SEQUENCE_BITS
9
const TIMESTAMP_SHIFT = SEQUENCE_BITS + NODE_BITS
10

11
export class SnowflakeGenerator {
12
  private nodeId: bigint
13
  private sequence: bigint = 0n
14
  private lastTimestamp: bigint = -1n
15

16
  constructor(nodeId: number) {
17
    if (nodeId < 0 || BigInt(nodeId) > MAX_NODE_ID) {
18
      throw new Error(`Node ID must be between 0 and ${MAX_NODE_ID}`)
19
    }
20
    this.nodeId = BigInt(nodeId)
21
  }
22

23
  generate(): bigint {
24
    let timestamp = BigInt(Date.now()) - EPOCH
25

26
    if (timestamp === this.lastTimestamp) {
27
      this.sequence = (this.sequence + 1n) & MAX_SEQUENCE
28
      if (this.sequence === 0n) {
29
        // Sequence exhausted, wait for next millisecond
30
        timestamp = this.waitNextMillis(this.lastTimestamp)
31
      }
32
    } else {
33
      this.sequence = 0n
34
    }
35

36
    this.lastTimestamp = timestamp
37

38
    return (timestamp << TIMESTAMP_SHIFT) | (this.nodeId << NODE_SHIFT) | this.sequence
39
  }
40

41
  private waitNextMillis(lastTimestamp: bigint): bigint {
42
    let timestamp = BigInt(Date.now()) - EPOCH
43
    while (timestamp <= lastTimestamp) {
44
      timestamp = BigInt(Date.now()) - EPOCH
45
    }
46
    return timestamp
47
  }
48
}

20 collapsed lines
1
import { BloomFilter } from "bloom-filters"
2

3
interface RedirectResult {
4
  found: boolean
5
  longUrl?: string
6
  isExpired?: boolean
7
  requiresPassword?: boolean
8
}
9

10
class RedirectService {
11
  private readonly redis: RedisCluster
12
  private readonly cassandra: CassandraClient
13
  private readonly bloomFilter: BloomFilter
14
  private readonly analytics: AnalyticsCollector
15

16
  constructor() {
17
    // Bloom filter: 1B items, 0.1% false positive rate
18
    // Memory: ~1.2GB
19
    this.bloomFilter = BloomFilter.create(1_000_000_000, 0.001)
20
  }
21

22
  async redirect(shortCode: string, context: RequestContext): Promise<RedirectResult> {
23
    // Step 1: Bloom filter check (prevents cache stampede on non-existent codes)
24
    if (!this.bloomFilter.has(shortCode)) {
25
      return { found: false }
26
    }
27

28
    // Step 2: Redis cache check
29
    const cached = await this.redis.hgetall(`url:${shortCode}`)
30
    if (cached && cached.long_url) {
31
      this.logClick(shortCode, context) // Async, non-blocking
32
      return this.buildResult(cached)
33
    }
34

35
    // Step 3: Database lookup
36
    const row = await this.cassandra.execute("SELECT * FROM url_mappings WHERE short_code = ?", [shortCode])
37

38
    if (!row || row.length === 0) {
39
      // False positive from bloom filter
40
      return { found: false }
41
    }
42

43
    const url = row[0]
44

45
    // Step 4: Cache the result
46
    await this.redis.hset(`url:${shortCode}`, {
47
      long_url: url.long_url,
48
      expires_at: url.expires_at?.toISOString() || "",
49
      password_hash: url.password_hash || "",
50
      is_active: url.is_active ? "1" : "0",
51
    })
52
    await this.redis.expire(`url:${shortCode}`, 3600) // 1 hour TTL
53

54
    this.logClick(shortCode, context)
55
    return this.buildResult(url)
56
  }
57

58
  private buildResult(data: any): RedirectResult {
59
    if (data.is_active === "0" || data.is_active === false) {
60
      return { found: false }
61
    }
62

63
    if (data.expires_at && new Date(data.expires_at) < new Date()) {
64
      return { found: true, isExpired: true }
65
    }
66

67
    if (data.password_hash) {
68
      return { found: true, requiresPassword: true, longUrl: data.long_url }
69
    }
70

71
    return { found: true, longUrl: data.long_url }
72
  }
73

74
  private logClick(shortCode: string, context: RequestContext): void {
75
    // Fire and forget - don't await
76
    this.analytics
77
      .log({
78
        shortCode,
79
        timestamp: Date.now(),
80
        ip: context.ip,
81
        userAgent: context.userAgent,
82
        referer: context.referer,
83
      })
84
      .catch((err) => console.error("Analytics error:", err))
85
  }
86
}

12 collapsed lines
1
interface RateLimitResult {
2
  allowed: boolean
3
  remaining: number
4
  resetAt: number
5
}
6

7
class SlidingWindowRateLimiter {
8
  private readonly redis: RedisCluster
9

10
  async checkLimit(key: string, limit: number, windowMs: number): Promise<RateLimitResult> {
11
    const now = Date.now()
12
    const windowStart = now - windowMs
13

14
    // Lua script for atomic sliding window
15
    const result = await this.redis.eval(
16
      `
17
      local key = KEYS[1]
18
      local now = tonumber(ARGV[1])
19
      local window_start = tonumber(ARGV[2])
20
      local limit = tonumber(ARGV[3])
21
      local window_ms = tonumber(ARGV[4])
22

23
      -- Remove old entries
24
      redis.call('ZREMRANGEBYSCORE', key, '-inf', window_start)
25

26
      -- Count current entries
27
      local count = redis.call('ZCARD', key)
28

29
      if count < limit then
30
        -- Add new entry
31
        redis.call('ZADD', key, now, now .. ':' .. math.random())
32
        redis.call('PEXPIRE', key, window_ms)
33
        return {1, limit - count - 1, now + window_ms}
34
      else
35
        -- Get oldest entry for reset time
36
        local oldest = redis.call('ZRANGE', key, 0, 0, 'WITHSCORES')
37
        local reset_at = oldest[2] + window_ms
38
        return {0, 0, reset_at}
39
      end
40
    `,
41
      [key],
42
      [now, windowStart, limit, windowMs],
43
    )
44

45
    return {
46
      allowed: result[0] === 1,
47
      remaining: result[1],
48
      resetAt: result[2],
49
    }
50
  }
51
}

15 collapsed lines
1
interface ScanResult {
2
  isSafe: boolean
3
  threats: string[]
4
  scanTime: number
5
}
6

7
class URLScanner {
8
  private readonly blocklist: BlocklistService
9
  private readonly googleSafeBrowsing: GoogleSafeBrowsingClient
10
  private readonly virusTotal: VirusTotalClient
11
  private readonly redis: RedisCluster
12

13
  async scan(url: string): Promise<ScanResult> {
14
    const urlHash = this.hashUrl(url)
15

16
    // Check cache first
17
    const cached = await this.redis.get(`scan:${urlHash}`)
18
    if (cached) {
19
      return JSON.parse(cached)
20
    }
21

22
    const domain = new URL(url).hostname
23
    const threats: string[] = []
24

25
    // Step 1: Local blocklist (fast)
26
    if (await this.blocklist.contains(domain)) {
27
      return this.cacheResult(urlHash, { isSafe: false, threats: ["blocklist"] })
28
    }
29

30
    // Step 2: Known-good allowlist
31
    if (await this.isKnownGood(domain)) {
32
      return this.cacheResult(urlHash, { isSafe: true, threats: [] })
33
    }
34

35
    // Step 3: Google Safe Browsing API
36
    const gsbResult = await this.googleSafeBrowsing.lookup(url)
37
    if (gsbResult.threats.length > 0) {
38
      threats.push(...gsbResult.threats)
39
    }
40

41
    // Step 4: VirusTotal (for suspicious domains)
42
    if (await this.isSuspicious(domain)) {
43
      const vtResult = await this.virusTotal.scan(url)
44
      if (vtResult.positives > 2) {
45
        threats.push("malware")
46
      }
47
    }
48

49
    const result: ScanResult = {
50
      isSafe: threats.length === 0,
51
      threats,
52
      scanTime: Date.now(),
53
    }
54

55
    return this.cacheResult(urlHash, result)
56
  }
57

58
  private async cacheResult(hash: string, result: ScanResult): Promise<ScanResult> {
59
    // Cache safe results longer than unsafe
60
    const ttl = result.isSafe ? 86400 : 3600 // 24h vs 1h
61
    await this.redis.setex(`scan:${hash}`, ttl, JSON.stringify(result))
62
    return result
63
  }
64

65
  private async isKnownGood(domain: string): Promise<boolean> {
66
    // Top 10K domains by traffic
67
    return this.redis.sismember("domains:allowlist", domain)
68
  }
69

70
  private async isSuspicious(domain: string): Promise<boolean> {
71
    // New domain, unusual TLD, etc.
72
    const domainAge = await this.getDomainAge(domain)
73
    return domainAge < 30 // Less than 30 days old
74
  }
75
}

10 collapsed lines
1
interface ClickEvent {
2
  shortCode: string
3
  timestamp: number
4
  ip: string
5
  userAgent: string
6
  referer: string | null
7
}
8

9
class AnalyticsCollector {
10
  private readonly kafka: KafkaProducer
11
  private readonly buffer: ClickEvent[] = []
12
  private readonly BUFFER_SIZE = 100
13
  private readonly FLUSH_INTERVAL = 1000 // 1 second
14

15
  constructor() {
16
    setInterval(() => this.flush(), this.FLUSH_INTERVAL)
17
  }
18

19
  async log(event: ClickEvent): Promise<void> {
20
    this.buffer.push(event)
21

22
    if (this.buffer.length >= this.BUFFER_SIZE) {
23
      await this.flush()
24
    }
25
  }
26

27
  private async flush(): Promise<void> {
28
    if (this.buffer.length === 0) return
29

30
    const events = this.buffer.splice(0)
31

32
    await this.kafka.sendBatch({
33
      topic: "clicks",
34
      messages: events.map((e) => ({
35
        key: e.shortCode,
36
        value: JSON.stringify(e),
37
        timestamp: e.timestamp.toString(),
38
      })),
39
    })
40
  }
41
}
42

43
// Stream processor (Kafka Consumer → ClickHouse)
44
class ClickProcessor {
45
  private readonly clickhouse: ClickHouseClient
46
  private readonly geoIP: GeoIPService
47
  private readonly deviceParser: DeviceParser
48
  private readonly botDetector: BotDetector
49

50
  async process(event: ClickEvent): Promise<EnrichedClick> {
51
    const geo = await this.geoIP.lookup(event.ip)
52
    const device = this.deviceParser.parse(event.userAgent)
53
    const isBot = this.botDetector.detect(event.userAgent, event.ip)
54

55
    return {
56
      short_code: event.shortCode,
57
      clicked_at: new Date(event.timestamp),
58
      ip_hash: this.hashIP(event.ip), // GDPR compliance
59
      country: geo.country,
60
      city: geo.city,
61
      device_type: device.type,
62
      browser: device.browser,
63
      os: device.os,
64
      referrer_domain: this.extractDomain(event.referer),
65
      referrer_path: this.extractPath(event.referer),
66
      is_bot: isBot.isBot ? 1 : 0,
67
      bot_type: isBot.type,
68
    }
69
  }
70

71
  private hashIP(ip: string): string {
72
    // GDPR-compliant anonymization
73
    return crypto
74
      .createHash("sha256")
75
      .update(ip + process.env.IP_SALT)
76
      .digest("hex")
77
      .substring(0, 32)
78
  }
79
}

8 collapsed lines
1
// Server-side redirect handler (minimal processing)
2
export async function handleRedirect(req: Request): Promise<Response> {
3
  const shortCode = req.url.split("/").pop()
4

5
  // Validate format before any I/O
6
  if (!isValidCode(shortCode)) {
7
    return new Response(null, { status: 404 })
8
  }
9

10
  const result = await redirectService.redirect(shortCode, {
11
    ip: req.headers.get("x-forwarded-for"),
12
    userAgent: req.headers.get("user-agent"),
13
    referer: req.headers.get("referer"),
14
  })
15

16
  if (!result.found) {
17
    return new Response(null, { status: 404 })
18
  }
19

20
  if (result.isExpired) {
21
    return new Response("Link expired", { status: 410 })
22
  }
23

24
  if (result.requiresPassword) {
25
    return renderPasswordPage(shortCode)
26
  }
27

28
  return new Response(null, {
29
    status: 302,
30
    headers: {
31
      Location: result.longUrl,
32
      "Cache-Control": "private, max-age=60",
33
      "X-Robots-Tag": "noindex",
34
    },
35
  })
36
}