Deconstructing JavaScript String Length: Unicode, UTF-16, and the Grapheme Cluster

console.log("👨‍👩‍👧‍👦".length) // 11 - UTF-16 code units
console.log(getGraphemeLength("👨‍👩‍👧‍👦")) // 1 - User-perceived characters

const logLengths = (...items) => console.log(items.map((item) => `${item}: ${item.length}`))

// Basic characters work as expected
logLengths("A", "a", "À", "⇐", "⇟")
// ['A: 1', 'a: 1', 'À: 1', '⇐: 1', '⇟: 1']

// Emoji require multiple code units
logLengths("🧘", "🌦", "😂", "😃", "🥖", "🚗")
// ['🧘: 2', '🌦: 2', '😂: 2', '😃: 2', '🥖: 2', '🚗: 2']

// Complex emoji sequences are even longer
logLengths("🧘", "🧘🏻‍♂️", "👨‍👩‍👧‍👦")
// ['🧘: 2', '🧘🏻‍♂️: 7', '👨‍👩‍👧‍👦: 11']

function getGraphemeLength(str, locale = "en") {
  return [...new Intl.Segmenter(locale, { granularity: "grapheme" }).segment(str)].length
}

console.log("👨‍👩‍👧‍👦".length) // 11
console.log(getGraphemeLength("👨‍👩‍👧‍👦")) // 1

// Iterate over grapheme clusters
const segmenter = new Intl.Segmenter("en", { granularity: "grapheme" })
for (const grapheme of segmenter.segment("👨‍👩‍👧‍👦🌦️🧘🏻‍♂️")) {
  console.log(`'${grapheme.segment}' at index ${grapheme.index}`)
}
// '👨‍👩‍👧‍👦' at index 0
// '🌦️' at index 11
// '🧘🏻‍♂️' at index 14

// ASCII characters (U+0000 to U+007F) are single UTF-16 code units
"A".charCodeAt(0) // 65 (U+0041)
"a".charCodeAt(0) // 97 (U+0061)

// Unicode code points vs. encoding
"€".codePointAt(0) // 8364 (U+20AC)
"💩".codePointAt(0) // 128169 (U+1F4A9)

// UTF-16 encoding examples
"€".length // 1 (BMP character)
"💩".length // 2 (supplementary plane - surrogate pair)
"👨‍👩‍👧‍👦".length // 11 (complex grapheme cluster)

// Surrogate pair encoding for U+1F4A9 (💩)
const highSurrogate = 0xd83d // U+D800 to U+DBFF
const lowSurrogate = 0xdca9 // U+DC00 to U+DFFF

// Mathematical transformation
const codePoint = 0x1f4a9
const temp = codePoint - 0x10000
const high = Math.floor(temp / 0x400) + 0xd800
const low = (temp % 0x400) + 0xdc00

console.log(high.toString(16), low.toString(16)) // 'd83d', 'dca9'

const emoji = "💩"

// Unsafe operations
emoji.length // 2 (code units)
emoji.charAt(0) // '\uD83D' (incomplete surrogate)
emoji.charCodeAt(0) // 55357 (high surrogate only)

// Safe operations
emoji.codePointAt(0) // 128169 (full code point)
[...emoji].length // 1 (code points)

const text = "A💩Z"

// Unsafe: iterates over code units
for (let i = 0; i < text.length; i++) {
  console.log(text[i]) // 'A', '\uD83D', '\uDCA9', 'Z'
}

// Safe: iterates over code points
for (const char of text) {
  console.log(char) // 'A', '💩', 'Z'
}

// Spread operator also works
console.log([...text]) // ['A', '💩', 'Z']

const family = "👨‍👩‍👧‍👦"
const segmenter = new Intl.Segmenter("en", { granularity: "grapheme" })

// Count grapheme clusters
console.log([...segmenter.segment(family)].length) // 1

// Iterate over grapheme clusters
for (const grapheme of segmenter.segment(family)) {
  console.log(grapheme.segment) // '👨‍👩‍👧‍👦'
}

// Without u flag: matches code units
/^.$/.test("💩") // false (2 code units)

// With u flag: matches code points
/^.$/u.test("💩") // true (1 code point)

// Unicode property escapes
/\p{Emoji}/u.test("💩") // true
/\p{Script=Latin}/u.test("A") // true

const e1 = "é" // U+00E9 (precomposed)
const e2 = "e\u0301" // U+0065 + U+0301 (decomposed)

console.log(e1 === e2) // false
console.log(e1.normalize() === e2.normalize()) // true

-- Legacy (incomplete UTF-8)
CREATE TABLE users (name VARCHAR(255) CHARACTER SET utf8);

-- Modern (complete UTF-8)
CREATE TABLE users (name VARCHAR(255) CHARACTER SET utf8mb4);

// API response with explicit encoding
res.setHeader("Content-Type", "application/json; charset=utf-8")

// Input normalization
const normalizedInput = userInput.normalize("NFC")

const emoji = "💩"

// Dangerous: splits surrogate pair
const corrupted = emoji.substring(0, 1) // '\uD83D' (invalid)

// Safe: use code point-aware methods
const safe = [...emoji][0] // '💩'

// Dangerous: assumes 1 byte per character
const buffer = Buffer.alloc(100)
buffer.write(text.slice(0, 100)) // May overflow with emoji

// Safe: use proper encoding
const safeBuffer = Buffer.from(text, "utf8")

// Cyrillic 'а' vs Latin 'a'
const cyrillicA = "а" // U+0430
const latinA = "a" // U+0061

console.log(cyrillicA === latinA) // false
console.log(cyrillicA.normalize() === latinA.normalize()) // false

class UnicodeSanctuary {
  // Decode on input
  static decode(input: Buffer, encoding: string = "utf8"): string {
    return input.toString(encoding).normalize("NFC")
  }

  // Process internally (always Unicode)
  static process(text: string): string {
    // All operations on normalized Unicode
    return text.toUpperCase()
  }

  // Encode on output
  static encode(text: string, encoding: string = "utf8"): Buffer {
    return Buffer.from(text, encoding)
  }
}

function validateUsername(username: string): boolean {
  // Normalize first
  const normalized = username.normalize("NFC")

  // Check for homographs
  const hasHomographs = /[\u0430-\u044F]/.test(normalized) // Cyrillic

  // Validate grapheme length
  const segmenter = new Intl.Segmenter("en", { granularity: "grapheme" })
  const graphemeCount = [...segmenter.segment(normalized)].length

  return !hasHomographs && graphemeCount >= 3 && graphemeCount <= 20
}