JavaScript String Length: Graphemes, UTF-16, and Unicode

1
const logLengths = (...items) => console.log(items.map((item) => `${item}: ${item.length}`))
2

3
// BMP characters (U+0000 to U+FFFF): 1 code unit each
4
logLengths("A", "a", "À", "⇐", "⇟")
5
// ['A: 1', 'a: 1', 'À: 1', '⇐: 1', '⇟: 1']
6

7
// Supplementary plane emoji (U+10000+): 2 code units (surrogate pair)
8
logLengths("🧘", "🌦", "😂", "😃", "🥖", "🚗")
9
// ['🧘: 2', '🌦: 2', '😂: 2', '😃: 2', '🥖: 2', '🚗: 2']
10

11
// ZWJ sequences: multiple code points joined by U+200D
12
logLengths("🧘", "🧘🏻‍♂️", "👨‍👩‍👧‍👦")
13
// ['🧘: 2', '🧘🏻‍♂️: 7', '👨‍👩‍👧‍👦: 11']

1
function getGraphemeLength(str: string, locale = "en"): number {
2
  return [...new Intl.Segmenter(locale, { granularity: "grapheme" }).segment(str)].length
3
}
4

5
console.log("👨‍👩‍👧‍👦".length) // 11 (code units)
6
console.log(getGraphemeLength("👨‍👩‍👧‍👦")) // 1 (grapheme)
7

8
// Iterate over grapheme clusters
9
const segmenter = new Intl.Segmenter("en", { granularity: "grapheme" })
10
for (const { segment, index } of segmenter.segment("👨‍👩‍👧‍👦🌦️🧘🏻‍♂️")) {
11
  console.log(`'${segment}' at code unit index ${index}`)
12
}
13
// '👨‍👩‍👧‍👦' at code unit index 0
14
// '🌦️' at code unit index 11
15
// '🧘🏻‍♂️' at code unit index 14

1
// Lone surrogate (invalid UTF-16, valid JavaScript string)
2
const invalid = "\uD800" // High surrogate without low surrogate
3
invalid.length // 1
4
invalid.isWellFormed() // false (ES2024)

1
// Encoding algorithm for code point → surrogate pair
2
function toSurrogatePair(codePoint: number): [number, number] {
3
  const temp = codePoint - 0x10000
4
  const high = Math.floor(temp / 0x400) + 0xd800 // D800-DBFF
5
  const low = (temp % 0x400) + 0xdc00 // DC00-DFFF
6
  return [high, low]
7
}
8

9
// U+1F4A9 (💩) → [0xD83D, 0xDCA9]
10
toSurrogatePair(0x1f4a9) // [55357, 56489]
11

12
// Verification
13
"💩".charCodeAt(0) // 55357 (0xD83D) - high surrogate
14
"💩".charCodeAt(1) // 56489 (0xDCA9) - low surrogate
15
"💩".codePointAt(0) // 128169 (0x1F4A9) - full code point

1
const text = "Hello 💩 World"
2

3
// Dangerous: arbitrary index may split surrogate pair
4
text.substring(0, 7) // "Hello �" - corrupted high surrogate
5

6
// Safe: slice at code point boundaries
7
const chars = [...text]
8
chars.slice(0, 7).join("") // "Hello 💩"

1
// Dangerous: splits surrogate pairs into invalid characters
2
"💩".split("") // ['\uD83D', '\uDCA9'] - two invalid strings
3

4
// Safe: spread preserves code points
5
[..."💩"] // ['💩']

1
const emoji = "💩"
2

3
emoji.length      // 2 (code units, not characters)
4
emoji.charAt(0)   // '\uD83D' (lone high surrogate - invalid)
5
emoji.charAt(1)   // '\uDCA9' (lone low surrogate - invalid)
6
emoji.charCodeAt(0) // 55357 (high surrogate value)
7

8
// Safe alternatives
9
emoji.codePointAt(0) // 128169 (full code point)
10
emoji.at(0)          // '\uD83D' (still code unit based - ES2022)
11
[...emoji][0]        // '💩' (code point)

1
const wellFormed = "Hello 💩"
2
const illFormed = "Hello \uD800" // lone high surrogate
3

4
wellFormed.isWellFormed() // true
5
illFormed.isWellFormed() // false
6

7
// Fix by replacing lone surrogates with U+FFFD (replacement character)
8
illFormed.toWellFormed() // "Hello �"
9

10
// Use case: encodeURI throws on ill-formed strings
11
encodeURI(illFormed) // URIError
12
encodeURI(illFormed.toWellFormed()) // "Hello%20%EF%BF%BD"

1
const text = "A💩Z"
2

3
// Code unit iteration (broken)
4
for (let i = 0; i < text.length; i++) {
5
  console.log(text[i]) // 'A', '\uD83D', '\uDCA9', 'Z' - 4 items
6
}
7

8
// Code point iteration (correct for supplementary characters)
9
for (const char of text) {
10
  console.log(char) // 'A', '💩', 'Z' - 3 items
11
}
12

13
// Spread also iterates code points
14
[...text] // ['A', '💩', 'Z']
15
[...text].length // 3 (code points)
16
text.length // 4 (code units)

1
// Without u flag: . matches one code unit
2
/^.$/.test("💩") // false (💩 is 2 code units)
3

4
// With u flag: . matches one code point
5
/^.$/u.test("💩") // true
6

7
// Unicode property escapes (ES2018)
8
/\p{Emoji}/u.test("💩")           // true
9
/\p{Script=Latin}/u.test("A")    // true
10
/\p{Script=Cyrillic}/u.test("а") // true (Cyrillic 'а', not Latin 'a')
11

12
// Extended grapheme cluster matching (ES2024 /v flag)
13
/^\p{RGI_Emoji}$/v.test("👨‍👩‍👧‍👦") // true - matches full ZWJ sequence

1
const e1 = "é" // U+00E9 (precomposed: single code point)
2
const e2 = "e\u0301" // U+0065 + U+0301 (decomposed: base + combining acute)
3

4
e1 === e2 // false (different code points)
5
e1.normalize() === e2.normalize() // true (NFC: canonical composition)
6
e1.length // 1
7
e2.length // 2
8

9
// Normalization forms
10
"é".normalize("NFC") // Canonical Composition (default)
11
"é".normalize("NFD") // Canonical Decomposition
12
"é".normalize("NFKC") // Compatibility Composition
13
"é".normalize("NFKD") // Compatibility Decomposition

1
-- Dangerous: silently truncates emoji or throws errors
2
CREATE TABLE users (name VARCHAR(255) CHARACTER SET utf8);
3
INSERT INTO users VALUES ('Hello 💩'); -- Error or truncation
4

5
-- Safe: full UTF-8 support (4 bytes per character max)
6
CREATE TABLE users (name VARCHAR(255) CHARACTER SET utf8mb4);

1
// Always specify encoding in Content-Type
2
res.setHeader("Content-Type", "application/json; charset=utf-8")
3

4
// Normalize input at system boundary
5
const sanitized = userInput
6
  .normalize("NFC") // Canonical composition
7
  .replace(/\p{C}/gu, "") // Remove control characters
8

9
// Validate well-formedness before external APIs
10
if (!sanitized.isWellFormed()) {
11
  throw new Error("Invalid Unicode input")
12
}

1
const MAX_GRAPHEMES = 280 // Twitter-style limit
2

3
function validateLength(text: string): boolean {
4
  const segmenter = new Intl.Segmenter("en", { granularity: "grapheme" })
5
  const graphemeCount = [...segmenter.segment(text)].length
6
  return graphemeCount <= MAX_GRAPHEMES
7
}
8

9
// "👨‍👩‍👧‍👦".repeat(280) passes - 280 graphemes
10
// But it's 11 * 280 = 3080 code units for storage

1
// User sees: "👨‍👩‍👧‍👦" (1 character)
2
// Code units: 11 positions
3
// Code points: 7 positions
4
// Valid cursor positions: 2 (before and after)
5

6
function getCursorPositions(text: string): number[] {
7
  const segmenter = new Intl.Segmenter("en", { granularity: "grapheme" })
8
  const positions = [0]
9
  for (const { index, segment } of segmenter.segment(text)) {
10
    positions.push(index + segment.length)
11
  }
12
  return positions
13
}
14

15
getCursorPositions("👨‍👩‍👧‍👦") // [0, 11] - only 2 valid positions

1
const cyrillicA = "а" // U+0430 (Cyrillic Small Letter A)
2
const latinA = "a" // U+0061 (Latin Small Letter A)
3

4
cyrillicA === latinA // false
5
cyrillicA.normalize() === latinA.normalize() // false (different scripts)
6

7
// Detecting mixed scripts
8
function detectMixedScripts(text: string): boolean {
9
  const scripts = new Set<string>()
10
  for (const char of text) {
11
    // Check script property (simplified)
12
    if (/\p{Script=Latin}/u.test(char)) scripts.add("Latin")
13
    if (/\p{Script=Cyrillic}/u.test(char)) scripts.add("Cyrillic")
14
    // Add more scripts as needed
15
  }
16
  return scripts.size > 1
17
}
18

19
detectMixedScripts("pаypal.com") // true - mixed Latin and Cyrillic

1
// Code units vs UTF-8 bytes
2
const text = "Hello 💩"
3
text.length // 8 code units
4
Buffer.byteLength(text, "utf8") // 10 bytes (💩 = 4 bytes in UTF-8)
5

6
// Safe buffer allocation
7
function safeBuffer(text: string): Buffer {
8
  return Buffer.from(text, "utf8") // Auto-sizes correctly
9
}
10

11
// Dangerous: assumes code units = bytes
12
const buf = Buffer.alloc(text.length) // Only 8 bytes
13
buf.write(text) // Truncation risk

1
const composed = "é" // U+00E9 (1 code point)
2
const decomposed = "e\u0301" // U+0065 + U+0301 (2 code points)
3

4
composed.length // 1
5
decomposed.length // 2
6

7
// Both render identically, but different lengths
8
// Always normalize before length checks

1
function truncateGraphemes(text: string, maxGraphemes: number, ellipsis = "…"): string {
2
  const segmenter = new Intl.Segmenter("en", { granularity: "grapheme" })
3
  const segments = [...segmenter.segment(text)]
4

5
  if (segments.length <= maxGraphemes) return text
6

7
  return (
8
    segments
9
      .slice(0, maxGraphemes - 1)
10
      .map((s) => s.segment)
11
      .join("") + ellipsis
12
  )
13
}
14

15
truncateGraphemes("Hello 👨‍👩‍👧‍👦 World", 8) // "Hello 👨‍👩‍👧‍👦…"
16
// Code unit truncation would break: "Hello 👨‍👩‍👧‍👦 World".slice(0, 8) = "Hello �"

1
function validateUsername(username: string): { valid: boolean; error?: string } {
2
  const normalized = username.normalize("NFC")
3

4
  // Check well-formedness
5
  if (!normalized.isWellFormed()) {
6
    return { valid: false, error: "Invalid Unicode" }
7
  }
8

9
  // Count graphemes
10
  const segmenter = new Intl.Segmenter("en", { granularity: "grapheme" })
11
  const graphemeCount = [...segmenter.segment(normalized)].length
12

13
  if (graphemeCount < 3 || graphemeCount > 20) {
14
    return { valid: false, error: "Username must be 3-20 characters" }
15
  }
16

17
  // Detect mixed scripts (homograph defense)
18
  const hasLatin = /\p{Script=Latin}/u.test(normalized)
19
  const hasCyrillic = /\p{Script=Cyrillic}/u.test(normalized)
20
  if (hasLatin && hasCyrillic) {
21
    return { valid: false, error: "Mixed scripts not allowed" }
22
  }
23

24
  return { valid: true }
25
}

1
class TextProcessor {
2
  // Normalize and validate on input
3
  static ingest(raw: string): string {
4
    const normalized = raw.normalize("NFC")
5
    if (!normalized.isWellFormed()) {
6
      throw new Error("Ill-formed Unicode input")
7
    }
8
    return normalized
9
  }
10

11
  // Internal processing on normalized text
12
  static process(text: string): string {
13
    // Safe to use code point iteration here
14
    return [...text].map((c) => c.toUpperCase()).join("")
15
  }
16

17
  // Ensure well-formed output
18
  static emit(text: string): string {
19
    return text.toWellFormed() // Replace any lone surrogates
20
  }
21
}