Design an Infinite Feed

1
SELECT * FROM posts ORDER BY created_at DESC LIMIT 20 OFFSET 40;

1
┌─────────────────────────────────────┐
2
│         Scrollable Container         │
3
├─────────────────────────────────────┤
4
│  [Top Sentinel - for bidirectional] │
5
│  ┌─────────────────────────────────┐│
6
│  │         Item 1                  ││
7
│  ├─────────────────────────────────┤│
8
│  │         Item 2                  ││
9
│  ├─────────────────────────────────┤│
10
│  │         ...                     ││
11
│  ├─────────────────────────────────┤│
12
│  │         Item N                  ││
13
│  └─────────────────────────────────┘│
14
│  [Bottom Sentinel - triggers load]  │
15
└─────────────────────────────────────┘

2 collapsed lines
1
type FeedItem = { id: string; content: string; createdAt: string }
2
type PageResponse = { items: FeedItem[]; nextCursor: string | null }
3

4
function createFeedObserver(onLoadMore: () => void, options: { rootMargin?: string; threshold?: number } = {}) {
5
  const { rootMargin = "0px 0px 200px 0px", threshold = 0 } = options
6

7
  return new IntersectionObserver(
8
    (entries) => {
9
      const [entry] = entries
10
      if (entry.isIntersecting) {
11
        onLoadMore()
12
      }
13
    },
14
    { rootMargin, threshold },
15
  )
16
}
17

18
// Usage
19
const sentinel = document.getElementById("load-more-sentinel")
20
const observer = createFeedObserver(() => fetchNextPage(currentCursor))
21
observer.observe(sentinel)

1
┌──────────────────────────────────────┐
2
│    Scroll Container (fixed height)    │
3
│  ┌────────────────────────────────┐  │
4
│  │   Spacer: items 0-14           │  │  ← height = 14 × itemHeight
5
│  ├────────────────────────────────┤  │
6
│  │   Rendered: items 15-25        │  │  ← actual DOM nodes
7
│  ├────────────────────────────────┤  │
8
│  │   Spacer: items 26-999         │  │  ← height = remaining × itemHeight
9
│  └────────────────────────────────┘  │
10
│  [Sentinel: triggers load at item 990]│
11
└──────────────────────────────────────┘

8 collapsed lines
1
import { useVirtualizer } from '@tanstack/react-virtual';
2
import { useInfiniteQuery } from '@tanstack/react-query';
3
import { useRef, useEffect } from 'react';
4

5
type FeedItem = { id: string; content: string };
6
type PageResponse = { items: FeedItem[]; nextCursor: string | null };
7

8
const ITEMS_PER_PAGE = 20;
9

10
function VirtualizedFeed() {
11
  const parentRef = useRef<HTMLDivElement>(null);
12

13
  const { data, fetchNextPage, hasNextPage, isFetchingNextPage } =
14
    useInfiniteQuery({
15
      queryKey: ['feed'],
16
      queryFn: ({ pageParam }) => fetchFeedPage(pageParam),
17
      getNextPageParam: (lastPage) => lastPage.nextCursor,
18
      initialPageParam: null as string | null,
19
    });
20

21
  const allItems = data?.pages.flatMap((page) => page.items) ?? [];
22

23
  const virtualizer = useVirtualizer({
24
    count: hasNextPage ? allItems.length + 1 : allItems.length,
25
    getScrollElement: () => parentRef.current,
26
    estimateSize: () => 100, // Estimated item height
27
    overscan: 5,
18 collapsed lines
28
  });
29

30
  // Trigger load when approaching end
31
  useEffect(() => {
32
    const lastItem = virtualizer.getVirtualItems().at(-1);
33
    if (!lastItem) return;
34

35
    if (
36
      lastItem.index >= allItems.length - 1 &&
37
      hasNextPage &&
38
      !isFetchingNextPage
39
    ) {
40
      fetchNextPage();
41
    }
42
  }, [virtualizer.getVirtualItems(), hasNextPage, isFetchingNextPage]);
43

44
  return (
45
    <div ref={parentRef} style={{ height: '100vh', overflow: 'auto' }}>
46
      <div style={{ height: virtualizer.getTotalSize(), position: 'relative' }}>
47
        {virtualizer.getVirtualItems().map((virtualItem) => (
48
          <div
49
            key={virtualItem.key}
50
            style={{
51
              position: 'absolute',
52
              top: 0,
53
              left: 0,
54
              width: '100%',
55
              transform: `translateY(${virtualItem.start}px)`,
56
            }}
57
          >
58
            {virtualItem.index < allItems.length ? (
59
              <FeedItemComponent item={allItems[virtualItem.index]} />
60
            ) : (
61
              <LoadingPlaceholder />
62
            )}
63
          </div>
64
        ))}
65
      </div>
66
    </div>
67
  );
68
}

5 collapsed lines
1
import { useState } from 'react';
2

3
type LoadMode = 'auto' | 'manual';
4
type FeedItem = { id: string; content: string };
5

6
function HybridFeed({ userPreference }: { userPreference: LoadMode }) {
7
  const [mode, setMode] = useState(userPreference);
8
  const { data, fetchNextPage, hasNextPage, isFetching } = useInfiniteQuery({
9
    /* ... */
10
  });
11

12
  return (
13
    <div>
14
      <ModeToggle mode={mode} onChange={setMode} />
15

16
      <FeedList items={data?.pages.flatMap((p) => p.items) ?? []} />
17

18
      {hasNextPage && mode === 'manual' && (
19
        <button onClick={() => fetchNextPage()} disabled={isFetching}>
20
          Load more
21
        </button>
22
      )}
23

24
      {hasNextPage && mode === 'auto' && (
25
        <IntersectionSentinel onVisible={() => fetchNextPage()} />
26
      )}
27
    </div>
28
  );
29
}

1
function encodeCursor(createdAt: Date, id: string): string {
2
  return Buffer.from(JSON.stringify({ createdAt: createdAt.toISOString(), id })).toString("base64")
3
}
4

5
function decodeCursor(cursor: string): { createdAt: Date; id: string } {
6
  const { createdAt, id } = JSON.parse(Buffer.from(cursor, "base64").toString())
7
  return { createdAt: new Date(createdAt), id }
8
}

1
-- Keyset pagination with compound cursor
2
SELECT id, created_at, content FROM posts
3
WHERE (created_at, id) < ('2024-01-15T12:00:00Z', 'abc123')
4
ORDER BY created_at DESC, id DESC
5
LIMIT 20;

3 collapsed lines
1
import crypto from "crypto"
2

3
const SECRET = process.env.CURSOR_SECRET!
4

5
function signCursor(payload: object): string {
6
  const data = JSON.stringify(payload)
7
  const signature = crypto.createHmac("sha256", SECRET).update(data).digest("hex")
8
  return Buffer.from(`${data}|${signature}`).toString("base64")
9
}
10

11
function verifyCursor(cursor: string): object | null {
12
  const decoded = Buffer.from(cursor, "base64").toString()
13
  const [data, signature] = decoded.split("|")
14
  const expected = crypto.createHmac("sha256", SECRET).update(data).digest("hex")
15

16
  if (!crypto.timingSafeEqual(Buffer.from(signature), Buffer.from(expected))) {
17
    return null // Tampered
18
  }
19
  return JSON.parse(data)
20
}

1
interface PaginatedResponse<T> {
2
  items: T[]
3
  nextCursor: string | null // null indicates end
4
  prevCursor?: string | null // For bidirectional
5
  hasMore: boolean // Explicit flag
6
  totalCount?: number // Optional, expensive to compute
7
}

1
const { data, fetchNextPage, fetchPreviousPage, hasNextPage, hasPreviousPage } = useInfiniteQuery({
2
  queryKey: ["feed"],
3
  queryFn: ({ pageParam }) => fetchPage(pageParam),
4
  getNextPageParam: (lastPage) => lastPage.nextCursor,
5
  getPreviousPageParam: (firstPage) => firstPage.prevCursor,
6
  initialPageParam: { cursor: startCursor, direction: "forward" },
7
})

1
const observer = new IntersectionObserver(callback, {
2
  root: null, // null = viewport
3
  rootMargin: "0px 0px 200px 0px", // Trigger 200px before visible
4
  threshold: 0, // Fire as soon as any part intersects
5
})

2 collapsed lines
1
type Direction = "forward" | "backward"
2
type LoadHandler = (direction: Direction) => void
3

4
function createBidirectionalObserver(onLoad: LoadHandler) {
5
  return new IntersectionObserver(
6
    (entries) => {
7
      entries.forEach((entry) => {
8
        if (!entry.isIntersecting) return
9

10
        const direction = entry.target.id === "top-sentinel" ? "backward" : "forward"
11
        onLoad(direction)
12
      })
13
    },
14
    { rootMargin: "200px" },
15
  )
16
}

4 collapsed lines
1
type AdaptiveConfig = {
2
  avgFetchTime: number
3
  scrollVelocity: number
4
}
5

6
function calculateRootMargin({ avgFetchTime, scrollVelocity }: AdaptiveConfig): string {
7
  // Load when user will reach end in ~2x fetch time
8
  const pixelsNeeded = scrollVelocity * (avgFetchTime / 1000) * 2
9
  const minMargin = 200
10
  const maxMargin = 800
11

12
  const margin = Math.max(minMargin, Math.min(maxMargin, pixelsNeeded))
13
  return `0px 0px ${margin}px 0px`
14
}

3 collapsed lines
1
let currentController: AbortController | null = null
2
let isLoading = false
3

4
async function fetchNextPageSafe(cursor: string) {
5
  if (isLoading) return // Already fetching
6

7
  // Cancel any stale request
8
  currentController?.abort()
9
  currentController = new AbortController()
10
  isLoading = true
11

12
  try {
13
    const response = await fetch(`/api/feed?cursor=${cursor}`, {
14
      signal: currentController.signal,
15
    })
16
    return await response.json()
17
  } catch (error) {
18
    if (error instanceof Error && error.name === "AbortError") {
19
      return null // Cancelled, ignore
20
    }
21
    throw error
22
  } finally {
23
    isLoading = false
24
  }
25
}

1
const startIndex = Math.floor(scrollTop / itemHeight)
2
const endIndex = Math.min(startIndex + Math.ceil(viewportHeight / itemHeight) + overscan, totalItems)
3
const offsetY = startIndex * itemHeight

4 collapsed lines
1
import { FixedSizeList } from 'react-window';
2

3
type ItemData = { items: Array<{ id: string; content: string }> };
4

5
function FeedList({ items }: { items: ItemData['items'] }) {
6
  return (
7
    <FixedSizeList
8
      height={600}
9
      itemCount={items.length}
10
      itemSize={80}
11
      width="100%"
12
      itemData={{ items }}
13
    >
14
      {({ index, style, data }) => (
15
        <div style={style}>
16
          <FeedItem item={data.items[index]} />
17
        </div>
18
      )}
19
    </FixedSizeList>
20
  );
21
}

3 collapsed lines
1
import { Virtuoso } from 'react-virtuoso';
2

3
type FeedItem = { id: string; content: string; hasImage: boolean };
4

5
function VariableFeed({ items }: { items: FeedItem[] }) {
6
  return (
7
    <Virtuoso
8
      data={items}
9
      itemContent={(index, item) => <FeedItem item={item} />}
10
      endReached={() => fetchNextPage()}
11
      overscan={200} // Pixels, not items
12
      increaseViewportBy={{ top: 200, bottom: 200 }}
13
    />
14
  );
15
}

1
{
2
  pages: [
3
    { items: [{ id: "1", author: { id: "a", name: "Alice" } }] },
4
    { items: [{ id: "2", author: { id: "a", name: "Alice" } }] },
5
  ]
6
}

1
{
2
  items: {
3
    byId: { '1': { id: '1', authorId: 'a' }, '2': { id: '2', authorId: 'a' } },
4
    allIds: ['1', '2']
5
  },
6
  authors: {
7
    byId: { 'a': { id: 'a', name: 'Alice' } }
8
  },
9
  feed: { itemIds: ['1', '2'], nextCursor: 'xyz' }
10
}

1
useInfiniteQuery({
2
  queryKey: ["feed"],
3
  queryFn: fetchPage,
4
  maxPages: 5, // Keep only 5 pages
5
  getNextPageParam: (last) => last.nextCursor,
6
  getPreviousPageParam: (first) => first.prevCursor,
7
})

1
function maybeEvictItem(item: FeedItem, isVisible: boolean): FeedItem | Tombstone {
2
  if (!isVisible && item.mediaUrls?.length) {
3
    return { id: item.id, tombstoned: true }
4
  }
5
  return item
6
}

5 collapsed lines
1
import { useState, useCallback } from 'react';
2

3
type FeedItem = { id: string };
4
type NewItemsBannerProps = { count: number; onShow: () => void };
5

6
function Feed() {
7
  const [newItemIds, setNewItemIds] = useState<string[]>([]);
8

9
  // Poll or WebSocket for new items
10
  useEffect(() => {
11
    const interval = setInterval(async () => {
12
      const latestId = items[0]?.id;
13
      const newItems = await checkForNewItems(latestId);
14
      if (newItems.length) {
15
        setNewItemIds((prev) => [...newItems.map((i) => i.id), ...prev]);
16
      }
17
    }, 30000);
18
    return () => clearInterval(interval);
19
  }, [items]);
20

21
  const showNewItems = useCallback(() => {
22
    prependItems(newItemIds);
23
    scrollToTop();
24
    setNewItemIds([]);
25
  }, [newItemIds]);
26

27
  return (
28
    <>
29
      {newItemIds.length > 0 && (
30
        <button className="new-items-banner" onClick={showNewItems}>
31
          {newItemIds.length} new posts
32
        </button>
33
      )}
34
      <FeedList items={items} />
35
    </>
36
  );
37
}

4 collapsed lines
1
import { useRef, useState, TouchEvent } from "react"
2

3
type PullState = "idle" | "pulling" | "refreshing"
4

5
function usePullToRefresh(onRefresh: () => Promise<void>) {
6
  const [state, setState] = useState<PullState>("idle")
7
  const [pullDistance, setPullDistance] = useState(0)
8
  const startY = useRef(0)
9

10
  const handleTouchStart = (e: TouchEvent) => {
11
    if (window.scrollY === 0) {
12
      startY.current = e.touches[0].pageY
13
      setState("pulling")
14
    }
15
  }
16

17
  const handleTouchMove = (e: TouchEvent) => {
18
    if (state !== "pulling") return
19
    const distance = e.touches[0].pageY - startY.current
20
    setPullDistance(Math.max(0, Math.min(distance, 150)))
21
  }
22

23
  const handleTouchEnd = async () => {
24
    if (pullDistance > 100) {
25
      setState("refreshing")
26
      await onRefresh()
27
    }
28
    setState("idle")
29
    setPullDistance(0)
30
  }
31

32
  return { state, pullDistance, handleTouchStart, handleTouchMove, handleTouchEnd }
33
}

1
body {
2
  overscroll-behavior-y: contain;
3
}

1
function detectGap(pages: Page[]): { gapAfterPage: number; cursor: string } | null {
2
  for (let i = 1; i < pages.length; i++) {
3
    const prevEnd = pages[i - 1].endCursor
4
    const currStart = pages[i].startCursor
5
    if (prevEnd !== currStart) {
6
      return { gapAfterPage: i - 1, cursor: prevEnd }
7
    }
8
  }
9
  return null
10
}

1
function calculateDelay(attempt: number): number {
2
  const baseDelay = 100
3
  const maxDelay = 30000
4
  const exponential = baseDelay * Math.pow(2, attempt)
5
  const jitter = Math.random() * 1000
6
  return Math.min(exponential + jitter, maxDelay)
7
}
8

9
// Delays: ~100ms, ~200ms, ~400ms, ~800ms... up to 30s

5 collapsed lines
1
type FeedState =
2
  | { status: 'loading' }
3
  | { status: 'error'; error: Error; retryCount: number }
4
  | { status: 'success' };
5

6
function FeedWithErrorBoundary() {
7
  const { data, error, refetch, isError, failureCount } = useInfiniteQuery({
8
    queryKey: ['feed'],
9
    queryFn: fetchPage,
10
    retry: 3,
11
    retryDelay: (attempt) => calculateDelay(attempt),
12
  });
13

14
  if (isError) {
15
    return (
16
      <div role="alert">
17
        <p>Failed to load feed: {error.message}</p>
18
        <button onClick={() => refetch()}>
19
          Retry ({failureCount}/3 attempts)
20
        </button>
21
      </div>
22
    );
23
  }
24

25
  return <FeedList pages={data?.pages ?? []} />;
26
}

1
const { data, isStale, isFetching } = useInfiniteQuery({
2
  queryKey: ['feed'],
3
  queryFn: fetchPage,
4
  staleTime: 60 * 1000,        // Fresh for 1 minute
5
  gcTime: 5 * 60 * 1000,       // Keep in cache for 5 minutes
6
  refetchOnWindowFocus: true,
7
  refetchOnReconnect: true,
8
});
9

10
return (
11
  <>
12
    {isStale && isFetching && <RefreshingIndicator />}
13
    <FeedList pages={data?.pages ?? []} />
14
  </>
15
);

1
// Disable browser's automatic (often wrong) restoration
2
if ("scrollRestoration" in history) {
3
  history.scrollRestoration = "manual"
4
}

5 collapsed lines
1
type ScrollState = {
2
  pageCount: number
3
  scrollY: number
4
  anchorItemId: string
5
  timestamp: number
6
}
7

8
function saveScrollState(key: string, state: ScrollState) {
9
  sessionStorage.setItem(key, JSON.stringify(state))
10
}
11

12
function restoreScrollState(key: string): ScrollState | null {
13
  const saved = sessionStorage.getItem(key)
14
  if (!saved) return null
15

16
  const state: ScrollState = JSON.parse(saved)
17

18
  // Expire after 5 minutes
19
  if (Date.now() - state.timestamp > 300000) {
20
    sessionStorage.removeItem(key)
21
    return null
22
  }
23

24
  return state
25
}
26

27
function useScrollRestoration(feedKey: string, setSize: (size: number) => void) {
28
  useEffect(() => {
29
    const state = restoreScrollState(feedKey)
30
    if (!state) return
31

32
    // Restore page count first
33
    setSize(state.pageCount)
34

35
    // Then scroll after render
36
    requestAnimationFrame(() => {
37
      window.scrollTo(0, state.scrollY)
38
      sessionStorage.removeItem(feedKey)
39
    })
40
  }, [feedKey, setSize])
41
}

1
// TanStack Virtual scroll restoration
2
const virtualizer = useVirtualizer({
3
  count: items.length,
4
  getScrollElement: () => parentRef.current,
5
  estimateSize: () => 100,
6
})
7

8
useEffect(() => {
9
  const savedIndex = sessionStorage.getItem("lastVisibleIndex")
10
  if (savedIndex) {
11
    virtualizer.scrollToIndex(parseInt(savedIndex, 10), {
12
      align: "start",
13
      behavior: "auto",
14
    })
15
  }
16
}, [])

1
<div role="feed" aria-busy="false" aria-label="News feed">
2
  <article role="article" aria-labelledby="post-1-title" aria-posinset="1" aria-setsize="-1" tabindex="0">
3
    <h2 id="post-1-title">Post Title</h2>
4
    <p>Post content...</p>
5
  </article>
6
  <!-- More articles -->
7
</div>

1
function handleFeedKeyDown(event: KeyboardEvent) {
2
  const current = document.activeElement?.closest('[role="article"]')
3
  if (!current) return
4

5
  switch (event.key) {
6
    case "ArrowDown":
7
    case "j":
8
      focusNextArticle(current)
9
      event.preventDefault()
10
      break
11
    case "ArrowUp":
12
    case "k":
13
      focusPreviousArticle(current)
14
      event.preventDefault()
15
      break
16
    case "Home":
17
      focusFirstArticle()
18
      event.preventDefault()
19
      break
20
    case "End":
21
      focusLastArticle()
22
      event.preventDefault()
23
      break
24
  }
25
}

1
<div role="status" aria-live="polite" class="visually-hidden">Loading more posts...</div>

1
// Bad: Forces layout recalculation
2
element.style.width = "100px"
3
const height = element.offsetHeight // Forced reflow
4

5
// Good: Batch reads, then writes
6
const height = element.offsetHeight
7
element.style.width = "100px"

1
element.addEventListener("scroll", handler, { passive: true })

1
function processInBackground(items: Item[]) {
2
  let index = 0
3

4
  function processChunk(deadline: IdleDeadline) {
5
    while (deadline.timeRemaining() > 0 && index < items.length) {
6
      processItem(items[index])
7
      index++
8
    }
9

10
    if (index < items.length) {
11
      requestIdleCallback(processChunk)
12
    }
13
  }
14

15
  requestIdleCallback(processChunk, { timeout: 2000 })
16
}

1
const requestIdleCallback =
2
  window.requestIdleCallback ||
3
  ((cb: IdleRequestCallback) => setTimeout(() => cb({ timeRemaining: () => 0 } as IdleDeadline), 1))

1
function checkMemoryPressure(): boolean {
2
  if (!performance.memory) return false
3

4
  const { usedJSHeapSize, jsHeapSizeLimit } = performance.memory
5
  const usage = usedJSHeapSize / jsHeapSizeLimit
6

7
  if (usage > 0.8) {
8
    console.warn("High memory usage:", (usage * 100).toFixed(1) + "%")
9
    return true
10
  }
11
  return false
12
}
13

14
// Trigger cleanup when memory is high
15
if (checkMemoryPressure()) {
16
  trimOldPages()
17
}