fix: handle long blob names by truncating them (#182)

* chore: repro ENAMETOOLONG

* refactor: extract encodeBlobKey function from build and handlers into shared impl

* fix: truncate long key names

* fix: lower max length

* fix: use base64url to prevent clashes with blobstore urls

* use filename short enough to be writable by next.js, but long enough to trigger ENAMETOOLONG once base64'd

* fix a bug and add test

* fix: fix test

* fix: remove revalidation from test

* fix: use .cjs file so that vitest doesn't stumble

* chore: supress TS error

* fix: update cache-handler test

* fix: also move to base64+url in test suite

* fix: use encodeBlobKey across full repo

* fix: let's try .ts file again

* same fix for static.test.ts

* fix: use webcrypto

---------

Co-authored-by: Rob Stanford <me@robstanford.com>

Author: Skn0tt

src/build/content/static.ts

@@ -1,10 +1,10 @@
-import { Buffer } from 'node:buffer'
 import { existsSync } from 'node:fs'
 import { cp, mkdir, rename, rm } from 'node:fs/promises'
 import { join } from 'node:path'
 
 import glob from 'fast-glob'
 
+import { encodeBlobKey } from '../../shared/blobkey.js'
 import { PluginContext } from '../plugin-context.js'
 
 /**
@@ -24,8 +24,7 @@ export const copyStaticContent = async (ctx: PluginContext): Promise<void> => {
       paths
         .filter((path) => !paths.includes(`${path.slice(0, -5)}.json`))
         .map(async (path) => {
-          const key = Buffer.from(path).toString('base64')
-          await cp(join(srcDir, path), join(destDir, key), { recursive: true })
+          await cp(join(srcDir, path), join(destDir, await encodeBlobKey(path)), { recursive: true })
         }),
     )
   } catch (error) {

src/build/plugin-context.ts

@@ -1,4 +1,3 @@
-import { Buffer } from 'node:buffer'
 import { readFileSync } from 'node:fs'
 import { mkdir, readFile, writeFile } from 'node:fs/promises'
 // Here we need to actually import `resolve` from node:path as we want to resolve the paths
@@ -15,6 +14,8 @@ import type { PrerenderManifest, RoutesManifest } from 'next/dist/build/index.js
 import type { MiddlewareManifest } from 'next/dist/build/webpack/plugins/middleware-plugin.js'
 import type { NextConfigComplete } from 'next/dist/server/config-shared.js'
 
+import { encodeBlobKey } from '../shared/blobkey.js'
+
 const MODULE_DIR = fileURLToPath(new URL('.', import.meta.url))
 const PLUGIN_DIR = join(MODULE_DIR, '../..')
 
@@ -185,11 +186,10 @@ export class PluginContext {
   }
 
   /**
-   * Write a cache entry to the blob upload directory using
-   * base64 keys to avoid collisions with directories
+   * Write a cache entry to the blob upload directory.
    */
-  async writeCacheEntry(key: string, value: CacheValue): Promise<void> {
-    const path = join(this.blobDir, Buffer.from(key).toString('base64'))
+  async writeCacheEntry(route: string, value: CacheValue): Promise<void> {
+    const path = join(this.blobDir, await encodeBlobKey(route))
     const entry = JSON.stringify({
       lastModified: Date.now(),
       value,

src/run/handlers/cache.cts

@@ -15,9 +15,6 @@ import type {
   IncrementalCache,
 } from 'next/dist/server/lib/incremental-cache/index.js'
 
-// @ts-expect-error only type import of an ESM file will be removed in compiled version
-import type { CacheEntry } from '../../build/plugin-context.ts'
-
 type TagManifest = { revalidatedAt: number }
 
 // load the prerender manifest
@@ -30,10 +27,6 @@ function toRoute(cacheKey: string): string {
   return cacheKey.replace(/\/$/, '').replace(/\/index$/, '') || '/'
 }
 
-function encodeBlobKey(key: string) {
-  return Buffer.from(key.replace(/^\//, '')).toString('base64')
-}
-
 const fetchBeforeNextPatchedIt = globalThis.fetch
 
 export class NetlifyCacheHandler implements CacheHandler {
@@ -47,12 +40,17 @@ export class NetlifyCacheHandler implements CacheHandler {
     this.blobStore = getDeployStore({ fetch: fetchBeforeNextPatchedIt })
   }
 
+  private async encodeBlobKey(key: string) {
+    const { encodeBlobKey } = await import("../../shared/blobkey.js")
+    return await encodeBlobKey(key)
+  }
+
   async get(...args: Parameters<CacheHandler['get']>): ReturnType<CacheHandler['get']> {
     const [key, ctx = {}] = args
 
     console.debug(`[NetlifyCacheHandler.get]: ${key}`)
 
-    const blob = (await this.blobStore.get(encodeBlobKey(key), {
+    const blob = (await this.blobStore.get(await this.encodeBlobKey(key), {
       type: 'json',
     })) as CacheEntry | null
 
@@ -106,7 +104,7 @@ export class NetlifyCacheHandler implements CacheHandler {
 
     console.debug(`[NetlifyCacheHandler.set]: ${key}`)
 
-    await this.blobStore.setJSON(encodeBlobKey(key), {
+    await this.blobStore.setJSON(await this.encodeBlobKey(key), {
       lastModified: Date.now(),
       value: data,
     })
@@ -121,7 +119,7 @@ export class NetlifyCacheHandler implements CacheHandler {
     }
 
     try {
-      await this.blobStore.setJSON(encodeBlobKey(tag), data)
+      await this.blobStore.setJSON(await this.encodeBlobKey(tag), data)
     } catch (error) {
       console.warn(`Failed to update tag manifest for ${tag}`, error)
     }
@@ -145,7 +143,7 @@ export class NetlifyCacheHandler implements CacheHandler {
     const allManifests = await Promise.all(
       cacheTags.map(async (tag) => {
         const res = await this.blobStore
-          .get(encodeBlobKey(tag), { type: 'json' })
+          .get(await this.encodeBlobKey(tag), { type: 'json' })
           .then((value: TagManifest) => ({ [tag]: value }))
           .catch(console.error)
         return res || { [tag]: null }

src/run/headers.ts

@@ -1,8 +1,8 @@
-import { Buffer } from 'node:buffer'
-
 import { getDeployStore } from '@netlify/blobs'
 import type { NextConfigComplete } from 'next/dist/server/config-shared.js'
 
+import { encodeBlobKey } from '../shared/blobkey.js'
+
 import type { TagsManifest } from './config.js'
 
 interface NetlifyVaryValues {
@@ -74,10 +74,6 @@ export const setVaryHeaders = (
   headers.set(`netlify-vary`, generateNetlifyVaryValues(netlifyVaryValues))
 }
 
-function encodeBlobKey(key: string) {
-  return Buffer.from(key.replace(/^\//, '')).toString('base64')
-}
-
 /**
  * Change the date header to be the last-modified date of the blob. This means the CDN
  * will use the correct expiry time for the response. e.g. if the blob was last modified
@@ -90,7 +86,7 @@ export const adjustDateHeader = async (headers: Headers, request: Request) => {
     return
   }
   const path = new URL(request.url).pathname
-  const key = encodeBlobKey(path)
+  const key = await encodeBlobKey(path)
   const blobStore = getDeployStore()
   // TODO: use metadata for this
   const { lastModified } = (await blobStore.get(key, { type: 'json' })) ?? {}

src/run/next.cts

@@ -1,5 +1,4 @@
 import fs from 'fs/promises'
-import { Buffer } from 'node:buffer'
 import { relative, resolve } from 'path'
 
 import { getDeployStore } from '@netlify/blobs'
@@ -13,6 +12,8 @@ export async function getMockedRequestHandlers(...args: Parameters<typeof getReq
   const store = getDeployStore()
   const ofs = { ...fs }
 
+  const { encodeBlobKey } = await import("../shared/blobkey.js")
+
   async function readFileFallbackBlobStore(...fsargs: Parameters<FS['promises']['readFile']>) {
     const [path, options] = fsargs
     try {
@@ -23,8 +24,7 @@ export async function getMockedRequestHandlers(...args: Parameters<typeof getReq
       // only try to get .html files from the blob store
       if (typeof path === 'string' && path.endsWith('.html')) {
         const relPath = relative(resolve('.next/server/pages'), path)
-        const blobKey = Buffer.from(relPath).toString('base64')
-        const file = await store.get(blobKey)
+        const file = await store.get(await encodeBlobKey(relPath))
         if (file !== null) {
           return file
         }

src/shared/blobkey.test.ts

@@ -0,0 +1,30 @@
+import { Buffer } from 'node:buffer'
+
+import { expect, describe, it } from 'vitest'
+
+import { encodeBlobKey } from './blobkey.js'
+
+describe('encodeBlobKey', () => {
+  it('ignores leading slashes', async () => {
+    expect(await encodeBlobKey('/foo')).toEqual(await encodeBlobKey('foo'))
+  })
+
+  const longKey = 'long'.repeat(100)
+
+  it('truncates long keys to 180 characters', async () => {
+    expect(await encodeBlobKey(longKey)).toHaveLength(180)
+  })
+
+  it('truncated keys also ignore leading slashes', async () => {
+    expect(await encodeBlobKey(`/${longKey}`)).toEqual(await encodeBlobKey(longKey))
+  })
+
+  it('adds a differentiating hash to truncated keys', async () => {
+    expect(await encodeBlobKey(`${longKey}a`)).not.toEqual(await encodeBlobKey(`${longKey}b`))
+  })
+
+  it('truncated keys keep having a readable start', async () => {
+    const key = await encodeBlobKey(`/products/${longKey}`)
+    expect(Buffer.from(key, 'base64url').toString().startsWith('products/')).toBe(true)
+  })
+})

src/shared/blobkey.ts

@@ -0,0 +1,23 @@
+import { Buffer } from 'node:buffer'
+// need to import this since globalThis.crypto isn't available on Node 18
+import { webcrypto as crypto } from 'node:crypto'
+
+const maxLength = 180
+
+/**
+ * Takes a blob key and returns a safe key for the file system.
+ * The returned key is a base64url encoded string with a maximum length of 180 characters.
+ * Longer keys are truncated and appended with a hash to ensure uniqueness.
+ */
+export async function encodeBlobKey(key: string): Promise<string> {
+  const buffer = Buffer.from(key.replace(/^\//, '') || 'index')
+  const base64 = buffer.toString('base64url')
+  if (base64.length <= maxLength) {
+    return base64
+  }
+
+  const digest = await crypto.subtle.digest('SHA-256', buffer)
+  const hash = Buffer.from(digest).toString('base64url')
+
+  return `${base64.slice(0, maxLength - hash.length - 1)}-${hash}`
+}

tests/e2e/page-router.test.ts

@@ -86,3 +86,13 @@ test('requesting a non existing page route that needs to be fetched from the blo
 
   expect(await page.textContent('h1')).toBe('404')
 })
+
+test('requesting a page with a very long name works', async ({ page }) => {
+  const response = await page.goto(
+    new URL(
+      '/products/an-incredibly-long-product-name-thats-impressively-repetetively-needlessly-overdimensioned-and-should-be-shortened-to-less-than-255-characters-for-the-sake-of-seo-and-ux-and-first-and-foremost-for-gods-sake-but-nobody-wont-ever-read-this-anyway',
+      ctx.url,
+    ).href,
+  )
+  expect(response?.status()).toBe(200)
+})

tests/fixtures/page-router/pages/products/[slug].js

@@ -0,0 +1,31 @@
+const Product = ({ time }) => (
+  <div>
+    <p>
+      This page uses getStaticProps() and getStaticPaths() to pre-fetch a Product
+      <span data-testid="date-now">{time}</span>
+    </p>
+  </div>
+)
+
+export async function getStaticProps() {
+  return {
+    props: {
+      time: new Date().toISOString(),
+    },
+  }
+}
+
+export const getStaticPaths = () => {
+  return {
+    paths: [
+      {
+        params: {
+          slug: 'an-incredibly-long-product-name-thats-impressively-repetetively-needlessly-overdimensioned-and-should-be-shortened-to-less-than-255-characters-for-the-sake-of-seo-and-ux-and-first-and-foremost-for-gods-sake-but-nobody-wont-ever-read-this-anyway',
+        },
+      },
+    ],
+    fallback: false, // false or "blocking"
+  }
+}
+
+export default Product

tests/integration/cache-handler.test.ts

@@ -40,9 +40,11 @@ describe('page router', () => {
     console.timeEnd('runPlugin')
     // check if the blob entries where successful set on the build plugin
     const blobEntries = await getBlobEntries(ctx)
-    expect(blobEntries.map(({ key }) => decodeBlobKey(key)).sort()).toEqual([
+    expect(blobEntries.map(({ key }) => decodeBlobKey(key.substring(0, 50))).sort()).toEqual([
       '404.html',
       '500.html',
+      // the real key is much longer and ends in a hash, but we only assert on the first 50 chars to make it easier
+      'products/an-incredibly-long-product-n',
       'static/revalidate-automatic',
       'static/revalidate-manual',
       'static/revalidate-slow',

tests/integration/static.test.ts

@@ -38,9 +38,11 @@ test<FixtureTestContext>('requesting a non existing page route that needs to be
   await runPlugin(ctx)
 
   const entries = await getBlobEntries(ctx)
-  expect(entries.map(({ key }) => decodeBlobKey(key)).sort()).toEqual([
+  expect(entries.map(({ key }) => decodeBlobKey(key.substring(0, 50))).sort()).toEqual([
     '404.html',
     '500.html',
+    // the real key is much longer and ends in a hash, but we only assert on the first 50 chars to make it easier
+    'products/an-incredibly-long-product-n',
     'static/revalidate-automatic',
     'static/revalidate-manual',
     'static/revalidate-slow',

tests/integration/turborepo.test.ts

@@ -1,17 +1,13 @@
-import { load } from 'cheerio'
 import { getLogger } from 'lambda-local'
 import { v4 } from 'uuid'
 import { beforeEach, expect, test, vi } from 'vitest'
 import {
   createFixture,
-  invokeFunction,
   runPlugin,
   type FixtureTestContext,
 } from '../utils/fixture.js'
 import {
-  encodeBlobKey,
   generateRandomObjectID,
-  getBlobEntries,
   startMockBlobStore,
 } from '../utils/helpers.js'
 import { glob } from 'fast-glob'

tests/utils/helpers.ts

@@ -94,12 +94,12 @@ export const getBlobEntries = async (ctx: FixtureTestContext) => {
 /**
  * Converts a string to base64 blob key
  */
-export const encodeBlobKey = (key: string) => Buffer.from(key).toString('base64')
+export const encodeBlobKey = (key: string) => Buffer.from(key).toString('base64url')
 
 /**
  * Converts a base64 blob key to a string
  */
-export const decodeBlobKey = (key: string) => Buffer.from(key, 'base64').toString('utf-8')
+export const decodeBlobKey = (key: string) => Buffer.from(key, 'base64url').toString('utf-8')
 
 /**
  * Fake build utils that are passed to a build plugin execution