fix: dynamic not-prerendered routes revalidate tracking (#2771)

* fix: dynamic not-prerendered routes revalidate tracking

* fix: correct typeof check

* test: adjust cache-status assertions for stale responses serverd by durable

* test: ensure we don't prerender API responses for og image in test fixture

Author: pieh

src/run/handlers/cache.cts

@@ -183,37 +183,56 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
 
   private async injectEntryToPrerenderManifest(
     key: string,
-    revalidate: NetlifyCachedPageValue['revalidate'],
+    { revalidate, cacheControl }: Pick<NetlifyCachedPageValue, 'revalidate' | 'cacheControl'>,
   ) {
-    if (this.options.serverDistDir && (typeof revalidate === 'number' || revalidate === false)) {
+    if (
+      this.options.serverDistDir &&
+      (typeof revalidate === 'number' ||
+        revalidate === false ||
+        typeof cacheControl !== 'undefined')
+    ) {
       try {
         const { loadManifest } = await import('next/dist/server/load-manifest.js')
         const prerenderManifest = loadManifest(
           join(this.options.serverDistDir, '..', 'prerender-manifest.json'),
         ) as PrerenderManifest
 
-        try {
-          const { normalizePagePath } = await import(
-            'next/dist/shared/lib/page-path/normalize-page-path.js'
+        if (typeof cacheControl !== 'undefined') {
+          // instead of `revalidate` property, we might get `cacheControls` ( https://github.com/vercel/next.js/pull/76207 )
+          // then we need to keep track of revalidate values via SharedCacheControls
+          const { SharedCacheControls } = await import(
+            // @ts-expect-error supporting multiple next version, this module is not resolvable with currently used dev dependency
+            // eslint-disable-next-line import/no-unresolved, n/no-missing-import
+            'next/dist/server/lib/incremental-cache/shared-cache-controls.js'
           )
-
-          prerenderManifest.routes[key] = {
-            experimentalPPR: undefined,
-            dataRoute: posixJoin('/_next/data', `${normalizePagePath(key)}.json`),
-            srcRoute: null, // FIXME: provide actual source route, however, when dynamically appending it doesn't really matter
-            initialRevalidateSeconds: revalidate,
-            // Pages routes do not have a prefetch data route.
-            prefetchDataRoute: undefined,
+          const sharedCacheControls = new SharedCacheControls(prerenderManifest)
+          sharedCacheControls.set(key, cacheControl)
+        } else if (typeof revalidate === 'number' || revalidate === false) {
+          // if we don't get cacheControls, but we still get revalidate, it should mean we are before
+          // https://github.com/vercel/next.js/pull/76207
+          try {
+            const { normalizePagePath } = await import(
+              'next/dist/shared/lib/page-path/normalize-page-path.js'
+            )
+
+            prerenderManifest.routes[key] = {
+              experimentalPPR: undefined,
+              dataRoute: posixJoin('/_next/data', `${normalizePagePath(key)}.json`),
+              srcRoute: null, // FIXME: provide actual source route, however, when dynamically appending it doesn't really matter
+              initialRevalidateSeconds: revalidate,
+              // Pages routes do not have a prefetch data route.
+              prefetchDataRoute: undefined,
+            }
+          } catch {
+            // depending on Next.js version - prerender manifest might not be mutable
+            // https://github.com/vercel/next.js/pull/64313
+            // if it's not mutable we will try to use SharedRevalidateTimings ( https://github.com/vercel/next.js/pull/64370) instead
+            const { SharedRevalidateTimings } = await import(
+              'next/dist/server/lib/incremental-cache/shared-revalidate-timings.js'
+            )
+            const sharedRevalidateTimings = new SharedRevalidateTimings(prerenderManifest)
+            sharedRevalidateTimings.set(key, revalidate)
           }
-        } catch {
-          // depending on Next.js version - prerender manifest might not be mutable
-          // https://github.com/vercel/next.js/pull/64313
-          // if it's not mutable we will try to use SharedRevalidateTimings ( https://github.com/vercel/next.js/pull/64370) instead
-          const { SharedRevalidateTimings } = await import(
-            'next/dist/server/lib/incremental-cache/shared-revalidate-timings.js'
-          )
-          const sharedRevalidateTimings = new SharedRevalidateTimings(prerenderManifest)
-          sharedRevalidateTimings.set(key, revalidate)
         }
       } catch {}
     }
@@ -315,7 +334,7 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
 
           span.addEvent(blob.value?.kind, { lastModified: blob.lastModified, revalidate, ttl })
 
-          await this.injectEntryToPrerenderManifest(key, revalidate)
+          await this.injectEntryToPrerenderManifest(key, blob.value)
 
           return {
             lastModified: blob.lastModified,
@@ -327,7 +346,7 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
 
           span.addEvent(blob.value?.kind, { lastModified: blob.lastModified, revalidate, ttl })
 
-          await this.injectEntryToPrerenderManifest(key, revalidate)
+          await this.injectEntryToPrerenderManifest(key, blob.value)
 
           return {
             lastModified: blob.lastModified,
@@ -355,22 +374,25 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
     if (isCachedRouteValue(data)) {
       return {
         ...data,
-        revalidate: context.revalidate,
+        revalidate: context.revalidate ?? context.cacheControl?.revalidate,
+        cacheControl: context.cacheControl,
         body: data.body.toString('base64'),
       }
     }
 
     if (isCachedPageValue(data)) {
       return {
         ...data,
-        revalidate: context.revalidate,
+        revalidate: context.revalidate ?? context.cacheControl?.revalidate,
+        cacheControl: context.cacheControl,
       }
     }
 
     if (data?.kind === 'APP_PAGE') {
       return {
         ...data,
-        revalidate: context.revalidate,
+        revalidate: context.revalidate ?? context.cacheControl?.revalidate,
+        cacheControl: context.cacheControl,
         rscData: data.rscData?.toString('base64'),
       }
     }

src/shared/cache-types.cts

@@ -12,6 +12,11 @@ import type {
 
 export type { CacheHandlerContext } from 'next/dist/server/lib/incremental-cache/index.js'
 
+type CacheControl = {
+  revalidate: Parameters<CacheHandler['set']>[2]['revalidate']
+  expire: number | undefined
+}
+
 /**
  * Shape of the cache value that is returned from CacheHandler.get or passed to CacheHandler.set
  */
@@ -28,6 +33,7 @@ export type NetlifyCachedRouteValue = Omit<CachedRouteValueForMultipleVersions,
   // Next.js doesn't produce cache-control tag we use to generate cdn cache control
   // so store needed values as part of cached response data
   revalidate?: Parameters<CacheHandler['set']>[2]['revalidate']
+  cacheControl?: CacheControl
 }
 
 /**
@@ -50,6 +56,7 @@ export type NetlifyCachedAppPageValue = Omit<
   // Next.js stores rscData as buffer, while we store it as base64 encoded string
   rscData: string | undefined
   revalidate?: Parameters<CacheHandler['set']>[2]['revalidate']
+  cacheControl?: CacheControl
 }
 
 /**
@@ -64,6 +71,7 @@ type IncrementalCachedPageValueForMultipleVersions = Omit<IncrementalCachedPageV
  */
 export type NetlifyCachedPageValue = IncrementalCachedPageValueForMultipleVersions & {
   revalidate?: Parameters<CacheHandler['set']>[2]['revalidate']
+  cacheControl?: CacheControl
 }
 
 export type CachedFetchValueForMultipleVersions = Omit<CachedFetchValue, 'kind'> & {
@@ -131,4 +139,18 @@ type MapCacheHandlerClassMethod<T> = T extends (...args: infer Args) => infer Re
 
 type MapCacheHandlerClass<T> = { [K in keyof T]: MapCacheHandlerClassMethod<T[K]> }
 
-export type CacheHandlerForMultipleVersions = MapCacheHandlerClass<CacheHandler>
+type BaseCacheHandlerForMultipleVersions = MapCacheHandlerClass<CacheHandler>
+
+type CacheHandlerSetContext = Parameters<CacheHandler['set']>[2]
+
+type CacheHandlerSetContextForMultipleVersions = CacheHandlerSetContext & {
+  cacheControl?: CacheControl
+}
+
+export type CacheHandlerForMultipleVersions = BaseCacheHandlerForMultipleVersions & {
+  set: (
+    key: Parameters<BaseCacheHandlerForMultipleVersions['set']>[0],
+    value: Parameters<BaseCacheHandlerForMultipleVersions['set']>[1],
+    context: CacheHandlerSetContextForMultipleVersions,
+  ) => ReturnType<BaseCacheHandlerForMultipleVersions['set']>
+}

tests/e2e/page-router.test.ts

@@ -413,12 +413,12 @@ test.describe('Simple Page Router (no basePath, no i18n)', () => {
     expect(date1.localeCompare(beforeFirstFetch)).toBeGreaterThan(0)
 
     // allow page to get stale
-    await page.waitForTimeout(60_000)
+    await page.waitForTimeout(61_000)
 
     const response2 = await page.goto(new URL(pathname, pageRouter.url).href)
     expect(response2?.status()).toBe(200)
     expect(response2?.headers()['cache-status']).toMatch(
-      /"Netlify (Edge|Durable)"; hit; fwd=stale/m,
+      /("Netlify Edge"; hit; fwd=stale|"Netlify Durable"; hit; ttl=-[0-9]+)/m,
     )
     expect(response2?.headers()['netlify-cdn-cache-control']).toMatch(
       /s-maxage=60, stale-while-revalidate=[0-9]+, durable/,
@@ -436,8 +436,8 @@ test.describe('Simple Page Router (no basePath, no i18n)', () => {
     const response3 = await page.goto(new URL(pathname, pageRouter.url).href)
     expect(response3?.status()).toBe(200)
     expect(response3?.headers()['cache-status']).toMatch(
-      // hit, without being followed by ';fwd=stale'
-      /"Netlify (Edge|Durable)"; hit(?!; fwd=stale)/m,
+      // hit, without being followed by ';fwd=stale' for edge or negative TTL for durable, optionally with fwd=stale
+      /("Netlify Edge"; hit(?!; fwd=stale)|"Netlify Durable"; hit(?!; ttl=-[0-9]+))/m,
     )
     expect(response3?.headers()['netlify-cdn-cache-control']).toMatch(
       /s-maxage=60, stale-while-revalidate=[0-9]+, durable/,

tests/fixtures/wasm-src/src/app/og-node/route.js

@@ -6,3 +6,5 @@ export async function GET() {
     height: 630,
   })
 }
+
+export const dynamic = 'force-dynamic'

tests/fixtures/wasm-src/src/app/og/route.js

@@ -8,3 +8,5 @@ export async function GET() {
 }
 
 export const runtime = 'edge'
+
+export const dynamic = 'force-dynamic'

tests/fixtures/wasm/app/og-node/route.js

@@ -6,3 +6,5 @@ export async function GET() {
     height: 630,
   })
 }
+
+export const dynamic = 'force-dynamic'

tests/fixtures/wasm/app/og/route.js

@@ -8,3 +8,5 @@ export async function GET() {
 }
 
 export const runtime = 'edge'
+
+export const dynamic = 'force-dynamic'