feat(gatsby-source-wordpress): opt out of stale node checks (#37920)

* opt out of stale node checks

* update error message

---------

Co-authored-by: Michal Piechowiak <misiek.piechowiak@gmail.com>

packages/gatsby-source-wordpress/src/steps/persist-cached-images.ts

@@ -2,6 +2,7 @@ import { Step } from "./../utils/run-steps"
 import store from "~/store"
 import { getGatsbyApi } from "~/utils/get-gatsby-api"
 import { getPersistentCache } from "~/utils/cache"
+import { needToTouchNodes } from "~/utils/gatsby-features"
 
 const persistPreviouslyCachedImages: Step = async (): Promise<void> => {
   const { helpers, pluginOptions } = getGatsbyApi()
@@ -11,9 +12,11 @@ const persistPreviouslyCachedImages: Step = async (): Promise<void> => {
     `${pluginOptions.schema.typePrefix}MediaItem`
   )
 
-  // and touch them so they aren't garbage collected.
-  // we will remove them as needed when receiving DELETE events from WP
-  mediaItemNodes.forEach(node => helpers.actions.touchNode(node))
+  if (needToTouchNodes) {
+    // and if needed touch them so they aren't garbage collected.
+    // we will remove them as needed when receiving DELETE events from WP
+    mediaItemNodes.forEach(node => helpers.actions.touchNode(node))
+  }
 
   const imageNodeMetaByUrl = await getPersistentCache({
     key: `image-node-meta-by-url`,

packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/fetch-and-create-non-node-root-fields.js

@@ -5,6 +5,7 @@ import { createNodeWithSideEffects } from "./create-nodes"
 import fetchReferencedMediaItemsAndCreateNodes from "../fetch-nodes/fetch-referenced-media-items"
 import { CREATED_NODE_IDS } from "~/constants"
 import { getPersistentCache, setPersistentCache } from "~/utils/cache"
+import { needToTouchNodes } from "../../../utils/gatsby-features"
 
 const fetchAndCreateNonNodeRootFields = async () => {
   const state = store.getState()
@@ -69,20 +70,22 @@ const fetchAndCreateNonNodeRootFields = async () => {
       referencedMediaItemNodeIds: newMediaItemIds,
     })
 
-    const previouslyCachedNodeIds = await getPersistentCache({
-      key: CREATED_NODE_IDS,
-    })
-
-    const createdNodeIds = [
-      ...new Set([
-        ...(previouslyCachedNodeIds || []),
-        ...referencedMediaItemNodeIdsArray,
-      ]),
-    ]
-
-    // save the node id's so we can touch them on the next build
-    // so that we don't have to refetch all nodes
-    await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
+    if (needToTouchNodes) {
+      const previouslyCachedNodeIds = await getPersistentCache({
+        key: CREATED_NODE_IDS,
+      })
+
+      const createdNodeIds = [
+        ...new Set([
+          ...(previouslyCachedNodeIds || []),
+          ...referencedMediaItemNodeIdsArray,
+        ]),
+      ]
+
+      // save the node id's so we can touch them on the next build
+      // so that we don't have to refetch all nodes
+      await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
+    }
 
     store.dispatch.logger.stopActivityTimer({
       typeName: `MediaItems`,

packages/gatsby-source-wordpress/src/steps/source-nodes/fetch-nodes/fetch-nodes.js

@@ -15,6 +15,7 @@ import {
   setPersistentCache,
 } from "~/utils/cache"
 import { buildTypeName } from "../../create-schema-customization/helpers"
+import { needToTouchNodes } from "../../../utils/gatsby-features"
 
 /**
  * fetchWPGQLContentNodes
@@ -227,7 +228,9 @@ export const fetchAndCreateAllNodes = async () => {
     })
   }
 
-  // save the node id's so we can touch them on the next build
-  // so that we don't have to refetch all nodes
-  await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
+  if (needToTouchNodes) {
+    // save the node id's so we can touch them on the next build
+    // so that we don't have to refetch all nodes
+    await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
+  }
 }

packages/gatsby-source-wordpress/src/steps/source-nodes/index.ts

@@ -8,9 +8,14 @@ import store from "~/store"
 import fetchAndCreateNonNodeRootFields from "./create-nodes/fetch-and-create-non-node-root-fields"
 import { allowFileDownloaderProgressBarToClear } from "./create-nodes/create-remote-file-node/progress-bar-promise"
 import { sourcePreviews } from "~/steps/preview"
+import { hasStatefulSourceNodes } from "~/utils/gatsby-features"
 
 const sourceNodes: Step = async helpers => {
-  const { cache, webhookBody, refetchAll } = helpers
+  const { cache, webhookBody, refetchAll, actions } = helpers
+
+  if (hasStatefulSourceNodes) {
+    actions.enableStatefulSourceNodes()
+  }
 
   // fetch non-node root fields such as settings.
   // For now, we're refetching them on every build

packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/fetch-node-updates.js

@@ -3,8 +3,13 @@ import { fetchAndRunWpActions } from "./wp-actions"
 import { formatLogMessage } from "~/utils/format-log-message"
 import { getGatsbyApi } from "~/utils/get-gatsby-api"
 import { getPersistentCache } from "~/utils/cache"
+import { needToTouchNodes } from "../../../utils/gatsby-features"
 
 export const touchValidNodes = async () => {
+  if (!needToTouchNodes) {
+    return
+  }
+
   const { helpers } = getGatsbyApi()
   const { actions } = helpers
 

packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/wp-actions/delete.js

@@ -10,16 +10,15 @@ import { fetchGraphql } from "~/utils/fetch-graphql"
 import { getQueryInfoBySingleFieldName } from "../../helpers"
 import { CREATED_NODE_IDS } from "~/constants"
 import { setPersistentCache, getPersistentCache } from "~/utils/cache"
+import { needToTouchNodes } from "../../../../utils/gatsby-features"
 
-const wpActionDELETE = async ({
-  helpers,
-  // cachedNodeIds,
-  wpAction,
-}) => {
+const wpActionDELETE = async ({ helpers, wpAction }) => {
   const { reporter, actions, getNode } = helpers
 
   try {
-    const cachedNodeIds = await getPersistentCache({ key: CREATED_NODE_IDS })
+    const cachedNodeIds = needToTouchNodes
+      ? await getPersistentCache({ key: CREATED_NODE_IDS })
+      : null
 
     // get the node ID from the WPGQL id
     const nodeId = wpAction.referencedNodeGlobalRelayID
@@ -60,7 +59,7 @@ const wpActionDELETE = async ({
           wpStore: store,
         })) || {}
 
-      if (additionalNodeIds && additionalNodeIds.length) {
+      if (needToTouchNodes && additionalNodeIds && additionalNodeIds.length) {
         additionalNodeIds.forEach(id => cachedNodeIds.push(id))
       }
     }
@@ -81,12 +80,12 @@ const wpActionDELETE = async ({
       reporter.log(``)
     }
 
-    // Remove this from cached node id's so we don't try to touch it
-    const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)
+    if (needToTouchNodes) {
+      // Remove this from cached node id's so we don't try to touch it
+      const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)
 
-    await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })
-
-    // return validNodeIds
+      await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })
+    }
   } catch (e) {
     Object.entries(wpAction).forEach(([key, value]) => {
       reporter.warn(`${key} ${JSON.stringify(value)}`)

packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/wp-actions/update.js

@@ -19,6 +19,7 @@ import {
 } from "~/steps/create-schema-customization/helpers"
 import { processNode } from "~/steps/source-nodes/create-nodes/process-node"
 import { getPersistentCache, setPersistentCache } from "~/utils/cache"
+import { needToTouchNodes } from "../../../../utils/gatsby-features"
 
 export const fetchAndCreateSingleNode = async ({
   singleName,
@@ -170,7 +171,7 @@ export const createSingleNode = async ({
 
   const { typeInfo } = getQueryInfoBySingleFieldName(singleName)
 
-  if (!cachedNodeIds) {
+  if (!cachedNodeIds && needToTouchNodes) {
     cachedNodeIds = await getPersistentCache({ key: CREATED_NODE_IDS })
   }
 
@@ -259,12 +260,11 @@ export const createSingleNode = async ({
 
   if (remoteNode) {
     actions.createNode(remoteNode)
+  }
 
+  if (remoteNode && needToTouchNodes) {
     cachedNodeIds.push(remoteNode.id)
-
-    if (additionalNodeIds && additionalNodeIds.length) {
-      additionalNodeIds.forEach(id => cachedNodeIds.push(id))
-    }
+    additionalNodeIds?.forEach(id => cachedNodeIds.push(id))
 
     await setPersistentCache({ key: CREATED_NODE_IDS, value: cachedNodeIds })
   }
@@ -304,13 +304,18 @@ const wpActionUPDATE = async ({ helpers, wpAction }) => {
   const existingNode = await getNode(nodeId)
 
   if (wpAction.referencedNodeStatus !== `publish`) {
-    // if the post status isn't publish anymore, we need to remove the node
-    // by removing it from cached nodes so it's garbage collected by Gatsby
-    const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)
+    if (needToTouchNodes) {
+      // if the post status isn't publish anymore, we need to remove the node
+      // by removing it from cached nodes so it's garbage collected by Gatsby
+      const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)
 
-    await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })
+      await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })
+    }
 
     if (existingNode) {
+      // calling touchNode here ensures owners is populated before we delete.
+      // In some cases calling delete node without touching the node first throws an error. this is a bug in Gatsby that has been fixed but this code remains to be backwards compatible with earlier versions that have the bug still.
+      // TODO remove in the next major version
       await actions.touchNode(existingNode)
       await actions.deleteNode(existingNode)
       reportUpdate({ setAction: `DELETE` })

packages/gatsby-source-wordpress/src/utils/gatsby-features.ts

@@ -0,0 +1,4 @@
+import { hasFeature } from "gatsby-plugin-utils"
+
+export const hasStatefulSourceNodes = hasFeature(`stateful-source-nodes`)
+export const needToTouchNodes = !hasStatefulSourceNodes