.circleci/config.yml

@@ -150,7 +150,7 @@ commands:
         default: false
       trigger_pattern:
         type: string
-        default: "packages/*|.circleci/*"
+        default: "packages/*|.circleci/*|scripts/e2e-test.sh"
       test_path:
         type: string
       test_command:
@@ -168,6 +168,9 @@ commands:
             - run: ./scripts/assert-changed-files.sh "<< parameters.trigger_pattern >>|<< parameters.test_path >>/*"
 
       - <<: *attach_to_bootstrap
+      - run:
+          name: Install gatsby-dev
+          command: yarn global add gatsby-dev-cli
       - run:
           name: Run tests (using defaults)
           command: ./scripts/e2e-test.sh "<< parameters.test_path >>" "<< parameters.test_command >>"
@@ -179,7 +182,7 @@ jobs:
     executor: node
     steps:
       - checkout
-      - run: ./scripts/assert-changed-files.sh "packages/*|(e2e|integration)-tests/*|.circleci/*|yarn.lock"
+      - run: ./scripts/assert-changed-files.sh "packages/*|(e2e|integration)-tests/*|.circleci/*|scripts/e2e-test.sh|yarn.lock"
       - <<: *restore_cache
       - <<: *install_node_modules
       - <<: *check_lockfile
@@ -228,8 +231,16 @@ jobs:
     <<: *test_template
 
   integration_tests_gatsby_source_wordpress:
-    executor: node
+    machine: true
     steps:
+      - run:
+          command: |
+            echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV
+            echo ' [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> $BASH_ENV
+      - run: nvm install v12
+      - run: nvm alias default v12
+      - run: nvm use v12
+      - run: npm i -g yarn@1.22.10
       - e2e-test:
           test_path: integration-tests/gatsby-source-wordpress
 
@@ -256,7 +267,7 @@ jobs:
     steps:
       - e2e-test:
           test_path: integration-tests/gatsby-cli
-          trigger_pattern: packages/gatsby-cli/*|packages/gatsby/*|integration-tests/gatsby-cli/*
+          trigger_pattern: packages/gatsby-cli/*|packages/gatsby/*|integration-tests/gatsby-cli/*|scripts/e2e-test.sh
 
   integration_tests_structured_logging:
     executor: node
@@ -369,8 +380,8 @@ jobs:
       CYPRESS_RECORD_KEY: 3fb49000-4143-4bd8-9ab4-219389060910
     steps:
       - e2e-test:
-          test_path: e2e-tests/themes/development-runtime
-          test_command: yarn test
+          test_path: e2e-tests/themes
+          test_command: cd development-runtime; yarn test
 
   themes_e2e_tests_production_runtime:
     <<: *e2e-executor
@@ -379,8 +390,8 @@ jobs:
       CYPRESS_RECORD_KEY: e4e7b3b8-e1e7-4a74-a0c9-9ac76585236b
     steps:
       - e2e-test:
-          test_path: e2e-tests/themes/production-runtime
-          test_command: yarn test
+          test_path: e2e-tests/themes
+          test_command: cd production-runtime; yarn test
 
   mdx_e2e_tests:
     <<: *e2e-executor
@@ -482,7 +493,7 @@ jobs:
     steps:
       - checkout
       - run:
-          command: ./scripts/assert-changed-files.sh "packages/*|(e2e|integration)-tests/*|.circleci/*|yarn.lock"
+          command: ./scripts/assert-changed-files.sh "packages/*|(e2e|integration)-tests/*|.circleci/*|scripts/e2e-test.sh|yarn.lock"
           shell: bash.exe
       # Restoring cache takes as long as installing node modules, so skipping
       # - restore_cache:

.github/ISSUE_TEMPLATE/bug_report.md

@@ -12,6 +12,7 @@ labels: "type: bug"
   - How to File an Issue: https://www.gatsbyjs.com/contributing/how-to-file-an-issue/
 
   Before opening a new issue, please search existing issues: https://github.com/gatsbyjs/gatsby/issues
+  And existing Discussions: https://github.com/gatsbyjs/gatsby/discussions
 -->
 
 ## Description
@@ -34,4 +35,4 @@ What happened.
 
 ### Environment
 
-Run `gatsby info --clipboard` in your project directory and paste the output here.
+Run `gatsby info --clipboard` in your project directory and paste the output here. Also name any `flags` you use inside `gatsby-config.js`.

README.md

@@ -175,6 +175,6 @@ Licensed under the [MIT License](./LICENSE).
 
 ## 💜 Thanks
 
-Thanks goes out to all our many contributors creating plugins, starters, videos, and blog posts. And a special appreciation for our community members helping with issues and PRs, or answering questions on Discord and GitHub Discussions.
+Thanks go out to all our many contributors creating plugins, starters, videos, and blog posts. And a special appreciation for our community members helping with issues and PRs, or answering questions on Discord and GitHub Discussions.
 
 A big part of what makes Gatsby great is each and every one of you in the community. Your contributions enrich the Gatsby experience and make it better every day.

benchmarks/create-pages/package.json

@@ -9,9 +9,9 @@
     "serve": "gatsby serve"
   },
   "dependencies": {
-    "gatsby": "^2.19.5",
-    "react": "^16.12.0",
-    "react-dom": "^16.12.0"
+    "gatsby": "^3.4.0",
+    "react": "^17.0.2",
+    "react-dom": "^17.0.2"
   },
   "devDependencies": {
     "gatsby-plugin-benchmark-reporting": "*"

benchmarks/query-filters-sort/README.md

@@ -0,0 +1,43 @@
+# Filters and sort benchmark
+
+Stress tests various query filters (with and without sorting by random value).
+
+# Usage
+
+```shell
+NUM_NODES=1000 NUM_PAGES=1000 FILTER=eq SORT=1 TEXT=1 yarn bench
+```
+
+Explanation:
+
+- `FILTER`: one of `eq`, `gt`, `gt-lt`, `in`, `lt`, `ne`, `nin`, `regex`, `elemMatch-eq` (default: `eq`).
+  See `src/templates` for specific queries.
+- `SORT`: when set, the benchmark will also add sorting to the query (by random integer value)
+- `TEXT`: all nodes are lightweight by default (just 5 fields with numeric values or short strings).\
+  When settings this env variable - each node will get additional field with 4k of text
+  (useful for memory usage measurements)
+- `NUM_NODES`: the number of nodes created (1000 by default)
+- `NUM_PAGES`: the number of pages created (1000 by default, must be >= `NUM_NODES`)
+
+# Example
+
+Let's figure out time complexity of `gt` filter. To make this happen - let's run the benchmark
+3 times with the same number of pages but growing number of nodes:
+
+### run 1:
+
+```shell
+NUM_NODES=1000 FILTER=gt yarn bench
+```
+
+### run 2:
+
+```shell
+NUM_NODES=10000 FILTER=gt yarn bench
+```
+
+### run 3:
+
+```shell
+NUM_NODES=100000 FILTER=gt yarn bench
+```

benchmarks/query-filters-sort/gatsby-config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  siteMetadata: {
+    title: `Gatsby Benchmark Filters & Sort`,
+    description: `The filters and sort benchmark`,
+    author: `@gatsbyjs`,
+  },
+  // plugins: [`gatsby-plugin-benchmark-reporting`],
+}

benchmarks/query-filters-sort/gatsby-node.js

@@ -0,0 +1,87 @@
+const NUM_PAGES = parseInt(process.env.NUM_PAGES || 1000, 10)
+const NUM_NODES = parseInt(process.env.NUM_NODES || NUM_PAGES, 10)
+const SORT = process.env.SORT
+const FILTER = process.env.FILTER || `eq`
+const TEXT = Boolean(process.env.TEXT)
+
+if (NUM_NODES < NUM_PAGES) {
+  throw new Error("Expecting NUM_NODES >= NUM_PAGES")
+}
+
+const nodesPerPage = Math.max(1, Math.round(NUM_NODES / NUM_PAGES))
+const ptop = require(`process-top`)()
+
+exports.sourceNodes = async ({ actions: { createNode } }) => {
+  console.log(`Creating ${NUM_NODES} nodes`)
+  for (let nodeNum = 0; nodeNum < NUM_NODES; nodeNum++) {
+    const pageNum = Math.floor(nodeNum / nodesPerPage)
+    createNode({
+      id: String(nodeNum),
+      nodeNum,
+      pageNum,
+      nodeNumReversed: NUM_NODES - nodeNum,
+      testEq: String(nodeNum),
+      testIn: [`foo`, `bar`, `baz`, `foobar`][nodeNum % 4],
+      testElemMatch: [
+        { testIn: [`foo`, `bar`, `baz`, `foobar`][nodeNum % 4] },
+        { testEq: String(nodeNum) },
+      ],
+      text: `${TEXT ? new Array(4128).join("*") : ``}${nodeNum}`,
+      sortRandom: Math.random() * NUM_NODES,
+      internal: {
+        type: `Test`,
+        contentDigest: String(nodeNum),
+      },
+    })
+    if (nodeNum % 50 === 0) {
+      await new Promise(resolve => setTimeout(resolve, 3))
+    }
+  }
+  if (global.gc) {
+    global.gc()
+  }
+  console.log(ptop.toString())
+}
+
+const pageTemplate = require.resolve(`./src/templates/${FILTER}.js`)
+exports.createPages = async ({ actions: { createPage } }) => {
+  console.log(`Creating ${NUM_PAGES} pages for filter: ${FILTER}`)
+  for (let pageNum = 0; pageNum < NUM_PAGES; pageNum++) {
+    createPage({
+      path: `/path/${pageNum}/`,
+      component: pageTemplate,
+      context: {
+        pageNumAsStr: String(pageNum),
+        fooBarValues: [
+          [`foo`, `bar`, `baz`, `foobar`][pageNum % 4],
+          [`foo`, `bar`, `baz`, `foobar`][(pageNum + 1) % 4],
+        ],
+        intValue: pageNum,
+        pageNum: pageNum,
+        pagesLeft: NUM_PAGES - pageNum,
+        limit: nodesPerPage,
+        skip: nodesPerPage * pageNum,
+        nodesTotal: NUM_NODES,
+        pagesTotal: NUM_PAGES,
+        sort: SORT
+          ? { fields: ["sortRandom"], order: SORT === `1` ? `ASC` : `DESC` }
+          : undefined,
+        regex: `/^${String(pageNum).slice(0, 1)}/`, // node id starts with the same number as page id
+      },
+    })
+    if (pageNum % 50 === 0) {
+      await new Promise(resolve => setTimeout(resolve, 3))
+    }
+  }
+  if (global.gc) {
+    global.gc()
+  }
+  console.log(ptop.toString())
+}
+
+exports.onPostBuild = () => {
+  if (global.gc) {
+    global.gc()
+  }
+  console.log(ptop.toString())
+}

benchmarks/query-filters-sort/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "gatsby-starter-hello-world",
+  "description": "Gatsby hello world starter",
+  "license": "MIT",
+  "scripts": {
+    "bench": "gatsby clean && rimraf .data && node --max_old_space_size=16384 --expose-gc node_modules/gatsby/dist/bin/gatsby.js build",
+    "develop": "gatsby develop",
+    "build": "gatsby build",
+    "serve": "gatsby serve"
+  },
+  "dependencies": {
+    "gatsby": "^3.4.1",
+    "process-top": "^1.2.0",
+    "react": "^17.0.2",
+    "react-dom": "^17.0.2",
+    "rimraf": "^3.0.2"
+  },
+  "devDependencies": {
+    "gatsby-plugin-benchmark-reporting": "*"
+  }
+}

benchmarks/query-filters-sort/src/pages/index.js

@@ -0,0 +1,3 @@
+import React from "react"
+
+export default () => <div>Hello world!</div>

benchmarks/query-filters-sort/src/templates/elemMatch-eq.js

@@ -0,0 +1,26 @@
+import React from "react"
+import { graphql } from "gatsby"
+
+export default ({ data }) => {
+  if (!data?.allTest?.nodes) {
+    throw new Error("Wrong data")
+  }
+  return <div>{JSON.stringify(data)}</div>
+}
+
+export const query = graphql`
+  query($pageNumAsStr: String, $sort: TestSortInput) {
+    allTest(
+      filter: {
+        testElemMatch: { elemMatch: { testEq: { eq: $pageNumAsStr } } }
+      }
+      sort: $sort
+      limit: 5
+    ) {
+      nodes {
+        nodeNum
+        text
+      }
+    }
+  }
+`

benchmarks/query-filters-sort/src/templates/eq.js

@@ -0,0 +1,20 @@
+import React from "react"
+import { graphql } from "gatsby"
+
+export default ({ data }) => {
+  if (!data?.allTest?.nodes) {
+    throw new Error("Bad query result")
+  }
+  return <div>{JSON.stringify(data)}</div>
+}
+
+export const query = graphql`
+  query($pageNumAsStr: String!, $sort: TestSortInput) {
+    allTest(filter: { testEq: { eq: $pageNumAsStr } }, sort: $sort, limit: 5) {
+      nodes {
+        nodeNum
+        text
+      }
+    }
+  }
+`

benchmarks/query-filters-sort/src/templates/gt-lt.js

@@ -0,0 +1,24 @@
+import React from "react"
+import { graphql } from "gatsby"
+
+export default ({ data }) => {
+  if (!data?.allTest?.nodes) {
+    throw new Error("Wrong data")
+  }
+  return <div>{JSON.stringify(data)}</div>
+}
+
+export const query = graphql`
+  query($pageNum: Int, $pagesTotal: Int, $sort: TestSortInput) {
+    allTest(
+      filter: { nodeNum: { gt: $pageNum, lt: $pagesTotal } }
+      sort: $sort
+      limit: 5
+    ) {
+      nodes {
+        nodeNum
+        text
+      }
+    }
+  }
+`

benchmarks/query-filters-sort/src/templates/gt.js

@@ -0,0 +1,20 @@
+import React from "react"
+import { graphql } from "gatsby"
+
+export default ({ data }) => {
+  if (!data?.allTest?.nodes) {
+    throw new Error("Wrong data")
+  }
+  return <div>{JSON.stringify(data)}</div>
+}
+
+export const query = graphql`
+  query($pagesLeft: Int, $sort: TestSortInput) {
+    allTest(filter: { nodeNum: { gt: $pagesLeft } }, sort: $sort, limit: 5) {
+      nodes {
+        nodeNum
+        text
+      }
+    }
+  }
+`