fix(devtools): rework path debugging

Fixes #189

Author: harlan-zw

Diff for: .playground/components/DisableIndexing.vue

@@ -0,0 +1,9 @@
+<script lang="ts" setup>
+import { disableIndexing } from '../composables/robots'
+
+disableIndexing()
+</script>
+
+<template>
+  <div>disable</div>
+</template>

Diff for: .playground/composables/robots.ts

@@ -0,0 +1,5 @@
+import { useRobotsRule } from '#robots/app/composables/useRobotsRule'
+
+export function disableIndexing() {
+  return useRobotsRule(false)
+}

Diff for: .playground/pages/composable-false.vue

@@ -0,0 +1,9 @@
+<script lang="ts" setup>
+import { useRobotsRule } from '#imports'
+
+const rule = useRobotsRule(false)
+</script>
+
+<template>
+  <div>rule: {{ rule }}</div>
+</template>

Diff for: .playground/pages/composable-nested.vue

@@ -0,0 +1,8 @@
+<script lang="ts" setup>
+</script>
+
+<template>
+  <div>
+    <DisableIndexing />
+  </div>
+</template>

Diff for: client/app.vue

@@ -216,17 +216,23 @@ const tab = useLocalStorage('nuxt-robots:tab', 'overview')
               <div v-else>
                 <div class="inline-flex gap-3 mb-5 items-center">
                   <div>
-                    <NIcon v-if="pathDebugData?.indexable" icon="carbon:checkmark-filled" class="text-green-300" />
+                    <NIcon v-if="pathDebugData?.indexable && pathDebugData.crawlable" icon="carbon:checkmark-filled" class="text-green-300" />
                     <NIcon v-else icon="carbon:warning-filled" class="text-red-300" />
                   </div>
-                  <div v-if="pathDebugData.indexable">
-                    Robots can crawl <code class="opacity-60 text-sm">{{ path }}</code>.
+                  <div v-if="!pathDebugData.crawlable">
+                    Robots are not allowed to access <code class="opacity-60 text-sm">{{ path }}</code>.
+                  </div>
+                  <div v-else-if="!pathDebugData.indexable">
+                    Robots are not allowed to index <code class="opacity-60 text-sm">{{ path }}</code>, however, they can access it.
                   </div>
                   <div v-else>
-                    Robots are blocked from crawling <code class="opacity-60 text-sm">{{ path }}</code>.
+                    Robots can access and crawl <code class="opacity-60 text-sm">{{ path }}</code>.
                   </div>
                 </div>
                 <OCodeBlock :code="metaTag" lang="html" />
+                <div v-if="pathDebugData.robotsHeader" class="mt-3">
+                  <OCodeBlock :code="`X-Robots-Tag: ${pathDebugData.robotsHeader}`" lang="bash" />
+                </div>
                 <div v-if="pathDebugData?.debug" class="mt-3 flex gap-2">
                   <div v-if="pathDebugData?.debug?.source" class="text-xs text-gray-300 mt-3 border-gray-600 rounded-xl border-1 px-2 py-1 inline-flex">
                     Source: {{ pathDebugData?.debug?.source }}

Diff for: client/composables/shiki.ts

@@ -16,6 +16,7 @@ export async function loadShiki() {
     langs: [
       'html',
       'json',
+      'bash',
       Object.freeze({
         displayName: 'robots.txt',
         name: 'robots-txt',

Diff for: src/module.ts

@@ -489,11 +489,9 @@ declare module 'nitropack' {
   }
 }
 declare module 'h3' {
+  import type { RobotsContext } from '#robots/types'
   interface H3EventContext {
-    robots: {
-      rule: string
-      indexable: boolean
-    }
+    robots: RobotsContext
   }
 }
 `

Diff for: src/runtime/app/composables/useRobotsRule.ts

@@ -1,17 +1,20 @@
 import type { MaybeRef } from 'vue'
+// @ts-expect-error untyped
+import { devRootDir } from '#build/nuxt.config.mjs'
 import { injectHead, useHead } from '@unhead/vue'
 import { setHeader } from 'h3'
 import {
   useRequestEvent,
   useRuntimeConfig,
 } from 'nuxt/app'
-import { computed, onBeforeUnmount, ref, toValue } from 'vue'
+import { computed, getCurrentInstance, onBeforeUnmount, ref, toValue } from 'vue'
 
 /**
  * Get and set the current robots rule.
  */
 export function useRobotsRule(rule?: MaybeRef<boolean | string>) {
   const head = injectHead()
+  const vm = getCurrentInstance()
   if (import.meta.client) {
     // bit hacky but should work fine
     const robotsRef = ref(document.querySelector('meta[name="robots"]')?.getAttribute('content') || '')
@@ -39,8 +42,9 @@ export function useRobotsRule(rule?: MaybeRef<boolean | string>) {
     useHead({
       meta: [
         {
-          name: 'robots',
-          content: _rule,
+          'name': 'robots',
+          'content': _rule,
+          'data-hint': import.meta.dev ? `useRobotsRule,.${vm ? vm.type?.__file?.split(devRootDir)[1] : ''}` : undefined,
         },
       ],
     }, {

Diff for: src/runtime/app/plugins/robot-meta.server.ts

@@ -15,7 +15,7 @@ export default defineNuxtPlugin({
         {
           'name': 'robots',
           'content': () => ctx.rule || '',
-          'data-hint': () => config['nuxt-robots']?.debug ? ctx.debug?.source : undefined,
+          'data-hint': () => config['nuxt-robots']?.debug && ctx.debug?.source ? `${ctx.debug?.source},${ctx.debug?.line}` : undefined,
         },
       ],
     })

Diff for: src/runtime/server/composables/getPathRobotConfig.ts

@@ -1,12 +1,13 @@
 import type { H3Event } from 'h3'
+import type { RobotsContext } from '../../types'
 import { getRequestHeader } from 'h3'
 import { useNitroApp, useRuntimeConfig } from 'nitropack/runtime'
 import { withoutTrailingSlash } from 'ufo'
 import { matchPathToRule, normaliseRobotsRouteRule } from '../../util'
 import { createNitroRouteRuleMatcher } from '../kit'
 import { getSiteRobotConfig } from './getSiteRobotConfig'
 
-export function getPathRobotConfig(e: H3Event, options?: { userAgent?: string, skipSiteIndexable?: boolean, path?: string }): { rule: string, indexable: boolean, debug?: { source: string, line: string } } {
+export function getPathRobotConfig(e: H3Event, options?: { userAgent?: string, skipSiteIndexable?: boolean, path?: string }): RobotsContext {
   // has already been resolved
   const { robotsDisabledValue, robotsEnabledValue, isNuxtContentV2 } = useRuntimeConfig()['nuxt-robots']
   if (!options?.skipSiteIndexable) {

Diff for: src/runtime/server/middleware/injectContext.ts

@@ -5,10 +5,10 @@ import { getPathRobotConfig } from '../composables/getPathRobotConfig'
 export default defineEventHandler(async (e) => {
   if (e.path === '/robots.txt' || e.path.startsWith('/__') || e.path.startsWith('/api') || e.path.startsWith('/_nuxt'))
     return
-  const robotConfig = getPathRobotConfig(e)
   const nuxtRobotsConfig = useRuntimeConfig(e)['nuxt-robots']
   if (nuxtRobotsConfig) {
     const { header } = nuxtRobotsConfig
+    const robotConfig = getPathRobotConfig(e)
     if (header) {
       setHeader(e, 'X-Robots-Tag', robotConfig.rule)
     }

Diff for: src/runtime/server/routes/__robots__/debug-path.ts

@@ -1,9 +1,29 @@
 import { defineEventHandler, getQuery } from 'h3'
-import { getPathRobotConfig } from '../../composables/getPathRobotConfig'
+import { withQuery } from 'ufo'
 
 export default defineEventHandler(async (e) => {
-  const path = getQuery(e).path as string
-  return getPathRobotConfig(e, {
+  const query = getQuery(e)
+  const path = query.path as string
+  delete query.path
+  // we have to fetch the path to know for sure
+  const res = await $fetch.raw(withQuery(path, query))
+  const html = res._data
+  const robotsHeader = String(res.headers.get('x-robots-tag'))
+  // get robots meta tag <meta name="robots" content="noindex, nofollow" data-hint="useRobotsRule">
+  // <meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1">
+  const robotsMeta = html.match(/<meta[^>]+name=["']robots["'][^>]+content=["']([^"']+)["'](?:[^>]+data-hint=["']([^"']+)["'])?[^>]*>/i)
+  const [, robotsContent = null, robotsHint = null] = robotsMeta || []
+  const [source, line] = robotsHint ? robotsHint.split(',') : [null, null]
+  return {
+    rule: robotsContent,
+    indexable: !(robotsContent?.includes('noindex') && robotsHeader?.includes('noindex')),
+    crawlable: !(source === '/robots.txt'),
     path,
-  })
+    debug: {
+      source,
+      line,
+    },
+    robotsHeader,
+    robotsContent,
+  }
 })

Diff for: src/runtime/types.ts

@@ -56,3 +56,9 @@ export interface AutoI18nConfig {
   defaultLocale: string
   strategy: 'prefix' | 'prefix_except_default' | 'prefix_and_default' | 'no_prefix'
 }
+
+export interface RobotsContext {
+  rule: string
+  indexable: boolean
+  debug?: { source: string, line?: string }
+}