Remove pinned builtin plugin versions from storage

builtin/logical/database/path_config_connection.go

@@ -11,6 +11,7 @@ import (
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/go-version"
 
+	"github.com/hashicorp/vault/helper/versions"
 	v5 "github.com/hashicorp/vault/sdk/database/dbplugin/v5"
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/helper/consts"
@@ -228,6 +229,12 @@ func (b *databaseBackend) connectionReadHandler() framework.OperationFunc {
 			}
 		}
 
+		if versions.IsBuiltinVersion(config.PluginVersion) {
+			// This gets treated as though it's empty when mounting, and will get
+			// overwritten to be empty when the config is next written. See #18051.
+			config.PluginVersion = ""
+		}
+
 		delete(config.ConnectionDetails, "password")
 		delete(config.ConnectionDetails, "private_key")
 
@@ -295,7 +302,10 @@ func (b *databaseBackend) connectionWriteHandler() framework.OperationFunc {
 			config.PluginVersion = pluginVersionRaw.(string)
 		}
 
-		unversionedPlugin, err := b.System().LookupPlugin(ctx, config.PluginName, consts.PluginTypeDatabase)
+		var builtinShadowed bool
+		if unversionedPlugin, err := b.System().LookupPlugin(ctx, config.PluginName, consts.PluginTypeDatabase); err == nil && !unversionedPlugin.Builtin {
+			builtinShadowed = true
+		}
 		switch {
 		case config.PluginVersion != "":
 			semanticVersion, err := version.NewVersion(config.PluginVersion)
@@ -305,7 +315,16 @@ func (b *databaseBackend) connectionWriteHandler() framework.OperationFunc {
 
 			// Canonicalize the version.
 			config.PluginVersion = "v" + semanticVersion.String()
-		case err == nil && !unversionedPlugin.Builtin:
+
+			if config.PluginVersion == versions.GetBuiltinVersion(consts.PluginTypeDatabase, config.PluginName) {
+				if builtinShadowed {
+					return logical.ErrorResponse("database plugin %q, version %s not found, as it is"+
+						" overridden by an unversioned plugin of the same name. Omit `plugin_version` to use the unversioned plugin", config.PluginName, config.PluginVersion), nil
+				}
+
+				config.PluginVersion = ""
+			}
+		case builtinShadowed:
 			// We'll select the unversioned plugin that's been registered.
 		case req.Operation == logical.CreateOperation:
 			// No version provided and no unversioned plugin of that name available.
@@ -439,6 +458,11 @@ func (b *databaseBackend) connectionWriteHandler() framework.OperationFunc {
 }
 
 func storeConfig(ctx context.Context, storage logical.Storage, name string, config *DatabaseConfig) error {
+	// 1.12.0 and 1.12.1 stored builtin plugins in storage, but 1.12.2 reverted
+	// that, so clean up any pre-existing stored builtin versions on write.
+	if versions.IsBuiltinVersion(config.PluginVersion) {
+		config.PluginVersion = ""
+	}
 	entry, err := logical.StorageEntryJSON(fmt.Sprintf("config/%s", name), config)
 	if err != nil {
 		return fmt.Errorf("unable to marshal object to JSON: %w", err)

builtin/logical/database/version_wrapper.go

@@ -6,6 +6,7 @@ import (
 
 	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/vault/helper/versions"
 	v4 "github.com/hashicorp/vault/sdk/database/dbplugin"
 	v5 "github.com/hashicorp/vault/sdk/database/dbplugin/v5"
 	"github.com/hashicorp/vault/sdk/helper/pluginutil"
@@ -22,8 +23,15 @@ type databaseVersionWrapper struct {
 var _ logical.PluginVersioner = databaseVersionWrapper{}
 
 // newDatabaseWrapper figures out which version of the database the pluginName is referring to and returns a wrapper object
-// that can be used to make operations on the underlying database plugin.
+// that can be used to make operations on the underlying database plugin. If a builtin pluginVersion is provided, it will
+// be ignored.
 func newDatabaseWrapper(ctx context.Context, pluginName string, pluginVersion string, sys pluginutil.LookRunnerUtil, logger log.Logger) (dbw databaseVersionWrapper, err error) {
+	// 1.12.0 and 1.12.1 stored plugin version in the config, but that stored
+	// builtin version may disappear from the plugin catalog when Vault is
+	// upgraded, so always reference builtin plugins by an empty version.
+	if versions.IsBuiltinVersion(pluginVersion) {
+		pluginVersion = ""
+	}
 	newDB, err := v5.PluginFactoryVersion(ctx, pluginName, pluginVersion, sys, logger)
 	if err == nil {
 		dbw = databaseVersionWrapper{

changelog/18051.txt

@@ -0,0 +1,6 @@
+```release-note:change
+plugins: Mounts can no longer be pinned to a specific _builtin_ version. Mounts previously pinned to a specific builtin version will now automatically upgrade to the latest builtin version, and may now be overridden if an unversioned plugin of the same name and type is registered. Mounts using plugin versions without `builtin` in their metadata remain unaffected.
+```
+```release-note:bug
+plugins: Vault upgrades will no longer fail if a mount has been created using an explicit builtin plugin version.
+```

helper/versions/version.go

@@ -6,14 +6,19 @@ import (
 	"strings"
 	"sync"
 
+	semver "github.com/hashicorp/go-version"
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/sdk/version"
 )
 
+const (
+	BuiltinMetadata = "builtin"
+)
+
 var (
 	buildInfoOnce         sync.Once // once is used to ensure we only parse build info once.
 	buildInfo             *debug.BuildInfo
-	DefaultBuiltinVersion = "v" + version.GetVersion().Version + "+builtin.vault"
+	DefaultBuiltinVersion = fmt.Sprintf("v%s+%s.vault", version.GetVersion().Version, BuiltinMetadata)
 )
 
 func GetBuiltinVersion(pluginType consts.PluginType, pluginName string) string {
@@ -46,9 +51,28 @@ func GetBuiltinVersion(pluginType consts.PluginType, pluginName string) string {
 
 	for _, dep := range buildInfo.Deps {
 		if dep.Path == pluginModulePath {
-			return dep.Version + "+builtin"
+			return dep.Version + "+" + BuiltinMetadata
 		}
 	}
 
 	return DefaultBuiltinVersion
 }
+
+// IsBuiltinVersion checks for the "builtin" metadata identifier in a plugin's
+// semantic version. Vault rejects any plugin registration requests with this
+// identifier, so we can be certain it's a builtin plugin if it's present.
+func IsBuiltinVersion(v string) bool {
+	semanticVersion, err := semver.NewSemver(v)
+	if err != nil {
+		return false
+	}
+
+	metadataIdentifiers := strings.Split(semanticVersion.Metadata(), ".")
+	for _, identifier := range metadataIdentifiers {
+		if identifier == BuiltinMetadata {
+			return true
+		}
+	}
+
+	return false
+}

vault/auth.go

@@ -636,6 +636,12 @@ func (c *Core) loadCredentials(ctx context.Context) error {
 			needPersist = true
 		}
 
+		// Don't store built-in version in the mount table, to make upgrades smoother.
+		if versions.IsBuiltinVersion(entry.Version) {
+			entry.Version = ""
+			needPersist = true
+		}
+
 		if entry.NamespaceID == "" {
 			entry.NamespaceID = namespace.RootNamespaceID
 			needPersist = true

vault/logical_system.go

@@ -635,20 +635,12 @@ func getVersion(d *framework.FieldData) (version string, builtin bool, err error
 			return "", false, fmt.Errorf("version %q is not a valid semantic version: %w", version, err)
 		}
 
-		metadataIdentifiers := strings.Split(semanticVersion.Metadata(), ".")
-		for _, identifier := range metadataIdentifiers {
-			if identifier == "builtin" {
-				builtin = true
-				break
-			}
-		}
-
 		// Canonicalize the version string.
 		// Add the 'v' back in, since semantic version strips it out, and we want to be consistent with internal plugins.
 		version = "v" + semanticVersion.String()
 	}
 
-	return version, builtin, nil
+	return version, versions.IsBuiltinVersion(version), nil
 }
 
 func (b *SystemBackend) handlePluginReloadUpdate(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
@@ -2636,6 +2628,19 @@ func (b *SystemBackend) validateVersion(ctx context.Context, version string, plu
 
 		// Canonicalize the version.
 		version = "v" + semanticVersion.String()
+
+		if version == versions.GetBuiltinVersion(pluginType, pluginName) {
+			unversionedPlugin, err := b.System().LookupPlugin(ctx, pluginName, pluginType)
+			if err == nil && !unversionedPlugin.Builtin {
+				// Builtin is overridden, return "not found" error.
+				return "", logical.ErrorResponse("%s plugin %q, version %s not found, as it is"+
+					" overridden by an unversioned plugin of the same name", pluginType.String(), pluginName, version), nil
+			}
+
+			// Don't put the builtin version in storage. Ensures that builtins
+			// can always be overridden, and upgrades are much simpler to handle.
+			version = ""
+		}
 	}
 
 	// if a non-builtin version is requested for a builtin plugin, return an error

vault/logical_system_test.go

@@ -5113,3 +5113,35 @@ func TestSortVersionedPlugins(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateVersion(t *testing.T) {
+	b := testSystemBackend(t).(*SystemBackend)
+	k8sAuthBuiltin := versions.GetBuiltinVersion(consts.PluginTypeCredential, "kubernetes")
+
+	for name, tc := range map[string]struct {
+		pluginName         string
+		pluginVersion      string
+		pluginType         consts.PluginType
+		expectLogicalError bool
+		expectedVersion    string
+	}{
+		"default, nothing in nothing out": {"kubernetes", "", consts.PluginTypeCredential, false, ""},
+		"builtin specified, empty out":    {"kubernetes", k8sAuthBuiltin, consts.PluginTypeCredential, false, ""},
+		"not canonical is ok":             {"kubernetes", "1.0.0", consts.PluginTypeCredential, false, "v1.0.0"},
+		"not a semantic version, error":   {"kubernetes", "not-a-version", consts.PluginTypeCredential, true, ""},
+	} {
+		t.Run(name, func(t *testing.T) {
+			version, resp, err := b.validateVersion(context.Background(), tc.pluginVersion, tc.pluginName, tc.pluginType)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if tc.expectLogicalError {
+				if resp == nil || !resp.IsError() || resp.Error() == nil {
+					t.Errorf("expected logical error but got none, resp: %#v", resp)
+				}
+			} else if version != tc.expectedVersion {
+				t.Errorf("expected version %q but got %q", tc.expectedVersion, version)
+			}
+		})
+	}
+}

vault/mount.go

@@ -727,8 +727,7 @@ func (c *Core) builtinTypeFromMountEntry(ctx context.Context, entry *MountEntry)
 		return consts.PluginTypeUnknown
 	}
 
-	// Builtin plugins should contain the "builtin" string in their RunningVersion
-	if !strings.Contains(entry.RunningVersion, "builtin") {
+	if !versions.IsBuiltinVersion(entry.RunningVersion) {
 		return consts.PluginTypeUnknown
 	}
 
@@ -1317,6 +1316,11 @@ func (c *Core) runMountUpdates(ctx context.Context, needPersist bool) error {
 			needPersist = true
 		}
 
+		// Don't store built-in version in the mount table, to make upgrades smoother.
+		if versions.IsBuiltinVersion(entry.Version) {
+			entry.Version = ""
+			needPersist = true
+		}
 	}
 	// Done if we have restored the mount table and we don't need
 	// to persist

vault/mount_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/vault/audit"
 	"github.com/hashicorp/vault/helper/metricsutil"
 	"github.com/hashicorp/vault/helper/namespace"
+	"github.com/hashicorp/vault/helper/versions"
 	"github.com/hashicorp/vault/sdk/helper/compressutil"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
 	"github.com/hashicorp/vault/sdk/logical"
@@ -206,7 +207,7 @@ func TestCore_Mount_secrets_builtin_RunningVersion(t *testing.T) {
 
 	raw, _ := c.router.root.Get(match)
 	// we override the running version of builtins
-	if !strings.Contains(raw.(*routeEntry).mountEntry.RunningVersion, "builtin") {
+	if !versions.IsBuiltinVersion(raw.(*routeEntry).mountEntry.RunningVersion) {
 		t.Errorf("Expected mount to have builtin version but got %s", raw.(*routeEntry).mountEntry.RunningVersion)
 	}
 }

vault/plugin_catalog.go

@@ -833,6 +833,14 @@ func (c *PluginCatalog) get(ctx context.Context, name string, pluginType consts.
 
 	builtinVersion := versions.GetBuiltinVersion(pluginType, name)
 	if version == "" || version == builtinVersion {
+		if version == builtinVersion {
+			// Don't return the builtin if it's shadowed by an unversioned plugin.
+			unversioned, err := c.get(ctx, name, pluginType, "")
+			if err == nil && unversioned != nil && !unversioned.Builtin {
+				return nil, nil
+			}
+		}
+
 		// Look for builtin plugins
 		if factory, ok := c.builtinRegistry.Get(name, pluginType); ok {
 			return &pluginutil.PluginRunner{

vault/plugin_catalog_test.go

@@ -383,7 +383,7 @@ func TestPluginCatalog_ListVersionedPlugins(t *testing.T) {
 			if !plugin.Builtin {
 				t.Fatalf("expected %v plugin to be builtin", plugin)
 			}
-			if plugin.SemanticVersion.Metadata() != "builtin" && plugin.SemanticVersion.Metadata() != "builtin.vault" {
+			if !versions.IsBuiltinVersion(plugin.Version) {
 				t.Fatalf("expected +builtin metadata but got %s", plugin.Version)
 			}
 		}