tls: add 'certs' option to TLS block for multi-certs support

```
tls {
  certs = [
    {
      cert_file: "./configs/certs/srva-cert.pem"
      key_file:  "./configs/certs/srva-key.pem"
    },
    {
      cert_file: "./configs/certs/srvb-cert.pem"
      key_file:  "./configs/certs/srvb-key.pem"
    }
  ]
  ca_file: "./configs/certs/ca.pem"
  verify:  true
  timeout: 2
}
```

Signed-off-by: Waldemar Quevedo <wally@nats.io>

server/config_check_test.go

@@ -1832,6 +1832,90 @@ func TestConfigCheck(t *testing.T) {
 			errorLine: 4,
 			errorPos:  6,
 		},
+		{
+			name: "TLS multiple certs",
+			config: `
+				port: -1
+				tls {
+					certs: [
+					  { cert_file: "configs/certs/server.pem", key_file: "configs/certs/key.pem"},
+					  { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"},
+					]
+				}
+			`,
+			err: nil,
+		},
+		{
+			name: "TLS multiple certs, bad type",
+			config: `
+				port: -1
+				tls {
+					certs: [
+					  { cert_file: "configs/certs/server.pem", key_file: 123 },
+					  { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"},
+					]
+				}
+			`,
+			err:       fmt.Errorf("error parsing certificates config: unsupported type int64"),
+			errorLine: 5,
+			errorPos:  49,
+		},
+		{
+			name: "TLS multiple certs, missing key_file",
+			config: `
+				port: -1
+				tls {
+					certs: [
+					  { cert_file: "configs/certs/server.pem" }
+					  { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"}
+					]
+				}
+			`,
+			err:       fmt.Errorf("error parsing certificates config: both 'cert_file' and 'cert_key' options are required"),
+			errorLine: 5,
+			errorPos:  10,
+		},
+		{
+			name: "TLS multiple certs and single cert options at the same time",
+			config: `
+				port: -1
+				tls {
+					cert_file: "configs/certs/server.pem"
+					key_file: "configs/certs/key.pem"
+					certs: [
+					  { cert_file: "configs/certs/server.pem", key_file: "configs/certs/key.pem"},
+					  { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"},
+					]
+				}
+			`,
+			err:       fmt.Errorf("error parsing tls config, cannot combine 'cert_file' option with 'certs' option"),
+			errorLine: 3,
+			errorPos:  5,
+		},
+		{
+			name: "TLS multiple certs used but not configured",
+			config: `
+				port: -1
+				tls {
+					certs: []
+				}
+			`,
+			err:       fmt.Errorf("error parsing tls config, at least one 'cert_file' must be configured to enable TLS"),
+			errorLine: 3,
+			errorPos:  5,
+		},
+		{
+			name: "TLS multiple certs used but not configured, but cert_file configured",
+			config: `
+				port: -1
+				tls {
+					cert_file: "configs/certs/server.pem"
+					key_file: "configs/certs/key.pem"
+					certs: []
+				}
+			`,
+			err: nil,
+		},
 	}
 
 	checkConfig := func(config string) error {

server/configs/reload/tls_multi_cert_1.conf

@@ -0,0 +1,13 @@
+# Simple TLS config file
+
+listen:   127.0.0.1:-1
+
+tls {
+  certs = [
+    {
+      cert_file: "../test/configs/certs/srva-cert.pem"
+      key_file:  "../test/configs/certs/srva-key.pem"
+    }
+  ]
+  timeout:    2
+}

server/configs/reload/tls_multi_cert_2.conf

@@ -0,0 +1,19 @@
+# Simple TLS config file
+
+listen:   127.0.0.1:-1
+
+tls {
+  certs = [
+    {
+      cert_file: "../test/configs/certs/srva-cert.pem"
+      key_file:  "../test/configs/certs/srva-key.pem"
+    },
+    {
+      cert_file: "../test/configs/certs/srvb-cert.pem"
+      key_file:  "../test/configs/certs/srvb-key.pem"
+    }
+  ]
+  ca_file: "../test/configs/certs/ca.pem"
+  verify:  true
+  timeout: 2
+}

server/configs/reload/tls_multi_cert_3.conf

@@ -0,0 +1,13 @@
+# Simple TLS config file
+
+listen:   127.0.0.1:-1
+
+tls {
+  certs = [
+    {
+      cert_file: "../test/configs/certs/srvb-cert.pem"
+      key_file:  "../test/configs/certs/srvb-key.pem"
+    }
+  ]
+  timeout: 2
+}

server/opts.go

@@ -669,6 +669,13 @@ type TLSConfigOpts struct {
 	CertMatchBy       certstore.MatchByType
 	CertMatch         string
 	OCSPPeerConfig    *certidp.OCSPPeerConfig
+	Certificates      []*TLSCertPairOpt
+}
+
+// TLSCertPairOpt are the paths to a certificate and private key.
+type TLSCertPairOpt struct {
+	CertFile string
+	KeyFile  string
 }
 
 // OCSPConfig represents the options of OCSP stapling options.
@@ -4180,7 +4187,7 @@ func parseTLS(v interface{}, isClientCtx bool) (t *TLSConfigOpts, retErr error)
 	)
 	defer convertPanicToError(&lt, &retErr)
 
-	_, v = unwrapValue(v, &lt)
+	tk, v := unwrapValue(v, &lt)
 	tlsm = v.(map[string]interface{})
 	for mk, mv := range tlsm {
 		tk, mv := unwrapValue(mv, &lt)
@@ -4381,10 +4388,49 @@ func parseTLS(v interface{}, isClientCtx bool) (t *TLSConfigOpts, retErr error)
 			default:
 				return nil, &configErr{tk, fmt.Sprintf("error parsing ocsp peer config: unsupported type %T", v)}
 			}
+		case "certs", "certificates":
+			certs, ok := mv.([]interface{})
+			if !ok {
+				return nil, &configErr{tk, fmt.Sprintf("error parsing certificates config: unsupported type %T", v)}
+			}
+			tc.Certificates = make([]*TLSCertPairOpt, len(certs))
+			for i, v := range certs {
+				tk, vv := unwrapValue(v, &lt)
+				pair, ok := vv.(map[string]interface{})
+				if !ok {
+					return nil, &configErr{tk, fmt.Sprintf("error parsing certificates config: unsupported type %T", vv)}
+				}
+				certPair := &TLSCertPairOpt{}
+				for k, v := range pair {
+					tk, vv = unwrapValue(v, &lt)
+					file, ok := vv.(string)
+					if !ok {
+						return nil, &configErr{tk, fmt.Sprintf("error parsing certificates config: unsupported type %T", vv)}
+					}
+					switch k {
+					case "cert_file":
+						certPair.CertFile = file
+					case "key_file":
+						certPair.KeyFile = file
+					default:
+						return nil, &configErr{tk, fmt.Sprintf("error parsing tls certs config, unknown field %q", k)}
+					}
+				}
+				if certPair.CertFile == "" || certPair.KeyFile == "" {
+					return nil, &configErr{tk, "error parsing certificates config: both 'cert_file' and 'cert_key' options are required"}
+				}
+				tc.Certificates[i] = certPair
+			}
 		default:
-			return nil, &configErr{tk, fmt.Sprintf("error parsing tls config, unknown field [%q]", mk)}
+			return nil, &configErr{tk, fmt.Sprintf("error parsing tls config, unknown field %q", mk)}
 		}
 	}
+	if len(tc.Certificates) == 0 && tc.CertFile == _EMPTY_ {
+		return nil, &configErr{tk, "error parsing tls config, at least one 'cert_file' must be configured to enable TLS"}
+	}
+	if len(tc.Certificates) > 0 && tc.CertFile != _EMPTY_ {
+		return nil, &configErr{tk, "error parsing tls config, cannot combine 'cert_file' option with 'certs' option"}
+	}
 
 	// If cipher suites were not specified then use the defaults
 	if tc.Ciphers == nil {
@@ -4681,19 +4727,26 @@ func GenTLSConfig(tc *TLSConfigOpts) (*tls.Config, error) {
 	case tc.CertFile == _EMPTY_ && tc.KeyFile != _EMPTY_:
 		return nil, fmt.Errorf("missing 'cert_file' in TLS configuration")
 	case tc.CertFile != _EMPTY_ && tc.KeyFile != _EMPTY_:
-		// To support multiple certs (for SANs) without rearchitecting our
-		// configuration, we accept a list, separated by the platform list
-		// separator; both specs must contain the same count of entries.
-		// (So, colon-separated on Unix, semi-colon on Windows).
-		certFiles := strings.Split(tc.CertFile, string(os.PathListSeparator))
-		keyFiles := strings.Split(tc.KeyFile, string(os.PathListSeparator))
-		if len(certFiles) != len(keyFiles) {
-			return nil, fmt.Errorf("TLS configuration has %d entries in cert_file but %d in key_file", len(certFiles), len(keyFiles))
-		}
-		config.Certificates = make([]tls.Certificate, len(certFiles))
-		// Now load in cert and private keys
-		for i := range certFiles {
-			cert, err := tls.LoadX509KeyPair(certFiles[i], keyFiles[i])
+		// Now load in cert and private key
+		cert, err := tls.LoadX509KeyPair(tc.CertFile, tc.KeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing X509 certificate/key pair: %v", err)
+		}
+		cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return nil, fmt.Errorf("error parsing certificate: %v", err)
+		}
+		config.Certificates = []tls.Certificate{cert}
+	case tc.CertStore != certstore.STOREEMPTY:
+		err := certstore.TLSConfig(tc.CertStore, tc.CertMatchBy, tc.CertMatch, &config)
+		if err != nil {
+			return nil, err
+		}
+	case tc.Certificates != nil:
+		// Multiple certificate support.
+		config.Certificates = make([]tls.Certificate, len(tc.Certificates))
+		for i, certPair := range tc.Certificates {
+			cert, err := tls.LoadX509KeyPair(certPair.CertFile, certPair.KeyFile)
 			if err != nil {
 				return nil, fmt.Errorf("error parsing X509 certificate/key pair: %v", err)
 			}
@@ -4703,11 +4756,6 @@ func GenTLSConfig(tc *TLSConfigOpts) (*tls.Config, error) {
 			}
 			config.Certificates[i] = cert
 		}
-	case tc.CertStore != certstore.STOREEMPTY:
-		err := certstore.TLSConfig(tc.CertStore, tc.CertMatchBy, tc.CertMatch, &config)
-		if err != nil {
-			return nil, err
-		}
 	}
 
 	// Require client certificates as needed

server/reload_test.go

@@ -492,6 +492,102 @@ func TestConfigReloadDisableTLS(t *testing.T) {
 	nc.Close()
 }
 
+func TestConfigReloadRotateTLSMultiCert(t *testing.T) {
+	server, opts, config := runReloadServerWithConfig(t, "./configs/reload/tls_multi_cert_1.conf")
+	defer server.Shutdown()
+
+	// Ensure we can connect as a sanity check.
+	addr := fmt.Sprintf("nats://%s:%d", opts.Host, server.Addr().(*net.TCPAddr).Port)
+
+	rawCerts := make(chan []byte, 3)
+	nc, err := nats.Connect(addr, nats.Secure(&tls.Config{
+		VerifyConnection: func(s tls.ConnectionState) error {
+			rawCerts <- s.PeerCertificates[0].Raw
+			return nil
+		},
+		InsecureSkipVerify: true,
+	}))
+	if err != nil {
+		t.Fatalf("Error creating client: %v", err)
+	}
+	defer nc.Close()
+	sub, err := nc.SubscribeSync("foo")
+	if err != nil {
+		t.Fatalf("Error subscribing: %v", err)
+	}
+	defer sub.Unsubscribe()
+
+	// Rotate cert and enable client verification.
+	changeCurrentConfigContent(t, config, "./configs/reload/tls_multi_cert_2.conf")
+	if err := server.Reload(); err != nil {
+		t.Fatalf("Error reloading config: %v", err)
+	}
+
+	// Ensure connecting fails.
+	if _, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true})); err == nil {
+		t.Fatal("Expected connect to fail")
+	}
+
+	// Ensure connecting succeeds when client presents cert.
+	cert := nats.ClientCert("../test/configs/certs/client-cert.pem", "../test/configs/certs/client-key.pem")
+	conn, err := nats.Connect(addr, cert, nats.RootCAs("../test/configs/certs/ca.pem"), nats.Secure(&tls.Config{
+		VerifyConnection: func(s tls.ConnectionState) error {
+			rawCerts <- s.PeerCertificates[0].Raw
+			return nil
+		},
+	}))
+	if err != nil {
+		t.Fatalf("Error creating client: %v", err)
+	}
+	conn.Close()
+
+	// Ensure the original connection can still publish/receive.
+	if err := nc.Publish("foo", []byte("hello")); err != nil {
+		t.Fatalf("Error publishing: %v", err)
+	}
+	nc.Flush()
+	msg, err := sub.NextMsg(2 * time.Second)
+	if err != nil {
+		t.Fatalf("Error receiving msg: %v", err)
+	}
+	if string(msg.Data) != "hello" {
+		t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hello"), msg.Data)
+	}
+
+	// Rotate cert and disable client verification.
+	changeCurrentConfigContent(t, config, "./configs/reload/tls_multi_cert_3.conf")
+	if err := server.Reload(); err != nil {
+		t.Fatalf("Error reloading config: %v", err)
+	}
+
+	nc, err = nats.Connect(addr, nats.Secure(&tls.Config{
+		VerifyConnection: func(s tls.ConnectionState) error {
+			rawCerts <- s.PeerCertificates[0].Raw
+			return nil
+		},
+		InsecureSkipVerify: true,
+	}))
+	if err != nil {
+		t.Fatalf("Error creating client: %v", err)
+	}
+	defer nc.Close()
+	sub, err = nc.SubscribeSync("foo")
+	if err != nil {
+		t.Fatalf("Error subscribing: %v", err)
+	}
+	defer sub.Unsubscribe()
+
+	certA := <-rawCerts
+	certB := <-rawCerts
+	certC := <-rawCerts
+	if !bytes.Equal(certA, certB) {
+		t.Error("Expected the same cert")
+	}
+	if bytes.Equal(certB, certC) {
+		t.Error("Expected a different cert")
+	}
+}
+
 // Ensure Reload supports single user authentication config changes. Test this
 // by starting a server with authentication enabled, connect to it to verify,
 // reload config using a different username/password, ensure reconnect fails,