xen: move vcpu_info to common, leave hook for setup

vcpu_info is crucial for the Xen event channel core.  Since both the
data and setup steps are identical between architectures, move them to
the common file.  Since there is no cross-architecture method to call
a function on every processor during bring-up, simply leave the setup
function.

The number of vcpu_info structures available on the shared information
page varies by architecture.  Instead of hard-coding the count use
nitems().  Add a warning message for this being used.

Switch to XEN_VCPUID() and use Xen's typedefs.

panic() on failure since >32 processors is no longer unusual.

royger: Specify 64-byte alignment for vcpu_info to try to defend
against vcpu_info crossing a page boundary.  Add detection for this
limit.

Reviewed by: royger

sys/x86/xen/hvm.c

@@ -88,8 +88,7 @@ int xen_vector_callback_enabled;
 bool xen_evtchn_needs_ack;
 
 /*------------------------------- Per-CPU Data -------------------------------*/
-DPCPU_DEFINE(struct vcpu_info, vcpu_local_info);
-DPCPU_DEFINE(struct vcpu_info *, vcpu_info);
+DPCPU_DECLARE(struct vcpu_info *, vcpu_info);
 
 /*------------------------------ Sysctl tunables -----------------------------*/
 int xen_disable_pv_disks = 0;
@@ -416,10 +415,8 @@ SYSINIT(xen_hvm_init, SI_SUB_HYPERVISOR, SI_ORDER_FIRST, xen_hvm_sysinit, NULL);
 static void
 xen_hvm_cpu_init(void)
 {
-	struct vcpu_register_vcpu_info info;
-	struct vcpu_info *vcpu_info;
 	uint32_t regs[4];
-	int cpu, rc;
+	int rc;
 
 	if (!xen_domain())
 		return;
@@ -459,23 +456,7 @@ xen_hvm_cpu_init(void)
 			    rc);
 	}
 
-	/*
-	 * Set the vCPU info.
-	 *
-	 * NB: the vCPU info for vCPUs < 32 can be fetched from the shared info
-	 * page, but in order to make sure the mapping code is correct always
-	 * attempt to map the vCPU info at a custom place.
-	 */
-	vcpu_info = DPCPU_PTR(vcpu_local_info);
-	cpu = PCPU_GET(vcpu_id);
-	info.mfn = vtophys(vcpu_info) >> PAGE_SHIFT;
-	info.offset = vtophys(vcpu_info) - trunc_page(vtophys(vcpu_info));
-
-	rc = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);
-	if (rc != 0)
-		DPCPU_SET(vcpu_info, &HYPERVISOR_shared_info->vcpu_info[cpu]);
-	else
-		DPCPU_SET(vcpu_info, vcpu_info);
+	xen_setup_vcpu_info();
 }
 SYSINIT(xen_hvm_cpu_init, SI_SUB_INTR, SI_ORDER_FIRST, xen_hvm_cpu_init, NULL);
 

sys/xen/xen-os.h

@@ -44,6 +44,11 @@
 #include <xen/hvm.h>
 #include <contrib/xen/event_channel.h>
 
+/*
+ * Setup function which needs to be called on each processor by architecture
+ */
+extern void xen_setup_vcpu_info(void);
+
 static inline vm_paddr_t
 xen_get_xenstore_mfn(void)
 {

sys/xen/xen_common.c

@@ -33,11 +33,16 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h> /* required by xen/xen-os.h */
 
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
 #include <machine/atomic.h> /* required by xen/xen-os.h */
 
 #include <xen/xen-os.h>
 #include <xen/hvm.h>
 
+#include <contrib/xen/vcpu.h>
+
 /*-------------------------------- Global Data -------------------------------*/
 enum xen_domain_type xen_domain_type = XEN_NATIVE;
 
@@ -49,3 +54,51 @@ uint32_t hvm_start_flags;
 
 /*------------------ Hypervisor Access Shared Memory Regions -----------------*/
 shared_info_t *HYPERVISOR_shared_info;
+
+/*------------------------------- Per-CPU Data -------------------------------*/
+DPCPU_DEFINE(struct vcpu_info *, vcpu_info);
+
+void
+xen_setup_vcpu_info(void)
+{
+	/* This isn't directly accessed outside this function */
+	DPCPU_DEFINE_STATIC(vcpu_info_t, vcpu_local_info)
+	    __attribute__((aligned(64)));
+
+	vcpu_info_t *vcpu_info = DPCPU_PTR(vcpu_local_info);
+	vcpu_register_vcpu_info_t info = {
+		.mfn = vtophys(vcpu_info) >> PAGE_SHIFT,
+		.offset = vtophys(vcpu_info) & PAGE_MASK,
+	};
+	unsigned int cpu = XEN_VCPUID();
+	int rc;
+
+	KASSERT(xen_domain(), ("%s(): invoked when not on Xen?", __func__));
+
+	_Static_assert(sizeof(struct vcpu_info) <= 64,
+	    "struct vcpu_info is larger than supported limit of 64 bytes");
+
+	/*
+	 * Set the vCPU info.
+	 *
+	 * NB: the vCPU info for some vCPUs can be fetched from the shared info
+	 * page, but in order to make sure the mapping code is correct always
+	 * attempt to map the vCPU info at a custom place.
+	 */
+	rc = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);
+	if (rc == 0)
+		DPCPU_SET(vcpu_info, vcpu_info);
+	else if (cpu < nitems(HYPERVISOR_shared_info->vcpu_info)) {
+		static bool warned = false;
+
+		DPCPU_SET(vcpu_info, &HYPERVISOR_shared_info->vcpu_info[cpu]);
+
+		if (bootverbose && !warned) {
+			warned = true;
+			printf(
+		"WARNING: Xen vCPU %u failed to setup vcpu_info rc = %d\n",
+			    cpu, rc);
+		}
+	} else
+		panic("Unable to register vCPU %u, rc=%d\n", cpu, rc);
+}