multikernel: manage LLC way isolation through device tree

Documentation/multikernel/usage.rst

@@ -47,6 +47,7 @@ Phase 1: Instance Creation (Automatic from DTB)
                   resources {
                       cpus = <1>;
                       memory-bytes = <0x20000000>;   // 512MB
+                      llc-way-mask = <0x0000ffff>;  // Lower half of LLC ways
                   };
               };
 
@@ -142,6 +143,7 @@ The multikernel device tree uses the ``/instances`` structure with ``multikernel
                 resources {
                     cpus = <1>;                      // CPU ID 1
                     memory-bytes = <0x20000000>;     // 512MB
+                    llc-way-mask = <0x0000ffff>;     // resctrl L3 CBM
                 };
             };
 
@@ -178,6 +180,7 @@ When viewing an instance's ``device_tree_source``, it appears in per-instance fo
         resources {
             cpus = <1>;
             memory-bytes = <0x20000000>; // 512 MB
+            llc-way-mask = <0x0000ffff>;
         };
     };
 
@@ -186,12 +189,51 @@ Resource Properties
 
 - **cpus**: Array of CPU IDs to assign to this instance
 - **memory-bytes**: Memory size in bytes (must be page-aligned)
+- **llc-way-mask**: Optional 32-bit resctrl L3 capacity bitmask. The baseline
+  value defines the LLC ways available to the multikernel pool; every instance
+  value must be a non-zero, non-overlapping subset of that pool.
 - **id**: Unique instance identifier used for kexec operations
 
+LLC Way Isolation
+-----------------
+
+``llc-way-mask`` uses the same bit numbering as the L3 CBM in a resctrl
+``schemata`` file.  Mount resctrl before applying a baseline that contains an
+LLC mask, and keep it mounted while any LLC-partitioned instance exists.  For
+example::
+
+    mkdir -p /dev/resctrl
+    mount -t resctrl none /dev/resctrl
+
+The baseline mask is the pool reserved for spawn kernels.  Multikernel assigns
+the complementary supported mask to the host's default resctrl group.  For a
+16-way LLC, this reserves the upper half for multikernel and leaves the lower
+half to the host::
+
+    /dts-v1/;
+    / {
+        compatible = "multikernel-v1";
+        resources {
+            cpus = <0x0 0x1>;
+            memory-base = /bits/ 64 <0x100000000>;
+            memory-bytes = /bits/ 64 <0x40000000>;
+            llc-way-mask = <0xff00>;
+        };
+    };
+
+An instance then requests a non-overlapping subset of ``0xff00``.  The host
+allocates a resctrl CLOSID, programs the mask on every L3 domain, and records
+the CLOSID in the generated instance DTB.  ``linux,resctrl-closid`` is internal
+host-generated metadata and must not be supplied in an input DTB.  A spawn
+kernel inherits that CLOSID and does not reset the package-wide CAT registers.
+Mounting resctrl inside a spawn kernel is rejected because those registers are
+shared with its parent.
+
 The system validates that:
 
 - CPU IDs are valid and available
 - Memory requests don't exceed available multikernel pool
+- LLC way masks stay within the baseline pool and don't overlap another instance
 - Instance IDs are unique
 - All values are properly aligned
 

arch/x86/include/asm/multikernel.h

@@ -90,6 +90,9 @@ struct mk_spawn_context {
 	u32 target_apic_id;		/* Target CPU's APIC ID */
 	u32 flags;			/* MK_SPAWN_F_* flags */
 	u32 ready;			/* Signal flag */
+	u32 resctrl_closid;		/* CLOSID to install before dispatch */
+	u32 resctrl_l3_mask;		/* L3 CBM to install before dispatch */
+	u32 resctrl_l3_cdp;		/* L3 CDP uses two CBM registers */
 	u32 reserved;			/* Padding for alignment */
 	/* Variable-size struct last - size depends on kernel config */
 	struct boot_params bp;		/* Standard x86 boot params */

arch/x86/kernel/asm-offsets.c

@@ -64,6 +64,9 @@ static void __used common(void)
 	OFFSET(MK_CTX_target_apic_id, mk_spawn_context, target_apic_id);
 	OFFSET(MK_CTX_flags, mk_spawn_context, flags);
 	OFFSET(MK_CTX_ready, mk_spawn_context, ready);
+	OFFSET(MK_CTX_resctrl_closid, mk_spawn_context, resctrl_closid);
+	OFFSET(MK_CTX_resctrl_l3_mask, mk_spawn_context, resctrl_l3_mask);
+	OFFSET(MK_CTX_resctrl_l3_cdp, mk_spawn_context, resctrl_l3_cdp);
 	OFFSET(MK_CTX_bp, mk_spawn_context, bp);
 #endif
 

arch/x86/kernel/cpu/resctrl/core.c

@@ -20,6 +20,7 @@
 #include <linux/slab.h>
 #include <linux/err.h>
 #include <linux/cpuhotplug.h>
+#include <linux/multikernel.h>
 
 #include <asm/cpu_device_id.h>
 #include <asm/msr.h>
@@ -409,13 +410,48 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_ctrl_domain *
 		return -ENOMEM;
 
 	hw_dom->ctrl_val = dc;
-	setup_default_ctrlval(r, dc);
+	if (multikernel_is_spawn_kernel()) {
+		u32 closid = multikernel_resctrl_closid();
+		u32 mask = multikernel_resctrl_l3_mask();
+		int i;
+
+		/* Import the package-wide controls programmed by the parent. */
+		for (i = 0; i < hw_res->num_closid; i++)
+			rdmsrq(hw_res->msr_base + i, dc[i]);
+
+		/*
+		 * A parent may have no online CPU in this LLC domain after
+		 * donating the whole domain.  Program our own slot locally as
+		 * the first CPU in the domain comes online.
+		 */
+		if (r->rid == RDT_RESOURCE_L3 && mask) {
+			m.res = r;
+			m.dom = d;
+			if (resctrl_arch_get_cdp_enabled(RDT_RESOURCE_L3)) {
+				m.low = resctrl_get_config_index(closid, CDP_DATA);
+				m.high = m.low + 2;
+				if (m.high > hw_res->num_closid)
+					return -ERANGE;
+				dc[m.low] = mask;
+				dc[m.low + 1] = mask;
+			} else {
+				if (closid >= hw_res->num_closid)
+					return -ERANGE;
+				m.low = closid;
+				m.high = closid + 1;
+				dc[closid] = mask;
+			}
+			hw_res->msr_update(&m);
+		}
+	} else {
+		setup_default_ctrlval(r, dc);
 
-	m.res = r;
-	m.dom = d;
-	m.low = 0;
-	m.high = hw_res->num_closid;
-	hw_res->msr_update(&m);
+		m.res = r;
+		m.dom = d;
+		m.low = 0;
+		m.high = hw_res->num_closid;
+		hw_res->msr_update(&m);
+	}
 	return 0;
 }
 
@@ -724,13 +760,13 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r)
 static void clear_closid_rmid(int cpu)
 {
 	struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state);
+	u32 closid = multikernel_resctrl_closid();
 
-	state->default_closid = RESCTRL_RESERVED_CLOSID;
+	state->default_closid = closid;
 	state->default_rmid = RESCTRL_RESERVED_RMID;
-	state->cur_closid = RESCTRL_RESERVED_CLOSID;
+	state->cur_closid = closid;
 	state->cur_rmid = RESCTRL_RESERVED_RMID;
-	wrmsr(MSR_IA32_PQR_ASSOC, RESCTRL_RESERVED_RMID,
-	      RESCTRL_RESERVED_CLOSID);
+	wrmsr(MSR_IA32_PQR_ASSOC, RESCTRL_RESERVED_RMID, closid);
 }
 
 static int resctrl_arch_online_cpu(unsigned int cpu)

arch/x86/multikernel/direct_boot.S

@@ -351,6 +351,34 @@ SYM_CODE_START_LOCAL_NOALIGN(mk_park_identity)
 	cmpl	%r13d, %eax
 	jne	.Lpark_relax
 
+	/* Program this LLC domain and select it before kernel code executes. */
+	movl	MK_CTX_resctrl_closid(%r12), %r10d
+	testl	%r10d, %r10d
+	jz	.Lpark_closid_done
+	movl	MK_CTX_resctrl_l3_mask(%r12), %eax
+	testl	%eax, %eax
+	jz	.Lpark_select_closid
+	movl	%r10d, %ecx
+	cmpl	$0, MK_CTX_resctrl_l3_cdp(%r12)
+	je	.Lpark_write_l3
+	shll	$1, %ecx
+	addl	$MSR_IA32_L3_CBM_BASE, %ecx
+	xorl	%edx, %edx
+	wrmsr
+	incl	%ecx
+	wrmsr
+	jmp	.Lpark_select_closid
+.Lpark_write_l3:
+	addl	$MSR_IA32_L3_CBM_BASE, %ecx
+	xorl	%edx, %edx
+	wrmsr
+.Lpark_select_closid:
+	movl	$MSR_IA32_PQR_ASSOC, %ecx
+	rdmsr
+	movl	%r10d, %edx
+	wrmsr
+.Lpark_closid_done:
+
 	/*
 	 * Copy the publication into registers BEFORE claiming: once the
 	 * claim clears ready, the publisher may reuse the slot for the

arch/x86/multikernel/spawn.c

@@ -31,6 +31,7 @@
 #include <linux/memblock.h>
 #include <linux/multikernel.h>
 #include <linux/percpu.h>
+#include <linux/resctrl.h>
 #include <linux/set_memory.h>
 #include <linux/sched.h>
 
@@ -328,6 +329,9 @@ int mk_spawn_cpu(struct mk_instance *instance, int cpu,
 	slot->kernel_entry = ctx->kernel_entry;
 	slot->trampoline_phys = ctx->trampoline_phys;
 	slot->self_phys = virt_to_phys(ctx);
+	slot->resctrl_closid = ctx->resctrl_closid;
+	slot->resctrl_l3_mask = ctx->resctrl_l3_mask;
+	slot->resctrl_l3_cdp = ctx->resctrl_l3_cdp;
 	slot->flags = 0;
 	ret = mk_slot_wake(slot, apic_id, "first spawn boot cpu");
 	if (!ret) {
@@ -447,6 +451,12 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance,
 			     (unsigned long)instance->trampoline_va,
 			     virt_to_phys(instance->trampoline_va),
 			     virt_to_phys(instance->park_va));
+	instance->spawn_ctx->resctrl_closid = instance->resctrl_closid_valid ?
+		instance->resctrl_closid : 0;
+	instance->spawn_ctx->resctrl_l3_mask = instance->llc_way_mask_valid ?
+		instance->llc_way_mask : 0;
+	instance->spawn_ctx->resctrl_l3_cdp =
+		resctrl_multikernel_l3_cdp_enabled();
 
 	return mk_spawn_cpu(instance, cpu, instance->spawn_ctx);
 }