From b24e05c5b419f6b87109c62a7e0dd097878f3dfa Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 23:46:02 +0300 Subject: [PATCH 01/16] x86/multikernel: inherit host timer calibration Spawn kernels cannot calibrate against host-owned PIT, PIC, or IO-APIC resources. Carry the host loops-per-jiffy, CPU and TSC frequencies, and local APIC timer calibration in the spawn boot context. Install fixed calibration callbacks before x86 timer initialization. Keep explicit command-line calibration authoritative. Signed-off-by: Nikolay Nikolaev --- arch/x86/include/asm/multikernel.h | 26 +++++++++++++++---- arch/x86/kernel/platform-quirks.c | 40 +++++++++++++++++++++++++++++- arch/x86/multikernel/spawn.c | 22 ++++++++++++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index da9d37264e410d..584490b6246b74 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -10,6 +10,8 @@ #ifndef __ASSEMBLY__ +#include +#include #include #include #include @@ -54,7 +56,7 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) * from a page written by the host while running on CPUs parked by (possibly * differently built) spawn kernels. * - * The fields fall into two classes that must not be mixed up: + * The fields fall into three classes that must not be mixed up: * * - Anchor fields (self_phys, park_phys, park_cr3, ctrl_phys, * ctrl_size): the context's own identity, written once when the @@ -62,9 +64,13 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) * CPU on halt or offline, so they must stay valid for the context's * whole lifetime. * - * - Dispatch fields (everything else): the wake mailbox, rewritten for - * every publication and staged into registers by the CPU that claims - * it. Reparking gets its own repark_* dispatch fields precisely so a + * - Primary boot data (bp and the calibration values appended after it): + * written before the boot CPU is released and consumed while that kernel + * initializes. Secondary and repark publications do not rewrite it. + * + * - Dispatch fields (the remaining fixed-size fields): the wake mailbox, + * rewritten for every publication and staged into registers by the CPU + * that claims it. Reparking gets its own repark_* dispatch fields so a * repark publication never overwrites the anchor: the two used to * share fields, and a repark left the anchor pointing at another * kernel's park area, which triple-faulted the next halt. @@ -91,10 +97,20 @@ struct mk_spawn_context { u32 flags; /* MK_SPAWN_F_* flags */ u32 ready; /* Signal flag */ u32 reserved; /* Padding for alignment */ - /* Variable-size struct last - size depends on kernel config */ + /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ + /* Optional boot data belongs after boot_params, in the zeroed tail. */ + unsigned long boot_lps; /* Host delay loops per second */ + unsigned long boot_cpu_khz; /* Host CPU frequency calibration */ + unsigned long boot_tsc_khz; /* Host TSC frequency calibration */ + unsigned long boot_apic_hz; /* Host local APIC timer frequency */ } __aligned(PAGE_SIZE); +static_assert(offsetof(struct mk_spawn_context, bp) == 144); +static_assert(offsetof(struct mk_spawn_context, boot_lps) == + 144 + sizeof(struct boot_params)); +static_assert(sizeof(struct mk_spawn_context) == 2 * PAGE_SIZE); + /* Pool park loop code, copied by the host into per-instance park pages */ extern char mk_pool_park_start[]; extern char mk_pool_park_end[]; diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index 95d2cd2ccf74f5..feea109497efde 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -14,8 +14,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -28,6 +30,35 @@ extern pmd_t *populate_extra_pmd(unsigned long vaddr); extern unsigned long orig_boot_params; #ifdef CONFIG_MULTIKERNEL +static unsigned long multikernel_cpu_khz; +static unsigned long multikernel_tsc_khz; + +static unsigned long multikernel_calibrate_cpu(void) +{ + return multikernel_cpu_khz; +} + +static unsigned long multikernel_calibrate_tsc(void) +{ + return multikernel_tsc_khz; +} + +static void __init multikernel_setup_calibration(void) +{ + phys_addr_t ctx_phys = orig_boot_params - + offsetof(struct mk_spawn_context, bp); + struct mk_spawn_context *ctx = __va(ctx_phys); + + if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + return; + + multikernel_tsc_khz = ctx->boot_tsc_khz; + multikernel_cpu_khz = ctx->boot_cpu_khz ?: ctx->boot_tsc_khz; + x86_platform.calibrate_cpu = multikernel_calibrate_cpu; + x86_platform.calibrate_tsc = multikernel_calibrate_tsc; + setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); +} + /* * Custom wakeup for multikernel spawn kernels. * Uses shared spawn table instead of realmode trampoline. @@ -105,6 +136,10 @@ static void __init multikernel_parse_smp_config(void) */ apic_update_callback(wakeup_secondary_cpu_64, multikernel_wakeup_cpu); } +#else +static inline void multikernel_setup_calibration(void) +{ +} #endif /* CONFIG_MULTIKERNEL */ void __init x86_early_init_platform_quirks(void) @@ -135,6 +170,7 @@ void __init x86_early_init_platform_quirks(void) x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; break; case X86_SUBARCH_MULTIKERNEL: + multikernel_setup_calibration(); x86_platform.legacy.devices.pnpbios = 0; x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; x86_platform.legacy.rtc = 0; @@ -175,7 +211,9 @@ void __init x86_early_init_platform_quirks(void) * the PIT - which belongs to the host - and then request * legacy IRQ0, which can never reach an instance CPU that * has neither a PIC nor an IO-APIC. Ticks come from the - * local APIC timer via setup_percpu_clockev() instead. + * local APIC timer initialized by setup_percpu_clockev(). + * Keeping global_clock_event unset bypasses LAPIC timer + * verification, whose fallback path requires legacy IRQ0. */ x86_init.timers.timer_init = x86_init_noop; x86_init.timers.wallclock_init = x86_init_noop; diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 55487d9ed5a1b2..293788aac5c02d 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include #include @@ -44,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -447,6 +450,14 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, (unsigned long)instance->trampoline_va, virt_to_phys(instance->trampoline_va), virt_to_phys(instance->park_va)); + instance->spawn_ctx->boot_lps = cpu_data(cpu).loops_per_jiffy; + if (!instance->spawn_ctx->boot_lps) + instance->spawn_ctx->boot_lps = loops_per_jiffy; + instance->spawn_ctx->boot_lps *= HZ; + instance->spawn_ctx->boot_cpu_khz = cpu_khz; + instance->spawn_ctx->boot_tsc_khz = tsc_khz; + instance->spawn_ctx->boot_apic_hz = + (unsigned long)lapic_timer_period * HZ; return mk_spawn_cpu(instance, cpu, instance->spawn_ctx); } @@ -705,6 +716,17 @@ void mk_init_boot_context(phys_addr_t ctx_phys) } mk_boot_context = ctx; + /* + * A spawn kernel cannot calibrate against legacy timers because they + * belong to the host. Reuse the selected physical CPU's delay and local + * APIC timer calibration, while keeping explicit command-line values + * authoritative. + */ + if (!preset_lpj && ctx->boot_lps) + preset_lpj = DIV_ROUND_CLOSEST_ULL(ctx->boot_lps, HZ); + if (!lapic_timer_period && ctx->boot_apic_hz) + lapic_timer_period = + DIV_ROUND_CLOSEST_ULL(ctx->boot_apic_hz, HZ); /* * The host's control area (this context, the trampoline and park From 80c1b232d5d50c2216f2df67905b10ad97509151 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 07:54:21 +0300 Subject: [PATCH 02/16] multikernel: carry assigned PCI resource metadata Describe only the PCI functions assigned to a spawn and retain the BAR shape discovered by the host. Validate exact BDF syntax, reject duplicate functions, and preserve the inventory across the baseline and instance device trees. Keep PCI support behind CONFIG_PCI so the control plane remains buildable without PCI. Signed-off-by: Nikolay Nikolaev --- arch/x86/multikernel/Makefile | 1 + include/linux/multikernel.h | 36 +++++-- kernel/multikernel/Makefile | 4 +- kernel/multikernel/baseline.c | 58 ++++++++--- kernel/multikernel/dts.c | 165 +++++++++++++++++++++++++++---- kernel/multikernel/instance_dt.c | 121 ++++------------------- kernel/multikernel/internal.h | 3 +- kernel/multikernel/overlay.c | 48 ++++++--- kernel/multikernel/pci.c | 77 +++++++++++++++ 9 files changed, 357 insertions(+), 156 deletions(-) create mode 100644 kernel/multikernel/pci.c diff --git a/arch/x86/multikernel/Makefile b/arch/x86/multikernel/Makefile index 331bd895af1f7b..1fcee985bcbfe8 100644 --- a/arch/x86/multikernel/Makefile +++ b/arch/x86/multikernel/Makefile @@ -4,3 +4,4 @@ # obj-y += spawn.o direct_boot.o head_64.o +obj-$(CONFIG_PCI) += pci.o diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 011374c92c8481..3f36f4a338eb72 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -15,6 +15,9 @@ #include #include +struct pci_bus; +struct pci_ops; + /** * Physical CPU identifiers * @@ -403,6 +406,12 @@ struct mk_memory_region { * Represents a single PCI device that should be accessible to an instance. * Format: vendor:device@domain:bus:slot.func */ +#define MK_PCI_RESOURCE_COUNT 6 +struct mk_pci_resource { + u64 start; + u64 end; + u64 flags; +}; struct mk_pci_device { char name[64]; /* Device name from DTB (e.g., "enp9s0_dev") */ u16 vendor; /* PCI vendor ID */ @@ -411,6 +420,8 @@ struct mk_pci_device { u8 bus; /* PCI bus number */ u8 slot; /* PCI slot number */ u8 func; /* PCI function number */ + struct mk_pci_resource resources[MK_PCI_RESOURCE_COUNT]; + bool resources_valid; struct list_head list; /* Link to device list */ }; @@ -460,7 +471,7 @@ struct mk_dt_config { bool platform_devices_valid; /* Whether platform device list is valid */ /* Extensibility: Reserved fields for future use */ - u32 reserved[7]; /* Reduced due to added fields */ + u32 reserved[4]; /* Raw device tree data */ void *dtb_data; @@ -764,7 +775,10 @@ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size); /* Device probe filtering against the instance's allowlist */ -bool mk_pci_should_probe(struct pci_bus *bus, int devfn); +bool mk_pci_should_probe(struct pci_bus *bus, int devfn, + const struct pci_ops *ops); +bool mk_pci_get_assigned_identity(struct pci_bus *bus, int devfn, + u16 *vendor, u16 *device); bool mk_platform_device_allowed(const char *name, const char *hid); /* Early CPU registration from the manifest (spawn kernels) */ @@ -812,7 +826,9 @@ static inline void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size) { } -static inline bool mk_pci_should_probe(struct pci_bus *bus, int devfn) + +static inline bool mk_pci_should_probe(struct pci_bus *bus, int devfn, + const struct pci_ops *ops) { return true; } @@ -913,17 +929,17 @@ int __init mk_instance_restore_from_manifest(void); */ /** - * mk_pci_should_probe() - Check if PCI probing should occur at a location + * mk_pci_get_assigned_identity() - Check and identify an assigned PCI function * @bus: PCI bus * @devfn: PCI device/function number + * @vendor: optional assigned vendor ID output + * @device: optional assigned device ID output * - * Called BEFORE any PCI config space reads to determine if probing - * should proceed. This prevents config space accesses to devices - * that are not in the whitelist, avoiding hardware conflicts on bare metal. - * - * Returns: true if probing should proceed, false to skip entirely + * Synthetic roots make assigned functions directly discoverable. Only exact + * assignment metadata matches may access config space; physical bridges and + * all other functions remain inaccessible. * - * Declared above with the CONFIG_MULTIKERNEL stubs. + * Returns: true for an exact assignment metadata match, false otherwise */ /** diff --git a/kernel/multikernel/Makefile b/kernel/multikernel/Makefile index 359cccdd20353e..8e5e9497b51474 100644 --- a/kernel/multikernel/Makefile +++ b/kernel/multikernel/Makefile @@ -3,7 +3,9 @@ # Makefile for multikernel support # -obj-y += core.o cpuset.o mem.o kernfs.o dts.o instance_dt.o manifest.o ipi.o messaging.o overlay.o hotplug.o baseline.o +obj-y += core.o cpuset.o mem.o kernfs.o dts.o instance_dt.o manifest.o +obj-y += ipi.o messaging.o overlay.o hotplug.o baseline.o +obj-$(CONFIG_PCI) += pci.o # DMA-BUF heap for multikernel memory allocation obj-$(CONFIG_DMABUF_HEAPS) += dma_heap.o diff --git a/kernel/multikernel/baseline.c b/kernel/multikernel/baseline.c index d4e2dd9be3a54c..89d97fe9e50aca 100644 --- a/kernel/multikernel/baseline.c +++ b/kernel/multikernel/baseline.c @@ -264,10 +264,14 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, } if (strcmp(device_type, "pci") == 0) { + struct mk_pci_device *existing; struct mk_pci_device *pci_dev; const char *pci_id_str; const fdt32_t *vendor_prop, *device_prop; - unsigned int domain, bus, slot, func; + u32 vendor, device; + u16 domain; + u8 bus, slot, func; + int ret; pci_id_str = fdt_getprop(fdt, dev_node, "pci-id", &len); if (!pci_id_str) { @@ -276,10 +280,12 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s' for device '%s'\n", - pci_id_str, dev_name); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s' for device '%s'\n", + len, pci_id_str, dev_name); + return ret; } vendor_prop = fdt_getprop(fdt, dev_node, "vendor-id", &len); @@ -295,6 +301,23 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, dev_name); return -EINVAL; } + vendor = fdt32_to_cpu(*vendor_prop); + device = fdt32_to_cpu(*device_prop); + if (vendor > U16_MAX || device > U16_MAX) { + pr_err("Out-of-range vendor-id or device-id for device '%s'\n", + dev_name); + return -ERANGE; + } + list_for_each_entry(existing, &instance->pci_devices, list) { + if (existing->domain == domain && + existing->bus == bus && existing->slot == slot && + existing->func == func) { + pr_err("Duplicate baseline PCI BDF %04x:%02x:%02x.%x\n", + domain, bus, slot, func); + mk_baseline_clear_resources(instance); + return -EEXIST; + } + } pci_dev = kzalloc(sizeof(*pci_dev), GFP_KERNEL); if (!pci_dev) { @@ -304,12 +327,12 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, strncpy(pci_dev->name, dev_name, sizeof(pci_dev->name) - 1); pci_dev->name[sizeof(pci_dev->name) - 1] = '\0'; - pci_dev->vendor = (u16)fdt32_to_cpu(*vendor_prop); - pci_dev->device = (u16)fdt32_to_cpu(*device_prop); - pci_dev->domain = (u16)domain; - pci_dev->bus = (u8)bus; - pci_dev->slot = (u8)slot; - pci_dev->func = (u8)func; + pci_dev->vendor = (u16)vendor; + pci_dev->device = (u16)device; + pci_dev->domain = domain; + pci_dev->bus = bus; + pci_dev->slot = slot; + pci_dev->func = func; list_add_tail(&pci_dev->list, &instance->pci_devices); instance->pci_device_count++; @@ -498,6 +521,7 @@ static int mk_baseline_initialize_cpus(void) return 0; } +#ifdef CONFIG_PCI static int mk_baseline_initialize_devices(const struct mk_instance *instance) { struct mk_pci_device *pci_dev; @@ -552,6 +576,18 @@ static int mk_baseline_initialize_devices(const struct mk_instance *instance) pr_info("Successfully unbound %d PCI devices for multikernel pool\n", unbound); return 0; } +#else +static int mk_baseline_initialize_devices(const struct mk_instance *instance) +{ + if (!instance->pci_device_count) { + pr_debug("No PCI devices in the multikernel pool\n"); + return 0; + } + + pr_err("Cannot initialize PCI devices without CONFIG_PCI\n"); + return -EOPNOTSUPP; +} +#endif int mk_baseline_validate_and_initialize(const void *fdt, size_t fdt_size) { diff --git a/kernel/multikernel/dts.c b/kernel/multikernel/dts.c index 51950eae735129..dc444a2d3e7865 100644 --- a/kernel/multikernel/dts.c +++ b/kernel/multikernel/dts.c @@ -17,12 +17,55 @@ #include #include #include +#include #include #include +#include #include - #include "internal.h" +static int mk_pci_parse_hex(const char *str, size_t digits, u32 *value) +{ + size_t i; + u32 parsed = 0; + + for (i = 0; i < digits; i++) { + int digit = hex_to_bin(str[i]); + + if (digit < 0) + return -EINVAL; + parsed = (parsed << 4) | digit; + } + + *value = parsed; + return 0; +} + +int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, + u8 *slot, u8 *func) +{ + u32 parsed_domain, parsed_bus, parsed_slot, parsed_func; + + if (len != (int)sizeof("0000:00:00.0") || pci_id[12] != '\0' || + pci_id[4] != ':' || pci_id[7] != ':' || pci_id[10] != '.') + return -EINVAL; + + if (mk_pci_parse_hex(pci_id, 4, &parsed_domain) || + mk_pci_parse_hex(pci_id + 5, 2, &parsed_bus) || + mk_pci_parse_hex(pci_id + 8, 2, &parsed_slot) || + mk_pci_parse_hex(pci_id + 11, 1, &parsed_func)) + return -EINVAL; + if (parsed_domain > U16_MAX || parsed_bus > U8_MAX || + parsed_slot > 31 || parsed_func > 7) + return -ERANGE; + + *domain = (u16)parsed_domain; + *bus = (u8)parsed_bus; + *slot = (u8)parsed_slot; + *func = (u8)parsed_func; + return 0; +} + static const void *mk_dt_get_base_fdt(void) { if (!root_instance || !root_instance->dtb_data) { @@ -208,10 +251,14 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, { const char *pci_id_str; const fdt32_t *vendor_prop, *device_prop; + const fdt64_t *resources_prop; + struct mk_pci_device *existing; struct mk_pci_device *pci_dev; - unsigned int domain, bus, slot, func; + u32 vendor, device; + u16 domain; + u8 bus, slot, func; const char *node_name; - int len; + int len, i, ret; node_name = fdt_get_name(source_fdt, dev_node, NULL); @@ -222,10 +269,11 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format: '%s' (expected domain:bus:slot.func)\n", - pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id: '%.*s' (expected domain:bus:slot.func)\n", + len, pci_id_str); + return ret; } vendor_prop = fdt_getprop(source_fdt, dev_node, "vendor-id", &len); @@ -241,6 +289,21 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, device_name, node_name ? node_name : ""); return -EINVAL; } + vendor = fdt32_to_cpu(*vendor_prop); + device = fdt32_to_cpu(*device_prop); + if (vendor > U16_MAX || device > U16_MAX) { + pr_err("Out-of-range vendor-id or device-id in device '%s'\n", + device_name); + return -ERANGE; + } + list_for_each_entry(existing, &config->pci_devices, list) { + if (existing->domain == domain && existing->bus == bus && + existing->slot == slot && existing->func == func) { + pr_err("Duplicate assigned PCI BDF %04x:%02x:%02x.%x\n", + domain, bus, slot, func); + return -EEXIST; + } + } pci_dev = kzalloc(sizeof(*pci_dev), GFP_KERNEL); if (!pci_dev) { @@ -248,12 +311,38 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, return -ENOMEM; } - pci_dev->vendor = (u16)fdt32_to_cpu(*vendor_prop); - pci_dev->device = (u16)fdt32_to_cpu(*device_prop); - pci_dev->domain = (u16)domain; - pci_dev->bus = (u8)bus; - pci_dev->slot = (u8)slot; - pci_dev->func = (u8)func; + strscpy(pci_dev->name, device_name, sizeof(pci_dev->name)); + pci_dev->vendor = (u16)vendor; + pci_dev->device = (u16)device; + pci_dev->domain = domain; + pci_dev->bus = bus; + pci_dev->slot = slot; + pci_dev->func = func; + resources_prop = fdt_getprop(source_fdt, dev_node, "bar-resources", &len); + if (resources_prop) { + if (len != MK_PCI_RESOURCE_COUNT * 3 * sizeof(*resources_prop)) { + pr_err("Invalid bar-resources in device '%s'\n", device_name); + kfree(pci_dev); + return -EINVAL; + } + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + u64 start = fdt64_to_cpu(resources_prop[i * 3]); + u64 end = fdt64_to_cpu(resources_prop[i * 3 + 1]); + u64 flags = fdt64_to_cpu(resources_prop[i * 3 + 2]); + + if ((start || end) && + (end < start || !(flags & (IORESOURCE_IO | IORESOURCE_MEM)))) { + pr_err("Invalid PCI BAR %d range in device '%s'\n", + i, device_name); + kfree(pci_dev); + return -EINVAL; + } + pci_dev->resources[i].start = start; + pci_dev->resources[i].end = end; + pci_dev->resources[i].flags = flags; + } + pci_dev->resources_valid = true; + } list_add_tail(&pci_dev->list, &config->pci_devices); config->pci_device_count++; @@ -564,7 +653,7 @@ int mk_dt_parse(const void *dtb_data, size_t dtb_size, return ret; } - pr_info("Successfully parsed multikernel device tree with %zu bytes memory, %u CPUs, %d PCI devices, and %d platform devices\n", + pr_info("Successfully parsed multikernel device tree with %zu bytes memory, %d CPUs, %d PCI devices, and %d platform devices\n", config->memory_size, mk_cpu_set_count(config->cpus), config->pci_device_count, config->platform_device_count); return 0; @@ -614,10 +703,10 @@ int mk_dt_parse_resources(const void *fdt, int resources_node, return ret; } - pr_info("Successfully parsed instance '%s': %zu bytes memory, %u CPUs, %d PCI devices, %d platform devices\n", + pr_info("Successfully parsed instance '%s': %zu bytes memory, %d CPUs, %d PCI devices, %d platform devices\n", instance_name, config->memory_size, - mk_cpu_set_count(config->cpus), - config->pci_device_count, config->platform_device_count); + mk_cpu_set_count(config->cpus), config->pci_device_count, + config->platform_device_count); return 0; } @@ -994,6 +1083,43 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, list_for_each_entry(pci_dev, &instance->pci_devices, list) { char node_name[64]; char pci_id_str[32]; + fdt64_t resources[MK_PCI_RESOURCE_COUNT * 3]; + struct pci_dev *live_dev = NULL; + unsigned int devfn; + int i; + + if (!pci_dev->resources_valid) { + devfn = PCI_DEVFN(pci_dev->slot, + pci_dev->func); + live_dev = + pci_get_domain_bus_and_slot(pci_dev->domain, + pci_dev->bus, devfn); + if (!live_dev) { + pr_err("PCI device %04x:%02x:%02x.%x disappeared before resource snapshot\n", + pci_dev->domain, pci_dev->bus, + pci_dev->slot, pci_dev->func); + ret = -ENODEV; + goto err_free; + } + } + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + u64 start, end, flags; + + if (live_dev) { + start = pci_resource_start(live_dev, i); + end = pci_resource_end(live_dev, i); + flags = pci_resource_flags(live_dev, i); + } else { + start = pci_dev->resources[i].start; + end = pci_dev->resources[i].end; + flags = pci_dev->resources[i].flags; + } + resources[i * 3] = cpu_to_fdt64(start); + resources[i * 3 + 1] = cpu_to_fdt64(end); + resources[i * 3 + 2] = cpu_to_fdt64(flags); + } + if (live_dev) + pci_dev_put(live_dev); snprintf(node_name, sizeof(node_name), "%s", pci_dev->name[0] ? pci_dev->name : "unnamed_pci"); @@ -1015,6 +1141,11 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, ret = fdt_property_u32(fdt, "device-id", pci_dev->device); if (ret) goto err_free; + ret = fdt_property(fdt, "bar-resources", resources, + sizeof(resources)); + if (ret) + goto err_free; + ret = fdt_end_node(fdt); if (ret) goto err_free; } diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index f21cc947149f10..b82ec1a987efd1 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -168,6 +168,7 @@ int mk_manifest_add_instance_dtb(struct kimage *image, void *fdt, int mk_id) */ int mk_manifest_add_host_ipi(struct kimage *image, void *fdt) { + mk_phys_cpu_t target_cpu = arch_cpu_physical_id(0); int ret = 0; if (!root_instance->ipi_data) { @@ -175,12 +176,15 @@ int mk_manifest_add_host_ipi(struct kimage *image, void *fdt) return 0; } - pr_info("Preserving host IPI buffer: phys=0x%llx, pages=%u\n", - (unsigned long long)root_instance->ipi_phys, root_instance->ipi_pages); + pr_info("Preserving host IPI buffer: phys=0x%llx, pages=%u, target CPU=%llu\n", + (unsigned long long)root_instance->ipi_phys, + root_instance->ipi_pages, + (unsigned long long)target_cpu); ret |= fdt_begin_node(fdt, "host-ipi-buffer"); ret |= fdt_property_u64(fdt, "phys-addr", root_instance->ipi_phys); ret |= fdt_property_u32(fdt, "pages", root_instance->ipi_pages); + ret |= fdt_property_u64(fdt, "target-cpu", target_cpu); ret |= fdt_end_node(fdt); if (ret) { @@ -452,19 +456,13 @@ static int __init mk_copy_pci_devices(const struct mk_dt_config *config, instance->pci_devices_valid = true; list_for_each_entry(src_dev, &config->pci_devices, list) { - dst_dev = kzalloc(sizeof(*dst_dev), GFP_KERNEL); + dst_dev = kmemdup(src_dev, sizeof(*dst_dev), GFP_KERNEL); if (!dst_dev) { pr_err("Failed to allocate PCI device entry\n"); return -ENOMEM; } - dst_dev->vendor = src_dev->vendor; - dst_dev->device = src_dev->device; - dst_dev->domain = src_dev->domain; - dst_dev->bus = src_dev->bus; - dst_dev->slot = src_dev->slot; - dst_dev->func = src_dev->func; - + INIT_LIST_HEAD(&dst_dev->list); list_add_tail(&dst_dev->list, &instance->pci_devices); instance->pci_device_count++; } @@ -564,8 +562,10 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest struct mk_instance *host_instance; int host_ipi_node; const fdt64_t *phys_prop; + const fdt64_t *cpu_prop; const fdt32_t *pages_prop; phys_addr_t host_ipi_phys = 0; + mk_phys_cpu_t host_ipi_cpu = MK_PHYS_CPU_INVALID; u32 host_ipi_pages = 0; size_t host_ipi_size = 0; int len; @@ -585,10 +585,15 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest host_ipi_pages = fdt32_to_cpu(*pages_prop); host_ipi_size = (size_t)host_ipi_pages << PAGE_SHIFT; } + cpu_prop = fdt_getprop(manifest, host_ipi_node, "target-cpu", &len); + if (cpu_prop && len == sizeof(*cpu_prop)) + host_ipi_cpu = fdt64_to_cpu(*cpu_prop); - if (!host_ipi_phys || !host_ipi_pages) { - pr_warn("Incomplete host IPI buffer info (phys=0x%llx, pages=%u)\n", - (unsigned long long)host_ipi_phys, host_ipi_pages); + if (!host_ipi_phys || !host_ipi_pages || + host_ipi_cpu == MK_PHYS_CPU_INVALID) { + pr_warn("Incomplete host IPI buffer info (phys=0x%llx, pages=%u, target CPU=%llu)\n", + (unsigned long long)host_ipi_phys, host_ipi_pages, + (unsigned long long)host_ipi_cpu); return NULL; } @@ -596,8 +601,7 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest if (!host_instance) return NULL; - /* Set physical CPU 0 as default target for host IPIs */ - if (mk_cpu_set_add(host_instance->cpus, 0)) { + if (mk_cpu_set_add(host_instance->cpus, host_ipi_cpu)) { kfree(host_instance->name); mk_cpu_set_free(host_instance->cpus); kfree(host_instance); @@ -615,9 +619,9 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest } host_instance->ipi_phys = host_ipi_phys; host_instance->ipi_pages = host_ipi_pages; - pr_info("Restored host IPI buffer: phys=0x%llx, virt=%px, pages=%u\n", + pr_info("Restored host IPI buffer: phys=0x%llx, virt=%p, pages=%u, target CPU=%llu\n", (unsigned long long)host_ipi_phys, host_instance->ipi_data, - host_ipi_pages); + host_ipi_pages, (unsigned long long)host_ipi_cpu); pr_info("Registered host instance (ID 0) for spawn→host communication\n"); return host_instance; @@ -828,89 +832,6 @@ int __init mk_instance_restore_from_manifest(void) /* Run at early_initcall to enforce CPU restrictions before per-CPU allocations */ early_initcall(mk_instance_restore_from_manifest); -/** - * mk_pci_should_probe - Check if PCI probing should occur at all - * @bus: PCI bus - * @devfn: device/function number - * - * Called BEFORE any PCI config space reads to determine if probing - * should proceed. This prevents config space accesses to devices - * that are not in the whitelist. - * - * Returns: true if probing should proceed, false to skip entirely - */ -bool mk_pci_should_probe(struct pci_bus *bus, int devfn) -{ - struct mk_pci_device *pci_dev; - u16 domain = pci_domain_nr(bus); - u8 bus_num = bus->number; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - u8 hdr_type; - - if (!root_instance) - return true; - - if (!root_instance->dtb_data) - return true; - - if (!root_instance->pci_devices_valid || root_instance->pci_device_count == 0) - return false; - - list_for_each_entry(pci_dev, &root_instance->pci_devices, list) { - if (pci_dev->domain != domain) - continue; - - /* Exact location match - always allow */ - if (pci_dev->bus == bus_num && - pci_dev->slot == slot && - pci_dev->func == func) - return true; - } - - /* - * Check if any whitelisted device is on a downstream bus. - * If so, this might be a bridge in the path to that device. - */ - list_for_each_entry(pci_dev, &root_instance->pci_devices, list) { - if (pci_dev->domain == domain && pci_dev->bus > bus_num) - goto check_bridge; - } - return false; - -check_bridge: - /* - * There's a whitelisted device on a downstream bus. Check if this - * is a bridge that serves it. - */ - if (pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type) == 0) { - bool is_bridge = ((hdr_type & PCI_HEADER_TYPE_MASK) == PCI_HEADER_TYPE_BRIDGE); - - if (is_bridge) { - u8 secondary_bus = 0, subordinate_bus = 0; - - pci_bus_read_config_byte(bus, devfn, PCI_SECONDARY_BUS, &secondary_bus); - pci_bus_read_config_byte(bus, devfn, PCI_SUBORDINATE_BUS, &subordinate_bus); - - /* - * Allow bridge if there's a whitelisted device on any bus - * between secondary and subordinate (inclusive). - */ - if (secondary_bus > 0 && subordinate_bus >= secondary_bus) { - list_for_each_entry(pci_dev, &root_instance->pci_devices, list) { - if (pci_dev->domain == domain && - pci_dev->bus >= secondary_bus && - pci_dev->bus <= subordinate_bus) - return true; - } - } - } - } - - return false; -} -EXPORT_SYMBOL_GPL(mk_pci_should_probe); - bool mk_platform_device_allowed(const char *name, const char *hid) { struct mk_platform_device *plat_dev; diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 75e4b8d25eb939..112ef5e08ce94f 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -18,7 +18,8 @@ int mk_dt_parse_resources(const void *fdt, int resources_node, const char *instance_name, struct mk_dt_config *config); int mk_dt_generate_instance_dtb(struct mk_instance *instance, void **out_dtb, size_t *out_size); - +int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, + u8 *slot, u8 *func); /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; int mk_overlay_init(void); diff --git a/kernel/multikernel/overlay.c b/kernel/multikernel/overlay.c index 5098918fee048a..1fafe915b0ecaf 100644 --- a/kernel/multikernel/overlay.c +++ b/kernel/multikernel/overlay.c @@ -581,7 +581,8 @@ static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, fdt_for_each_subnode(item_node, fdt, op_node) { const char *name = fdt_get_name(fdt, item_node, NULL); const char *pci_id_str; - unsigned int domain, bus, slot, func; + u16 domain; + u8 bus, slot, func; if (strncmp(name, "pci@", 4) != 0) continue; @@ -593,9 +594,12 @@ static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s'\n", pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s'\n", + len, pci_id_str); + return ret; } pr_info("Overlay tx%d: -device %04x:%02x:%02x.%x from %s\n", @@ -624,7 +628,8 @@ static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, const char *name = fdt_get_name(fdt, item_node, NULL); const char *pci_id_str; const char *driver_name = NULL; - unsigned int domain, bus, slot, func; + u16 domain; + u8 bus, slot, func; u32 flags = 0; if (strncmp(name, "pci@", 4) != 0) @@ -637,9 +642,12 @@ static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s'\n", pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s'\n", + len, pci_id_str); + return ret; } /* Get optional driver override */ @@ -799,7 +807,8 @@ static int mk_overlay_parse_and_rollback(struct mk_overlay_tx *tx, fdt_for_each_subnode(item_node, fdt, op_node) { const char *name = fdt_get_name(fdt, item_node, NULL); const char *pci_id_str; - unsigned int domain, bus, slot, func; + u16 domain; + u8 bus, slot, func; if (strncmp(name, "pci@", 4) != 0) continue; @@ -810,9 +819,12 @@ static int mk_overlay_parse_and_rollback(struct mk_overlay_tx *tx, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s'\n", pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s'\n", + len, pci_id_str); + return ret; } pr_info("Rollback tx%d: -device %04x:%02x:%02x.%x from %s\n", @@ -845,7 +857,8 @@ static int mk_overlay_parse_and_rollback(struct mk_overlay_tx *tx, const char *name = fdt_get_name(fdt, item_node, NULL); const char *pci_id_str; const char *driver_name = NULL; - unsigned int domain, bus, slot, func; + u16 domain; + u8 bus, slot, func; u32 flags = 0; if (strncmp(name, "pci@", 4) != 0) @@ -857,9 +870,12 @@ static int mk_overlay_parse_and_rollback(struct mk_overlay_tx *tx, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s'\n", pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s'\n", + len, pci_id_str); + return ret; } /* Get optional driver override */ diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c new file mode 100644 index 00000000000000..61c5f9cbe15e10 --- /dev/null +++ b/kernel/multikernel/pci.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Multikernel PCI assignment policy + * + * Keeps assigned-device discovery, identity presentation, bridge traversal, + * and resource restoration independent from the manifest that populated + * the current instance. + */ + +#include +#include + +#include "internal.h" + +static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) +{ + struct mk_pci_device *device; + u16 domain = pci_domain_nr(bus); + u8 slot = PCI_SLOT(devfn); + u8 func = PCI_FUNC(devfn); + + if (!root_instance || !root_instance->dtb_data || + !root_instance->pci_devices_valid) + return NULL; + + list_for_each_entry(device, &root_instance->pci_devices, list) { + if (device->domain == domain && device->bus == bus->number && + device->slot == slot && device->func == func) + return device; + } + + return NULL; +} + +/** + * mk_pci_get_assigned_identity - Get the identity presented to an instance + * @bus: PCI bus + * @devfn: device/function number + * @vendor: assigned Vendor ID + * @device_id: assigned Device ID + * + * Returns: true when assignment metadata contains an exact location match. + */ +bool mk_pci_get_assigned_identity(struct pci_bus *bus, int devfn, + u16 *vendor, u16 *device_id) +{ + struct mk_pci_device *device = mk_pci_find_assigned(bus, devfn); + + if (!device) + return false; + + if (vendor) + *vendor = device->vendor; + if (device_id) + *device_id = device->device; + return true; +} + +static void mk_pci_restore_resources(struct pci_dev *dev) +{ + struct mk_pci_device *device; + int i; + + device = mk_pci_find_assigned(dev->bus, dev->devfn); + if (!device || !device->resources_valid) + return; + + dev->non_compliant_bars = true; + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + dev->resource[i].start = device->resources[i].start; + dev->resource[i].end = device->resources[i].end; + dev->resource[i].flags = device->resources[i].flags; + } + pr_info("Restored PCI BAR resources for %s\n", pci_name(dev)); +} + +DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, mk_pci_restore_resources); From 20cd77fc7a58013eaa7f40e26fc8eca4e862546c Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 07:55:41 +0300 Subject: [PATCH 03/16] x86/multikernel: build PCI roots for assigned functions Create one synthetic root bus for each assigned domain and bus, then scan only the explicitly assigned devfns. Supply an exact bus-number resource when x86 root resources do not provide one. Spawn kernels do not inherit or map host ECAM windows. Configuration is mediated by the filtered backend introduced with the assigned roots. Signed-off-by: Nikolay Nikolaev --- arch/x86/include/asm/multikernel.h | 6 ++ arch/x86/kernel/platform-quirks.c | 1 + arch/x86/multikernel/pci.c | 152 +++++++++++++++++++++++++++++ drivers/pci/probe.c | 9 -- 4 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 arch/x86/multikernel/pci.c diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index 584490b6246b74..bff6074bdc8fc1 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -11,6 +11,7 @@ #ifndef __ASSEMBLY__ #include +#include #include #include #include @@ -177,6 +178,11 @@ int multikernel_wakeup_secondary_cpu_64(u32 apicid, unsigned long start_eip, int multikernel_restore_ap(unsigned int cpu, unsigned long cr3, unsigned long gs_base, unsigned long stack, unsigned long entry); +#if defined(CONFIG_MULTIKERNEL) && defined(CONFIG_PCI) +void __init x86_multikernel_pci_platform_init(void); +#else +static inline void x86_multikernel_pci_platform_init(void) { } +#endif #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index feea109497efde..5227f8131a93a9 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -171,6 +171,7 @@ void __init x86_early_init_platform_quirks(void) break; case X86_SUBARCH_MULTIKERNEL: multikernel_setup_calibration(); + x86_multikernel_pci_platform_init(); x86_platform.legacy.devices.pnpbios = 0; x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; x86_platform.legacy.rtc = 0; diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c new file mode 100644 index 00000000000000..85d2dba737acd7 --- /dev/null +++ b/arch/x86/multikernel/pci.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * x86 PCI support for multikernel spawn kernels. + * + * Spawn kernels discover only assigned BDFs. Configuration accesses are + * filtered here and become host-mediated once the RPC transport is installed. + */ +#include +#include +#include +#include +#include + +#include +#include + +static struct pci_ops mk_pci_native_ops; +static bool mk_pci_roots_ready; + +static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, + u32 *value) +{ + u32 identity; + u32 mask; + + if (where < PCI_VENDOR_ID || where + size > PCI_COMMAND) + return false; + identity = vendor | (u32)device << 16; + mask = size == sizeof(identity) ? ~0U : (1U << (size * 8)) - 1; + *value = (identity >> (where * 8)) & mask; + return true; +} + +static int mk_pci_read(struct pci_bus *bus, unsigned int devfn, int where, + int size, u32 *value) +{ + u16 vendor, device; + + if (!mk_pci_get_assigned_identity(bus, devfn, &vendor, &device)) { + *value = ~0U; + return PCIBIOS_DEVICE_NOT_FOUND; + } + if (mk_pci_identity_read(vendor, device, where, size, value)) + return PCIBIOS_SUCCESSFUL; + return mk_pci_native_ops.read(bus, devfn, where, size, value); +} + +static int mk_pci_write(struct pci_bus *bus, unsigned int devfn, int where, + int size, u32 value) +{ + if (!mk_pci_get_assigned_identity(bus, devfn, NULL, NULL)) + return PCIBIOS_DEVICE_NOT_FOUND; + return mk_pci_native_ops.write(bus, devfn, where, size, value); +} + +static int __init x86_multikernel_pci_arch_init(void) +{ + if (!root_instance || !root_instance->pci_devices_valid) + return 0; + + mk_pci_native_ops = pci_root_ops; + pci_root_ops.read = mk_pci_read; + pci_root_ops.write = mk_pci_write; + mk_pci_roots_ready = true; + pr_notice("Multikernel selected filtered PCI config access\n"); + return 0; +} + +static struct pci_bus * __init mk_pci_get_root(u16 domain, u8 bus_number) +{ + struct resource_entry *window; + struct pci_sysdata *sd; + struct pci_bus *bus; + bool has_busn_res = false; + LIST_HEAD(resources); + + if (domain && !pci_domains_supported) { + pr_err("Multikernel cannot scan PCI root %04x:%02x without domain support\n", + domain, bus_number); + return ERR_PTR(-EOPNOTSUPP); + } + bus = pci_find_bus(domain, bus_number); + if (bus) + return bus; + + sd = kzalloc_obj(*sd, GFP_KERNEL); + if (!sd) + return ERR_PTR(-ENOMEM); + sd->domain = domain; + sd->node = x86_pci_root_bus_node(bus_number); + x86_pci_root_bus_resources(bus_number, &resources); + resource_list_for_each_entry(window, &resources) { + if (window->res->flags & IORESOURCE_BUS) { + has_busn_res = true; + break; + } + } + bus = pci_create_root_bus(NULL, bus_number, &pci_root_ops, sd, + &resources); + if (!bus) { + pci_free_resource_list(&resources); + kfree(sd); + return ERR_PTR(-ENOMEM); + } + if (!has_busn_res && + !pci_bus_insert_busn_res(bus, bus_number, bus_number)) { + pci_remove_root_bus(bus); + kfree(sd); + return ERR_PTR(-EBUSY); + } + pr_notice("Multikernel created synthetic PCI root %04x:%02x\n", + domain, bus_number); + return bus; +} + +static int __init x86_multikernel_pci_init(void) +{ + const struct mk_pci_device *device; + struct pci_bus *bus; + struct pci_dev *pdev; + + if (!root_instance) + panic("Multikernel lost restored instance metadata"); + if (!root_instance->pci_device_count) + return 0; + if (!root_instance->pci_devices_valid || !mk_pci_roots_ready) + panic("Multikernel assigned PCI inventory is unavailable"); + + list_for_each_entry(device, &root_instance->pci_devices, list) { + bus = mk_pci_get_root(device->domain, device->bus); + if (IS_ERR(bus)) + panic("Multikernel failed to create synthetic PCI root %04x:%02x: %ld", + device->domain, device->bus, PTR_ERR(bus)); + pdev = pci_scan_single_device(bus, + PCI_DEVFN(device->slot, device->func)); + if (!pdev) + panic("Multikernel failed to enumerate assigned PCI device %04x:%02x:%02x.%x", + device->domain, device->bus, device->slot, + device->func); + pci_bus_add_devices(bus); + } + + /* Suppress legacy bus 0 probing after every assigned function is present. */ + return 0; +} + +void __init x86_multikernel_pci_platform_init(void) +{ + pci_probe = PCI_PROBE_NOEARLY; + x86_init.pci.arch_init = x86_multikernel_pci_arch_init; + x86_init.pci.init = x86_multikernel_pci_init; +} diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 7661517ebdf0df..bccc7a4bdd7943 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -24,7 +24,6 @@ #include #include #include -#include #include "pci.h" static struct resource busn_resource = { @@ -2601,14 +2600,6 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn) struct pci_dev *dev; u32 l; - /* - * For multikernel spawns, check if we should even probe this location - * BEFORE any config space access. This prevents hardware conflicts - * when the host kernel is also using PCI devices. - */ - if (IS_ENABLED(CONFIG_MULTIKERNEL) && !mk_pci_should_probe(bus, devfn)) - return NULL; - if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000)) return NULL; From 081793500fb3d3a1c326e37c20a030027bfc53c7 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 4 Aug 2026 08:26:40 +0300 Subject: [PATCH 04/16] x86/multikernel: filter raw PCI config operations Synthetic root filtering alone leaves raw x86 configuration backends able to reach functions outside the spawn inventory. Apply the assigned-BDF filter to both raw configuration entry points. Identity reads come from validated metadata while other accesses use the selected backend only for an assigned function. Signed-off-by: Nikolay Nikolaev --- arch/x86/multikernel/pci.c | 70 ++++++++++++++++++++++++++++++------- include/linux/multikernel.h | 30 +++++++--------- kernel/multikernel/pci.c | 27 +++++++++----- 3 files changed, 89 insertions(+), 38 deletions(-) diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c index 85d2dba737acd7..2a3d9a84375386 100644 --- a/arch/x86/multikernel/pci.c +++ b/arch/x86/multikernel/pci.c @@ -14,7 +14,8 @@ #include #include -static struct pci_ops mk_pci_native_ops; +static const struct pci_raw_ops *mk_pci_native_raw_ops; +static const struct pci_raw_ops *mk_pci_native_raw_ext_ops; static bool mk_pci_roots_ready; static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, @@ -31,38 +32,81 @@ static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, return true; } -static int mk_pci_read(struct pci_bus *bus, unsigned int devfn, int where, - int size, u32 *value) +static int mk_pci_raw_read(const struct pci_raw_ops *native, + unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + u32 *value) { u16 vendor, device; - if (!mk_pci_get_assigned_identity(bus, devfn, &vendor, &device)) { + if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, &vendor, + &device)) { *value = ~0U; return PCIBIOS_DEVICE_NOT_FOUND; } if (mk_pci_identity_read(vendor, device, where, size, value)) return PCIBIOS_SUCCESSFUL; - return mk_pci_native_ops.read(bus, devfn, where, size, value); + return native->read(domain, bus, devfn, where, size, value); } -static int mk_pci_write(struct pci_bus *bus, unsigned int devfn, int where, - int size, u32 value) +static int mk_pci_raw_write(const struct pci_raw_ops *native, + unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + u32 value) { - if (!mk_pci_get_assigned_identity(bus, devfn, NULL, NULL)) + if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, NULL, NULL)) return PCIBIOS_DEVICE_NOT_FOUND; - return mk_pci_native_ops.write(bus, devfn, where, size, value); + return native->write(domain, bus, devfn, where, size, value); } +static int mk_pci_read(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, u32 *value) +{ + return mk_pci_raw_read(mk_pci_native_raw_ops, domain, bus, devfn, + where, size, value); +} + +static int mk_pci_write(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, u32 value) +{ + return mk_pci_raw_write(mk_pci_native_raw_ops, domain, bus, devfn, + where, size, value); +} + +static int mk_pci_ext_read(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, u32 *value) +{ + return mk_pci_raw_read(mk_pci_native_raw_ext_ops, domain, bus, devfn, + where, size, value); +} + +static int mk_pci_ext_write(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, u32 value) +{ + return mk_pci_raw_write(mk_pci_native_raw_ext_ops, domain, bus, devfn, + where, size, value); +} + +static const struct pci_raw_ops mk_pci_filtered_raw_ops = { + .read = mk_pci_read, + .write = mk_pci_write, +}; + +static const struct pci_raw_ops mk_pci_filtered_raw_ext_ops = { + .read = mk_pci_ext_read, + .write = mk_pci_ext_write, +}; static int __init x86_multikernel_pci_arch_init(void) { if (!root_instance || !root_instance->pci_devices_valid) return 0; - mk_pci_native_ops = pci_root_ops; - pci_root_ops.read = mk_pci_read; - pci_root_ops.write = mk_pci_write; + mk_pci_native_raw_ops = raw_pci_ops; + mk_pci_native_raw_ext_ops = raw_pci_ext_ops; + raw_pci_ops = &mk_pci_filtered_raw_ops; + raw_pci_ext_ops = &mk_pci_filtered_raw_ext_ops; mk_pci_roots_ready = true; - pr_notice("Multikernel selected filtered PCI config access\n"); + pr_notice("Multikernel selected filtered raw PCI config access\n"); return 0; } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 3f36f4a338eb72..3fbd41e6957fd0 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -16,7 +16,6 @@ #include struct pci_bus; -struct pci_ops; /** * Physical CPU identifiers @@ -774,11 +773,10 @@ void mk_instance_set_state(struct mk_instance *instance, void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size); -/* Device probe filtering against the instance's allowlist */ -bool mk_pci_should_probe(struct pci_bus *bus, int devfn, - const struct pci_ops *ops); -bool mk_pci_get_assigned_identity(struct pci_bus *bus, int devfn, - u16 *vendor, u16 *device); +/* Device filtering against the instance metadata */ +bool mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, + u16 *device); bool mk_platform_device_allowed(const char *name, const char *hid); /* Early CPU registration from the manifest (spawn kernels) */ @@ -827,11 +825,6 @@ static inline void mk_kimage_free(struct kimage *image, void *virt_addr, { } -static inline bool mk_pci_should_probe(struct pci_bus *bus, int devfn, - const struct pci_ops *ops) -{ - return true; -} static inline bool mk_platform_device_allowed(const char *name, const char *hid) { return true; @@ -925,19 +918,22 @@ int __init mk_instance_restore_from_manifest(void); */ /** - * PCI Device Enforcement Functions + * PCI Device Filtering Functions */ /** - * mk_pci_get_assigned_identity() - Check and identify an assigned PCI function - * @bus: PCI bus + * mk_pci_get_assigned_identity_bdf() - Identify an assigned PCI function + * @domain: PCI domain number + * @bus: PCI bus number * @devfn: PCI device/function number * @vendor: optional assigned vendor ID output * @device: optional assigned device ID output * - * Synthetic roots make assigned functions directly discoverable. Only exact - * assignment metadata matches may access config space; physical bridges and - * all other functions remain inaccessible. + * The raw x86 PCI configuration wrappers use this BDF-only lookup before + * reaching their hardware backend. Synthetic roots make assigned functions + * directly discoverable. A privileged spawn kernel can bypass those wrappers, + * so this check prevents accidental access rather than isolating a hostile + * kernel. * * Returns: true for an exact assignment metadata match, false otherwise */ diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 61c5f9cbe15e10..23b713d9aeedb9 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -12,10 +12,10 @@ #include "internal.h" -static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) +static struct mk_pci_device * +mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) { struct mk_pci_device *device; - u16 domain = pci_domain_nr(bus); u8 slot = PCI_SLOT(devfn); u8 func = PCI_FUNC(devfn); @@ -24,7 +24,7 @@ static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn return NULL; list_for_each_entry(device, &root_instance->pci_devices, list) { - if (device->domain == domain && device->bus == bus->number && + if (device->domain == domain && device->bus == bus && device->slot == slot && device->func == func) return device; } @@ -32,19 +32,30 @@ static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn return NULL; } +static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) +{ + return mk_pci_find_assigned_bdf(pci_domain_nr(bus), bus->number, devfn); +} + /** - * mk_pci_get_assigned_identity - Get the identity presented to an instance - * @bus: PCI bus + * mk_pci_get_assigned_identity_bdf - Get an assigned function's identity + * @domain: PCI domain number + * @bus: PCI bus number * @devfn: device/function number * @vendor: assigned Vendor ID * @device_id: assigned Device ID * * Returns: true when assignment metadata contains an exact location match. */ -bool mk_pci_get_assigned_identity(struct pci_bus *bus, int devfn, - u16 *vendor, u16 *device_id) +bool mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, + u16 *device_id) { - struct mk_pci_device *device = mk_pci_find_assigned(bus, devfn); + struct mk_pci_device *device; + + if (domain != (u16)domain || bus != (u8)bus || devfn != (u8)devfn) + return false; + device = mk_pci_find_assigned_bdf(domain, bus, devfn); if (!device) return false; From 5577293f16f2e9253fb7fc8c8e0d4cffd58c13b1 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Thu, 30 Jul 2026 08:17:44 +0300 Subject: [PATCH 05/16] pci/multikernel: add exclusive SR-IOV VF leases Reserve assigned VFs transactionally under a host-wide lease lock. Detach the host driver, mark the device assigned, and publish ownership only when the complete request succeeds. Propagate host-driver restoration failures and retain failed instance state when cleanup cannot be completed safely. Signed-off-by: Nikolay Nikolaev --- Documentation/multikernel/usage.rst | 17 + include/linux/multikernel.h | 2 + kernel/multikernel/baseline.c | 60 +-- kernel/multikernel/core.c | 296 +++++---------- kernel/multikernel/hotplug.c | 131 ++----- kernel/multikernel/instance_dt.c | 1 + kernel/multikernel/internal.h | 14 + kernel/multikernel/kernfs.c | 1 + kernel/multikernel/pci.c | 564 +++++++++++++++++++++++++++- 9 files changed, 745 insertions(+), 341 deletions(-) diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index a2ec8d56ca1d06..fdb65094eacb81 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -122,6 +122,23 @@ Phase 2: Kernel Loading (Kexec Integration) - Restore the instance's DTB and recreate the instance structure - Re-reserve the same memory and CPU resources +SR-IOV Assignment Boundary +========================== + +SR-IOV assignment is intended for cooperative spawned kernels. Filtering +configuration-space access and enumerating only assigned BDFs prevents +accidental access by those kernels; it is not a security boundary against a +privileged kernel that deliberately issues configuration cycles or maps host +physical windows. The host-owned IOMMU domain is the boundary that constrains +device-initiated DMA. + +The host fails assignment closed unless it can establish the complete device +lifecycle: the device is an SR-IOV VF in a singleton IOMMU group, reset is +available, MSI or MSI-X programming remains isolated and host-owned, and one +coherent host-owned IOMMU domain covers the assigned memory. Reset, interrupt, +or DMA teardown uncertainty leaves the instance failed rather than returning +the VF to use. + Device Tree Format ================== diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 3fbd41e6957fd0..6fd49ff9b13fa7 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -502,6 +502,8 @@ struct mk_instance { struct list_head pci_devices; /* List of struct mk_pci_device */ int pci_device_count; /* Number of PCI devices */ bool pci_devices_valid; /* Whether PCI device list is valid */ + /* Host-only live PCI assignment leases (private elements). */ + struct list_head pci_assignments; /* Platform device resources */ struct list_head platform_devices; /* List of struct mk_platform_device */ diff --git a/kernel/multikernel/baseline.c b/kernel/multikernel/baseline.c index 89d97fe9e50aca..c2e88e2ad84122 100644 --- a/kernel/multikernel/baseline.c +++ b/kernel/multikernel/baseline.c @@ -526,54 +526,54 @@ static int mk_baseline_initialize_devices(const struct mk_instance *instance) { struct mk_pci_device *pci_dev; struct pci_dev *dev; - int failed = 0, unbound = 0; + int failed = 0; + int available = 0; - if (instance->pci_device_count == 0) { - pr_debug("No PCI devices in baseline to unbind\n"); + if (!instance->pci_device_count) { + pr_debug("No PCI devices in the multikernel pool\n"); return 0; } - pr_info("Unbinding %d PCI devices for multikernel pool\n", + pr_info("Validating %d PCI devices for the multikernel pool\n", instance->pci_device_count); + pci_lock_rescan_remove(); list_for_each_entry(pci_dev, &instance->pci_devices, list) { - dev = pci_get_domain_bus_and_slot(pci_dev->domain, pci_dev->bus, - PCI_DEVFN(pci_dev->slot, pci_dev->func)); + dev = pci_get_domain_bus_and_slot(pci_dev->domain, + pci_dev->bus, + PCI_DEVFN(pci_dev->slot, + pci_dev->func)); if (!dev) { pr_warn("PCI device %04x:%04x@%04x:%02x:%02x.%x not found in system\n", - pci_dev->vendor, pci_dev->device, pci_dev->domain, - pci_dev->bus, pci_dev->slot, pci_dev->func); + pci_dev->vendor, pci_dev->device, + pci_dev->domain, pci_dev->bus, + pci_dev->slot, pci_dev->func); failed++; continue; } - if (!dev->driver) { - pr_debug("PCI device %04x:%04x@%04x:%02x:%02x.%x already unbound\n", - pci_dev->vendor, pci_dev->device, pci_dev->domain, - pci_dev->bus, pci_dev->slot, pci_dev->func); - pci_dev_put(dev); - unbound++; - continue; + if (dev->vendor != pci_dev->vendor || + dev->device != pci_dev->device || + pci_dev_is_disconnected(dev) || + !pci_device_is_present(dev)) { + pr_warn("PCI device %04x:%04x@%04x:%02x:%02x.%x is not available\n", + pci_dev->vendor, pci_dev->device, + pci_dev->domain, pci_dev->bus, + pci_dev->slot, pci_dev->func); + failed++; + } else { + available++; } - - const char *driver_name = dev->driver->name; - - device_release_driver(&dev->dev); - - pr_info("Unbound PCI device %04x:%04x@%04x:%02x:%02x.%x (was: %s) for multikernel pool\n", - pci_dev->vendor, pci_dev->device, pci_dev->domain, - pci_dev->bus, pci_dev->slot, pci_dev->func, - driver_name); - pci_dev_put(dev); - unbound++; } + pci_unlock_rescan_remove(); - if (failed > 0) { - pr_warn("Failed to find %d PCI devices in system\n", failed); - } + if (failed) + pr_warn("%d PCI devices in the multikernel pool are unavailable\n", + failed); - pr_info("Successfully unbound %d PCI devices for multikernel pool\n", unbound); + pr_info("Validated %d PCI devices; host drivers remain bound until assignment\n", + available); return 0; } #else diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index f91763107cf950..bb9b1fc1499f5f 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -26,16 +26,20 @@ static void mk_instance_return_all_cpus(struct mk_instance *instance) mk_instance_return_cpus(instance, instance->cpus); } -static void mk_instance_return_pci_devices(struct mk_instance *instance) +static int mk_instance_return_pci_devices(struct mk_instance *instance) { struct mk_pci_device *pci_dev, *pci_tmp; int returned_count = 0; + int ret; - if (!instance || !instance->pci_devices_valid) - return; + if (!instance || instance == root_instance || instance->id == 0) + return 0; - if (instance == root_instance || instance->id == 0) - return; + ret = mk_pci_release_assignments(instance); + if (ret) + return ret; + if (!instance->pci_devices_valid) + return 0; if (!root_instance) { pr_warn("Cannot return PCI devices from instance %d (%s): no root instance\n", @@ -43,43 +47,28 @@ static void mk_instance_return_pci_devices(struct mk_instance *instance) goto cleanup; } - list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, list) { - struct mk_pci_device *root_dev; - - root_dev = kzalloc(sizeof(*root_dev), GFP_KERNEL); - if (!root_dev) { - pr_warn("Failed to allocate PCI device entry for root instance\n"); - continue; - } - - *root_dev = *pci_dev; - INIT_LIST_HEAD(&root_dev->list); - - list_add_tail(&root_dev->list, &root_instance->pci_devices); + list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, + list) { + list_move_tail(&pci_dev->list, &root_instance->pci_devices); root_instance->pci_device_count++; root_instance->pci_devices_valid = true; - - pr_debug("Returned PCI device %04x:%02x:%02x.%d from instance %d to root\n", - root_dev->domain, root_dev->bus, root_dev->slot, - root_dev->func, instance->id); - returned_count++; } - if (returned_count > 0) { + if (returned_count) pr_info("Returned %d PCI devices from instance %d (%s) to root instance\n", returned_count, instance->id, instance->name); - } cleanup: - list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, list) { + list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, + list) { list_del(&pci_dev->list); kfree(pci_dev); } instance->pci_device_count = 0; instance->pci_devices_valid = false; + return 0; } - static void mk_instance_return_platform_devices(struct mk_instance *instance) { struct mk_platform_device *plat_dev, *plat_tmp; @@ -133,18 +122,39 @@ static void mk_instance_return_platform_devices(struct mk_instance *instance) instance->platform_devices_valid = false; } +int mk_instance_release_resources(struct mk_instance *instance) +{ + int ret; + + if (!instance || instance == root_instance || instance->id == 0) + return 0; + ret = mk_instance_return_pci_devices(instance); + if (ret) + return ret; + mk_instance_return_platform_devices(instance); + mk_instance_return_all_cpus(instance); + mk_instance_free_memory(instance); + return 0; +} + static void mk_instance_release(struct kref *kref) { struct mk_instance *instance = container_of(kref, struct mk_instance, refcount); + int ret; pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", instance->id, instance->name); - - mk_instance_return_all_cpus(instance); - mk_instance_return_pci_devices(instance); - mk_instance_return_platform_devices(instance); - mk_instance_free_memory(instance); - + ret = mk_instance_release_resources(instance); + if (WARN_ON_ONCE(ret)) { + /* + * A failed PCI release can leave assignments linked to this + * instance. Retain the backing object rather than leave those + * assignments with a dangling instance pointer. + */ + pr_crit("Retaining multikernel instance %d (%s) after resource release failed: %d\n", + instance->id, instance->name, ret); + return; + } mk_cpu_set_free(instance->cpus); kfree(instance->dtb_data); kfree(instance->name); @@ -459,81 +469,21 @@ static int mk_instance_transfer_pci_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count) { - struct mk_pci_device *req_dev, *root_dev, *tmp; - int transferred = 0; - int not_found = 0; - bool found; - if (!root_instance || !root_instance->pci_devices_valid) { pr_err("No root instance or PCI devices not initialized\n"); return -EINVAL; } - if (requested_count == 0 || list_empty(requested_devices)) { + if (!requested_count || list_empty(requested_devices)) { pr_info("No PCI devices requested for instance %d (%s)\n", instance->id, instance->name); instance->pci_devices_valid = true; return 0; } - list_for_each_entry(req_dev, requested_devices, list) { - found = false; - list_for_each_entry(root_dev, &root_instance->pci_devices, list) { - if (root_dev->vendor == req_dev->vendor && - root_dev->device == req_dev->device && - root_dev->domain == req_dev->domain && - root_dev->bus == req_dev->bus && - root_dev->slot == req_dev->slot && - root_dev->func == req_dev->func) { - found = true; - break; - } - } - if (!found) { - pr_err("PCI device %04x:%04x@%04x:%02x:%02x.%x not available in root pool\n", - req_dev->vendor, req_dev->device, req_dev->domain, - req_dev->bus, req_dev->slot, req_dev->func); - not_found++; - } - } - - if (not_found > 0) { - pr_err("Instance %d (%s): %d PCI devices not available\n", - instance->id, instance->name, not_found); - return -ENOENT; - } - - list_for_each_entry(req_dev, requested_devices, list) { - list_for_each_entry_safe(root_dev, tmp, &root_instance->pci_devices, list) { - if (root_dev->vendor == req_dev->vendor && - root_dev->device == req_dev->device && - root_dev->domain == req_dev->domain && - root_dev->bus == req_dev->bus && - root_dev->slot == req_dev->slot && - root_dev->func == req_dev->func) { - - list_del(&root_dev->list); - list_add_tail(&root_dev->list, &instance->pci_devices); - root_instance->pci_device_count--; - instance->pci_device_count++; - transferred++; - - pr_debug("Transferred PCI device %04x:%04x@%04x:%02x:%02x.%x to instance %d\n", - root_dev->vendor, root_dev->device, root_dev->domain, - root_dev->bus, root_dev->slot, root_dev->func, - instance->id); - break; - } - } - } - - instance->pci_devices_valid = true; - pr_info("Transferred %d PCI devices from root to instance %d (%s), root pool remaining: %d devices\n", - transferred, instance->id, instance->name, root_instance->pci_device_count); - - return 0; + return mk_pci_assign_devices(instance, requested_devices, + requested_count); } - static int mk_instance_reserve_pci_devices(struct mk_instance *instance, const struct mk_dt_config *config) { @@ -646,37 +596,14 @@ static int mk_instance_reserve_platform_devices(struct mk_instance *instance, int mk_instance_add_pci_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { - struct mk_pci_device *root_dev, *tmp; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - - if (!root_instance || !root_instance->pci_devices_valid) { - pr_err("No root instance or PCI devices not initialized\n"); - return -EINVAL; - } - - list_for_each_entry_safe(root_dev, tmp, &root_instance->pci_devices, list) { - if (root_dev->domain == domain && - root_dev->bus == bus && - root_dev->slot == slot && - root_dev->func == func) { - - list_del(&root_dev->list); - list_add_tail(&root_dev->list, &instance->pci_devices); - root_instance->pci_device_count--; - instance->pci_device_count++; - instance->pci_devices_valid = true; - - pr_info("Transferred PCI device %04x:%04x@%04x:%02x:%02x.%x to instance %d\n", - root_dev->vendor, root_dev->device, domain, bus, slot, func, - instance->id); - return 0; - } - } + int ret; - pr_err("PCI device %04x:%02x:%02x.%x not found in root pool\n", - domain, bus, slot, func); - return -ENOENT; + ret = mk_pci_assign_device(instance, domain, bus, devfn); + if (!ret) + pr_info("Leased PCI VF %04x:%02x:%02x.%x to instance %d\n", + domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), + instance->id); + return ret; } /** @@ -687,63 +614,22 @@ int mk_instance_add_pci_device(struct mk_instance *instance, * @devfn: PCI device and function (combined) * * Returns a single PCI device from the specified instance back to root instance. - * Used for dynamic PCI device hotplug from non-running instances. + * Dynamic assignment changes are accepted only while the instance is ready. * * Returns: 0 on success, negative error code on failure */ int mk_instance_remove_pci_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { - struct mk_pci_device *inst_dev, *tmp; - struct mk_pci_device *root_dev; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - - if (!instance->pci_devices_valid) { - pr_err("Instance %d PCI devices not initialized\n", instance->id); - return -EINVAL; - } - - if (!root_instance) { - pr_err("Cannot return PCI device: no root instance\n"); - return -EINVAL; - } - - list_for_each_entry_safe(inst_dev, tmp, &instance->pci_devices, list) { - if (inst_dev->domain == domain && - inst_dev->bus == bus && - inst_dev->slot == slot && - inst_dev->func == func) { - - root_dev = kzalloc(sizeof(*root_dev), GFP_KERNEL); - if (!root_dev) { - pr_err("Failed to allocate PCI device entry for root instance\n"); - return -ENOMEM; - } - - *root_dev = *inst_dev; - INIT_LIST_HEAD(&root_dev->list); - - list_add_tail(&root_dev->list, &root_instance->pci_devices); - root_instance->pci_device_count++; - root_instance->pci_devices_valid = true; - - list_del(&inst_dev->list); - kfree(inst_dev); - instance->pci_device_count--; - - pr_info("Returned PCI device %04x:%04x@%04x:%02x:%02x.%x from instance %d to root\n", - root_dev->vendor, root_dev->device, domain, bus, slot, func, - instance->id); - return 0; - } - } + int ret; - pr_err("PCI device %04x:%02x:%02x.%x not found in instance %d\n", - domain, bus, slot, func, instance->id); - return -ENOENT; + ret = mk_pci_unassign_device(instance, domain, bus, devfn); + if (!ret) + pr_info("Released PCI VF %04x:%02x:%02x.%x from instance %d\n", + domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), + instance->id); + return ret; } - /** * Memory management functions for instances */ @@ -1387,8 +1273,8 @@ int multikernel_halt_by_id(int mk_id) } /** - * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI - * @mk_id: Instance ID to halt + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop * * Forces a spawn kernel's CPUs to stop by queuing a shutdown message in the * IPI ring buffer and sending NMIs directly to each CPU. The NMI handler @@ -1399,64 +1285,70 @@ int multikernel_halt_by_id(int mk_id) * * Returns: 0 on success, negative error code on failure */ -int multikernel_force_halt_by_id(int mk_id) +int mk_instance_force_halt(struct mk_instance *instance) { - struct mk_instance *instance; struct mk_shutdown_payload payload; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); - if (!instance) - return -ENOENT; - if (instance->state != MK_STATE_ACTIVE) { pr_err("Instance %d not active (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } if (mk_cpu_set_empty(instance->cpus)) { - pr_err("Instance %d has no CPUs assigned\n", mk_id); - mk_instance_put(instance); + pr_err("Instance %d has no CPUs assigned\n", instance->id); return -EINVAL; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); + pr_info("Force halting multikernel instance %d via NMI\n", + instance->id); - /* Queue shutdown message - NMI handler will check for this */ payload.flags = MK_SHUTDOWN_IMMEDIATE; payload.sender_instance_id = root_instance->id; - ret = mk_send_message(mk_id, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, + ret = mk_send_message(instance->id, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, &payload, sizeof(payload)); if (ret < 0) - pr_err("Failed to queue shutdown message: %d (sending NMI anyway)\n", ret); + pr_err("Failed to queue shutdown message: %d (sending NMI anyway)\n", + ret); - /* Send NMI to each CPU in the instance */ mk_cpu_set_for_each(i, phys_cpu, instance->cpus) { mk_force_stop_cpu(phys_cpu); cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); - - /* - * The NMI handler parks each CPU on the instance's context. Wait - * for them to arrive before reporting the instance re-spawnable, - * exactly as the graceful path does after its shutdown ACK. - */ + pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, + instance->id); mk_instance_settle_halted(instance); - mk_instance_put(instance); return 0; } +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + + ret = mk_instance_force_halt(instance); + mk_instance_put(instance); + return ret; +} static int __init multikernel_init(void) { int ret; + ret = mk_pci_lease_system_init(); + if (ret) { + pr_err("Failed to initialize PCI assignment leases: %d\n", ret); + return ret; + } + /* Register NMI handler for forcible shutdown */ ret = mk_register_stop_nmi_handler(); if (ret < 0) { @@ -1467,6 +1359,7 @@ static int __init multikernel_init(void) ret = mk_messaging_init(); if (ret < 0) { pr_err("Failed to initialize multikernel messaging: %d\n", ret); + mk_pci_lease_system_cleanup(); return ret; } @@ -1474,6 +1367,7 @@ static int __init multikernel_init(void) if (ret < 0) { pr_err("Failed to register system message handler: %d\n", ret); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } @@ -1482,6 +1376,7 @@ static int __init multikernel_init(void) pr_err("Failed to initialize multikernel hotplug: %d\n", ret); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } @@ -1491,6 +1386,7 @@ static int __init multikernel_init(void) mk_hotplug_cleanup(); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 8ad4a9411ecbc3..ff1cd381a38145 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -1370,117 +1370,66 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) } /** - * mk_send_device_add - Add PCI device to instance and wait for completion + * mk_send_device_add - Assign a PCI device to an instance * @instance_id: Target instance ID * @domain: PCI domain * @bus: PCI bus * @devfn: PCI device and function (combined) - * @driver_override: Target driver name for binding (can be NULL) - * @flags: Additional flags + * @driver_override: Target driver name for a root-kernel add + * @flags: Additional root-kernel add flags * - * For local instance, executes addition synchronously. - * For remote instance, sends IPI and waits for ACK response. - * For instances that are not yet running (MK_STATE_READY/LOADED), - * adds device to instance's device list. + * Remote assignment changes are permitted only while the target instance is + * ready. Active instances must be stopped and returned to ready state first. * * Returns: 0 on success, negative error code on failure */ int mk_send_device_add(int instance_id, u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { - struct mk_device_resource_payload payload = { - .domain = domain, - .bus = bus, - .devfn = devfn, - .flags = flags, - .sender_instance_id = root_instance->id - }; - struct mk_pending_msg *pending; struct mk_instance *target_instance; int ret; - u32 resource_id; - - if (driver_override) - strscpy(payload.driver_override, driver_override, sizeof(payload.driver_override)); - else - payload.driver_override[0] = '\0'; - - resource_id = (domain << 16) | (bus << 8) | devfn; + if (!root_instance) + return -ENODEV; if (instance_id == root_instance->id) - return mk_do_device_add(domain, bus, devfn, driver_override, flags); + return mk_do_device_add(domain, bus, devfn, driver_override, + flags); target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; - if (target_instance->state != MK_STATE_ACTIVE) { - ret = mk_instance_add_pci_device(target_instance, domain, bus, devfn); - goto out; - } - - pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_DEVICE_ADD, resource_id); - if (!pending) { - ret = -ENOMEM; - goto out; - } - - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_DEVICE_ADD, - &payload, sizeof(payload)); - if (ret < 0) { - mk_msg_pending_wait(pending, 0); - goto out; - } - - ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) - goto out; - - ret = mk_instance_add_pci_device(target_instance, domain, bus, devfn); - if (ret < 0) { - pr_warn("Device added to target but failed to update tracking: %d\n", ret); + if (target_instance->state != MK_STATE_READY) { + pr_err("PCI assignment changes require instance %d to be ready\n", + instance_id); + ret = -EBUSY; + } else { + ret = mk_instance_add_pci_device(target_instance, domain, bus, + devfn); } - - pr_info("Multikernel hotplug: Device %04x:%02x:%02x.%x successfully added to instance %d\n", - domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), instance_id); - - ret = 0; -out: mk_instance_put(target_instance); return ret; } /** - * mk_send_device_remove - Remove PCI device from instance and wait for completion + * mk_send_device_remove - Release a PCI device from an instance * @instance_id: Target instance ID * @domain: PCI domain * @bus: PCI bus * @devfn: PCI device and function (combined) * - * For local instance, executes removal synchronously. - * For remote instance, sends IPI and waits for ACK response. - * For instances that are not yet running (MK_STATE_READY/LOADED), - * removes device from instance's device list. + * Remote assignment changes are permitted only while the target instance is + * ready. Active instances must be stopped and returned to ready state first. * * Returns: 0 on success, negative error code on failure */ int mk_send_device_remove(int instance_id, u16 domain, u8 bus, u8 devfn) { - struct mk_device_resource_payload payload = { - .domain = domain, - .bus = bus, - .devfn = devfn, - .flags = 0, - .sender_instance_id = root_instance->id - }; - struct mk_pending_msg *pending; struct mk_instance *target_instance; int ret; - u32 resource_id; - - payload.driver_override[0] = '\0'; - resource_id = (domain << 16) | (bus << 8) | devfn; + if (!root_instance) + return -ENODEV; if (instance_id == root_instance->id) return mk_do_device_remove(domain, bus, devfn); @@ -1488,38 +1437,14 @@ int mk_send_device_remove(int instance_id, u16 domain, u8 bus, u8 devfn) if (!target_instance) return -ENODEV; - if (target_instance->state != MK_STATE_ACTIVE) { - ret = mk_instance_remove_pci_device(target_instance, domain, bus, devfn); - goto out; - } - - pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_DEVICE_REMOVE, resource_id); - if (!pending) { - ret = -ENOMEM; - goto out; - } - - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_DEVICE_REMOVE, - &payload, sizeof(payload)); - if (ret < 0) { - mk_msg_pending_wait(pending, 0); - goto out; - } - - ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) - goto out; - - ret = mk_instance_remove_pci_device(target_instance, domain, bus, devfn); - if (ret < 0) { - pr_warn("Device removed from target but failed to update tracking: %d\n", ret); + if (target_instance->state != MK_STATE_READY) { + pr_err("PCI assignment changes require instance %d to be ready\n", + instance_id); + ret = -EBUSY; + } else { + ret = mk_instance_remove_pci_device(target_instance, domain, bus, + devfn); } - - pr_info("Multikernel hotplug: Device %04x:%02x:%02x.%x successfully removed from instance %d\n", - domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), instance_id); - - ret = 0; -out: mk_instance_put(target_instance); return ret; } diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index b82ec1a987efd1..a97cb1b6ea0ed0 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -407,6 +407,7 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char INIT_LIST_HEAD(&instance->list); kref_init(&instance->refcount); INIT_LIST_HEAD(&instance->pci_devices); + mk_pci_lease_instance_init(instance); instance->pci_devices_valid = false; instance->pci_device_count = 0; INIT_LIST_HEAD(&instance->platform_devices); diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 112ef5e08ce94f..1d67f8a26e89c2 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -20,6 +20,20 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, void **out_dtb, size_t *out_size); int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, u8 *slot, u8 *func); + +/* pci.c */ +int mk_pci_lease_system_init(void); +void mk_pci_lease_system_cleanup(void); +void mk_pci_lease_instance_init(struct mk_instance *instance); +int mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count); +int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn); +int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn); +int mk_pci_release_assignments(struct mk_instance *instance); +int mk_instance_force_halt(struct mk_instance *instance); /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; int mk_overlay_init(void); diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index a0573fc23a1c99..3ca8f8a2c7fae4 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -269,6 +269,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); + mk_pci_lease_instance_init(instance); INIT_LIST_HEAD(&instance->platform_devices); kref_init(&instance->refcount); diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 23b713d9aeedb9..d94d20e7c1e42e 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -7,31 +7,579 @@ * the current instance. */ +#include +#include #include #include +#include +#include #include "internal.h" +struct mk_pci_assignment { + struct list_head instance_node; + struct list_head active_node; + struct list_head transaction_node; + struct mk_instance *instance; + struct mk_pci_device *inventory; + struct pci_dev *vf; + struct pci_dev *pf; + const struct device_driver *host_driver; + struct work_struct failure_work; + atomic_t failure_pending; + bool assigned; + bool inventory_moved; + bool expected_unbind; +}; + +static DEFINE_MUTEX(mk_pci_lease_mutex); +static DEFINE_SPINLOCK(mk_pci_active_lock); +static LIST_HEAD(mk_pci_active_assignments); +static bool mk_pci_notifier_registered; + +static bool mk_pci_device_live(struct pci_dev *pdev) +{ + return device_is_registered(&pdev->dev) && + !pci_dev_is_disconnected(pdev) && + pci_device_is_present(pdev); +} + +static bool mk_pci_device_matches_bdf(const struct mk_pci_device *device, + u16 domain, u8 bus, u8 devfn) +{ + return device->domain == domain && device->bus == bus && + device->slot == PCI_SLOT(devfn) && + device->func == PCI_FUNC(devfn); +} + static struct mk_pci_device * -mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) +mk_pci_find_device_bdf(struct list_head *devices, u16 domain, u8 bus, u8 devfn) { struct mk_pci_device *device; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - if (!root_instance || !root_instance->dtb_data || - !root_instance->pci_devices_valid) - return NULL; + list_for_each_entry(device, devices, list) { + if (mk_pci_device_matches_bdf(device, domain, bus, devfn)) + return device; + } + + return NULL; +} + +static bool mk_pci_inventory_matches(const struct mk_pci_device *left, + const struct mk_pci_device *right) +{ + return left->vendor == right->vendor && + left->device == right->device && + mk_pci_device_matches_bdf(left, right->domain, right->bus, + PCI_DEVFN(right->slot, right->func)); +} + +static struct mk_pci_device * +mk_pci_find_root_inventory(const struct mk_pci_device *requested) +{ + struct mk_pci_device *device; list_for_each_entry(device, &root_instance->pci_devices, list) { - if (device->domain == domain && device->bus == bus && - device->slot == slot && device->func == func) + if (mk_pci_inventory_matches(device, requested)) return device; } return NULL; } +static struct mk_pci_device * +mk_pci_find_root_bdf(u16 domain, u8 bus, u8 devfn) +{ + return mk_pci_find_device_bdf(&root_instance->pci_devices, domain, bus, + devfn); +} + +static struct mk_pci_assignment * +mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_assignment *assignment; + + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + if (mk_pci_device_matches_bdf(assignment->inventory, domain, bus, + devfn)) + return assignment; + } + + return NULL; +} + +static void mk_pci_assignment_failure_work(struct work_struct *work) +{ + struct mk_pci_assignment *assignment = + container_of(work, struct mk_pci_assignment, failure_work); + struct mk_instance *instance = assignment->instance; + int ret; + + pr_err("PCI assignment lease for %s was lost by instance %d (%s)\n", + pci_name(assignment->vf), instance->id, instance->name); + + if (READ_ONCE(instance->state) == MK_STATE_ACTIVE) { + ret = mk_instance_force_halt(instance); + if (ret) + pr_err("Failed to force halt instance %d after PCI lease loss: %d\n", + instance->id, ret); + } + + mk_instance_set_state(instance, MK_STATE_FAILED); +} + +static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment) +{ + if (atomic_cmpxchg(&assignment->failure_pending, 0, 1)) + return; + + if (!schedule_work(&assignment->failure_work)) + atomic_set(&assignment->failure_pending, 0); +} + +static int mk_pci_bus_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct pci_dev *pdev = to_pci_dev(data); + struct mk_pci_assignment *assignment; + unsigned long flags; + + if (action != BUS_NOTIFY_DEL_DEVICE && + action != BUS_NOTIFY_REMOVED_DEVICE && + action != BUS_NOTIFY_UNBOUND_DRIVER) + return NOTIFY_DONE; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + list_for_each_entry(assignment, &mk_pci_active_assignments, + active_node) { + if (pdev != assignment->vf && pdev != assignment->pf) + continue; + if (pdev == assignment->vf && + action == BUS_NOTIFY_UNBOUND_DRIVER && + assignment->expected_unbind) + continue; + mk_pci_schedule_failure(assignment); + } + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + return NOTIFY_OK; +} + +static struct notifier_block mk_pci_bus_notifier = { + .notifier_call = mk_pci_bus_notify, +}; + +static int +mk_pci_prepare_assignment(struct mk_instance *instance, + const struct mk_pci_device *requested, + struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + struct mk_pci_device *inventory; + struct pci_dev *vf; + struct pci_dev *pf; + struct pci_dev *physfn; + + inventory = mk_pci_find_root_inventory(requested); + if (!inventory) { + pr_err("PCI device %04x:%04x@%04x:%02x:%02x.%x is not available in the root pool\n", + requested->vendor, requested->device, requested->domain, + requested->bus, requested->slot, requested->func); + return -ENOENT; + } + + vf = pci_get_domain_bus_and_slot(inventory->domain, + inventory->bus, + PCI_DEVFN(inventory->slot, + inventory->func)); + if (!vf) + return -ENODEV; + + if (vf->vendor != inventory->vendor || + vf->device != inventory->device) { + pr_err("PCI identity changed for %s: expected %04x:%04x, found %04x:%04x\n", + pci_name(vf), inventory->vendor, inventory->device, + vf->vendor, vf->device); + pci_dev_put(vf); + return -ENODEV; + } + + physfn = pci_physfn(vf); + if (!vf->is_virtfn || physfn == vf) { + pr_err("PCI assignment only supports SR-IOV VFs, rejecting %s\n", + pci_name(vf)); + pci_dev_put(vf); + return -EOPNOTSUPP; + } + + if (!mk_pci_device_live(vf) || !mk_pci_device_live(physfn)) { + pci_dev_put(vf); + return -ENODEV; + } + + if (pci_is_dev_assigned(vf) || + mk_pci_find_assignment(instance, inventory->domain, + inventory->bus, + PCI_DEVFN(inventory->slot, + inventory->func))) { + pci_dev_put(vf); + return -EBUSY; + } + + pf = pci_dev_get(physfn); + assignment = kzalloc_obj(*assignment, GFP_KERNEL); + if (!assignment) { + pci_dev_put(pf); + pci_dev_put(vf); + return -ENOMEM; + } + + assignment->instance = instance; + assignment->inventory = inventory; + assignment->vf = vf; + assignment->pf = pf; + assignment->host_driver = vf->dev.driver; + if (assignment->host_driver && assignment->host_driver->owner && + !try_module_get(assignment->host_driver->owner)) { + kfree(assignment); + pci_dev_put(pf); + pci_dev_put(vf); + return -ENODEV; + } + + INIT_LIST_HEAD(&assignment->instance_node); + INIT_LIST_HEAD(&assignment->active_node); + INIT_LIST_HEAD(&assignment->transaction_node); + INIT_WORK(&assignment->failure_work, mk_pci_assignment_failure_work); + atomic_set(&assignment->failure_pending, 0); + list_add_tail(&assignment->instance_node, &instance->pci_assignments); + list_add_tail(&assignment->transaction_node, transaction); + + return 0; +} + +static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + unsigned long flags; + int i; + + if (!mk_pci_device_live(vf) || !mk_pci_device_live(assignment->pf)) + return -ENODEV; + + if (vf->dev.driver != assignment->host_driver || + pci_is_dev_assigned(vf)) + return -EBUSY; + + pci_set_dev_assigned(vf); + assignment->assigned = true; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + list_add_tail(&assignment->active_node, &mk_pci_active_assignments); + assignment->expected_unbind = true; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + if (assignment->host_driver) + device_release_driver(&vf->dev); + + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + if (vf->dev.driver) + return -EBUSY; + + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + assignment->inventory->resources[i].start = + vf->resource[i].start; + assignment->inventory->resources[i].end = + vf->resource[i].end; + assignment->inventory->resources[i].flags = + vf->resource[i].flags; + } + assignment->inventory->resources_valid = true; + + list_move_tail(&assignment->inventory->list, + &assignment->instance->pci_devices); + root_instance->pci_device_count--; + assignment->instance->pci_device_count++; + assignment->instance->pci_devices_valid = true; + assignment->inventory_moved = true; + + pr_info("Leased SR-IOV VF %s to instance %d (%s)\n", + pci_name(vf), assignment->instance->id, + assignment->instance->name); + return 0; +} + +static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) +{ + struct mk_instance *instance = assignment->instance; + struct pci_dev *vf = assignment->vf; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + if (!list_empty(&assignment->active_node)) + list_del_init(&assignment->active_node); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + cancel_work_sync(&assignment->failure_work); + + if (assignment->assigned) { + pci_clear_dev_assigned(vf); + assignment->assigned = false; + } + + if (assignment->host_driver && mk_pci_device_live(vf)) { + if (!vf->dev.driver) { + ret = device_driver_attach(assignment->host_driver, + &vf->dev); + if (ret) + pr_err("Failed to restore driver %s to %s: %d\n", + assignment->host_driver->name, + pci_name(vf), ret); + } else if (vf->dev.driver != assignment->host_driver) { + pr_err("Cannot restore driver %s to %s: device is bound to %s\n", + assignment->host_driver->name, pci_name(vf), + vf->dev.driver->name); + ret = -EBUSY; + } + } + + if (assignment->inventory_moved && root_instance) { + list_move_tail(&assignment->inventory->list, + &root_instance->pci_devices); + instance->pci_device_count--; + root_instance->pci_device_count++; + root_instance->pci_devices_valid = true; + assignment->inventory_moved = false; + } + + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + if (!list_empty(&assignment->transaction_node)) + list_del_init(&assignment->transaction_node); + list_del_init(&assignment->instance_node); + pci_dev_put(assignment->pf); + pci_dev_put(vf); + kfree(assignment); + + return ret; +} + +static void mk_pci_rollback_transaction(struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + + while (!list_empty(transaction)) { + assignment = list_last_entry(transaction, + struct mk_pci_assignment, + transaction_node); + mk_pci_release_assignment(assignment); + } +} + +static int mk_pci_commit_transaction(struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + int ret; + + list_for_each_entry(assignment, transaction, transaction_node) { + ret = mk_pci_commit_assignment(assignment); + if (ret) + return ret; + } + + while (!list_empty(transaction)) { + assignment = list_first_entry(transaction, + struct mk_pci_assignment, + transaction_node); + list_del_init(&assignment->transaction_node); + } + + return 0; +} + +void mk_pci_lease_instance_init(struct mk_instance *instance) +{ + INIT_LIST_HEAD(&instance->pci_assignments); +} + +int mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count) +{ + struct mk_pci_device *requested; + LIST_HEAD(transaction); + int prepared = 0; + int ret = 0; + + if (!instance || instance == root_instance || !requested_devices || + requested_count < 0) + return -EINVAL; + if (!root_instance || !root_instance->pci_devices_valid) + return -EINVAL; + + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + list_for_each_entry(requested, requested_devices, list) { + ret = mk_pci_prepare_assignment(instance, requested, + &transaction); + if (ret) + goto rollback; + prepared++; + } + + if (prepared != requested_count) { + ret = -EINVAL; + goto rollback; + } + + ret = mk_pci_commit_transaction(&transaction); + if (ret) + goto rollback; + goto out; + +rollback: + mk_pci_rollback_transaction(&transaction); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + return ret; +} + +int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_device *inventory; + LIST_HEAD(transaction); + int ret; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + if (instance->state != MK_STATE_READY) { + ret = -EBUSY; + goto out; + } + if (!root_instance || !root_instance->pci_devices_valid) { + ret = -EINVAL; + goto out; + } + + inventory = mk_pci_find_root_bdf(domain, bus, devfn); + if (!inventory) { + ret = -ENOENT; + goto out; + } + + ret = mk_pci_prepare_assignment(instance, inventory, &transaction); + if (ret) + goto rollback; + ret = mk_pci_commit_transaction(&transaction); + if (ret) + goto rollback; + goto out; + +rollback: + mk_pci_rollback_transaction(&transaction); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + return ret; +} + +int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_assignment *assignment; + int ret; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + if (instance->state != MK_STATE_READY) { + ret = -EBUSY; + goto out; + } + + assignment = mk_pci_find_assignment(instance, domain, bus, devfn); + if (!assignment) { + ret = -ENOENT; + goto out; + } + + ret = mk_pci_release_assignment(assignment); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + return ret; +} + +int mk_pci_release_assignments(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + int ret = 0; + int release_ret; + + if (!instance || instance == root_instance) + return 0; + + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + while (!list_empty(&instance->pci_assignments)) { + assignment = list_last_entry(&instance->pci_assignments, + struct mk_pci_assignment, + instance_node); + release_ret = mk_pci_release_assignment(assignment); + if (release_ret && !ret) + ret = release_ret; + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + return ret; +} + +int mk_pci_lease_system_init(void) +{ + int ret; + + ret = bus_register_notifier(&pci_bus_type, &mk_pci_bus_notifier); + if (!ret) + mk_pci_notifier_registered = true; + return ret; +} + +void mk_pci_lease_system_cleanup(void) +{ + if (!mk_pci_notifier_registered) + return; + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; +} + +static struct mk_pci_device * +mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) +{ + if (!root_instance || !root_instance->dtb_data || + !root_instance->pci_devices_valid) + return NULL; + + return mk_pci_find_device_bdf(&root_instance->pci_devices, + domain, bus, devfn); +} + static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) { return mk_pci_find_assigned_bdf(pci_domain_nr(bus), bus->number, devfn); From efdf7779696e7a6435ce8815bbe0e7a4d9a3b1db Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Thu, 30 Jul 2026 08:21:22 +0300 Subject: [PATCH 06/16] multikernel: make resource reservation atomic Instance creation reserves memory, CPUs, PCI devices, host-bridge metadata, and platform devices. Returning an error after any one of those transfers can otherwise expose a partially populated instance and leak resources from the root. Validate configuration counts and lists before transfer, acquire each resource class in a fixed order, and unwind every completed step in reverse order. Centralize release so create failure, instance deletion, and final reference teardown share the same all-or-nothing semantics. Balance references acquired for remote memory add and remove operations on every success and error path so resource hotplug cannot pin a deleted instance. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 12 +- kernel/multikernel/core.c | 258 +++++++++++++++++++--------------- kernel/multikernel/internal.h | 1 + kernel/multikernel/kernfs.c | 2 +- 4 files changed, 152 insertions(+), 121 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 6fd49ff9b13fa7..d089558e415f4a 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -692,14 +692,14 @@ struct mk_instance *mk_instance_get(struct mk_instance *instance); void __noreturn mk_halt_to_pool(void); /** - * mk_instance_reserve_resources() - Reserve CPU and memory resources for instance - * @instance: Instance to reserve resources for - * @config: Device tree configuration with memory size and CPU assignment + * mk_instance_reserve_resources() - Atomically reserve instance resources + * @instance: Empty instance to populate + * @config: Parsed memory, CPU, PCI, and platform resource configuration * - * Allocates the specified memory size from the multikernel pool, creates - * memory regions, and copies CPU assignment. + * Reserves every configured resource class or returns all resources acquired + * by the attempt. A failure never leaves a partially populated instance. * - * Returns 0 on success, negative error code on failure. + * Returns: 0 on success, negative error code on failure */ int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config); diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index bb9b1fc1499f5f..ea01fc1725fe1a 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -71,13 +71,13 @@ static int mk_instance_return_pci_devices(struct mk_instance *instance) } static void mk_instance_return_platform_devices(struct mk_instance *instance) { - struct mk_platform_device *plat_dev, *plat_tmp; - int returned_count = 0; + struct mk_platform_device *device, *tmp; + int returned = 0; - if (!instance || !instance->platform_devices_valid) + if (!instance || instance == root_instance || instance->id == 0) return; - - if (instance == root_instance || instance->id == 0) + if (!instance->platform_devices_valid && + list_empty(&instance->platform_devices)) return; if (!root_instance) { @@ -86,37 +86,24 @@ static void mk_instance_return_platform_devices(struct mk_instance *instance) goto cleanup; } - list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { - struct mk_platform_device *root_dev; - - root_dev = kzalloc(sizeof(*root_dev), GFP_KERNEL); - if (!root_dev) { - pr_warn("Failed to allocate platform device entry for root instance\n"); - continue; - } - - *root_dev = *plat_dev; - INIT_LIST_HEAD(&root_dev->list); - - list_add_tail(&root_dev->list, &root_instance->platform_devices); + list_for_each_entry_safe(device, tmp, &instance->platform_devices, + list) { + list_move_tail(&device->list, + &root_instance->platform_devices); root_instance->platform_device_count++; root_instance->platform_devices_valid = true; - - pr_debug("Returned platform device '%s' from instance %d to root\n", - root_dev->name, instance->id); - - returned_count++; + returned++; } - if (returned_count > 0) { + if (returned) pr_info("Returned %d platform devices from instance %d (%s) to root instance\n", - returned_count, instance->id, instance->name); - } + returned, instance->id, instance->name); cleanup: - list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { - list_del(&plat_dev->list); - kfree(plat_dev); + list_for_each_entry_safe(device, tmp, &instance->platform_devices, + list) { + list_del(&device->list); + kfree(device); } instance->platform_device_count = 0; instance->platform_devices_valid = false; @@ -124,10 +111,12 @@ static void mk_instance_return_platform_devices(struct mk_instance *instance) int mk_instance_release_resources(struct mk_instance *instance) { + const char *failed_resource; int ret; if (!instance || instance == root_instance || instance->id == 0) return 0; + ret = mk_instance_return_pci_devices(instance); if (ret) return ret; @@ -160,7 +149,6 @@ static void mk_instance_release(struct kref *kref) kfree(instance->name); kfree(instance); } - /** * Instance reference counting */ @@ -457,9 +445,9 @@ static int mk_instance_reserve_cpus(struct mk_instance *instance, const struct mk_dt_config *config) { if (!config->cpus) { - pr_warn("No CPU configuration for instance %d (%s)\n", - instance->id, instance->name); - return 0; + pr_err("No CPU configuration for instance %d (%s)\n", + instance->id, instance->name); + return -ENOMEM; } return mk_instance_transfer_cpus(instance, config->cpus); @@ -487,13 +475,25 @@ static int mk_instance_transfer_pci_devices(struct mk_instance *instance, static int mk_instance_reserve_pci_devices(struct mk_instance *instance, const struct mk_dt_config *config) { - if (!config->pci_devices_valid || config->pci_device_count == 0) { + if (!config->pci_devices_valid) { + if (config->pci_device_count || + !list_empty(&config->pci_devices)) + return -EINVAL; + instance->pci_devices_valid = true; + return 0; + } + + if (!config->pci_device_count) { + if (!list_empty(&config->pci_devices)) + return -EINVAL; instance->pci_devices_valid = true; instance->pci_device_count = 0; pr_debug("No PCI devices to reserve for instance %d (%s)\n", instance->id, instance->name); return 0; } + if (list_empty(&config->pci_devices)) + return -EINVAL; return mk_instance_transfer_pci_devices(instance, &config->pci_devices, @@ -501,86 +501,111 @@ static int mk_instance_reserve_pci_devices(struct mk_instance *instance, } static int mk_instance_transfer_platform_devices(struct mk_instance *instance, - const struct list_head *requested_devices, - int requested_count) + const struct list_head *requested_devices, + int requested_count) { - struct mk_platform_device *req_dev, *root_dev, *tmp; + struct mk_platform_device *requested, *other, *root_device; + int actual_count = 0; int transferred = 0; - int not_found = 0; - bool found; if (!root_instance || !root_instance->platform_devices_valid) { pr_err("No root instance or platform devices not initialized\n"); return -EINVAL; } + if (requested_count <= 0 || list_empty(requested_devices)) + return -EINVAL; - if (requested_count == 0 || list_empty(requested_devices)) { - pr_info("No platform devices requested for instance %d (%s)\n", - instance->id, instance->name); - instance->platform_devices_valid = true; - return 0; - } + list_for_each_entry(requested, requested_devices, list) { + actual_count++; + list_for_each_entry(other, requested_devices, list) { + if (other == requested) + break; + if (!strcmp(other->name, requested->name)) { + pr_err("Platform device %s is requested more than once\n", + requested->name); + return -EINVAL; + } + } - list_for_each_entry(req_dev, requested_devices, list) { - found = false; - list_for_each_entry(root_dev, &root_instance->platform_devices, list) { - if (strcmp(root_dev->name, req_dev->name) == 0) { - found = true; + root_device = NULL; + list_for_each_entry(other, &root_instance->platform_devices, + list) { + if (!strcmp(other->name, requested->name)) { + root_device = other; break; } } - if (!found) { - pr_err("Platform device '%s' not available in root pool\n", - req_dev->name); - not_found++; + if (!root_device) { + pr_err("Platform device %s not available in root pool\n", + requested->name); + return -ENOENT; } } - if (not_found > 0) { - pr_err("Instance %d (%s): %d platform devices not available\n", - instance->id, instance->name, not_found); - return -ENOENT; + if (actual_count != requested_count) { + pr_err("Platform device count mismatch: metadata=%d list=%d\n", + requested_count, actual_count); + return -EINVAL; } - list_for_each_entry(req_dev, requested_devices, list) { - list_for_each_entry_safe(root_dev, tmp, &root_instance->platform_devices, list) { - if (strcmp(root_dev->name, req_dev->name) == 0) { - list_del(&root_dev->list); - list_add_tail(&root_dev->list, &instance->platform_devices); - root_instance->platform_device_count--; - instance->platform_device_count++; - transferred++; - - pr_debug("Transferred platform device '%s' to instance %d\n", - root_dev->name, instance->id); + list_for_each_entry(requested, requested_devices, list) { + root_device = NULL; + list_for_each_entry(other, &root_instance->platform_devices, + list) { + if (!strcmp(other->name, requested->name)) { + root_device = other; break; } } + if (!root_device) + goto rollback; + + list_move_tail(&root_device->list, + &instance->platform_devices); + root_instance->platform_device_count--; + instance->platform_device_count++; + transferred++; } instance->platform_devices_valid = true; - pr_info("Transferred %d platform devices from root to instance %d (%s), root pool remaining: %d devices\n", - transferred, instance->id, instance->name, root_instance->platform_device_count); - + pr_info("Transferred %d platform devices from root to instance %d (%s)\n", + transferred, instance->id, instance->name); return 0; + +rollback: + pr_err("Platform inventory changed during reservation for instance %d\n", + instance->id); + mk_instance_return_platform_devices(instance); + return -EIO; } static int mk_instance_reserve_platform_devices(struct mk_instance *instance, - const struct mk_dt_config *config) + const struct mk_dt_config *config) { - if (!config->platform_devices_valid || config->platform_device_count == 0) { + if (!config->platform_devices_valid) { + if (config->platform_device_count || + !list_empty(&config->platform_devices)) + return -EINVAL; + instance->platform_devices_valid = true; + return 0; + } + + if (!config->platform_device_count) { + if (!list_empty(&config->platform_devices)) + return -EINVAL; instance->platform_devices_valid = true; instance->platform_device_count = 0; pr_debug("No platform devices to reserve for instance %d (%s)\n", instance->id, instance->name); return 0; } + if (list_empty(&config->platform_devices)) + return -EINVAL; return mk_instance_transfer_platform_devices(instance, &config->platform_devices, config->platform_device_count); } - /** * mk_instance_add_pci_device - Add a single PCI device to an instance * @instance: Target instance @@ -825,67 +850,72 @@ void mk_instance_free_memory(struct mk_instance *instance) instance->id, instance->name); } +static bool mk_instance_resources_empty(const struct mk_instance *instance) +{ + return list_empty(&instance->memory_regions) && + !instance->instance_pool && !instance->region_count && + mk_cpu_set_empty(instance->cpus) && + list_empty(&instance->pci_devices) && + list_empty(&instance->pci_assignments) && + !instance->pci_device_count && + list_empty(&instance->platform_devices) && + !instance->platform_device_count; +} + /** - * mk_instance_reserve_resources() - Reserve memory and CPU resources for an instance + * mk_instance_reserve_resources() - Atomically reserve instance resources * @instance: Instance to reserve resources for - * @config: Device tree configuration with memory regions and CPU assignment + * @config: Parsed resource configuration * - * Reserves all memory regions specified in the device tree configuration, - * makes them children of the main multikernel_res, and copies CPU assignment. + * Each resource class is acquired only after the preceding class succeeds. + * Any error returns every acquired resource to the root instance in reverse + * order, so callers never observe a partially populated instance. * - * Returns 0 on success, negative error code on failure. + * Returns: 0 on success, negative error code on failure */ int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config) { int ret; - if (!config || !instance) { + if (!config || !instance || !instance->cpus) { pr_err("Invalid parameters to mk_instance_reserve_resources\n"); return -EINVAL; } + if (!mk_instance_resources_empty(instance)) { + pr_err("Instance %d (%s) already owns resources\n", + instance->id, instance->name); + return -EBUSY; + } - /* Free any existing memory regions first */ - mk_instance_free_memory(instance); - - /* Reserve memory regions */ + failed_resource = "memory"; ret = mk_instance_reserve_memory(instance, config); - if (ret) { - pr_err("Failed to reserve memory regions for instance %d (%s): %d\n", - instance->id, instance->name, ret); - return ret; - } + if (ret) + goto rollback; - /* Reserve CPU resources */ + failed_resource = "CPU"; ret = mk_instance_reserve_cpus(instance, config); - if (ret) { - pr_err("Failed to reserve CPU resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for CPU reservation failure */ - pr_warn("Continuing without CPU assignment\n"); - } - - /* Reserve PCI device resources */ - ret = mk_instance_reserve_pci_devices(instance, config); - if (ret) { - pr_err("Failed to reserve PCI device resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for PCI reservation failure */ - pr_warn("Continuing without PCI device assignment\n"); - } + if (ret) + goto rollback; - /* Reserve platform device resources */ + failed_resource = "platform device"; ret = mk_instance_reserve_platform_devices(instance, config); - if (ret) { - pr_err("Failed to reserve platform device resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for platform reservation failure */ - pr_warn("Continuing without platform device assignment\n"); - } + if (ret) + goto rollback; + + failed_resource = "PCI device"; + ret = mk_instance_reserve_pci_devices(instance, config); + if (ret) + goto rollback; return 0; -} +rollback: + pr_err("Failed to reserve %s resources for instance %d (%s): %d\n", + failed_resource, instance->id, instance->name, ret); + mk_instance_release_resources(instance); + return ret; +} /** * Per-instance memory pool management */ diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 1d67f8a26e89c2..42ea2e1f4ace5f 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -12,6 +12,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, int resources_node, size_t dtb_size); struct mk_instance *mk_instance_find_by_name(const char *name); int mk_instance_destroy(struct mk_instance *instance); +void mk_instance_release_resources(struct mk_instance *instance); /* dts.c */ int mk_dt_parse_resources(const void *fdt, int resources_node, diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index 3ca8f8a2c7fae4..3b3a34318064c1 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -334,7 +334,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, kfree(instance->dtb_data); instance->dtb_data = NULL; err_free_resources: - mk_instance_free_memory(instance); + mk_instance_release_resources(instance); err_free_idr: idr_remove(&mk_instance_idr, instance->id); err_remove_dir: From 760874fd55cad1dcaa7b93c35eb8175ac8c38b76 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Thu, 30 Jul 2026 08:23:56 +0300 Subject: [PATCH 07/16] pci/multikernel: isolate VF DMA with a host IOMMU domain Exclusive VF ownership does not constrain DMA. Allocate a host-owned domain for each assignment and map only the memory owned by the instance. Update mappings with memory hotplug under the lease lifetime. Detach and destroy the domain before returning the VF to the host. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 3 + kernel/multikernel/Kconfig | 3 + kernel/multikernel/hotplug.c | 25 ++ kernel/multikernel/internal.h | 1 + kernel/multikernel/mem.c | 44 +++- kernel/multikernel/pci.c | 476 +++++++++++++++++++++++++++++++++- 6 files changed, 535 insertions(+), 17 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index d089558e415f4a..ecb08792f13c03 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -487,6 +488,8 @@ struct mk_instance { int id; /* Kernel-assigned instance ID */ char *name; /* User-provided instance name */ enum mk_instance_state state; /* Current state */ + /* Serializes memory topology changes with PCI assignment leases. */ + struct mutex resource_mutex; /* Resource management - list of reserved memory regions */ struct list_head memory_regions; /* List of struct mk_memory_region */ diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index d704be265ae70c..4f4669d041f41e 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -24,4 +24,7 @@ config MULTIKERNEL - A manifest handed to each spawn kernel on multikernel's own boot channel, describing its resources and message rings + SR-IOV VF assignment additionally requires PCI_IOV and an active + hardware IOMMU with isolated interrupt delivery. Assignment fails + closed when those isolation facilities are unavailable. If unsure, say N. diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index ff1cd381a38145..77901c564d2171 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -1229,6 +1229,21 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl return ret; } +static int mk_memory_change_allowed(struct mk_instance *instance) +{ + bool iommu_active; + + mutex_lock(&instance->resource_mutex); + iommu_active = mk_pci_iommu_lease_active_locked(instance); + mutex_unlock(&instance->resource_mutex); + if (!iommu_active) + return 0; + + pr_err("Cannot change memory for instance %d while an IOMMU lease is active\n", + instance->id); + return -EBUSY; +} + /** * mk_send_mem_add - Add memory to instance * @instance_id: Target instance ID @@ -1265,6 +1280,11 @@ int mk_send_mem_add(int instance_id, u64 start_pfn, u64 nr_pages, target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; + ret = mk_memory_change_allowed(target_instance); + if (ret) { + mk_instance_put(target_instance); + return ret; + } /* For non-running instances, allocate memory from pool and add to instance */ if (target_instance->state != MK_STATE_ACTIVE) { @@ -1331,6 +1351,11 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; + ret = mk_memory_change_allowed(target_instance); + if (ret) { + mk_instance_put(target_instance); + return ret; + } /* For non-running instances, just remove the memory region from the instance */ if (target_instance->state != MK_STATE_ACTIVE) { diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 42ea2e1f4ace5f..c185b388851f3a 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -26,6 +26,7 @@ int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, int mk_pci_lease_system_init(void); void mk_pci_lease_system_cleanup(void); void mk_pci_lease_instance_init(struct mk_instance *instance); +bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance); int mk_pci_assign_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count); diff --git a/kernel/multikernel/mem.c b/kernel/multikernel/mem.c index 5c2da8dadb12ed..001d75bcc0368d 100644 --- a/kernel/multikernel/mem.c +++ b/kernel/multikernel/mem.c @@ -285,17 +285,30 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) phys_addr_t phys_addr; int ret; + if (!instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + if (mk_pci_iommu_lease_active_locked(instance)) { + pr_err("Cannot add memory to instance %d while an IOMMU lease is active\n", + instance->id); + ret = -EBUSY; + goto out_unlock; + } + phys_addr = multikernel_alloc(size); if (!phys_addr) { pr_err("Failed to allocate %zu bytes from multikernel pool for instance %d\n", size, instance->id); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region = kzalloc(sizeof(*region), GFP_KERNEL); if (!region) { multikernel_free(phys_addr, size); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region->res.name = kasprintf(GFP_KERNEL, "mk-instance-%d-%s-region-%d", @@ -303,7 +316,8 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) if (!region->res.name) { kfree(region); multikernel_free(phys_addr, size); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region->res.start = phys_addr; @@ -318,7 +332,7 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) kfree(region->res.name); kfree(region); multikernel_free(phys_addr, size); - return ret; + goto out_unlock; } INIT_LIST_HEAD(®ion->list); @@ -329,7 +343,10 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), size >> 20, instance->id, instance->name); - return 0; + ret = 0; +out_unlock: + mutex_unlock(&instance->resource_mutex); + return ret; } /** @@ -350,10 +367,19 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, { struct mk_memory_region *region, *tmp; bool found = false; + int ret; if (!instance) return -EINVAL; + mutex_lock(&instance->resource_mutex); + if (mk_pci_iommu_lease_active_locked(instance)) { + pr_err("Cannot remove memory from instance %d while an IOMMU lease is active\n", + instance->id); + ret = -EBUSY; + goto out_unlock; + } + list_for_each_entry_safe(region, tmp, &instance->memory_regions, list) { if (region->res.start == phys_addr && resource_size(®ion->res) == size) { @@ -381,10 +407,14 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), instance->id, instance->name); - return -ENOENT; + ret = -ENOENT; + } else { + ret = 0; } - return 0; +out_unlock: + mutex_unlock(&instance->resource_mutex); + return ret; } /** diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index d94d20e7c1e42e..3fda9d87c6ef7b 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -8,8 +8,11 @@ */ #include +#include +#include #include #include +#include #include #include #include @@ -25,6 +28,14 @@ struct mk_pci_assignment { struct pci_dev *vf; struct pci_dev *pf; const struct device_driver *host_driver; + struct iommu_group *iommu_group; + struct iommu_domain *iommu_domain; + char *host_driver_override; + struct mutex iommu_mutex; /* Serializes IOMMU activation and teardown. */ + unsigned int iommu_mapped_regions; + bool iommu_dma_owner; + bool iommu_attached; + bool iommu_override_active; struct work_struct failure_work; atomic_t failure_pending; bool assigned; @@ -171,6 +182,400 @@ static struct notifier_block mk_pci_bus_notifier = { .notifier_call = mk_pci_bus_notify, }; +#if IS_ENABLED(CONFIG_IOMMU_API) +#define MK_PCI_ASSIGNMENT_DRIVER_NAME "multikernel-pci-assignment" + +static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, + const struct pci_device_id *id); +static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev); + +static struct pci_driver mk_pci_assignment_driver = { + .name = MK_PCI_ASSIGNMENT_DRIVER_NAME, + .probe = mk_pci_iommu_assignment_probe, + .remove = mk_pci_iommu_assignment_remove, + .driver_managed_dma = true, +}; + +struct mk_pci_iommu_group_check { + struct device *vf; + unsigned int count; +}; + +static int mk_pci_iommu_check_group_device(struct device *dev, void *data) +{ + struct mk_pci_iommu_group_check *check = data; + + check->count++; + return dev == check->vf ? 0 : -EXDEV; +} + +static void mk_pci_iommu_free_resv_regions(struct list_head *regions) +{ + struct iommu_resv_region *region, *tmp; + + list_for_each_entry_safe(region, tmp, regions, list) { + list_del(®ion->list); + kfree(region); + } +} + +static int mk_pci_iommu_validate_group(struct mk_pci_assignment *assignment) +{ + struct mk_pci_iommu_group_check check = { + .vf = &assignment->vf->dev, + }; + struct iommu_resv_region *region; + LIST_HEAD(resv_regions); + int ret; + + ret = iommu_group_for_each_dev(assignment->iommu_group, &check, + mk_pci_iommu_check_group_device); + if (ret || check.count != 1) { + pr_err("IOMMU group %d for %s is not an isolated singleton group\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf)); + return ret ?: -EXDEV; + } + + if (!iommu_group_has_isolated_msi(assignment->iommu_group)) { + pr_err("IOMMU group %d for %s lacks isolated MSI delivery\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf)); + return -EPERM; + } + + ret = iommu_get_group_resv_regions(assignment->iommu_group, + &resv_regions); + if (ret) { + mk_pci_iommu_free_resv_regions(&resv_regions); + return ret; + } + + list_for_each_entry(region, &resv_regions, list) { + if (region->type != IOMMU_RESV_DIRECT && + region->type != IOMMU_RESV_DIRECT_RELAXABLE && + region->type != IOMMU_RESV_SW_MSI) + continue; + pr_err("IOMMU group %d for %s requires unsupported reserved region %#llx-%#llx type %u\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf), + (unsigned long long)region->start, + (unsigned long long)(region->start + region->length - 1), + region->type); + ret = -EPERM; + break; + } + + mk_pci_iommu_free_resv_regions(&resv_regions); + return ret; +} + +static int +mk_pci_iommu_validate_region(struct mk_pci_assignment *assignment, + const struct mk_memory_region *region) +{ + struct iommu_domain *domain = assignment->iommu_domain; + resource_size_t start = region->res.start; + resource_size_t size = resource_size(®ion->res); + u64 dma_mask = dma_get_mask(&assignment->vf->dev); + u64 end; + unsigned long min_page_size; + + if (!size || check_add_overflow((u64)start, (u64)size - 1, &end)) + return -EOVERFLOW; + if (!domain->pgsize_bitmap) + return -EOPNOTSUPP; + + min_page_size = 1UL << __ffs(domain->pgsize_bitmap); + if (!IS_ALIGNED(start, min_page_size) || + !IS_ALIGNED(size, min_page_size)) { + pr_err("Instance %d memory %#llx-%#llx is not aligned to IOMMU page size %#lx\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, min_page_size); + return -EINVAL; + } + if (start > ULONG_MAX || end > ULONG_MAX || end > dma_mask) { + pr_err("Instance %d memory %#llx-%#llx exceeds DMA addressability of %s\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, pci_name(assignment->vf)); + return -ERANGE; + } + if (domain->geometry.force_aperture && + (start < domain->geometry.aperture_start || + end > domain->geometry.aperture_end)) { + pr_err("Instance %d memory %#llx-%#llx is outside the IOMMU aperture for %s\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, pci_name(assignment->vf)); + return -ERANGE; + } + + return 0; +} + +static void mk_pci_iommu_unmap_regions(struct mk_pci_assignment *assignment) +{ + struct mk_memory_region *region; + unsigned int remaining = assignment->iommu_mapped_regions; + + list_for_each_entry(region, &assignment->instance->memory_regions, list) { + resource_size_t size; + size_t unmapped; + + if (!remaining) + break; + size = resource_size(®ion->res); + unmapped = iommu_unmap(assignment->iommu_domain, + region->res.start, size); + if (unmapped != size) + pr_err("IOMMU unmapped only %#zx of %#llx bytes for instance %d at %#llx\n", + unmapped, (unsigned long long)size, + assignment->instance->id, + (unsigned long long)region->res.start); + remaining--; + } + if (remaining) + pr_err("IOMMU lease for %s lost %u mapped instance regions\n", + pci_name(assignment->vf), remaining); + assignment->iommu_mapped_regions = 0; +} + +static void +__mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ + if (assignment->iommu_attached) { + iommu_detach_group(assignment->iommu_domain, + assignment->iommu_group); + assignment->iommu_attached = false; + } + if (assignment->iommu_dma_owner) { + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + } +} + +static void +mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ + mutex_lock(&assignment->iommu_mutex); + __mk_pci_iommu_deactivate_assignment(assignment); + mutex_unlock(&assignment->iommu_mutex); +} + +static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment) +{ + mutex_lock(&assignment->iommu_mutex); + __mk_pci_iommu_deactivate_assignment(assignment); + + if (assignment->iommu_domain) { + mk_pci_iommu_unmap_regions(assignment); + iommu_domain_free(assignment->iommu_domain); + assignment->iommu_domain = NULL; + } + if (assignment->iommu_group) { + iommu_group_put(assignment->iommu_group); + assignment->iommu_group = NULL; + } + mutex_unlock(&assignment->iommu_mutex); +} + +static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) +{ + struct mk_memory_region *region; + int ret; + + if (!device_iommu_mapped(&assignment->vf->dev)) { + pr_err("Cannot assign %s without an active hardware IOMMU\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; + } + if (!assignment->instance->region_count || + list_empty(&assignment->instance->memory_regions)) + return -EINVAL; + + assignment->iommu_group = iommu_group_get(&assignment->vf->dev); + if (!assignment->iommu_group) + return -ENODEV; + + ret = mk_pci_iommu_validate_group(assignment); + if (ret) + goto err_release; + + assignment->iommu_domain = + iommu_paging_domain_alloc(&assignment->vf->dev); + if (IS_ERR(assignment->iommu_domain)) { + ret = PTR_ERR(assignment->iommu_domain); + assignment->iommu_domain = NULL; + goto err_release; + } + + list_for_each_entry(region, &assignment->instance->memory_regions, list) { + resource_size_t size = resource_size(®ion->res); + + ret = mk_pci_iommu_validate_region(assignment, region); + if (ret) + goto err_release; + ret = iommu_map(assignment->iommu_domain, region->res.start, + region->res.start, size, IOMMU_READ | IOMMU_WRITE, + GFP_KERNEL); + if (ret) + goto err_release; + assignment->iommu_mapped_regions++; + } + + /* + * The domain blocks DMA outside these mappings, but translation-fault + * notification is not portable. In particular, Intel VT-d reports primary + * faults through dmar_fault() without invoking a legacy domain handler. + * Do not claim automatic instance failure on an IOMMU fault here. + */ + pr_info("Prepared host IOMMU domain for %s with %u instance regions\n", + pci_name(assignment->vf), assignment->iommu_mapped_regions); + return 0; + +err_release: + mk_pci_iommu_release_assignment(assignment); + return ret; +} + +static int mk_pci_iommu_commit_assignment(struct mk_pci_assignment *assignment) +{ + int ret; + + if (!assignment->iommu_domain) + return 0; + + if (assignment->vf->driver_override) { + assignment->host_driver_override = + kstrdup(assignment->vf->driver_override, GFP_KERNEL); + if (!assignment->host_driver_override) + return -ENOMEM; + } + + ret = driver_set_override(&assignment->vf->dev, + &assignment->vf->driver_override, + MK_PCI_ASSIGNMENT_DRIVER_NAME, + strlen(MK_PCI_ASSIGNMENT_DRIVER_NAME)); + if (ret) + return ret; + assignment->iommu_override_active = true; + pci_set_drvdata(assignment->vf, assignment); + ret = device_driver_attach(&mk_pci_assignment_driver.driver, + &assignment->vf->dev); + if (ret) + return ret; + if (assignment->vf->dev.driver != &mk_pci_assignment_driver.driver) + return -ENODEV; + return 0; +} + +static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + int ret; + + if (!assignment || assignment->vf != pdev || !assignment->iommu_domain) + return -ENODEV; + ret = iommu_device_claim_dma_owner(&assignment->vf->dev, assignment); + if (ret) + return ret; + assignment->iommu_dma_owner = true; + + ret = iommu_attach_group(assignment->iommu_domain, + assignment->iommu_group); + if (ret) + return ret; + assignment->iommu_attached = true; + pr_info("Attached %s to host-owned IOMMU domain for instance %d\n", + pci_name(assignment->vf), assignment->instance->id); + return 0; +} + +static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) +{ + struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + + if (assignment && assignment->vf == pdev) { + mk_pci_iommu_deactivate_assignment(assignment); + pci_set_drvdata(pdev, NULL); + } +} + +static int +mk_pci_iommu_restore_host_binding(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + const char *override = assignment->host_driver_override ?: ""; + unsigned long flags; + int ret = 0; + + if (vf->dev.driver == &mk_pci_assignment_driver.driver) { + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = true; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + device_release_driver(&vf->dev); + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + } else if (vf->dev.driver) { + pr_err("Cannot release assignment driver from %s: device is bound to %s\n", + pci_name(vf), vf->dev.driver->name); + return -EBUSY; + } + + pci_set_drvdata(vf, NULL); + if (assignment->iommu_override_active) { + ret = driver_set_override(&vf->dev, &vf->driver_override, + override, strlen(override)); + if (ret) + return ret; + assignment->iommu_override_active = false; + } + return 0; +} + +static int mk_pci_iommu_system_init(void) +{ + return pci_register_driver(&mk_pci_assignment_driver); +} + +static void mk_pci_iommu_system_cleanup(void) +{ + pci_unregister_driver(&mk_pci_assignment_driver); +} +#else +static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) +{ + pr_err("Cannot assign %s without CONFIG_IOMMU_API\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; +} + +static int mk_pci_iommu_commit_assignment(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment) +{ +} + +static int +mk_pci_iommu_restore_host_binding(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static int mk_pci_iommu_system_init(void) +{ + return 0; +} + +static void mk_pci_iommu_system_cleanup(void) +{ +} +#endif + static int mk_pci_prepare_assignment(struct mk_instance *instance, const struct mk_pci_device *requested, @@ -181,6 +586,7 @@ mk_pci_prepare_assignment(struct mk_instance *instance, struct pci_dev *vf; struct pci_dev *pf; struct pci_dev *physfn; + int ret; inventory = mk_pci_find_root_inventory(requested); if (!inventory) { @@ -252,18 +658,33 @@ mk_pci_prepare_assignment(struct mk_instance *instance, INIT_LIST_HEAD(&assignment->instance_node); INIT_LIST_HEAD(&assignment->active_node); INIT_LIST_HEAD(&assignment->transaction_node); + mutex_init(&assignment->iommu_mutex); INIT_WORK(&assignment->failure_work, mk_pci_assignment_failure_work); atomic_set(&assignment->failure_pending, 0); + + ret = mk_pci_iommu_prepare_assignment(assignment); + if (ret) + goto err_module; + list_add_tail(&assignment->instance_node, &instance->pci_assignments); list_add_tail(&assignment->transaction_node, transaction); return 0; + +err_module: + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + kfree(assignment); + pci_dev_put(pf); + pci_dev_put(vf); + return ret; } static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) { struct pci_dev *vf = assignment->vf; unsigned long flags; + int ret; int i; if (!mk_pci_device_live(vf) || !mk_pci_device_live(assignment->pf)) @@ -291,6 +712,10 @@ static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) if (vf->dev.driver) return -EBUSY; + ret = mk_pci_iommu_commit_assignment(assignment); + if (ret) + return ret; + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { assignment->inventory->resources[i].start = vf->resource[i].start; @@ -319,6 +744,7 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) struct mk_instance *instance = assignment->instance; struct pci_dev *vf = assignment->vf; unsigned long flags; + int binding_ret; int ret = 0; spin_lock_irqsave(&mk_pci_active_lock, flags); @@ -327,14 +753,18 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) assignment->expected_unbind = false; spin_unlock_irqrestore(&mk_pci_active_lock, flags); - cancel_work_sync(&assignment->failure_work); - if (assignment->assigned) { pci_clear_dev_assigned(vf); assignment->assigned = false; } - if (assignment->host_driver && mk_pci_device_live(vf)) { + mk_pci_iommu_release_assignment(assignment); + cancel_work_sync(&assignment->failure_work); + binding_ret = mk_pci_iommu_restore_host_binding(assignment); + if (binding_ret) + ret = binding_ret; + + if (!binding_ret && assignment->host_driver && mk_pci_device_live(vf)) { if (!vf->dev.driver) { ret = device_driver_attach(assignment->host_driver, &vf->dev); @@ -359,6 +789,7 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) assignment->inventory_moved = false; } + kfree(assignment->host_driver_override); if (assignment->host_driver && assignment->host_driver->owner) module_put(assignment->host_driver->owner); if (!list_empty(&assignment->transaction_node)) @@ -406,9 +837,19 @@ static int mk_pci_commit_transaction(struct list_head *transaction) void mk_pci_lease_instance_init(struct mk_instance *instance) { + mutex_init(&instance->resource_mutex); INIT_LIST_HEAD(&instance->pci_assignments); } +bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance) +{ + if (!instance) + return false; + + lockdep_assert_held(&instance->resource_mutex); + return !list_empty(&instance->pci_assignments); +} + int mk_pci_assign_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count) @@ -424,6 +865,7 @@ int mk_pci_assign_devices(struct mk_instance *instance, if (!root_instance || !root_instance->pci_devices_valid) return -EINVAL; + mutex_lock(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); @@ -450,6 +892,7 @@ int mk_pci_assign_devices(struct mk_instance *instance, out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); return ret; } @@ -463,6 +906,7 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, if (!instance || instance == root_instance) return -EINVAL; + mutex_lock(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); @@ -494,6 +938,7 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); return ret; } @@ -506,6 +951,7 @@ int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, if (!instance || instance == root_instance) return -EINVAL; + mutex_lock(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); @@ -524,6 +970,7 @@ int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); return ret; } @@ -536,6 +983,7 @@ int mk_pci_release_assignments(struct mk_instance *instance) if (!instance || instance == root_instance) return 0; + mutex_lock(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); while (!list_empty(&instance->pci_assignments)) { @@ -548,6 +996,7 @@ int mk_pci_release_assignments(struct mk_instance *instance) } pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); return ret; } @@ -555,18 +1004,25 @@ int mk_pci_lease_system_init(void) { int ret; + ret = mk_pci_iommu_system_init(); + if (ret) + return ret; ret = bus_register_notifier(&pci_bus_type, &mk_pci_bus_notifier); - if (!ret) - mk_pci_notifier_registered = true; - return ret; + if (ret) { + mk_pci_iommu_system_cleanup(); + return ret; + } + mk_pci_notifier_registered = true; + return 0; } void mk_pci_lease_system_cleanup(void) { - if (!mk_pci_notifier_registered) - return; - bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); - mk_pci_notifier_registered = false; + if (mk_pci_notifier_registered) { + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; + } + mk_pci_iommu_system_cleanup(); } static struct mk_pci_device * From 09f38f74121e080f1270eb4e77b2ff4a7d47b73d Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Thu, 30 Jul 2026 08:23:56 +0300 Subject: [PATCH 08/16] pci/multikernel: quiesce VFs before releasing leases Returning a VF while it can still issue DMA races the IOMMU teardown and host-driver reprobe. Unexpected PF or VF removal must also stop an active instance instead of silently losing its assigned device. Clear bus mastering, wait for pending transactions, and issue function-level reset while the assignment domain is still attached. Then detach and free the domain, restore the saved driver override and host driver, and only afterwards return the inventory to the root. Lease-loss notifications force an active instance to halt and mark it failed. Rollback entries that were prepared but never committed skip device quiesce and driver restoration, releasing only their prepared IOMMU resources. Signed-off-by: Nikolay Nikolaev --- kernel/multikernel/core.c | 10 +- kernel/multikernel/internal.h | 2 +- kernel/multikernel/kernfs.c | 22 +++- kernel/multikernel/pci.c | 199 ++++++++++++++++++++++++---------- 4 files changed, 173 insertions(+), 60 deletions(-) diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index ea01fc1725fe1a..d5fcf99cd0065e 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -111,7 +111,6 @@ static void mk_instance_return_platform_devices(struct mk_instance *instance) int mk_instance_release_resources(struct mk_instance *instance) { - const char *failed_resource; int ret; if (!instance || instance == root_instance || instance->id == 0) @@ -128,7 +127,8 @@ int mk_instance_release_resources(struct mk_instance *instance) static void mk_instance_release(struct kref *kref) { - struct mk_instance *instance = container_of(kref, struct mk_instance, refcount); + struct mk_instance *instance = + container_of(kref, struct mk_instance, refcount); int ret; pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", @@ -876,6 +876,8 @@ static bool mk_instance_resources_empty(const struct mk_instance *instance) int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config) { + const char *failed_resource; + int release_ret; int ret; if (!config || !instance || !instance->cpus) { @@ -913,7 +915,9 @@ int mk_instance_reserve_resources(struct mk_instance *instance, rollback: pr_err("Failed to reserve %s resources for instance %d (%s): %d\n", failed_resource, instance->id, instance->name, ret); - mk_instance_release_resources(instance); + release_ret = mk_instance_release_resources(instance); + if (release_ret) + return release_ret; return ret; } /** diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index c185b388851f3a..6a220a7c533fc9 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -12,7 +12,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, int resources_node, size_t dtb_size); struct mk_instance *mk_instance_find_by_name(const char *name); int mk_instance_destroy(struct mk_instance *instance); -void mk_instance_release_resources(struct mk_instance *instance); +int mk_instance_release_resources(struct mk_instance *instance); /* dts.c */ int mk_dt_parse_resources(const void *fdt, int resources_node, diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index 3b3a34318064c1..3d152e613a32ac 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -239,6 +239,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, struct kernfs_node *kn; struct mk_dt_config config; void *dtb_copy; + int release_ret; int ret; int allocated_id; @@ -334,7 +335,16 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, kfree(instance->dtb_data); instance->dtb_data = NULL; err_free_resources: - mk_instance_release_resources(instance); + release_ret = mk_instance_release_resources(instance); + if (release_ret) { + pr_crit("Retaining failed instance '%s' because PCI cleanup failed: %d\n", + name, release_ret); + list_add_tail(&instance->list, &mk_instance_list); + kernfs_activate(kn); + mk_instance_set_state(instance, MK_STATE_FAILED); + mk_dt_config_free(&config); + return release_ret; + } err_free_idr: idr_remove(&mk_instance_idr, instance->id); err_remove_dir: @@ -436,6 +446,8 @@ static int mk_kernfs_rmdir(struct kernfs_node *kn) */ int mk_instance_destroy(struct mk_instance *instance) { + int ret; + lockdep_assert_held(&mk_instance_mutex); if (!instance) { @@ -455,6 +467,14 @@ int mk_instance_destroy(struct mk_instance *instance) return -EBUSY; } + ret = mk_instance_release_resources(instance); + if (ret) { + pr_crit("Cannot remove instance '%s' while PCI cleanup is unsafe: %d\n", + instance->name, ret); + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; + } + list_del(&instance->list); idr_remove(&mk_instance_idr, instance->id); if (instance->kn) { diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 3fda9d87c6ef7b..f394c6a8638a51 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -182,6 +182,20 @@ static struct notifier_block mk_pci_bus_notifier = { .notifier_call = mk_pci_bus_notify, }; +static void +mk_pci_release_bound_driver(struct mk_pci_assignment *assignment) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = true; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + device_release_driver(&assignment->vf->dev); + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); +} + #if IS_ENABLED(CONFIG_IOMMU_API) #define MK_PCI_ASSIGNMENT_DRIVER_NAME "multikernel-pci-assignment" @@ -339,6 +353,43 @@ static void mk_pci_iommu_unmap_regions(struct mk_pci_assignment *assignment) assignment->iommu_mapped_regions = 0; } +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + bool transactions_drained; + int ret; + + if (!mk_pci_device_live(vf)) + return 0; + + /* + * Releasing DMA ownership restores the group's default domain. Stop new + * DMA first, drain requests already issued, and reset the VF while the + * assignment domain still contains any stragglers. + */ + pci_clear_master(vf); + transactions_drained = pci_wait_for_pending_transaction(vf); + ret = pcie_reset_flr(vf, false); + if (!ret) + return 0; + if (ret == -ENOTTY && transactions_drained) + return 0; + + if (!transactions_drained) + pr_err("Timed out draining DMA from assigned VF %s\n", + pci_name(vf)); + if (ret != -ENOTTY) + pr_err("Failed to reset assigned VF %s: %d\n", + pci_name(vf), ret); + + /* + * Keep the assignment domain attached when the device cannot be made + * safe. The lease owner can retry teardown after the instance halts. + */ + return ret == -ENOTTY ? -ETIMEDOUT : ret; +} + static void __mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) { @@ -494,29 +545,35 @@ static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) { struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + int ret; - if (assignment && assignment->vf == pdev) { - mk_pci_iommu_deactivate_assignment(assignment); + if (!assignment || assignment->vf != pdev) + return; + + if (READ_ONCE(assignment->expected_unbind)) { pci_set_drvdata(pdev, NULL); + return; } + + ret = mk_pci_quiesce_assignment(assignment); + if (ret) { + pr_crit("Keeping IOMMU containment for %s after unsafe driver removal: %d\n", + pci_name(pdev), ret); + mk_pci_schedule_failure(assignment); + } else { + mk_pci_iommu_deactivate_assignment(assignment); + } + pci_set_drvdata(pdev, NULL); } -static int -mk_pci_iommu_restore_host_binding(struct mk_pci_assignment *assignment) +static int mk_pci_restore_host_binding(struct mk_pci_assignment *assignment) { struct pci_dev *vf = assignment->vf; const char *override = assignment->host_driver_override ?: ""; - unsigned long flags; int ret = 0; if (vf->dev.driver == &mk_pci_assignment_driver.driver) { - spin_lock_irqsave(&mk_pci_active_lock, flags); - assignment->expected_unbind = true; - spin_unlock_irqrestore(&mk_pci_active_lock, flags); - device_release_driver(&vf->dev); - spin_lock_irqsave(&mk_pci_active_lock, flags); - assignment->expected_unbind = false; - spin_unlock_irqrestore(&mk_pci_active_lock, flags); + mk_pci_release_bound_driver(assignment); } else if (vf->dev.driver) { pr_err("Cannot release assignment driver from %s: device is bound to %s\n", pci_name(vf), vf->dev.driver->name); @@ -531,6 +588,24 @@ mk_pci_iommu_restore_host_binding(struct mk_pci_assignment *assignment) return ret; assignment->iommu_override_active = false; } + + mk_pci_iommu_deactivate_assignment(assignment); + if (assignment->host_driver && mk_pci_device_live(vf)) { + if (!vf->dev.driver) { + ret = device_driver_attach(assignment->host_driver, &vf->dev); + if (ret) { + pr_err("Failed to restore driver %s to %s: %d\n", + assignment->host_driver->name, + pci_name(vf), ret); + return ret; + } + } else if (vf->dev.driver != assignment->host_driver) { + pr_err("Cannot restore driver %s to %s: device is bound to %s\n", + assignment->host_driver->name, pci_name(vf), + vf->dev.driver->name); + return -EBUSY; + } + } return 0; } @@ -544,6 +619,17 @@ static void mk_pci_iommu_system_cleanup(void) pci_unregister_driver(&mk_pci_assignment_driver); } #else +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static void +mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ +} + static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) { pr_err("Cannot assign %s without CONFIG_IOMMU_API\n", @@ -560,8 +646,7 @@ static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment { } -static int -mk_pci_iommu_restore_host_binding(struct mk_pci_assignment *assignment) +static int mk_pci_restore_host_binding(struct mk_pci_assignment *assignment) { return 0; } @@ -699,15 +784,10 @@ static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) spin_lock_irqsave(&mk_pci_active_lock, flags); list_add_tail(&assignment->active_node, &mk_pci_active_assignments); - assignment->expected_unbind = true; spin_unlock_irqrestore(&mk_pci_active_lock, flags); if (assignment->host_driver) - device_release_driver(&vf->dev); - - spin_lock_irqsave(&mk_pci_active_lock, flags); - assignment->expected_unbind = false; - spin_unlock_irqrestore(&mk_pci_active_lock, flags); + mk_pci_release_bound_driver(assignment); if (vf->dev.driver) return -EBUSY; @@ -744,8 +824,18 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) struct mk_instance *instance = assignment->instance; struct pci_dev *vf = assignment->vf; unsigned long flags; - int binding_ret; - int ret = 0; + int ret; + + if (!assignment->assigned) + goto release_resources; + + ret = mk_pci_quiesce_assignment(assignment); + if (ret) + return ret; + + ret = mk_pci_restore_host_binding(assignment); + if (ret) + return ret; spin_lock_irqsave(&mk_pci_active_lock, flags); if (!list_empty(&assignment->active_node)) @@ -758,27 +848,9 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) assignment->assigned = false; } +release_resources: mk_pci_iommu_release_assignment(assignment); cancel_work_sync(&assignment->failure_work); - binding_ret = mk_pci_iommu_restore_host_binding(assignment); - if (binding_ret) - ret = binding_ret; - - if (!binding_ret && assignment->host_driver && mk_pci_device_live(vf)) { - if (!vf->dev.driver) { - ret = device_driver_attach(assignment->host_driver, - &vf->dev); - if (ret) - pr_err("Failed to restore driver %s to %s: %d\n", - assignment->host_driver->name, - pci_name(vf), ret); - } else if (vf->dev.driver != assignment->host_driver) { - pr_err("Cannot restore driver %s to %s: device is bound to %s\n", - assignment->host_driver->name, pci_name(vf), - vf->dev.driver->name); - ret = -EBUSY; - } - } if (assignment->inventory_moved && root_instance) { list_move_tail(&assignment->inventory->list, @@ -799,19 +871,28 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) pci_dev_put(vf); kfree(assignment); - return ret; + return 0; } -static void mk_pci_rollback_transaction(struct list_head *transaction) +static int mk_pci_rollback_transaction(struct list_head *transaction) { - struct mk_pci_assignment *assignment; + struct mk_pci_assignment *assignment, *tmp; + int rollback_ret = 0; + int ret; - while (!list_empty(transaction)) { - assignment = list_last_entry(transaction, - struct mk_pci_assignment, - transaction_node); - mk_pci_release_assignment(assignment); + list_for_each_entry_safe_reverse(assignment, tmp, transaction, + transaction_node) { + ret = mk_pci_release_assignment(assignment); + if (!ret) + continue; + pr_crit("Failed to roll back PCI assignment for %s: %d\n", + pci_name(assignment->vf), ret); + list_del_init(&assignment->transaction_node); + if (!rollback_ret) + rollback_ret = ret; } + + return rollback_ret; } static int mk_pci_commit_transaction(struct list_head *transaction) @@ -858,6 +939,7 @@ int mk_pci_assign_devices(struct mk_instance *instance, LIST_HEAD(transaction); int prepared = 0; int ret = 0; + int rollback_ret; if (!instance || instance == root_instance || !requested_devices || requested_count < 0) @@ -888,7 +970,9 @@ int mk_pci_assign_devices(struct mk_instance *instance, goto out; rollback: - mk_pci_rollback_transaction(&transaction); + rollback_ret = mk_pci_rollback_transaction(&transaction); + if (rollback_ret) + ret = rollback_ret; out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); @@ -902,6 +986,7 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, struct mk_pci_device *inventory; LIST_HEAD(transaction); int ret; + int rollback_ret; if (!instance || instance == root_instance) return -EINVAL; @@ -934,7 +1019,9 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, goto out; rollback: - mk_pci_rollback_transaction(&transaction); + rollback_ret = mk_pci_rollback_transaction(&transaction); + if (rollback_ret) + ret = rollback_ret; out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); @@ -978,7 +1065,6 @@ int mk_pci_release_assignments(struct mk_instance *instance) { struct mk_pci_assignment *assignment; int ret = 0; - int release_ret; if (!instance || instance == root_instance) return 0; @@ -990,9 +1076,12 @@ int mk_pci_release_assignments(struct mk_instance *instance) assignment = list_last_entry(&instance->pci_assignments, struct mk_pci_assignment, instance_node); - release_ret = mk_pci_release_assignment(assignment); - if (release_ret && !ret) - ret = release_ret; + ret = mk_pci_release_assignment(assignment); + if (ret) { + pr_crit("Instance %d retains unsafe PCI lease for %s: %d\n", + instance->id, pci_name(assignment->vf), ret); + break; + } } pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); From 85575f64d1bf64818b60485728aaef7332b74e7c Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 4 Aug 2026 00:50:57 +0300 Subject: [PATCH 09/16] pci/multikernel: reset assigned VFs before respawn A stopped or force-halted instance can leave bus mastering enabled and DMA in flight while its next boot rewrites the same memory. Lease teardown resets the VF, but a respawn retains the lease and previously skipped that protection. Require assigned VFs to support FLR. After confirming that all instance CPUs are parked, clear bus mastering, drain pending transactions, and reset every leased VF while its restrictive IOMMU domain remains attached. Abort the restart if any device cannot be made safe. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 1 + kernel/kexec_core.c | 12 +++++++ kernel/multikernel/pci.c | 62 +++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index ecb08792f13c03..394a88f3cc9c72 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -792,6 +792,7 @@ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); +int mk_pci_prepare_instance_start(struct mk_instance *instance); #else static inline bool multikernel_allow_emergency_restart(void) { diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 01382f40aeb724..63da42ffb7b67f 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1733,6 +1733,18 @@ int multikernel_kexec_by_id(int mk_id) goto unlock; } + /* + * Stop and reset every leased VF before rewriting instance memory. A + * force-halted kernel may have left bus mastering enabled and DMA in + * flight into the image that is about to be reused. + */ + rc = mk_pci_prepare_instance_start(instance); + if (rc) { + pr_err("Failed to prepare PCI assignments for instance %d restart: %d\n", + mk_id, rc); + goto unlock; + } + /* * Booting consumes the image: the spawn kernel writes its .data and * patches its own text, so the copy in instance memory is spent once diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index f394c6a8638a51..af68dce079c7c9 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -390,6 +390,38 @@ mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) return ret == -ENOTTY ? -ETIMEDOUT : ret; } +static int +mk_pci_reset_assignment_for_start(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + int ret; + + if (!assignment->assigned || !assignment->iommu_attached) + return -EINVAL; + if (!mk_pci_device_live(vf)) + return -ENODEV; + + /* + * A stopped instance may have left DMA active. Keep its restrictive + * domain attached while stopping new requests, draining old ones, and + * resetting device state before the instance image is reused. + */ + pci_clear_master(vf); + if (!pci_wait_for_pending_transaction(vf)) { + pr_err("Timed out draining assigned VF %s before instance restart\n", + pci_name(vf)); + return -ETIMEDOUT; + } + + ret = pcie_reset_flr(vf, false); + if (ret) { + pr_err("Failed to reset assigned VF %s before instance restart: %d\n", + pci_name(vf), ret); + return ret == -ENOTTY ? -EOPNOTSUPP : ret; + } + return 0; +} + static void __mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) { @@ -709,6 +741,13 @@ mk_pci_prepare_assignment(struct mk_instance *instance, pci_dev_put(vf); return -ENODEV; } + ret = pcie_reset_flr(vf, true); + if (ret) { + pr_err("PCI assignment requires FLR for safe instance restart, rejecting %s\n", + pci_name(vf)); + pci_dev_put(vf); + return -EOPNOTSUPP; + } if (pci_is_dev_assigned(vf) || mk_pci_find_assignment(instance, inventory->domain, @@ -1089,6 +1128,29 @@ int mk_pci_release_assignments(struct mk_instance *instance) return ret; } +int mk_pci_prepare_instance_start(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + int ret = 0; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + ret = mk_pci_reset_assignment_for_start(assignment); + if (ret) + break; + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + int mk_pci_lease_system_init(void) { int ret; From 0f999799d018c21b79fe573b7faf181f425e90e8 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Wed, 5 Aug 2026 10:35:52 +0300 Subject: [PATCH 10/16] multikernel: serialize CPU ownership transfers Protect CPU membership and pool ownership with a dedicated mutex nested inside an operation-wide transaction lock. Keep add and remove transactions serialized across reservation, acknowledgment, and repark so a CPU cannot be transferred twice. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 25 ++-- kernel/multikernel/core.c | 29 ++++ kernel/multikernel/cpuset.c | 257 +++++++++++++++++++++++++++------- kernel/multikernel/hotplug.c | 199 +++++++++++++++++++------- kernel/multikernel/internal.h | 7 + 5 files changed, 396 insertions(+), 121 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 394a88f3cc9c72..f0800c98c1e116 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -15,6 +15,7 @@ #include #include #include +#include struct pci_bus; @@ -32,6 +33,7 @@ typedef u64 mk_phys_cpu_t; #define MK_PHYS_CPU_INVALID (~(mk_phys_cpu_t)0) struct mk_cpu_set { + raw_spinlock_t lock; unsigned int nr; /* Entries in use */ unsigned int cap; /* Allocated capacity */ mk_phys_cpu_t *ids; @@ -46,26 +48,15 @@ bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id); bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id); int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src); int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set); - -static inline unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) -{ - return set ? set->nr : 0; -} - -static inline bool mk_cpu_set_empty(const struct mk_cpu_set *set) -{ - return mk_cpu_set_count(set) == 0; -} - -static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) -{ - return mk_cpu_set_empty(set) ? MK_PHYS_CPU_INVALID : set->ids[0]; -} +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set); +bool mk_cpu_set_empty(const struct mk_cpu_set *set); +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set); +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id); #define mk_cpu_set_for_each(i, id, set) \ for ((i) = 0; \ - (set) && (i) < (set)->nr && \ - (((id) = (set)->ids[(i)]), true); \ + mk_cpu_set_get((set), (i), &(id)); \ (i)++) /** diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index d5fcf99cd0065e..a94a325f7873d9 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -15,6 +15,35 @@ #include #include "internal.h" +/* CPU moves hold the transaction lock before the ownership lock. */ +static DEFINE_MUTEX(mk_cpu_transaction_mutex); +static DEFINE_MUTEX(mk_cpu_ownership_mutex); + +void mk_cpu_transaction_lock(void) +{ + mutex_lock(&mk_cpu_transaction_mutex); +} + +void mk_cpu_transaction_unlock(void) +{ + mutex_unlock(&mk_cpu_transaction_mutex); +} + +void mk_cpu_ownership_lock(void) +{ + mutex_lock(&mk_cpu_ownership_mutex); +} + +void mk_cpu_ownership_unlock(void) +{ + mutex_unlock(&mk_cpu_ownership_mutex); +} + +void mk_cpu_ownership_assert_held(void) +{ + lockdep_assert_held(&mk_cpu_ownership_mutex); +} + static void mk_instance_return_all_cpus(struct mk_instance *instance) { if (!instance || mk_cpu_set_empty(instance->cpus)) diff --git a/kernel/multikernel/cpuset.c b/kernel/multikernel/cpuset.c index ad36a4f94fc65a..ceb54e7435cc20 100644 --- a/kernel/multikernel/cpuset.c +++ b/kernel/multikernel/cpuset.c @@ -13,31 +13,22 @@ #include #include -struct mk_cpu_set *mk_cpu_set_alloc(void) -{ - return kzalloc(sizeof(struct mk_cpu_set), GFP_KERNEL); -} - -void mk_cpu_set_free(struct mk_cpu_set *set) +static void mk_cpu_set_lock(const struct mk_cpu_set *set, unsigned long *flags) { - if (!set) - return; - - kfree(set->ids); - kfree(set); + raw_spin_lock_irqsave((raw_spinlock_t *)&set->lock, *flags); } -void mk_cpu_set_clear(struct mk_cpu_set *set) +static void mk_cpu_set_unlock(const struct mk_cpu_set *set, unsigned long flags) { - if (set) - set->nr = 0; + raw_spin_unlock_irqrestore((raw_spinlock_t *)&set->lock, flags); } -static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) +static int mk_cpu_set_index_locked(const struct mk_cpu_set *set, + mk_phys_cpu_t id) { unsigned int i; - for (i = 0; set && i < set->nr; i++) { + for (i = 0; i < set->nr; i++) { if (set->ids[i] == id) return i; } @@ -45,9 +36,35 @@ static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) return -1; } -bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +struct mk_cpu_set *mk_cpu_set_alloc(void) { - return mk_cpu_set_index(set, id) >= 0; + struct mk_cpu_set *set; + + set = kzalloc_obj(*set, GFP_KERNEL); + if (set) + raw_spin_lock_init(&set->lock); + return set; +} + +void mk_cpu_set_free(struct mk_cpu_set *set) +{ + if (!set) + return; + + kfree(set->ids); + kfree(set); +} + +void mk_cpu_set_clear(struct mk_cpu_set *set) +{ + unsigned long flags; + + if (!set) + return; + + mk_cpu_set_lock(set, &flags); + set->nr = 0; + mk_cpu_set_unlock(set, flags); } /** @@ -61,64 +78,195 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) */ int mk_cpu_set_reserve(struct mk_cpu_set *set, unsigned int extra) { - unsigned int cap = set->nr + extra; - mk_phys_cpu_t *ids; + mk_phys_cpu_t *ids = NULL; + mk_phys_cpu_t *old_ids; + unsigned int cap; + unsigned long flags; - if (cap <= set->cap) - return 0; + if (!set) + return -EINVAL; + + for (;;) { + mk_cpu_set_lock(set, &flags); + cap = set->nr + extra; + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + mk_cpu_set_unlock(set, flags); - cap = max_t(unsigned int, cap, 8); - ids = krealloc_array(set->ids, cap, sizeof(*ids), GFP_KERNEL); - if (!ids) - return -ENOMEM; + cap = max_t(unsigned int, cap, 8); + kfree(ids); + ids = kcalloc(cap, sizeof(*ids), GFP_KERNEL); + if (!ids) + return -ENOMEM; - set->ids = ids; - set->cap = cap; - return 0; + mk_cpu_set_lock(set, &flags); + if (set->nr + extra > cap) { + mk_cpu_set_unlock(set, flags); + continue; + } + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + + memcpy(ids, set->ids, set->nr * sizeof(*ids)); + old_ids = set->ids; + set->ids = ids; + set->cap = cap; + mk_cpu_set_unlock(set, flags); + kfree(old_ids); + return 0; + } } /* Idempotent: adding an ID already in the set succeeds without effect */ int mk_cpu_set_add(struct mk_cpu_set *set, mk_phys_cpu_t id) { + unsigned long flags; int ret; - if (mk_cpu_set_contains(set, id)) - return 0; + if (!set) + return -EINVAL; - ret = mk_cpu_set_reserve(set, 1); - if (ret) - return ret; + for (;;) { + mk_cpu_set_lock(set, &flags); + if (mk_cpu_set_index_locked(set, id) >= 0) { + mk_cpu_set_unlock(set, flags); + return 0; + } + if (set->nr < set->cap) { + set->ids[set->nr++] = id; + mk_cpu_set_unlock(set, flags); + return 0; + } + mk_cpu_set_unlock(set, flags); - set->ids[set->nr++] = id; - return 0; + ret = mk_cpu_set_reserve(set, 1); + if (ret) + return ret; + } } bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id) { - int idx = mk_cpu_set_index(set, id); + unsigned long flags; + int idx; - if (idx < 0) + if (!set) return false; + mk_cpu_set_lock(set, &flags); + idx = mk_cpu_set_index_locked(set, id); + if (idx < 0) { + mk_cpu_set_unlock(set, flags); + return false; + } + memmove(&set->ids[idx], &set->ids[idx + 1], (set->nr - idx - 1) * sizeof(set->ids[0])); set->nr--; + mk_cpu_set_unlock(set, flags); return true; } +bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +{ + unsigned long flags; + bool found; + + if (!set) + return false; + + mk_cpu_set_lock(set, &flags); + found = mk_cpu_set_index_locked(set, id) >= 0; + mk_cpu_set_unlock(set, flags); + return found; +} + +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) +{ + unsigned long flags; + unsigned int nr; + + if (!set) + return 0; + + mk_cpu_set_lock(set, &flags); + nr = set->nr; + mk_cpu_set_unlock(set, flags); + return nr; +} + +bool mk_cpu_set_empty(const struct mk_cpu_set *set) +{ + return mk_cpu_set_count(set) == 0; +} + +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) +{ + unsigned long flags; + mk_phys_cpu_t id; + + if (!set) + return MK_PHYS_CPU_INVALID; + + mk_cpu_set_lock(set, &flags); + id = set->nr ? set->ids[0] : MK_PHYS_CPU_INVALID; + mk_cpu_set_unlock(set, flags); + return id; +} + +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id) +{ + unsigned long flags; + bool found = false; + + if (!set || !id) + return false; + + mk_cpu_set_lock(set, &flags); + if (index < set->nr) { + *id = set->ids[index]; + found = true; + } + mk_cpu_set_unlock(set, flags); + return found; +} + int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) { - unsigned int nr = mk_cpu_set_count(src); + unsigned long src_flags; + unsigned long dst_flags; + unsigned int nr; int ret; - dst->nr = 0; - ret = mk_cpu_set_reserve(dst, nr); - if (ret) - return ret; + if (!dst || !src) + return -EINVAL; - memcpy(dst->ids, src->ids, nr * sizeof(dst->ids[0])); - dst->nr = nr; - return 0; + for (;;) { + nr = mk_cpu_set_count(src); + ret = mk_cpu_set_reserve(dst, nr); + if (ret) + return ret; + + mk_cpu_set_lock(src, &src_flags); + if (src->nr > dst->cap) { + mk_cpu_set_unlock(src, src_flags); + continue; + } + + mk_cpu_set_lock(dst, &dst_flags); + memcpy(dst->ids, src->ids, src->nr * sizeof(dst->ids[0])); + dst->nr = src->nr; + mk_cpu_set_unlock(dst, dst_flags); + mk_cpu_set_unlock(src, src_flags); + return 0; + } } /** @@ -127,22 +275,31 @@ int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) * @size: Buffer size * @set: Set to format * - * Writes "none" for an empty set, a comma-separated list of physical + * Writes none for an empty set, a comma-separated list of physical * IDs otherwise. Output is truncated to @size. Returns the number of * characters written. */ int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set) { + unsigned long flags; unsigned int i; int len = 0; - if (mk_cpu_set_empty(set)) + if (!buf || !size) + return 0; + if (!set) return scnprintf(buf, size, "none"); - for (i = 0; i < set->nr; i++) { + mk_cpu_set_lock(set, &flags); + if (!set->nr) { + mk_cpu_set_unlock(set, flags); + return scnprintf(buf, size, "none"); + } + + for (i = 0; i < set->nr && len < size; i++) { len += scnprintf(buf + len, size - len, "%s%llu", i ? "," : "", set->ids[i]); } - + mk_cpu_set_unlock(set, flags); return len; } diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 77901c564d2171..faf6c180019780 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -77,25 +77,6 @@ struct mk_cpu_hotplug_work { u32 operation; /* MK_RES_CPU_ADD or MK_RES_CPU_REMOVE */ }; -/* - * Ownership tracking for this kernel's own hotplug: root_instance->cpus - * is the set of CPUs this kernel owns, in the host and in spawn kernels - * alike. The assignable-pool bookkeeping (mk_cpu_pool) is not done here; - * it belongs to the mk_send_cpu_* initiator paths of the kernel that - * manages the pool. - */ -static void mk_account_cpu_online(mk_phys_cpu_t cpu_id) -{ - if (root_instance->cpus && mk_cpu_set_add(root_instance->cpus, cpu_id)) - pr_warn("Multikernel hotplug: Failed to track CPU %llu\n", - cpu_id); -} - -static void mk_account_cpu_offline(mk_phys_cpu_t cpu_id) -{ - mk_cpu_set_del(root_instance->cpus, cpu_id); -} - /** * Search present CPUs (not possible CPUs) to find the logical CPU with matching * physical ID. Using present CPUs is important because topology can change during @@ -124,6 +105,7 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) pr_info("Multikernel hotplug: Adding CPU %llu (numa=%u, flags=0x%x)\n", cpu_id, numa_node, flags); + mk_cpu_transaction_lock(); logical_cpu = mk_cpu_to_logical(cpu_id); if (logical_cpu < 0) { /* @@ -136,7 +118,8 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (logical_cpu < 0) { pr_err("Multikernel hotplug: CPU %llu is not in this kernel's pool\n", cpu_id); - return -ENODEV; + ret = -ENODEV; + goto unlock_transaction; } set_cpu_present(logical_cpu, true); } @@ -144,18 +127,37 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (cpu_online(logical_cpu)) { pr_warn("Multikernel hotplug: CPU %d (phys %llu) already online\n", logical_cpu, cpu_id); - mk_account_cpu_online(cpu_id); - return 0; + mk_cpu_ownership_lock(); + if (root_instance->cpus) { + ret = mk_cpu_set_add(root_instance->cpus, cpu_id); + if (ret) + pr_warn("Multikernel hotplug: Failed to track CPU %llu in root pool\n", + cpu_id); + } + mk_cpu_ownership_unlock(); + ret = 0; + goto unlock_transaction; } + mk_cpu_ownership_lock(); + if (root_instance->cpus) { + ret = mk_cpu_set_reserve(root_instance->cpus, 1); + if (ret) { + mk_cpu_ownership_unlock(); + goto unlock_transaction; + } + } + mk_cpu_ownership_unlock(); + if (!get_cpu_device(logical_cpu)) { struct cpu *c = &per_cpu(cpu_devices, logical_cpu); + c->hotpluggable = true; ret = register_cpu(c, logical_cpu); if (ret) { pr_err("Multikernel hotplug: Failed to register CPU %d: %d\n", logical_cpu, ret); - return ret; + goto unlock_transaction; } } @@ -163,10 +165,19 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (ret < 0) { pr_err("Multikernel hotplug: Failed to add CPU %d (phys %llu): %d\n", logical_cpu, cpu_id, ret); - return ret; + goto unlock_transaction; } - mk_account_cpu_online(cpu_id); + mk_cpu_ownership_lock(); + if (root_instance->cpus) { + ret = mk_cpu_set_add(root_instance->cpus, cpu_id); + if (ret) + pr_warn("Multikernel hotplug: Failed to track CPU %llu in root pool\n", + cpu_id); + } + mk_cpu_ownership_unlock(); + if (ret) + goto unlock_transaction; /* Track the operation for potential rollback */ op = kzalloc(sizeof(*op), GFP_KERNEL); @@ -183,7 +194,10 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) pr_info("Multikernel hotplug: Successfully added CPU %d (phys %llu)\n", logical_cpu, cpu_id); - return 0; + ret = 0; +unlock_transaction: + mk_cpu_transaction_unlock(); + return ret; } static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) @@ -191,25 +205,45 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) int logical_cpu; int ret; struct mk_hotplug_op *op; + bool tracked; + mk_cpu_transaction_lock(); logical_cpu = mk_cpu_to_logical(cpu_id); if (logical_cpu < 0) { pr_err("Multikernel hotplug: Physical CPU %llu not found\n", cpu_id); - return -ENODEV; + ret = -ENODEV; + goto unlock_transaction; } + mk_cpu_ownership_lock(); + tracked = root_instance->cpus && + mk_cpu_set_contains(root_instance->cpus, cpu_id); + if (!cpu_online(logical_cpu)) { pr_warn("Multikernel hotplug: CPU %d (phys %llu) already offline\n", logical_cpu, cpu_id); - mk_account_cpu_offline(cpu_id); - return 0; + mk_cpu_set_del(root_instance->cpus, cpu_id); + mk_cpu_ownership_unlock(); + ret = 0; + goto unlock_transaction; + } + + if (!tracked) { + pr_err("Multikernel hotplug: CPU %llu is not tracked in root pool\n", + cpu_id); + mk_cpu_ownership_unlock(); + ret = -EINVAL; + goto unlock_transaction; } /* Don't allow removing CPU 0 (boot processor) */ if (logical_cpu == 0) { pr_err("Multikernel hotplug: Cannot remove boot CPU\n"); - return -EINVAL; + mk_cpu_ownership_unlock(); + ret = -EINVAL; + goto unlock_transaction; } + mk_cpu_ownership_unlock(); mk_set_pool_cpu(logical_cpu, true); @@ -218,10 +252,12 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) pr_err("Multikernel hotplug: Failed to remove CPU %d (phys %llu): %d\n", logical_cpu, cpu_id, ret); mk_set_pool_cpu(logical_cpu, false); - return ret; + goto unlock_transaction; } - mk_account_cpu_offline(cpu_id); + mk_cpu_ownership_lock(); + mk_cpu_set_del(root_instance->cpus, cpu_id); + mk_cpu_ownership_unlock(); /* * Clear CPU from present mask to prevent host kernel from trying @@ -244,7 +280,10 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) pr_info("Multikernel hotplug: Successfully removed CPU %d (phys %llu)\n", logical_cpu, cpu_id); - return 0; + ret = 0; +unlock_transaction: + mk_cpu_transaction_unlock(); + return ret; } static void mk_cpu_add_work_fn(struct work_struct *work) @@ -1049,6 +1088,7 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) if (target_instance->state != MK_STATE_ACTIVE) { struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; + raw_spin_lock_init(&cpus.lock); /* * A CPU the instance has already run on is parked on that * instance's context. Bring it back to the host slot before @@ -1068,22 +1108,39 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) goto out; } + mk_cpu_transaction_lock(); pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, cpu_id); if (!pending) { ret = -ENOMEM; - goto out; + goto unlock_transaction; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, - &payload, sizeof(payload)); + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { + pr_err("Multikernel hotplug: CPU %llu not assigned to instance %d\n", + cpu_id, instance_id); + mk_msg_pending_wait(pending, 0); + ret = -EINVAL; + goto unlock_ownership; + } + + ret = mk_cpu_set_reserve(mk_cpu_pool, 1); + if (ret) { + mk_msg_pending_wait(pending, 0); + goto unlock_ownership; + } + mk_cpu_ownership_unlock(); + + ret = mk_send_message(target_instance->id, MK_MSG_RESOURCE, + MK_RES_CPU_REMOVE, &payload, sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ - goto out; + goto unlock_transaction; } ret = mk_msg_pending_wait(pending, 10000); if (ret < 0) - goto out; + goto unlock_transaction; /* * The spawn kernel parked the CPU on its own context when it went @@ -1097,15 +1154,27 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) if (ret < 0) { pr_err("Multikernel hotplug: CPU %llu offline in instance %d but not reparked to host: %d\n", cpu_id, instance_id, ret); - goto out; + goto unlock_transaction; } - mk_cpu_set_del(target_instance->cpus, cpu_id); - if (!mk_cpu_pool || mk_cpu_set_add(mk_cpu_pool, cpu_id)) + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { + ret = -ESTALE; + goto unlock_ownership; + } + ret = mk_cpu_set_add(mk_cpu_pool, cpu_id); + if (ret) { pr_warn("Multikernel hotplug: Failed to track CPU %llu in pool\n", cpu_id); + goto unlock_ownership; + } + mk_cpu_set_del(target_instance->cpus, cpu_id); ret = 0; +unlock_ownership: + mk_cpu_ownership_unlock(); +unlock_transaction: + mk_cpu_transaction_unlock(); out: mk_instance_put(target_instance); return ret; @@ -1159,26 +1228,36 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl instance_id); return -ENODEV; } + if (arch_cpu_from_physical_id(cpu_id) == 0) { + pr_err("Multikernel hotplug: CPU %llu is reserved for host control\n", + cpu_id); + ret = -EINVAL; + goto out; + } /* For non-running instances, transfer CPU from root using existing API */ if (target_instance->state != MK_STATE_ACTIVE) { struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; + raw_spin_lock_init(&cpus.lock); ret = mk_instance_transfer_cpus(target_instance, &cpus); goto out; } - /* - * Only a CPU from the assignable pool is parked on the host slot; - * publishing a wakeup for any other CPU can only time out. - */ + mk_cpu_transaction_lock(); + mk_cpu_ownership_lock(); if (!mk_cpu_set_contains(mk_cpu_pool, cpu_id)) { - pr_err("Multikernel hotplug: CPU %llu is not in this kernel's pool\n", + pr_err("Multikernel hotplug: CPU %llu not available in the pool\n", cpu_id); ret = -EBUSY; - goto out; + goto unlock_ownership; } + ret = mk_cpu_set_reserve(target_instance->cpus, 1); + if (ret) + goto unlock_ownership; + mk_cpu_ownership_unlock(); + /* * The CPU is parked on the host slot, where the spawn kernel's * secondary wakeup cannot reach it. Point it at the instance's @@ -1188,22 +1267,22 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl if (ret < 0) { pr_err("Multikernel hotplug: Failed to repark CPU %llu to instance %d: %d\n", cpu_id, instance_id, ret); - goto out; + goto unlock_transaction; } pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_ADD, cpu_id); if (!pending) { mk_repark_cpu_to_host(target_instance, cpu_id); ret = -ENOMEM; - goto out; + goto unlock_transaction; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_ADD, - &payload, sizeof(payload)); + ret = mk_send_message(target_instance->id, MK_MSG_RESOURCE, + MK_RES_CPU_ADD, &payload, sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ mk_repark_cpu_to_host(target_instance, cpu_id); - goto out; + goto unlock_transaction; } ret = mk_msg_pending_wait(pending, 10000); @@ -1215,15 +1294,27 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl * watching the context and this times out harmlessly. */ mk_repark_cpu_to_host(target_instance, cpu_id); - goto out; + goto unlock_transaction; } - if (mk_cpu_set_add(target_instance->cpus, cpu_id)) + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(mk_cpu_pool, cpu_id)) { + ret = -ESTALE; + goto unlock_ownership; + } + ret = mk_cpu_set_add(target_instance->cpus, cpu_id); + if (ret) { pr_warn("Multikernel hotplug: Failed to track CPU %llu in instance %d\n", cpu_id, instance_id); + goto unlock_ownership; + } mk_cpu_set_del(mk_cpu_pool, cpu_id); ret = 0; +unlock_ownership: + mk_cpu_ownership_unlock(); +unlock_transaction: + mk_cpu_transaction_unlock(); out: mk_instance_put(target_instance); return ret; diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 6a220a7c533fc9..e4455937b6639e 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -22,6 +22,13 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, u8 *slot, u8 *func); +/* CPU ownership serialization: transaction must be acquired first. */ +void mk_cpu_transaction_lock(void); +void mk_cpu_transaction_unlock(void); +void mk_cpu_ownership_lock(void); +void mk_cpu_ownership_unlock(void); +void mk_cpu_ownership_assert_held(void); + /* pci.c */ int mk_pci_lease_system_init(void); void mk_pci_lease_system_cleanup(void); From cf09af6108ff303da9acc36945cb9c38bec4f63d Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Wed, 12 Aug 2026 13:51:10 +0300 Subject: [PATCH 11/16] multikernel: make IPI publication ordered and recoverable Serialize shared-ring producers with a bounded owner-aware gate. Preserve FIFO publication and recover a gate only after its producer CPU is known to be parked. The gate and slot-state protocol change the private shared transport layout. Add an exact pre-launch layout and protocol check at this boundary, then require a transport initialization acknowledgment after both rings have been validated and before marking the instance active. Fail invalid manifests and missing acknowledgments closed. A spawn started by a host without the pre-launch check validates the boot-context anchor before using any shifted field and enters a local interrupt-disabled halt loop on mismatch without trusting shared park state or touching reset and APIC hardware. Signed-off-by: Nikolay Nikolaev --- arch/x86/boot/header.S | 8 +- arch/x86/include/asm/multikernel.h | 6 +- arch/x86/include/uapi/asm/bootparam.h | 2 + arch/x86/kernel/kexec-bzimage64.c | 5 + arch/x86/kernel/kexec-vmlinux.c | 24 +- arch/x86/kernel/platform-quirks.c | 4 +- arch/x86/multikernel/head_64.S | 3 + arch/x86/multikernel/spawn.c | 132 ++++-- include/linux/multikernel.h | 82 +++- include/linux/multikernel_abi.h | 9 + kernel/kexec_core.c | 56 ++- kernel/multikernel/core.c | 102 ++++- kernel/multikernel/instance_dt.c | 65 ++- kernel/multikernel/internal.h | 5 + kernel/multikernel/ipi.c | 583 +++++++++++++++++++------- kernel/multikernel/manifest.c | 14 +- 16 files changed, 863 insertions(+), 237 deletions(-) create mode 100644 include/linux/multikernel_abi.h diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9bea5a1e2c52cb..56f1e2fd9471a8 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -379,7 +379,13 @@ xloadflags: #define XLF56 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 +#ifdef CONFIG_MULTIKERNEL +# define XLF9 XLF_MULTIKERNEL_IPI_V3 +#else +# define XLF9 0 +#endif + + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF9 cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index bff6074bdc8fc1..d69471790823ef 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -105,11 +105,14 @@ struct mk_spawn_context { unsigned long boot_cpu_khz; /* Host CPU frequency calibration */ unsigned long boot_tsc_khz; /* Host TSC frequency calibration */ unsigned long boot_apic_hz; /* Host local APIC timer frequency */ + u64 abi_magic; /* Validated context producer */ } __aligned(PAGE_SIZE); static_assert(offsetof(struct mk_spawn_context, bp) == 144); static_assert(offsetof(struct mk_spawn_context, boot_lps) == 144 + sizeof(struct boot_params)); +static_assert(offsetof(struct mk_spawn_context, abi_magic) == + 144 + sizeof(struct boot_params) + 4 * sizeof(unsigned long)); static_assert(sizeof(struct mk_spawn_context) == 2 * PAGE_SIZE); /* Pool park loop code, copied by the host into per-instance park pages */ @@ -156,7 +159,8 @@ void mk_set_spawn_context(struct mk_spawn_context *ctx, int mk_spawn_cpu(struct mk_instance *instance, int cpu, struct mk_spawn_context *ctx); -/* Initialize boot context tracking in spawn kernel */ +/* Validate and initialize boot context tracking in spawn kernel */ +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys); void mk_init_boot_context(phys_addr_t ctx_phys); /* Identity page table and trampoline setup */ diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index c70be687a3ecc7..bc33a5105e5142 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -25,6 +25,8 @@ #define XLF_5LEVEL (1<<5) #define XLF_5LEVEL_ENABLED (1<<6) #define XLF_MEM_ENCRYPTION (1<<7) +#define XLF_MULTIKERNEL_IPI_V2 0x0100 +#define XLF_MULTIKERNEL_IPI_V3 0x0200 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 0759f9d6b64c23..5b82205d1ae2d7 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -611,6 +611,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel, .buf_max = ULONG_MAX, .top_down = true }; header = (struct setup_header *)(kernel + setup_hdr_offset); + if (image->type == KEXEC_TYPE_MULTIKERNEL && + !(header->xloadflags & XLF_MULTIKERNEL_IPI_V3)) { + pr_err("Loaded kernel lacks the required shared transport layout\n"); + return ERR_PTR(-EPROTONOSUPPORT); + } setup_sects = header->setup_sects; if (setup_sects == 0) setup_sects = 4; diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index e8279270018c30..cbc449fce37078 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -60,6 +60,12 @@ struct elf_kernel_info { unsigned long reloc_size; /* Size of relocation data */ }; +struct mk_elf_note_desc { + u64 entry; + u32 ipi_abi_version; + u32 reserved; +}; + /* * Find multikernel entry point from PT_NOTE section. * Looks for note with name "Linux" and type 0x4d4b ('MK'). @@ -93,12 +99,20 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, if (nhdr->n_type == 0x4d4b && nhdr->n_namesz == 6 && - nhdr->n_descsz == sizeof(u64) && + nhdr->n_descsz == sizeof(struct mk_elf_note_desc) && !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) { - u64 entry = *(u64 *)(ptr + sizeof(*nhdr) + - ALIGN(nhdr->n_namesz, 4)); - pr_info("multikernel: entry=0x%llx\n", entry); - return entry; + const struct mk_elf_note_desc *desc; + + desc = ptr + sizeof(*nhdr) + + ALIGN(nhdr->n_namesz, 4); + if (desc->ipi_abi_version != MK_IPI_ABI_VERSION) { + pr_err("multikernel IPI ABI %u is not supported\n", + desc->ipi_abi_version); + return 0; + } + pr_info("multikernel: entry=0x%llx, IPI ABI=%u\n", + desc->entry, desc->ipi_abi_version); + return desc->entry; } ptr += note_size; } diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index 5227f8131a93a9..a1d1b8749112a6 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -47,9 +47,9 @@ static void __init multikernel_setup_calibration(void) { phys_addr_t ctx_phys = orig_boot_params - offsetof(struct mk_spawn_context, bp); - struct mk_spawn_context *ctx = __va(ctx_phys); + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); - if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + if (!ctx || !ctx->boot_tsc_khz) return; multikernel_tsc_khz = ctx->boot_tsc_khz; diff --git a/arch/x86/multikernel/head_64.S b/arch/x86/multikernel/head_64.S index 3784147fd82f62..b726e74ff3e402 100644 --- a/arch/x86/multikernel/head_64.S +++ b/arch/x86/multikernel/head_64.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -273,4 +274,6 @@ SYM_CODE_END(multikernel_secondary_startup) 1: .asciz "Linux" 2: .balign 4 3: .quad multikernel_startup_64 - __START_KERNEL_map + .long MK_IPI_ABI_VERSION + .long 0 4: .balign 4 diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 293788aac5c02d..c1cbd0d8aa9358 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -71,6 +71,7 @@ /* Set in spawn kernels: the context this kernel booted from */ static struct mk_spawn_context *mk_boot_context; +static phys_addr_t mk_boot_context_phys; /* * Spawn kernel's own trampoline for secondary CPU wakeup. @@ -84,6 +85,16 @@ static struct mk_spawn_context *mk_boot_context; */ static void *spawn_trampoline_va; static unsigned long spawn_trampoline_phys; +static bool spawn_trampoline_prepared; +static bool spawn_pool_park_prepared; +static bool spawn_park_ready; +static int spawn_park_error; + +bool mk_arch_park_ready(void) +{ + /* Pair with publication after both executable park mappings succeed. */ + return smp_load_acquire(&spawn_park_ready); +} extern char multikernel_relocate_kernel_start[]; extern char multikernel_relocate_kernel_end[]; @@ -458,6 +469,7 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, instance->spawn_ctx->boot_tsc_khz = tsc_khz; instance->spawn_ctx->boot_apic_hz = (unsigned long)lapic_timer_period * HZ; + instance->spawn_ctx->abi_magic = MK_BOOT_CONTEXT_MAGIC; return mk_spawn_cpu(instance, cpu, instance->spawn_ctx); } @@ -680,18 +692,38 @@ void __init mk_arch_register_cpu(u64 phys_id) topology_register_apic((u32)phys_id, CPU_ACPIID_INVALID, true); } -/* - * Initialize boot context tracking in spawn kernel. - * Called early during spawn kernel boot. - */ -void mk_init_boot_context(phys_addr_t ctx_phys) +static __noreturn void mk_reject_spawn_context(void) +{ + /* + * The host context layout is unknown, so neither its park state nor any + * shared context field is safe to use. Keep this CPU local and inert. An NMI + * can wake HLT, but returns to this loop with maskable interrupts still + * disabled; disable them again before every halt for defense in depth. + */ + for (;;) { + native_irq_disable(); + native_halt(); + } +} + +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys) { struct mk_spawn_context *ctx; + phys_addr_t stamped_phys; + u64 abi_magic; if (!ctx_phys) { pr_err("mk_spawn: Boot context physical address is 0!\n"); - return; + return NULL; } + if (mk_boot_context) { + if (ctx_phys != mk_boot_context_phys) + mk_reject_spawn_context(); + return mk_boot_context; + } + /* Reject an invalid derived address before mapping or dereferencing it. */ + if (!IS_ALIGNED(ctx_phys, PAGE_SIZE)) + mk_reject_spawn_context(); /* * The spawn context is in the multikernel pool which is regular RAM, @@ -708,14 +740,28 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * work and then fails much later, when this kernel shuts down and * its CPUs park on nonsense addresses. */ - if (ctx->self_phys != ctx_phys) { - pr_err("mk_spawn: Boot context at %pa is stamped %pa\n", - &ctx_phys, &ctx->self_phys); - pr_err("mk_spawn: Spawn context layout mismatch - host and spawn kernels must be built from the same source\n"); - return; - } - + stamped_phys = READ_ONCE(ctx->self_phys); + if (stamped_phys != ctx_phys) + mk_reject_spawn_context(); + abi_magic = READ_ONCE(ctx->abi_magic); + if (abi_magic != MK_BOOT_CONTEXT_MAGIC) + mk_reject_spawn_context(); + + mk_boot_context_phys = ctx_phys; mk_boot_context = ctx; + return ctx; +} + +/* + * Initialize boot context tracking in spawn kernel. + * Called early during spawn kernel boot. + */ +void mk_init_boot_context(phys_addr_t ctx_phys) +{ + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx) + return; /* * A spawn kernel cannot calibrate against legacy timers because they * belong to the host. Reuse the selected physical CPU's delay and local @@ -751,17 +797,25 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * * One physical page serves every wake path of this instance: the host * allocates it once in mk_setup_trampoline() and reuses it across - * re-spawns, and mk_prepare_trampoline() places our own trampoline copy + * re-spawns, and mk_arch_prepare_park() places our own trampoline copy * (including the secondary entry) in the same page. */ -static int __init mk_prepare_trampoline(void) +int __init mk_arch_prepare_park(void) { struct mk_spawn_context *ctx = mk_boot_context; unsigned long virt; int ret; + if (mk_arch_park_ready()) + return 0; + if (spawn_park_error) + return spawn_park_error; if (!ctx) return 0; + if (!ctx->trampoline_phys || !ctx->park_phys || !ctx->park_cr3) { + ret = -EINVAL; + goto fail; + } /* * Put our own copy of the trampoline in the page the host set @@ -769,34 +823,48 @@ static int __init mk_prepare_trampoline(void) * is entered from an offline CPU, where changing page attributes * is not allowed. */ - spawn_trampoline_phys = ctx->trampoline_phys; - spawn_trampoline_va = __va(spawn_trampoline_phys); - memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, - multikernel_relocate_kernel_end - multikernel_relocate_kernel_start); + if (!spawn_trampoline_prepared) { + spawn_trampoline_phys = ctx->trampoline_phys; + spawn_trampoline_va = __va(spawn_trampoline_phys); + memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, + multikernel_relocate_kernel_end - + multikernel_relocate_kernel_start); - /* - * Both pages are executed from the direct map, which is writable, - * so drop write before adding execute. Leaving them writable and - * executable trips the kernel's own W^X check. - */ - virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; - ret = set_memory_ro(virt, 1); - if (!ret) - ret = set_memory_x(virt, 1); - if (ret) - return ret; + /* + * Both pages are executed from the direct map, which is writable, + * so drop write before adding execute. Leaving them writable and + * executable trips the kernel's own W^X check. + */ + virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; + ret = set_memory_ro(virt, 1); + if (!ret) + ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_trampoline_prepared = true; + } /* The pool park page is entered the same way when this kernel dies */ - if (ctx->park_phys) { + if (!spawn_pool_park_prepared) { virt = (unsigned long)__va(ctx->park_phys) & PAGE_MASK; ret = set_memory_ro(virt, 1); if (!ret) ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_pool_park_prepared = true; } + /* Publish executable mappings before any reject or abort can park. */ + smp_store_release(&spawn_park_ready, true); + return 0; + +fail: + /* A partial W^X transition is not safe to retry. */ + spawn_park_error = ret; return ret; } -early_initcall(mk_prepare_trampoline); +early_initcall(mk_arch_prepare_park); /* * Add a 2MB executable mapping to a page table. diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index f0800c98c1e116..f6d6692c385f43 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -16,6 +16,7 @@ #include #include #include +#include struct pci_bus; @@ -69,8 +70,17 @@ bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, /* IPI ring buffer size - must be power of 2 for efficient modulo */ #define MK_IPI_RING_SIZE 64 +#define MK_IPI_SLOT_EMPTY 0 +#define MK_IPI_SLOT_WRITING 1 +#define MK_IPI_SLOT_READY 2 +#define MK_IPI_SLOT_CONSUMING 3 +#define MK_IPI_SLOT_CANCELLED 4 +#define MK_IPI_ABI_MAGIC 0x4d4b495049303033ULL /* "MKIPI003" */ +#define MK_IPI_READY_TIMEOUT_MS 120000 + /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + atomic_t state; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ @@ -79,16 +89,62 @@ struct mk_ipi_data { /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ + atomic_t head; /* Producer allocation cursor */ + atomic_t tail; /* Consumer scan cursor */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ + /* Appended shared ABI: do not move fields above this line. */ + atomic64_t producer_gate; /* Owner CPU and claimed slot */ + atomic_t producer_contention; /* Sends that observed a busy gate */ + atomic_t full_failures; /* Sends rejected by a full ring */ + atomic_t invalid_state; /* Invalid slot state observations */ + atomic_t cancelled_writes; /* Halted producer writes recovered */ }; /* Shared memory structures - per-instance design */ struct mk_shared_data { + atomic_t emergency_shutdown; struct mk_ipi_ring ring; /* IPI message ring buffer */ + /* Appended ABI handshake; existing shared offsets stay unchanged. */ + u64 abi_magic; + u32 abi_version; + u32 abi_size; + s32 ready_instance_id; + atomic_t ready; }; +static inline void mk_ipi_ring_reset_contents(struct mk_ipi_ring *ring) +{ + unsigned int i; + + atomic_set(&ring->head, 0); + atomic_set(&ring->tail, 0); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + WRITE_ONCE(ring->entries[i].data_size, 0); + atomic_set(&ring->entries[i].state, MK_IPI_SLOT_EMPTY); + } + atomic_set(&ring->producer_contention, 0); + atomic_set(&ring->full_failures, 0); + atomic_set(&ring->invalid_state, 0); + atomic_set(&ring->cancelled_writes, 0); +} + +static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + mk_ipi_ring_reset_contents(ring); + atomic64_set(&ring->producer_gate, 0); +} + +static inline void mk_shared_data_reset(struct mk_shared_data *shared) +{ + atomic_set(&shared->emergency_shutdown, 0); + mk_ipi_ring_reset(&shared->ring); + WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); + WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); + WRITE_ONCE(shared->abi_size, sizeof(*shared)); + WRITE_ONCE(shared->ready_instance_id, -1); + atomic_set(&shared->ready, 0); +} + /* Function pointer type for IPI callbacks */ typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *data, void *ctx); @@ -131,8 +187,14 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns void generic_multikernel_interrupt(void); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +int mk_ipi_shared_validate(const struct mk_shared_data *shared); +int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id); +int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, + unsigned int timeout_ms); +int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared); + +/* Recover a producer only after every CPU in @halted_cpus is parked. */ +int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus); /* * Multikernel Messaging System @@ -764,6 +826,7 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); @@ -780,6 +843,7 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); @@ -832,6 +896,11 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + +static inline bool mk_manifest_rejected(void) +{ + return false; +} #endif /** @@ -839,7 +908,8 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 -#define MK_FDT_COMPATIBLE "multikernel-v1" +/* Bumped whenever the shared-memory layout or message semantics change. */ +#define MK_FDT_COMPATIBLE "multikernel-v3" /** * Property Names @@ -988,6 +1058,8 @@ void mk_set_pool_cpu(int cpu, bool is_pool); /* Park the calling CPU in the pool wait loop; never returns */ void __noreturn mk_enter_pool_state(void *info); +int __init mk_arch_prepare_park(void); +bool mk_arch_park_ready(void); /* * Forcible stop of another instance's CPUs (NMI on x86). Registration diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h new file mode 100644 index 00000000000000..9f64fe3c398de1 --- /dev/null +++ b/include/linux/multikernel_abi.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MULTIKERNEL_ABI_H +#define _LINUX_MULTIKERNEL_ABI_H + +/* Private host/spawn transport compatibility constants. */ +#define MK_IPI_ABI_VERSION 3 +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b435458303032ULL /* "MKCTX002" */ + +#endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 63da42ffb7b67f..61b68e38b15055 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1702,6 +1702,12 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + pr_err("Multikernel instance %d is not loadable (state=%d)\n", + mk_id, instance->state); + rc = -EINVAL; + goto unlock; + } if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1768,10 +1774,11 @@ int multikernel_kexec_by_id(int mk_id) } rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + if (rc) { + pr_err("Manifest finalization failed: %d\n", rc); + goto unlock; + } + pr_info("Manifest finalized for multikernel instance\n"); /* * Point at the ring this image actually carries. Every load @@ -1787,26 +1794,33 @@ int multikernel_kexec_by_id(int mk_id) } /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. - */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); - - /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. + * Start the instance with an empty downlink after its CPUs have been + * confirmed parked. The host is the only producer for this ring, so no + * publisher can race the reset once the old receiver is quiesced. */ - mk_ipi_ring_drop_pending(); - + if (instance->ipi_data) { + rc = mk_ipi_shared_reset_downlink(instance->ipi_data); + if (rc) { + pr_err("Failed to reset instance %d IPI downlink: %d\n", + mk_id, rc); + goto unlock; + } + } rc = mk_arch_spawn_instance(mk_image, instance, cpu); if (rc == 0) { + rc = mk_ipi_shared_wait_ready(instance->ipi_data, mk_id, + MK_IPI_READY_TIMEOUT_MS); + if (rc) { + int abort_ret; + + pr_err("Instance %d did not acknowledge IPI ABI %u: %d\n", + mk_id, MK_IPI_ABI_VERSION, rc); + abort_ret = mk_instance_abort_spawn(instance); + if (abort_ret) + pr_crit("Instance %d IPI ABI timeout abort failed: %d\n", + mk_id, abort_ret); + goto unlock; + } rc = mk_instance_set_kexec_active(mk_image->mk_id); if (rc) pr_warn("Failed to set instance %d as active: %d\n", mk_image->mk_id, rc); diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index a94a325f7873d9..1077a85763ac43 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -313,6 +313,7 @@ bool multikernel_allow_emergency_restart(void) */ int mk_instance_confirm_parked(struct mk_instance *instance) { + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int ret, failed = 0; @@ -320,8 +321,19 @@ int mk_instance_confirm_parked(struct mk_instance *instance) /* Never started, so nothing of it is running */ if (!instance->spawn_ctx) return 0; + if (!instance->cpus_on_slot) + return 0; - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + ret = mk_cpu_set_copy(snapshot, instance->cpus_on_slot); + if (ret) { + mk_cpu_set_free(snapshot); + return ret; + } + + mk_cpu_set_for_each(i, phys_cpu, snapshot) { ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -329,6 +341,15 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } + if (!failed) { + ret = mk_ipi_ring_recover_halted(snapshot); + if (ret) { + pr_err("Instance %d (%s): failed to recover halted IPI producer: %d\n", + instance->id, instance->name, ret); + failed++; + } + } + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -1335,20 +1356,8 @@ int multikernel_halt_by_id(int mk_id) return ret; } -/** - * mk_instance_force_halt - Forcibly stop an instance via NMI - * @instance: Instance to stop - * - * Forces a spawn kernel's CPUs to stop by queuing a shutdown message in the - * IPI ring buffer and sending NMIs directly to each CPU. The NMI handler - * checks for the pending shutdown message and stops if found. - * - * Use when: The spawn kernel is stuck/crashed and not responding to graceful - * shutdown, or when graceful shutdown has failed. - * - * Returns: 0 on success, negative error code on failure - */ -int mk_instance_force_halt(struct mk_instance *instance) +static int __mk_instance_force_halt(struct mk_instance *instance, + bool allow_loaded) { struct mk_shutdown_payload payload; mk_phys_cpu_t phys_cpu; @@ -1356,7 +1365,11 @@ int mk_instance_force_halt(struct mk_instance *instance) int cpu_count = 0; int ret; - if (instance->state != MK_STATE_ACTIVE) { + if (!instance) + return -EINVAL; + + if (instance->state != MK_STATE_ACTIVE && + (!allow_loaded || instance->state != MK_STATE_LOADED)) { pr_err("Instance %d not active (state=%d), nothing to force halt\n", instance->id, instance->state); return -EINVAL; @@ -1370,25 +1383,62 @@ int mk_instance_force_halt(struct mk_instance *instance) pr_info("Force halting multikernel instance %d via NMI\n", instance->id); + /* Queue shutdown message - NMI handler will check for this */ payload.flags = MK_SHUTDOWN_IMMEDIATE; payload.sender_instance_id = root_instance->id; ret = mk_send_message(instance->id, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, &payload, sizeof(payload)); if (ret < 0) - pr_err("Failed to queue shutdown message: %d (sending NMI anyway)\n", - ret); + pr_err("Failed to queue shutdown message: %d (sending NMI anyway)\n", ret); + /* Send NMI to each CPU in the instance */ mk_cpu_set_for_each(i, phys_cpu, instance->cpus) { mk_force_stop_cpu(phys_cpu); cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, - instance->id); + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); + + ret = mk_instance_confirm_parked(instance); + if (ret) { + pr_err("Instance %d CPUs did not park after force halt: %d\n", + instance->id, ret); + return ret; + } + mk_instance_settle_halted(instance); return 0; } +int mk_instance_abort_spawn(struct mk_instance *instance) +{ + int ret; + + ret = __mk_instance_force_halt(instance, true); + if (ret && instance) + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; +} + +/** + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop + * + * Forces a spawn kernel's CPUs to stop by queuing a shutdown message in the + * IPI ring buffer and sending NMIs directly to each CPU. The NMI handler + * checks for the pending shutdown message and stops if found. + * + * Use when: The spawn kernel is stuck/crashed and not responding to graceful + * shutdown, or when graceful shutdown has failed. + * + * Returns: 0 on success, negative error code on failure + */ +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); +} + int multikernel_force_halt_by_id(int mk_id) { struct mk_instance *instance; @@ -1397,7 +1447,6 @@ int multikernel_force_halt_by_id(int mk_id) instance = mk_instance_find(mk_id); if (!instance) return -ENOENT; - ret = mk_instance_force_halt(instance); mk_instance_put(instance); return ret; @@ -1453,6 +1502,17 @@ static int __init multikernel_init(void) return ret; } + ret = mk_ipi_shared_mark_ready(root_instance->ipi_data, + root_instance->id); + if (ret < 0) { + pr_err("Failed to publish multikernel IPI readiness: %d\n", ret); + mk_kernfs_cleanup(); + mk_hotplug_cleanup(); + mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); + mk_messaging_cleanup(); + return ret; + } + pr_info("Multikernel support initialized\n"); return 0; } diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index a97cb1b6ea0ed0..95cbcef01fd6fb 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "internal.h" #define PROP_SUB_FDT "fdt" @@ -34,6 +35,23 @@ struct mk_instance *root_instance = NULL; EXPORT_SYMBOL_GPL(root_instance); +static void __init __noreturn mk_manifest_reject_and_park(int error) +{ + int ret; + + ret = mk_arch_prepare_park(); + if (ret || !mk_arch_park_ready()) + panic("multikernel: rejected manifest before park path became ready"); + ret = mk_register_stop_nmi_handler(); + if (ret) + pr_emerg("multikernel: stop-NMI registration failed while rejecting manifest: %d\n", + ret); + pr_emerg("multikernel: parking CPUs after rejecting supplied manifest: %d\n", + error); + smp_call_function(mk_enter_pool_state, NULL, 0); + mk_enter_pool_state(NULL); +} + /* * Collect every CPU the instance might receive through hotplug later: * the unassigned pool plus every other kernel's CPUs (the host's and @@ -390,6 +408,7 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char pr_err("Failed to allocate IPI buffer for instance %d\n", instance_id); goto err_free_name; } + mk_shared_data_reset(instance->ipi_data); instance->ipi_phys = virt_to_phys(instance->ipi_data); instance->ipi_pages = (sizeof(struct mk_shared_data) + PAGE_SIZE - 1) / PAGE_SIZE; @@ -542,6 +561,12 @@ static int __init mk_restore_instance_ipi(const void *manifest, struct mk_instan (unsigned long long)ipi_phys, ipi_pages); return 0; } + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small for ABI %u: %zu < %zu\n", + MK_IPI_ABI_VERSION, ipi_size, + sizeof(struct mk_shared_data)); + return -EPROTO; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -597,6 +622,12 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest (unsigned long long)host_ipi_cpu); return NULL; } + if (host_ipi_size < sizeof(struct mk_shared_data)) { + pr_err("Host IPI buffer is too small for ABI %u: %zu < %zu\n", + MK_IPI_ABI_VERSION, host_ipi_size, + sizeof(struct mk_shared_data)); + return NULL; + } host_instance = alloc_mk_instance(0, "", false); if (!host_instance) @@ -650,6 +681,9 @@ int __init mk_instance_restore_from_manifest(void) const void *manifest = NULL; phys_addr_t fdt_phys; + if (mk_manifest_rejected()) + mk_manifest_reject_and_park(-EPROTO); + fdt_phys = mk_manifest_phys(); if (!fdt_phys) { pr_info("No manifest available for multikernel DTB restoration\n"); @@ -687,15 +721,15 @@ int __init mk_instance_restore_from_manifest(void) int mk_node = fdt_subnode_offset(manifest, 0, "multikernel"); if (mk_node < 0) { - pr_info("No multikernel node found in manifest\n"); - ret = 0; + pr_err("No multikernel node found in supplied manifest\n"); + ret = -EINVAL; goto cleanup_fdt; } const void *dtb_data = fdt_getprop(manifest, mk_node, "dtb-data", &dtb_len); if (!dtb_data || dtb_len <= 0) { - pr_info("No dtb-data property found in multikernel node\n"); - ret = 0; + pr_err("No dtb-data property found in multikernel node\n"); + ret = -EINVAL; goto cleanup_fdt; } @@ -792,8 +826,25 @@ int __init mk_instance_restore_from_manifest(void) host_instance = mk_restore_host_instance(manifest); if (!host_instance) - pr_warn("Failed to restore host instance (spawn→host communication unavailable)\n"); - + mk_manifest_reject_and_park(-ENODEV); + + ret = mk_ipi_shared_validate(instance->ipi_data); + if (ret) + mk_manifest_reject_and_park(ret); + ret = mk_ipi_shared_validate(host_instance->ipi_data); + if (ret) + mk_manifest_reject_and_park(ret); + if (!atomic_read_acquire(&host_instance->ipi_data->ready) || + READ_ONCE(host_instance->ipi_data->ready_instance_id) != 0) + mk_manifest_reject_and_park(-EHOSTDOWN); + ret = mk_arch_prepare_park(); + if (ret) + mk_manifest_reject_and_park(ret); + if (!mk_arch_park_ready()) + mk_manifest_reject_and_park(-EIO); + ret = mk_register_stop_nmi_handler(); + if (ret) + mk_manifest_reject_and_park(ret); pr_info("Successfully restored multikernel root instance %d ('%s') from manifest (%d bytes)\n", instance_id, instance_name, dtb_len); mk_dt_config_free(&config); @@ -827,6 +878,8 @@ int __init mk_instance_restore_from_manifest(void) kfree(dtb_virt); cleanup_fdt: early_memunmap((void *)manifest, PAGE_SIZE); + if (ret) + mk_manifest_reject_and_park(ret); return ret; } diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index e4455937b6639e..145447cb67a987 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -5,6 +5,11 @@ extern struct idr mk_instance_idr; extern struct list_head mk_instance_list; extern struct mk_instance *root_instance; +/* ipi.c */ +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); +void mk_poll_ipi_messages(void); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 7f4ba78b39b06a..b29604088a2d1e 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "internal.h" /* Callback management */ @@ -20,41 +21,363 @@ static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lo static void mk_ipi_drain_ring(void); +#define MK_IPI_PRODUCER_RETRIES 10000 +#define MK_IPI_GATE_INDEX_BITS 6 +#define MK_IPI_GATE_INDEX_MASK (MK_IPI_RING_SIZE - 1) + +/* + * A nonzero gate records both the physical producer CPU and the slot at head. + * This makes the serialization recoverable after that exact CPU is confirmed + * parked. A boolean shared lock would be unsafe because the force-stop NMI may + * prevent its owner from ever returning to release it. + */ +static u64 mk_ipi_gate_token(mk_phys_cpu_t owner, unsigned int idx) +{ + BUILD_BUG_ON(BIT(MK_IPI_GATE_INDEX_BITS) != MK_IPI_RING_SIZE); + if (owner >= (U64_MAX >> MK_IPI_GATE_INDEX_BITS)) + return 0; + + return ((owner + 1) << MK_IPI_GATE_INDEX_BITS) | idx; +} + +static mk_phys_cpu_t mk_ipi_gate_owner(u64 token) +{ + return (token >> MK_IPI_GATE_INDEX_BITS) - 1; +} + +static unsigned int mk_ipi_gate_index(u64 token) +{ + return token & MK_IPI_GATE_INDEX_MASK; +} + /* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. + * Publish the complete message while local IRQs and preemption are disabled. + * No NMI path sends general messages; force halt uses emergency_shutdown. + * Advancing head before READY lets recovery distinguish both interruption + * windows without allowing another producer to pass the gate. */ -static inline unsigned int mk_ring_idx(unsigned int i) +static int mk_ipi_ring_publish(struct mk_shared_data *shared, int instance_id, + const void *data, size_t data_size, + unsigned long type) +{ + struct mk_ipi_ring *ring = &shared->ring; + struct mk_ipi_data *slot; + mk_phys_cpu_t owner; + unsigned long flags; + bool contended = false; + unsigned int retry; + unsigned int idx; + u64 token, old; + int state; + int head; + int ret; + + preempt_disable(); + local_irq_save(flags); + owner = arch_cpu_physical_id(smp_processor_id()); + + for (retry = 0; retry < MK_IPI_PRODUCER_RETRIES; retry++) { + head = atomic_read(&ring->head); + idx = head & MK_IPI_GATE_INDEX_MASK; + token = mk_ipi_gate_token(owner, idx); + if (!token) { + ret = -EOVERFLOW; + goto out_restore; + } + + old = atomic64_cmpxchg_acquire(&ring->producer_gate, 0, token); + if (!old) { + if ((atomic_read(&ring->head) & + MK_IPI_GATE_INDEX_MASK) != idx) { + atomic64_set_release(&ring->producer_gate, 0); + contended = true; + cpu_relax(); + continue; + } + break; + } + contended = true; + if (mk_ipi_gate_owner(old) == owner) { + ret = -EDEADLK; + goto out_count_contention; + } + cpu_relax(); + } + + if (retry == MK_IPI_PRODUCER_RETRIES) { + ret = -EAGAIN; + goto out_count_contention; + } + if (contended) + atomic_inc(&ring->producer_contention); + if (!atomic_read_acquire(&shared->ready) || + READ_ONCE(shared->ready_instance_id) != instance_id) { + ret = -ESHUTDOWN; + goto out_release_gate; + } + + slot = &ring->entries[idx]; + state = atomic_cmpxchg(&slot->state, MK_IPI_SLOT_EMPTY, + MK_IPI_SLOT_WRITING); + if (state != MK_IPI_SLOT_EMPTY) { + if (state == MK_IPI_SLOT_READY || + state == MK_IPI_SLOT_CONSUMING || + state == MK_IPI_SLOT_CANCELLED) { + atomic_inc(&ring->full_failures); + ret = -ENOSPC; + } else { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + goto out_release_gate; + } + + WRITE_ONCE(slot->data_size, 0); + WRITE_ONCE(slot->sender_cpu, owner); + WRITE_ONCE(slot->type, type); + if (data_size) + memcpy(slot->buffer, data, data_size); + WRITE_ONCE(slot->data_size, data_size); + atomic_set(&ring->head, (idx + 1) & MK_IPI_GATE_INDEX_MASK); + atomic_set_release(&slot->state, MK_IPI_SLOT_READY); + ret = 0; + +out_release_gate: + atomic64_set_release(&ring->producer_gate, 0); +out_restore: + local_irq_restore(flags); + preempt_enable(); + return ret; + +out_count_contention: + atomic_inc(&ring->producer_contention); + goto out_restore; +} + +static bool mk_ipi_slot_is_pending(int state) +{ + return state == MK_IPI_SLOT_READY || + state == MK_IPI_SLOT_CANCELLED; +} + +static void mk_ipi_slot_release(struct mk_ipi_data *slot) +{ + WRITE_ONCE(slot->data_size, 0); + atomic_set_release(&slot->state, MK_IPI_SLOT_EMPTY); +} + +int mk_ipi_shared_validate(const struct mk_shared_data *shared) +{ + if (!shared) + return -ENODEV; + if (READ_ONCE(shared->abi_magic) != MK_IPI_ABI_MAGIC || + READ_ONCE(shared->abi_version) != MK_IPI_ABI_VERSION || + READ_ONCE(shared->abi_size) != sizeof(*shared)) + return -EPROTO; + + return 0; +} + +int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id) { - return i & (MK_IPI_RING_SIZE - 1); + int ret; + + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + WRITE_ONCE(shared->ready_instance_id, instance_id); + atomic_set_release(&shared->ready, 1); + return 0; +} + +int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, + unsigned int timeout_ms) +{ + unsigned long deadline; + int ret; + + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + deadline = jiffies + msecs_to_jiffies(timeout_ms); + do { + if (atomic_read_acquire(&shared->ready)) + return READ_ONCE(shared->ready_instance_id) == instance_id ? + 0 : -EPROTO; + msleep(20); + } while (time_before(jiffies, deadline)); + + return -ETIMEDOUT; +} + +int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared) +{ + struct mk_ipi_ring *ring; + mk_phys_cpu_t owner; + unsigned long flags; + unsigned int retry; + unsigned int idx; + u64 token, old; + int head; + int ret = 0; + + if (!shared) + return -EINVAL; + + /* Exclude new publishers before waiting for an in-flight one. */ + atomic_set_release(&shared->ready, 0); + /* Pair exclusion with the readiness recheck after gate acquisition. */ + smp_mb(); + ring = &shared->ring; + preempt_disable(); + local_irq_save(flags); + owner = arch_cpu_physical_id(smp_processor_id()); + for (retry = 0; retry < MK_IPI_PRODUCER_RETRIES; retry++) { + head = atomic_read(&ring->head); + idx = head & MK_IPI_GATE_INDEX_MASK; + token = mk_ipi_gate_token(owner, idx); + if (!token) { + ret = -EOVERFLOW; + goto out_restore; + } + old = atomic64_cmpxchg_acquire(&ring->producer_gate, 0, token); + if (!old) + break; + if (mk_ipi_gate_owner(old) == owner) { + ret = -EDEADLK; + goto out_restore; + } + cpu_relax(); + } + if (retry == MK_IPI_PRODUCER_RETRIES) { + ret = -EAGAIN; + goto out_restore; + } + + /* The old receiver is parked and every pre-existing publisher drained. */ + atomic_set(&shared->emergency_shutdown, 0); + mk_ipi_ring_reset_contents(ring); + WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); + WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); + WRITE_ONCE(shared->abi_size, sizeof(*shared)); + WRITE_ONCE(shared->ready_instance_id, -1); + atomic_set(&shared->ready, 0); + atomic64_set_release(&ring->producer_gate, 0); + +out_restore: + local_irq_restore(flags); + preempt_enable(); + return ret; } /** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring + * mk_ipi_ring_recover_halted - Recover a ring producer after it is parked + * @halted_cpus: Exact set of CPUs confirmed parked by the caller * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. + * A force-stop NMI can park a producer while it owns the shared gate. Only its + * receiver may recover the write, and only after the owner CPU is proven not + * to be executing it. Other instances may still publish into the host ring, + * so this function never resets the ring or touches another owner's gate. */ -void mk_ipi_ring_drop_pending(void) +int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus) { + struct mk_ipi_data *slot; struct mk_ipi_ring *ring; - unsigned int head, tail; - - if (!root_instance || !root_instance->ipi_data) - return; + mk_phys_cpu_t owner; + mk_phys_cpu_t target; + unsigned int idx; + unsigned int next; + u64 token; + int state; + int head; + int ret = 0; + + if (!halted_cpus || !root_instance || !root_instance->ipi_data) + return -EINVAL; ring = &root_instance->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); + token = atomic64_read_acquire(&ring->producer_gate); + if (!token) + goto kick; + if (!(token >> MK_IPI_GATE_INDEX_BITS)) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + goto kick; + } + + owner = mk_ipi_gate_owner(token); + if (!mk_cpu_set_contains(halted_cpus, owner)) + goto kick; + + idx = mk_ipi_gate_index(token); + next = (idx + 1) & MK_IPI_GATE_INDEX_MASK; + slot = &ring->entries[idx]; + state = atomic_read_acquire(&slot->state); + head = atomic_read(&ring->head) & MK_IPI_GATE_INDEX_MASK; + switch (state) { + case MK_IPI_SLOT_EMPTY: + if (head != idx && head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + case MK_IPI_SLOT_WRITING: + if (head == idx) { + atomic_set(&ring->head, next); + } else if (head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + break; + } + atomic_set_release(&slot->state, MK_IPI_SLOT_CANCELLED); + atomic_inc(&ring->cancelled_writes); + break; + case MK_IPI_SLOT_READY: + case MK_IPI_SLOT_CANCELLED: + if (head == idx) { + /* Repair an interrupted publication before releasing its gate. */ + atomic_set(&ring->head, next); + atomic_inc(&ring->invalid_state); + } else if (head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + case MK_IPI_SLOT_CONSUMING: + /* + * The consumer may claim a full-ring tail while this producer + * waits to test it, leaving head at idx with nothing published. + * Head at next means the previous publication was claimed before + * its now-stale gate could be released. Both cursors are valid. + */ + if (head != idx && head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + default: + atomic_inc(&ring->invalid_state); + ret = -EIO; + break; + } + + /* Keep the gate closed when cursor repair cannot make the FIFO safe. */ + if (ret) + goto kick; + if (atomic64_cmpxchg_release(&ring->producer_gate, token, 0) != token) { + atomic_inc(&ring->invalid_state); + ret = -EAGAIN; + } - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; +kick: + /* The producer may have published and parked before ringing the bell. */ + target = mk_cpu_set_first(root_instance->cpus); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); - atomic_set(&ring->tail, head); + return ret; } /** @@ -119,7 +442,7 @@ EXPORT_SYMBOL(multikernel_unregister_handler); /** * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID + * @instance: Target multikernel instance * @data: Pointer to data to send * @data_size: Size of data * @type: User-defined type identifier @@ -129,24 +452,22 @@ EXPORT_SYMBOL(multikernel_unregister_handler); * * Returns 0 on success, negative error code on failure */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) { - struct mk_ipi_data *slot; - struct mk_instance *instance = mk_instance_find(instance_id); - unsigned int head, next_head, tail; mk_phys_cpu_t target; + int instance_id; + int ret; if (!instance) return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); + instance_id = instance->id; + if (data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; - } target = mk_cpu_set_first(instance->cpus); if (target == MK_PHYS_CPU_INVALID) { pr_err("Instance %d has no CPUs to receive the IPI\n", instance_id); - mk_instance_put(instance); return -ENODEV; } @@ -165,98 +486,98 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns if (!instance->ipi_data) { pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); return -ENODEV; } + ret = mk_ipi_shared_validate(instance->ipi_data); + if (ret) + return ret; + if (!atomic_read_acquire(&instance->ipi_data->ready)) + return -EAGAIN; + if (READ_ONCE(instance->ipi_data->ready_instance_id) != instance_id) + return -EPROTO; + + ret = mk_ipi_ring_publish(instance->ipi_data, instance_id, data, + data_size, type); + if (ret) { + /* + * A doorbell can be coalesced while the target is draining this + * ring. Kick it again before reporting backpressure so READY + * entries cannot remain stranded without another notification. + */ + mk_arch_send_ipi(target); + if (ret == -ENOSPC) + pr_warn_ratelimited("multikernel: IPI ring full for instance %d\n", + instance_id); + else if (ret == -EAGAIN) + pr_warn_ratelimited("multikernel: IPI producer busy for instance %d\n", + instance_id); + else if (ret != -EDEADLK) + pr_err_ratelimited("multikernel: IPI publish failed for instance %d: %d\n", + instance_id, ret); + return ret; + } + mk_arch_send_ipi(target); - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&instance->ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&instance->ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - mk_instance_put(instance); - return -ENOSPC; - } - - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&instance->ipi_data->ring.head, head, next_head) != head); - - /* We've claimed slot 'head', now fill it */ - slot = &instance->ipi_data->ring.entries[head]; - - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; - - if (data && data_size > 0) - memcpy(slot->buffer, data, data_size); - - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); + return 0; +} - mk_arch_send_ipi(target); +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + struct mk_instance *instance; + int ret; + instance = mk_instance_find(instance_id); + if (!instance) + return -EINVAL; + ret = mk_send_ipi_data(instance, data, data_size, type); mk_instance_put(instance); - return 0; + return ret; } static void mk_ipi_drain_ring(void) { struct mk_ipi_data *slot; struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; + struct mk_ipi_ring *ring; + unsigned int tail, idx; size_t data_size; + int state; int messages_processed = 0; if (!root_instance || !root_instance->ipi_data) return; - while (1) { - tail = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.head)); - - if (tail == head) + ring = &root_instance->ipi_data->ring; + while (messages_processed < MK_IPI_RING_SIZE) { + tail = atomic_read(&ring->tail); + idx = tail & (MK_IPI_RING_SIZE - 1); + slot = &ring->entries[idx]; + + state = atomic_read_acquire(&slot->state); + if (!mk_ipi_slot_is_pending(state)) { + if (state != MK_IPI_SLOT_EMPTY && + state != MK_IPI_SLOT_WRITING && + state != MK_IPI_SLOT_CONSUMING) { + atomic_inc(&ring->invalid_state); + pr_warn_once("Multikernel IPI slot %u has bad state %d\n", + idx, state); + } break; + } - slot = &root_instance->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + if (atomic_cmpxchg_acquire(&slot->state, state, + MK_IPI_SLOT_CONSUMING) != state) break; + if (state == MK_IPI_SLOT_CANCELLED) + goto advance_tail; + + data_size = READ_ONCE(slot->data_size); if (data_size > MK_MAX_DATA_SIZE) { pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); - continue; + idx, data_size); + goto advance_tail; } /* Dispatch to registered handler */ @@ -274,17 +595,29 @@ static void mk_ipi_drain_ring(void) raw_spin_unlock(&mk_handlers_lock); advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); + mk_ipi_slot_release(slot); + atomic_set(&ring->tail, (idx + 1) & (MK_IPI_RING_SIZE - 1)); messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; } } +void mk_poll_ipi_messages(void) +{ + unsigned long flags; + mk_phys_cpu_t target; + + if (!root_instance) + return; + target = mk_cpu_set_first(root_instance->cpus); + if (target == MK_PHYS_CPU_INVALID || + target != arch_cpu_physical_id(smp_processor_id())) + return; + + local_irq_save(flags); + mk_ipi_drain_ring(); + local_irq_restore(flags); +} + /** * multikernel_interrupt_handler - Handle the multikernel IPI * @@ -320,8 +653,9 @@ void generic_multikernel_interrupt(void) /** * mk_has_pending_shutdown - Check if there's a pending shutdown message * - * Peeks at the IPI ring buffer to check for a MK_SYS_SHUTDOWN message - * with MK_SHUTDOWN_IMMEDIATE flag. Used by NMI handler for force halt. + * Checks the dedicated emergency flag published before force-halt NMIs. + * This path must not depend on claiming a normal IPI ring slot because a + * crashed producer can leave slots unavailable. * * Safe to call from NMI context (no locks, read-only peek). * @@ -329,41 +663,8 @@ void generic_multikernel_interrupt(void) */ bool mk_has_pending_shutdown(void) { - struct mk_ipi_data *slot; - struct mk_message *msg; - struct mk_shutdown_payload *payload; - unsigned int head, tail, idx, scanned; - if (!root_instance || !root_instance->ipi_data) return false; - tail = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.head)); - - /* - * Scan pending messages without consuming them. The trip count is - * bounded by the ring size rather than by the indices alone: this - * runs in NMI context, where a never-terminating loop takes the CPU - * out permanently with NMIs latched. - */ - for (scanned = 0, idx = tail; - idx != head && scanned < MK_IPI_RING_SIZE; - idx = mk_ring_idx(idx + 1), scanned++) { - slot = &root_instance->ipi_data->ring.entries[idx]; - - if (slot->data_size < sizeof(struct mk_message)) - continue; - - msg = (struct mk_message *)slot->buffer; - if (msg->msg_type != MK_MSG_SYSTEM || msg->msg_subtype != MK_SYS_SHUTDOWN) - continue; - - if (msg->payload_len >= sizeof(struct mk_shutdown_payload)) { - payload = (struct mk_shutdown_payload *)msg->payload; - if (payload->flags & MK_SHUTDOWN_IMMEDIATE) - return true; - } - } - - return false; + return atomic_read_acquire(&root_instance->ipi_data->emergency_shutdown); } diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index 89a8ee0d2c59b3..19cdc625e9e04d 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -23,12 +23,18 @@ /* Physical address of the manifest this kernel booted with, 0 if none */ static phys_addr_t mk_manifest_fdt_phys; +static bool mk_manifest_fdt_rejected; phys_addr_t mk_manifest_phys(void) { return mk_manifest_fdt_phys; } +bool mk_manifest_rejected(void) +{ + return READ_ONCE(mk_manifest_fdt_rejected); +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT @@ -50,6 +56,7 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) if (!fdt) { pr_warn("multikernel: failed to memremap manifest (0x%llx)\n", fdt_phys); + err = -ENOMEM; goto out; } @@ -68,14 +75,17 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) } mk_manifest_fdt_phys = fdt_phys; + mk_manifest_fdt_rejected = false; pr_info("multikernel: manifest accepted\n"); out: if (fdt) early_memunmap(fdt, fdt_len); - if (err) - pr_warn("multikernel: ignoring invalid manifest\n"); + if (err) { + mk_manifest_fdt_rejected = true; + pr_warn("multikernel: supplied manifest rejected: %d\n", err); + } } /** From 1e113168dbf939bbc44f03d67781a8385160025b Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 03:28:14 +0300 Subject: [PATCH 12/16] multikernel: pin control routes across CPU ownership changes Cache the IRQ forwarding CPU for hardirq-safe routing. Serialize route, ownership, reload, halt, and teardown mutations while active users hold a route reference. Drain assignment IRQ producers before publishing a replacement route or reparking the old CPU. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 26 ++++- kernel/kexec_core.c | 30 +++++- kernel/multikernel/core.c | 176 +++++++++++++++++++++++++------ kernel/multikernel/hotplug.c | 56 ++++++++-- kernel/multikernel/instance_dt.c | 2 + kernel/multikernel/internal.h | 23 ++++ kernel/multikernel/ipi.c | 37 +++++-- kernel/multikernel/kernfs.c | 2 + kernel/multikernel/messaging.c | 42 +++++++- net/vmw_vsock/mk_transport.c | 3 +- 10 files changed, 340 insertions(+), 57 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index f6d6692c385f43..395c6cf5674689 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -17,8 +17,10 @@ #include #include #include +#include struct pci_bus; +struct mk_instance; /** * Physical CPU identifiers @@ -553,6 +555,9 @@ struct mk_instance { /* CPU resources */ struct mk_cpu_set *cpus; /* Set of assigned physical CPU IDs */ + /* Pins the CPU selected for control messages and forwarded IRQs. */ + struct rw_semaphore control_route_sem; + mk_phys_cpu_t irq_route_cpu; /* IRQ-safe cached forwarding target */ /* PCI device resources */ struct list_head pci_devices; /* List of struct mk_pci_device */ @@ -614,6 +619,20 @@ struct mk_instance { struct kref refcount; /* Reference count for cleanup */ }; +static inline mk_phys_cpu_t +mk_instance_irq_route_load(const struct mk_instance *instance) +{ + /* Pairs with route publication before old-target IRQs are drained. */ + return smp_load_acquire(&instance->irq_route_cpu); +} + +static inline void mk_instance_irq_route_store(struct mk_instance *instance, + mk_phys_cpu_t target) +{ + /* Publish the new target before synchronize_irq() drains old users. */ + smp_store_release(&instance->irq_route_cpu, target); +} + /** * Device Tree Parsing Functions */ @@ -771,9 +790,11 @@ void mk_instance_free_memory(struct mk_instance *instance); int mk_instance_transfer_cpus(struct mk_instance *instance, - const struct mk_cpu_set *cpus); + const struct mk_cpu_set *cpus); int mk_instance_return_cpus(struct mk_instance *instance, - const struct mk_cpu_set *cpus); + const struct mk_cpu_set *cpus); +void mk_cpu_transaction_lock(void); +void mk_cpu_transaction_unlock(void); int mk_instance_add_memory_region(struct mk_instance *instance, size_t size); int mk_instance_remove_memory_region(struct mk_instance *instance, phys_addr_t phys_addr, size_t size); @@ -826,6 +847,7 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +void mk_instance_mark_failed(struct mk_instance *instance); int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 61b68e38b15055..d1451197bb97eb 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -595,6 +595,7 @@ void kimage_free(struct kimage *image) { kimage_entry_t *ptr, entry; kimage_entry_t ind = 0; + struct mk_instance *route_instance = NULL; if (!image) return; @@ -609,6 +610,10 @@ void kimage_free(struct kimage *image) if (image->type == KEXEC_TYPE_MULTIKERNEL) { unsigned long i; + route_instance = image->mk_instance; + if (route_instance) + down_write(&route_instance->control_route_sem); + for (i = 0; i < image->nr_segments; i++) { void *virt_addr = phys_to_virt(image->segment[i].mem); @@ -629,7 +634,6 @@ void kimage_free(struct kimage *image) image->mk_instance->ipi_phys = 0; image->mk_instance->kimage = NULL; mk_instance_set_state(image->mk_instance, MK_STATE_READY); - mk_instance_put(image->mk_instance); image->mk_instance = NULL; } @@ -644,6 +648,10 @@ void kimage_free(struct kimage *image) __free_pages(phys_to_page(image->mk_ipi), order); image->mk_ipi = 0; } + if (route_instance) { + up_write(&route_instance->control_route_sem); + mk_instance_put(route_instance); + } } #ifdef CONFIG_CRASH_DUMP if (image->vmcoreinfo_data_copy) { @@ -1688,6 +1696,8 @@ int multikernel_kexec_by_id(int mk_id) { struct kimage *mk_image; struct mk_instance *instance; + bool transaction_locked = false; + bool route_locked = false; int cpu = -1; int i, rc; @@ -1708,9 +1718,16 @@ int multikernel_kexec_by_id(int mk_id) rc = -EINVAL; goto unlock; } + mk_cpu_transaction_lock(); + transaction_locked = true; + down_write(&instance->control_route_sem); + route_locked = true; if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); + if (!mk_cpu_set_contains(instance->cpus, + mk_instance_irq_route_load(instance))) + mk_instance_irq_route_store(instance, phys_cpu); cpu = arch_cpu_from_physical_id(phys_cpu); if (cpu < 0) { pr_err("Physical CPU %llu not found in logical CPU map\n", phys_cpu); @@ -1815,6 +1832,13 @@ int multikernel_kexec_by_id(int mk_id) pr_err("Instance %d did not acknowledge IPI ABI %u: %d\n", mk_id, MK_IPI_ABI_VERSION, rc); + mutex_lock(&instance->resource_mutex); + mk_instance_set_state(instance, MK_STATE_FAILED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + route_locked = false; + mk_cpu_transaction_unlock(); + transaction_locked = false; abort_ret = mk_instance_abort_spawn(instance); if (abort_ret) pr_crit("Instance %d IPI ABI timeout abort failed: %d\n", @@ -1827,6 +1851,10 @@ int multikernel_kexec_by_id(int mk_id) } unlock: + if (route_locked) + up_write(&instance->control_route_sem); + if (transaction_locked) + mk_cpu_transaction_unlock(); kexec_unlock(); return rc; } diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 1077a85763ac43..285d2607c7e85d 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -15,7 +15,7 @@ #include #include "internal.h" -/* CPU moves hold the transaction lock before the ownership lock. */ +/* Lock order: transaction -> route write -> ownership -> resources. */ static DEFINE_MUTEX(mk_cpu_transaction_mutex); static DEFINE_MUTEX(mk_cpu_ownership_mutex); @@ -44,6 +44,55 @@ void mk_cpu_ownership_assert_held(void) lockdep_assert_held(&mk_cpu_ownership_mutex); } +static int __mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing) +{ + mk_phys_cpu_t replacement = MK_PHYS_CPU_INVALID; + mk_phys_cpu_t route_cpu; + mk_phys_cpu_t phys_cpu; + unsigned int i; + + lockdep_assert_held_write(&instance->control_route_sem); + route_cpu = mk_instance_irq_route_load(instance); + if (route_cpu == MK_PHYS_CPU_INVALID || + !mk_cpu_set_contains(removing, route_cpu)) + return 0; + + mk_cpu_ownership_lock(); + mk_cpu_set_for_each(i, phys_cpu, instance->cpus) { + if (!mk_cpu_set_contains(removing, phys_cpu)) { + replacement = phys_cpu; + break; + } + } + mk_cpu_ownership_unlock(); + if (replacement == MK_PHYS_CPU_INVALID && + READ_ONCE(instance->state) == MK_STATE_ACTIVE) + return -EBUSY; + + mutex_lock(&instance->resource_mutex); + if (replacement != MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, replacement); + mk_pci_sync_instance_irq_route(instance); + if (replacement == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); + mutex_unlock(&instance->resource_mutex); + return 0; +} + +int mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing) +{ + int ret; + + if (!instance || !removing) + return -EINVAL; + down_write(&instance->control_route_sem); + ret = __mk_instance_migrate_irq_route(instance, removing); + up_write(&instance->control_route_sem); + return ret; +} + static void mk_instance_return_all_cpus(struct mk_instance *instance) { if (!instance || mk_cpu_set_empty(instance->cpus)) @@ -217,6 +266,17 @@ void mk_instance_set_state(struct mk_instance *instance, */ } +void mk_instance_mark_failed(struct mk_instance *instance) +{ + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + mk_instance_set_state(instance, MK_STATE_FAILED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); +} + struct mk_instance *mk_instance_find_by_name(const char *name) { struct mk_instance *instance; @@ -367,8 +427,10 @@ int mk_instance_confirm_parked(struct mk_instance *instance) int mk_instance_transfer_cpus(struct mk_instance *instance, const struct mk_cpu_set *cpus) { + struct mk_cpu_set *snapshot; unsigned int i, requested_count; mk_phys_cpu_t phys_cpu; + int logical_cpu; int unavailable = 0; char buf[256]; int ret; @@ -377,15 +439,25 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, pr_err("Invalid CPU sets for transfer\n"); return -EINVAL; } + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mk_cpu_ownership_lock(); + ret = mk_cpu_set_copy(snapshot, cpus); + if (ret) + goto unlock; - requested_count = mk_cpu_set_count(cpus); + requested_count = mk_cpu_set_count(snapshot); if (requested_count == 0) { pr_info("No CPUs requested for instance %d (%s)\n", instance->id, instance->name); - return 0; + ret = 0; + goto unlock; } - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, snapshot) { if (!mk_cpu_set_contains(mk_cpu_pool, phys_cpu)) { pr_err("CPU %llu not available in the pool\n", phys_cpu); @@ -393,33 +465,48 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, continue; } - if (arch_cpu_from_physical_id(phys_cpu) < 0) { + logical_cpu = arch_cpu_from_physical_id(phys_cpu); + if (logical_cpu < 0) { pr_err("Physical CPU %llu not found in logical CPU map\n", phys_cpu); unavailable++; + } else if (logical_cpu == 0) { + pr_err("Physical CPU %llu is reserved for host control\n", + phys_cpu); + unavailable++; } } if (unavailable > 0) { pr_err("Instance %d (%s): %d CPUs are not available\n", instance->id, instance->name, unavailable); - return -EBUSY; + ret = -EBUSY; + goto unlock; } ret = mk_cpu_set_reserve(instance->cpus, requested_count); if (ret) - return ret; + goto unlock; - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, snapshot) { mk_cpu_set_del(mk_cpu_pool, phys_cpu); mk_cpu_set_add(instance->cpus, phys_cpu); } + if (mk_instance_irq_route_load(instance) == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, + mk_cpu_set_first(instance->cpus)); mk_cpu_set_format(buf, sizeof(buf), instance->cpus); pr_info("Transferred %u CPUs from pool to instance %d (%s): %s\n", requested_count, instance->id, instance->name, buf); - return 0; + ret = 0; +unlock: + mk_cpu_ownership_unlock(); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + mk_cpu_set_free(snapshot); + return ret; } /** @@ -435,6 +522,7 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, int mk_instance_return_cpus(struct mk_instance *instance, const struct mk_cpu_set *cpus) { + struct mk_cpu_set *snapshot; unsigned int i, requested_count; mk_phys_cpu_t phys_cpu; int not_found = 0; @@ -445,16 +533,26 @@ int mk_instance_return_cpus(struct mk_instance *instance, pr_err("Invalid CPU sets for return\n"); return -EINVAL; } + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mk_cpu_ownership_lock(); + ret = mk_cpu_set_copy(snapshot, cpus); + if (ret) + goto unlock; - requested_count = mk_cpu_set_count(cpus); + requested_count = mk_cpu_set_count(snapshot); if (requested_count == 0) { pr_info("No CPUs requested to return from instance %d (%s)\n", instance->id, instance->name); - return 0; + ret = 0; + goto unlock; } /* Validate all CPUs are assigned to this instance */ - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, snapshot) { if (!mk_cpu_set_contains(instance->cpus, phys_cpu)) { pr_err("CPU %llu not assigned to instance %d (%s)\n", phys_cpu, instance->id, instance->name); @@ -465,22 +563,21 @@ int mk_instance_return_cpus(struct mk_instance *instance, if (not_found > 0) { pr_err("Instance %d (%s): %d CPUs are not assigned to this instance\n", instance->id, instance->name, not_found); - return -EINVAL; + ret = -EINVAL; + goto unlock; } ret = mk_cpu_set_reserve(mk_cpu_pool, requested_count); if (ret) - return ret; - - mk_cpu_set_format(buf, sizeof(buf), cpus); + goto unlock; - /* - * @cpus may alias instance->cpus (returning everything on - * teardown), so walk it back-to-front: a deletion then never - * shifts entries the walk has yet to visit. - */ - for (i = requested_count; i-- > 0; ) { - phys_cpu = cpus->ids[i]; + mk_cpu_set_format(buf, sizeof(buf), snapshot); + mk_cpu_ownership_unlock(); + ret = __mk_instance_migrate_irq_route(instance, snapshot); + if (ret) + goto unlock_route; + mk_cpu_ownership_lock(); + mk_cpu_set_for_each(i, phys_cpu, snapshot) { mk_cpu_set_add(mk_cpu_pool, phys_cpu); mk_cpu_set_del(instance->cpus, phys_cpu); } @@ -488,7 +585,14 @@ int mk_instance_return_cpus(struct mk_instance *instance, pr_info("Returned %u CPUs from instance %d (%s) to the pool: %s\n", requested_count, instance->id, instance->name, buf); - return 0; + ret = 0; +unlock: + mk_cpu_ownership_unlock(); +unlock_route: + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + mk_cpu_set_free(snapshot); + return ret; } static int mk_instance_reserve_cpus(struct mk_instance *instance, @@ -1200,11 +1304,21 @@ static void mk_shutdown_work_fn(struct work_struct *work) * it corrupts the single-producer mailbox. The kexec path confirms the * CPUs are parked before it rewrites the image. */ -static void mk_instance_settle_halted(struct mk_instance *instance) +static void mk_instance_settle_halted(struct mk_instance *instance, + bool transaction_held) { pr_info("Instance %d (%s) halted, CPUs parking in pool\n", instance->id, instance->name); + if (!transaction_held) + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); mk_instance_set_state(instance, MK_STATE_LOADED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + if (!transaction_held) + mk_cpu_transaction_unlock(); } struct mk_halted_work { @@ -1220,7 +1334,7 @@ static void mk_halted_work_fn(struct work_struct *work) instance = mk_instance_find(aw->instance_id); if (instance) { - mk_instance_settle_halted(instance); + mk_instance_settle_halted(instance, false); mk_instance_put(instance); } else { pr_warn("Shutdown ACK from unknown instance %d\n", @@ -1348,7 +1462,7 @@ int multikernel_halt_by_id(int mk_id) pr_warn("Multikernel instance %d halted with CPUs unaccounted for\n", mk_id); - mk_instance_set_state(instance, MK_STATE_LOADED); + mk_instance_settle_halted(instance, false); pr_info("Multikernel instance %d halted (graceful)\n", mk_id); } @@ -1369,7 +1483,9 @@ static int __mk_instance_force_halt(struct mk_instance *instance, return -EINVAL; if (instance->state != MK_STATE_ACTIVE && - (!allow_loaded || instance->state != MK_STATE_LOADED)) { + (!allow_loaded || + (instance->state != MK_STATE_LOADED && + instance->state != MK_STATE_FAILED))) { pr_err("Instance %d not active (state=%d), nothing to force halt\n", instance->id, instance->state); return -EINVAL; @@ -1407,7 +1523,7 @@ static int __mk_instance_force_halt(struct mk_instance *instance, return ret; } - mk_instance_settle_halted(instance); + mk_instance_settle_halted(instance, false); return 0; } @@ -1417,7 +1533,7 @@ int mk_instance_abort_spawn(struct mk_instance *instance) ret = __mk_instance_force_halt(instance, true); if (ret && instance) - mk_instance_set_state(instance, MK_STATE_FAILED); + mk_instance_mark_failed(instance); return ret; } diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index faf6c180019780..cc2cc02354f308 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -663,6 +663,7 @@ static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 pay * PCI Device Hotplug Operations */ +#ifdef CONFIG_PCI static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { @@ -795,6 +796,18 @@ static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) return 0; } +#else +static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, + const char *driver_override, u32 flags) +{ + return -EOPNOTSUPP; +} + +static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) +{ + return -EOPNOTSUPP; +} +#endif struct mk_device_hotplug_work { struct work_struct work; @@ -1057,6 +1070,7 @@ void mk_hotplug_cleanup(void) */ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) { + struct mk_cpu_set removing = { .nr = 1, .cap = 1, .ids = &cpu_id }; struct mk_cpu_resource_payload payload = { .cpu_id = cpu_id, .numa_node = 0, @@ -1065,8 +1079,11 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) }; struct mk_pending_msg *pending; struct mk_instance *target_instance; + mk_phys_cpu_t route_cpu; int ret; + raw_spin_lock_init(&removing.lock); + /* For self-removal, execute directly (we're in process context) */ if (instance_id == root_instance->id) { ret = mk_do_cpu_remove(cpu_id); @@ -1119,20 +1136,29 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { pr_err("Multikernel hotplug: CPU %llu not assigned to instance %d\n", cpu_id, instance_id); - mk_msg_pending_wait(pending, 0); ret = -EINVAL; - goto unlock_ownership; + mk_cpu_ownership_unlock(); + mk_msg_pending_wait(pending, 0); + goto unlock_transaction; } ret = mk_cpu_set_reserve(mk_cpu_pool, 1); if (ret) { + mk_cpu_ownership_unlock(); mk_msg_pending_wait(pending, 0); - goto unlock_ownership; + goto unlock_transaction; } mk_cpu_ownership_unlock(); + ret = mk_instance_migrate_irq_route(target_instance, &removing); + if (ret) { + mk_msg_pending_wait(pending, 0); + goto unlock_transaction; + } + route_cpu = mk_instance_irq_route_load(target_instance); - ret = mk_send_message(target_instance->id, MK_MSG_RESOURCE, - MK_RES_CPU_REMOVE, &payload, sizeof(payload)); + ret = mk_send_message_to_cpu(target_instance, route_cpu, + MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, + &payload, sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ goto unlock_transaction; @@ -1157,6 +1183,7 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) goto unlock_transaction; } + down_write(&target_instance->control_route_sem); mk_cpu_ownership_lock(); if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { ret = -ESTALE; @@ -1173,6 +1200,7 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) ret = 0; unlock_ownership: mk_cpu_ownership_unlock(); + up_write(&target_instance->control_route_sem); unlock_transaction: mk_cpu_transaction_unlock(); out: @@ -1250,12 +1278,15 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl pr_err("Multikernel hotplug: CPU %llu not available in the pool\n", cpu_id); ret = -EBUSY; - goto unlock_ownership; + mk_cpu_ownership_unlock(); + goto unlock_transaction; } ret = mk_cpu_set_reserve(target_instance->cpus, 1); - if (ret) - goto unlock_ownership; + if (ret) { + mk_cpu_ownership_unlock(); + goto unlock_transaction; + } mk_cpu_ownership_unlock(); /* @@ -1277,8 +1308,9 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl goto unlock_transaction; } - ret = mk_send_message(target_instance->id, MK_MSG_RESOURCE, - MK_RES_CPU_ADD, &payload, sizeof(payload)); + ret = mk_send_message_to_instance(target_instance, MK_MSG_RESOURCE, + MK_RES_CPU_ADD, &payload, + sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ mk_repark_cpu_to_host(target_instance, cpu_id); @@ -1297,6 +1329,7 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl goto unlock_transaction; } + down_write(&target_instance->control_route_sem); mk_cpu_ownership_lock(); if (!mk_cpu_set_contains(mk_cpu_pool, cpu_id)) { ret = -ESTALE; @@ -1309,10 +1342,13 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl goto unlock_ownership; } mk_cpu_set_del(mk_cpu_pool, cpu_id); + if (mk_instance_irq_route_load(target_instance) == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(target_instance, cpu_id); ret = 0; unlock_ownership: mk_cpu_ownership_unlock(); + up_write(&target_instance->control_route_sem); unlock_transaction: mk_cpu_transaction_unlock(); out: diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 95cbcef01fd6fb..df804639af6028 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -422,6 +422,8 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char goto err_free_ipi; instance->state = MK_STATE_READY; + init_rwsem(&instance->control_route_sem); + instance->irq_route_cpu = MK_PHYS_CPU_INVALID; INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); kref_init(&instance->refcount); diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 145447cb67a987..15517f521cf7ca 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -8,8 +8,18 @@ extern struct mk_instance *root_instance; /* ipi.c */ int mk_send_ipi_data(struct mk_instance *instance, void *data, size_t data_size, unsigned long type); +int mk_send_ipi_data_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, void *data, + size_t data_size, unsigned long type); void mk_poll_ipi_messages(void); +/* messaging.c */ +int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, + u32 subtype, void *payload, u32 payload_len); +int mk_send_message_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, u32 msg_type, u32 subtype, + void *payload, u32 payload_len); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; @@ -18,6 +28,14 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, struct mk_instance *mk_instance_find_by_name(const char *name); int mk_instance_destroy(struct mk_instance *instance); int mk_instance_release_resources(struct mk_instance *instance); +void mk_cpu_transaction_lock(void); +void mk_cpu_transaction_unlock(void); +void mk_cpu_ownership_lock(void); +void mk_cpu_ownership_unlock(void); +void mk_cpu_ownership_assert_held(void); +/* Caller serializes CPU ownership changes with mk_cpu_transaction_lock(). */ +int mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing); /* dts.c */ int mk_dt_parse_resources(const void *fdt, int resources_node, @@ -47,6 +65,11 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn); int mk_pci_release_assignments(struct mk_instance *instance); +static inline unsigned int +mk_pci_sync_instance_irq_route(struct mk_instance *instance) +{ + return 0; +} int mk_instance_force_halt(struct mk_instance *instance); /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index b29604088a2d1e..c2ff6fb37884c0 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -452,25 +452,19 @@ EXPORT_SYMBOL(multikernel_unregister_handler); * * Returns 0 on success, negative error code on failure */ -int mk_send_ipi_data(struct mk_instance *instance, void *data, - size_t data_size, unsigned long type) +static int __mk_send_ipi_data(struct mk_instance *instance, + mk_phys_cpu_t target, void *data, + size_t data_size, unsigned long type) { - mk_phys_cpu_t target; int instance_id; int ret; - if (!instance) + if (!instance || target == MK_PHYS_CPU_INVALID) return -EINVAL; instance_id = instance->id; if (data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; - target = mk_cpu_set_first(instance->cpus); - if (target == MK_PHYS_CPU_INVALID) { - pr_err("Instance %d has no CPUs to receive the IPI\n", instance_id); - return -ENODEV; - } - if (!instance->ipi_data) { struct mk_shared_data *ipi_data = NULL; @@ -521,6 +515,29 @@ int mk_send_ipi_data(struct mk_instance *instance, void *data, return 0; } +int mk_send_ipi_data_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, void *data, + size_t data_size, unsigned long type) +{ + return __mk_send_ipi_data(instance, target, data, data_size, type); +} + +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) +{ + mk_phys_cpu_t target; + + if (!instance) + return -EINVAL; + target = mk_cpu_set_first(instance->cpus); + if (target == MK_PHYS_CPU_INVALID) { + pr_err("Instance %d has no CPUs to receive the IPI\n", + instance->id); + return -ENODEV; + } + return __mk_send_ipi_data(instance, target, data, data_size, type); +} + int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) { diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index 3d152e613a32ac..5759df97df5c88 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -269,6 +269,8 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); + init_rwsem(&instance->control_route_sem); + instance->irq_route_cpu = MK_PHYS_CPU_INVALID; INIT_LIST_HEAD(&instance->pci_devices); mk_pci_lease_instance_init(instance); INIT_LIST_HEAD(&instance->platform_devices); diff --git a/kernel/multikernel/messaging.c b/kernel/multikernel/messaging.c index 2cd2b7682f8d3e..208fc22367c32d 100644 --- a/kernel/multikernel/messaging.c +++ b/kernel/multikernel/messaging.c @@ -11,6 +11,7 @@ #include #include #include +#include "internal.h" /* Pending message tracking for request-response pattern */ struct mk_pending_msg { @@ -190,8 +191,10 @@ int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms * * Returns 0 on success, negative error code on failure */ -int mk_send_message(int instance_id, u32 msg_type, u32 subtype, - void *payload, u32 payload_len) +static int __mk_send_message(struct mk_instance *instance, int instance_id, + mk_phys_cpu_t target, + u32 msg_type, u32 subtype, void *payload, + u32 payload_len) { struct mk_message *msg; size_t total_size; @@ -223,7 +226,14 @@ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, memcpy(msg->payload, payload, payload_len); /* Send via IPI using the message type as IPI type */ - ret = multikernel_send_ipi_data(instance_id, msg, total_size, msg_type); + if (instance && target != MK_PHYS_CPU_INVALID) + ret = mk_send_ipi_data_to_cpu(instance, target, msg, total_size, + msg_type); + else if (instance) + ret = mk_send_ipi_data(instance, msg, total_size, msg_type); + else + ret = multikernel_send_ipi_data(instance_id, msg, total_size, + msg_type); /* Clean up temporary buffer */ kfree(msg); @@ -238,6 +248,32 @@ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, return 0; } + +int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, + u32 subtype, void *payload, u32 payload_len) +{ + if (!instance) + return -EINVAL; + return __mk_send_message(instance, instance->id, MK_PHYS_CPU_INVALID, + msg_type, subtype, payload, payload_len); +} + +int mk_send_message_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, u32 msg_type, u32 subtype, + void *payload, u32 payload_len) +{ + if (!instance || target == MK_PHYS_CPU_INVALID) + return -EINVAL; + return __mk_send_message(instance, instance->id, target, msg_type, + subtype, payload, payload_len); +} + +int mk_send_message(int instance_id, u32 msg_type, u32 subtype, + void *payload, u32 payload_len) +{ + return __mk_send_message(NULL, instance_id, MK_PHYS_CPU_INVALID, + msg_type, subtype, payload, payload_len); +} EXPORT_SYMBOL(mk_send_message); /** diff --git a/net/vmw_vsock/mk_transport.c b/net/vmw_vsock/mk_transport.c index f75dbf3f2c8209..e7594eaee1393f 100644 --- a/net/vmw_vsock/mk_transport.c +++ b/net/vmw_vsock/mk_transport.c @@ -279,7 +279,8 @@ static void mk_vsock_rx_work(struct work_struct *work) } static void mk_vsock_ipi_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx) { struct sk_buff *skb; struct virtio_vsock_hdr *hdr; From 119c06d213d2b06c9de0ea9a6e310080e7ef37c4 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 4 Aug 2026 11:04:33 +0300 Subject: [PATCH 13/16] multikernel: mediate PCI config with direct replies Proxy spawn PCI configuration through the host so only a live VF lease selected by an assigned BDF can access hardware. Return results through preallocated generation-tagged reply slots. Atomic callers wait only on their slot with a bounded deadline and never drain the general ring. Bump the exact transport ABI to version 4. Signed-off-by: Nikolay Nikolaev --- arch/x86/boot/header.S | 6 +- arch/x86/include/uapi/asm/bootparam.h | 1 + arch/x86/kernel/kexec-bzimage64.c | 2 +- arch/x86/multikernel/pci.c | 146 ++++++---- include/linux/multikernel.h | 113 +++++++- include/linux/multikernel_abi.h | 2 +- kernel/multikernel/core.c | 3 +- kernel/multikernel/hotplug.c | 3 +- kernel/multikernel/ipi.c | 369 ++++++++++++++++++++++++-- kernel/multikernel/messaging.c | 3 +- kernel/multikernel/pci.c | 264 ++++++++++++++++++ 11 files changed, 837 insertions(+), 75 deletions(-) diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 56f1e2fd9471a8..18ee3cb89340bc 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -380,12 +380,12 @@ xloadflags: #endif #ifdef CONFIG_MULTIKERNEL -# define XLF9 XLF_MULTIKERNEL_IPI_V3 +# define XLF10 XLF_MULTIKERNEL_IPI_V4 #else -# define XLF9 0 +# define XLF10 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF9 + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF10 cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index bc33a5105e5142..5dc95ebdf1bfc0 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -27,6 +27,7 @@ #define XLF_MEM_ENCRYPTION (1<<7) #define XLF_MULTIKERNEL_IPI_V2 0x0100 #define XLF_MULTIKERNEL_IPI_V3 0x0200 +#define XLF_MULTIKERNEL_IPI_V4 0x0400 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 5b82205d1ae2d7..6615d0b04aa05a 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -612,7 +612,7 @@ static void *bzImage64_load(struct kimage *image, char *kernel, header = (struct setup_header *)(kernel + setup_hdr_offset); if (image->type == KEXEC_TYPE_MULTIKERNEL && - !(header->xloadflags & XLF_MULTIKERNEL_IPI_V3)) { + !(header->xloadflags & XLF_MULTIKERNEL_IPI_V4)) { pr_err("Loaded kernel lacks the required shared transport layout\n"); return ERR_PTR(-EPROTONOSUPPORT); } diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c index 2a3d9a84375386..ab9917a0cdd86d 100644 --- a/arch/x86/multikernel/pci.c +++ b/arch/x86/multikernel/pci.c @@ -6,6 +6,7 @@ * filtered here and become host-mediated once the RPC transport is installed. */ #include +#include #include #include #include @@ -14,9 +15,84 @@ #include #include -static const struct pci_raw_ops *mk_pci_native_raw_ops; -static const struct pci_raw_ops *mk_pci_native_raw_ext_ops; static bool mk_pci_roots_ready; +static atomic64_t mk_pci_request_id = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_count = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_total_ns = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_max_ns = ATOMIC64_INIT(0); +static struct mk_instance *mk_pci_host_instance; + +static void mk_pci_record_latency(u64 start) +{ + u64 elapsed = ktime_get_mono_fast_ns() - start; + u64 old_max = atomic64_read(&mk_pci_cfg_max_ns); + + atomic64_inc(&mk_pci_cfg_count); + atomic64_add(elapsed, &mk_pci_cfg_total_ns); + while (elapsed > old_max) { + u64 previous = atomic64_cmpxchg(&mk_pci_cfg_max_ns, old_max, + elapsed); + + if (previous == old_max) + break; + old_max = previous; + } +} + +static int mk_pci_remote_config(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + bool write, u32 *value) +{ + struct mk_pci_cfg_request request = { + .request_id = atomic64_inc_return(&mk_pci_request_id), + .sender_instance_id = root_instance ? root_instance->id : -1, + .domain = domain, + .bus = bus, + .devfn = devfn, + .reg = where, + .len = size, + .write = write, + .value = *value, + }; + struct mk_reply_handle reply; + s32 status; + u32 response_value; + u64 start = ktime_get_mono_fast_ns(); + int ret; + + ret = mk_reply_reserve(root_instance->ipi_data, MK_REPLY_PCI_CFG, + request.request_id, &reply); + if (ret) + goto out_error; + request.reply_slot = reply.slot; + request.reply_generation = reply.generation; + ret = mk_send_message(0, MK_MSG_PCI, MK_PCI_CFG_REQUEST, + &request, sizeof(request)); + if (ret) { + mk_reply_release(root_instance->ipi_data, &reply); + goto out_error; + } + ret = mk_reply_wait_atomic(root_instance->ipi_data, &reply, 20000, + &status, &response_value); + if (ret) + goto out_error; + if (status < 0) { + ret = status; + goto out_error; + } + if (!write) + *value = response_value; + mk_pci_record_latency(start); + return status; + +out_error: + if (ret < 0) { + pr_err_ratelimited("Multikernel PCI config request timed out or failed to send: %d\n", + ret); + return PCIBIOS_SET_FAILED; + } + return PCIBIOS_SET_FAILED; +} static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, u32 *value) @@ -32,8 +108,7 @@ static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, return true; } -static int mk_pci_raw_read(const struct pci_raw_ops *native, - unsigned int domain, unsigned int bus, +static int mk_pci_raw_read(unsigned int domain, unsigned int bus, unsigned int devfn, int where, int size, u32 *value) { @@ -46,65 +121,35 @@ static int mk_pci_raw_read(const struct pci_raw_ops *native, } if (mk_pci_identity_read(vendor, device, where, size, value)) return PCIBIOS_SUCCESSFUL; - return native->read(domain, bus, devfn, where, size, value); + return mk_pci_remote_config(domain, bus, devfn, where, size, false, + value); } -static int mk_pci_raw_write(const struct pci_raw_ops *native, - unsigned int domain, unsigned int bus, +static int mk_pci_raw_write(unsigned int domain, unsigned int bus, unsigned int devfn, int where, int size, u32 value) { if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, NULL, NULL)) return PCIBIOS_DEVICE_NOT_FOUND; - return native->write(domain, bus, devfn, where, size, value); -} - -static int mk_pci_read(unsigned int domain, unsigned int bus, - unsigned int devfn, int where, int size, u32 *value) -{ - return mk_pci_raw_read(mk_pci_native_raw_ops, domain, bus, devfn, - where, size, value); -} - -static int mk_pci_write(unsigned int domain, unsigned int bus, - unsigned int devfn, int where, int size, u32 value) -{ - return mk_pci_raw_write(mk_pci_native_raw_ops, domain, bus, devfn, - where, size, value); -} - -static int mk_pci_ext_read(unsigned int domain, unsigned int bus, - unsigned int devfn, int where, int size, u32 *value) -{ - return mk_pci_raw_read(mk_pci_native_raw_ext_ops, domain, bus, devfn, - where, size, value); -} - -static int mk_pci_ext_write(unsigned int domain, unsigned int bus, - unsigned int devfn, int where, int size, u32 value) -{ - return mk_pci_raw_write(mk_pci_native_raw_ext_ops, domain, bus, devfn, - where, size, value); + return mk_pci_remote_config(domain, bus, devfn, where, size, true, + &value); } static const struct pci_raw_ops mk_pci_filtered_raw_ops = { - .read = mk_pci_read, - .write = mk_pci_write, -}; - -static const struct pci_raw_ops mk_pci_filtered_raw_ext_ops = { - .read = mk_pci_ext_read, - .write = mk_pci_ext_write, + .read = mk_pci_raw_read, + .write = mk_pci_raw_write, }; static int __init x86_multikernel_pci_arch_init(void) { if (!root_instance || !root_instance->pci_devices_valid) return 0; - - mk_pci_native_raw_ops = raw_pci_ops; - mk_pci_native_raw_ext_ops = raw_pci_ext_ops; + mk_pci_host_instance = mk_instance_find(0); + if (!mk_pci_host_instance) { + pr_err("Multikernel has no restored host instance for PCI control\n"); + return 0; + } raw_pci_ops = &mk_pci_filtered_raw_ops; - raw_pci_ext_ops = &mk_pci_filtered_raw_ext_ops; + raw_pci_ext_ops = &mk_pci_filtered_raw_ops; mk_pci_roots_ready = true; pr_notice("Multikernel selected filtered raw PCI config access\n"); return 0; @@ -183,6 +228,13 @@ static int __init x86_multikernel_pci_init(void) device->func); pci_bus_add_devices(bus); } + if (atomic64_read(&mk_pci_cfg_count)) { + u64 count = atomic64_read(&mk_pci_cfg_count); + + pr_notice("Multikernel PCI control plane: %llu config requests, average %llu ns, max %llu ns\n", + count, atomic64_read(&mk_pci_cfg_total_ns) / count, + atomic64_read(&mk_pci_cfg_max_ns)); + } /* Suppress legacy bus 0 probing after every assigned function is present. */ return 0; diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 395c6cf5674689..981fd94bbf84b4 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -77,9 +77,52 @@ bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, #define MK_IPI_SLOT_READY 2 #define MK_IPI_SLOT_CONSUMING 3 #define MK_IPI_SLOT_CANCELLED 4 -#define MK_IPI_ABI_MAGIC 0x4d4b495049303033ULL /* "MKIPI003" */ +#define MK_IPI_ABI_MAGIC 0x4d4b495049303034ULL /* "MKIPI004" */ #define MK_IPI_READY_TIMEOUT_MS 120000 +#define MK_REPLY_SLOTS 16 +#define MK_REPLY_STATE_BITS 3 + +enum mk_reply_state { + MK_REPLY_FREE = 0, + MK_REPLY_RESERVED, + MK_REPLY_WRITING, + MK_REPLY_EXECUTING, + MK_REPLY_COMMITTED, + MK_REPLY_READY, + MK_REPLY_ABANDONED, +}; + +enum mk_reply_kind { + MK_REPLY_PCI_CFG = 1, + MK_REPLY_PCI_IRQ, +}; + +struct mk_reply_slot { + atomic64_t state_generation; + u64 request_id; + u32 kind; + s32 status; + u32 value; + u32 reserved; +}; + +struct mk_reply_table { + struct mk_reply_slot slots[MK_REPLY_SLOTS]; + atomic_t late_replies; + atomic_t cancelled_slots; + atomic_t atomic_timeouts; + atomic_t indeterminate_timeouts; + atomic_t occupied_failures; +}; + +struct mk_reply_handle { + u32 slot; + u32 kind; + u64 request_id; + u64 generation; +}; + /* Data structure for passing parameters via IPI */ struct mk_ipi_data { atomic_t state; @@ -112,8 +155,28 @@ struct mk_shared_data { u32 abi_size; s32 ready_instance_id; atomic_t ready; + /* Appended direct synchronous replies; keep all older offsets stable. */ + struct mk_reply_table replies; }; +static inline void mk_reply_table_reset(struct mk_reply_table *table) +{ + unsigned int i; + + for (i = 0; i < MK_REPLY_SLOTS; i++) { + atomic64_set(&table->slots[i].state_generation, MK_REPLY_FREE); + WRITE_ONCE(table->slots[i].request_id, 0); + WRITE_ONCE(table->slots[i].kind, 0); + WRITE_ONCE(table->slots[i].status, 0); + WRITE_ONCE(table->slots[i].value, 0); + } + atomic_set(&table->late_replies, 0); + atomic_set(&table->cancelled_slots, 0); + atomic_set(&table->atomic_timeouts, 0); + atomic_set(&table->indeterminate_timeouts, 0); + atomic_set(&table->occupied_failures, 0); +} + static inline void mk_ipi_ring_reset_contents(struct mk_ipi_ring *ring) { unsigned int i; @@ -145,6 +208,7 @@ static inline void mk_shared_data_reset(struct mk_shared_data *shared) WRITE_ONCE(shared->abi_size, sizeof(*shared)); WRITE_ONCE(shared->ready_instance_id, -1); atomic_set(&shared->ready, 0); + mk_reply_table_reset(&shared->replies); } /* Function pointer type for IPI callbacks */ @@ -212,6 +276,7 @@ int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus); #define MK_MSG_SYSTEM 0x3000 #define MK_MSG_USER 0x4000 #define MK_MSG_NETWORK 0x5000 +#define MK_MSG_PCI 0x6000 /* I/O interrupt forwarding subtypes */ #define MK_IO_IRQ_FORWARD (MK_MSG_IO + 1) @@ -238,6 +303,9 @@ int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus); /* Network/vsock subtypes */ #define MK_NET_VSOCK_PKT (MK_MSG_NETWORK + 1) /* vsock packet */ #define MK_NET_DATA_READY (MK_MSG_NETWORK + 2) /* Data available notification */ +/* Host-mediated PCI control-plane subtypes */ +#define MK_PCI_CFG_REQUEST (MK_MSG_PCI + 1) +#define MK_PCI_CFG_RESPONSE (MK_MSG_PCI + 2) /** * Core message structure @@ -262,6 +330,27 @@ struct mk_io_irq_payload { u32 flags; /* Control flags (priority, etc.) */ }; +struct mk_pci_cfg_request { + u64 request_id; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u16 reg; + u8 len; + u8 write; + u32 value; + u32 reply_slot; + u32 reply_reserved; + u64 reply_generation; +}; + +struct mk_pci_cfg_response { + u64 request_id; + s32 status; + u32 value; +}; + /* IRQ control flags */ #define MK_IRQ_HIGH_PRIORITY 0x01 #define MK_IRQ_LOW_LATENCY 0x02 @@ -330,7 +419,8 @@ struct mk_shutdown_payload { * Message handler callback type */ typedef void (*mk_msg_handler_t)(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx); + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx); /* Opaque type for pending message tracking */ struct mk_pending_msg; @@ -370,6 +460,23 @@ int mk_register_msg_handler(u32 msg_type, mk_msg_handler_t handler, void *ctx); * Returns 0 on success, negative error code on failure */ int mk_unregister_msg_handler(u32 msg_type, mk_msg_handler_t handler); +int mk_reply_reserve(struct mk_shared_data *shared, u32 kind, u64 request_id, + struct mk_reply_handle *reply); +int mk_reply_claim(struct mk_instance *instance, + const struct mk_reply_handle *reply); +int mk_reply_begin_execute(struct mk_instance *instance, + const struct mk_reply_handle *reply); +int mk_reply_publish(struct mk_instance *instance, + const struct mk_reply_handle *reply, s32 status, u32 value); +int mk_reply_wait_atomic(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_us, + s32 *status, u32 *value); +int mk_reply_wait(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_ms, + s32 *status, u32 *value); +void mk_reply_release(struct mk_shared_data *shared, + struct mk_reply_handle *reply); +void mk_reply_scan(struct mk_shared_data *shared); /* Pending message tracking for request-response pattern */ struct mk_pending_msg *mk_msg_pending_add(u32 msg_type, u32 operation, u64 resource_id); @@ -931,7 +1038,7 @@ static inline bool mk_manifest_rejected(void) #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 /* Bumped whenever the shared-memory layout or message semantics change. */ -#define MK_FDT_COMPATIBLE "multikernel-v3" +#define MK_FDT_COMPATIBLE "multikernel-v4" /** * Property Names diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h index 9f64fe3c398de1..54df3e77834c5d 100644 --- a/include/linux/multikernel_abi.h +++ b/include/linux/multikernel_abi.h @@ -3,7 +3,7 @@ #define _LINUX_MULTIKERNEL_ABI_H /* Private host/spawn transport compatibility constants. */ -#define MK_IPI_ABI_VERSION 3 +#define MK_IPI_ABI_VERSION 4 #define MK_BOOT_CONTEXT_MAGIC 0x4d4b435458303032ULL /* "MKCTX002" */ #endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 285d2607c7e85d..f241bfff11e58d 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -1345,7 +1345,8 @@ static void mk_halted_work_fn(struct work_struct *work) } static void mk_system_msg_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx) { if (msg_type != MK_MSG_SYSTEM) return; diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index cc2cc02354f308..a234b0ff6cf0d0 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -943,7 +943,8 @@ static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u * message subtype. */ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx) { int ret = 0; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index c2ff6fb37884c0..1b302f644f1c98 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include "internal.h" /* Callback management */ @@ -20,10 +22,28 @@ static struct mk_ipi_handler *mk_handlers; static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); static void mk_ipi_drain_ring(void); +static DECLARE_WAIT_QUEUE_HEAD(mk_reply_waitq); #define MK_IPI_PRODUCER_RETRIES 10000 #define MK_IPI_GATE_INDEX_BITS 6 #define MK_IPI_GATE_INDEX_MASK (MK_IPI_RING_SIZE - 1) +#define MK_REPLY_STATE_MASK (BIT(MK_REPLY_STATE_BITS) - 1) +#define MK_REPLY_GENERATION_MAX (U64_MAX >> MK_REPLY_STATE_BITS) + +static u64 mk_reply_token(u64 generation, enum mk_reply_state state) +{ + return generation << MK_REPLY_STATE_BITS | state; +} + +static u64 mk_reply_generation(u64 token) +{ + return token >> MK_REPLY_STATE_BITS; +} + +static enum mk_reply_state mk_reply_state(u64 token) +{ + return token & MK_REPLY_STATE_MASK; +} /* * A nonzero gate records both the physical producer CPU and the slot at head. @@ -191,6 +211,336 @@ int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id) return 0; } +int mk_reply_reserve(struct mk_shared_data *shared, u32 kind, u64 request_id, + struct mk_reply_handle *reply) +{ + struct mk_reply_table *table; + unsigned int i; + int ret; + + if (!reply || !request_id || !kind) + return -EINVAL; + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + table = &shared->replies; + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &table->slots[i]; + u64 generation; + u64 claim; + u64 old; + + old = atomic64_read(&slot->state_generation); + if (mk_reply_state(old) != MK_REPLY_FREE) + continue; + generation = mk_reply_generation(old) + 1; + if (!generation || generation > MK_REPLY_GENERATION_MAX) + generation = 1; + claim = mk_reply_token(generation, MK_REPLY_WRITING); + if (atomic64_cmpxchg_acquire(&slot->state_generation, old, + claim) != old) + continue; + + WRITE_ONCE(slot->request_id, request_id); + WRITE_ONCE(slot->kind, kind); + WRITE_ONCE(slot->status, -ETIMEDOUT); + WRITE_ONCE(slot->value, ~0U); + atomic64_set_release(&slot->state_generation, + mk_reply_token(generation, + MK_REPLY_RESERVED)); + reply->slot = i; + reply->kind = kind; + reply->request_id = request_id; + reply->generation = generation; + return 0; + } + + atomic_inc(&table->occupied_failures); + return -ENOSPC; +} + +static int mk_reply_take_ready(struct mk_shared_data *shared, + struct mk_reply_handle *reply, + s32 *status, u32 *value) +{ + struct mk_reply_slot *slot = &shared->replies.slots[reply->slot]; + u64 ready = mk_reply_token(reply->generation, MK_REPLY_READY); + u64 free = mk_reply_token(reply->generation, MK_REPLY_FREE); + + if (atomic64_read_acquire(&slot->state_generation) != ready) + return -EAGAIN; + if (READ_ONCE(slot->request_id) != reply->request_id || + READ_ONCE(slot->kind) != reply->kind) + return -EPROTO; + if (status) + *status = READ_ONCE(slot->status); + if (value) + *value = READ_ONCE(slot->value); + if (atomic64_cmpxchg_release(&slot->state_generation, ready, free) != + ready) + return -EAGAIN; + return 0; +} + +static bool mk_reply_cancel(struct mk_shared_data *shared, + struct mk_reply_handle *reply, bool atomic_timeout) +{ + struct mk_reply_table *table = &shared->replies; + struct mk_reply_slot *slot = &table->slots[reply->slot]; + u64 reserved = mk_reply_token(reply->generation, MK_REPLY_RESERVED); + u64 writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + u64 executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + u64 committed = mk_reply_token(reply->generation, MK_REPLY_COMMITTED); + u64 abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + u64 ready = mk_reply_token(reply->generation, MK_REPLY_READY); + u64 free = mk_reply_token(reply->generation, MK_REPLY_FREE); + u64 token; + + if (atomic64_cmpxchg_release(&slot->state_generation, reserved, free) == + reserved) + goto cancelled; + + for (;;) { + token = atomic64_read_acquire(&slot->state_generation); + if (token == ready) + return true; + if (token == writing && + atomic64_cmpxchg_release(&slot->state_generation, writing, + abandoned) == writing) + break; + if (token == executing && + atomic64_cmpxchg_release(&slot->state_generation, executing, + committed) == executing) { + atomic_inc(&table->indeterminate_timeouts); + goto timed_out; + } + if (token != writing && token != executing) + break; + } + +cancelled: + atomic_inc(&table->cancelled_slots); +timed_out: + if (atomic_timeout) + atomic_inc(&table->atomic_timeouts); + return false; +} + +static bool mk_reply_wait_done(struct mk_shared_data *shared, + const struct mk_reply_handle *reply) +{ + struct mk_reply_slot *slot = &shared->replies.slots[reply->slot]; + u64 token; + + token = atomic64_read_acquire(&slot->state_generation); + return token == mk_reply_token(reply->generation, MK_REPLY_READY) || + mk_reply_generation(token) != reply->generation || + mk_reply_state(token) == MK_REPLY_FREE; +} + +int mk_reply_wait_atomic(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_us, + s32 *status, u32 *value) +{ + u64 deadline; + + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + deadline = ktime_get_mono_fast_ns() + (u64)timeout_us * NSEC_PER_USEC; + for (;;) { + int ret = mk_reply_take_ready(shared, reply, status, value); + + if (!ret) + return 0; + if (ret != -EAGAIN) + return ret; + if (ktime_get_mono_fast_ns() >= deadline) + break; + cpu_relax(); + } + + if (mk_reply_cancel(shared, reply, true)) + return mk_reply_take_ready(shared, reply, status, value); + return -ETIMEDOUT; +} + +int mk_reply_wait(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_ms, + s32 *status, u32 *value) +{ + long waited; + int ret; + + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + waited = wait_event_timeout(mk_reply_waitq, + mk_reply_wait_done(shared, reply), + msecs_to_jiffies(timeout_ms)); + if (!waited) { + if (mk_reply_cancel(shared, reply, false)) + return mk_reply_take_ready(shared, reply, status, value); + return -ETIMEDOUT; + } + ret = mk_reply_take_ready(shared, reply, status, value); + if (ret == -EAGAIN) + ret = -ESTALE; + return ret; +} + +void mk_reply_release(struct mk_shared_data *shared, + struct mk_reply_handle *reply) +{ + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return; + mk_reply_cancel(shared, reply, false); +} + +int mk_reply_claim(struct mk_instance *instance, + const struct mk_reply_handle *reply) +{ + struct mk_shared_data *shared; + struct mk_reply_slot *slot; + u64 writing; + u64 abandoned; + u64 reserved; + u64 free; + int ret; + + if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + shared = instance->ipi_data; + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + if (!atomic_read_acquire(&shared->ready) || + READ_ONCE(shared->ready_instance_id) != instance->id) + return -ESHUTDOWN; + + slot = &shared->replies.slots[reply->slot]; + reserved = mk_reply_token(reply->generation, MK_REPLY_RESERVED); + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + free = mk_reply_token(reply->generation, MK_REPLY_FREE); + if (atomic64_cmpxchg_acquire(&slot->state_generation, reserved, + writing) != reserved) { + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (READ_ONCE(slot->request_id) != reply->request_id || + READ_ONCE(slot->kind) != reply->kind) { + u64 old; + + old = atomic64_cmpxchg_release(&slot->state_generation, writing, + free); + if (old == abandoned) + old = atomic64_cmpxchg_release(&slot->state_generation, + abandoned, free); + if (old == writing || old == abandoned) + wake_up_all(&mk_reply_waitq); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + return 0; +} + +int mk_reply_begin_execute(struct mk_instance *instance, + const struct mk_reply_handle *reply) +{ + struct mk_reply_slot *slot; + u64 writing; + u64 executing; + u64 old; + + if (!instance || !instance->ipi_data || !reply || + reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + slot = &instance->ipi_data->replies.slots[reply->slot]; + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + old = atomic64_cmpxchg_acquire(&slot->state_generation, writing, + executing); + if (old == writing) + return 0; + if (old == mk_reply_token(reply->generation, MK_REPLY_ABANDONED)) + return -ECANCELED; + return -ESTALE; +} + +int mk_reply_publish(struct mk_instance *instance, + const struct mk_reply_handle *reply, s32 status, u32 value) +{ + struct mk_shared_data *shared; + struct mk_reply_slot *slot; + mk_phys_cpu_t target; + u64 writing; + u64 executing; + u64 committed; + u64 abandoned; + u64 ready; + u64 free; + u64 old; + int ret; + + if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + shared = instance->ipi_data; + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + slot = &shared->replies.slots[reply->slot]; + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + committed = mk_reply_token(reply->generation, MK_REPLY_COMMITTED); + abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + ready = mk_reply_token(reply->generation, MK_REPLY_READY); + free = mk_reply_token(reply->generation, MK_REPLY_FREE); + + WRITE_ONCE(slot->status, status); + WRITE_ONCE(slot->value, value); + old = atomic64_cmpxchg_release(&slot->state_generation, executing, ready); + if (old == writing) + old = atomic64_cmpxchg_release(&slot->state_generation, writing, + ready); + if (old == abandoned) { + atomic64_set_release(&slot->state_generation, free); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (old == committed) { + atomic64_set_release(&slot->state_generation, free); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (old != executing && old != writing) { + atomic_inc(&shared->replies.late_replies); + return -EIO; + } + + target = mk_cpu_set_first(instance->cpus); + if (target == MK_PHYS_CPU_INVALID) + return -ENODEV; + mk_arch_send_ipi(target); + return 0; +} + +void mk_reply_scan(struct mk_shared_data *shared) +{ + unsigned int i; + + if (!shared) + return; + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &shared->replies.slots[i]; + u64 token = atomic64_read_acquire(&slot->state_generation); + + if (mk_reply_state(token) == MK_REPLY_READY) { + wake_up_all(&mk_reply_waitq); + return; + } + } +} + int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, unsigned int timeout_ms) { @@ -259,6 +609,7 @@ int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared) /* The old receiver is parked and every pre-existing publisher drained. */ atomic_set(&shared->emergency_shutdown, 0); mk_ipi_ring_reset_contents(ring); + mk_reply_table_reset(&shared->replies); WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); WRITE_ONCE(shared->abi_size, sizeof(*shared)); @@ -618,23 +969,6 @@ static void mk_ipi_drain_ring(void) } } -void mk_poll_ipi_messages(void) -{ - unsigned long flags; - mk_phys_cpu_t target; - - if (!root_instance) - return; - target = mk_cpu_set_first(root_instance->cpus); - if (target == MK_PHYS_CPU_INVALID || - target != arch_cpu_physical_id(smp_processor_id())) - return; - - local_irq_save(flags); - mk_ipi_drain_ring(); - local_irq_restore(flags); -} - /** * multikernel_interrupt_handler - Handle the multikernel IPI * @@ -645,6 +979,7 @@ static void multikernel_interrupt_handler(void) { if (!root_instance || !root_instance->ipi_data) return; + mk_reply_scan(root_instance->ipi_data); /* * Drain here rather than from irq_work. We are already in interrupt diff --git a/kernel/multikernel/messaging.c b/kernel/multikernel/messaging.c index 208fc22367c32d..1963613325ac0c 100644 --- a/kernel/multikernel/messaging.c +++ b/kernel/multikernel/messaging.c @@ -87,7 +87,8 @@ static void mk_message_type_ipi_callback(struct mk_ipi_data *data, void *ctx) msg_type, msg_subtype, payload_len, data->sender_cpu); /* Call the registered handler for this message type */ - type_handler->msg_handler(msg_type, msg_subtype, payload, payload_len, type_handler->context); + type_handler->msg_handler(msg_type, msg_subtype, payload, payload_len, + data->sender_cpu, type_handler->context); } /* diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index af68dce079c7c9..f03723d2ffba46 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include #include @@ -47,6 +49,68 @@ static DEFINE_MUTEX(mk_pci_lease_mutex); static DEFINE_SPINLOCK(mk_pci_active_lock); static LIST_HEAD(mk_pci_active_assignments); static bool mk_pci_notifier_registered; +static bool mk_pci_control_registered; +static mempool_t *mk_pci_control_pool; +static struct workqueue_struct *mk_pci_control_wq; +static DEFINE_SPINLOCK(mk_pci_control_lock); +static DECLARE_WAIT_QUEUE_HEAD(mk_pci_control_waitq); +static unsigned int mk_pci_control_active; +static bool mk_pci_control_shutdown = true; +static atomic64_t mk_pci_control_pool_exhausted = ATOMIC64_INIT(0); + +struct mk_pci_control_work { + struct work_struct work; + struct mk_pci_cfg_request request; + mk_phys_cpu_t sender_cpu; +}; + +static bool mk_pci_control_handler_get(void) +{ + unsigned long flags; + bool acquired = false; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + if (!mk_pci_control_shutdown) { + mk_pci_control_active++; + acquired = true; + } + spin_unlock_irqrestore(&mk_pci_control_lock, flags); + return acquired; +} + +static void mk_pci_control_handler_put(void) +{ + unsigned long flags; + bool drained; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + WARN_ON_ONCE(!mk_pci_control_active); + if (mk_pci_control_active) + mk_pci_control_active--; + drained = !mk_pci_control_active; + spin_unlock_irqrestore(&mk_pci_control_lock, flags); + if (drained) + wake_up_all(&mk_pci_control_waitq); +} + +static void mk_pci_control_shutdown_begin(void) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + mk_pci_control_shutdown = true; + spin_unlock_irqrestore(&mk_pci_control_lock, flags); +} + +static void mk_pci_control_shutdown_end(void) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + mk_pci_control_active = 0; + mk_pci_control_shutdown = false; + spin_unlock_irqrestore(&mk_pci_control_lock, flags); +} static bool mk_pci_device_live(struct pci_dev *pdev) { @@ -121,6 +185,156 @@ mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, return NULL; } +static bool mk_pci_request_route_stale(struct mk_instance *instance, + mk_phys_cpu_t sender_cpu) +{ + if (!instance || instance == root_instance) + return true; + if (READ_ONCE(instance->state) != MK_STATE_ACTIVE) + return true; + + mk_cpu_ownership_assert_held(); + return !mk_cpu_set_contains(instance->cpus, sender_cpu); +} + +static int mk_pci_config_access(struct mk_instance *instance, + const struct mk_pci_cfg_request *request, + u32 *value, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + struct pci_dev *vf; + int ret; + + if (request->len != 1 && request->len != 2 && request->len != 4) + return PCIBIOS_BAD_REGISTER_NUMBER; + if (request->reg > PCI_CFG_SPACE_EXP_SIZE - request->len) + return PCIBIOS_BAD_REGISTER_NUMBER; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf)) { + ret = PCIBIOS_DEVICE_NOT_FOUND; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + + vf = assignment->vf; + if (request->write) { + switch (request->len) { + case 1: + ret = pci_write_config_byte(vf, request->reg, + request->value); + break; + case 2: + ret = pci_write_config_word(vf, request->reg, + request->value); + break; + default: + ret = pci_write_config_dword(vf, request->reg, + request->value); + break; + } + } else { + switch (request->len) { + case 1: { + u8 data; + + ret = pci_read_config_byte(vf, request->reg, &data); + *value = data; + break; + } + case 2: { + u16 data; + + ret = pci_read_config_word(vf, request->reg, &data); + *value = data; + break; + } + default: + ret = pci_read_config_dword(vf, request->reg, value); + break; + } + } +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_cfg_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_reply_handle reply = { + .slot = control_work->request.reply_slot, + .kind = MK_REPLY_PCI_CFG, + .request_id = control_work->request.request_id, + .generation = control_work->request.reply_generation, + }; + struct mk_instance *instance; + u32 value = ~0U; + s32 status; + + instance = mk_instance_find(control_work->request.sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + mk_cpu_ownership_lock(); + status = mk_pci_request_route_stale(instance, control_work->sender_cpu) ? + -ESTALE : 0; + mk_cpu_ownership_unlock(); + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_config_access(instance, &control_work->request, + &value, &reply); + if (mk_reply_publish(instance, &reply, status, value)) + pr_warn_ratelimited("Failed to return PCI config response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + mempool_free(control_work, mk_pci_control_pool); +} + +static void mk_pci_control_msg_handler(u32 msg_type, u32 subtype, + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx) +{ + struct mk_pci_control_work *work; + + if (msg_type != MK_MSG_PCI || subtype != MK_PCI_CFG_REQUEST || + payload_len != sizeof(work->request)) + return; + if (!mk_pci_control_handler_get()) + return; + + /* One reserve object exists for every valid outstanding reply slot. */ + work = mempool_alloc(mk_pci_control_pool, GFP_ATOMIC); + if (!work) { + atomic64_inc(&mk_pci_control_pool_exhausted); + pr_warn_ratelimited("Multikernel PCI control work pool exhausted\n"); + goto out; + } + INIT_WORK(&work->work, mk_pci_cfg_work_fn); + memcpy(&work->request, payload, sizeof(work->request)); + work->sender_cpu = sender_cpu; + if (!queue_work(mk_pci_control_wq, &work->work)) + mempool_free(work, mk_pci_control_pool); +out: + mk_pci_control_handler_put(); +} + static void mk_pci_assignment_failure_work(struct work_struct *work) { struct mk_pci_assignment *assignment = @@ -1153,6 +1367,8 @@ int mk_pci_prepare_instance_start(struct mk_instance *instance) int mk_pci_lease_system_init(void) { + unsigned int pool_size; + size_t work_size = sizeof(struct mk_pci_control_work); int ret; ret = mk_pci_iommu_system_init(); @@ -1164,11 +1380,59 @@ int mk_pci_lease_system_init(void) return ret; } mk_pci_notifier_registered = true; + if (root_instance && root_instance->id == 0) { + pool_size = max_t(unsigned int, num_possible_cpus(), 1) * + MK_REPLY_SLOTS; + mk_pci_control_pool = mempool_create_kmalloc_pool(pool_size, work_size); + if (!mk_pci_control_pool) { + ret = -ENOMEM; + goto unregister_notifier; + } + mk_pci_control_wq = + alloc_workqueue("mk-pci-control", + WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!mk_pci_control_wq) { + ret = -ENOMEM; + goto destroy_pool; + } + mk_pci_control_shutdown_end(); + ret = mk_register_msg_handler(MK_MSG_PCI, + mk_pci_control_msg_handler, NULL); + if (ret) + goto destroy_workqueue; + mk_pci_control_registered = true; + } return 0; + +destroy_workqueue: + mk_pci_control_shutdown_begin(); + destroy_workqueue(mk_pci_control_wq); + mk_pci_control_wq = NULL; +destroy_pool: + mempool_destroy(mk_pci_control_pool); + mk_pci_control_pool = NULL; +unregister_notifier: + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; + mk_pci_iommu_system_cleanup(); + return ret; } void mk_pci_lease_system_cleanup(void) { + if (mk_pci_control_registered) { + mk_pci_control_shutdown_begin(); + mk_unregister_msg_handler(MK_MSG_PCI, + mk_pci_control_msg_handler); + mk_pci_control_registered = false; + wait_event(mk_pci_control_waitq, !READ_ONCE(mk_pci_control_active)); + } + if (mk_pci_control_wq) { + destroy_workqueue(mk_pci_control_wq); + mk_pci_control_wq = NULL; + } + mempool_destroy(mk_pci_control_pool); + mk_pci_control_pool = NULL; if (mk_pci_notifier_registered) { bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); mk_pci_notifier_registered = false; From 13f95617d180b314cdcc6a0b966c86ae2e3f7de1 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 09:39:15 +0300 Subject: [PATCH 14/16] multikernel: move assigned MSI control to process context Keep MSI and MSI-X programming out of irqchip callbacks. Setup, bind, activation, restore, and teardown use the process-context PCI control path; write_msg only updates the cached message. Route controlled VF FLR through a separate process-context direct reply, with an independent spawn epoch and serial generation. Reject raw config-space FLR writes so reset cannot run inside the bounded atomic config path. Reject stale operations without side effects and fail closed on incomplete activation or reset. Pre-mask controlled MSI-X tables before host activation. Bump the exact transport ABI to version 6. Signed-off-by: Nikolay Nikolaev --- Documentation/multikernel/usage.rst | 3 + arch/x86/boot/header.S | 6 +- arch/x86/include/uapi/asm/bootparam.h | 2 + arch/x86/kernel/kexec-bzimage64.c | 2 +- arch/x86/multikernel/pci.c | 462 +++++++++++++++- drivers/pci/msi/api.c | 9 + drivers/pci/msi/irqdomain.c | 27 +- drivers/pci/msi/msi.c | 22 +- drivers/pci/pci.c | 3 + include/linux/multikernel.h | 192 ++++++- include/linux/multikernel_abi.h | 2 +- include/linux/pci.h | 9 + kernel/multikernel/internal.h | 7 +- kernel/multikernel/ipi.c | 30 +- kernel/multikernel/pci.c | 739 +++++++++++++++++++++++++- 15 files changed, 1460 insertions(+), 55 deletions(-) diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index fdb65094eacb81..9f420791d26e79 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -230,3 +230,6 @@ The new kernfs interface has the following restrictions: - **No direct DTB upload to instances**: Instances don't have writable ``device_tree`` files - **Centralized DTB management**: All instances must be created via the root ``device_tree`` file - **Read-only instance files**: All instance attributes are read-only for consistency +- **Host control CPU**: Logical CPU 0 is the PCI forwarding and control CPU and + must remain online while assigned devices are active. Selecting another + control CPU is a future policy extension. diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 18ee3cb89340bc..93279c8daf7452 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -380,12 +380,12 @@ xloadflags: #endif #ifdef CONFIG_MULTIKERNEL -# define XLF10 XLF_MULTIKERNEL_IPI_V4 +# define XLF12 XLF_MULTIKERNEL_IPI_V6 #else -# define XLF10 0 +# define XLF12 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF10 + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF12 cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index 5dc95ebdf1bfc0..27f0d219061771 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -28,6 +28,8 @@ #define XLF_MULTIKERNEL_IPI_V2 0x0100 #define XLF_MULTIKERNEL_IPI_V3 0x0200 #define XLF_MULTIKERNEL_IPI_V4 0x0400 +#define XLF_MULTIKERNEL_IPI_V5 0x0800 +#define XLF_MULTIKERNEL_IPI_V6 0x1000 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 6615d0b04aa05a..35cec98efd479d 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -612,7 +612,7 @@ static void *bzImage64_load(struct kimage *image, char *kernel, header = (struct setup_header *)(kernel + setup_hdr_offset); if (image->type == KEXEC_TYPE_MULTIKERNEL && - !(header->xloadflags & XLF_MULTIKERNEL_IPI_V4)) { + !(header->xloadflags & XLF_MULTIKERNEL_IPI_V6)) { pr_err("Loaded kernel lacks the required shared transport layout\n"); return ERR_PTR(-EPROTONOSUPPORT); } diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c index ab9917a0cdd86d..d154cd5d423236 100644 --- a/arch/x86/multikernel/pci.c +++ b/arch/x86/multikernel/pci.c @@ -2,11 +2,16 @@ /* * x86 PCI support for multikernel spawn kernels. * - * Spawn kernels discover only assigned BDFs. Configuration accesses are - * filtered here and become host-mediated once the RPC transport is installed. + * Spawn kernels discover only assigned BDFs and proxy all configuration + * accesses to the host kernel. */ + #include +#include +#include +#include #include +#include #include #include #include @@ -20,8 +25,428 @@ static atomic64_t mk_pci_request_id = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_count = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_total_ns = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_max_ns = ATOMIC64_INIT(0); +#define MK_PCI_RESET_TIMEOUT_MS 70000 +#ifdef CONFIG_PCI_MSI static struct mk_instance *mk_pci_host_instance; +static bool mk_pci_message_from_host(mk_phys_cpu_t sender_cpu) +{ + return mk_pci_host_instance && + mk_cpu_set_contains(mk_pci_host_instance->cpus, sender_cpu); +} + +static void mk_pci_forward_irq_noop(struct irq_data *data) +{ +} + +static void mk_pci_forward_irq_write_msg(struct irq_data *data, + struct msi_msg *msg) +{ +} + +static struct irq_chip mk_pci_forward_irq_chip = { + .name = "multikernel-pci-forward", + .irq_ack = mk_pci_forward_irq_noop, + /* Host process-context lifecycle owns physical mask state. */ + .irq_mask = mk_pci_forward_irq_noop, + .irq_unmask = mk_pci_forward_irq_noop, + .irq_write_msi_msg = mk_pci_forward_irq_write_msg, +}; + +static void mk_pci_bind_local_irqs(unsigned int irq, unsigned int count) +{ + unsigned int i; + + for (i = 0; i < count; i++) + irq_set_chip_and_handler(irq + i, &mk_pci_forward_irq_chip, + handle_edge_irq); +} + +static bool mk_pci_forward_irq_matches(const struct mk_io_irq_payload *irq, + struct irq_data **irq_data) +{ + struct irq_data *data = irq_get_irq_data(irq->irq_number); + struct msi_desc *desc; + struct pci_dev *dev; + unsigned int offset; + + if (!data) + return false; + desc = irq_data_get_msi_desc(data); + if (!desc || irq->vector < desc->msi_index) + return false; + + dev = msi_desc_to_pci_dev(desc); + offset = irq->vector - desc->msi_index; + if (offset >= desc->nvec_used || desc->irq + offset != irq->irq_number || + pci_domain_nr(dev->bus) != MK_PCI_IRQ_ID_DOMAIN(irq->device_id) || + dev->bus->number != MK_PCI_IRQ_ID_BUS(irq->device_id) || + dev->devfn != MK_PCI_IRQ_ID_DEVFN(irq->device_id)) + return false; + + *irq_data = data; + return true; +} + +static void mk_pci_irq_forward_handler(u32 msg_type, u32 subtype, + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx) +{ + struct mk_io_irq_payload *irq = payload; + struct irq_data *irq_data; + struct pci_dev *dev; + + if (msg_type != MK_MSG_IO || subtype != MK_IO_IRQ_FORWARD || + payload_len != sizeof(*irq) || + !mk_pci_message_from_host(sender_cpu)) + return; + if (!mk_pci_forward_irq_matches(irq, &irq_data)) { + pr_warn_ratelimited("Rejected host-forwarded PCI IRQ %u with unmatched identity %#x vector %u\n", + irq->irq_number, irq->device_id, + irq->vector); + return; + } + dev = msi_desc_to_pci_dev(irq_data_get_msi_desc(irq_data)); + if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE || + irq->lifecycle_generation != + READ_ONCE(dev->multikernel_msi_generation) || + !root_instance->ipi_data || + irq->lifecycle_epoch != + READ_ONCE(root_instance->ipi_data->spawn_epoch)) + return; + + if (irq_data_get_irq_chip(irq_data) != &mk_pci_forward_irq_chip) { + pr_warn_ratelimited("Rejected host-forwarded PCI IRQ %u for %s vector %u before local binding\n", + irq->irq_number, pci_name(dev), + irq->vector); + return; + } + + if (generic_handle_irq_safe(irq->irq_number)) + pr_warn_ratelimited("Failed to dispatch host-forwarded PCI IRQ %u\n", + irq->irq_number); +} + +static int mk_pci_send_irq_request(struct mk_pci_irq_request *request) +{ + struct mk_reply_handle reply; + s32 status; + int ret; + + if (WARN_ON_ONCE(irqs_disabled() || !in_task())) + return -EWOULDBLOCK; + might_sleep(); + if (!request->lifecycle_generation || !root_instance || + !root_instance->ipi_data) + return -EINVAL; + request->lifecycle_epoch = + READ_ONCE(root_instance->ipi_data->spawn_epoch); + if (!request->lifecycle_epoch) + return -EPROTO; + + request->request_id = atomic64_inc_return(&mk_pci_request_id); + request->sender_instance_id = root_instance ? root_instance->id : -1; + ret = mk_reply_reserve(root_instance->ipi_data, MK_REPLY_PCI_IRQ, + request->request_id, &reply); + if (ret) + return ret; + request->reply_slot = reply.slot; + request->reply_generation = reply.generation; + + ret = mk_send_message(0, MK_MSG_PCI, MK_PCI_IRQ_REQUEST, + request, sizeof(*request)); + if (ret) { + mk_reply_release(root_instance->ipi_data, &reply); + return ret; + } + + ret = mk_reply_wait(root_instance->ipi_data, &reply, 1000, + &status, NULL); + return ret ? ret : status; +} + +bool mk_pci_msi_controlled(struct pci_dev *dev) +{ + return mk_pci_controlled(dev); +} + +static int mk_pci_msi_teardown_generation(struct pci_dev *dev, + u32 generation); + +int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) +{ + u32 generation; + u8 state; + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_SETUP, + .nr_vectors = nvec, + .msix = type == PCI_CAP_ID_MSIX, + }; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + state = READ_ONCE(dev->multikernel_msi_state); + if (state != MK_PCI_MSI_IDLE && state != MK_PCI_MSI_FAILED) + return -EBUSY; + generation = READ_ONCE(dev->multikernel_msi_generation) + 1; + if (!generation) + generation = 1; + WRITE_ONCE(dev->multikernel_msi_generation, generation); + request.lifecycle_generation = generation; + ret = mk_pci_send_irq_request(&request); + if (ret) { + if (ret == -EINPROGRESS) { + int cleanup_ret; + + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : + MK_PCI_MSI_IDLE); + } + return ret; + } + WRITE_ONCE(dev->multikernel_msi_nvec, nvec); + WRITE_ONCE(dev->multikernel_msi_msix, type == PCI_CAP_ID_MSIX); + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_PREPARED); + return 0; +} + +static int mk_pci_msi_bind(struct pci_dev *dev, unsigned int index, + unsigned int irq, unsigned int nvec, bool msix, + u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_BIND, + .vector = index, + .nr_vectors = nvec, + .msix = msix, + .local_irq = irq, + .lifecycle_generation = generation, + }; + + /* The local descriptor must be dispatchable before the host unmasks. */ + mk_pci_bind_local_irqs(irq, msix ? 1 : nvec); + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_teardown_generation(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_TEARDOWN, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_commit(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_COMMIT, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_host_activate(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_ACTIVATE, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_restore_begin(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_RESTORE_BEGIN, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_bind_all(struct pci_dev *dev, u32 generation) +{ + struct msi_desc *desc; + unsigned int expected = READ_ONCE(dev->multikernel_msi_nvec); + unsigned int next = 0; + bool msix = READ_ONCE(dev->multikernel_msi_msix); + int ret; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = msix ? 1 : desc->nvec_used; + + if (next >= expected || desc->msi_index != next || !count || + count > expected - next) + return -EINVAL; + ret = mk_pci_msi_bind(dev, desc->msi_index, desc->irq, + desc->nvec_used, msix, generation); + if (ret) + return ret; + next += count; + } + if (next != expected) + return -EINVAL; + return mk_pci_msi_commit(dev, generation); +} + +int mk_pci_msi_activate(struct pci_dev *dev) +{ + u32 generation; + int cleanup_ret; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_PREPARED) + return -EIO; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_bind_all(dev, generation); + if (ret) { + pr_err("Failed to activate host-owned MSI vectors for %s: %d\n", + pci_name(dev), ret); + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; + } + + /* The guest must be able to consume the first edge before host unmask. */ + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_ACTIVE); + ret = mk_pci_msi_host_activate(dev, generation); + if (ret) { + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; + } + return 0; +} + +int mk_pci_msi_restore(struct pci_dev *dev) +{ + u32 generation; + int cleanup_ret; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE) + return -EIO; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_restore_begin(dev, generation); + if (!ret) + ret = mk_pci_msi_bind_all(dev, generation); + if (!ret) + ret = mk_pci_msi_host_activate(dev, generation); + if (ret) { + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_FAILED); + if (cleanup_ret) + pr_err("Failed to quiesce host-owned MSI after restore failure for %s: %d\n", + pci_name(dev), cleanup_ret); + return ret; + } + return 0; +} + +int mk_pci_msi_teardown(struct pci_dev *dev) +{ + u32 generation; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) == MK_PCI_MSI_IDLE) + return 0; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; +} +#endif /* CONFIG_PCI_MSI */ + +bool mk_pci_controlled(struct pci_dev *dev) +{ + return root_instance && root_instance->id != 0 && + mk_pci_get_assigned_identity_bdf(pci_domain_nr(dev->bus), + dev->bus->number, dev->devfn, + NULL, NULL); +} + +int mk_pci_reset_flr(struct pci_dev *dev) +{ + struct mk_pci_reset_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + }; + struct mk_reply_handle reply; + u32 generation; + s32 status; + int ret; + + if (WARN_ON_ONCE(irqs_disabled() || !in_task())) + return -EWOULDBLOCK; + might_sleep(); + if (!root_instance || !root_instance->ipi_data) + return -ENODEV; + request.lifecycle_epoch = + READ_ONCE(root_instance->ipi_data->spawn_epoch); + if (!request.lifecycle_epoch) + return -EPROTO; + + generation = READ_ONCE(dev->multikernel_reset_generation) + 1; + if (!generation) + generation = 1; + WRITE_ONCE(dev->multikernel_reset_generation, generation); + request.reset_generation = generation; + request.request_id = atomic64_inc_return(&mk_pci_request_id); + request.sender_instance_id = root_instance->id; + ret = mk_reply_reserve(root_instance->ipi_data, MK_REPLY_PCI_RESET, + request.request_id, &reply); + if (ret) + return ret; + request.reply_slot = reply.slot; + request.reply_generation = reply.generation; + + ret = mk_send_message(0, MK_MSG_PCI, MK_PCI_RESET_REQUEST, + &request, sizeof(request)); + if (ret) { + mk_reply_release(root_instance->ipi_data, &reply); + return ret; + } + + ret = mk_reply_wait(root_instance->ipi_data, &reply, + MK_PCI_RESET_TIMEOUT_MS, + &status, NULL); + return ret ? ret : status; +} + static void mk_pci_record_latency(u64 start) { u64 elapsed = ktime_get_mono_fast_ns() - start; @@ -66,6 +491,7 @@ static int mk_pci_remote_config(unsigned int domain, unsigned int bus, goto out_error; request.reply_slot = reply.slot; request.reply_generation = reply.generation; + ret = mk_send_message(0, MK_MSG_PCI, MK_PCI_CFG_REQUEST, &request, sizeof(request)); if (ret) { @@ -89,7 +515,6 @@ static int mk_pci_remote_config(unsigned int domain, unsigned int bus, if (ret < 0) { pr_err_ratelimited("Multikernel PCI config request timed out or failed to send: %d\n", ret); - return PCIBIOS_SET_FAILED; } return PCIBIOS_SET_FAILED; } @@ -102,6 +527,7 @@ static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, if (where < PCI_VENDOR_ID || where + size > PCI_COMMAND) return false; + identity = vendor | (u32)device << 16; mask = size == sizeof(identity) ? ~0U : (1U << (size * 8)) - 1; *value = (identity >> (where * 8)) & mask; @@ -121,6 +547,7 @@ static int mk_pci_raw_read(unsigned int domain, unsigned int bus, } if (mk_pci_identity_read(vendor, device, where, size, value)) return PCIBIOS_SUCCESSFUL; + return mk_pci_remote_config(domain, bus, devfn, where, size, false, value); } @@ -131,6 +558,7 @@ static int mk_pci_raw_write(unsigned int domain, unsigned int bus, { if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, NULL, NULL)) return PCIBIOS_DEVICE_NOT_FOUND; + return mk_pci_remote_config(domain, bus, devfn, where, size, true, &value); } @@ -139,19 +567,31 @@ static const struct pci_raw_ops mk_pci_filtered_raw_ops = { .read = mk_pci_raw_read, .write = mk_pci_raw_write, }; + static int __init x86_multikernel_pci_arch_init(void) { if (!root_instance || !root_instance->pci_devices_valid) return 0; +#ifdef CONFIG_PCI_MSI mk_pci_host_instance = mk_instance_find(0); if (!mk_pci_host_instance) { pr_err("Multikernel has no restored host instance for PCI control\n"); return 0; } +#endif +#ifdef CONFIG_PCI_MSI + if (mk_register_msg_handler(MK_MSG_IO, mk_pci_irq_forward_handler, + NULL)) { + pr_err("Multikernel failed to register PCI IRQ forwarding handler\n"); + return 0; + } +#endif + raw_pci_ops = &mk_pci_filtered_raw_ops; raw_pci_ext_ops = &mk_pci_filtered_raw_ops; mk_pci_roots_ready = true; - pr_notice("Multikernel selected filtered raw PCI config access\n"); + pr_notice("Multikernel selected host-mediated PCI config access\n"); + return 0; } @@ -191,11 +631,12 @@ static struct pci_bus * __init mk_pci_get_root(u16 domain, u8 bus_number) kfree(sd); return ERR_PTR(-ENOMEM); } - if (!has_busn_res && - !pci_bus_insert_busn_res(bus, bus_number, bus_number)) { - pci_remove_root_bus(bus); - kfree(sd); - return ERR_PTR(-EBUSY); + if (!has_busn_res) { + if (!pci_bus_insert_busn_res(bus, bus_number, bus_number)) { + pci_remove_root_bus(bus); + kfree(sd); + return ERR_PTR(-EBUSY); + } } pr_notice("Multikernel created synthetic PCI root %04x:%02x\n", domain, bus_number); @@ -236,10 +677,9 @@ static int __init x86_multikernel_pci_init(void) atomic64_read(&mk_pci_cfg_max_ns)); } - /* Suppress legacy bus 0 probing after every assigned function is present. */ + /* Suppress legacy bus 0 probing after every assigned root is present. */ return 0; } - void __init x86_multikernel_pci_platform_init(void) { pci_probe = PCI_PROBE_NOEARLY; diff --git a/drivers/pci/msi/api.c b/drivers/pci/msi/api.c index 818d55fbad0dfa..ca7195c58ef8e0 100644 --- a/drivers/pci/msi/api.c +++ b/drivers/pci/msi/api.c @@ -10,6 +10,7 @@ #include #include +#include #include "msi.h" @@ -123,6 +124,8 @@ bool pci_msix_can_alloc_dyn(struct pci_dev *dev) { if (!dev->msix_cap) return false; + if (mk_pci_msi_controlled(dev)) + return false; return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY); } @@ -388,6 +391,12 @@ EXPORT_SYMBOL(pci_free_irq_vectors); */ void pci_restore_msi_state(struct pci_dev *dev) { + if (mk_pci_msi_controlled(dev)) { + if (mk_pci_msi_restore(dev)) + pr_err("Failed to restore host-owned MSI vectors for %s\n", + pci_name(dev)); + return; + } __pci_restore_msi_state(dev); __pci_restore_msix_state(dev); } diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index 6e65f0f44112e6..e79bf05f07a325 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -4,6 +4,7 @@ */ #include #include +#include #include #include "msi.h" @@ -11,18 +12,38 @@ int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) { struct irq_domain *domain; + bool controlled = mk_pci_msi_controlled(dev); + int ret; + + if (controlled) { + ret = mk_pci_msi_prepare(dev, nvec, type); + if (ret) + return ret; + } domain = dev_get_msi_domain(&dev->dev); if (domain && irq_domain_is_hierarchy(domain)) - return msi_domain_alloc_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN, nvec); - - return pci_msi_legacy_setup_msi_irqs(dev, nvec, type); + ret = msi_domain_alloc_irqs_all_locked(&dev->dev, + MSI_DEFAULT_DOMAIN, nvec); + else + ret = pci_msi_legacy_setup_msi_irqs(dev, nvec, type); + if (!ret && controlled) + ret = mk_pci_msi_activate(dev); + + if (ret && controlled && mk_pci_msi_teardown(dev)) + pr_err("Failed to roll back host-owned MSI setup for %s\n", + pci_name(dev)); + return ret; } void pci_msi_teardown_msi_irqs(struct pci_dev *dev) { struct irq_domain *domain; + if (mk_pci_msi_controlled(dev) && mk_pci_msi_teardown(dev)) + pr_err("Failed to tear down host-owned MSI vectors for %s\n", + pci_name(dev)); + domain = dev_get_msi_domain(&dev->dev); if (domain && irq_domain_is_hierarchy(domain)) { msi_domain_free_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN); diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index e2412175d7af20..6694a75e8ac92b 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "../pci.h" #include "msi.h" @@ -240,6 +241,14 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { struct pci_dev *dev = msi_desc_to_pci_dev(entry); + if (mk_pci_msi_controlled(dev)) { + /* Host programming is committed later from the setup context. */ + entry->msg = *msg; + if (entry->write_msi_msg) + entry->write_msi_msg(entry, entry->write_msi_msg_data); + return; + } + if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) { /* Don't touch the hardware now */ } else if (entry->pci.msi_attrib.is_msix) { @@ -714,6 +723,7 @@ static int msix_setup_interrupts(struct pci_dev *dev, struct msix_entry *entries static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, int nvec, struct irq_affinity *affd) { + bool controlled = mk_pci_msi_controlled(dev); int ret, tsize; u16 control; @@ -736,6 +746,15 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, ret = -ENOMEM; goto out_disable; } + if (controlled && + !pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { + /* + * The controlled setup path activates host-owned vectors from + * msix_setup_interrupts(). Clear stale table entries while the + * function-wide mask is still set, before host activation. + */ + msix_mask_all(dev->msix_base, tsize); + } ret = msix_setup_interrupts(dev, entries, nvec, affd); if (ret) @@ -744,7 +763,8 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, /* Disable INTX */ pci_intx_for_msi(dev, 0); - if (!pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { + if (!controlled && + !pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { /* * Ensure that all table entries are masked to prevent * stale entries from firing in a crash kernel. diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 8479c2e1f74f10..7a9f7a327971ef 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -4380,6 +4381,8 @@ int pcie_reset_flr(struct pci_dev *dev, bool probe) if (probe) return 0; + if (mk_pci_controlled(dev)) + return mk_pci_reset_flr(dev); return pcie_flr(dev); } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 981fd94bbf84b4..e2b096184f3dbd 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -18,8 +18,8 @@ #include #include #include - struct pci_bus; +struct pci_dev; struct mk_instance; /** @@ -77,7 +77,7 @@ bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, #define MK_IPI_SLOT_READY 2 #define MK_IPI_SLOT_CONSUMING 3 #define MK_IPI_SLOT_CANCELLED 4 -#define MK_IPI_ABI_MAGIC 0x4d4b495049303034ULL /* "MKIPI004" */ +#define MK_IPI_ABI_MAGIC 0x4d4b495049303036ULL /* "MKIPI006" */ #define MK_IPI_READY_TIMEOUT_MS 120000 #define MK_REPLY_SLOTS 16 @@ -96,6 +96,7 @@ enum mk_reply_state { enum mk_reply_kind { MK_REPLY_PCI_CFG = 1, MK_REPLY_PCI_IRQ, + MK_REPLY_PCI_RESET, }; struct mk_reply_slot { @@ -157,6 +158,8 @@ struct mk_shared_data { atomic_t ready; /* Appended direct synchronous replies; keep all older offsets stable. */ struct mk_reply_table replies; + /* Changes on every host launch; tags process-context device lifecycles. */ + u64 spawn_epoch; }; static inline void mk_reply_table_reset(struct mk_reply_table *table) @@ -209,6 +212,7 @@ static inline void mk_shared_data_reset(struct mk_shared_data *shared) WRITE_ONCE(shared->ready_instance_id, -1); atomic_set(&shared->ready, 0); mk_reply_table_reset(&shared->replies); + WRITE_ONCE(shared->spawn_epoch, 0); } /* Function pointer type for IPI callbacks */ @@ -306,6 +310,10 @@ int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus); /* Host-mediated PCI control-plane subtypes */ #define MK_PCI_CFG_REQUEST (MK_MSG_PCI + 1) #define MK_PCI_CFG_RESPONSE (MK_MSG_PCI + 2) +#define MK_PCI_IRQ_REQUEST (MK_MSG_PCI + 3) +#define MK_PCI_IRQ_RESPONSE (MK_MSG_PCI + 4) +#define MK_PCI_RESET_REQUEST (MK_MSG_PCI + 5) +#define MK_PCI_RESET_RESPONSE (MK_MSG_PCI + 6) /** * Core message structure @@ -328,8 +336,18 @@ struct mk_io_irq_payload { u32 vector; /* Interrupt vector */ u32 device_id; /* Device identifier (optional) */ u32 flags; /* Control flags (priority, etc.) */ + u32 lifecycle_generation; + u32 reserved; + u64 lifecycle_epoch; }; +/* Pack a PCI segment and BDF into mk_io_irq_payload::device_id. */ +#define MK_PCI_IRQ_ID(domain, bus, devfn) \ + (((u32)(domain) << 16) | ((u32)(bus) << 8) | (u32)(devfn)) +#define MK_PCI_IRQ_ID_DOMAIN(id) ((u16)((id) >> 16)) +#define MK_PCI_IRQ_ID_BUS(id) ((u8)((id) >> 8)) +#define MK_PCI_IRQ_ID_DEVFN(id) ((u8)(id)) + struct mk_pci_cfg_request { u64 request_id; s32 sender_instance_id; @@ -351,6 +369,64 @@ struct mk_pci_cfg_response { u32 value; }; +enum mk_pci_irq_operation { + MK_PCI_IRQ_SETUP = 1, + MK_PCI_IRQ_RESTORE_BEGIN, + MK_PCI_IRQ_BIND, + MK_PCI_IRQ_COMMIT, + MK_PCI_IRQ_ACTIVATE, + MK_PCI_IRQ_TEARDOWN, +}; + +enum mk_pci_msi_lifecycle { + MK_PCI_MSI_IDLE = 0, + MK_PCI_MSI_PREPARED, + MK_PCI_MSI_COMMITTED, + MK_PCI_MSI_ACTIVE, + MK_PCI_MSI_FAILED, +}; + +struct mk_pci_irq_request { + u64 request_id; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u16 operation; + u16 vector; + u16 nr_vectors; + u8 msix; + u8 reserved; + u32 local_irq; + u32 reply_slot; + u32 lifecycle_generation; + u64 reply_generation; + u64 lifecycle_epoch; +}; + +struct mk_pci_irq_response { + u64 request_id; + s32 status; +}; + +struct mk_pci_reset_request { + u64 request_id; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u32 reset_generation; + u32 reply_slot; + u32 reserved; + u64 reply_generation; + u64 lifecycle_epoch; +}; + +struct mk_pci_reset_response { + u64 request_id; + s32 status; +}; + /* IRQ control flags */ #define MK_IRQ_HIGH_PRIORITY 0x01 #define MK_IRQ_LOW_LATENCY 0x02 @@ -941,6 +1017,7 @@ int mk_instance_set_kexec_active(int mk_id); */ struct kimage; struct pci_bus; +struct pci_dev; #ifdef CONFIG_MULTIKERNEL bool multikernel_allow_emergency_restart(void); @@ -962,9 +1039,75 @@ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size); /* Device filtering against the instance metadata */ +#ifdef CONFIG_PCI bool mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, unsigned int devfn, u16 *vendor, u16 *device); +#if defined(CONFIG_X86) +bool mk_pci_controlled(struct pci_dev *dev); +int mk_pci_reset_flr(struct pci_dev *dev); +#else +static inline bool mk_pci_controlled(struct pci_dev *dev) +{ + return false; +} + +static inline int mk_pci_reset_flr(struct pci_dev *dev) +{ + return -EOPNOTSUPP; +} +#endif +#else +static inline bool +mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, u16 *device) +{ + return false; +} + +static inline bool mk_pci_controlled(struct pci_dev *dev) +{ + return false; +} + +static inline int mk_pci_reset_flr(struct pci_dev *dev) +{ + return -EOPNOTSUPP; +} +#endif +#if defined(CONFIG_X86) && defined(CONFIG_PCI) && defined(CONFIG_PCI_MSI) +bool mk_pci_msi_controlled(struct pci_dev *dev); +int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type); +int mk_pci_msi_activate(struct pci_dev *dev); +int mk_pci_msi_restore(struct pci_dev *dev); +int mk_pci_msi_teardown(struct pci_dev *dev); +#else +static inline bool mk_pci_msi_controlled(struct pci_dev *dev) +{ + return false; +} + +static inline int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) +{ + return 0; +} + +static inline int mk_pci_msi_activate(struct pci_dev *dev) +{ + return 0; +} + +static inline int mk_pci_msi_restore(struct pci_dev *dev) +{ + return 0; +} + +static inline int mk_pci_msi_teardown(struct pci_dev *dev) +{ + return 0; +} +#endif + bool mk_platform_device_allowed(const char *name, const char *hid); /* Early CPU registration from the manifest (spawn kernels) */ @@ -976,8 +1119,15 @@ bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); +#ifdef CONFIG_PCI int mk_pci_prepare_instance_start(struct mk_instance *instance); #else +static inline int mk_pci_prepare_instance_start(struct mk_instance *instance) +{ + return 0; +} +#endif +#else static inline bool multikernel_allow_emergency_restart(void) { return true; @@ -1019,6 +1169,42 @@ static inline bool mk_platform_device_allowed(const char *name, const char *hid) { return true; } + +static inline bool mk_pci_controlled(struct pci_dev *dev) +{ + return false; +} + +static inline int mk_pci_reset_flr(struct pci_dev *dev) +{ + return -EOPNOTSUPP; +} + +static inline bool mk_pci_msi_controlled(struct pci_dev *dev) +{ + return false; +} + +static inline int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) +{ + return 0; +} + +static inline int mk_pci_msi_activate(struct pci_dev *dev) +{ + return 0; +} + +static inline int mk_pci_msi_restore(struct pci_dev *dev) +{ + return 0; +} + +static inline int mk_pci_msi_teardown(struct pci_dev *dev) +{ + return 0; +} + static inline void mk_register_cpus_from_manifest(void) { } @@ -1038,7 +1224,7 @@ static inline bool mk_manifest_rejected(void) #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 /* Bumped whenever the shared-memory layout or message semantics change. */ -#define MK_FDT_COMPATIBLE "multikernel-v4" +#define MK_FDT_COMPATIBLE "multikernel-v6" /** * Property Names diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h index 54df3e77834c5d..f99b8bba23ed19 100644 --- a/include/linux/multikernel_abi.h +++ b/include/linux/multikernel_abi.h @@ -3,7 +3,7 @@ #define _LINUX_MULTIKERNEL_ABI_H /* Private host/spawn transport compatibility constants. */ -#define MK_IPI_ABI_VERSION 4 +#define MK_IPI_ABI_VERSION 6 #define MK_BOOT_CONTEXT_MAGIC 0x4d4b435458303032ULL /* "MKCTX002" */ #endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/include/linux/pci.h b/include/linux/pci.h index 1c270f1d512301..591b35215bdd08 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -503,6 +503,9 @@ struct pci_dev { unsigned int non_mappable_bars:1; /* BARs can't be mapped to user-space */ pci_dev_flags_t dev_flags; atomic_t enable_cnt; /* pci_enable_device has been called */ +#ifdef CONFIG_MULTIKERNEL + u32 multikernel_reset_generation; +#endif spinlock_t pcie_cap_lock; /* Protects RMW ops in capability accessors */ u32 saved_config_space[16]; /* Config space saved at suspend time */ @@ -524,6 +527,12 @@ struct pci_dev { #ifdef CONFIG_PCI_MSI void __iomem *msix_base; raw_spinlock_t msi_lock; +#ifdef CONFIG_MULTIKERNEL + u32 multikernel_msi_generation; + u16 multikernel_msi_nvec; + u8 multikernel_msi_state; + u8 multikernel_msi_msix; +#endif #endif struct pci_vpd vpd; #ifdef CONFIG_PCIE_DPC diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 15517f521cf7ca..c803eca9b05f24 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -65,11 +65,8 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn); int mk_pci_release_assignments(struct mk_instance *instance); -static inline unsigned int -mk_pci_sync_instance_irq_route(struct mk_instance *instance) -{ - return 0; -} +void mk_pci_quiesce_instance_irqs(struct mk_instance *instance); +unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance); int mk_instance_force_halt(struct mk_instance *instance); /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 1b302f644f1c98..51e70f848559d5 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -283,8 +283,8 @@ static int mk_reply_take_ready(struct mk_shared_data *shared, return 0; } -static bool mk_reply_cancel(struct mk_shared_data *shared, - struct mk_reply_handle *reply, bool atomic_timeout) +static int mk_reply_cancel(struct mk_shared_data *shared, + struct mk_reply_handle *reply, bool atomic_timeout) { struct mk_reply_table *table = &shared->replies; struct mk_reply_slot *slot = &table->slots[reply->slot]; @@ -304,7 +304,7 @@ static bool mk_reply_cancel(struct mk_shared_data *shared, for (;;) { token = atomic64_read_acquire(&slot->state_generation); if (token == ready) - return true; + return 1; if (token == writing && atomic64_cmpxchg_release(&slot->state_generation, writing, abandoned) == writing) @@ -313,7 +313,9 @@ static bool mk_reply_cancel(struct mk_shared_data *shared, atomic64_cmpxchg_release(&slot->state_generation, executing, committed) == executing) { atomic_inc(&table->indeterminate_timeouts); - goto timed_out; + if (atomic_timeout) + atomic_inc(&table->atomic_timeouts); + return -EINPROGRESS; } if (token != writing && token != executing) break; @@ -321,10 +323,9 @@ static bool mk_reply_cancel(struct mk_shared_data *shared, cancelled: atomic_inc(&table->cancelled_slots); -timed_out: if (atomic_timeout) atomic_inc(&table->atomic_timeouts); - return false; + return 0; } static bool mk_reply_wait_done(struct mk_shared_data *shared, @@ -344,6 +345,7 @@ int mk_reply_wait_atomic(struct mk_shared_data *shared, s32 *status, u32 *value) { u64 deadline; + int cancelled; if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) return -EINVAL; @@ -360,8 +362,11 @@ int mk_reply_wait_atomic(struct mk_shared_data *shared, cpu_relax(); } - if (mk_reply_cancel(shared, reply, true)) + cancelled = mk_reply_cancel(shared, reply, true); + if (cancelled > 0) return mk_reply_take_ready(shared, reply, status, value); + if (cancelled < 0) + return cancelled; return -ETIMEDOUT; } @@ -370,6 +375,7 @@ int mk_reply_wait(struct mk_shared_data *shared, s32 *status, u32 *value) { long waited; + int cancelled; int ret; if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) @@ -378,8 +384,11 @@ int mk_reply_wait(struct mk_shared_data *shared, mk_reply_wait_done(shared, reply), msecs_to_jiffies(timeout_ms)); if (!waited) { - if (mk_reply_cancel(shared, reply, false)) + cancelled = mk_reply_cancel(shared, reply, false); + if (cancelled > 0) return mk_reply_take_ready(shared, reply, status, value); + if (cancelled < 0) + return cancelled; return -ETIMEDOUT; } ret = mk_reply_take_ready(shared, reply, status, value); @@ -569,6 +578,7 @@ int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared) unsigned long flags; unsigned int retry; unsigned int idx; + u64 epoch; u64 token, old; int head; int ret = 0; @@ -610,6 +620,10 @@ int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared) atomic_set(&shared->emergency_shutdown, 0); mk_ipi_ring_reset_contents(ring); mk_reply_table_reset(&shared->replies); + epoch = READ_ONCE(shared->spawn_epoch) + 1; + if (!epoch) + epoch = 1; + WRITE_ONCE(shared->spawn_epoch, epoch); WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); WRITE_ONCE(shared->abi_size, sizeof(*shared)); diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index f03723d2ffba46..fe42193f1ee6fd 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -7,6 +7,8 @@ * the current instance. */ +#include +#include #include #include #include @@ -21,6 +23,19 @@ #include "internal.h" +struct mk_pci_assignment; + +#define MK_PCI_FLR_SETTLE_MS 100 + +struct mk_pci_irq_vector { + struct mk_pci_assignment *assignment; + unsigned int host_irq; + u32 local_irq; + atomic64_t forwarded; + bool requested; + bool disabled; +}; + struct mk_pci_assignment { struct list_head instance_node; struct list_head active_node; @@ -38,6 +53,17 @@ struct mk_pci_assignment { bool iommu_dma_owner; bool iommu_attached; bool iommu_override_active; + struct mk_pci_irq_vector *irq_vectors; + unsigned long *irq_bound_map; + unsigned int nr_irq_vectors; + u64 irq_epoch; + u32 irq_generation; + u32 reset_generation; + u8 irq_state; + bool irq_msix; + bool irq_needs_reprogram; + unsigned long irq_flr_deadline; + bool device_enabled; struct work_struct failure_work; atomic_t failure_pending; bool assigned; @@ -60,10 +86,16 @@ static atomic64_t mk_pci_control_pool_exhausted = ATOMIC64_INIT(0); struct mk_pci_control_work { struct work_struct work; - struct mk_pci_cfg_request request; mk_phys_cpu_t sender_cpu; + union { + struct mk_pci_cfg_request cfg; + struct mk_pci_irq_request irq; + struct mk_pci_reset_request reset; + } request; }; +static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment); + static bool mk_pci_control_handler_get(void) { unsigned long flags; @@ -119,6 +151,15 @@ static bool mk_pci_device_live(struct pci_dev *pdev) pci_device_is_present(pdev); } +static int mk_pci_forwarding_cpu(void) +{ + /* The host IPI manifest publishes logical CPU 0's physical ID. */ + if (!cpu_online(0)) + return -ENODEV; + + return 0; +} + static bool mk_pci_device_matches_bdf(const struct mk_pci_device *device, u16 domain, u8 bus, u8 devfn) { @@ -185,6 +226,408 @@ mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, return NULL; } +static irqreturn_t mk_pci_forward_irq(int irq, void *data) +{ + struct mk_pci_irq_vector *vector = data; + struct mk_pci_assignment *assignment = vector->assignment; + struct mk_io_irq_payload payload = { + .vector = vector - assignment->irq_vectors, + .device_id = MK_PCI_IRQ_ID(pci_domain_nr(assignment->vf->bus), + assignment->vf->bus->number, + assignment->vf->devfn), + .flags = MK_IRQ_LOW_LATENCY | MK_IRQ_EDGE_TRIGGERED, + .lifecycle_generation = READ_ONCE(assignment->irq_generation), + .lifecycle_epoch = READ_ONCE(assignment->irq_epoch), + }; + u32 local_irq = READ_ONCE(vector->local_irq); + + if (READ_ONCE(assignment->instance->state) != MK_STATE_ACTIVE || + READ_ONCE(assignment->irq_state) != MK_PCI_MSI_ACTIVE || + !local_irq) + return IRQ_HANDLED; + + payload.irq_number = local_irq; + if (atomic64_inc_return(&vector->forwarded) == 1) + pr_info("Forwarding host IRQ %u as instance IRQ %u for %s vector %u\n", + irq, local_irq, pci_name(assignment->vf), + payload.vector); + if (mk_send_message_to_instance(assignment->instance, MK_MSG_IO, + MK_IO_IRQ_FORWARD, &payload, + sizeof(payload))) + pr_warn_ratelimited("Failed to forward IRQ %u for %s to instance %d\n", + irq, pci_name(assignment->vf), + assignment->instance->id); + return IRQ_HANDLED; +} + +static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment) +{ + unsigned int disabled = 0; + unsigned int i; + + for (i = 0; i < assignment->nr_irq_vectors; i++) + WRITE_ONCE(assignment->irq_vectors[i].local_irq, 0); + + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->requested && !vector->disabled) { + disable_irq(vector->host_irq); + vector->disabled = true; + disabled++; + } + } + if (assignment->irq_bound_map) + bitmap_zero(assignment->irq_bound_map, + assignment->nr_irq_vectors); + if (assignment->irq_state != MK_PCI_MSI_IDLE && + assignment->irq_state != MK_PCI_MSI_FAILED) + assignment->irq_state = MK_PCI_MSI_PREPARED; + + return disabled; +} + +static void mk_pci_release_irqs(struct mk_pci_assignment *assignment) +{ + unsigned int i; + + mk_pci_quiesce_irqs(assignment); + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->requested) + free_irq(vector->host_irq, vector); + } + if (assignment->nr_irq_vectors) + pci_free_irq_vectors(assignment->vf); + kfree(assignment->irq_vectors); + bitmap_free(assignment->irq_bound_map); + assignment->irq_vectors = NULL; + assignment->irq_bound_map = NULL; + assignment->nr_irq_vectors = 0; + assignment->irq_msix = false; + assignment->irq_needs_reprogram = false; + assignment->irq_flr_deadline = 0; +} + +void mk_pci_quiesce_instance_irqs(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + unsigned int disabled = 0; + + if (!instance || instance == root_instance) + return; + + lockdep_assert_held(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) + disabled += mk_pci_quiesce_irqs(assignment); + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + if (disabled) + pr_info("Quiesced %u host-owned PCI IRQ vectors for instance %d\n", + disabled, instance->id); +} + +unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + unsigned int requested = 0; + unsigned int i; + + lockdep_assert_held(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = + &assignment->irq_vectors[i]; + + if (vector->requested) { + requested++; + synchronize_irq(vector->host_irq); + } + } + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + return requested; +} + +static bool mk_pci_generation_after(u32 generation, u32 last) +{ + return !last || (s32)(generation - last) > 0; +} + +static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + struct mk_pci_irq_vector *vectors; + unsigned long *bound_map; + unsigned int flags; + int forwarding_cpu; + int i; + int nvec; + int ret; + + if (!request->nr_vectors) + return -EINVAL; + if (!request->lifecycle_generation) + return -EINVAL; + if (assignment->irq_epoch && + assignment->irq_epoch != request->lifecycle_epoch) + return -ESTALE; + forwarding_cpu = mk_pci_forwarding_cpu(); + if (forwarding_cpu < 0) { + pr_err("Host control CPU is unavailable for %s MSI forwarding\n", + pci_name(assignment->vf)); + return forwarding_cpu; + } + if (assignment->irq_generation == request->lifecycle_generation) { + if ((assignment->irq_state == MK_PCI_MSI_PREPARED || + assignment->irq_state == MK_PCI_MSI_COMMITTED || + assignment->irq_state == MK_PCI_MSI_ACTIVE) && + assignment->nr_irq_vectors == request->nr_vectors && + assignment->irq_msix == request->msix) + return 0; + return -ESTALE; + } + if (!mk_pci_generation_after(request->lifecycle_generation, + assignment->irq_generation)) + return -ESTALE; + if (assignment->nr_irq_vectors) + mk_pci_release_irqs(assignment); + assignment->irq_epoch = request->lifecycle_epoch; + assignment->irq_generation = request->lifecycle_generation; + assignment->irq_state = MK_PCI_MSI_FAILED; + + vectors = kcalloc(request->nr_vectors, sizeof(*vectors), GFP_KERNEL); + if (!vectors) + return -ENOMEM; + bound_map = bitmap_zalloc(request->nr_vectors, GFP_KERNEL); + if (!bound_map) { + kfree(vectors); + return -ENOMEM; + } + flags = request->msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI; + nvec = pci_alloc_irq_vectors(assignment->vf, request->nr_vectors, + request->nr_vectors, flags); + if (nvec < 0) { + bitmap_free(bound_map); + kfree(vectors); + return nvec; + } + + assignment->irq_vectors = vectors; + assignment->irq_bound_map = bound_map; + assignment->nr_irq_vectors = nvec; + assignment->irq_msix = request->msix; + for (i = 0; i < nvec; i++) { + vectors[i].assignment = assignment; + vectors[i].host_irq = pci_irq_vector(assignment->vf, i); + } + for (i = 0; i < nvec; i++) { + ret = request_irq(vectors[i].host_irq, mk_pci_forward_irq, + IRQF_NO_AUTOEN | IRQF_NOBALANCING, + "multikernel-pci-forward", &vectors[i]); + if (ret) + goto err_release; + vectors[i].requested = true; + vectors[i].disabled = true; + ret = irq_set_affinity(vectors[i].host_irq, + cpumask_of(forwarding_cpu)); + if (ret) + goto err_release; + } + assignment->irq_state = MK_PCI_MSI_PREPARED; + pr_info("Allocated %d host-owned %s vectors for %s (instance %d, CPU %d)\n", + nvec, request->msix ? "MSI-X" : "MSI", + pci_name(assignment->vf), assignment->instance->id, + forwarding_cpu); + return 0; + +err_release: + pr_err("Failed to configure host-owned IRQ for %s vector %d: %d\n", + pci_name(assignment->vf), i, ret); + mk_pci_release_irqs(assignment); + return ret; +} + +static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + struct mk_pci_irq_vector *vector; + unsigned long delay; + unsigned int count; + unsigned int i; + u32 last_irq; + u32 local_irq; + + if (mk_instance_irq_route_load(assignment->instance) == + MK_PHYS_CPU_INVALID) + return -ENODEV; + if (!request->lifecycle_generation || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_epoch != request->lifecycle_epoch || + (assignment->irq_state != MK_PCI_MSI_PREPARED && + assignment->irq_state != MK_PCI_MSI_ACTIVE)) + return -ESTALE; + if (!assignment->irq_vectors || + assignment->irq_msix != request->msix) + return -EINVAL; + count = request->msix ? 1 : request->nr_vectors; + if (!count || request->vector >= assignment->nr_irq_vectors || + count > assignment->nr_irq_vectors - request->vector) + return -EINVAL; + last_irq = request->local_irq + count - 1; + if (!request->local_irq || last_irq < request->local_irq) + return -EINVAL; + + for (i = 0; i < count; i++) { + vector = &assignment->irq_vectors[request->vector + i]; + local_irq = READ_ONCE(vector->local_irq); + if (!vector->requested) + return -EINVAL; + if (local_irq && local_irq != request->local_irq + i) + return -EBUSY; + } + + if (assignment->irq_needs_reprogram) { + if (time_before(jiffies, assignment->irq_flr_deadline)) { + delay = assignment->irq_flr_deadline - jiffies; + msleep(jiffies_to_msecs(delay) + 1); + } + pci_restore_msi_state(assignment->vf); + assignment->irq_needs_reprogram = false; + assignment->irq_flr_deadline = 0; + pr_info("Reprogrammed host-owned MSI state for %s during restore\n", + pci_name(assignment->vf)); + } + + for (i = 0; i < count; i++) { + vector = &assignment->irq_vectors[request->vector + i]; + WRITE_ONCE(vector->local_irq, request->local_irq + i); + } + bitmap_set(assignment->irq_bound_map, request->vector, count); + return 0; +} + +static int +mk_pci_restore_irqs_begin(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_state == MK_PCI_MSI_PREPARED && + assignment->irq_needs_reprogram) + return 0; + if (assignment->irq_state != MK_PCI_MSI_ACTIVE) + return -ESTALE; + + mk_pci_quiesce_irqs(assignment); + assignment->irq_needs_reprogram = true; + return 0; +} + +static int mk_pci_commit_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + unsigned int i; + + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation || + (assignment->irq_state != MK_PCI_MSI_PREPARED && + assignment->irq_state != MK_PCI_MSI_ACTIVE) || + !assignment->irq_bound_map || + !bitmap_full(assignment->irq_bound_map, + assignment->nr_irq_vectors)) + return -ESTALE; + + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!READ_ONCE(vector->local_irq)) + return -EINVAL; + } + if (assignment->irq_state == MK_PCI_MSI_ACTIVE) + return 0; + assignment->irq_state = MK_PCI_MSI_COMMITTED; + return 0; +} + +static int mk_pci_activate_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + unsigned int i; + + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_state == MK_PCI_MSI_ACTIVE) + return 0; + if (assignment->irq_state != MK_PCI_MSI_COMMITTED || + !assignment->nr_irq_vectors || !assignment->irq_bound_map || + !bitmap_full(assignment->irq_bound_map, + assignment->nr_irq_vectors)) + return -ESTALE; + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!vector->requested || !READ_ONCE(vector->local_irq)) + return -EINVAL; + } + assignment->irq_state = MK_PCI_MSI_ACTIVE; + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->disabled) { + enable_irq(vector->host_irq); + vector->disabled = false; + } + } + return 0; +} + +static int mk_pci_teardown_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + if (assignment->irq_epoch && + assignment->irq_epoch != request->lifecycle_epoch) + return -ESTALE; + if (assignment->irq_generation != request->lifecycle_generation && + !mk_pci_generation_after(request->lifecycle_generation, + assignment->irq_generation)) + return -ESTALE; + + if (assignment->nr_irq_vectors) + mk_pci_release_irqs(assignment); + assignment->irq_epoch = request->lifecycle_epoch; + assignment->irq_generation = request->lifecycle_generation; + assignment->irq_state = MK_PCI_MSI_IDLE; + return 0; +} + +static bool mk_pci_is_flr_write(struct pci_dev *vf, + const struct mk_pci_cfg_request *request) +{ + u16 flr_byte; + unsigned int bit; + + if (!request->write || !pci_is_pcie(vf)) + return false; + flr_byte = pci_pcie_cap(vf) + PCI_EXP_DEVCTL + 1; + if (request->reg > flr_byte || + request->reg + request->len <= flr_byte) + return false; + bit = (flr_byte - request->reg) * 8 + 7; + return request->value & BIT(bit); +} + static bool mk_pci_request_route_stale(struct mk_instance *instance, mk_phys_cpu_t sender_cpu) { @@ -204,6 +647,7 @@ static int mk_pci_config_access(struct mk_instance *instance, { struct mk_pci_assignment *assignment; struct pci_dev *vf; + bool flr; int ret; if (request->len != 1 && request->len != 2 && request->len != 4) @@ -224,8 +668,14 @@ static int mk_pci_config_access(struct mk_instance *instance, ret = mk_reply_begin_execute(instance, reply); if (ret) goto out; + /* Committed: complete and let publish reclaim an indeterminate timeout. */ vf = assignment->vf; + flr = mk_pci_is_flr_write(vf, request); + if (flr) { + ret = PCIBIOS_SET_FAILED; + goto out; + } if (request->write) { switch (request->len) { case 1: @@ -273,17 +723,18 @@ static void mk_pci_cfg_work_fn(struct work_struct *work) { struct mk_pci_control_work *control_work = container_of(work, struct mk_pci_control_work, work); + struct mk_pci_cfg_request *request = &control_work->request.cfg; struct mk_reply_handle reply = { - .slot = control_work->request.reply_slot, + .slot = request->reply_slot, .kind = MK_REPLY_PCI_CFG, - .request_id = control_work->request.request_id, - .generation = control_work->request.reply_generation, + .request_id = request->request_id, + .generation = request->reply_generation, }; struct mk_instance *instance; u32 value = ~0U; s32 status; - instance = mk_instance_find(control_work->request.sender_instance_id); + instance = mk_instance_find(request->sender_instance_id); if (!instance) goto out; down_read(&instance->control_route_sem); @@ -291,12 +742,12 @@ static void mk_pci_cfg_work_fn(struct work_struct *work) status = mk_pci_request_route_stale(instance, control_work->sender_cpu) ? -ESTALE : 0; mk_cpu_ownership_unlock(); + /* An untrusted payload ID must never select another instance's slot. */ if (status) goto unlock_route; if (mk_reply_claim(instance, &reply)) goto unlock_route; - status = mk_pci_config_access(instance, &control_work->request, - &value, &reply); + status = mk_pci_config_access(instance, request, &value, &reply); if (mk_reply_publish(instance, &reply, status, value)) pr_warn_ratelimited("Failed to return PCI config response to instance %d\n", instance->id); @@ -307,30 +758,253 @@ static void mk_pci_cfg_work_fn(struct work_struct *work) mempool_free(control_work, mk_pci_control_pool); } +static int mk_pci_irq_access(struct mk_instance *instance, + const struct mk_pci_irq_request *request, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + int ret; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + if (request->operation != MK_PCI_IRQ_TEARDOWN && + READ_ONCE(instance->state) != MK_STATE_ACTIVE) { + ret = -ESHUTDOWN; + goto out; + } + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf)) { + ret = -ENODEV; + goto out; + } + if (!instance->ipi_data || !request->lifecycle_epoch || + request->lifecycle_epoch != + READ_ONCE(instance->ipi_data->spawn_epoch)) { + ret = -ESTALE; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + /* + * This CAS is the sole cancellation boundary. IRQ programming is + * non-cancellable once EXECUTING; a timed-out waiter moves the exact + * generation to COMMITTED for mk_reply_publish() to reclaim afterwards. + */ + + switch (request->operation) { + case MK_PCI_IRQ_SETUP: + ret = mk_pci_setup_irqs(assignment, request); + break; + case MK_PCI_IRQ_RESTORE_BEGIN: + ret = mk_pci_restore_irqs_begin(assignment, request); + break; + case MK_PCI_IRQ_BIND: + ret = mk_pci_bind_irq(assignment, request); + break; + case MK_PCI_IRQ_COMMIT: + ret = mk_pci_commit_irqs(assignment, request); + break; + case MK_PCI_IRQ_ACTIVATE: + ret = mk_pci_activate_irqs(assignment, request); + break; + case MK_PCI_IRQ_TEARDOWN: + ret = mk_pci_teardown_irqs(assignment, request); + break; + default: + ret = -EINVAL; + break; + } +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_irq_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_pci_irq_request *request = &control_work->request.irq; + struct mk_reply_handle reply = { + .slot = request->reply_slot, + .kind = MK_REPLY_PCI_IRQ, + .request_id = request->request_id, + .generation = request->reply_generation, + }; + struct mk_instance *instance; + s32 status; + + instance = mk_instance_find(request->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + mk_cpu_ownership_lock(); + status = mk_pci_request_route_stale(instance, control_work->sender_cpu) ? + -ESTALE : 0; + mk_cpu_ownership_unlock(); + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_irq_access(instance, request, &reply); + if (mk_reply_publish(instance, &reply, status, 0)) + pr_warn_ratelimited("Failed to return PCI IRQ response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + mempool_free(control_work, mk_pci_control_pool); +} + +static int mk_pci_reset_access(struct mk_instance *instance, + const struct mk_pci_reset_request *request, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + int ret; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + if (READ_ONCE(instance->state) != MK_STATE_ACTIVE) { + ret = -ESHUTDOWN; + goto out; + } + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf) || !assignment->vf->is_virtfn) { + ret = -ENODEV; + goto out; + } + if (!instance->ipi_data || !request->lifecycle_epoch || + request->lifecycle_epoch != + READ_ONCE(instance->ipi_data->spawn_epoch) || + !request->reset_generation || + !mk_pci_generation_after(request->reset_generation, + assignment->reset_generation)) { + ret = -ESTALE; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + + /* Tombstone this serial before side effects so delayed replays reject. */ + assignment->reset_generation = request->reset_generation; + mk_pci_quiesce_irqs(assignment); + ret = pcie_reset_flr(assignment->vf, false); + if (ret) { + assignment->irq_state = MK_PCI_MSI_FAILED; + mk_pci_schedule_failure(assignment); + goto out; + } + if (assignment->nr_irq_vectors) { + assignment->irq_needs_reprogram = true; + assignment->irq_flr_deadline = + jiffies + msecs_to_jiffies(MK_PCI_FLR_SETTLE_MS); + } + pr_info("Invalidated host-owned MSI bindings for spawn-triggered FLR of %s\n", + pci_name(assignment->vf)); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_reset_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_pci_reset_request *request = &control_work->request.reset; + struct mk_reply_handle reply = { + .slot = request->reply_slot, + .kind = MK_REPLY_PCI_RESET, + .request_id = request->request_id, + .generation = request->reply_generation, + }; + struct mk_instance *instance; + s32 status; + + instance = mk_instance_find(request->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + mk_cpu_ownership_lock(); + status = mk_pci_request_route_stale(instance, control_work->sender_cpu) ? + -ESTALE : 0; + mk_cpu_ownership_unlock(); + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_reset_access(instance, request, &reply); + if (mk_reply_publish(instance, &reply, status, 0)) + pr_warn_ratelimited("Failed to return PCI reset response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + mempool_free(control_work, mk_pci_control_pool); +} + static void mk_pci_control_msg_handler(u32 msg_type, u32 subtype, void *payload, u32 payload_len, mk_phys_cpu_t sender_cpu, void *ctx) { - struct mk_pci_control_work *work; + struct mk_pci_control_work *control_work; + size_t request_size; + work_func_t work_fn; - if (msg_type != MK_MSG_PCI || subtype != MK_PCI_CFG_REQUEST || - payload_len != sizeof(work->request)) + if (msg_type != MK_MSG_PCI) return; if (!mk_pci_control_handler_get()) return; - /* One reserve object exists for every valid outstanding reply slot. */ - work = mempool_alloc(mk_pci_control_pool, GFP_ATOMIC); - if (!work) { + switch (subtype) { + case MK_PCI_CFG_REQUEST: + request_size = sizeof(control_work->request.cfg); + work_fn = mk_pci_cfg_work_fn; + break; + case MK_PCI_IRQ_REQUEST: + request_size = sizeof(control_work->request.irq); + work_fn = mk_pci_irq_work_fn; + break; + case MK_PCI_RESET_REQUEST: + request_size = sizeof(control_work->request.reset); + work_fn = mk_pci_reset_work_fn; + break; + default: + goto out; + } + if (payload_len != request_size) + goto out; + /* + * Valid senders reserve one of MK_REPLY_SLOTS before publishing. Every + * active instance owns at least one disjoint possible CPU, so the pool + * covers the maximum number of valid requests across all instances. This + * keeps the hardirq receive path allocation-safe without changing reply + * or route validation. Duplicate traffic is outside the cooperative ABI. + */ + control_work = mempool_alloc(mk_pci_control_pool, GFP_ATOMIC); + if (!control_work) { atomic64_inc(&mk_pci_control_pool_exhausted); pr_warn_ratelimited("Multikernel PCI control work pool exhausted\n"); goto out; } - INIT_WORK(&work->work, mk_pci_cfg_work_fn); - memcpy(&work->request, payload, sizeof(work->request)); - work->sender_cpu = sender_cpu; - if (!queue_work(mk_pci_control_wq, &work->work)) - mempool_free(work, mk_pci_control_pool); + INIT_WORK(&control_work->work, work_fn); + memcpy(&control_work->request, payload, request_size); + control_work->sender_cpu = sender_cpu; + if (!queue_work(mk_pci_control_wq, &control_work->work)) + mempool_free(control_work, mk_pci_control_pool); out: mk_pci_control_handler_put(); } @@ -574,6 +1248,7 @@ mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) bool transactions_drained; int ret; + mk_pci_release_irqs(assignment); if (!mk_pci_device_live(vf)) return 0; @@ -610,6 +1285,11 @@ mk_pci_reset_assignment_for_start(struct mk_pci_assignment *assignment) struct pci_dev *vf = assignment->vf; int ret; + mk_pci_release_irqs(assignment); + assignment->irq_epoch = 0; + assignment->irq_generation = 0; + assignment->reset_generation = 0; + assignment->irq_state = MK_PCI_MSI_IDLE; if (!assignment->assigned || !assignment->iommu_attached) return -EINVAL; if (!mk_pci_device_live(vf)) @@ -780,9 +1460,22 @@ static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, ret = iommu_attach_group(assignment->iommu_domain, assignment->iommu_group); - if (ret) + if (ret) { + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; return ret; + } assignment->iommu_attached = true; + ret = pci_enable_device(pdev); + if (ret) { + iommu_detach_group(assignment->iommu_domain, + assignment->iommu_group); + assignment->iommu_attached = false; + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + return ret; + } + assignment->device_enabled = true; pr_info("Attached %s to host-owned IOMMU domain for instance %d\n", pci_name(assignment->vf), assignment->instance->id); return 0; @@ -797,6 +1490,10 @@ static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) return; if (READ_ONCE(assignment->expected_unbind)) { + if (assignment->device_enabled) { + pci_disable_device(pdev); + assignment->device_enabled = false; + } pci_set_drvdata(pdev, NULL); return; } @@ -809,6 +1506,10 @@ static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) } else { mk_pci_iommu_deactivate_assignment(assignment); } + if (assignment->device_enabled) { + pci_disable_device(pdev); + assignment->device_enabled = false; + } pci_set_drvdata(pdev, NULL); } From 51bf535374b04bb9c0820c90c5ebf595446ff251 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 05:10:33 +0300 Subject: [PATCH 15/16] multikernel: add lossless pending IRQ mailboxes A bounded shared ring cannot guarantee delivery when a host interrupt arrives in hardirq context. Record assigned interrupts in preallocated per-instance pending slots and use the IPI only as a doorbell. Protect each slot with spawn epoch, lifecycle generation, and an atomic pending, masked, and consuming token. Coalesce while masked and retry lost doorbells until the guest drains the slot. Bump the exact transport ABI to version 7. Signed-off-by: Nikolay Nikolaev --- Documentation/multikernel/usage.rst | 24 ++ arch/x86/boot/header.S | 6 +- arch/x86/include/uapi/asm/bootparam.h | 1 + arch/x86/kernel/kexec-bzimage64.c | 2 +- arch/x86/multikernel/pci.c | 389 ++++++++++++++++--- include/linux/multikernel.h | 98 ++++- include/linux/multikernel_abi.h | 2 +- kernel/multikernel/core.c | 172 ++++++--- kernel/multikernel/internal.h | 87 ++++- kernel/multikernel/ipi.c | 2 + kernel/multikernel/pci.c | 518 ++++++++++++++++++++++---- 11 files changed, 1096 insertions(+), 205 deletions(-) diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index 9f420791d26e79..e74f19f4346631 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -93,6 +93,30 @@ Phase 1: Instance Creation (Automatic from DTB) cat /sys/fs/multikernel/instances/web-server/status # Output: ready + cat /sys/fs/multikernel/instances/web-server/stats + + ``stats`` is a read-only, append-only key/value interface. Version 1 + starts with ``stats_version 1`` and reports the shared ordered-IPI, direct + reply, and pending-IRQ transport counters for the instance. New keys may + be appended; readers must ignore keys they do not understand. + + The output is an observational snapshot rather than an atomic transaction. + Shared transport counters are unsigned 32-bit values and may wrap. They + are reinitialized for a new ``spawn_epoch``, are not writable or resettable + through this interface, and must only be compared as modulo-32-bit deltas + between samples carrying the same nonzero epoch. + +Shared Transport Compatibility +============================== + +The private kernel-to-kernel shared transport requires an exact match between +the host and spawn. The loader compares the spawn image note before launch; +the spawn manifest and transport initialization acknowledgment confirm the +same layout before shared memory is used. These checks establish transport +compatibility and do not authenticate the kernel artifact. A mismatched +bzImage or vmlinux is rejected; there is no translation or backward- +compatibility promise between transport versions. + # View instance device tree cat /sys/fs/multikernel/instances/web-server/device_tree_source # Output: DTS format showing the instance configuration diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 93279c8daf7452..8553df38269bc4 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -380,12 +380,12 @@ xloadflags: #endif #ifdef CONFIG_MULTIKERNEL -# define XLF12 XLF_MULTIKERNEL_IPI_V6 +# define XLF13 XLF_MULTIKERNEL_IPI_V7 #else -# define XLF12 0 +# define XLF13 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF12 + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF13 cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index 27f0d219061771..062a70099e77d3 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -30,6 +30,7 @@ #define XLF_MULTIKERNEL_IPI_V4 0x0400 #define XLF_MULTIKERNEL_IPI_V5 0x0800 #define XLF_MULTIKERNEL_IPI_V6 0x1000 +#define XLF_MULTIKERNEL_IPI_V7 0x2000 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 35cec98efd479d..4f840e8ce74221 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -612,7 +612,7 @@ static void *bzImage64_load(struct kimage *image, char *kernel, header = (struct setup_header *)(kernel + setup_hdr_offset); if (image->type == KEXEC_TYPE_MULTIKERNEL && - !(header->xloadflags & XLF_MULTIKERNEL_IPI_V6)) { + !(header->xloadflags & XLF_MULTIKERNEL_IPI_V7)) { pr_err("Loaded kernel lacks the required shared transport layout\n"); return ERR_PTR(-EPROTONOSUPPORT); } diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c index d154cd5d423236..931d8d7bf44477 100644 --- a/arch/x86/multikernel/pci.c +++ b/arch/x86/multikernel/pci.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -27,16 +28,22 @@ static atomic64_t mk_pci_cfg_total_ns = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_max_ns = ATOMIC64_INIT(0); #define MK_PCI_RESET_TIMEOUT_MS 70000 #ifdef CONFIG_PCI_MSI -static struct mk_instance *mk_pci_host_instance; +static void mk_pci_forward_irq_set_mask(struct irq_data *data, bool masked); +static void mk_pci_irq_mailbox_requeue(struct mk_irq_mailbox *mailbox, + unsigned int slot); -static bool mk_pci_message_from_host(mk_phys_cpu_t sender_cpu) +static void mk_pci_forward_irq_noop(struct irq_data *data) { - return mk_pci_host_instance && - mk_cpu_set_contains(mk_pci_host_instance->cpus, sender_cpu); } -static void mk_pci_forward_irq_noop(struct irq_data *data) +static void mk_pci_forward_irq_mask(struct irq_data *data) +{ + mk_pci_forward_irq_set_mask(data, true); +} + +static void mk_pci_forward_irq_unmask(struct irq_data *data) { + mk_pci_forward_irq_set_mask(data, false); } static void mk_pci_forward_irq_write_msg(struct irq_data *data, @@ -48,8 +55,8 @@ static struct irq_chip mk_pci_forward_irq_chip = { .name = "multikernel-pci-forward", .irq_ack = mk_pci_forward_irq_noop, /* Host process-context lifecycle owns physical mask state. */ - .irq_mask = mk_pci_forward_irq_noop, - .irq_unmask = mk_pci_forward_irq_noop, + .irq_mask = mk_pci_forward_irq_mask, + .irq_unmask = mk_pci_forward_irq_unmask, .irq_write_msi_msg = mk_pci_forward_irq_write_msg, }; @@ -62,10 +69,11 @@ static void mk_pci_bind_local_irqs(unsigned int irq, unsigned int count) handle_edge_irq); } -static bool mk_pci_forward_irq_matches(const struct mk_io_irq_payload *irq, +static bool mk_pci_forward_irq_matches(u32 irq_number, u32 vector, + u32 device_id, struct irq_data **irq_data) { - struct irq_data *data = irq_get_irq_data(irq->irq_number); + struct irq_data *data = irq_get_irq_data(irq_number); struct msi_desc *desc; struct pci_dev *dev; unsigned int offset; @@ -73,58 +81,251 @@ static bool mk_pci_forward_irq_matches(const struct mk_io_irq_payload *irq, if (!data) return false; desc = irq_data_get_msi_desc(data); - if (!desc || irq->vector < desc->msi_index) + if (!desc || vector < desc->msi_index) return false; dev = msi_desc_to_pci_dev(desc); - offset = irq->vector - desc->msi_index; - if (offset >= desc->nvec_used || desc->irq + offset != irq->irq_number || - pci_domain_nr(dev->bus) != MK_PCI_IRQ_ID_DOMAIN(irq->device_id) || - dev->bus->number != MK_PCI_IRQ_ID_BUS(irq->device_id) || - dev->devfn != MK_PCI_IRQ_ID_DEVFN(irq->device_id)) + offset = vector - desc->msi_index; + if (offset >= desc->nvec_used || desc->irq + offset != irq_number || + pci_domain_nr(dev->bus) != MK_PCI_IRQ_ID_DOMAIN(device_id) || + dev->bus->number != MK_PCI_IRQ_ID_BUS(device_id) || + dev->devfn != MK_PCI_IRQ_ID_DEVFN(device_id)) return false; *irq_data = data; return true; } -static void mk_pci_irq_forward_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, - mk_phys_cpu_t sender_cpu, void *ctx) +static void mk_pci_forward_irq_set_mask(struct irq_data *data, bool masked) { - struct mk_io_irq_payload *irq = payload; - struct irq_data *irq_data; + struct msi_desc *desc = irq_data_get_msi_desc(data); + struct mk_irq_mailbox *mailbox; struct pci_dev *dev; + u32 device_id; + u32 generation; + u32 vector; + unsigned int slot; + u64 epoch; - if (msg_type != MK_MSG_IO || subtype != MK_IO_IRQ_FORWARD || - payload_len != sizeof(*irq) || - !mk_pci_message_from_host(sender_cpu)) + if (!desc || !root_instance || !root_instance->ipi_data) return; - if (!mk_pci_forward_irq_matches(irq, &irq_data)) { - pr_warn_ratelimited("Rejected host-forwarded PCI IRQ %u with unmatched identity %#x vector %u\n", - irq->irq_number, irq->device_id, - irq->vector); + dev = msi_desc_to_pci_dev(desc); + generation = READ_ONCE(dev->multikernel_msi_generation); + epoch = READ_ONCE(root_instance->ipi_data->spawn_epoch); + device_id = MK_PCI_IRQ_ID(pci_domain_nr(dev->bus), dev->bus->number, + dev->devfn); + vector = desc->msi_index + data->irq - desc->irq; + mailbox = &root_instance->ipi_data->irq_mailbox; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry = &mailbox->entries[slot]; + u64 old, new; + u32 slot_generation; + + old = atomic64_read_acquire(&entry->pending_generation); + if (!mk_irq_mailbox_generation(old) || + READ_ONCE(entry->lifecycle_epoch) != epoch || + READ_ONCE(entry->lifecycle_generation) != generation || + READ_ONCE(entry->device_id) != device_id || + READ_ONCE(entry->local_irq) != data->irq || + READ_ONCE(entry->vector) != vector) + continue; + slot_generation = mk_irq_mailbox_generation(old); + for (;;) { + new = masked ? old | MK_IRQ_MAILBOX_MASKED : + old & ~MK_IRQ_MAILBOX_MASKED; + if (new == old || + atomic64_cmpxchg(&entry->pending_generation, + old, new) == old) + break; + old = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(old) != slot_generation) + return; + } + if (!masked && mk_irq_mailbox_pending(new)) { + mk_phys_cpu_t target; + + mk_pci_irq_mailbox_requeue(mailbox, slot); + target = arch_cpu_physical_id(smp_processor_id()); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } return; } +} + +static void mk_pci_irq_mailbox_requeue(struct mk_irq_mailbox *mailbox, + unsigned int slot) +{ + atomic64_or(BIT_ULL(slot & 63), + &mailbox->pending_bitmap[slot / 64]); +} + +static void mk_pci_irq_mailbox_drain_slot(struct mk_shared_data *shared, + unsigned int slot) +{ + struct mk_irq_mailbox *mailbox = &shared->irq_mailbox; + struct mk_irq_mailbox_entry *entry = &mailbox->entries[slot]; + struct irq_data *irq_data; + struct pci_dev *dev; + u64 lifecycle_epoch; + u64 token, base, claim; + u32 lifecycle_generation; + u32 pending; + u32 device_id; + u32 local_irq; + u16 vector; + + token = atomic64_read_acquire(&entry->pending_generation); + pending = mk_irq_mailbox_pending(token); + if (!mk_irq_mailbox_generation(token) || !pending) + return; + if (mk_irq_mailbox_consuming(token)) { + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + lifecycle_epoch = READ_ONCE(entry->lifecycle_epoch); + lifecycle_generation = READ_ONCE(entry->lifecycle_generation); + device_id = READ_ONCE(entry->device_id); + local_irq = READ_ONCE(entry->local_irq); + vector = READ_ONCE(entry->vector); + base = token & ~MK_IRQ_MAILBOX_PENDING_MASK; + + if (!local_irq || !root_instance || !root_instance->ipi_data || + lifecycle_epoch != READ_ONCE(shared->spawn_epoch)) + goto stale; + if (mk_irq_mailbox_masked(token)) { + atomic_inc(&mailbox->masked_deferred); + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + claim = base | MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_acquire(&entry->pending_generation, + token, claim) != token) { + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim)) + goto claimed_stale; + if (mk_irq_mailbox_masked(token)) { + atomic_inc(&mailbox->masked_deferred); + goto claimed_defer; + } + if (!mk_pci_forward_irq_matches(local_irq, vector, device_id, + &irq_data)) + goto claimed_stale; dev = msi_desc_to_pci_dev(irq_data_get_msi_desc(irq_data)); - if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE || - irq->lifecycle_generation != + if (lifecycle_epoch != READ_ONCE(entry->lifecycle_epoch) || + lifecycle_generation != + READ_ONCE(entry->lifecycle_generation) || + device_id != READ_ONCE(entry->device_id) || + local_irq != READ_ONCE(entry->local_irq) || + vector != READ_ONCE(entry->vector) || + READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE || + lifecycle_generation != READ_ONCE(dev->multikernel_msi_generation) || - !root_instance->ipi_data || - irq->lifecycle_epoch != - READ_ONCE(root_instance->ipi_data->spawn_epoch)) + irq_data_get_irq_chip(irq_data) != &mk_pci_forward_irq_chip) + goto claimed_stale; + if (pending > 1) + atomic_add(pending - 1, &mailbox->coalesced); + if (generic_handle_irq_safe(local_irq)) + atomic_inc(&mailbox->dispatch_failed); + + for (;;) { + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new = token & ~MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + if (mk_irq_mailbox_pending(new)) + mk_pci_irq_mailbox_requeue(mailbox, slot); return; + } + +claimed_defer: + for (;;) { + u32 new_pending; + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new_pending = mk_irq_mailbox_pending(token); + if (new_pending < MK_IRQ_MAILBOX_PENDING_MASK) + new_pending++; + new = mk_irq_mailbox_token(mk_irq_mailbox_generation(token), + new_pending) | + (token & MK_IRQ_MAILBOX_MASKED); + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + mk_pci_irq_mailbox_requeue(mailbox, slot); + if (!mk_irq_mailbox_masked(new)) { + mk_phys_cpu_t target = + arch_cpu_physical_id(smp_processor_id()); + + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } + return; + } - if (irq_data_get_irq_chip(irq_data) != &mk_pci_forward_irq_chip) { - pr_warn_ratelimited("Rejected host-forwarded PCI IRQ %u for %s vector %u before local binding\n", - irq->irq_number, pci_name(dev), - irq->vector); +claimed_stale: + atomic_inc(&mailbox->stale); + for (;;) { + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new = token & ~MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + if (mk_irq_mailbox_pending(new)) + mk_pci_irq_mailbox_requeue(mailbox, slot); return; } + return; - if (generic_handle_irq_safe(irq->irq_number)) - pr_warn_ratelimited("Failed to dispatch host-forwarded PCI IRQ %u\n", - irq->irq_number); +stale: + if (atomic64_cmpxchg_acquire(&entry->pending_generation, + token, base) != token) + mk_pci_irq_mailbox_requeue(mailbox, slot); + atomic_inc(&mailbox->stale); +} + +void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) +{ + unsigned int word; + + if (!shared) + return; + for (word = 0; word < MK_IRQ_MAILBOX_WORDS; word++) { + atomic64_t *pending_bitmap; + unsigned long bits; + + pending_bitmap = &shared->irq_mailbox.pending_bitmap[word]; + bits = atomic64_xchg_acquire(pending_bitmap, 0); + + while (bits) { + unsigned int bit = __ffs(bits); + + bits &= bits - 1; + mk_pci_irq_mailbox_drain_slot(shared, word * 64 + bit); + } + } } static int mk_pci_send_irq_request(struct mk_pci_irq_request *request) @@ -230,10 +431,26 @@ static int mk_pci_msi_bind(struct pci_dev *dev, unsigned int index, .local_irq = irq, .lifecycle_generation = generation, }; + unsigned int count = msix ? 1 : nvec; + unsigned int i; + int ret; /* The local descriptor must be dispatchable before the host unmasks. */ - mk_pci_bind_local_irqs(irq, msix ? 1 : nvec); - return mk_pci_send_irq_request(&request); + mk_pci_bind_local_irqs(irq, count); + ret = mk_pci_send_irq_request(&request); + if (ret) + return ret; + /* Publish the irqdesc's initial logical mask state into the token. */ + for (i = 0; i < count; i++) { + struct irq_data *data = irq_get_irq_data(irq + i); + + if (!data) + return -EINVAL; + mk_pci_forward_irq_set_mask(data, + irqd_irq_disabled(data) || + irqd_irq_masked(data)); + } + return 0; } static int mk_pci_msi_teardown_generation(struct pci_dev *dev, u32 generation) @@ -313,6 +530,69 @@ static int mk_pci_msi_bind_all(struct pci_dev *dev, u32 generation) return mk_pci_msi_commit(dev, generation); } +static int mk_pci_msi_mask_mailbox(struct pci_dev *dev) +{ + struct msi_desc *desc; + unsigned long deadline; + u64 epoch; + u32 device_id; + u32 generation; + unsigned int slot; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = desc->pci.msi_attrib.is_msix ? + 1 : desc->nvec_used; + unsigned int i; + + for (i = 0; i < count; i++) { + struct irq_data *data = irq_get_irq_data(desc->irq + i); + + if (data) + mk_pci_forward_irq_set_mask(data, true); + } + } + + epoch = READ_ONCE(root_instance->ipi_data->spawn_epoch); + generation = READ_ONCE(dev->multikernel_msi_generation); + device_id = MK_PCI_IRQ_ID(pci_domain_nr(dev->bus), dev->bus->number, + dev->devfn); + deadline = jiffies + msecs_to_jiffies(1000); + for (;;) { + bool consuming = false; + + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry = + &root_instance->ipi_data->irq_mailbox.entries[slot]; + u64 token; + + token = atomic64_read_acquire(&entry->pending_generation); + + if (mk_irq_mailbox_generation(token) && + mk_irq_mailbox_consuming(token) && + READ_ONCE(entry->lifecycle_epoch) == epoch && + READ_ONCE(entry->lifecycle_generation) == generation && + READ_ONCE(entry->device_id) == device_id) { + consuming = true; + break; + } + } + if (!consuming) + break; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(50, 100); + } + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = desc->pci.msi_attrib.is_msix ? + 1 : desc->nvec_used; + unsigned int i; + + for (i = 0; i < count; i++) + synchronize_irq(desc->irq + i); + } + return 0; +} + int mk_pci_msi_activate(struct pci_dev *dev) { u32 generation; @@ -338,7 +618,9 @@ int mk_pci_msi_activate(struct pci_dev *dev) WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_ACTIVE); ret = mk_pci_msi_host_activate(dev, generation); if (ret) { - cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + cleanup_ret = mk_pci_msi_mask_mailbox(dev); + if (!cleanup_ret) + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); WRITE_ONCE(dev->multikernel_msi_state, cleanup_ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); return ret; @@ -357,6 +639,11 @@ int mk_pci_msi_restore(struct pci_dev *dev) if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE) return -EIO; generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_mask_mailbox(dev); + if (ret) { + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_FAILED); + return ret; + } ret = mk_pci_msi_restore_begin(dev, generation); if (!ret) ret = mk_pci_msi_bind_all(dev, generation); @@ -383,7 +670,9 @@ int mk_pci_msi_teardown(struct pci_dev *dev) if (READ_ONCE(dev->multikernel_msi_state) == MK_PCI_MSI_IDLE) return 0; generation = READ_ONCE(dev->multikernel_msi_generation); - ret = mk_pci_msi_teardown_generation(dev, generation); + ret = mk_pci_msi_mask_mailbox(dev); + if (!ret) + ret = mk_pci_msi_teardown_generation(dev, generation); WRITE_ONCE(dev->multikernel_msi_state, ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); return ret; @@ -572,20 +861,6 @@ static int __init x86_multikernel_pci_arch_init(void) { if (!root_instance || !root_instance->pci_devices_valid) return 0; -#ifdef CONFIG_PCI_MSI - mk_pci_host_instance = mk_instance_find(0); - if (!mk_pci_host_instance) { - pr_err("Multikernel has no restored host instance for PCI control\n"); - return 0; - } -#endif -#ifdef CONFIG_PCI_MSI - if (mk_register_msg_handler(MK_MSG_IO, mk_pci_irq_forward_handler, - NULL)) { - pr_err("Multikernel failed to register PCI IRQ forwarding handler\n"); - return 0; - } -#endif raw_pci_ops = &mk_pci_filtered_raw_ops; raw_pci_ext_ops = &mk_pci_filtered_raw_ops; diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index e2b096184f3dbd..9f73ca860353e8 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -77,11 +77,18 @@ bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, #define MK_IPI_SLOT_READY 2 #define MK_IPI_SLOT_CONSUMING 3 #define MK_IPI_SLOT_CANCELLED 4 -#define MK_IPI_ABI_MAGIC 0x4d4b495049303036ULL /* "MKIPI006" */ +#define MK_IPI_ABI_MAGIC 0x4d4b495049303037ULL /* "MKIPI007" */ #define MK_IPI_READY_TIMEOUT_MS 120000 #define MK_REPLY_SLOTS 16 #define MK_REPLY_STATE_BITS 3 +#define MK_IRQ_MAILBOX_SLOTS 256 +#define MK_IRQ_MAILBOX_WORDS (MK_IRQ_MAILBOX_SLOTS / 64) +#define MK_IRQ_MAILBOX_SLOT_INVALID ((u32)~0U) +#define MK_IRQ_MAILBOX_PENDING_MASK 0x3fffffffULL +#define MK_IRQ_MAILBOX_CONSUMING 0x40000000ULL +#define MK_IRQ_MAILBOX_MASKED 0x80000000ULL +#define MK_IRQ_MAILBOX_GENERATION_SHIFT 32 enum mk_reply_state { MK_REPLY_FREE = 0, @@ -146,6 +153,54 @@ struct mk_ipi_ring { atomic_t cancelled_writes; /* Halted producer writes recovered */ }; +struct mk_irq_mailbox_entry { + /* Upper 32 bits are slot generation; lower 32 bits are pending count. */ + atomic64_t pending_generation; + u64 lifecycle_epoch; + u32 lifecycle_generation; + u32 device_id; + u32 local_irq; + u16 vector; + u16 reserved; +}; + +struct mk_irq_mailbox { + atomic64_t pending_bitmap[MK_IRQ_MAILBOX_WORDS]; + struct mk_irq_mailbox_entry entries[MK_IRQ_MAILBOX_SLOTS]; + atomic_t next_generation; + atomic_t recorded; + atomic_t coalesced; + atomic_t masked_deferred; + atomic_t stale; + atomic_t dispatch_failed; + atomic_t saturated; +}; + +static inline u64 mk_irq_mailbox_token(u32 generation, u32 pending) +{ + return (u64)generation << MK_IRQ_MAILBOX_GENERATION_SHIFT | pending; +} + +static inline u32 mk_irq_mailbox_generation(u64 token) +{ + return token >> MK_IRQ_MAILBOX_GENERATION_SHIFT; +} + +static inline u32 mk_irq_mailbox_pending(u64 token) +{ + return token & MK_IRQ_MAILBOX_PENDING_MASK; +} + +static inline bool mk_irq_mailbox_masked(u64 token) +{ + return token & MK_IRQ_MAILBOX_MASKED; +} + +static inline bool mk_irq_mailbox_consuming(u64 token) +{ + return token & MK_IRQ_MAILBOX_CONSUMING; +} + /* Shared memory structures - per-instance design */ struct mk_shared_data { atomic_t emergency_shutdown; @@ -160,6 +215,8 @@ struct mk_shared_data { struct mk_reply_table replies; /* Changes on every host launch; tags process-context device lifecycles. */ u64 spawn_epoch; + /* Preallocated hardirq-safe PCI interrupt forwarding transport. */ + struct mk_irq_mailbox irq_mailbox; }; static inline void mk_reply_table_reset(struct mk_reply_table *table) @@ -180,6 +237,32 @@ static inline void mk_reply_table_reset(struct mk_reply_table *table) atomic_set(&table->occupied_failures, 0); } +static inline void mk_irq_mailbox_reset(struct mk_irq_mailbox *mailbox) +{ + unsigned int i; + + for (i = 0; i < MK_IRQ_MAILBOX_WORDS; i++) + atomic64_set(&mailbox->pending_bitmap[i], 0); + for (i = 0; i < MK_IRQ_MAILBOX_SLOTS; i++) { + struct mk_irq_mailbox_entry *entry = &mailbox->entries[i]; + + atomic64_set(&entry->pending_generation, 0); + WRITE_ONCE(entry->lifecycle_epoch, 0); + WRITE_ONCE(entry->lifecycle_generation, 0); + WRITE_ONCE(entry->device_id, 0); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, 0); + WRITE_ONCE(entry->reserved, 0); + } + atomic_set(&mailbox->next_generation, 0); + atomic_set(&mailbox->recorded, 0); + atomic_set(&mailbox->coalesced, 0); + atomic_set(&mailbox->masked_deferred, 0); + atomic_set(&mailbox->stale, 0); + atomic_set(&mailbox->dispatch_failed, 0); + atomic_set(&mailbox->saturated, 0); +} + static inline void mk_ipi_ring_reset_contents(struct mk_ipi_ring *ring) { unsigned int i; @@ -213,6 +296,7 @@ static inline void mk_shared_data_reset(struct mk_shared_data *shared) atomic_set(&shared->ready, 0); mk_reply_table_reset(&shared->replies); WRITE_ONCE(shared->spawn_epoch, 0); + mk_irq_mailbox_reset(&shared->irq_mailbox); } /* Function pointer type for IPI callbacks */ @@ -741,6 +825,7 @@ struct mk_instance { /* Pins the CPU selected for control messages and forwarded IRQs. */ struct rw_semaphore control_route_sem; mk_phys_cpu_t irq_route_cpu; /* IRQ-safe cached forwarding target */ + struct delayed_work irq_retry_work; /* Re-rings pending IRQ mailboxes. */ /* PCI device resources */ struct list_head pci_devices; /* List of struct mk_pci_device */ @@ -1081,6 +1166,7 @@ int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type); int mk_pci_msi_activate(struct pci_dev *dev); int mk_pci_msi_restore(struct pci_dev *dev); int mk_pci_msi_teardown(struct pci_dev *dev); +void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared); #else static inline bool mk_pci_msi_controlled(struct pci_dev *dev) { @@ -1106,6 +1192,10 @@ static inline int mk_pci_msi_teardown(struct pci_dev *dev) { return 0; } + +static inline void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) +{ +} #endif bool mk_platform_device_allowed(const char *name, const char *hid); @@ -1205,6 +1295,10 @@ static inline int mk_pci_msi_teardown(struct pci_dev *dev) return 0; } +static inline void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) +{ +} + static inline void mk_register_cpus_from_manifest(void) { } @@ -1224,7 +1318,7 @@ static inline bool mk_manifest_rejected(void) #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 /* Bumped whenever the shared-memory layout or message semantics change. */ -#define MK_FDT_COMPATIBLE "multikernel-v6" +#define MK_FDT_COMPATIBLE "multikernel-v7" /** * Property Names diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h index f99b8bba23ed19..82285c84b8a02e 100644 --- a/include/linux/multikernel_abi.h +++ b/include/linux/multikernel_abi.h @@ -3,7 +3,7 @@ #define _LINUX_MULTIKERNEL_ABI_H /* Private host/spawn transport compatibility constants. */ -#define MK_IPI_ABI_VERSION 6 +#define MK_IPI_ABI_VERSION 7 #define MK_BOOT_CONTEXT_MAGIC 0x4d4b435458303032ULL /* "MKCTX002" */ #endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index f241bfff11e58d..3e540db03bb6e2 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -15,7 +15,12 @@ #include #include "internal.h" -/* Lock order: transaction -> route write -> ownership -> resources. */ +/* + * CPU moves take transaction, then the affected instance's route write side, + * then ownership while inspecting or publishing paired CPU sets. PCI request + * work takes the route read side, validates under ownership, drops ownership + * before sleeping, and retains the route pin through reply publication. + */ static DEFINE_MUTEX(mk_cpu_transaction_mutex); static DEFINE_MUTEX(mk_cpu_ownership_mutex); @@ -277,6 +282,26 @@ void mk_instance_mark_failed(struct mk_instance *instance) mk_cpu_transaction_unlock(); } +static int mk_instance_finish_halt(struct mk_instance *instance, + bool transaction_held) +{ + int ret; + + if (!transaction_held) + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + /* Every caller has confirmed that all spawn CPUs are parked. */ + ret = mk_pci_quiesce_instance_irqs(instance, true); + mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); + mk_instance_set_state(instance, ret ? MK_STATE_FAILED : MK_STATE_LOADED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + if (!transaction_held) + mk_cpu_transaction_unlock(); + return ret; +} + struct mk_instance *mk_instance_find_by_name(const char *name) { struct mk_instance *instance; @@ -409,8 +434,8 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } - mk_cpu_set_free(snapshot); + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -439,9 +464,11 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, pr_err("Invalid CPU sets for transfer\n"); return -EINVAL; } + snapshot = mk_cpu_set_alloc(); if (!snapshot) return -ENOMEM; + mk_cpu_transaction_lock(); down_write(&instance->control_route_sem); mk_cpu_ownership_lock(); @@ -500,7 +527,6 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, pr_info("Transferred %u CPUs from pool to instance %d (%s): %s\n", requested_count, instance->id, instance->name, buf); - ret = 0; unlock: mk_cpu_ownership_unlock(); up_write(&instance->control_route_sem); @@ -533,9 +559,11 @@ int mk_instance_return_cpus(struct mk_instance *instance, pr_err("Invalid CPU sets for return\n"); return -EINVAL; } + snapshot = mk_cpu_set_alloc(); if (!snapshot) return -ENOMEM; + mk_cpu_transaction_lock(); down_write(&instance->control_route_sem); mk_cpu_ownership_lock(); @@ -577,6 +605,7 @@ int mk_instance_return_cpus(struct mk_instance *instance, if (ret) goto unlock_route; mk_cpu_ownership_lock(); + mk_cpu_set_for_each(i, phys_cpu, snapshot) { mk_cpu_set_add(mk_cpu_pool, phys_cpu); mk_cpu_set_del(instance->cpus, phys_cpu); @@ -585,7 +614,6 @@ int mk_instance_return_cpus(struct mk_instance *instance, pr_info("Returned %u CPUs from instance %d (%s) to the pool: %s\n", requested_count, instance->id, instance->name, buf); - ret = 0; unlock: mk_cpu_ownership_unlock(); unlock_route: @@ -1298,27 +1326,37 @@ static void mk_shutdown_work_fn(struct work_struct *work) } /* - * Mark a halted instance re-spawnable. No wakeups are published here: - * the instance's CPUs may still be on their way to the park loop, and - * poking its context while its (old or next) kernel also publishes on - * it corrupts the single-producer mailbox. The kexec path confirms the - * CPUs are parked before it rewrites the image. + * Mark a halted instance re-spawnable only after every CPU is parked. This + * proof permits recovery of a mailbox consumer interrupted by shutdown and + * prevents the next image from racing the old kernel in shared memory. */ -static void mk_instance_settle_halted(struct mk_instance *instance, - bool transaction_held) +static int mk_instance_settle_halted(struct mk_instance *instance, + bool transaction_held, + bool parked_confirmed) { + int ret; + pr_info("Instance %d (%s) halted, CPUs parking in pool\n", instance->id, instance->name); if (!transaction_held) mk_cpu_transaction_lock(); - down_write(&instance->control_route_sem); - mutex_lock(&instance->resource_mutex); - mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); - mk_instance_set_state(instance, MK_STATE_LOADED); - mutex_unlock(&instance->resource_mutex); - up_write(&instance->control_route_sem); + if (!parked_confirmed) { + ret = mk_instance_confirm_parked(instance); + if (ret) { + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + mk_instance_set_state(instance, MK_STATE_FAILED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + if (!transaction_held) + mk_cpu_transaction_unlock(); + return ret; + } + } + ret = mk_instance_finish_halt(instance, true); if (!transaction_held) mk_cpu_transaction_unlock(); + return ret; } struct mk_halted_work { @@ -1334,7 +1372,9 @@ static void mk_halted_work_fn(struct work_struct *work) instance = mk_instance_find(aw->instance_id); if (instance) { - mk_instance_settle_halted(instance, false); + if (mk_instance_settle_halted(instance, false, false)) + pr_err("Instance %d halted but could not be made reusable\n", + instance->id); mk_instance_put(instance); } else { pr_warn("Shutdown ACK from unknown instance %d\n", @@ -1459,22 +1499,34 @@ int multikernel_halt_by_id(int mk_id) ret = mk_msg_pending_wait(pending, 30000); if (ret == 0) { - if (mk_instance_confirm_parked(instance)) - pr_warn("Multikernel instance %d halted with CPUs unaccounted for\n", + ret = mk_instance_settle_halted(instance, false, false); + if (!ret) + pr_info("Multikernel instance %d halted (graceful)\n", mk_id); - - mk_instance_settle_halted(instance, false); - pr_info("Multikernel instance %d halted (graceful)\n", mk_id); } mk_instance_put(instance); return ret; } +/** + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop + * + * Forces a spawn kernel's CPUs to stop by queuing a shutdown message in the + * IPI ring buffer and sending NMIs directly to each CPU. The NMI handler + * checks for the pending shutdown message and stops if found. + * + * Use when: The spawn kernel is stuck/crashed and not responding to graceful + * shutdown, or when graceful shutdown has failed. + * + * Returns: 0 on success, negative error code on failure + */ static int __mk_instance_force_halt(struct mk_instance *instance, bool allow_loaded) { struct mk_shutdown_payload payload; + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; @@ -1482,7 +1534,6 @@ static int __mk_instance_force_halt(struct mk_instance *instance, if (!instance) return -EINVAL; - if (instance->state != MK_STATE_ACTIVE && (!allow_loaded || (instance->state != MK_STATE_LOADED && @@ -1492,40 +1543,68 @@ static int __mk_instance_force_halt(struct mk_instance *instance, return -EINVAL; } - if (mk_cpu_set_empty(instance->cpus)) { + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + mk_cpu_transaction_lock(); + mk_cpu_ownership_lock(); + ret = mk_cpu_set_copy(snapshot, instance->cpus); + mk_cpu_ownership_unlock(); + if (ret) { + mk_cpu_transaction_unlock(); + mk_cpu_set_free(snapshot); + return ret; + } + + if (mk_cpu_set_empty(snapshot)) { pr_err("Instance %d has no CPUs assigned\n", instance->id); + mk_cpu_transaction_unlock(); + mk_cpu_set_free(snapshot); return -EINVAL; } - pr_info("Force halting multikernel instance %d via NMI\n", - instance->id); - - /* Queue shutdown message - NMI handler will check for this */ + pr_info("Force halting multikernel instance %d via NMI\n", instance->id); + if (!instance->ipi_data) { + mk_cpu_transaction_unlock(); + mk_cpu_set_free(snapshot); + return -ENODEV; + } + atomic_set_release(&instance->ipi_data->emergency_shutdown, 1); payload.flags = MK_SHUTDOWN_IMMEDIATE; - payload.sender_instance_id = root_instance->id; - ret = mk_send_message(instance->id, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, - &payload, sizeof(payload)); + payload.sender_instance_id = root_instance ? root_instance->id : 0; + ret = mk_send_message_to_instance(instance, MK_MSG_SYSTEM, + MK_SYS_SHUTDOWN, &payload, + sizeof(payload)); if (ret < 0) - pr_err("Failed to queue shutdown message: %d (sending NMI anyway)\n", ret); + pr_err("Failed to queue shutdown message: %d (sending NMI anyway)\n", + ret); /* Send NMI to each CPU in the instance */ - mk_cpu_set_for_each(i, phys_cpu, instance->cpus) { + mk_cpu_set_for_each(i, phys_cpu, snapshot) { mk_force_stop_cpu(phys_cpu); cpu_count++; } pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, instance->id); - ret = mk_instance_confirm_parked(instance); if (ret) { pr_err("Instance %d CPUs did not park after force halt: %d\n", instance->id, ret); + mk_cpu_set_free(snapshot); + mk_cpu_transaction_unlock(); return ret; } + mk_cpu_set_free(snapshot); + /* Quiesce host-owned resources before making the instance reusable. */ + ret = mk_instance_settle_halted(instance, true, true); + mk_cpu_transaction_unlock(); + return ret; +} - mk_instance_settle_halted(instance, false); - return 0; +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); } int mk_instance_abort_spawn(struct mk_instance *instance) @@ -1538,24 +1617,6 @@ int mk_instance_abort_spawn(struct mk_instance *instance) return ret; } -/** - * mk_instance_force_halt - Forcibly stop an instance via NMI - * @instance: Instance to stop - * - * Forces a spawn kernel's CPUs to stop by queuing a shutdown message in the - * IPI ring buffer and sending NMIs directly to each CPU. The NMI handler - * checks for the pending shutdown message and stops if found. - * - * Use when: The spawn kernel is stuck/crashed and not responding to graceful - * shutdown, or when graceful shutdown has failed. - * - * Returns: 0 on success, negative error code on failure - */ -int mk_instance_force_halt(struct mk_instance *instance) -{ - return __mk_instance_force_halt(instance, false); -} - int multikernel_force_halt_by_id(int mk_id) { struct mk_instance *instance; @@ -1627,6 +1688,7 @@ static int __init multikernel_init(void) mk_hotplug_cleanup(); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index c803eca9b05f24..bb51152d7ef2f4 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -11,8 +11,6 @@ int mk_send_ipi_data(struct mk_instance *instance, void *data, int mk_send_ipi_data_to_cpu(struct mk_instance *instance, mk_phys_cpu_t target, void *data, size_t data_size, unsigned long type); -void mk_poll_ipi_messages(void); - /* messaging.c */ int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, u32 subtype, void *payload, u32 payload_len); @@ -28,14 +26,6 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, struct mk_instance *mk_instance_find_by_name(const char *name); int mk_instance_destroy(struct mk_instance *instance); int mk_instance_release_resources(struct mk_instance *instance); -void mk_cpu_transaction_lock(void); -void mk_cpu_transaction_unlock(void); -void mk_cpu_ownership_lock(void); -void mk_cpu_ownership_unlock(void); -void mk_cpu_ownership_assert_held(void); -/* Caller serializes CPU ownership changes with mk_cpu_transaction_lock(). */ -int mk_instance_migrate_irq_route(struct mk_instance *instance, - const struct mk_cpu_set *removing); /* dts.c */ int mk_dt_parse_resources(const void *fdt, int resources_node, @@ -45,14 +35,18 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, u8 *slot, u8 *func); -/* CPU ownership serialization: transaction must be acquired first. */ +/* CPU ownership serialization: transaction must be acquired before ownership. */ void mk_cpu_transaction_lock(void); void mk_cpu_transaction_unlock(void); void mk_cpu_ownership_lock(void); void mk_cpu_ownership_unlock(void); void mk_cpu_ownership_assert_held(void); +/* Caller serializes CPU ownership changes with mk_cpu_transaction_lock(). */ +int mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing); /* pci.c */ +#ifdef CONFIG_PCI int mk_pci_lease_system_init(void); void mk_pci_lease_system_cleanup(void); void mk_pci_lease_instance_init(struct mk_instance *instance); @@ -65,9 +59,78 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn); int mk_pci_release_assignments(struct mk_instance *instance); -void mk_pci_quiesce_instance_irqs(struct mk_instance *instance); + +/* Caller must hold instance->resource_mutex. */ +int mk_pci_quiesce_instance_irqs(struct mk_instance *instance, + bool parked_force); unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance); +#else +static inline int mk_pci_lease_system_init(void) +{ + return 0; +} + +static inline void mk_pci_lease_system_cleanup(void) +{ +} + +static inline void mk_pci_lease_instance_init(struct mk_instance *instance) +{ + mutex_init(&instance->resource_mutex); + INIT_LIST_HEAD(&instance->pci_assignments); +} + +static inline bool +mk_pci_iommu_lease_active_locked(struct mk_instance *instance) +{ + return false; +} + +static inline int +mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count) +{ + if (requested_count < 0) + return -EINVAL; + + return requested_count ? -EOPNOTSUPP : 0; +} + +static inline int mk_pci_assign_device(struct mk_instance *instance, + u16 domain, u8 bus, u8 devfn) +{ + return -EOPNOTSUPP; +} + +static inline int mk_pci_unassign_device(struct mk_instance *instance, + u16 domain, u8 bus, u8 devfn) +{ + return -EOPNOTSUPP; +} + +static inline int mk_pci_release_assignments(struct mk_instance *instance) +{ + return 0; +} + +/* Caller must hold instance->resource_mutex. */ +static inline int +mk_pci_quiesce_instance_irqs(struct mk_instance *instance, bool parked_force) +{ + return 0; +} + +static inline unsigned int +mk_pci_sync_instance_irq_route(struct mk_instance *instance) +{ + return 0; +} + +#endif + int mk_instance_force_halt(struct mk_instance *instance); + /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; int mk_overlay_init(void); diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 51e70f848559d5..786505dba0e47f 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -620,6 +620,7 @@ int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared) atomic_set(&shared->emergency_shutdown, 0); mk_ipi_ring_reset_contents(ring); mk_reply_table_reset(&shared->replies); + mk_irq_mailbox_reset(&shared->irq_mailbox); epoch = READ_ONCE(shared->spawn_epoch) + 1; if (!epoch) epoch = 1; @@ -994,6 +995,7 @@ static void multikernel_interrupt_handler(void) if (!root_instance || !root_instance->ipi_data) return; mk_reply_scan(root_instance->ipi_data); + mk_pci_irq_mailbox_drain(root_instance->ipi_data); /* * Drain here rather than from irq_work. We are already in interrupt diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index fe42193f1ee6fd..e31fd04b6b2b95 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -5,6 +5,12 @@ * Keeps assigned-device discovery, identity presentation, bridge traversal, * and resource restoration independent from the manifest that populated * the current instance. + * + * Configuration-space filtering constrains normal PCI access by a cooperative + * spawn kernel. It is not a security boundary: a privileged kernel can issue + * configuration cycles or map physical configuration windows directly. The + * host-owned IOMMU domain separately constrains DMA initiated by an assigned + * device. */ #include @@ -26,11 +32,19 @@ struct mk_pci_assignment; #define MK_PCI_FLR_SETTLE_MS 100 +#define MK_PCI_MAILBOX_QUIESCE_MS 1000 + +static unsigned long mk_pci_mailbox_deadline(void) +{ + return jiffies + msecs_to_jiffies(MK_PCI_MAILBOX_QUIESCE_MS); +} struct mk_pci_irq_vector { struct mk_pci_assignment *assignment; unsigned int host_irq; u32 local_irq; + u32 mailbox_slot; + u32 mailbox_generation; atomic64_t forwarded; bool requested; bool disabled; @@ -62,6 +76,7 @@ struct mk_pci_assignment { u8 irq_state; bool irq_msix; bool irq_needs_reprogram; + bool irq_mailbox_failed; unsigned long irq_flr_deadline; bool device_enabled; struct work_struct failure_work; @@ -226,45 +241,249 @@ mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, return NULL; } +static void mk_pci_mailbox_clear_bit(struct mk_irq_mailbox *mailbox, + unsigned int slot) +{ + atomic64_andnot(BIT_ULL(slot & 63), + &mailbox->pending_bitmap[slot / 64]); +} + +static bool mk_pci_mailbox_quiesce(struct mk_pci_irq_vector *vector, + bool parked_force, + unsigned long deadline) +{ + struct mk_shared_data *shared = vector->assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u64 base, token; + + if (!shared || vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return true; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + base = mk_irq_mailbox_token(vector->mailbox_generation, 0) | + MK_IRQ_MAILBOX_MASKED; + for (;;) { + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return false; + } + if (mk_irq_mailbox_consuming(token)) { + if (parked_force) { + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, base) == token) + break; + continue; + } + if (!mk_irq_mailbox_masked(token)) + atomic64_cmpxchg(&entry->pending_generation, + token, + token | MK_IRQ_MAILBOX_MASKED); + if (time_after_eq(jiffies, deadline)) + return false; + usleep_range(50, 100); + continue; + } + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, base) == token) + break; + cpu_relax(); + } + mk_pci_mailbox_clear_bit(mailbox, vector->mailbox_slot); + WRITE_ONCE(entry->local_irq, 0); + return true; +} + +static void mk_pci_mailbox_release(struct mk_pci_irq_vector *vector) +{ + struct mk_shared_data *shared = vector->assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u64 token; + + if (!shared || vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + for (;;) { + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return; + } + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, 0) == token) + break; + cpu_relax(); + } + mk_pci_mailbox_clear_bit(mailbox, vector->mailbox_slot); + WRITE_ONCE(entry->lifecycle_epoch, 0); + WRITE_ONCE(entry->lifecycle_generation, 0); + WRITE_ONCE(entry->device_id, 0); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, 0); + vector->mailbox_slot = MK_IRQ_MAILBOX_SLOT_INVALID; + vector->mailbox_generation = 0; +} + +static int mk_pci_mailbox_reserve(struct mk_pci_irq_vector *vector, + unsigned int vector_index) +{ + struct mk_pci_assignment *assignment = vector->assignment; + struct mk_shared_data *shared = assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u32 generation; + u32 device_id; + unsigned int slot; + u64 base; + + if (!shared) + return -ENODEV; + mailbox = &shared->irq_mailbox; + generation = (u32)atomic_inc_return(&mailbox->next_generation); + if (!generation) + generation = (u32)atomic_inc_return(&mailbox->next_generation); + if (!generation) + return -EOVERFLOW; + device_id = MK_PCI_IRQ_ID(pci_domain_nr(assignment->vf->bus), + assignment->vf->bus->number, + assignment->vf->devfn); + base = mk_irq_mailbox_token(generation, 0) | MK_IRQ_MAILBOX_MASKED; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + entry = &mailbox->entries[slot]; + if (atomic64_read(&entry->pending_generation)) + continue; + if (atomic64_cmpxchg_release(&entry->pending_generation, 0, + base)) + continue; + WRITE_ONCE(entry->lifecycle_epoch, assignment->irq_epoch); + WRITE_ONCE(entry->lifecycle_generation, + assignment->irq_generation); + WRITE_ONCE(entry->device_id, device_id); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, vector_index); + vector->mailbox_slot = slot; + vector->mailbox_generation = generation; + return 0; + } + return -ENOSPC; +} + +static void mk_pci_irq_retry_workfn(struct work_struct *work) +{ + struct mk_instance *instance = container_of(to_delayed_work(work), + struct mk_instance, + irq_retry_work); + struct mk_shared_data *shared = READ_ONCE(instance->ipi_data); + bool any_pending = false; + bool unmasked_pending = false; + mk_phys_cpu_t target; + unsigned int slot; + + if (!shared) + return; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry; + u64 token; + + entry = &shared->irq_mailbox.entries[slot]; + token = atomic64_read_acquire(&entry->pending_generation); + + if (!mk_irq_mailbox_generation(token)) + continue; + if (mk_irq_mailbox_pending(token)) { + any_pending = true; + atomic64_or(BIT_ULL(slot & 63), + &shared->irq_mailbox.pending_bitmap[slot / 64]); + if (!mk_irq_mailbox_masked(token)) + unmasked_pending = true; + } else if (mk_irq_mailbox_consuming(token)) { + any_pending = true; + } + } + if (unmasked_pending) { + target = mk_instance_irq_route_load(instance); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } + if (any_pending) + mod_delayed_work(system_wq, &instance->irq_retry_work, + msecs_to_jiffies(unmasked_pending ? 10 : 100)); +} + static irqreturn_t mk_pci_forward_irq(int irq, void *data) { struct mk_pci_irq_vector *vector = data; struct mk_pci_assignment *assignment = vector->assignment; - struct mk_io_irq_payload payload = { - .vector = vector - assignment->irq_vectors, - .device_id = MK_PCI_IRQ_ID(pci_domain_nr(assignment->vf->bus), - assignment->vf->bus->number, - assignment->vf->devfn), - .flags = MK_IRQ_LOW_LATENCY | MK_IRQ_EDGE_TRIGGERED, - .lifecycle_generation = READ_ONCE(assignment->irq_generation), - .lifecycle_epoch = READ_ONCE(assignment->irq_epoch), - }; - u32 local_irq = READ_ONCE(vector->local_irq); + struct mk_shared_data *shared = assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + mk_phys_cpu_t target; + u64 old, new; + u32 pending; + u32 local_irq; + + /* Pair with BIND's publication of the shared mailbox identity. */ + local_irq = smp_load_acquire(&vector->local_irq); if (READ_ONCE(assignment->instance->state) != MK_STATE_ACTIVE || READ_ONCE(assignment->irq_state) != MK_PCI_MSI_ACTIVE || - !local_irq) + !local_irq || !shared || + vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) return IRQ_HANDLED; - - payload.irq_number = local_irq; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + for (;;) { + old = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(old) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return IRQ_HANDLED; + } + pending = mk_irq_mailbox_pending(old); + if (pending == MK_IRQ_MAILBOX_PENDING_MASK) { + atomic_inc(&mailbox->saturated); + break; + } + new = mk_irq_mailbox_token(vector->mailbox_generation, + pending + 1) | + (old & (MK_IRQ_MAILBOX_MASKED | + MK_IRQ_MAILBOX_CONSUMING)); + if (atomic64_cmpxchg(&entry->pending_generation, old, new) == old) + break; + cpu_relax(); + } + atomic64_or(BIT_ULL(vector->mailbox_slot & 63), + &mailbox->pending_bitmap[vector->mailbox_slot / 64]); + atomic_inc(&mailbox->recorded); + if (!pending) + mod_delayed_work(system_wq, &assignment->instance->irq_retry_work, + msecs_to_jiffies(10)); if (atomic64_inc_return(&vector->forwarded) == 1) pr_info("Forwarding host IRQ %u as instance IRQ %u for %s vector %u\n", irq, local_irq, pci_name(assignment->vf), - payload.vector); - if (mk_send_message_to_instance(assignment->instance, MK_MSG_IO, - MK_IO_IRQ_FORWARD, &payload, - sizeof(payload))) - pr_warn_ratelimited("Failed to forward IRQ %u for %s to instance %d\n", - irq, pci_name(assignment->vf), - assignment->instance->id); + (unsigned int)(vector - assignment->irq_vectors)); + target = mk_instance_irq_route_load(assignment->instance); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); return IRQ_HANDLED; } -static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment) +static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment, + bool parked_force, + unsigned long deadline) { unsigned int disabled = 0; unsigned int i; + assignment->irq_mailbox_failed = false; for (i = 0; i < assignment->nr_irq_vectors; i++) WRITE_ONCE(assignment->irq_vectors[i].local_irq, 0); @@ -277,6 +496,12 @@ static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment) disabled++; } } + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!mk_pci_mailbox_quiesce(vector, parked_force, deadline)) + assignment->irq_mailbox_failed = true; + } if (assignment->irq_bound_map) bitmap_zero(assignment->irq_bound_map, assignment->nr_irq_vectors); @@ -287,16 +512,21 @@ static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment) return disabled; } -static void mk_pci_release_irqs(struct mk_pci_assignment *assignment) +static int mk_pci_release_irqs(struct mk_pci_assignment *assignment, + bool parked_force) { + unsigned long deadline = mk_pci_mailbox_deadline(); unsigned int i; - mk_pci_quiesce_irqs(assignment); + mk_pci_quiesce_irqs(assignment, parked_force, deadline); + if (assignment->irq_mailbox_failed) + return -ETIMEDOUT; for (i = 0; i < assignment->nr_irq_vectors; i++) { struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; if (vector->requested) free_irq(vector->host_irq, vector); + mk_pci_mailbox_release(vector); } if (assignment->nr_irq_vectors) pci_free_irq_vectors(assignment->vf); @@ -308,27 +538,37 @@ static void mk_pci_release_irqs(struct mk_pci_assignment *assignment) assignment->irq_msix = false; assignment->irq_needs_reprogram = false; assignment->irq_flr_deadline = 0; + assignment->irq_mailbox_failed = false; + return 0; } -void mk_pci_quiesce_instance_irqs(struct mk_instance *instance) +int mk_pci_quiesce_instance_irqs(struct mk_instance *instance, + bool parked_force) { struct mk_pci_assignment *assignment; + unsigned long deadline = mk_pci_mailbox_deadline(); unsigned int disabled = 0; + int ret = 0; if (!instance || instance == root_instance) - return; + return 0; lockdep_assert_held(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); list_for_each_entry(assignment, &instance->pci_assignments, - instance_node) - disabled += mk_pci_quiesce_irqs(assignment); + instance_node) { + disabled += mk_pci_quiesce_irqs(assignment, parked_force, + deadline); + if (assignment->irq_mailbox_failed) + ret = -ETIMEDOUT; + } pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); if (disabled) pr_info("Quiesced %u host-owned PCI IRQ vectors for instance %d\n", disabled, instance->id); + return ret; } unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance) @@ -354,6 +594,12 @@ unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance) } pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); + if (requested) { + mk_phys_cpu_t target = mk_instance_irq_route_load(instance); + + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } return requested; } @@ -373,7 +619,8 @@ static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, int nvec; int ret; - if (!request->nr_vectors) + if (!request->nr_vectors || + request->nr_vectors > MK_IRQ_MAILBOX_SLOTS) return -EINVAL; if (!request->lifecycle_generation) return -EINVAL; @@ -398,8 +645,11 @@ static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, if (!mk_pci_generation_after(request->lifecycle_generation, assignment->irq_generation)) return -ESTALE; - if (assignment->nr_irq_vectors) - mk_pci_release_irqs(assignment); + if (assignment->nr_irq_vectors) { + ret = mk_pci_release_irqs(assignment, false); + if (ret) + return ret; + } assignment->irq_epoch = request->lifecycle_epoch; assignment->irq_generation = request->lifecycle_generation; assignment->irq_state = MK_PCI_MSI_FAILED; @@ -428,6 +678,10 @@ static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, for (i = 0; i < nvec; i++) { vectors[i].assignment = assignment; vectors[i].host_irq = pci_irq_vector(assignment->vf, i); + vectors[i].mailbox_slot = MK_IRQ_MAILBOX_SLOT_INVALID; + ret = mk_pci_mailbox_reserve(&vectors[i], i); + if (ret) + goto err_release; } for (i = 0; i < nvec; i++) { ret = request_irq(vectors[i].host_irq, mk_pci_forward_irq, @@ -452,13 +706,15 @@ static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, err_release: pr_err("Failed to configure host-owned IRQ for %s vector %d: %d\n", pci_name(assignment->vf), i, ret); - mk_pci_release_irqs(assignment); + if (mk_pci_release_irqs(assignment, false)) + return -ETIMEDOUT; return ret; } static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, const struct mk_pci_irq_request *request) { + struct mk_shared_data *shared = assignment->instance->ipi_data; struct mk_pci_irq_vector *vector; unsigned long delay; unsigned int count; @@ -476,6 +732,7 @@ static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, (assignment->irq_state != MK_PCI_MSI_PREPARED && assignment->irq_state != MK_PCI_MSI_ACTIVE)) return -ESTALE; + if (!assignment->irq_vectors || assignment->irq_msix != request->msix) return -EINVAL; @@ -488,10 +745,24 @@ static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, return -EINVAL; for (i = 0; i < count; i++) { + struct mk_irq_mailbox_entry *entry; + u64 token; + vector = &assignment->irq_vectors[request->vector + i]; local_irq = READ_ONCE(vector->local_irq); - if (!vector->requested) + if (!vector->requested || + vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) return -EINVAL; + entry = &shared->irq_mailbox.entries[vector->mailbox_slot]; + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation || + READ_ONCE(entry->lifecycle_epoch) != assignment->irq_epoch || + READ_ONCE(entry->lifecycle_generation) != + assignment->irq_generation || + READ_ONCE(entry->vector) != request->vector + i) + return -ESTALE; if (local_irq && local_irq != request->local_irq + i) return -EBUSY; } @@ -509,8 +780,13 @@ static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, } for (i = 0; i < count; i++) { + struct mk_irq_mailbox_entry *entry; + vector = &assignment->irq_vectors[request->vector + i]; - WRITE_ONCE(vector->local_irq, request->local_irq + i); + entry = &shared->irq_mailbox.entries[vector->mailbox_slot]; + WRITE_ONCE(entry->local_irq, request->local_irq + i); + /* Make the shared identity visible before routing this vector. */ + smp_store_release(&vector->local_irq, request->local_irq + i); } bitmap_set(assignment->irq_bound_map, request->vector, count); return 0; @@ -529,7 +805,9 @@ mk_pci_restore_irqs_begin(struct mk_pci_assignment *assignment, if (assignment->irq_state != MK_PCI_MSI_ACTIVE) return -ESTALE; - mk_pci_quiesce_irqs(assignment); + mk_pci_quiesce_irqs(assignment, false, mk_pci_mailbox_deadline()); + if (assignment->irq_mailbox_failed) + return -ETIMEDOUT; assignment->irq_needs_reprogram = true; return 0; } @@ -604,8 +882,12 @@ static int mk_pci_teardown_irqs(struct mk_pci_assignment *assignment, assignment->irq_generation)) return -ESTALE; - if (assignment->nr_irq_vectors) - mk_pci_release_irqs(assignment); + if (assignment->nr_irq_vectors) { + int ret = mk_pci_release_irqs(assignment, false); + + if (ret) + return ret; + } assignment->irq_epoch = request->lifecycle_epoch; assignment->irq_generation = request->lifecycle_generation; assignment->irq_state = MK_PCI_MSI_IDLE; @@ -898,7 +1180,13 @@ static int mk_pci_reset_access(struct mk_instance *instance, /* Tombstone this serial before side effects so delayed replays reject. */ assignment->reset_generation = request->reset_generation; - mk_pci_quiesce_irqs(assignment); + mk_pci_quiesce_irqs(assignment, false, mk_pci_mailbox_deadline()); + if (assignment->irq_mailbox_failed) { + assignment->irq_state = MK_PCI_MSI_FAILED; + mk_pci_schedule_failure(assignment); + ret = -ETIMEDOUT; + goto out; + } ret = pcie_reset_flr(assignment->vf, false); if (ret) { assignment->irq_state = MK_PCI_MSI_FAILED; @@ -1026,7 +1314,7 @@ static void mk_pci_assignment_failure_work(struct work_struct *work) instance->id, ret); } - mk_instance_set_state(instance, MK_STATE_FAILED); + mk_instance_mark_failed(instance); } static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment) @@ -1126,8 +1414,6 @@ static int mk_pci_iommu_validate_group(struct mk_pci_assignment *assignment) struct mk_pci_iommu_group_check check = { .vf = &assignment->vf->dev, }; - struct iommu_resv_region *region; - LIST_HEAD(resv_regions); int ret; ret = iommu_group_for_each_dev(assignment->iommu_group, &check, @@ -1146,28 +1432,65 @@ static int mk_pci_iommu_validate_group(struct mk_pci_assignment *assignment) return -EPERM; } + return 0; +} + +static int +mk_pci_iommu_validate_resv_regions(struct mk_pci_assignment *assignment) +{ + struct iommu_resv_region *region; + struct mk_memory_region *memory; + LIST_HEAD(resv_regions); + u64 memory_end, resv_end; + int ret; + ret = iommu_get_group_resv_regions(assignment->iommu_group, &resv_regions); - if (ret) { - mk_pci_iommu_free_resv_regions(&resv_regions); - return ret; - } + if (ret) + goto out; list_for_each_entry(region, &resv_regions, list) { - if (region->type != IOMMU_RESV_DIRECT && - region->type != IOMMU_RESV_DIRECT_RELAXABLE && - region->type != IOMMU_RESV_SW_MSI) + if (region->type == IOMMU_RESV_DIRECT_RELAXABLE) continue; - pr_err("IOMMU group %d for %s requires unsupported reserved region %#llx-%#llx type %u\n", - iommu_group_id(assignment->iommu_group), - pci_name(assignment->vf), - (unsigned long long)region->start, - (unsigned long long)(region->start + region->length - 1), - region->type); - ret = -EPERM; - break; + if (!region->length || + check_add_overflow((u64)region->start, + (u64)region->length - 1, &resv_end)) { + pr_err("IOMMU group %d for %s has invalid reserved region at %#llx\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf), + (unsigned long long)region->start); + ret = -EOVERFLOW; + goto out; + } + + list_for_each_entry(memory, + &assignment->instance->memory_regions, list) { + resource_size_t size = resource_size(&memory->res); + + if (!size || + check_add_overflow((u64)memory->res.start, + (u64)size - 1, &memory_end)) { + ret = -EOVERFLOW; + goto out; + } + if ((u64)memory->res.start > resv_end || + memory_end < (u64)region->start) + continue; + + pr_err("Instance %d IOVA %#llx-%#llx overlaps IOMMU reserved region %#llx-%#llx type %u for %s\n", + assignment->instance->id, + (unsigned long long)memory->res.start, + (unsigned long long)memory_end, + (unsigned long long)region->start, + (unsigned long long)resv_end, region->type, + pci_name(assignment->vf)); + ret = -EPERM; + goto out; + } } + ret = 0; +out: mk_pci_iommu_free_resv_regions(&resv_regions); return ret; } @@ -1248,7 +1571,9 @@ mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) bool transactions_drained; int ret; - mk_pci_release_irqs(assignment); + ret = mk_pci_release_irqs(assignment, false); + if (ret) + return ret; if (!mk_pci_device_live(vf)) return 0; @@ -1285,7 +1610,9 @@ mk_pci_reset_assignment_for_start(struct mk_pci_assignment *assignment) struct pci_dev *vf = assignment->vf; int ret; - mk_pci_release_irqs(assignment); + ret = mk_pci_release_irqs(assignment, true); + if (ret) + return ret; assignment->irq_epoch = 0; assignment->irq_generation = 0; assignment->reset_generation = 0; @@ -1365,6 +1692,12 @@ static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) pci_name(assignment->vf)); return -EOPNOTSUPP; } + if (!device_iommu_capable(&assignment->vf->dev, + IOMMU_CAP_CACHE_COHERENCY)) { + pr_err("Cannot assign %s without coherent IOMMU mappings\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; + } if (!assignment->instance->region_count || list_empty(&assignment->instance->memory_regions)) return -EINVAL; @@ -1374,6 +1707,9 @@ static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) return -ENODEV; ret = mk_pci_iommu_validate_group(assignment); + if (ret) + goto err_release; + ret = mk_pci_iommu_validate_resv_regions(assignment); if (ret) goto err_release; @@ -1392,8 +1728,8 @@ static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) if (ret) goto err_release; ret = iommu_map(assignment->iommu_domain, region->res.start, - region->res.start, size, IOMMU_READ | IOMMU_WRITE, - GFP_KERNEL); + region->res.start, size, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE, GFP_KERNEL); if (ret) goto err_release; assignment->iommu_mapped_regions++; @@ -1773,7 +2109,8 @@ static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) return 0; } -static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) +static int mk_pci_release_assignment(struct mk_pci_assignment *assignment, + struct list_head *released) { struct mk_instance *instance = assignment->instance; struct pci_dev *vf = assignment->vf; @@ -1804,9 +2141,9 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) release_resources: mk_pci_iommu_release_assignment(assignment); - cancel_work_sync(&assignment->failure_work); if (assignment->inventory_moved && root_instance) { + assignment->inventory->resources_valid = false; list_move_tail(&assignment->inventory->list, &root_instance->pci_devices); instance->pci_device_count--; @@ -1815,20 +2152,32 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) assignment->inventory_moved = false; } - kfree(assignment->host_driver_override); - if (assignment->host_driver && assignment->host_driver->owner) - module_put(assignment->host_driver->owner); if (!list_empty(&assignment->transaction_node)) list_del_init(&assignment->transaction_node); list_del_init(&assignment->instance_node); - pci_dev_put(assignment->pf); - pci_dev_put(vf); - kfree(assignment); + list_add_tail(&assignment->transaction_node, released); return 0; } -static int mk_pci_rollback_transaction(struct list_head *transaction) +static void mk_pci_finalize_releases(struct list_head *released) +{ + struct mk_pci_assignment *assignment, *tmp; + + list_for_each_entry_safe(assignment, tmp, released, transaction_node) { + list_del_init(&assignment->transaction_node); + cancel_work_sync(&assignment->failure_work); + kfree(assignment->host_driver_override); + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + pci_dev_put(assignment->pf); + pci_dev_put(assignment->vf); + kfree(assignment); + } +} + +static int mk_pci_rollback_transaction(struct list_head *transaction, + struct list_head *released) { struct mk_pci_assignment *assignment, *tmp; int rollback_ret = 0; @@ -1836,7 +2185,7 @@ static int mk_pci_rollback_transaction(struct list_head *transaction) list_for_each_entry_safe_reverse(assignment, tmp, transaction, transaction_node) { - ret = mk_pci_release_assignment(assignment); + ret = mk_pci_release_assignment(assignment, released); if (!ret) continue; pr_crit("Failed to roll back PCI assignment for %s: %d\n", @@ -1874,6 +2223,7 @@ void mk_pci_lease_instance_init(struct mk_instance *instance) { mutex_init(&instance->resource_mutex); INIT_LIST_HEAD(&instance->pci_assignments); + INIT_DELAYED_WORK(&instance->irq_retry_work, mk_pci_irq_retry_workfn); } bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance) @@ -1890,6 +2240,7 @@ int mk_pci_assign_devices(struct mk_instance *instance, int requested_count) { struct mk_pci_device *requested; + LIST_HEAD(released); LIST_HEAD(transaction); int prepared = 0; int ret = 0; @@ -1924,13 +2275,14 @@ int mk_pci_assign_devices(struct mk_instance *instance, goto out; rollback: - rollback_ret = mk_pci_rollback_transaction(&transaction); + rollback_ret = mk_pci_rollback_transaction(&transaction, &released); if (rollback_ret) ret = rollback_ret; out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); return ret; } @@ -1938,6 +2290,7 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { struct mk_pci_device *inventory; + LIST_HEAD(released); LIST_HEAD(transaction); int ret; int rollback_ret; @@ -1973,13 +2326,14 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, goto out; rollback: - rollback_ret = mk_pci_rollback_transaction(&transaction); + rollback_ret = mk_pci_rollback_transaction(&transaction, &released); if (rollback_ret) ret = rollback_ret; out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); return ret; } @@ -1987,6 +2341,7 @@ int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { struct mk_pci_assignment *assignment; + LIST_HEAD(released); int ret; if (!instance || instance == root_instance) @@ -2007,22 +2362,26 @@ int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, goto out; } - ret = mk_pci_release_assignment(assignment); + ret = mk_pci_release_assignment(assignment, &released); out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); return ret; } int mk_pci_release_assignments(struct mk_instance *instance) { struct mk_pci_assignment *assignment; + LIST_HEAD(released); int ret = 0; if (!instance || instance == root_instance) return 0; + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); mutex_lock(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); @@ -2030,7 +2389,7 @@ int mk_pci_release_assignments(struct mk_instance *instance) assignment = list_last_entry(&instance->pci_assignments, struct mk_pci_assignment, instance_node); - ret = mk_pci_release_assignment(assignment); + ret = mk_pci_release_assignment(assignment, &released); if (ret) { pr_crit("Instance %d retains unsafe PCI lease for %s: %d\n", instance->id, pci_name(assignment->vf), ret); @@ -2040,6 +2399,16 @@ int mk_pci_release_assignments(struct mk_instance *instance) pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + /* + * The assignment is no longer reachable by routed requests. Cancel its + * failure work after dropping the route and transaction locks because the + * worker may itself force-halt the instance and take both locks. + */ + mk_pci_finalize_releases(&released); + if (!ret) + cancel_delayed_work_sync(&instance->irq_retry_work); return ret; } @@ -2144,7 +2513,8 @@ void mk_pci_lease_system_cleanup(void) static struct mk_pci_device * mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) { - if (!root_instance || !root_instance->dtb_data || + if (!root_instance || root_instance->id == 0 || + !root_instance->dtb_data || !root_instance->pci_devices_valid) return NULL; From 29c6e3140c0578ba24e6517b89c83edacef46511 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 06:34:14 +0300 Subject: [PATCH 16/16] multikernel: expose transport reliability stats Expose a versioned per-instance snapshot for the ordered IPI ring, direct reply table, and pending IRQ mailbox. Document every counter, gauge, reset boundary, and the non-atomic modulo-u32 snapshot semantics. Signed-off-by: Nikolay Nikolaev --- Documentation/ABI/testing/multikernel | 78 +++++++++++++ Documentation/multikernel/usage.rst | 11 -- kernel/multikernel/kernfs.c | 161 ++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 11 deletions(-) create mode 100644 Documentation/ABI/testing/multikernel diff --git a/Documentation/ABI/testing/multikernel b/Documentation/ABI/testing/multikernel new file mode 100644 index 00000000000000..67986e09ea88bd --- /dev/null +++ b/Documentation/ABI/testing/multikernel @@ -0,0 +1,78 @@ +What: /sys/fs/multikernel/instances//stats +Date: August 2026 +Contact: Nikolay Nikolaev +Description: + Read-only reliability snapshot for one multikernel instance. The + format is one ``key value`` pair per line. The first line is always + ``stats_version 1``. Versioned formats are append-only; readers must + ignore unknown keys. + + The snapshot is observational, not atomic with concurrent producers. + Cumulative transport counters are unsigned 32-bit event counts and + wrap modulo 2^32. They reset when the shared buffer is initialized + for a new ``spawn_epoch`` and cannot be reset through this file. + Compare deltas only between samples with the same nonzero epoch. + Gauges are unsigned point-in-time counts and have no reset operation. + + Metadata keys (class; unit; meaning): + + ``stats_version`` (metadata; version; this file format, currently 1), + ``abi_version`` (metadata; version; exact shared transport ABI), + ``snapshot_atomic`` (metadata; boolean; always 0), + ``transport_counter_bits`` (metadata; bits; always 32), + ``transport_available`` (gauge; boolean; shared buffer is mapped), and + ``spawn_epoch`` (metadata; launch identifier; changes on each start). + + ``abi_version`` identifies the exact private kernel-to-kernel transport + contract used by the running instance. Exposing it here supports + diagnostics; it is not a stable userspace ABI or a compatibility promise + between transport versions. + + Ordered IPI keys (class; unit; increment or sampled condition; reset): + + ``ipi.producer_contention`` (counter; sends; producer gate observed + busy or recursively owned; new epoch), ``ipi.full_failures`` (counter; + sends; head slot prevented publication; new epoch), + ``ipi.invalid_state`` (counter; observations; impossible slot/cursor + state; new epoch), ``ipi.cancelled_writes`` (counter; writes; parked + producer recovered as CANCELLED; new epoch), ``ipi.gate_busy`` (gauge; + gates; producer gate nonzero; sampled), ``ipi.slot_writing`` (gauge; + slots; WRITING; sampled), ``ipi.slot_ready`` (gauge; slots; READY; + sampled), ``ipi.slot_consuming`` (gauge; slots; CONSUMING; sampled), + and ``ipi.slot_cancelled`` (gauge; slots; CANCELLED; sampled). + + Direct reply keys (class; unit; increment or sampled condition; reset): + + ``reply.late_replies`` (counter; replies; stale exact-token claim or + completion was rejected; new epoch), ``reply.cancelled_slots`` + (counter; slots; timeout + cancelled RESERVED/WRITING; new epoch), ``reply.atomic_timeouts`` + (counter; waits; bounded atomic wait expired in cancellable + RESERVED/WRITING or indeterminate EXECUTING state; new epoch), + ``reply.indeterminate_timeouts`` (counter; waits; subset where + EXECUTING won and changed to COMMITTED; new epoch), + ``reply.occupied_failures`` (counter; + reservations; no FREE slot; new epoch), ``reply.slot_busy`` (gauge; + slots; state is not FREE; sampled), ``reply.slot_reserved`` (gauge; + slots; RESERVED; sampled), ``reply.slot_writing`` (gauge; slots; + WRITING; sampled), ``reply.slot_executing`` (gauge; slots; EXECUTING; + sampled), ``reply.slot_committed`` (gauge; slots; COMMITTED; sampled), + ``reply.slot_ready`` (gauge; slots; READY; sampled), and + ``reply.slot_abandoned`` (gauge; slots; ABANDONED; sampled). + + Pending IRQ keys (class; unit; increment or sampled condition; reset): + + ``irq.recorded`` (counter; IRQs; pending count recorded; new epoch), + ``irq.coalesced`` (counter; IRQs; recorded onto an already-pending + slot; new epoch), ``irq.masked_deferred`` (counter; IRQs; delivery + deferred while logically masked; new epoch), ``irq.stale`` (counter; + observations; epoch/generation/route validation rejected work; new + epoch), ``irq.dispatch_failed`` (counter; IRQs; validated local dispatch + failed; new epoch), ``irq.saturated`` (counter; IRQs; pending count hit + its representable limit; new epoch), ``irq.slot_active`` (gauge; slots; + nonzero generation; sampled), ``irq.slot_pending`` (gauge; slots; + nonzero pending count; sampled), ``irq.slot_masked`` (gauge; slots; + MASKED set; sampled), ``irq.slot_consuming`` (gauge; slots; CONSUMING + set; sampled), and ``irq.pending_total`` (gauge; IRQs; sum of pending + counts across active slots; sampled). +Users: multikernel management and diagnostic tools diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index e74f19f4346631..9fe81edde86e26 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -106,17 +106,6 @@ Phase 1: Instance Creation (Automatic from DTB) through this interface, and must only be compared as modulo-32-bit deltas between samples carrying the same nonzero epoch. -Shared Transport Compatibility -============================== - -The private kernel-to-kernel shared transport requires an exact match between -the host and spawn. The loader compares the spawn image note before launch; -the spawn manifest and transport initialization acknowledgment confirm the -same layout before shared memory is used. These checks establish transport -compatibility and do not authenticate the kernel artifact. A mismatched -bzImage or vmlinux is rejected; there is no translation or backward- -compatibility promise between transport versions. - # View instance device tree cat /sys/fs/multikernel/instances/web-server/device_tree_source # Output: DTS format showing the instance configuration diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index 5759df97df5c88..43ad19a4a6edaf 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -126,6 +126,156 @@ static int status_seq_show(struct seq_file *sf, void *v) return 0; } +/* + * This is an observational snapshot, not a transaction: producers may update + * fields while the file is read. Shared transport counters are u32 values + * and restart with a new spawn_epoch, so consumers compare modulo-u32 deltas + * only between samples from the same epoch. + */ +static int stats_seq_show(struct seq_file *sf, void *v) +{ + struct kernfs_open_file *of = sf->private; + struct mk_instance *instance = of->kn->priv; + struct mk_shared_data *shared; + struct { + u64 spawn_epoch; + u64 irq_pending_total; + u32 ipi_producer_contention; + u32 ipi_full_failures; + u32 ipi_invalid_state; + u32 ipi_cancelled_writes; + u32 reply_late_replies; + u32 reply_cancelled_slots; + u32 reply_atomic_timeouts; + u32 reply_indeterminate_timeouts; + u32 reply_occupied_failures; + u32 irq_recorded; + u32 irq_coalesced; + u32 irq_masked_deferred; + u32 irq_stale; + u32 irq_dispatch_failed; + u32 irq_saturated; + u32 ipi_slots[4]; + u32 reply_states[MK_REPLY_ABANDONED + 1]; + u32 irq_active; + u32 irq_pending; + u32 irq_masked; + u32 irq_consuming; + u32 reply_busy; + u32 ipi_gate_busy; + bool available; + } stats = {}; + unsigned int i; + + down_read(&instance->control_route_sem); + shared = READ_ONCE(instance->ipi_data); + if (!shared) + goto unlock; + stats.available = true; + stats.spawn_epoch = READ_ONCE(shared->spawn_epoch); + stats.ipi_producer_contention = + atomic_read(&shared->ring.producer_contention); + stats.ipi_full_failures = atomic_read(&shared->ring.full_failures); + stats.ipi_invalid_state = atomic_read(&shared->ring.invalid_state); + stats.ipi_cancelled_writes = atomic_read(&shared->ring.cancelled_writes); + stats.ipi_gate_busy = !!atomic64_read_acquire(&shared->ring.producer_gate); + stats.reply_late_replies = atomic_read(&shared->replies.late_replies); + stats.reply_cancelled_slots = atomic_read(&shared->replies.cancelled_slots); + stats.reply_atomic_timeouts = atomic_read(&shared->replies.atomic_timeouts); + stats.reply_indeterminate_timeouts = + atomic_read(&shared->replies.indeterminate_timeouts); + stats.reply_occupied_failures = + atomic_read(&shared->replies.occupied_failures); + stats.irq_recorded = atomic_read(&shared->irq_mailbox.recorded); + stats.irq_coalesced = atomic_read(&shared->irq_mailbox.coalesced); + stats.irq_masked_deferred = + atomic_read(&shared->irq_mailbox.masked_deferred); + stats.irq_stale = atomic_read(&shared->irq_mailbox.stale); + stats.irq_dispatch_failed = + atomic_read(&shared->irq_mailbox.dispatch_failed); + stats.irq_saturated = atomic_read(&shared->irq_mailbox.saturated); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + u32 state = atomic_read_acquire(&shared->ring.entries[i].state); + + if (state >= MK_IPI_SLOT_WRITING && state <= MK_IPI_SLOT_CANCELLED) + stats.ipi_slots[state - MK_IPI_SLOT_WRITING]++; + } + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &shared->replies.slots[i]; + u64 token = atomic64_read_acquire(&slot->state_generation); + u32 state = token & (BIT(MK_REPLY_STATE_BITS) - 1); + + if (state <= MK_REPLY_ABANDONED) + stats.reply_states[state]++; + if (state != MK_REPLY_FREE) + stats.reply_busy++; + } + for (i = 0; i < MK_IRQ_MAILBOX_SLOTS; i++) { + struct mk_irq_mailbox_entry *entry = + &shared->irq_mailbox.entries[i]; + u64 token = atomic64_read_acquire(&entry->pending_generation); + + if (!mk_irq_mailbox_generation(token)) + continue; + stats.irq_active++; + if (mk_irq_mailbox_pending(token)) { + stats.irq_pending++; + stats.irq_pending_total += mk_irq_mailbox_pending(token); + } + if (mk_irq_mailbox_masked(token)) + stats.irq_masked++; + if (mk_irq_mailbox_consuming(token)) + stats.irq_consuming++; + } +unlock: + up_read(&instance->control_route_sem); + + seq_puts(sf, "stats_version 1\n"); + seq_printf(sf, "abi_version %u\n", MK_IPI_ABI_VERSION); + seq_puts(sf, "snapshot_atomic 0\n"); + seq_puts(sf, "transport_counter_bits 32\n"); + seq_printf(sf, "transport_available %u\n", stats.available); + seq_printf(sf, "spawn_epoch %llu\n", + (unsigned long long)stats.spawn_epoch); +#define MK_STATS_VALUE(name, value) seq_printf(sf, name " %u\n", value) + MK_STATS_VALUE("ipi.producer_contention", stats.ipi_producer_contention); + MK_STATS_VALUE("ipi.full_failures", stats.ipi_full_failures); + MK_STATS_VALUE("ipi.invalid_state", stats.ipi_invalid_state); + MK_STATS_VALUE("ipi.cancelled_writes", stats.ipi_cancelled_writes); + MK_STATS_VALUE("ipi.gate_busy", stats.ipi_gate_busy); + MK_STATS_VALUE("ipi.slot_writing", stats.ipi_slots[0]); + MK_STATS_VALUE("ipi.slot_ready", stats.ipi_slots[1]); + MK_STATS_VALUE("ipi.slot_consuming", stats.ipi_slots[2]); + MK_STATS_VALUE("ipi.slot_cancelled", stats.ipi_slots[3]); + MK_STATS_VALUE("reply.late_replies", stats.reply_late_replies); + MK_STATS_VALUE("reply.cancelled_slots", stats.reply_cancelled_slots); + MK_STATS_VALUE("reply.atomic_timeouts", stats.reply_atomic_timeouts); + MK_STATS_VALUE("reply.indeterminate_timeouts", + stats.reply_indeterminate_timeouts); + MK_STATS_VALUE("reply.occupied_failures", stats.reply_occupied_failures); + MK_STATS_VALUE("reply.slot_busy", stats.reply_busy); + MK_STATS_VALUE("reply.slot_reserved", stats.reply_states[MK_REPLY_RESERVED]); + MK_STATS_VALUE("reply.slot_writing", stats.reply_states[MK_REPLY_WRITING]); + MK_STATS_VALUE("reply.slot_executing", stats.reply_states[MK_REPLY_EXECUTING]); + MK_STATS_VALUE("reply.slot_committed", stats.reply_states[MK_REPLY_COMMITTED]); + MK_STATS_VALUE("reply.slot_ready", stats.reply_states[MK_REPLY_READY]); + MK_STATS_VALUE("reply.slot_abandoned", stats.reply_states[MK_REPLY_ABANDONED]); + MK_STATS_VALUE("irq.recorded", stats.irq_recorded); + MK_STATS_VALUE("irq.coalesced", stats.irq_coalesced); + MK_STATS_VALUE("irq.masked_deferred", stats.irq_masked_deferred); + MK_STATS_VALUE("irq.stale", stats.irq_stale); + MK_STATS_VALUE("irq.dispatch_failed", stats.irq_dispatch_failed); + MK_STATS_VALUE("irq.saturated", stats.irq_saturated); + MK_STATS_VALUE("irq.slot_active", stats.irq_active); + MK_STATS_VALUE("irq.slot_pending", stats.irq_pending); + MK_STATS_VALUE("irq.slot_masked", stats.irq_masked); + MK_STATS_VALUE("irq.slot_consuming", stats.irq_consuming); +#undef MK_STATS_VALUE + seq_printf(sf, "irq.pending_total %llu\n", + (unsigned long long)stats.irq_pending_total); + return 0; +} + /* Root-level device_tree attribute - shows current baseline pool from kernel structures */ static int root_device_tree_seq_show(struct seq_file *sf, void *v) { @@ -373,6 +523,10 @@ static const struct kernfs_ops mk_status_ops = { .seq_show = status_seq_show, }; +static const struct kernfs_ops mk_stats_ops = { + .seq_show = stats_seq_show, +}; + /* Root-level device_tree operations */ static const struct kernfs_ops mk_root_device_tree_ops = { .seq_show = root_device_tree_seq_show, @@ -407,6 +561,13 @@ static int mk_create_instance_files(struct mk_instance *instance) pr_err("Failed to create status file for instance %s\n", instance->name); return PTR_ERR(kn); } + kn = __kernfs_create_file(instance->kn, "stats", 0444, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, + &mk_stats_ops, instance, NULL, NULL); + if (IS_ERR(kn)) { + pr_err("Failed to create stats file for instance %s\n", instance->name); + return PTR_ERR(kn); + } kn = __kernfs_create_file(instance->kn, "device_tree", 0444, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0,