diff --git a/Documentation/ABI/testing/multikernel b/Documentation/ABI/testing/multikernel new file mode 100644 index 00000000000000..67986e09ea88bd --- /dev/null +++ b/Documentation/ABI/testing/multikernel @@ -0,0 +1,78 @@ +What: /sys/fs/multikernel/instances//stats +Date: August 2026 +Contact: Nikolay Nikolaev +Description: + Read-only reliability snapshot for one multikernel instance. The + format is one ``key value`` pair per line. The first line is always + ``stats_version 1``. Versioned formats are append-only; readers must + ignore unknown keys. + + The snapshot is observational, not atomic with concurrent producers. + Cumulative transport counters are unsigned 32-bit event counts and + wrap modulo 2^32. They reset when the shared buffer is initialized + for a new ``spawn_epoch`` and cannot be reset through this file. + Compare deltas only between samples with the same nonzero epoch. + Gauges are unsigned point-in-time counts and have no reset operation. + + Metadata keys (class; unit; meaning): + + ``stats_version`` (metadata; version; this file format, currently 1), + ``abi_version`` (metadata; version; exact shared transport ABI), + ``snapshot_atomic`` (metadata; boolean; always 0), + ``transport_counter_bits`` (metadata; bits; always 32), + ``transport_available`` (gauge; boolean; shared buffer is mapped), and + ``spawn_epoch`` (metadata; launch identifier; changes on each start). + + ``abi_version`` identifies the exact private kernel-to-kernel transport + contract used by the running instance. Exposing it here supports + diagnostics; it is not a stable userspace ABI or a compatibility promise + between transport versions. + + Ordered IPI keys (class; unit; increment or sampled condition; reset): + + ``ipi.producer_contention`` (counter; sends; producer gate observed + busy or recursively owned; new epoch), ``ipi.full_failures`` (counter; + sends; head slot prevented publication; new epoch), + ``ipi.invalid_state`` (counter; observations; impossible slot/cursor + state; new epoch), ``ipi.cancelled_writes`` (counter; writes; parked + producer recovered as CANCELLED; new epoch), ``ipi.gate_busy`` (gauge; + gates; producer gate nonzero; sampled), ``ipi.slot_writing`` (gauge; + slots; WRITING; sampled), ``ipi.slot_ready`` (gauge; slots; READY; + sampled), ``ipi.slot_consuming`` (gauge; slots; CONSUMING; sampled), + and ``ipi.slot_cancelled`` (gauge; slots; CANCELLED; sampled). + + Direct reply keys (class; unit; increment or sampled condition; reset): + + ``reply.late_replies`` (counter; replies; stale exact-token claim or + completion was rejected; new epoch), ``reply.cancelled_slots`` + (counter; slots; timeout + cancelled RESERVED/WRITING; new epoch), ``reply.atomic_timeouts`` + (counter; waits; bounded atomic wait expired in cancellable + RESERVED/WRITING or indeterminate EXECUTING state; new epoch), + ``reply.indeterminate_timeouts`` (counter; waits; subset where + EXECUTING won and changed to COMMITTED; new epoch), + ``reply.occupied_failures`` (counter; + reservations; no FREE slot; new epoch), ``reply.slot_busy`` (gauge; + slots; state is not FREE; sampled), ``reply.slot_reserved`` (gauge; + slots; RESERVED; sampled), ``reply.slot_writing`` (gauge; slots; + WRITING; sampled), ``reply.slot_executing`` (gauge; slots; EXECUTING; + sampled), ``reply.slot_committed`` (gauge; slots; COMMITTED; sampled), + ``reply.slot_ready`` (gauge; slots; READY; sampled), and + ``reply.slot_abandoned`` (gauge; slots; ABANDONED; sampled). + + Pending IRQ keys (class; unit; increment or sampled condition; reset): + + ``irq.recorded`` (counter; IRQs; pending count recorded; new epoch), + ``irq.coalesced`` (counter; IRQs; recorded onto an already-pending + slot; new epoch), ``irq.masked_deferred`` (counter; IRQs; delivery + deferred while logically masked; new epoch), ``irq.stale`` (counter; + observations; epoch/generation/route validation rejected work; new + epoch), ``irq.dispatch_failed`` (counter; IRQs; validated local dispatch + failed; new epoch), ``irq.saturated`` (counter; IRQs; pending count hit + its representable limit; new epoch), ``irq.slot_active`` (gauge; slots; + nonzero generation; sampled), ``irq.slot_pending`` (gauge; slots; + nonzero pending count; sampled), ``irq.slot_masked`` (gauge; slots; + MASKED set; sampled), ``irq.slot_consuming`` (gauge; slots; CONSUMING + set; sampled), and ``irq.pending_total`` (gauge; IRQs; sum of pending + counts across active slots; sampled). +Users: multikernel management and diagnostic tools diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index a2ec8d56ca1d06..9fe81edde86e26 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -93,6 +93,19 @@ Phase 1: Instance Creation (Automatic from DTB) cat /sys/fs/multikernel/instances/web-server/status # Output: ready + cat /sys/fs/multikernel/instances/web-server/stats + + ``stats`` is a read-only, append-only key/value interface. Version 1 + starts with ``stats_version 1`` and reports the shared ordered-IPI, direct + reply, and pending-IRQ transport counters for the instance. New keys may + be appended; readers must ignore keys they do not understand. + + The output is an observational snapshot rather than an atomic transaction. + Shared transport counters are unsigned 32-bit values and may wrap. They + are reinitialized for a new ``spawn_epoch``, are not writable or resettable + through this interface, and must only be compared as modulo-32-bit deltas + between samples carrying the same nonzero epoch. + # View instance device tree cat /sys/fs/multikernel/instances/web-server/device_tree_source # Output: DTS format showing the instance configuration @@ -122,6 +135,23 @@ Phase 2: Kernel Loading (Kexec Integration) - Restore the instance's DTB and recreate the instance structure - Re-reserve the same memory and CPU resources +SR-IOV Assignment Boundary +========================== + +SR-IOV assignment is intended for cooperative spawned kernels. Filtering +configuration-space access and enumerating only assigned BDFs prevents +accidental access by those kernels; it is not a security boundary against a +privileged kernel that deliberately issues configuration cycles or maps host +physical windows. The host-owned IOMMU domain is the boundary that constrains +device-initiated DMA. + +The host fails assignment closed unless it can establish the complete device +lifecycle: the device is an SR-IOV VF in a singleton IOMMU group, reset is +available, MSI or MSI-X programming remains isolated and host-owned, and one +coherent host-owned IOMMU domain covers the assigned memory. Reset, interrupt, +or DMA teardown uncertainty leaves the instance failed rather than returning +the VF to use. + Device Tree Format ================== @@ -213,3 +243,6 @@ The new kernfs interface has the following restrictions: - **No direct DTB upload to instances**: Instances don't have writable ``device_tree`` files - **Centralized DTB management**: All instances must be created via the root ``device_tree`` file - **Read-only instance files**: All instance attributes are read-only for consistency +- **Host control CPU**: Logical CPU 0 is the PCI forwarding and control CPU and + must remain online while assigned devices are active. Selecting another + control CPU is a future policy extension. diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9bea5a1e2c52cb..8553df38269bc4 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -379,7 +379,13 @@ xloadflags: #define XLF56 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 +#ifdef CONFIG_MULTIKERNEL +# define XLF13 XLF_MULTIKERNEL_IPI_V7 +#else +# define XLF13 0 +#endif + + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF13 cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index da9d37264e410d..d69471790823ef 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -10,6 +10,9 @@ #ifndef __ASSEMBLY__ +#include +#include +#include #include #include #include @@ -54,7 +57,7 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) * from a page written by the host while running on CPUs parked by (possibly * differently built) spawn kernels. * - * The fields fall into two classes that must not be mixed up: + * The fields fall into three classes that must not be mixed up: * * - Anchor fields (self_phys, park_phys, park_cr3, ctrl_phys, * ctrl_size): the context's own identity, written once when the @@ -62,9 +65,13 @@ static inline int arch_cpu_from_physical_id(u64 phys_id) * CPU on halt or offline, so they must stay valid for the context's * whole lifetime. * - * - Dispatch fields (everything else): the wake mailbox, rewritten for - * every publication and staged into registers by the CPU that claims - * it. Reparking gets its own repark_* dispatch fields precisely so a + * - Primary boot data (bp and the calibration values appended after it): + * written before the boot CPU is released and consumed while that kernel + * initializes. Secondary and repark publications do not rewrite it. + * + * - Dispatch fields (the remaining fixed-size fields): the wake mailbox, + * rewritten for every publication and staged into registers by the CPU + * that claims it. Reparking gets its own repark_* dispatch fields so a * repark publication never overwrites the anchor: the two used to * share fields, and a repark left the anchor pointing at another * kernel's park area, which triple-faulted the next halt. @@ -91,10 +98,23 @@ struct mk_spawn_context { u32 flags; /* MK_SPAWN_F_* flags */ u32 ready; /* Signal flag */ u32 reserved; /* Padding for alignment */ - /* Variable-size struct last - size depends on kernel config */ + /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ + /* Optional boot data belongs after boot_params, in the zeroed tail. */ + unsigned long boot_lps; /* Host delay loops per second */ + unsigned long boot_cpu_khz; /* Host CPU frequency calibration */ + unsigned long boot_tsc_khz; /* Host TSC frequency calibration */ + unsigned long boot_apic_hz; /* Host local APIC timer frequency */ + u64 abi_magic; /* Validated context producer */ } __aligned(PAGE_SIZE); +static_assert(offsetof(struct mk_spawn_context, bp) == 144); +static_assert(offsetof(struct mk_spawn_context, boot_lps) == + 144 + sizeof(struct boot_params)); +static_assert(offsetof(struct mk_spawn_context, abi_magic) == + 144 + sizeof(struct boot_params) + 4 * sizeof(unsigned long)); +static_assert(sizeof(struct mk_spawn_context) == 2 * PAGE_SIZE); + /* Pool park loop code, copied by the host into per-instance park pages */ extern char mk_pool_park_start[]; extern char mk_pool_park_end[]; @@ -139,7 +159,8 @@ void mk_set_spawn_context(struct mk_spawn_context *ctx, int mk_spawn_cpu(struct mk_instance *instance, int cpu, struct mk_spawn_context *ctx); -/* Initialize boot context tracking in spawn kernel */ +/* Validate and initialize boot context tracking in spawn kernel */ +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys); void mk_init_boot_context(phys_addr_t ctx_phys); /* Identity page table and trampoline setup */ @@ -161,6 +182,11 @@ int multikernel_wakeup_secondary_cpu_64(u32 apicid, unsigned long start_eip, int multikernel_restore_ap(unsigned int cpu, unsigned long cr3, unsigned long gs_base, unsigned long stack, unsigned long entry); +#if defined(CONFIG_MULTIKERNEL) && defined(CONFIG_PCI) +void __init x86_multikernel_pci_platform_init(void); +#else +static inline void x86_multikernel_pci_platform_init(void) { } +#endif #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index c70be687a3ecc7..062a70099e77d3 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -25,6 +25,12 @@ #define XLF_5LEVEL (1<<5) #define XLF_5LEVEL_ENABLED (1<<6) #define XLF_MEM_ENCRYPTION (1<<7) +#define XLF_MULTIKERNEL_IPI_V2 0x0100 +#define XLF_MULTIKERNEL_IPI_V3 0x0200 +#define XLF_MULTIKERNEL_IPI_V4 0x0400 +#define XLF_MULTIKERNEL_IPI_V5 0x0800 +#define XLF_MULTIKERNEL_IPI_V6 0x1000 +#define XLF_MULTIKERNEL_IPI_V7 0x2000 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 0759f9d6b64c23..4f840e8ce74221 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -611,6 +611,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel, .buf_max = ULONG_MAX, .top_down = true }; header = (struct setup_header *)(kernel + setup_hdr_offset); + if (image->type == KEXEC_TYPE_MULTIKERNEL && + !(header->xloadflags & XLF_MULTIKERNEL_IPI_V7)) { + pr_err("Loaded kernel lacks the required shared transport layout\n"); + return ERR_PTR(-EPROTONOSUPPORT); + } setup_sects = header->setup_sects; if (setup_sects == 0) setup_sects = 4; diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index e8279270018c30..cbc449fce37078 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -60,6 +60,12 @@ struct elf_kernel_info { unsigned long reloc_size; /* Size of relocation data */ }; +struct mk_elf_note_desc { + u64 entry; + u32 ipi_abi_version; + u32 reserved; +}; + /* * Find multikernel entry point from PT_NOTE section. * Looks for note with name "Linux" and type 0x4d4b ('MK'). @@ -93,12 +99,20 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, if (nhdr->n_type == 0x4d4b && nhdr->n_namesz == 6 && - nhdr->n_descsz == sizeof(u64) && + nhdr->n_descsz == sizeof(struct mk_elf_note_desc) && !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) { - u64 entry = *(u64 *)(ptr + sizeof(*nhdr) + - ALIGN(nhdr->n_namesz, 4)); - pr_info("multikernel: entry=0x%llx\n", entry); - return entry; + const struct mk_elf_note_desc *desc; + + desc = ptr + sizeof(*nhdr) + + ALIGN(nhdr->n_namesz, 4); + if (desc->ipi_abi_version != MK_IPI_ABI_VERSION) { + pr_err("multikernel IPI ABI %u is not supported\n", + desc->ipi_abi_version); + return 0; + } + pr_info("multikernel: entry=0x%llx, IPI ABI=%u\n", + desc->entry, desc->ipi_abi_version); + return desc->entry; } ptr += note_size; } diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index 95d2cd2ccf74f5..a1d1b8749112a6 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -14,8 +14,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -28,6 +30,35 @@ extern pmd_t *populate_extra_pmd(unsigned long vaddr); extern unsigned long orig_boot_params; #ifdef CONFIG_MULTIKERNEL +static unsigned long multikernel_cpu_khz; +static unsigned long multikernel_tsc_khz; + +static unsigned long multikernel_calibrate_cpu(void) +{ + return multikernel_cpu_khz; +} + +static unsigned long multikernel_calibrate_tsc(void) +{ + return multikernel_tsc_khz; +} + +static void __init multikernel_setup_calibration(void) +{ + phys_addr_t ctx_phys = orig_boot_params - + offsetof(struct mk_spawn_context, bp); + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx || !ctx->boot_tsc_khz) + return; + + multikernel_tsc_khz = ctx->boot_tsc_khz; + multikernel_cpu_khz = ctx->boot_cpu_khz ?: ctx->boot_tsc_khz; + x86_platform.calibrate_cpu = multikernel_calibrate_cpu; + x86_platform.calibrate_tsc = multikernel_calibrate_tsc; + setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); +} + /* * Custom wakeup for multikernel spawn kernels. * Uses shared spawn table instead of realmode trampoline. @@ -105,6 +136,10 @@ static void __init multikernel_parse_smp_config(void) */ apic_update_callback(wakeup_secondary_cpu_64, multikernel_wakeup_cpu); } +#else +static inline void multikernel_setup_calibration(void) +{ +} #endif /* CONFIG_MULTIKERNEL */ void __init x86_early_init_platform_quirks(void) @@ -135,6 +170,8 @@ void __init x86_early_init_platform_quirks(void) x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; break; case X86_SUBARCH_MULTIKERNEL: + multikernel_setup_calibration(); + x86_multikernel_pci_platform_init(); x86_platform.legacy.devices.pnpbios = 0; x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; x86_platform.legacy.rtc = 0; @@ -175,7 +212,9 @@ void __init x86_early_init_platform_quirks(void) * the PIT - which belongs to the host - and then request * legacy IRQ0, which can never reach an instance CPU that * has neither a PIC nor an IO-APIC. Ticks come from the - * local APIC timer via setup_percpu_clockev() instead. + * local APIC timer initialized by setup_percpu_clockev(). + * Keeping global_clock_event unset bypasses LAPIC timer + * verification, whose fallback path requires legacy IRQ0. */ x86_init.timers.timer_init = x86_init_noop; x86_init.timers.wallclock_init = x86_init_noop; diff --git a/arch/x86/multikernel/Makefile b/arch/x86/multikernel/Makefile index 331bd895af1f7b..1fcee985bcbfe8 100644 --- a/arch/x86/multikernel/Makefile +++ b/arch/x86/multikernel/Makefile @@ -4,3 +4,4 @@ # obj-y += spawn.o direct_boot.o head_64.o +obj-$(CONFIG_PCI) += pci.o diff --git a/arch/x86/multikernel/head_64.S b/arch/x86/multikernel/head_64.S index 3784147fd82f62..b726e74ff3e402 100644 --- a/arch/x86/multikernel/head_64.S +++ b/arch/x86/multikernel/head_64.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -273,4 +274,6 @@ SYM_CODE_END(multikernel_secondary_startup) 1: .asciz "Linux" 2: .balign 4 3: .quad multikernel_startup_64 - __START_KERNEL_map + .long MK_IPI_ABI_VERSION + .long 0 4: .balign 4 diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c new file mode 100644 index 00000000000000..931d8d7bf44477 --- /dev/null +++ b/arch/x86/multikernel/pci.c @@ -0,0 +1,963 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * x86 PCI support for multikernel spawn kernels. + * + * Spawn kernels discover only assigned BDFs and proxy all configuration + * accesses to the host kernel. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static bool mk_pci_roots_ready; +static atomic64_t mk_pci_request_id = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_count = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_total_ns = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_max_ns = ATOMIC64_INIT(0); +#define MK_PCI_RESET_TIMEOUT_MS 70000 +#ifdef CONFIG_PCI_MSI +static void mk_pci_forward_irq_set_mask(struct irq_data *data, bool masked); +static void mk_pci_irq_mailbox_requeue(struct mk_irq_mailbox *mailbox, + unsigned int slot); + +static void mk_pci_forward_irq_noop(struct irq_data *data) +{ +} + +static void mk_pci_forward_irq_mask(struct irq_data *data) +{ + mk_pci_forward_irq_set_mask(data, true); +} + +static void mk_pci_forward_irq_unmask(struct irq_data *data) +{ + mk_pci_forward_irq_set_mask(data, false); +} + +static void mk_pci_forward_irq_write_msg(struct irq_data *data, + struct msi_msg *msg) +{ +} + +static struct irq_chip mk_pci_forward_irq_chip = { + .name = "multikernel-pci-forward", + .irq_ack = mk_pci_forward_irq_noop, + /* Host process-context lifecycle owns physical mask state. */ + .irq_mask = mk_pci_forward_irq_mask, + .irq_unmask = mk_pci_forward_irq_unmask, + .irq_write_msi_msg = mk_pci_forward_irq_write_msg, +}; + +static void mk_pci_bind_local_irqs(unsigned int irq, unsigned int count) +{ + unsigned int i; + + for (i = 0; i < count; i++) + irq_set_chip_and_handler(irq + i, &mk_pci_forward_irq_chip, + handle_edge_irq); +} + +static bool mk_pci_forward_irq_matches(u32 irq_number, u32 vector, + u32 device_id, + struct irq_data **irq_data) +{ + struct irq_data *data = irq_get_irq_data(irq_number); + struct msi_desc *desc; + struct pci_dev *dev; + unsigned int offset; + + if (!data) + return false; + desc = irq_data_get_msi_desc(data); + if (!desc || vector < desc->msi_index) + return false; + + dev = msi_desc_to_pci_dev(desc); + offset = vector - desc->msi_index; + if (offset >= desc->nvec_used || desc->irq + offset != irq_number || + pci_domain_nr(dev->bus) != MK_PCI_IRQ_ID_DOMAIN(device_id) || + dev->bus->number != MK_PCI_IRQ_ID_BUS(device_id) || + dev->devfn != MK_PCI_IRQ_ID_DEVFN(device_id)) + return false; + + *irq_data = data; + return true; +} + +static void mk_pci_forward_irq_set_mask(struct irq_data *data, bool masked) +{ + struct msi_desc *desc = irq_data_get_msi_desc(data); + struct mk_irq_mailbox *mailbox; + struct pci_dev *dev; + u32 device_id; + u32 generation; + u32 vector; + unsigned int slot; + u64 epoch; + + if (!desc || !root_instance || !root_instance->ipi_data) + return; + dev = msi_desc_to_pci_dev(desc); + generation = READ_ONCE(dev->multikernel_msi_generation); + epoch = READ_ONCE(root_instance->ipi_data->spawn_epoch); + device_id = MK_PCI_IRQ_ID(pci_domain_nr(dev->bus), dev->bus->number, + dev->devfn); + vector = desc->msi_index + data->irq - desc->irq; + mailbox = &root_instance->ipi_data->irq_mailbox; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry = &mailbox->entries[slot]; + u64 old, new; + u32 slot_generation; + + old = atomic64_read_acquire(&entry->pending_generation); + if (!mk_irq_mailbox_generation(old) || + READ_ONCE(entry->lifecycle_epoch) != epoch || + READ_ONCE(entry->lifecycle_generation) != generation || + READ_ONCE(entry->device_id) != device_id || + READ_ONCE(entry->local_irq) != data->irq || + READ_ONCE(entry->vector) != vector) + continue; + slot_generation = mk_irq_mailbox_generation(old); + for (;;) { + new = masked ? old | MK_IRQ_MAILBOX_MASKED : + old & ~MK_IRQ_MAILBOX_MASKED; + if (new == old || + atomic64_cmpxchg(&entry->pending_generation, + old, new) == old) + break; + old = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(old) != slot_generation) + return; + } + if (!masked && mk_irq_mailbox_pending(new)) { + mk_phys_cpu_t target; + + mk_pci_irq_mailbox_requeue(mailbox, slot); + target = arch_cpu_physical_id(smp_processor_id()); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } + return; + } +} + +static void mk_pci_irq_mailbox_requeue(struct mk_irq_mailbox *mailbox, + unsigned int slot) +{ + atomic64_or(BIT_ULL(slot & 63), + &mailbox->pending_bitmap[slot / 64]); +} + +static void mk_pci_irq_mailbox_drain_slot(struct mk_shared_data *shared, + unsigned int slot) +{ + struct mk_irq_mailbox *mailbox = &shared->irq_mailbox; + struct mk_irq_mailbox_entry *entry = &mailbox->entries[slot]; + struct irq_data *irq_data; + struct pci_dev *dev; + u64 lifecycle_epoch; + u64 token, base, claim; + u32 lifecycle_generation; + u32 pending; + u32 device_id; + u32 local_irq; + u16 vector; + + token = atomic64_read_acquire(&entry->pending_generation); + pending = mk_irq_mailbox_pending(token); + if (!mk_irq_mailbox_generation(token) || !pending) + return; + if (mk_irq_mailbox_consuming(token)) { + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + lifecycle_epoch = READ_ONCE(entry->lifecycle_epoch); + lifecycle_generation = READ_ONCE(entry->lifecycle_generation); + device_id = READ_ONCE(entry->device_id); + local_irq = READ_ONCE(entry->local_irq); + vector = READ_ONCE(entry->vector); + base = token & ~MK_IRQ_MAILBOX_PENDING_MASK; + + if (!local_irq || !root_instance || !root_instance->ipi_data || + lifecycle_epoch != READ_ONCE(shared->spawn_epoch)) + goto stale; + if (mk_irq_mailbox_masked(token)) { + atomic_inc(&mailbox->masked_deferred); + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + claim = base | MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_acquire(&entry->pending_generation, + token, claim) != token) { + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim)) + goto claimed_stale; + if (mk_irq_mailbox_masked(token)) { + atomic_inc(&mailbox->masked_deferred); + goto claimed_defer; + } + if (!mk_pci_forward_irq_matches(local_irq, vector, device_id, + &irq_data)) + goto claimed_stale; + dev = msi_desc_to_pci_dev(irq_data_get_msi_desc(irq_data)); + if (lifecycle_epoch != READ_ONCE(entry->lifecycle_epoch) || + lifecycle_generation != + READ_ONCE(entry->lifecycle_generation) || + device_id != READ_ONCE(entry->device_id) || + local_irq != READ_ONCE(entry->local_irq) || + vector != READ_ONCE(entry->vector) || + READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE || + lifecycle_generation != + READ_ONCE(dev->multikernel_msi_generation) || + irq_data_get_irq_chip(irq_data) != &mk_pci_forward_irq_chip) + goto claimed_stale; + if (pending > 1) + atomic_add(pending - 1, &mailbox->coalesced); + if (generic_handle_irq_safe(local_irq)) + atomic_inc(&mailbox->dispatch_failed); + + for (;;) { + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new = token & ~MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + if (mk_irq_mailbox_pending(new)) + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + +claimed_defer: + for (;;) { + u32 new_pending; + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new_pending = mk_irq_mailbox_pending(token); + if (new_pending < MK_IRQ_MAILBOX_PENDING_MASK) + new_pending++; + new = mk_irq_mailbox_token(mk_irq_mailbox_generation(token), + new_pending) | + (token & MK_IRQ_MAILBOX_MASKED); + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + mk_pci_irq_mailbox_requeue(mailbox, slot); + if (!mk_irq_mailbox_masked(new)) { + mk_phys_cpu_t target = + arch_cpu_physical_id(smp_processor_id()); + + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } + return; + } + +claimed_stale: + atomic_inc(&mailbox->stale); + for (;;) { + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new = token & ~MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + if (mk_irq_mailbox_pending(new)) + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + return; + +stale: + if (atomic64_cmpxchg_acquire(&entry->pending_generation, + token, base) != token) + mk_pci_irq_mailbox_requeue(mailbox, slot); + atomic_inc(&mailbox->stale); +} + +void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) +{ + unsigned int word; + + if (!shared) + return; + for (word = 0; word < MK_IRQ_MAILBOX_WORDS; word++) { + atomic64_t *pending_bitmap; + unsigned long bits; + + pending_bitmap = &shared->irq_mailbox.pending_bitmap[word]; + bits = atomic64_xchg_acquire(pending_bitmap, 0); + + while (bits) { + unsigned int bit = __ffs(bits); + + bits &= bits - 1; + mk_pci_irq_mailbox_drain_slot(shared, word * 64 + bit); + } + } +} + +static int mk_pci_send_irq_request(struct mk_pci_irq_request *request) +{ + struct mk_reply_handle reply; + s32 status; + int ret; + + if (WARN_ON_ONCE(irqs_disabled() || !in_task())) + return -EWOULDBLOCK; + might_sleep(); + if (!request->lifecycle_generation || !root_instance || + !root_instance->ipi_data) + return -EINVAL; + request->lifecycle_epoch = + READ_ONCE(root_instance->ipi_data->spawn_epoch); + if (!request->lifecycle_epoch) + return -EPROTO; + + request->request_id = atomic64_inc_return(&mk_pci_request_id); + request->sender_instance_id = root_instance ? root_instance->id : -1; + ret = mk_reply_reserve(root_instance->ipi_data, MK_REPLY_PCI_IRQ, + request->request_id, &reply); + if (ret) + return ret; + request->reply_slot = reply.slot; + request->reply_generation = reply.generation; + + ret = mk_send_message(0, MK_MSG_PCI, MK_PCI_IRQ_REQUEST, + request, sizeof(*request)); + if (ret) { + mk_reply_release(root_instance->ipi_data, &reply); + return ret; + } + + ret = mk_reply_wait(root_instance->ipi_data, &reply, 1000, + &status, NULL); + return ret ? ret : status; +} + +bool mk_pci_msi_controlled(struct pci_dev *dev) +{ + return mk_pci_controlled(dev); +} + +static int mk_pci_msi_teardown_generation(struct pci_dev *dev, + u32 generation); + +int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) +{ + u32 generation; + u8 state; + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_SETUP, + .nr_vectors = nvec, + .msix = type == PCI_CAP_ID_MSIX, + }; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + state = READ_ONCE(dev->multikernel_msi_state); + if (state != MK_PCI_MSI_IDLE && state != MK_PCI_MSI_FAILED) + return -EBUSY; + generation = READ_ONCE(dev->multikernel_msi_generation) + 1; + if (!generation) + generation = 1; + WRITE_ONCE(dev->multikernel_msi_generation, generation); + request.lifecycle_generation = generation; + ret = mk_pci_send_irq_request(&request); + if (ret) { + if (ret == -EINPROGRESS) { + int cleanup_ret; + + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : + MK_PCI_MSI_IDLE); + } + return ret; + } + WRITE_ONCE(dev->multikernel_msi_nvec, nvec); + WRITE_ONCE(dev->multikernel_msi_msix, type == PCI_CAP_ID_MSIX); + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_PREPARED); + return 0; +} + +static int mk_pci_msi_bind(struct pci_dev *dev, unsigned int index, + unsigned int irq, unsigned int nvec, bool msix, + u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_BIND, + .vector = index, + .nr_vectors = nvec, + .msix = msix, + .local_irq = irq, + .lifecycle_generation = generation, + }; + unsigned int count = msix ? 1 : nvec; + unsigned int i; + int ret; + + /* The local descriptor must be dispatchable before the host unmasks. */ + mk_pci_bind_local_irqs(irq, count); + ret = mk_pci_send_irq_request(&request); + if (ret) + return ret; + /* Publish the irqdesc's initial logical mask state into the token. */ + for (i = 0; i < count; i++) { + struct irq_data *data = irq_get_irq_data(irq + i); + + if (!data) + return -EINVAL; + mk_pci_forward_irq_set_mask(data, + irqd_irq_disabled(data) || + irqd_irq_masked(data)); + } + return 0; +} + +static int mk_pci_msi_teardown_generation(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_TEARDOWN, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_commit(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_COMMIT, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_host_activate(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_ACTIVATE, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_restore_begin(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_RESTORE_BEGIN, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_bind_all(struct pci_dev *dev, u32 generation) +{ + struct msi_desc *desc; + unsigned int expected = READ_ONCE(dev->multikernel_msi_nvec); + unsigned int next = 0; + bool msix = READ_ONCE(dev->multikernel_msi_msix); + int ret; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = msix ? 1 : desc->nvec_used; + + if (next >= expected || desc->msi_index != next || !count || + count > expected - next) + return -EINVAL; + ret = mk_pci_msi_bind(dev, desc->msi_index, desc->irq, + desc->nvec_used, msix, generation); + if (ret) + return ret; + next += count; + } + if (next != expected) + return -EINVAL; + return mk_pci_msi_commit(dev, generation); +} + +static int mk_pci_msi_mask_mailbox(struct pci_dev *dev) +{ + struct msi_desc *desc; + unsigned long deadline; + u64 epoch; + u32 device_id; + u32 generation; + unsigned int slot; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = desc->pci.msi_attrib.is_msix ? + 1 : desc->nvec_used; + unsigned int i; + + for (i = 0; i < count; i++) { + struct irq_data *data = irq_get_irq_data(desc->irq + i); + + if (data) + mk_pci_forward_irq_set_mask(data, true); + } + } + + epoch = READ_ONCE(root_instance->ipi_data->spawn_epoch); + generation = READ_ONCE(dev->multikernel_msi_generation); + device_id = MK_PCI_IRQ_ID(pci_domain_nr(dev->bus), dev->bus->number, + dev->devfn); + deadline = jiffies + msecs_to_jiffies(1000); + for (;;) { + bool consuming = false; + + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry = + &root_instance->ipi_data->irq_mailbox.entries[slot]; + u64 token; + + token = atomic64_read_acquire(&entry->pending_generation); + + if (mk_irq_mailbox_generation(token) && + mk_irq_mailbox_consuming(token) && + READ_ONCE(entry->lifecycle_epoch) == epoch && + READ_ONCE(entry->lifecycle_generation) == generation && + READ_ONCE(entry->device_id) == device_id) { + consuming = true; + break; + } + } + if (!consuming) + break; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(50, 100); + } + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = desc->pci.msi_attrib.is_msix ? + 1 : desc->nvec_used; + unsigned int i; + + for (i = 0; i < count; i++) + synchronize_irq(desc->irq + i); + } + return 0; +} + +int mk_pci_msi_activate(struct pci_dev *dev) +{ + u32 generation; + int cleanup_ret; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_PREPARED) + return -EIO; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_bind_all(dev, generation); + if (ret) { + pr_err("Failed to activate host-owned MSI vectors for %s: %d\n", + pci_name(dev), ret); + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; + } + + /* The guest must be able to consume the first edge before host unmask. */ + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_ACTIVE); + ret = mk_pci_msi_host_activate(dev, generation); + if (ret) { + cleanup_ret = mk_pci_msi_mask_mailbox(dev); + if (!cleanup_ret) + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; + } + return 0; +} + +int mk_pci_msi_restore(struct pci_dev *dev) +{ + u32 generation; + int cleanup_ret; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE) + return -EIO; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_mask_mailbox(dev); + if (ret) { + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_FAILED); + return ret; + } + ret = mk_pci_msi_restore_begin(dev, generation); + if (!ret) + ret = mk_pci_msi_bind_all(dev, generation); + if (!ret) + ret = mk_pci_msi_host_activate(dev, generation); + if (ret) { + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_FAILED); + if (cleanup_ret) + pr_err("Failed to quiesce host-owned MSI after restore failure for %s: %d\n", + pci_name(dev), cleanup_ret); + return ret; + } + return 0; +} + +int mk_pci_msi_teardown(struct pci_dev *dev) +{ + u32 generation; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) == MK_PCI_MSI_IDLE) + return 0; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_mask_mailbox(dev); + if (!ret) + ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; +} +#endif /* CONFIG_PCI_MSI */ + +bool mk_pci_controlled(struct pci_dev *dev) +{ + return root_instance && root_instance->id != 0 && + mk_pci_get_assigned_identity_bdf(pci_domain_nr(dev->bus), + dev->bus->number, dev->devfn, + NULL, NULL); +} + +int mk_pci_reset_flr(struct pci_dev *dev) +{ + struct mk_pci_reset_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + }; + struct mk_reply_handle reply; + u32 generation; + s32 status; + int ret; + + if (WARN_ON_ONCE(irqs_disabled() || !in_task())) + return -EWOULDBLOCK; + might_sleep(); + if (!root_instance || !root_instance->ipi_data) + return -ENODEV; + request.lifecycle_epoch = + READ_ONCE(root_instance->ipi_data->spawn_epoch); + if (!request.lifecycle_epoch) + return -EPROTO; + + generation = READ_ONCE(dev->multikernel_reset_generation) + 1; + if (!generation) + generation = 1; + WRITE_ONCE(dev->multikernel_reset_generation, generation); + request.reset_generation = generation; + request.request_id = atomic64_inc_return(&mk_pci_request_id); + request.sender_instance_id = root_instance->id; + ret = mk_reply_reserve(root_instance->ipi_data, MK_REPLY_PCI_RESET, + request.request_id, &reply); + if (ret) + return ret; + request.reply_slot = reply.slot; + request.reply_generation = reply.generation; + + ret = mk_send_message(0, MK_MSG_PCI, MK_PCI_RESET_REQUEST, + &request, sizeof(request)); + if (ret) { + mk_reply_release(root_instance->ipi_data, &reply); + return ret; + } + + ret = mk_reply_wait(root_instance->ipi_data, &reply, + MK_PCI_RESET_TIMEOUT_MS, + &status, NULL); + return ret ? ret : status; +} + +static void mk_pci_record_latency(u64 start) +{ + u64 elapsed = ktime_get_mono_fast_ns() - start; + u64 old_max = atomic64_read(&mk_pci_cfg_max_ns); + + atomic64_inc(&mk_pci_cfg_count); + atomic64_add(elapsed, &mk_pci_cfg_total_ns); + while (elapsed > old_max) { + u64 previous = atomic64_cmpxchg(&mk_pci_cfg_max_ns, old_max, + elapsed); + + if (previous == old_max) + break; + old_max = previous; + } +} + +static int mk_pci_remote_config(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + bool write, u32 *value) +{ + struct mk_pci_cfg_request request = { + .request_id = atomic64_inc_return(&mk_pci_request_id), + .sender_instance_id = root_instance ? root_instance->id : -1, + .domain = domain, + .bus = bus, + .devfn = devfn, + .reg = where, + .len = size, + .write = write, + .value = *value, + }; + struct mk_reply_handle reply; + s32 status; + u32 response_value; + u64 start = ktime_get_mono_fast_ns(); + int ret; + + ret = mk_reply_reserve(root_instance->ipi_data, MK_REPLY_PCI_CFG, + request.request_id, &reply); + if (ret) + goto out_error; + request.reply_slot = reply.slot; + request.reply_generation = reply.generation; + + ret = mk_send_message(0, MK_MSG_PCI, MK_PCI_CFG_REQUEST, + &request, sizeof(request)); + if (ret) { + mk_reply_release(root_instance->ipi_data, &reply); + goto out_error; + } + ret = mk_reply_wait_atomic(root_instance->ipi_data, &reply, 20000, + &status, &response_value); + if (ret) + goto out_error; + if (status < 0) { + ret = status; + goto out_error; + } + if (!write) + *value = response_value; + mk_pci_record_latency(start); + return status; + +out_error: + if (ret < 0) { + pr_err_ratelimited("Multikernel PCI config request timed out or failed to send: %d\n", + ret); + } + return PCIBIOS_SET_FAILED; +} + +static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, + u32 *value) +{ + u32 identity; + u32 mask; + + if (where < PCI_VENDOR_ID || where + size > PCI_COMMAND) + return false; + + identity = vendor | (u32)device << 16; + mask = size == sizeof(identity) ? ~0U : (1U << (size * 8)) - 1; + *value = (identity >> (where * 8)) & mask; + return true; +} + +static int mk_pci_raw_read(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + u32 *value) +{ + u16 vendor, device; + + if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, &vendor, + &device)) { + *value = ~0U; + return PCIBIOS_DEVICE_NOT_FOUND; + } + if (mk_pci_identity_read(vendor, device, where, size, value)) + return PCIBIOS_SUCCESSFUL; + + return mk_pci_remote_config(domain, bus, devfn, where, size, false, + value); +} + +static int mk_pci_raw_write(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + u32 value) +{ + if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, NULL, NULL)) + return PCIBIOS_DEVICE_NOT_FOUND; + + return mk_pci_remote_config(domain, bus, devfn, where, size, true, + &value); +} + +static const struct pci_raw_ops mk_pci_filtered_raw_ops = { + .read = mk_pci_raw_read, + .write = mk_pci_raw_write, +}; + +static int __init x86_multikernel_pci_arch_init(void) +{ + if (!root_instance || !root_instance->pci_devices_valid) + return 0; + + raw_pci_ops = &mk_pci_filtered_raw_ops; + raw_pci_ext_ops = &mk_pci_filtered_raw_ops; + mk_pci_roots_ready = true; + pr_notice("Multikernel selected host-mediated PCI config access\n"); + + return 0; +} + +static struct pci_bus * __init mk_pci_get_root(u16 domain, u8 bus_number) +{ + struct resource_entry *window; + struct pci_sysdata *sd; + struct pci_bus *bus; + bool has_busn_res = false; + LIST_HEAD(resources); + + if (domain && !pci_domains_supported) { + pr_err("Multikernel cannot scan PCI root %04x:%02x without domain support\n", + domain, bus_number); + return ERR_PTR(-EOPNOTSUPP); + } + bus = pci_find_bus(domain, bus_number); + if (bus) + return bus; + + sd = kzalloc_obj(*sd, GFP_KERNEL); + if (!sd) + return ERR_PTR(-ENOMEM); + sd->domain = domain; + sd->node = x86_pci_root_bus_node(bus_number); + x86_pci_root_bus_resources(bus_number, &resources); + resource_list_for_each_entry(window, &resources) { + if (window->res->flags & IORESOURCE_BUS) { + has_busn_res = true; + break; + } + } + bus = pci_create_root_bus(NULL, bus_number, &pci_root_ops, sd, + &resources); + if (!bus) { + pci_free_resource_list(&resources); + kfree(sd); + return ERR_PTR(-ENOMEM); + } + if (!has_busn_res) { + if (!pci_bus_insert_busn_res(bus, bus_number, bus_number)) { + pci_remove_root_bus(bus); + kfree(sd); + return ERR_PTR(-EBUSY); + } + } + pr_notice("Multikernel created synthetic PCI root %04x:%02x\n", + domain, bus_number); + return bus; +} + +static int __init x86_multikernel_pci_init(void) +{ + const struct mk_pci_device *device; + struct pci_bus *bus; + struct pci_dev *pdev; + + if (!root_instance) + panic("Multikernel lost restored instance metadata"); + if (!root_instance->pci_device_count) + return 0; + if (!root_instance->pci_devices_valid || !mk_pci_roots_ready) + panic("Multikernel assigned PCI inventory is unavailable"); + + list_for_each_entry(device, &root_instance->pci_devices, list) { + bus = mk_pci_get_root(device->domain, device->bus); + if (IS_ERR(bus)) + panic("Multikernel failed to create synthetic PCI root %04x:%02x: %ld", + device->domain, device->bus, PTR_ERR(bus)); + pdev = pci_scan_single_device(bus, + PCI_DEVFN(device->slot, device->func)); + if (!pdev) + panic("Multikernel failed to enumerate assigned PCI device %04x:%02x:%02x.%x", + device->domain, device->bus, device->slot, + device->func); + pci_bus_add_devices(bus); + } + if (atomic64_read(&mk_pci_cfg_count)) { + u64 count = atomic64_read(&mk_pci_cfg_count); + + pr_notice("Multikernel PCI control plane: %llu config requests, average %llu ns, max %llu ns\n", + count, atomic64_read(&mk_pci_cfg_total_ns) / count, + atomic64_read(&mk_pci_cfg_max_ns)); + } + + /* Suppress legacy bus 0 probing after every assigned root is present. */ + return 0; +} +void __init x86_multikernel_pci_platform_init(void) +{ + pci_probe = PCI_PROBE_NOEARLY; + x86_init.pci.arch_init = x86_multikernel_pci_arch_init; + x86_init.pci.init = x86_multikernel_pci_init; +} diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 55487d9ed5a1b2..c1cbd0d8aa9358 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include #include @@ -44,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -68,6 +71,7 @@ /* Set in spawn kernels: the context this kernel booted from */ static struct mk_spawn_context *mk_boot_context; +static phys_addr_t mk_boot_context_phys; /* * Spawn kernel's own trampoline for secondary CPU wakeup. @@ -81,6 +85,16 @@ static struct mk_spawn_context *mk_boot_context; */ static void *spawn_trampoline_va; static unsigned long spawn_trampoline_phys; +static bool spawn_trampoline_prepared; +static bool spawn_pool_park_prepared; +static bool spawn_park_ready; +static int spawn_park_error; + +bool mk_arch_park_ready(void) +{ + /* Pair with publication after both executable park mappings succeed. */ + return smp_load_acquire(&spawn_park_ready); +} extern char multikernel_relocate_kernel_start[]; extern char multikernel_relocate_kernel_end[]; @@ -447,6 +461,15 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, (unsigned long)instance->trampoline_va, virt_to_phys(instance->trampoline_va), virt_to_phys(instance->park_va)); + instance->spawn_ctx->boot_lps = cpu_data(cpu).loops_per_jiffy; + if (!instance->spawn_ctx->boot_lps) + instance->spawn_ctx->boot_lps = loops_per_jiffy; + instance->spawn_ctx->boot_lps *= HZ; + instance->spawn_ctx->boot_cpu_khz = cpu_khz; + instance->spawn_ctx->boot_tsc_khz = tsc_khz; + instance->spawn_ctx->boot_apic_hz = + (unsigned long)lapic_timer_period * HZ; + instance->spawn_ctx->abi_magic = MK_BOOT_CONTEXT_MAGIC; return mk_spawn_cpu(instance, cpu, instance->spawn_ctx); } @@ -669,18 +692,38 @@ void __init mk_arch_register_cpu(u64 phys_id) topology_register_apic((u32)phys_id, CPU_ACPIID_INVALID, true); } -/* - * Initialize boot context tracking in spawn kernel. - * Called early during spawn kernel boot. - */ -void mk_init_boot_context(phys_addr_t ctx_phys) +static __noreturn void mk_reject_spawn_context(void) +{ + /* + * The host context layout is unknown, so neither its park state nor any + * shared context field is safe to use. Keep this CPU local and inert. An NMI + * can wake HLT, but returns to this loop with maskable interrupts still + * disabled; disable them again before every halt for defense in depth. + */ + for (;;) { + native_irq_disable(); + native_halt(); + } +} + +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys) { struct mk_spawn_context *ctx; + phys_addr_t stamped_phys; + u64 abi_magic; if (!ctx_phys) { pr_err("mk_spawn: Boot context physical address is 0!\n"); - return; + return NULL; + } + if (mk_boot_context) { + if (ctx_phys != mk_boot_context_phys) + mk_reject_spawn_context(); + return mk_boot_context; } + /* Reject an invalid derived address before mapping or dereferencing it. */ + if (!IS_ALIGNED(ctx_phys, PAGE_SIZE)) + mk_reject_spawn_context(); /* * The spawn context is in the multikernel pool which is regular RAM, @@ -697,14 +740,39 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * work and then fails much later, when this kernel shuts down and * its CPUs park on nonsense addresses. */ - if (ctx->self_phys != ctx_phys) { - pr_err("mk_spawn: Boot context at %pa is stamped %pa\n", - &ctx_phys, &ctx->self_phys); - pr_err("mk_spawn: Spawn context layout mismatch - host and spawn kernels must be built from the same source\n"); - return; - } - + stamped_phys = READ_ONCE(ctx->self_phys); + if (stamped_phys != ctx_phys) + mk_reject_spawn_context(); + abi_magic = READ_ONCE(ctx->abi_magic); + if (abi_magic != MK_BOOT_CONTEXT_MAGIC) + mk_reject_spawn_context(); + + mk_boot_context_phys = ctx_phys; mk_boot_context = ctx; + return ctx; +} + +/* + * Initialize boot context tracking in spawn kernel. + * Called early during spawn kernel boot. + */ +void mk_init_boot_context(phys_addr_t ctx_phys) +{ + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx) + return; + /* + * A spawn kernel cannot calibrate against legacy timers because they + * belong to the host. Reuse the selected physical CPU's delay and local + * APIC timer calibration, while keeping explicit command-line values + * authoritative. + */ + if (!preset_lpj && ctx->boot_lps) + preset_lpj = DIV_ROUND_CLOSEST_ULL(ctx->boot_lps, HZ); + if (!lapic_timer_period && ctx->boot_apic_hz) + lapic_timer_period = + DIV_ROUND_CLOSEST_ULL(ctx->boot_apic_hz, HZ); /* * The host's control area (this context, the trampoline and park @@ -729,17 +797,25 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * * One physical page serves every wake path of this instance: the host * allocates it once in mk_setup_trampoline() and reuses it across - * re-spawns, and mk_prepare_trampoline() places our own trampoline copy + * re-spawns, and mk_arch_prepare_park() places our own trampoline copy * (including the secondary entry) in the same page. */ -static int __init mk_prepare_trampoline(void) +int __init mk_arch_prepare_park(void) { struct mk_spawn_context *ctx = mk_boot_context; unsigned long virt; int ret; + if (mk_arch_park_ready()) + return 0; + if (spawn_park_error) + return spawn_park_error; if (!ctx) return 0; + if (!ctx->trampoline_phys || !ctx->park_phys || !ctx->park_cr3) { + ret = -EINVAL; + goto fail; + } /* * Put our own copy of the trampoline in the page the host set @@ -747,34 +823,48 @@ static int __init mk_prepare_trampoline(void) * is entered from an offline CPU, where changing page attributes * is not allowed. */ - spawn_trampoline_phys = ctx->trampoline_phys; - spawn_trampoline_va = __va(spawn_trampoline_phys); - memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, - multikernel_relocate_kernel_end - multikernel_relocate_kernel_start); + if (!spawn_trampoline_prepared) { + spawn_trampoline_phys = ctx->trampoline_phys; + spawn_trampoline_va = __va(spawn_trampoline_phys); + memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, + multikernel_relocate_kernel_end - + multikernel_relocate_kernel_start); - /* - * Both pages are executed from the direct map, which is writable, - * so drop write before adding execute. Leaving them writable and - * executable trips the kernel's own W^X check. - */ - virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; - ret = set_memory_ro(virt, 1); - if (!ret) - ret = set_memory_x(virt, 1); - if (ret) - return ret; + /* + * Both pages are executed from the direct map, which is writable, + * so drop write before adding execute. Leaving them writable and + * executable trips the kernel's own W^X check. + */ + virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; + ret = set_memory_ro(virt, 1); + if (!ret) + ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_trampoline_prepared = true; + } /* The pool park page is entered the same way when this kernel dies */ - if (ctx->park_phys) { + if (!spawn_pool_park_prepared) { virt = (unsigned long)__va(ctx->park_phys) & PAGE_MASK; ret = set_memory_ro(virt, 1); if (!ret) ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_pool_park_prepared = true; } + /* Publish executable mappings before any reject or abort can park. */ + smp_store_release(&spawn_park_ready, true); + return 0; + +fail: + /* A partial W^X transition is not safe to retry. */ + spawn_park_error = ret; return ret; } -early_initcall(mk_prepare_trampoline); +early_initcall(mk_arch_prepare_park); /* * Add a 2MB executable mapping to a page table. diff --git a/drivers/pci/msi/api.c b/drivers/pci/msi/api.c index 818d55fbad0dfa..ca7195c58ef8e0 100644 --- a/drivers/pci/msi/api.c +++ b/drivers/pci/msi/api.c @@ -10,6 +10,7 @@ #include #include +#include #include "msi.h" @@ -123,6 +124,8 @@ bool pci_msix_can_alloc_dyn(struct pci_dev *dev) { if (!dev->msix_cap) return false; + if (mk_pci_msi_controlled(dev)) + return false; return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY); } @@ -388,6 +391,12 @@ EXPORT_SYMBOL(pci_free_irq_vectors); */ void pci_restore_msi_state(struct pci_dev *dev) { + if (mk_pci_msi_controlled(dev)) { + if (mk_pci_msi_restore(dev)) + pr_err("Failed to restore host-owned MSI vectors for %s\n", + pci_name(dev)); + return; + } __pci_restore_msi_state(dev); __pci_restore_msix_state(dev); } diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index 6e65f0f44112e6..e79bf05f07a325 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -4,6 +4,7 @@ */ #include #include +#include #include #include "msi.h" @@ -11,18 +12,38 @@ int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) { struct irq_domain *domain; + bool controlled = mk_pci_msi_controlled(dev); + int ret; + + if (controlled) { + ret = mk_pci_msi_prepare(dev, nvec, type); + if (ret) + return ret; + } domain = dev_get_msi_domain(&dev->dev); if (domain && irq_domain_is_hierarchy(domain)) - return msi_domain_alloc_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN, nvec); - - return pci_msi_legacy_setup_msi_irqs(dev, nvec, type); + ret = msi_domain_alloc_irqs_all_locked(&dev->dev, + MSI_DEFAULT_DOMAIN, nvec); + else + ret = pci_msi_legacy_setup_msi_irqs(dev, nvec, type); + if (!ret && controlled) + ret = mk_pci_msi_activate(dev); + + if (ret && controlled && mk_pci_msi_teardown(dev)) + pr_err("Failed to roll back host-owned MSI setup for %s\n", + pci_name(dev)); + return ret; } void pci_msi_teardown_msi_irqs(struct pci_dev *dev) { struct irq_domain *domain; + if (mk_pci_msi_controlled(dev) && mk_pci_msi_teardown(dev)) + pr_err("Failed to tear down host-owned MSI vectors for %s\n", + pci_name(dev)); + domain = dev_get_msi_domain(&dev->dev); if (domain && irq_domain_is_hierarchy(domain)) { msi_domain_free_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN); diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index e2412175d7af20..6694a75e8ac92b 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "../pci.h" #include "msi.h" @@ -240,6 +241,14 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { struct pci_dev *dev = msi_desc_to_pci_dev(entry); + if (mk_pci_msi_controlled(dev)) { + /* Host programming is committed later from the setup context. */ + entry->msg = *msg; + if (entry->write_msi_msg) + entry->write_msi_msg(entry, entry->write_msi_msg_data); + return; + } + if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) { /* Don't touch the hardware now */ } else if (entry->pci.msi_attrib.is_msix) { @@ -714,6 +723,7 @@ static int msix_setup_interrupts(struct pci_dev *dev, struct msix_entry *entries static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, int nvec, struct irq_affinity *affd) { + bool controlled = mk_pci_msi_controlled(dev); int ret, tsize; u16 control; @@ -736,6 +746,15 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, ret = -ENOMEM; goto out_disable; } + if (controlled && + !pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { + /* + * The controlled setup path activates host-owned vectors from + * msix_setup_interrupts(). Clear stale table entries while the + * function-wide mask is still set, before host activation. + */ + msix_mask_all(dev->msix_base, tsize); + } ret = msix_setup_interrupts(dev, entries, nvec, affd); if (ret) @@ -744,7 +763,8 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, /* Disable INTX */ pci_intx_for_msi(dev, 0); - if (!pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { + if (!controlled && + !pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { /* * Ensure that all table entries are masked to prevent * stale entries from firing in a crash kernel. diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 8479c2e1f74f10..7a9f7a327971ef 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -4380,6 +4381,8 @@ int pcie_reset_flr(struct pci_dev *dev, bool probe) if (probe) return 0; + if (mk_pci_controlled(dev)) + return mk_pci_reset_flr(dev); return pcie_flr(dev); } diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 7661517ebdf0df..bccc7a4bdd7943 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -24,7 +24,6 @@ #include #include #include -#include #include "pci.h" static struct resource busn_resource = { @@ -2601,14 +2600,6 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn) struct pci_dev *dev; u32 l; - /* - * For multikernel spawns, check if we should even probe this location - * BEFORE any config space access. This prevents hardware conflicts - * when the host kernel is also using PCI devices. - */ - if (IS_ENABLED(CONFIG_MULTIKERNEL) && !mk_pci_should_probe(bus, devfn)) - return NULL; - if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000)) return NULL; diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 011374c92c8481..9f73ca860353e8 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -10,10 +10,17 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include +struct pci_bus; +struct pci_dev; +struct mk_instance; /** * Physical CPU identifiers @@ -29,6 +36,7 @@ typedef u64 mk_phys_cpu_t; #define MK_PHYS_CPU_INVALID (~(mk_phys_cpu_t)0) struct mk_cpu_set { + raw_spinlock_t lock; unsigned int nr; /* Entries in use */ unsigned int cap; /* Allocated capacity */ mk_phys_cpu_t *ids; @@ -43,26 +51,15 @@ bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id); bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id); int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src); int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set); - -static inline unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) -{ - return set ? set->nr : 0; -} - -static inline bool mk_cpu_set_empty(const struct mk_cpu_set *set) -{ - return mk_cpu_set_count(set) == 0; -} - -static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) -{ - return mk_cpu_set_empty(set) ? MK_PHYS_CPU_INVALID : set->ids[0]; -} +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set); +bool mk_cpu_set_empty(const struct mk_cpu_set *set); +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set); +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id); #define mk_cpu_set_for_each(i, id, set) \ for ((i) = 0; \ - (set) && (i) < (set)->nr && \ - (((id) = (set)->ids[(i)]), true); \ + mk_cpu_set_get((set), (i), &(id)); \ (i)++) /** @@ -75,8 +72,68 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* IPI ring buffer size - must be power of 2 for efficient modulo */ #define MK_IPI_RING_SIZE 64 +#define MK_IPI_SLOT_EMPTY 0 +#define MK_IPI_SLOT_WRITING 1 +#define MK_IPI_SLOT_READY 2 +#define MK_IPI_SLOT_CONSUMING 3 +#define MK_IPI_SLOT_CANCELLED 4 +#define MK_IPI_ABI_MAGIC 0x4d4b495049303037ULL /* "MKIPI007" */ +#define MK_IPI_READY_TIMEOUT_MS 120000 + +#define MK_REPLY_SLOTS 16 +#define MK_REPLY_STATE_BITS 3 +#define MK_IRQ_MAILBOX_SLOTS 256 +#define MK_IRQ_MAILBOX_WORDS (MK_IRQ_MAILBOX_SLOTS / 64) +#define MK_IRQ_MAILBOX_SLOT_INVALID ((u32)~0U) +#define MK_IRQ_MAILBOX_PENDING_MASK 0x3fffffffULL +#define MK_IRQ_MAILBOX_CONSUMING 0x40000000ULL +#define MK_IRQ_MAILBOX_MASKED 0x80000000ULL +#define MK_IRQ_MAILBOX_GENERATION_SHIFT 32 + +enum mk_reply_state { + MK_REPLY_FREE = 0, + MK_REPLY_RESERVED, + MK_REPLY_WRITING, + MK_REPLY_EXECUTING, + MK_REPLY_COMMITTED, + MK_REPLY_READY, + MK_REPLY_ABANDONED, +}; + +enum mk_reply_kind { + MK_REPLY_PCI_CFG = 1, + MK_REPLY_PCI_IRQ, + MK_REPLY_PCI_RESET, +}; + +struct mk_reply_slot { + atomic64_t state_generation; + u64 request_id; + u32 kind; + s32 status; + u32 value; + u32 reserved; +}; + +struct mk_reply_table { + struct mk_reply_slot slots[MK_REPLY_SLOTS]; + atomic_t late_replies; + atomic_t cancelled_slots; + atomic_t atomic_timeouts; + atomic_t indeterminate_timeouts; + atomic_t occupied_failures; +}; + +struct mk_reply_handle { + u32 slot; + u32 kind; + u64 request_id; + u64 generation; +}; + /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + atomic_t state; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ @@ -85,16 +142,163 @@ struct mk_ipi_data { /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ + atomic_t head; /* Producer allocation cursor */ + atomic_t tail; /* Consumer scan cursor */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ + /* Appended shared ABI: do not move fields above this line. */ + atomic64_t producer_gate; /* Owner CPU and claimed slot */ + atomic_t producer_contention; /* Sends that observed a busy gate */ + atomic_t full_failures; /* Sends rejected by a full ring */ + atomic_t invalid_state; /* Invalid slot state observations */ + atomic_t cancelled_writes; /* Halted producer writes recovered */ }; +struct mk_irq_mailbox_entry { + /* Upper 32 bits are slot generation; lower 32 bits are pending count. */ + atomic64_t pending_generation; + u64 lifecycle_epoch; + u32 lifecycle_generation; + u32 device_id; + u32 local_irq; + u16 vector; + u16 reserved; +}; + +struct mk_irq_mailbox { + atomic64_t pending_bitmap[MK_IRQ_MAILBOX_WORDS]; + struct mk_irq_mailbox_entry entries[MK_IRQ_MAILBOX_SLOTS]; + atomic_t next_generation; + atomic_t recorded; + atomic_t coalesced; + atomic_t masked_deferred; + atomic_t stale; + atomic_t dispatch_failed; + atomic_t saturated; +}; + +static inline u64 mk_irq_mailbox_token(u32 generation, u32 pending) +{ + return (u64)generation << MK_IRQ_MAILBOX_GENERATION_SHIFT | pending; +} + +static inline u32 mk_irq_mailbox_generation(u64 token) +{ + return token >> MK_IRQ_MAILBOX_GENERATION_SHIFT; +} + +static inline u32 mk_irq_mailbox_pending(u64 token) +{ + return token & MK_IRQ_MAILBOX_PENDING_MASK; +} + +static inline bool mk_irq_mailbox_masked(u64 token) +{ + return token & MK_IRQ_MAILBOX_MASKED; +} + +static inline bool mk_irq_mailbox_consuming(u64 token) +{ + return token & MK_IRQ_MAILBOX_CONSUMING; +} + /* Shared memory structures - per-instance design */ struct mk_shared_data { + atomic_t emergency_shutdown; struct mk_ipi_ring ring; /* IPI message ring buffer */ + /* Appended ABI handshake; existing shared offsets stay unchanged. */ + u64 abi_magic; + u32 abi_version; + u32 abi_size; + s32 ready_instance_id; + atomic_t ready; + /* Appended direct synchronous replies; keep all older offsets stable. */ + struct mk_reply_table replies; + /* Changes on every host launch; tags process-context device lifecycles. */ + u64 spawn_epoch; + /* Preallocated hardirq-safe PCI interrupt forwarding transport. */ + struct mk_irq_mailbox irq_mailbox; }; +static inline void mk_reply_table_reset(struct mk_reply_table *table) +{ + unsigned int i; + + for (i = 0; i < MK_REPLY_SLOTS; i++) { + atomic64_set(&table->slots[i].state_generation, MK_REPLY_FREE); + WRITE_ONCE(table->slots[i].request_id, 0); + WRITE_ONCE(table->slots[i].kind, 0); + WRITE_ONCE(table->slots[i].status, 0); + WRITE_ONCE(table->slots[i].value, 0); + } + atomic_set(&table->late_replies, 0); + atomic_set(&table->cancelled_slots, 0); + atomic_set(&table->atomic_timeouts, 0); + atomic_set(&table->indeterminate_timeouts, 0); + atomic_set(&table->occupied_failures, 0); +} + +static inline void mk_irq_mailbox_reset(struct mk_irq_mailbox *mailbox) +{ + unsigned int i; + + for (i = 0; i < MK_IRQ_MAILBOX_WORDS; i++) + atomic64_set(&mailbox->pending_bitmap[i], 0); + for (i = 0; i < MK_IRQ_MAILBOX_SLOTS; i++) { + struct mk_irq_mailbox_entry *entry = &mailbox->entries[i]; + + atomic64_set(&entry->pending_generation, 0); + WRITE_ONCE(entry->lifecycle_epoch, 0); + WRITE_ONCE(entry->lifecycle_generation, 0); + WRITE_ONCE(entry->device_id, 0); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, 0); + WRITE_ONCE(entry->reserved, 0); + } + atomic_set(&mailbox->next_generation, 0); + atomic_set(&mailbox->recorded, 0); + atomic_set(&mailbox->coalesced, 0); + atomic_set(&mailbox->masked_deferred, 0); + atomic_set(&mailbox->stale, 0); + atomic_set(&mailbox->dispatch_failed, 0); + atomic_set(&mailbox->saturated, 0); +} + +static inline void mk_ipi_ring_reset_contents(struct mk_ipi_ring *ring) +{ + unsigned int i; + + atomic_set(&ring->head, 0); + atomic_set(&ring->tail, 0); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + WRITE_ONCE(ring->entries[i].data_size, 0); + atomic_set(&ring->entries[i].state, MK_IPI_SLOT_EMPTY); + } + atomic_set(&ring->producer_contention, 0); + atomic_set(&ring->full_failures, 0); + atomic_set(&ring->invalid_state, 0); + atomic_set(&ring->cancelled_writes, 0); +} + +static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + mk_ipi_ring_reset_contents(ring); + atomic64_set(&ring->producer_gate, 0); +} + +static inline void mk_shared_data_reset(struct mk_shared_data *shared) +{ + atomic_set(&shared->emergency_shutdown, 0); + mk_ipi_ring_reset(&shared->ring); + WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); + WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); + WRITE_ONCE(shared->abi_size, sizeof(*shared)); + WRITE_ONCE(shared->ready_instance_id, -1); + atomic_set(&shared->ready, 0); + mk_reply_table_reset(&shared->replies); + WRITE_ONCE(shared->spawn_epoch, 0); + mk_irq_mailbox_reset(&shared->irq_mailbox); +} + /* Function pointer type for IPI callbacks */ typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *data, void *ctx); @@ -137,8 +341,14 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns void generic_multikernel_interrupt(void); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +int mk_ipi_shared_validate(const struct mk_shared_data *shared); +int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id); +int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, + unsigned int timeout_ms); +int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared); + +/* Recover a producer only after every CPU in @halted_cpus is parked. */ +int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus); /* * Multikernel Messaging System @@ -154,6 +364,7 @@ void mk_ipi_ring_drop_pending(void); #define MK_MSG_SYSTEM 0x3000 #define MK_MSG_USER 0x4000 #define MK_MSG_NETWORK 0x5000 +#define MK_MSG_PCI 0x6000 /* I/O interrupt forwarding subtypes */ #define MK_IO_IRQ_FORWARD (MK_MSG_IO + 1) @@ -180,6 +391,13 @@ void mk_ipi_ring_drop_pending(void); /* Network/vsock subtypes */ #define MK_NET_VSOCK_PKT (MK_MSG_NETWORK + 1) /* vsock packet */ #define MK_NET_DATA_READY (MK_MSG_NETWORK + 2) /* Data available notification */ +/* Host-mediated PCI control-plane subtypes */ +#define MK_PCI_CFG_REQUEST (MK_MSG_PCI + 1) +#define MK_PCI_CFG_RESPONSE (MK_MSG_PCI + 2) +#define MK_PCI_IRQ_REQUEST (MK_MSG_PCI + 3) +#define MK_PCI_IRQ_RESPONSE (MK_MSG_PCI + 4) +#define MK_PCI_RESET_REQUEST (MK_MSG_PCI + 5) +#define MK_PCI_RESET_RESPONSE (MK_MSG_PCI + 6) /** * Core message structure @@ -202,6 +420,95 @@ struct mk_io_irq_payload { u32 vector; /* Interrupt vector */ u32 device_id; /* Device identifier (optional) */ u32 flags; /* Control flags (priority, etc.) */ + u32 lifecycle_generation; + u32 reserved; + u64 lifecycle_epoch; +}; + +/* Pack a PCI segment and BDF into mk_io_irq_payload::device_id. */ +#define MK_PCI_IRQ_ID(domain, bus, devfn) \ + (((u32)(domain) << 16) | ((u32)(bus) << 8) | (u32)(devfn)) +#define MK_PCI_IRQ_ID_DOMAIN(id) ((u16)((id) >> 16)) +#define MK_PCI_IRQ_ID_BUS(id) ((u8)((id) >> 8)) +#define MK_PCI_IRQ_ID_DEVFN(id) ((u8)(id)) + +struct mk_pci_cfg_request { + u64 request_id; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u16 reg; + u8 len; + u8 write; + u32 value; + u32 reply_slot; + u32 reply_reserved; + u64 reply_generation; +}; + +struct mk_pci_cfg_response { + u64 request_id; + s32 status; + u32 value; +}; + +enum mk_pci_irq_operation { + MK_PCI_IRQ_SETUP = 1, + MK_PCI_IRQ_RESTORE_BEGIN, + MK_PCI_IRQ_BIND, + MK_PCI_IRQ_COMMIT, + MK_PCI_IRQ_ACTIVATE, + MK_PCI_IRQ_TEARDOWN, +}; + +enum mk_pci_msi_lifecycle { + MK_PCI_MSI_IDLE = 0, + MK_PCI_MSI_PREPARED, + MK_PCI_MSI_COMMITTED, + MK_PCI_MSI_ACTIVE, + MK_PCI_MSI_FAILED, +}; + +struct mk_pci_irq_request { + u64 request_id; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u16 operation; + u16 vector; + u16 nr_vectors; + u8 msix; + u8 reserved; + u32 local_irq; + u32 reply_slot; + u32 lifecycle_generation; + u64 reply_generation; + u64 lifecycle_epoch; +}; + +struct mk_pci_irq_response { + u64 request_id; + s32 status; +}; + +struct mk_pci_reset_request { + u64 request_id; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u32 reset_generation; + u32 reply_slot; + u32 reserved; + u64 reply_generation; + u64 lifecycle_epoch; +}; + +struct mk_pci_reset_response { + u64 request_id; + s32 status; }; /* IRQ control flags */ @@ -272,7 +579,8 @@ struct mk_shutdown_payload { * Message handler callback type */ typedef void (*mk_msg_handler_t)(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx); + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx); /* Opaque type for pending message tracking */ struct mk_pending_msg; @@ -312,6 +620,23 @@ int mk_register_msg_handler(u32 msg_type, mk_msg_handler_t handler, void *ctx); * Returns 0 on success, negative error code on failure */ int mk_unregister_msg_handler(u32 msg_type, mk_msg_handler_t handler); +int mk_reply_reserve(struct mk_shared_data *shared, u32 kind, u64 request_id, + struct mk_reply_handle *reply); +int mk_reply_claim(struct mk_instance *instance, + const struct mk_reply_handle *reply); +int mk_reply_begin_execute(struct mk_instance *instance, + const struct mk_reply_handle *reply); +int mk_reply_publish(struct mk_instance *instance, + const struct mk_reply_handle *reply, s32 status, u32 value); +int mk_reply_wait_atomic(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_us, + s32 *status, u32 *value); +int mk_reply_wait(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_ms, + s32 *status, u32 *value); +void mk_reply_release(struct mk_shared_data *shared, + struct mk_reply_handle *reply); +void mk_reply_scan(struct mk_shared_data *shared); /* Pending message tracking for request-response pattern */ struct mk_pending_msg *mk_msg_pending_add(u32 msg_type, u32 operation, u64 resource_id); @@ -403,6 +728,12 @@ struct mk_memory_region { * Represents a single PCI device that should be accessible to an instance. * Format: vendor:device@domain:bus:slot.func */ +#define MK_PCI_RESOURCE_COUNT 6 +struct mk_pci_resource { + u64 start; + u64 end; + u64 flags; +}; struct mk_pci_device { char name[64]; /* Device name from DTB (e.g., "enp9s0_dev") */ u16 vendor; /* PCI vendor ID */ @@ -411,6 +742,8 @@ struct mk_pci_device { u8 bus; /* PCI bus number */ u8 slot; /* PCI slot number */ u8 func; /* PCI function number */ + struct mk_pci_resource resources[MK_PCI_RESOURCE_COUNT]; + bool resources_valid; struct list_head list; /* Link to device list */ }; @@ -460,7 +793,7 @@ struct mk_dt_config { bool platform_devices_valid; /* Whether platform device list is valid */ /* Extensibility: Reserved fields for future use */ - u32 reserved[7]; /* Reduced due to added fields */ + u32 reserved[4]; /* Raw device tree data */ void *dtb_data; @@ -477,6 +810,8 @@ struct mk_instance { int id; /* Kernel-assigned instance ID */ char *name; /* User-provided instance name */ enum mk_instance_state state; /* Current state */ + /* Serializes memory topology changes with PCI assignment leases. */ + struct mutex resource_mutex; /* Resource management - list of reserved memory regions */ struct list_head memory_regions; /* List of struct mk_memory_region */ @@ -487,11 +822,17 @@ struct mk_instance { /* CPU resources */ struct mk_cpu_set *cpus; /* Set of assigned physical CPU IDs */ + /* Pins the CPU selected for control messages and forwarded IRQs. */ + struct rw_semaphore control_route_sem; + mk_phys_cpu_t irq_route_cpu; /* IRQ-safe cached forwarding target */ + struct delayed_work irq_retry_work; /* Re-rings pending IRQ mailboxes. */ /* PCI device resources */ struct list_head pci_devices; /* List of struct mk_pci_device */ int pci_device_count; /* Number of PCI devices */ bool pci_devices_valid; /* Whether PCI device list is valid */ + /* Host-only live PCI assignment leases (private elements). */ + struct list_head pci_assignments; /* Platform device resources */ struct list_head platform_devices; /* List of struct mk_platform_device */ @@ -546,6 +887,20 @@ struct mk_instance { struct kref refcount; /* Reference count for cleanup */ }; +static inline mk_phys_cpu_t +mk_instance_irq_route_load(const struct mk_instance *instance) +{ + /* Pairs with route publication before old-target IRQs are drained. */ + return smp_load_acquire(&instance->irq_route_cpu); +} + +static inline void mk_instance_irq_route_store(struct mk_instance *instance, + mk_phys_cpu_t target) +{ + /* Publish the new target before synchronize_irq() drains old users. */ + smp_store_release(&instance->irq_route_cpu, target); +} + /** * Device Tree Parsing Functions */ @@ -680,14 +1035,14 @@ struct mk_instance *mk_instance_get(struct mk_instance *instance); void __noreturn mk_halt_to_pool(void); /** - * mk_instance_reserve_resources() - Reserve CPU and memory resources for instance - * @instance: Instance to reserve resources for - * @config: Device tree configuration with memory size and CPU assignment + * mk_instance_reserve_resources() - Atomically reserve instance resources + * @instance: Empty instance to populate + * @config: Parsed memory, CPU, PCI, and platform resource configuration * - * Allocates the specified memory size from the multikernel pool, creates - * memory regions, and copies CPU assignment. + * Reserves every configured resource class or returns all resources acquired + * by the attempt. A failure never leaves a partially populated instance. * - * Returns 0 on success, negative error code on failure. + * Returns: 0 on success, negative error code on failure */ int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config); @@ -703,9 +1058,11 @@ void mk_instance_free_memory(struct mk_instance *instance); int mk_instance_transfer_cpus(struct mk_instance *instance, - const struct mk_cpu_set *cpus); + const struct mk_cpu_set *cpus); int mk_instance_return_cpus(struct mk_instance *instance, - const struct mk_cpu_set *cpus); + const struct mk_cpu_set *cpus); +void mk_cpu_transaction_lock(void); +void mk_cpu_transaction_unlock(void); int mk_instance_add_memory_region(struct mk_instance *instance, size_t size); int mk_instance_remove_memory_region(struct mk_instance *instance, phys_addr_t phys_addr, size_t size); @@ -745,6 +1102,7 @@ int mk_instance_set_kexec_active(int mk_id); */ struct kimage; struct pci_bus; +struct pci_dev; #ifdef CONFIG_MULTIKERNEL bool multikernel_allow_emergency_restart(void); @@ -758,13 +1116,88 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +void mk_instance_mark_failed(struct mk_instance *instance); +int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size); -/* Device probe filtering against the instance's allowlist */ -bool mk_pci_should_probe(struct pci_bus *bus, int devfn); +/* Device filtering against the instance metadata */ +#ifdef CONFIG_PCI +bool mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, + u16 *device); +#if defined(CONFIG_X86) +bool mk_pci_controlled(struct pci_dev *dev); +int mk_pci_reset_flr(struct pci_dev *dev); +#else +static inline bool mk_pci_controlled(struct pci_dev *dev) +{ + return false; +} + +static inline int mk_pci_reset_flr(struct pci_dev *dev) +{ + return -EOPNOTSUPP; +} +#endif +#else +static inline bool +mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, u16 *device) +{ + return false; +} + +static inline bool mk_pci_controlled(struct pci_dev *dev) +{ + return false; +} + +static inline int mk_pci_reset_flr(struct pci_dev *dev) +{ + return -EOPNOTSUPP; +} +#endif +#if defined(CONFIG_X86) && defined(CONFIG_PCI) && defined(CONFIG_PCI_MSI) +bool mk_pci_msi_controlled(struct pci_dev *dev); +int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type); +int mk_pci_msi_activate(struct pci_dev *dev); +int mk_pci_msi_restore(struct pci_dev *dev); +int mk_pci_msi_teardown(struct pci_dev *dev); +void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared); +#else +static inline bool mk_pci_msi_controlled(struct pci_dev *dev) +{ + return false; +} + +static inline int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) +{ + return 0; +} + +static inline int mk_pci_msi_activate(struct pci_dev *dev) +{ + return 0; +} + +static inline int mk_pci_msi_restore(struct pci_dev *dev) +{ + return 0; +} + +static inline int mk_pci_msi_teardown(struct pci_dev *dev) +{ + return 0; +} + +static inline void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) +{ +} +#endif + bool mk_platform_device_allowed(const char *name, const char *hid); /* Early CPU registration from the manifest (spawn kernels) */ @@ -772,9 +1205,18 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); +#ifdef CONFIG_PCI +int mk_pci_prepare_instance_start(struct mk_instance *instance); +#else +static inline int mk_pci_prepare_instance_start(struct mk_instance *instance) +{ + return 0; +} +#endif #else static inline bool multikernel_allow_emergency_restart(void) { @@ -812,20 +1254,62 @@ static inline void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size) { } -static inline bool mk_pci_should_probe(struct pci_bus *bus, int devfn) + +static inline bool mk_platform_device_allowed(const char *name, const char *hid) { return true; } -static inline bool mk_platform_device_allowed(const char *name, const char *hid) + +static inline bool mk_pci_controlled(struct pci_dev *dev) { - return true; + return false; +} + +static inline int mk_pci_reset_flr(struct pci_dev *dev) +{ + return -EOPNOTSUPP; +} + +static inline bool mk_pci_msi_controlled(struct pci_dev *dev) +{ + return false; } + +static inline int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) +{ + return 0; +} + +static inline int mk_pci_msi_activate(struct pci_dev *dev) +{ + return 0; +} + +static inline int mk_pci_msi_restore(struct pci_dev *dev) +{ + return 0; +} + +static inline int mk_pci_msi_teardown(struct pci_dev *dev) +{ + return 0; +} + +static inline void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) +{ +} + static inline void mk_register_cpus_from_manifest(void) { } static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + +static inline bool mk_manifest_rejected(void) +{ + return false; +} #endif /** @@ -833,7 +1317,8 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 -#define MK_FDT_COMPATIBLE "multikernel-v1" +/* Bumped whenever the shared-memory layout or message semantics change. */ +#define MK_FDT_COMPATIBLE "multikernel-v7" /** * Property Names @@ -909,21 +1394,24 @@ int __init mk_instance_restore_from_manifest(void); */ /** - * PCI Device Enforcement Functions + * PCI Device Filtering Functions */ /** - * mk_pci_should_probe() - Check if PCI probing should occur at a location - * @bus: PCI bus + * mk_pci_get_assigned_identity_bdf() - Identify an assigned PCI function + * @domain: PCI domain number + * @bus: PCI bus number * @devfn: PCI device/function number + * @vendor: optional assigned vendor ID output + * @device: optional assigned device ID output * - * Called BEFORE any PCI config space reads to determine if probing - * should proceed. This prevents config space accesses to devices - * that are not in the whitelist, avoiding hardware conflicts on bare metal. + * The raw x86 PCI configuration wrappers use this BDF-only lookup before + * reaching their hardware backend. Synthetic roots make assigned functions + * directly discoverable. A privileged spawn kernel can bypass those wrappers, + * so this check prevents accidental access rather than isolating a hostile + * kernel. * - * Returns: true if probing should proceed, false to skip entirely - * - * Declared above with the CONFIG_MULTIKERNEL stubs. + * Returns: true for an exact assignment metadata match, false otherwise */ /** @@ -979,6 +1467,8 @@ void mk_set_pool_cpu(int cpu, bool is_pool); /* Park the calling CPU in the pool wait loop; never returns */ void __noreturn mk_enter_pool_state(void *info); +int __init mk_arch_prepare_park(void); +bool mk_arch_park_ready(void); /* * Forcible stop of another instance's CPUs (NMI on x86). Registration diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h new file mode 100644 index 00000000000000..82285c84b8a02e --- /dev/null +++ b/include/linux/multikernel_abi.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MULTIKERNEL_ABI_H +#define _LINUX_MULTIKERNEL_ABI_H + +/* Private host/spawn transport compatibility constants. */ +#define MK_IPI_ABI_VERSION 7 +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b435458303032ULL /* "MKCTX002" */ + +#endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/include/linux/pci.h b/include/linux/pci.h index 1c270f1d512301..591b35215bdd08 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -503,6 +503,9 @@ struct pci_dev { unsigned int non_mappable_bars:1; /* BARs can't be mapped to user-space */ pci_dev_flags_t dev_flags; atomic_t enable_cnt; /* pci_enable_device has been called */ +#ifdef CONFIG_MULTIKERNEL + u32 multikernel_reset_generation; +#endif spinlock_t pcie_cap_lock; /* Protects RMW ops in capability accessors */ u32 saved_config_space[16]; /* Config space saved at suspend time */ @@ -524,6 +527,12 @@ struct pci_dev { #ifdef CONFIG_PCI_MSI void __iomem *msix_base; raw_spinlock_t msi_lock; +#ifdef CONFIG_MULTIKERNEL + u32 multikernel_msi_generation; + u16 multikernel_msi_nvec; + u8 multikernel_msi_state; + u8 multikernel_msi_msix; +#endif #endif struct pci_vpd vpd; #ifdef CONFIG_PCIE_DPC diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 01382f40aeb724..d1451197bb97eb 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -595,6 +595,7 @@ void kimage_free(struct kimage *image) { kimage_entry_t *ptr, entry; kimage_entry_t ind = 0; + struct mk_instance *route_instance = NULL; if (!image) return; @@ -609,6 +610,10 @@ void kimage_free(struct kimage *image) if (image->type == KEXEC_TYPE_MULTIKERNEL) { unsigned long i; + route_instance = image->mk_instance; + if (route_instance) + down_write(&route_instance->control_route_sem); + for (i = 0; i < image->nr_segments; i++) { void *virt_addr = phys_to_virt(image->segment[i].mem); @@ -629,7 +634,6 @@ void kimage_free(struct kimage *image) image->mk_instance->ipi_phys = 0; image->mk_instance->kimage = NULL; mk_instance_set_state(image->mk_instance, MK_STATE_READY); - mk_instance_put(image->mk_instance); image->mk_instance = NULL; } @@ -644,6 +648,10 @@ void kimage_free(struct kimage *image) __free_pages(phys_to_page(image->mk_ipi), order); image->mk_ipi = 0; } + if (route_instance) { + up_write(&route_instance->control_route_sem); + mk_instance_put(route_instance); + } } #ifdef CONFIG_CRASH_DUMP if (image->vmcoreinfo_data_copy) { @@ -1688,6 +1696,8 @@ int multikernel_kexec_by_id(int mk_id) { struct kimage *mk_image; struct mk_instance *instance; + bool transaction_locked = false; + bool route_locked = false; int cpu = -1; int i, rc; @@ -1702,9 +1712,22 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + pr_err("Multikernel instance %d is not loadable (state=%d)\n", + mk_id, instance->state); + rc = -EINVAL; + goto unlock; + } + mk_cpu_transaction_lock(); + transaction_locked = true; + down_write(&instance->control_route_sem); + route_locked = true; if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); + if (!mk_cpu_set_contains(instance->cpus, + mk_instance_irq_route_load(instance))) + mk_instance_irq_route_store(instance, phys_cpu); cpu = arch_cpu_from_physical_id(phys_cpu); if (cpu < 0) { pr_err("Physical CPU %llu not found in logical CPU map\n", phys_cpu); @@ -1733,6 +1756,18 @@ int multikernel_kexec_by_id(int mk_id) goto unlock; } + /* + * Stop and reset every leased VF before rewriting instance memory. A + * force-halted kernel may have left bus mastering enabled and DMA in + * flight into the image that is about to be reused. + */ + rc = mk_pci_prepare_instance_start(instance); + if (rc) { + pr_err("Failed to prepare PCI assignments for instance %d restart: %d\n", + mk_id, rc); + goto unlock; + } + /* * Booting consumes the image: the spawn kernel writes its .data and * patches its own text, so the copy in instance memory is spent once @@ -1756,10 +1791,11 @@ int multikernel_kexec_by_id(int mk_id) } rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + if (rc) { + pr_err("Manifest finalization failed: %d\n", rc); + goto unlock; + } + pr_info("Manifest finalized for multikernel instance\n"); /* * Point at the ring this image actually carries. Every load @@ -1775,32 +1811,50 @@ int multikernel_kexec_by_id(int mk_id) } /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. + * Start the instance with an empty downlink after its CPUs have been + * confirmed parked. The host is the only producer for this ring, so no + * publisher can race the reset once the old receiver is quiesced. */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); - - /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. - */ - mk_ipi_ring_drop_pending(); - + if (instance->ipi_data) { + rc = mk_ipi_shared_reset_downlink(instance->ipi_data); + if (rc) { + pr_err("Failed to reset instance %d IPI downlink: %d\n", + mk_id, rc); + goto unlock; + } + } rc = mk_arch_spawn_instance(mk_image, instance, cpu); if (rc == 0) { + rc = mk_ipi_shared_wait_ready(instance->ipi_data, mk_id, + MK_IPI_READY_TIMEOUT_MS); + if (rc) { + int abort_ret; + + pr_err("Instance %d did not acknowledge IPI ABI %u: %d\n", + mk_id, MK_IPI_ABI_VERSION, rc); + mutex_lock(&instance->resource_mutex); + mk_instance_set_state(instance, MK_STATE_FAILED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + route_locked = false; + mk_cpu_transaction_unlock(); + transaction_locked = false; + abort_ret = mk_instance_abort_spawn(instance); + if (abort_ret) + pr_crit("Instance %d IPI ABI timeout abort failed: %d\n", + mk_id, abort_ret); + goto unlock; + } rc = mk_instance_set_kexec_active(mk_image->mk_id); if (rc) pr_warn("Failed to set instance %d as active: %d\n", mk_image->mk_id, rc); } unlock: + if (route_locked) + up_write(&instance->control_route_sem); + if (transaction_locked) + mk_cpu_transaction_unlock(); kexec_unlock(); return rc; } diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index d704be265ae70c..4f4669d041f41e 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -24,4 +24,7 @@ config MULTIKERNEL - A manifest handed to each spawn kernel on multikernel's own boot channel, describing its resources and message rings + SR-IOV VF assignment additionally requires PCI_IOV and an active + hardware IOMMU with isolated interrupt delivery. Assignment fails + closed when those isolation facilities are unavailable. If unsure, say N. diff --git a/kernel/multikernel/Makefile b/kernel/multikernel/Makefile index 359cccdd20353e..8e5e9497b51474 100644 --- a/kernel/multikernel/Makefile +++ b/kernel/multikernel/Makefile @@ -3,7 +3,9 @@ # Makefile for multikernel support # -obj-y += core.o cpuset.o mem.o kernfs.o dts.o instance_dt.o manifest.o ipi.o messaging.o overlay.o hotplug.o baseline.o +obj-y += core.o cpuset.o mem.o kernfs.o dts.o instance_dt.o manifest.o +obj-y += ipi.o messaging.o overlay.o hotplug.o baseline.o +obj-$(CONFIG_PCI) += pci.o # DMA-BUF heap for multikernel memory allocation obj-$(CONFIG_DMABUF_HEAPS) += dma_heap.o diff --git a/kernel/multikernel/baseline.c b/kernel/multikernel/baseline.c index d4e2dd9be3a54c..c2e88e2ad84122 100644 --- a/kernel/multikernel/baseline.c +++ b/kernel/multikernel/baseline.c @@ -264,10 +264,14 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, } if (strcmp(device_type, "pci") == 0) { + struct mk_pci_device *existing; struct mk_pci_device *pci_dev; const char *pci_id_str; const fdt32_t *vendor_prop, *device_prop; - unsigned int domain, bus, slot, func; + u32 vendor, device; + u16 domain; + u8 bus, slot, func; + int ret; pci_id_str = fdt_getprop(fdt, dev_node, "pci-id", &len); if (!pci_id_str) { @@ -276,10 +280,12 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s' for device '%s'\n", - pci_id_str, dev_name); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s' for device '%s'\n", + len, pci_id_str, dev_name); + return ret; } vendor_prop = fdt_getprop(fdt, dev_node, "vendor-id", &len); @@ -295,6 +301,23 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, dev_name); return -EINVAL; } + vendor = fdt32_to_cpu(*vendor_prop); + device = fdt32_to_cpu(*device_prop); + if (vendor > U16_MAX || device > U16_MAX) { + pr_err("Out-of-range vendor-id or device-id for device '%s'\n", + dev_name); + return -ERANGE; + } + list_for_each_entry(existing, &instance->pci_devices, list) { + if (existing->domain == domain && + existing->bus == bus && existing->slot == slot && + existing->func == func) { + pr_err("Duplicate baseline PCI BDF %04x:%02x:%02x.%x\n", + domain, bus, slot, func); + mk_baseline_clear_resources(instance); + return -EEXIST; + } + } pci_dev = kzalloc(sizeof(*pci_dev), GFP_KERNEL); if (!pci_dev) { @@ -304,12 +327,12 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, strncpy(pci_dev->name, dev_name, sizeof(pci_dev->name) - 1); pci_dev->name[sizeof(pci_dev->name) - 1] = '\0'; - pci_dev->vendor = (u16)fdt32_to_cpu(*vendor_prop); - pci_dev->device = (u16)fdt32_to_cpu(*device_prop); - pci_dev->domain = (u16)domain; - pci_dev->bus = (u8)bus; - pci_dev->slot = (u8)slot; - pci_dev->func = (u8)func; + pci_dev->vendor = (u16)vendor; + pci_dev->device = (u16)device; + pci_dev->domain = domain; + pci_dev->bus = bus; + pci_dev->slot = slot; + pci_dev->func = func; list_add_tail(&pci_dev->list, &instance->pci_devices); instance->pci_device_count++; @@ -498,60 +521,73 @@ static int mk_baseline_initialize_cpus(void) return 0; } +#ifdef CONFIG_PCI static int mk_baseline_initialize_devices(const struct mk_instance *instance) { struct mk_pci_device *pci_dev; struct pci_dev *dev; - int failed = 0, unbound = 0; + int failed = 0; + int available = 0; - if (instance->pci_device_count == 0) { - pr_debug("No PCI devices in baseline to unbind\n"); + if (!instance->pci_device_count) { + pr_debug("No PCI devices in the multikernel pool\n"); return 0; } - pr_info("Unbinding %d PCI devices for multikernel pool\n", + pr_info("Validating %d PCI devices for the multikernel pool\n", instance->pci_device_count); + pci_lock_rescan_remove(); list_for_each_entry(pci_dev, &instance->pci_devices, list) { - dev = pci_get_domain_bus_and_slot(pci_dev->domain, pci_dev->bus, - PCI_DEVFN(pci_dev->slot, pci_dev->func)); + dev = pci_get_domain_bus_and_slot(pci_dev->domain, + pci_dev->bus, + PCI_DEVFN(pci_dev->slot, + pci_dev->func)); if (!dev) { pr_warn("PCI device %04x:%04x@%04x:%02x:%02x.%x not found in system\n", - pci_dev->vendor, pci_dev->device, pci_dev->domain, - pci_dev->bus, pci_dev->slot, pci_dev->func); + pci_dev->vendor, pci_dev->device, + pci_dev->domain, pci_dev->bus, + pci_dev->slot, pci_dev->func); failed++; continue; } - if (!dev->driver) { - pr_debug("PCI device %04x:%04x@%04x:%02x:%02x.%x already unbound\n", - pci_dev->vendor, pci_dev->device, pci_dev->domain, - pci_dev->bus, pci_dev->slot, pci_dev->func); - pci_dev_put(dev); - unbound++; - continue; + if (dev->vendor != pci_dev->vendor || + dev->device != pci_dev->device || + pci_dev_is_disconnected(dev) || + !pci_device_is_present(dev)) { + pr_warn("PCI device %04x:%04x@%04x:%02x:%02x.%x is not available\n", + pci_dev->vendor, pci_dev->device, + pci_dev->domain, pci_dev->bus, + pci_dev->slot, pci_dev->func); + failed++; + } else { + available++; } - - const char *driver_name = dev->driver->name; - - device_release_driver(&dev->dev); - - pr_info("Unbound PCI device %04x:%04x@%04x:%02x:%02x.%x (was: %s) for multikernel pool\n", - pci_dev->vendor, pci_dev->device, pci_dev->domain, - pci_dev->bus, pci_dev->slot, pci_dev->func, - driver_name); - pci_dev_put(dev); - unbound++; } + pci_unlock_rescan_remove(); - if (failed > 0) { - pr_warn("Failed to find %d PCI devices in system\n", failed); - } + if (failed) + pr_warn("%d PCI devices in the multikernel pool are unavailable\n", + failed); - pr_info("Successfully unbound %d PCI devices for multikernel pool\n", unbound); + pr_info("Validated %d PCI devices; host drivers remain bound until assignment\n", + available); return 0; } +#else +static int mk_baseline_initialize_devices(const struct mk_instance *instance) +{ + if (!instance->pci_device_count) { + pr_debug("No PCI devices in the multikernel pool\n"); + return 0; + } + + pr_err("Cannot initialize PCI devices without CONFIG_PCI\n"); + return -EOPNOTSUPP; +} +#endif int mk_baseline_validate_and_initialize(const void *fdt, size_t fdt_size) { diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index f91763107cf950..3e540db03bb6e2 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -15,6 +15,89 @@ #include #include "internal.h" +/* + * CPU moves take transaction, then the affected instance's route write side, + * then ownership while inspecting or publishing paired CPU sets. PCI request + * work takes the route read side, validates under ownership, drops ownership + * before sleeping, and retains the route pin through reply publication. + */ +static DEFINE_MUTEX(mk_cpu_transaction_mutex); +static DEFINE_MUTEX(mk_cpu_ownership_mutex); + +void mk_cpu_transaction_lock(void) +{ + mutex_lock(&mk_cpu_transaction_mutex); +} + +void mk_cpu_transaction_unlock(void) +{ + mutex_unlock(&mk_cpu_transaction_mutex); +} + +void mk_cpu_ownership_lock(void) +{ + mutex_lock(&mk_cpu_ownership_mutex); +} + +void mk_cpu_ownership_unlock(void) +{ + mutex_unlock(&mk_cpu_ownership_mutex); +} + +void mk_cpu_ownership_assert_held(void) +{ + lockdep_assert_held(&mk_cpu_ownership_mutex); +} + +static int __mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing) +{ + mk_phys_cpu_t replacement = MK_PHYS_CPU_INVALID; + mk_phys_cpu_t route_cpu; + mk_phys_cpu_t phys_cpu; + unsigned int i; + + lockdep_assert_held_write(&instance->control_route_sem); + route_cpu = mk_instance_irq_route_load(instance); + if (route_cpu == MK_PHYS_CPU_INVALID || + !mk_cpu_set_contains(removing, route_cpu)) + return 0; + + mk_cpu_ownership_lock(); + mk_cpu_set_for_each(i, phys_cpu, instance->cpus) { + if (!mk_cpu_set_contains(removing, phys_cpu)) { + replacement = phys_cpu; + break; + } + } + mk_cpu_ownership_unlock(); + if (replacement == MK_PHYS_CPU_INVALID && + READ_ONCE(instance->state) == MK_STATE_ACTIVE) + return -EBUSY; + + mutex_lock(&instance->resource_mutex); + if (replacement != MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, replacement); + mk_pci_sync_instance_irq_route(instance); + if (replacement == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); + mutex_unlock(&instance->resource_mutex); + return 0; +} + +int mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing) +{ + int ret; + + if (!instance || !removing) + return -EINVAL; + down_write(&instance->control_route_sem); + ret = __mk_instance_migrate_irq_route(instance, removing); + up_write(&instance->control_route_sem); + return ret; +} + static void mk_instance_return_all_cpus(struct mk_instance *instance) { if (!instance || mk_cpu_set_empty(instance->cpus)) @@ -26,16 +109,20 @@ static void mk_instance_return_all_cpus(struct mk_instance *instance) mk_instance_return_cpus(instance, instance->cpus); } -static void mk_instance_return_pci_devices(struct mk_instance *instance) +static int mk_instance_return_pci_devices(struct mk_instance *instance) { struct mk_pci_device *pci_dev, *pci_tmp; int returned_count = 0; + int ret; - if (!instance || !instance->pci_devices_valid) - return; + if (!instance || instance == root_instance || instance->id == 0) + return 0; - if (instance == root_instance || instance->id == 0) - return; + ret = mk_pci_release_assignments(instance); + if (ret) + return ret; + if (!instance->pci_devices_valid) + return 0; if (!root_instance) { pr_warn("Cannot return PCI devices from instance %d (%s): no root instance\n", @@ -43,52 +130,37 @@ static void mk_instance_return_pci_devices(struct mk_instance *instance) goto cleanup; } - list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, list) { - struct mk_pci_device *root_dev; - - root_dev = kzalloc(sizeof(*root_dev), GFP_KERNEL); - if (!root_dev) { - pr_warn("Failed to allocate PCI device entry for root instance\n"); - continue; - } - - *root_dev = *pci_dev; - INIT_LIST_HEAD(&root_dev->list); - - list_add_tail(&root_dev->list, &root_instance->pci_devices); + list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, + list) { + list_move_tail(&pci_dev->list, &root_instance->pci_devices); root_instance->pci_device_count++; root_instance->pci_devices_valid = true; - - pr_debug("Returned PCI device %04x:%02x:%02x.%d from instance %d to root\n", - root_dev->domain, root_dev->bus, root_dev->slot, - root_dev->func, instance->id); - returned_count++; } - if (returned_count > 0) { + if (returned_count) pr_info("Returned %d PCI devices from instance %d (%s) to root instance\n", returned_count, instance->id, instance->name); - } cleanup: - list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, list) { + list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, + list) { list_del(&pci_dev->list); kfree(pci_dev); } instance->pci_device_count = 0; instance->pci_devices_valid = false; + return 0; } - static void mk_instance_return_platform_devices(struct mk_instance *instance) { - struct mk_platform_device *plat_dev, *plat_tmp; - int returned_count = 0; + struct mk_platform_device *device, *tmp; + int returned = 0; - if (!instance || !instance->platform_devices_valid) + if (!instance || instance == root_instance || instance->id == 0) return; - - if (instance == root_instance || instance->id == 0) + if (!instance->platform_devices_valid && + list_empty(&instance->platform_devices)) return; if (!root_instance) { @@ -97,60 +169,69 @@ static void mk_instance_return_platform_devices(struct mk_instance *instance) goto cleanup; } - list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { - struct mk_platform_device *root_dev; - - root_dev = kzalloc(sizeof(*root_dev), GFP_KERNEL); - if (!root_dev) { - pr_warn("Failed to allocate platform device entry for root instance\n"); - continue; - } - - *root_dev = *plat_dev; - INIT_LIST_HEAD(&root_dev->list); - - list_add_tail(&root_dev->list, &root_instance->platform_devices); + list_for_each_entry_safe(device, tmp, &instance->platform_devices, + list) { + list_move_tail(&device->list, + &root_instance->platform_devices); root_instance->platform_device_count++; root_instance->platform_devices_valid = true; - - pr_debug("Returned platform device '%s' from instance %d to root\n", - root_dev->name, instance->id); - - returned_count++; + returned++; } - if (returned_count > 0) { + if (returned) pr_info("Returned %d platform devices from instance %d (%s) to root instance\n", - returned_count, instance->id, instance->name); - } + returned, instance->id, instance->name); cleanup: - list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { - list_del(&plat_dev->list); - kfree(plat_dev); + list_for_each_entry_safe(device, tmp, &instance->platform_devices, + list) { + list_del(&device->list); + kfree(device); } instance->platform_device_count = 0; instance->platform_devices_valid = false; } -static void mk_instance_release(struct kref *kref) +int mk_instance_release_resources(struct mk_instance *instance) { - struct mk_instance *instance = container_of(kref, struct mk_instance, refcount); + int ret; - pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", - instance->id, instance->name); + if (!instance || instance == root_instance || instance->id == 0) + return 0; - mk_instance_return_all_cpus(instance); - mk_instance_return_pci_devices(instance); + ret = mk_instance_return_pci_devices(instance); + if (ret) + return ret; mk_instance_return_platform_devices(instance); + mk_instance_return_all_cpus(instance); mk_instance_free_memory(instance); + return 0; +} + +static void mk_instance_release(struct kref *kref) +{ + struct mk_instance *instance = + container_of(kref, struct mk_instance, refcount); + int ret; + pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", + instance->id, instance->name); + ret = mk_instance_release_resources(instance); + if (WARN_ON_ONCE(ret)) { + /* + * A failed PCI release can leave assignments linked to this + * instance. Retain the backing object rather than leave those + * assignments with a dangling instance pointer. + */ + pr_crit("Retaining multikernel instance %d (%s) after resource release failed: %d\n", + instance->id, instance->name, ret); + return; + } mk_cpu_set_free(instance->cpus); kfree(instance->dtb_data); kfree(instance->name); kfree(instance); } - /** * Instance reference counting */ @@ -190,6 +271,37 @@ void mk_instance_set_state(struct mk_instance *instance, */ } +void mk_instance_mark_failed(struct mk_instance *instance) +{ + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + mk_instance_set_state(instance, MK_STATE_FAILED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); +} + +static int mk_instance_finish_halt(struct mk_instance *instance, + bool transaction_held) +{ + int ret; + + if (!transaction_held) + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + /* Every caller has confirmed that all spawn CPUs are parked. */ + ret = mk_pci_quiesce_instance_irqs(instance, true); + mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); + mk_instance_set_state(instance, ret ? MK_STATE_FAILED : MK_STATE_LOADED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + if (!transaction_held) + mk_cpu_transaction_unlock(); + return ret; +} + struct mk_instance *mk_instance_find_by_name(const char *name) { struct mk_instance *instance; @@ -286,6 +398,7 @@ bool multikernel_allow_emergency_restart(void) */ int mk_instance_confirm_parked(struct mk_instance *instance) { + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int ret, failed = 0; @@ -293,8 +406,19 @@ int mk_instance_confirm_parked(struct mk_instance *instance) /* Never started, so nothing of it is running */ if (!instance->spawn_ctx) return 0; + if (!instance->cpus_on_slot) + return 0; + + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + ret = mk_cpu_set_copy(snapshot, instance->cpus_on_slot); + if (ret) { + mk_cpu_set_free(snapshot); + return ret; + } - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + mk_cpu_set_for_each(i, phys_cpu, snapshot) { ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -302,7 +426,16 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } + if (!failed) { + ret = mk_ipi_ring_recover_halted(snapshot); + if (ret) { + pr_err("Instance %d (%s): failed to recover halted IPI producer: %d\n", + instance->id, instance->name, ret); + failed++; + } + } + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -319,8 +452,10 @@ int mk_instance_confirm_parked(struct mk_instance *instance) int mk_instance_transfer_cpus(struct mk_instance *instance, const struct mk_cpu_set *cpus) { + struct mk_cpu_set *snapshot; unsigned int i, requested_count; mk_phys_cpu_t phys_cpu; + int logical_cpu; int unavailable = 0; char buf[256]; int ret; @@ -330,14 +465,26 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, return -EINVAL; } - requested_count = mk_cpu_set_count(cpus); + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mk_cpu_ownership_lock(); + ret = mk_cpu_set_copy(snapshot, cpus); + if (ret) + goto unlock; + + requested_count = mk_cpu_set_count(snapshot); if (requested_count == 0) { pr_info("No CPUs requested for instance %d (%s)\n", instance->id, instance->name); - return 0; + ret = 0; + goto unlock; } - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, snapshot) { if (!mk_cpu_set_contains(mk_cpu_pool, phys_cpu)) { pr_err("CPU %llu not available in the pool\n", phys_cpu); @@ -345,33 +492,47 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, continue; } - if (arch_cpu_from_physical_id(phys_cpu) < 0) { + logical_cpu = arch_cpu_from_physical_id(phys_cpu); + if (logical_cpu < 0) { pr_err("Physical CPU %llu not found in logical CPU map\n", phys_cpu); unavailable++; + } else if (logical_cpu == 0) { + pr_err("Physical CPU %llu is reserved for host control\n", + phys_cpu); + unavailable++; } } if (unavailable > 0) { pr_err("Instance %d (%s): %d CPUs are not available\n", instance->id, instance->name, unavailable); - return -EBUSY; + ret = -EBUSY; + goto unlock; } ret = mk_cpu_set_reserve(instance->cpus, requested_count); if (ret) - return ret; + goto unlock; - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, snapshot) { mk_cpu_set_del(mk_cpu_pool, phys_cpu); mk_cpu_set_add(instance->cpus, phys_cpu); } + if (mk_instance_irq_route_load(instance) == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, + mk_cpu_set_first(instance->cpus)); mk_cpu_set_format(buf, sizeof(buf), instance->cpus); pr_info("Transferred %u CPUs from pool to instance %d (%s): %s\n", requested_count, instance->id, instance->name, buf); - return 0; +unlock: + mk_cpu_ownership_unlock(); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + mk_cpu_set_free(snapshot); + return ret; } /** @@ -387,6 +548,7 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, int mk_instance_return_cpus(struct mk_instance *instance, const struct mk_cpu_set *cpus) { + struct mk_cpu_set *snapshot; unsigned int i, requested_count; mk_phys_cpu_t phys_cpu; int not_found = 0; @@ -398,15 +560,27 @@ int mk_instance_return_cpus(struct mk_instance *instance, return -EINVAL; } - requested_count = mk_cpu_set_count(cpus); + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mk_cpu_ownership_lock(); + ret = mk_cpu_set_copy(snapshot, cpus); + if (ret) + goto unlock; + + requested_count = mk_cpu_set_count(snapshot); if (requested_count == 0) { pr_info("No CPUs requested to return from instance %d (%s)\n", instance->id, instance->name); - return 0; + ret = 0; + goto unlock; } /* Validate all CPUs are assigned to this instance */ - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, snapshot) { if (!mk_cpu_set_contains(instance->cpus, phys_cpu)) { pr_err("CPU %llu not assigned to instance %d (%s)\n", phys_cpu, instance->id, instance->name); @@ -417,22 +591,22 @@ int mk_instance_return_cpus(struct mk_instance *instance, if (not_found > 0) { pr_err("Instance %d (%s): %d CPUs are not assigned to this instance\n", instance->id, instance->name, not_found); - return -EINVAL; + ret = -EINVAL; + goto unlock; } ret = mk_cpu_set_reserve(mk_cpu_pool, requested_count); if (ret) - return ret; + goto unlock; - mk_cpu_set_format(buf, sizeof(buf), cpus); + mk_cpu_set_format(buf, sizeof(buf), snapshot); + mk_cpu_ownership_unlock(); + ret = __mk_instance_migrate_irq_route(instance, snapshot); + if (ret) + goto unlock_route; + mk_cpu_ownership_lock(); - /* - * @cpus may alias instance->cpus (returning everything on - * teardown), so walk it back-to-front: a deletion then never - * shifts entries the walk has yet to visit. - */ - for (i = requested_count; i-- > 0; ) { - phys_cpu = cpus->ids[i]; + mk_cpu_set_for_each(i, phys_cpu, snapshot) { mk_cpu_set_add(mk_cpu_pool, phys_cpu); mk_cpu_set_del(instance->cpus, phys_cpu); } @@ -440,16 +614,22 @@ int mk_instance_return_cpus(struct mk_instance *instance, pr_info("Returned %u CPUs from instance %d (%s) to the pool: %s\n", requested_count, instance->id, instance->name, buf); - return 0; +unlock: + mk_cpu_ownership_unlock(); +unlock_route: + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + mk_cpu_set_free(snapshot); + return ret; } static int mk_instance_reserve_cpus(struct mk_instance *instance, const struct mk_dt_config *config) { if (!config->cpus) { - pr_warn("No CPU configuration for instance %d (%s)\n", - instance->id, instance->name); - return 0; + pr_err("No CPU configuration for instance %d (%s)\n", + instance->id, instance->name); + return -ENOMEM; } return mk_instance_transfer_cpus(instance, config->cpus); @@ -459,91 +639,43 @@ static int mk_instance_transfer_pci_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count) { - struct mk_pci_device *req_dev, *root_dev, *tmp; - int transferred = 0; - int not_found = 0; - bool found; - if (!root_instance || !root_instance->pci_devices_valid) { pr_err("No root instance or PCI devices not initialized\n"); return -EINVAL; } - if (requested_count == 0 || list_empty(requested_devices)) { + if (!requested_count || list_empty(requested_devices)) { pr_info("No PCI devices requested for instance %d (%s)\n", instance->id, instance->name); instance->pci_devices_valid = true; return 0; } - list_for_each_entry(req_dev, requested_devices, list) { - found = false; - list_for_each_entry(root_dev, &root_instance->pci_devices, list) { - if (root_dev->vendor == req_dev->vendor && - root_dev->device == req_dev->device && - root_dev->domain == req_dev->domain && - root_dev->bus == req_dev->bus && - root_dev->slot == req_dev->slot && - root_dev->func == req_dev->func) { - found = true; - break; - } - } - if (!found) { - pr_err("PCI device %04x:%04x@%04x:%02x:%02x.%x not available in root pool\n", - req_dev->vendor, req_dev->device, req_dev->domain, - req_dev->bus, req_dev->slot, req_dev->func); - not_found++; - } - } - - if (not_found > 0) { - pr_err("Instance %d (%s): %d PCI devices not available\n", - instance->id, instance->name, not_found); - return -ENOENT; - } - - list_for_each_entry(req_dev, requested_devices, list) { - list_for_each_entry_safe(root_dev, tmp, &root_instance->pci_devices, list) { - if (root_dev->vendor == req_dev->vendor && - root_dev->device == req_dev->device && - root_dev->domain == req_dev->domain && - root_dev->bus == req_dev->bus && - root_dev->slot == req_dev->slot && - root_dev->func == req_dev->func) { - - list_del(&root_dev->list); - list_add_tail(&root_dev->list, &instance->pci_devices); - root_instance->pci_device_count--; - instance->pci_device_count++; - transferred++; - - pr_debug("Transferred PCI device %04x:%04x@%04x:%02x:%02x.%x to instance %d\n", - root_dev->vendor, root_dev->device, root_dev->domain, - root_dev->bus, root_dev->slot, root_dev->func, - instance->id); - break; - } - } - } - - instance->pci_devices_valid = true; - pr_info("Transferred %d PCI devices from root to instance %d (%s), root pool remaining: %d devices\n", - transferred, instance->id, instance->name, root_instance->pci_device_count); - - return 0; + return mk_pci_assign_devices(instance, requested_devices, + requested_count); } - static int mk_instance_reserve_pci_devices(struct mk_instance *instance, const struct mk_dt_config *config) { - if (!config->pci_devices_valid || config->pci_device_count == 0) { + if (!config->pci_devices_valid) { + if (config->pci_device_count || + !list_empty(&config->pci_devices)) + return -EINVAL; + instance->pci_devices_valid = true; + return 0; + } + + if (!config->pci_device_count) { + if (!list_empty(&config->pci_devices)) + return -EINVAL; instance->pci_devices_valid = true; instance->pci_device_count = 0; pr_debug("No PCI devices to reserve for instance %d (%s)\n", instance->id, instance->name); return 0; } + if (list_empty(&config->pci_devices)) + return -EINVAL; return mk_instance_transfer_pci_devices(instance, &config->pci_devices, @@ -551,86 +683,111 @@ static int mk_instance_reserve_pci_devices(struct mk_instance *instance, } static int mk_instance_transfer_platform_devices(struct mk_instance *instance, - const struct list_head *requested_devices, - int requested_count) + const struct list_head *requested_devices, + int requested_count) { - struct mk_platform_device *req_dev, *root_dev, *tmp; + struct mk_platform_device *requested, *other, *root_device; + int actual_count = 0; int transferred = 0; - int not_found = 0; - bool found; if (!root_instance || !root_instance->platform_devices_valid) { pr_err("No root instance or platform devices not initialized\n"); return -EINVAL; } + if (requested_count <= 0 || list_empty(requested_devices)) + return -EINVAL; - if (requested_count == 0 || list_empty(requested_devices)) { - pr_info("No platform devices requested for instance %d (%s)\n", - instance->id, instance->name); - instance->platform_devices_valid = true; - return 0; - } + list_for_each_entry(requested, requested_devices, list) { + actual_count++; + list_for_each_entry(other, requested_devices, list) { + if (other == requested) + break; + if (!strcmp(other->name, requested->name)) { + pr_err("Platform device %s is requested more than once\n", + requested->name); + return -EINVAL; + } + } - list_for_each_entry(req_dev, requested_devices, list) { - found = false; - list_for_each_entry(root_dev, &root_instance->platform_devices, list) { - if (strcmp(root_dev->name, req_dev->name) == 0) { - found = true; + root_device = NULL; + list_for_each_entry(other, &root_instance->platform_devices, + list) { + if (!strcmp(other->name, requested->name)) { + root_device = other; break; } } - if (!found) { - pr_err("Platform device '%s' not available in root pool\n", - req_dev->name); - not_found++; + if (!root_device) { + pr_err("Platform device %s not available in root pool\n", + requested->name); + return -ENOENT; } } - if (not_found > 0) { - pr_err("Instance %d (%s): %d platform devices not available\n", - instance->id, instance->name, not_found); - return -ENOENT; + if (actual_count != requested_count) { + pr_err("Platform device count mismatch: metadata=%d list=%d\n", + requested_count, actual_count); + return -EINVAL; } - list_for_each_entry(req_dev, requested_devices, list) { - list_for_each_entry_safe(root_dev, tmp, &root_instance->platform_devices, list) { - if (strcmp(root_dev->name, req_dev->name) == 0) { - list_del(&root_dev->list); - list_add_tail(&root_dev->list, &instance->platform_devices); - root_instance->platform_device_count--; - instance->platform_device_count++; - transferred++; - - pr_debug("Transferred platform device '%s' to instance %d\n", - root_dev->name, instance->id); + list_for_each_entry(requested, requested_devices, list) { + root_device = NULL; + list_for_each_entry(other, &root_instance->platform_devices, + list) { + if (!strcmp(other->name, requested->name)) { + root_device = other; break; } } + if (!root_device) + goto rollback; + + list_move_tail(&root_device->list, + &instance->platform_devices); + root_instance->platform_device_count--; + instance->platform_device_count++; + transferred++; } instance->platform_devices_valid = true; - pr_info("Transferred %d platform devices from root to instance %d (%s), root pool remaining: %d devices\n", - transferred, instance->id, instance->name, root_instance->platform_device_count); - + pr_info("Transferred %d platform devices from root to instance %d (%s)\n", + transferred, instance->id, instance->name); return 0; + +rollback: + pr_err("Platform inventory changed during reservation for instance %d\n", + instance->id); + mk_instance_return_platform_devices(instance); + return -EIO; } static int mk_instance_reserve_platform_devices(struct mk_instance *instance, - const struct mk_dt_config *config) + const struct mk_dt_config *config) { - if (!config->platform_devices_valid || config->platform_device_count == 0) { + if (!config->platform_devices_valid) { + if (config->platform_device_count || + !list_empty(&config->platform_devices)) + return -EINVAL; + instance->platform_devices_valid = true; + return 0; + } + + if (!config->platform_device_count) { + if (!list_empty(&config->platform_devices)) + return -EINVAL; instance->platform_devices_valid = true; instance->platform_device_count = 0; pr_debug("No platform devices to reserve for instance %d (%s)\n", instance->id, instance->name); return 0; } + if (list_empty(&config->platform_devices)) + return -EINVAL; return mk_instance_transfer_platform_devices(instance, &config->platform_devices, config->platform_device_count); } - /** * mk_instance_add_pci_device - Add a single PCI device to an instance * @instance: Target instance @@ -646,37 +803,14 @@ static int mk_instance_reserve_platform_devices(struct mk_instance *instance, int mk_instance_add_pci_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { - struct mk_pci_device *root_dev, *tmp; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - - if (!root_instance || !root_instance->pci_devices_valid) { - pr_err("No root instance or PCI devices not initialized\n"); - return -EINVAL; - } - - list_for_each_entry_safe(root_dev, tmp, &root_instance->pci_devices, list) { - if (root_dev->domain == domain && - root_dev->bus == bus && - root_dev->slot == slot && - root_dev->func == func) { - - list_del(&root_dev->list); - list_add_tail(&root_dev->list, &instance->pci_devices); - root_instance->pci_device_count--; - instance->pci_device_count++; - instance->pci_devices_valid = true; - - pr_info("Transferred PCI device %04x:%04x@%04x:%02x:%02x.%x to instance %d\n", - root_dev->vendor, root_dev->device, domain, bus, slot, func, - instance->id); - return 0; - } - } + int ret; - pr_err("PCI device %04x:%02x:%02x.%x not found in root pool\n", - domain, bus, slot, func); - return -ENOENT; + ret = mk_pci_assign_device(instance, domain, bus, devfn); + if (!ret) + pr_info("Leased PCI VF %04x:%02x:%02x.%x to instance %d\n", + domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), + instance->id); + return ret; } /** @@ -687,63 +821,22 @@ int mk_instance_add_pci_device(struct mk_instance *instance, * @devfn: PCI device and function (combined) * * Returns a single PCI device from the specified instance back to root instance. - * Used for dynamic PCI device hotplug from non-running instances. + * Dynamic assignment changes are accepted only while the instance is ready. * * Returns: 0 on success, negative error code on failure */ int mk_instance_remove_pci_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { - struct mk_pci_device *inst_dev, *tmp; - struct mk_pci_device *root_dev; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - - if (!instance->pci_devices_valid) { - pr_err("Instance %d PCI devices not initialized\n", instance->id); - return -EINVAL; - } - - if (!root_instance) { - pr_err("Cannot return PCI device: no root instance\n"); - return -EINVAL; - } - - list_for_each_entry_safe(inst_dev, tmp, &instance->pci_devices, list) { - if (inst_dev->domain == domain && - inst_dev->bus == bus && - inst_dev->slot == slot && - inst_dev->func == func) { - - root_dev = kzalloc(sizeof(*root_dev), GFP_KERNEL); - if (!root_dev) { - pr_err("Failed to allocate PCI device entry for root instance\n"); - return -ENOMEM; - } - - *root_dev = *inst_dev; - INIT_LIST_HEAD(&root_dev->list); - - list_add_tail(&root_dev->list, &root_instance->pci_devices); - root_instance->pci_device_count++; - root_instance->pci_devices_valid = true; - - list_del(&inst_dev->list); - kfree(inst_dev); - instance->pci_device_count--; - - pr_info("Returned PCI device %04x:%04x@%04x:%02x:%02x.%x from instance %d to root\n", - root_dev->vendor, root_dev->device, domain, bus, slot, func, - instance->id); - return 0; - } - } + int ret; - pr_err("PCI device %04x:%02x:%02x.%x not found in instance %d\n", - domain, bus, slot, func, instance->id); - return -ENOENT; + ret = mk_pci_unassign_device(instance, domain, bus, devfn); + if (!ret) + pr_info("Released PCI VF %04x:%02x:%02x.%x from instance %d\n", + domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), + instance->id); + return ret; } - /** * Memory management functions for instances */ @@ -939,67 +1032,76 @@ void mk_instance_free_memory(struct mk_instance *instance) instance->id, instance->name); } +static bool mk_instance_resources_empty(const struct mk_instance *instance) +{ + return list_empty(&instance->memory_regions) && + !instance->instance_pool && !instance->region_count && + mk_cpu_set_empty(instance->cpus) && + list_empty(&instance->pci_devices) && + list_empty(&instance->pci_assignments) && + !instance->pci_device_count && + list_empty(&instance->platform_devices) && + !instance->platform_device_count; +} + /** - * mk_instance_reserve_resources() - Reserve memory and CPU resources for an instance + * mk_instance_reserve_resources() - Atomically reserve instance resources * @instance: Instance to reserve resources for - * @config: Device tree configuration with memory regions and CPU assignment + * @config: Parsed resource configuration * - * Reserves all memory regions specified in the device tree configuration, - * makes them children of the main multikernel_res, and copies CPU assignment. + * Each resource class is acquired only after the preceding class succeeds. + * Any error returns every acquired resource to the root instance in reverse + * order, so callers never observe a partially populated instance. * - * Returns 0 on success, negative error code on failure. + * Returns: 0 on success, negative error code on failure */ int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config) { + const char *failed_resource; + int release_ret; int ret; - if (!config || !instance) { + if (!config || !instance || !instance->cpus) { pr_err("Invalid parameters to mk_instance_reserve_resources\n"); return -EINVAL; } + if (!mk_instance_resources_empty(instance)) { + pr_err("Instance %d (%s) already owns resources\n", + instance->id, instance->name); + return -EBUSY; + } - /* Free any existing memory regions first */ - mk_instance_free_memory(instance); - - /* Reserve memory regions */ + failed_resource = "memory"; ret = mk_instance_reserve_memory(instance, config); - if (ret) { - pr_err("Failed to reserve memory regions for instance %d (%s): %d\n", - instance->id, instance->name, ret); - return ret; - } + if (ret) + goto rollback; - /* Reserve CPU resources */ + failed_resource = "CPU"; ret = mk_instance_reserve_cpus(instance, config); - if (ret) { - pr_err("Failed to reserve CPU resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for CPU reservation failure */ - pr_warn("Continuing without CPU assignment\n"); - } - - /* Reserve PCI device resources */ - ret = mk_instance_reserve_pci_devices(instance, config); - if (ret) { - pr_err("Failed to reserve PCI device resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for PCI reservation failure */ - pr_warn("Continuing without PCI device assignment\n"); - } + if (ret) + goto rollback; - /* Reserve platform device resources */ + failed_resource = "platform device"; ret = mk_instance_reserve_platform_devices(instance, config); - if (ret) { - pr_err("Failed to reserve platform device resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for platform reservation failure */ - pr_warn("Continuing without platform device assignment\n"); - } + if (ret) + goto rollback; + + failed_resource = "PCI device"; + ret = mk_instance_reserve_pci_devices(instance, config); + if (ret) + goto rollback; return 0; -} +rollback: + pr_err("Failed to reserve %s resources for instance %d (%s): %d\n", + failed_resource, instance->id, instance->name, ret); + release_ret = mk_instance_release_resources(instance); + if (release_ret) + return release_ret; + return ret; +} /** * Per-instance memory pool management */ @@ -1224,17 +1326,37 @@ static void mk_shutdown_work_fn(struct work_struct *work) } /* - * Mark a halted instance re-spawnable. No wakeups are published here: - * the instance's CPUs may still be on their way to the park loop, and - * poking its context while its (old or next) kernel also publishes on - * it corrupts the single-producer mailbox. The kexec path confirms the - * CPUs are parked before it rewrites the image. + * Mark a halted instance re-spawnable only after every CPU is parked. This + * proof permits recovery of a mailbox consumer interrupted by shutdown and + * prevents the next image from racing the old kernel in shared memory. */ -static void mk_instance_settle_halted(struct mk_instance *instance) +static int mk_instance_settle_halted(struct mk_instance *instance, + bool transaction_held, + bool parked_confirmed) { + int ret; + pr_info("Instance %d (%s) halted, CPUs parking in pool\n", instance->id, instance->name); - mk_instance_set_state(instance, MK_STATE_LOADED); + if (!transaction_held) + mk_cpu_transaction_lock(); + if (!parked_confirmed) { + ret = mk_instance_confirm_parked(instance); + if (ret) { + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + mk_instance_set_state(instance, MK_STATE_FAILED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + if (!transaction_held) + mk_cpu_transaction_unlock(); + return ret; + } + } + ret = mk_instance_finish_halt(instance, true); + if (!transaction_held) + mk_cpu_transaction_unlock(); + return ret; } struct mk_halted_work { @@ -1250,7 +1372,9 @@ static void mk_halted_work_fn(struct work_struct *work) instance = mk_instance_find(aw->instance_id); if (instance) { - mk_instance_settle_halted(instance); + if (mk_instance_settle_halted(instance, false, false)) + pr_err("Instance %d halted but could not be made reusable\n", + instance->id); mk_instance_put(instance); } else { pr_warn("Shutdown ACK from unknown instance %d\n", @@ -1261,7 +1385,8 @@ static void mk_halted_work_fn(struct work_struct *work) } static void mk_system_msg_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx) { if (msg_type != MK_MSG_SYSTEM) return; @@ -1374,12 +1499,10 @@ int multikernel_halt_by_id(int mk_id) ret = mk_msg_pending_wait(pending, 30000); if (ret == 0) { - if (mk_instance_confirm_parked(instance)) - pr_warn("Multikernel instance %d halted with CPUs unaccounted for\n", + ret = mk_instance_settle_halted(instance, false, false); + if (!ret) + pr_info("Multikernel instance %d halted (graceful)\n", mk_id); - - mk_instance_set_state(instance, MK_STATE_LOADED); - pr_info("Multikernel instance %d halted (graceful)\n", mk_id); } mk_instance_put(instance); @@ -1387,8 +1510,8 @@ int multikernel_halt_by_id(int mk_id) } /** - * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI - * @mk_id: Instance ID to halt + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop * * Forces a spawn kernel's CPUs to stop by queuing a shutdown message in the * IPI ring buffer and sending NMIs directly to each CPU. The NMI handler @@ -1399,64 +1522,123 @@ int multikernel_halt_by_id(int mk_id) * * Returns: 0 on success, negative error code on failure */ -int multikernel_force_halt_by_id(int mk_id) +static int __mk_instance_force_halt(struct mk_instance *instance, + bool allow_loaded) { - struct mk_instance *instance; struct mk_shutdown_payload payload; + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); if (!instance) - return -ENOENT; - - if (instance->state != MK_STATE_ACTIVE) { + return -EINVAL; + if (instance->state != MK_STATE_ACTIVE && + (!allow_loaded || + (instance->state != MK_STATE_LOADED && + instance->state != MK_STATE_FAILED))) { pr_err("Instance %d not active (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } - if (mk_cpu_set_empty(instance->cpus)) { - pr_err("Instance %d has no CPUs assigned\n", mk_id); - mk_instance_put(instance); - return -EINVAL; + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + mk_cpu_transaction_lock(); + mk_cpu_ownership_lock(); + ret = mk_cpu_set_copy(snapshot, instance->cpus); + mk_cpu_ownership_unlock(); + if (ret) { + mk_cpu_transaction_unlock(); + mk_cpu_set_free(snapshot); + return ret; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); + if (mk_cpu_set_empty(snapshot)) { + pr_err("Instance %d has no CPUs assigned\n", instance->id); + mk_cpu_transaction_unlock(); + mk_cpu_set_free(snapshot); + return -EINVAL; + } - /* Queue shutdown message - NMI handler will check for this */ + pr_info("Force halting multikernel instance %d via NMI\n", instance->id); + if (!instance->ipi_data) { + mk_cpu_transaction_unlock(); + mk_cpu_set_free(snapshot); + return -ENODEV; + } + atomic_set_release(&instance->ipi_data->emergency_shutdown, 1); payload.flags = MK_SHUTDOWN_IMMEDIATE; - payload.sender_instance_id = root_instance->id; - ret = mk_send_message(mk_id, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, - &payload, sizeof(payload)); + payload.sender_instance_id = root_instance ? root_instance->id : 0; + ret = mk_send_message_to_instance(instance, MK_MSG_SYSTEM, + MK_SYS_SHUTDOWN, &payload, + sizeof(payload)); if (ret < 0) - pr_err("Failed to queue shutdown message: %d (sending NMI anyway)\n", ret); + pr_err("Failed to queue shutdown message: %d (sending NMI anyway)\n", + ret); /* Send NMI to each CPU in the instance */ - mk_cpu_set_for_each(i, phys_cpu, instance->cpus) { + mk_cpu_set_for_each(i, phys_cpu, snapshot) { mk_force_stop_cpu(phys_cpu); cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); + ret = mk_instance_confirm_parked(instance); + if (ret) { + pr_err("Instance %d CPUs did not park after force halt: %d\n", + instance->id, ret); + mk_cpu_set_free(snapshot); + mk_cpu_transaction_unlock(); + return ret; + } + mk_cpu_set_free(snapshot); + /* Quiesce host-owned resources before making the instance reusable. */ + ret = mk_instance_settle_halted(instance, true, true); + mk_cpu_transaction_unlock(); + return ret; +} - /* - * The NMI handler parks each CPU on the instance's context. Wait - * for them to arrive before reporting the instance re-spawnable, - * exactly as the graceful path does after its shutdown ACK. - */ - mk_instance_settle_halted(instance); - mk_instance_put(instance); - return 0; +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); +} + +int mk_instance_abort_spawn(struct mk_instance *instance) +{ + int ret; + + ret = __mk_instance_force_halt(instance, true); + if (ret && instance) + mk_instance_mark_failed(instance); + return ret; } +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + ret = mk_instance_force_halt(instance); + mk_instance_put(instance); + return ret; +} static int __init multikernel_init(void) { int ret; + ret = mk_pci_lease_system_init(); + if (ret) { + pr_err("Failed to initialize PCI assignment leases: %d\n", ret); + return ret; + } + /* Register NMI handler for forcible shutdown */ ret = mk_register_stop_nmi_handler(); if (ret < 0) { @@ -1467,6 +1649,7 @@ static int __init multikernel_init(void) ret = mk_messaging_init(); if (ret < 0) { pr_err("Failed to initialize multikernel messaging: %d\n", ret); + mk_pci_lease_system_cleanup(); return ret; } @@ -1474,6 +1657,7 @@ static int __init multikernel_init(void) if (ret < 0) { pr_err("Failed to register system message handler: %d\n", ret); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } @@ -1482,6 +1666,7 @@ static int __init multikernel_init(void) pr_err("Failed to initialize multikernel hotplug: %d\n", ret); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } @@ -1491,6 +1676,19 @@ static int __init multikernel_init(void) mk_hotplug_cleanup(); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); + return ret; + } + + ret = mk_ipi_shared_mark_ready(root_instance->ipi_data, + root_instance->id); + if (ret < 0) { + pr_err("Failed to publish multikernel IPI readiness: %d\n", ret); + mk_kernfs_cleanup(); + mk_hotplug_cleanup(); + mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); + mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } diff --git a/kernel/multikernel/cpuset.c b/kernel/multikernel/cpuset.c index ad36a4f94fc65a..ceb54e7435cc20 100644 --- a/kernel/multikernel/cpuset.c +++ b/kernel/multikernel/cpuset.c @@ -13,31 +13,22 @@ #include #include -struct mk_cpu_set *mk_cpu_set_alloc(void) -{ - return kzalloc(sizeof(struct mk_cpu_set), GFP_KERNEL); -} - -void mk_cpu_set_free(struct mk_cpu_set *set) +static void mk_cpu_set_lock(const struct mk_cpu_set *set, unsigned long *flags) { - if (!set) - return; - - kfree(set->ids); - kfree(set); + raw_spin_lock_irqsave((raw_spinlock_t *)&set->lock, *flags); } -void mk_cpu_set_clear(struct mk_cpu_set *set) +static void mk_cpu_set_unlock(const struct mk_cpu_set *set, unsigned long flags) { - if (set) - set->nr = 0; + raw_spin_unlock_irqrestore((raw_spinlock_t *)&set->lock, flags); } -static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) +static int mk_cpu_set_index_locked(const struct mk_cpu_set *set, + mk_phys_cpu_t id) { unsigned int i; - for (i = 0; set && i < set->nr; i++) { + for (i = 0; i < set->nr; i++) { if (set->ids[i] == id) return i; } @@ -45,9 +36,35 @@ static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) return -1; } -bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +struct mk_cpu_set *mk_cpu_set_alloc(void) { - return mk_cpu_set_index(set, id) >= 0; + struct mk_cpu_set *set; + + set = kzalloc_obj(*set, GFP_KERNEL); + if (set) + raw_spin_lock_init(&set->lock); + return set; +} + +void mk_cpu_set_free(struct mk_cpu_set *set) +{ + if (!set) + return; + + kfree(set->ids); + kfree(set); +} + +void mk_cpu_set_clear(struct mk_cpu_set *set) +{ + unsigned long flags; + + if (!set) + return; + + mk_cpu_set_lock(set, &flags); + set->nr = 0; + mk_cpu_set_unlock(set, flags); } /** @@ -61,64 +78,195 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) */ int mk_cpu_set_reserve(struct mk_cpu_set *set, unsigned int extra) { - unsigned int cap = set->nr + extra; - mk_phys_cpu_t *ids; + mk_phys_cpu_t *ids = NULL; + mk_phys_cpu_t *old_ids; + unsigned int cap; + unsigned long flags; - if (cap <= set->cap) - return 0; + if (!set) + return -EINVAL; + + for (;;) { + mk_cpu_set_lock(set, &flags); + cap = set->nr + extra; + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + mk_cpu_set_unlock(set, flags); - cap = max_t(unsigned int, cap, 8); - ids = krealloc_array(set->ids, cap, sizeof(*ids), GFP_KERNEL); - if (!ids) - return -ENOMEM; + cap = max_t(unsigned int, cap, 8); + kfree(ids); + ids = kcalloc(cap, sizeof(*ids), GFP_KERNEL); + if (!ids) + return -ENOMEM; - set->ids = ids; - set->cap = cap; - return 0; + mk_cpu_set_lock(set, &flags); + if (set->nr + extra > cap) { + mk_cpu_set_unlock(set, flags); + continue; + } + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + + memcpy(ids, set->ids, set->nr * sizeof(*ids)); + old_ids = set->ids; + set->ids = ids; + set->cap = cap; + mk_cpu_set_unlock(set, flags); + kfree(old_ids); + return 0; + } } /* Idempotent: adding an ID already in the set succeeds without effect */ int mk_cpu_set_add(struct mk_cpu_set *set, mk_phys_cpu_t id) { + unsigned long flags; int ret; - if (mk_cpu_set_contains(set, id)) - return 0; + if (!set) + return -EINVAL; - ret = mk_cpu_set_reserve(set, 1); - if (ret) - return ret; + for (;;) { + mk_cpu_set_lock(set, &flags); + if (mk_cpu_set_index_locked(set, id) >= 0) { + mk_cpu_set_unlock(set, flags); + return 0; + } + if (set->nr < set->cap) { + set->ids[set->nr++] = id; + mk_cpu_set_unlock(set, flags); + return 0; + } + mk_cpu_set_unlock(set, flags); - set->ids[set->nr++] = id; - return 0; + ret = mk_cpu_set_reserve(set, 1); + if (ret) + return ret; + } } bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id) { - int idx = mk_cpu_set_index(set, id); + unsigned long flags; + int idx; - if (idx < 0) + if (!set) return false; + mk_cpu_set_lock(set, &flags); + idx = mk_cpu_set_index_locked(set, id); + if (idx < 0) { + mk_cpu_set_unlock(set, flags); + return false; + } + memmove(&set->ids[idx], &set->ids[idx + 1], (set->nr - idx - 1) * sizeof(set->ids[0])); set->nr--; + mk_cpu_set_unlock(set, flags); return true; } +bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +{ + unsigned long flags; + bool found; + + if (!set) + return false; + + mk_cpu_set_lock(set, &flags); + found = mk_cpu_set_index_locked(set, id) >= 0; + mk_cpu_set_unlock(set, flags); + return found; +} + +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) +{ + unsigned long flags; + unsigned int nr; + + if (!set) + return 0; + + mk_cpu_set_lock(set, &flags); + nr = set->nr; + mk_cpu_set_unlock(set, flags); + return nr; +} + +bool mk_cpu_set_empty(const struct mk_cpu_set *set) +{ + return mk_cpu_set_count(set) == 0; +} + +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) +{ + unsigned long flags; + mk_phys_cpu_t id; + + if (!set) + return MK_PHYS_CPU_INVALID; + + mk_cpu_set_lock(set, &flags); + id = set->nr ? set->ids[0] : MK_PHYS_CPU_INVALID; + mk_cpu_set_unlock(set, flags); + return id; +} + +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id) +{ + unsigned long flags; + bool found = false; + + if (!set || !id) + return false; + + mk_cpu_set_lock(set, &flags); + if (index < set->nr) { + *id = set->ids[index]; + found = true; + } + mk_cpu_set_unlock(set, flags); + return found; +} + int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) { - unsigned int nr = mk_cpu_set_count(src); + unsigned long src_flags; + unsigned long dst_flags; + unsigned int nr; int ret; - dst->nr = 0; - ret = mk_cpu_set_reserve(dst, nr); - if (ret) - return ret; + if (!dst || !src) + return -EINVAL; - memcpy(dst->ids, src->ids, nr * sizeof(dst->ids[0])); - dst->nr = nr; - return 0; + for (;;) { + nr = mk_cpu_set_count(src); + ret = mk_cpu_set_reserve(dst, nr); + if (ret) + return ret; + + mk_cpu_set_lock(src, &src_flags); + if (src->nr > dst->cap) { + mk_cpu_set_unlock(src, src_flags); + continue; + } + + mk_cpu_set_lock(dst, &dst_flags); + memcpy(dst->ids, src->ids, src->nr * sizeof(dst->ids[0])); + dst->nr = src->nr; + mk_cpu_set_unlock(dst, dst_flags); + mk_cpu_set_unlock(src, src_flags); + return 0; + } } /** @@ -127,22 +275,31 @@ int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) * @size: Buffer size * @set: Set to format * - * Writes "none" for an empty set, a comma-separated list of physical + * Writes none for an empty set, a comma-separated list of physical * IDs otherwise. Output is truncated to @size. Returns the number of * characters written. */ int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set) { + unsigned long flags; unsigned int i; int len = 0; - if (mk_cpu_set_empty(set)) + if (!buf || !size) + return 0; + if (!set) return scnprintf(buf, size, "none"); - for (i = 0; i < set->nr; i++) { + mk_cpu_set_lock(set, &flags); + if (!set->nr) { + mk_cpu_set_unlock(set, flags); + return scnprintf(buf, size, "none"); + } + + for (i = 0; i < set->nr && len < size; i++) { len += scnprintf(buf + len, size - len, "%s%llu", i ? "," : "", set->ids[i]); } - + mk_cpu_set_unlock(set, flags); return len; } diff --git a/kernel/multikernel/dts.c b/kernel/multikernel/dts.c index 51950eae735129..dc444a2d3e7865 100644 --- a/kernel/multikernel/dts.c +++ b/kernel/multikernel/dts.c @@ -17,12 +17,55 @@ #include #include #include +#include #include #include +#include #include - #include "internal.h" +static int mk_pci_parse_hex(const char *str, size_t digits, u32 *value) +{ + size_t i; + u32 parsed = 0; + + for (i = 0; i < digits; i++) { + int digit = hex_to_bin(str[i]); + + if (digit < 0) + return -EINVAL; + parsed = (parsed << 4) | digit; + } + + *value = parsed; + return 0; +} + +int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, + u8 *slot, u8 *func) +{ + u32 parsed_domain, parsed_bus, parsed_slot, parsed_func; + + if (len != (int)sizeof("0000:00:00.0") || pci_id[12] != '\0' || + pci_id[4] != ':' || pci_id[7] != ':' || pci_id[10] != '.') + return -EINVAL; + + if (mk_pci_parse_hex(pci_id, 4, &parsed_domain) || + mk_pci_parse_hex(pci_id + 5, 2, &parsed_bus) || + mk_pci_parse_hex(pci_id + 8, 2, &parsed_slot) || + mk_pci_parse_hex(pci_id + 11, 1, &parsed_func)) + return -EINVAL; + if (parsed_domain > U16_MAX || parsed_bus > U8_MAX || + parsed_slot > 31 || parsed_func > 7) + return -ERANGE; + + *domain = (u16)parsed_domain; + *bus = (u8)parsed_bus; + *slot = (u8)parsed_slot; + *func = (u8)parsed_func; + return 0; +} + static const void *mk_dt_get_base_fdt(void) { if (!root_instance || !root_instance->dtb_data) { @@ -208,10 +251,14 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, { const char *pci_id_str; const fdt32_t *vendor_prop, *device_prop; + const fdt64_t *resources_prop; + struct mk_pci_device *existing; struct mk_pci_device *pci_dev; - unsigned int domain, bus, slot, func; + u32 vendor, device; + u16 domain; + u8 bus, slot, func; const char *node_name; - int len; + int len, i, ret; node_name = fdt_get_name(source_fdt, dev_node, NULL); @@ -222,10 +269,11 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format: '%s' (expected domain:bus:slot.func)\n", - pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id: '%.*s' (expected domain:bus:slot.func)\n", + len, pci_id_str); + return ret; } vendor_prop = fdt_getprop(source_fdt, dev_node, "vendor-id", &len); @@ -241,6 +289,21 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, device_name, node_name ? node_name : ""); return -EINVAL; } + vendor = fdt32_to_cpu(*vendor_prop); + device = fdt32_to_cpu(*device_prop); + if (vendor > U16_MAX || device > U16_MAX) { + pr_err("Out-of-range vendor-id or device-id in device '%s'\n", + device_name); + return -ERANGE; + } + list_for_each_entry(existing, &config->pci_devices, list) { + if (existing->domain == domain && existing->bus == bus && + existing->slot == slot && existing->func == func) { + pr_err("Duplicate assigned PCI BDF %04x:%02x:%02x.%x\n", + domain, bus, slot, func); + return -EEXIST; + } + } pci_dev = kzalloc(sizeof(*pci_dev), GFP_KERNEL); if (!pci_dev) { @@ -248,12 +311,38 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, return -ENOMEM; } - pci_dev->vendor = (u16)fdt32_to_cpu(*vendor_prop); - pci_dev->device = (u16)fdt32_to_cpu(*device_prop); - pci_dev->domain = (u16)domain; - pci_dev->bus = (u8)bus; - pci_dev->slot = (u8)slot; - pci_dev->func = (u8)func; + strscpy(pci_dev->name, device_name, sizeof(pci_dev->name)); + pci_dev->vendor = (u16)vendor; + pci_dev->device = (u16)device; + pci_dev->domain = domain; + pci_dev->bus = bus; + pci_dev->slot = slot; + pci_dev->func = func; + resources_prop = fdt_getprop(source_fdt, dev_node, "bar-resources", &len); + if (resources_prop) { + if (len != MK_PCI_RESOURCE_COUNT * 3 * sizeof(*resources_prop)) { + pr_err("Invalid bar-resources in device '%s'\n", device_name); + kfree(pci_dev); + return -EINVAL; + } + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + u64 start = fdt64_to_cpu(resources_prop[i * 3]); + u64 end = fdt64_to_cpu(resources_prop[i * 3 + 1]); + u64 flags = fdt64_to_cpu(resources_prop[i * 3 + 2]); + + if ((start || end) && + (end < start || !(flags & (IORESOURCE_IO | IORESOURCE_MEM)))) { + pr_err("Invalid PCI BAR %d range in device '%s'\n", + i, device_name); + kfree(pci_dev); + return -EINVAL; + } + pci_dev->resources[i].start = start; + pci_dev->resources[i].end = end; + pci_dev->resources[i].flags = flags; + } + pci_dev->resources_valid = true; + } list_add_tail(&pci_dev->list, &config->pci_devices); config->pci_device_count++; @@ -564,7 +653,7 @@ int mk_dt_parse(const void *dtb_data, size_t dtb_size, return ret; } - pr_info("Successfully parsed multikernel device tree with %zu bytes memory, %u CPUs, %d PCI devices, and %d platform devices\n", + pr_info("Successfully parsed multikernel device tree with %zu bytes memory, %d CPUs, %d PCI devices, and %d platform devices\n", config->memory_size, mk_cpu_set_count(config->cpus), config->pci_device_count, config->platform_device_count); return 0; @@ -614,10 +703,10 @@ int mk_dt_parse_resources(const void *fdt, int resources_node, return ret; } - pr_info("Successfully parsed instance '%s': %zu bytes memory, %u CPUs, %d PCI devices, %d platform devices\n", + pr_info("Successfully parsed instance '%s': %zu bytes memory, %d CPUs, %d PCI devices, %d platform devices\n", instance_name, config->memory_size, - mk_cpu_set_count(config->cpus), - config->pci_device_count, config->platform_device_count); + mk_cpu_set_count(config->cpus), config->pci_device_count, + config->platform_device_count); return 0; } @@ -994,6 +1083,43 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, list_for_each_entry(pci_dev, &instance->pci_devices, list) { char node_name[64]; char pci_id_str[32]; + fdt64_t resources[MK_PCI_RESOURCE_COUNT * 3]; + struct pci_dev *live_dev = NULL; + unsigned int devfn; + int i; + + if (!pci_dev->resources_valid) { + devfn = PCI_DEVFN(pci_dev->slot, + pci_dev->func); + live_dev = + pci_get_domain_bus_and_slot(pci_dev->domain, + pci_dev->bus, devfn); + if (!live_dev) { + pr_err("PCI device %04x:%02x:%02x.%x disappeared before resource snapshot\n", + pci_dev->domain, pci_dev->bus, + pci_dev->slot, pci_dev->func); + ret = -ENODEV; + goto err_free; + } + } + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + u64 start, end, flags; + + if (live_dev) { + start = pci_resource_start(live_dev, i); + end = pci_resource_end(live_dev, i); + flags = pci_resource_flags(live_dev, i); + } else { + start = pci_dev->resources[i].start; + end = pci_dev->resources[i].end; + flags = pci_dev->resources[i].flags; + } + resources[i * 3] = cpu_to_fdt64(start); + resources[i * 3 + 1] = cpu_to_fdt64(end); + resources[i * 3 + 2] = cpu_to_fdt64(flags); + } + if (live_dev) + pci_dev_put(live_dev); snprintf(node_name, sizeof(node_name), "%s", pci_dev->name[0] ? pci_dev->name : "unnamed_pci"); @@ -1015,6 +1141,11 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, ret = fdt_property_u32(fdt, "device-id", pci_dev->device); if (ret) goto err_free; + ret = fdt_property(fdt, "bar-resources", resources, + sizeof(resources)); + if (ret) + goto err_free; + ret = fdt_end_node(fdt); if (ret) goto err_free; } diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 8ad4a9411ecbc3..a234b0ff6cf0d0 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -77,25 +77,6 @@ struct mk_cpu_hotplug_work { u32 operation; /* MK_RES_CPU_ADD or MK_RES_CPU_REMOVE */ }; -/* - * Ownership tracking for this kernel's own hotplug: root_instance->cpus - * is the set of CPUs this kernel owns, in the host and in spawn kernels - * alike. The assignable-pool bookkeeping (mk_cpu_pool) is not done here; - * it belongs to the mk_send_cpu_* initiator paths of the kernel that - * manages the pool. - */ -static void mk_account_cpu_online(mk_phys_cpu_t cpu_id) -{ - if (root_instance->cpus && mk_cpu_set_add(root_instance->cpus, cpu_id)) - pr_warn("Multikernel hotplug: Failed to track CPU %llu\n", - cpu_id); -} - -static void mk_account_cpu_offline(mk_phys_cpu_t cpu_id) -{ - mk_cpu_set_del(root_instance->cpus, cpu_id); -} - /** * Search present CPUs (not possible CPUs) to find the logical CPU with matching * physical ID. Using present CPUs is important because topology can change during @@ -124,6 +105,7 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) pr_info("Multikernel hotplug: Adding CPU %llu (numa=%u, flags=0x%x)\n", cpu_id, numa_node, flags); + mk_cpu_transaction_lock(); logical_cpu = mk_cpu_to_logical(cpu_id); if (logical_cpu < 0) { /* @@ -136,7 +118,8 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (logical_cpu < 0) { pr_err("Multikernel hotplug: CPU %llu is not in this kernel's pool\n", cpu_id); - return -ENODEV; + ret = -ENODEV; + goto unlock_transaction; } set_cpu_present(logical_cpu, true); } @@ -144,18 +127,37 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (cpu_online(logical_cpu)) { pr_warn("Multikernel hotplug: CPU %d (phys %llu) already online\n", logical_cpu, cpu_id); - mk_account_cpu_online(cpu_id); - return 0; + mk_cpu_ownership_lock(); + if (root_instance->cpus) { + ret = mk_cpu_set_add(root_instance->cpus, cpu_id); + if (ret) + pr_warn("Multikernel hotplug: Failed to track CPU %llu in root pool\n", + cpu_id); + } + mk_cpu_ownership_unlock(); + ret = 0; + goto unlock_transaction; + } + + mk_cpu_ownership_lock(); + if (root_instance->cpus) { + ret = mk_cpu_set_reserve(root_instance->cpus, 1); + if (ret) { + mk_cpu_ownership_unlock(); + goto unlock_transaction; + } } + mk_cpu_ownership_unlock(); if (!get_cpu_device(logical_cpu)) { struct cpu *c = &per_cpu(cpu_devices, logical_cpu); + c->hotpluggable = true; ret = register_cpu(c, logical_cpu); if (ret) { pr_err("Multikernel hotplug: Failed to register CPU %d: %d\n", logical_cpu, ret); - return ret; + goto unlock_transaction; } } @@ -163,10 +165,19 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (ret < 0) { pr_err("Multikernel hotplug: Failed to add CPU %d (phys %llu): %d\n", logical_cpu, cpu_id, ret); - return ret; + goto unlock_transaction; } - mk_account_cpu_online(cpu_id); + mk_cpu_ownership_lock(); + if (root_instance->cpus) { + ret = mk_cpu_set_add(root_instance->cpus, cpu_id); + if (ret) + pr_warn("Multikernel hotplug: Failed to track CPU %llu in root pool\n", + cpu_id); + } + mk_cpu_ownership_unlock(); + if (ret) + goto unlock_transaction; /* Track the operation for potential rollback */ op = kzalloc(sizeof(*op), GFP_KERNEL); @@ -183,7 +194,10 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) pr_info("Multikernel hotplug: Successfully added CPU %d (phys %llu)\n", logical_cpu, cpu_id); - return 0; + ret = 0; +unlock_transaction: + mk_cpu_transaction_unlock(); + return ret; } static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) @@ -191,25 +205,45 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) int logical_cpu; int ret; struct mk_hotplug_op *op; + bool tracked; + mk_cpu_transaction_lock(); logical_cpu = mk_cpu_to_logical(cpu_id); if (logical_cpu < 0) { pr_err("Multikernel hotplug: Physical CPU %llu not found\n", cpu_id); - return -ENODEV; + ret = -ENODEV; + goto unlock_transaction; } + mk_cpu_ownership_lock(); + tracked = root_instance->cpus && + mk_cpu_set_contains(root_instance->cpus, cpu_id); + if (!cpu_online(logical_cpu)) { pr_warn("Multikernel hotplug: CPU %d (phys %llu) already offline\n", logical_cpu, cpu_id); - mk_account_cpu_offline(cpu_id); - return 0; + mk_cpu_set_del(root_instance->cpus, cpu_id); + mk_cpu_ownership_unlock(); + ret = 0; + goto unlock_transaction; + } + + if (!tracked) { + pr_err("Multikernel hotplug: CPU %llu is not tracked in root pool\n", + cpu_id); + mk_cpu_ownership_unlock(); + ret = -EINVAL; + goto unlock_transaction; } /* Don't allow removing CPU 0 (boot processor) */ if (logical_cpu == 0) { pr_err("Multikernel hotplug: Cannot remove boot CPU\n"); - return -EINVAL; + mk_cpu_ownership_unlock(); + ret = -EINVAL; + goto unlock_transaction; } + mk_cpu_ownership_unlock(); mk_set_pool_cpu(logical_cpu, true); @@ -218,10 +252,12 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) pr_err("Multikernel hotplug: Failed to remove CPU %d (phys %llu): %d\n", logical_cpu, cpu_id, ret); mk_set_pool_cpu(logical_cpu, false); - return ret; + goto unlock_transaction; } - mk_account_cpu_offline(cpu_id); + mk_cpu_ownership_lock(); + mk_cpu_set_del(root_instance->cpus, cpu_id); + mk_cpu_ownership_unlock(); /* * Clear CPU from present mask to prevent host kernel from trying @@ -244,7 +280,10 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) pr_info("Multikernel hotplug: Successfully removed CPU %d (phys %llu)\n", logical_cpu, cpu_id); - return 0; + ret = 0; +unlock_transaction: + mk_cpu_transaction_unlock(); + return ret; } static void mk_cpu_add_work_fn(struct work_struct *work) @@ -624,6 +663,7 @@ static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 pay * PCI Device Hotplug Operations */ +#ifdef CONFIG_PCI static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { @@ -756,6 +796,18 @@ static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) return 0; } +#else +static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, + const char *driver_override, u32 flags) +{ + return -EOPNOTSUPP; +} + +static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) +{ + return -EOPNOTSUPP; +} +#endif struct mk_device_hotplug_work { struct work_struct work; @@ -891,7 +943,8 @@ static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u * message subtype. */ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx) { int ret = 0; @@ -1018,6 +1071,7 @@ void mk_hotplug_cleanup(void) */ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) { + struct mk_cpu_set removing = { .nr = 1, .cap = 1, .ids = &cpu_id }; struct mk_cpu_resource_payload payload = { .cpu_id = cpu_id, .numa_node = 0, @@ -1026,8 +1080,11 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) }; struct mk_pending_msg *pending; struct mk_instance *target_instance; + mk_phys_cpu_t route_cpu; int ret; + raw_spin_lock_init(&removing.lock); + /* For self-removal, execute directly (we're in process context) */ if (instance_id == root_instance->id) { ret = mk_do_cpu_remove(cpu_id); @@ -1049,6 +1106,7 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) if (target_instance->state != MK_STATE_ACTIVE) { struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; + raw_spin_lock_init(&cpus.lock); /* * A CPU the instance has already run on is parked on that * instance's context. Bring it back to the host slot before @@ -1068,22 +1126,48 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) goto out; } + mk_cpu_transaction_lock(); pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, cpu_id); if (!pending) { ret = -ENOMEM; - goto out; + goto unlock_transaction; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, - &payload, sizeof(payload)); + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { + pr_err("Multikernel hotplug: CPU %llu not assigned to instance %d\n", + cpu_id, instance_id); + ret = -EINVAL; + mk_cpu_ownership_unlock(); + mk_msg_pending_wait(pending, 0); + goto unlock_transaction; + } + + ret = mk_cpu_set_reserve(mk_cpu_pool, 1); + if (ret) { + mk_cpu_ownership_unlock(); + mk_msg_pending_wait(pending, 0); + goto unlock_transaction; + } + mk_cpu_ownership_unlock(); + ret = mk_instance_migrate_irq_route(target_instance, &removing); + if (ret) { + mk_msg_pending_wait(pending, 0); + goto unlock_transaction; + } + route_cpu = mk_instance_irq_route_load(target_instance); + + ret = mk_send_message_to_cpu(target_instance, route_cpu, + MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, + &payload, sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ - goto out; + goto unlock_transaction; } ret = mk_msg_pending_wait(pending, 10000); if (ret < 0) - goto out; + goto unlock_transaction; /* * The spawn kernel parked the CPU on its own context when it went @@ -1097,15 +1181,29 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) if (ret < 0) { pr_err("Multikernel hotplug: CPU %llu offline in instance %d but not reparked to host: %d\n", cpu_id, instance_id, ret); - goto out; + goto unlock_transaction; } - mk_cpu_set_del(target_instance->cpus, cpu_id); - if (!mk_cpu_pool || mk_cpu_set_add(mk_cpu_pool, cpu_id)) + down_write(&target_instance->control_route_sem); + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { + ret = -ESTALE; + goto unlock_ownership; + } + ret = mk_cpu_set_add(mk_cpu_pool, cpu_id); + if (ret) { pr_warn("Multikernel hotplug: Failed to track CPU %llu in pool\n", cpu_id); + goto unlock_ownership; + } + mk_cpu_set_del(target_instance->cpus, cpu_id); ret = 0; +unlock_ownership: + mk_cpu_ownership_unlock(); + up_write(&target_instance->control_route_sem); +unlock_transaction: + mk_cpu_transaction_unlock(); out: mk_instance_put(target_instance); return ret; @@ -1159,26 +1257,39 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl instance_id); return -ENODEV; } + if (arch_cpu_from_physical_id(cpu_id) == 0) { + pr_err("Multikernel hotplug: CPU %llu is reserved for host control\n", + cpu_id); + ret = -EINVAL; + goto out; + } /* For non-running instances, transfer CPU from root using existing API */ if (target_instance->state != MK_STATE_ACTIVE) { struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; + raw_spin_lock_init(&cpus.lock); ret = mk_instance_transfer_cpus(target_instance, &cpus); goto out; } - /* - * Only a CPU from the assignable pool is parked on the host slot; - * publishing a wakeup for any other CPU can only time out. - */ + mk_cpu_transaction_lock(); + mk_cpu_ownership_lock(); if (!mk_cpu_set_contains(mk_cpu_pool, cpu_id)) { - pr_err("Multikernel hotplug: CPU %llu is not in this kernel's pool\n", + pr_err("Multikernel hotplug: CPU %llu not available in the pool\n", cpu_id); ret = -EBUSY; - goto out; + mk_cpu_ownership_unlock(); + goto unlock_transaction; } + ret = mk_cpu_set_reserve(target_instance->cpus, 1); + if (ret) { + mk_cpu_ownership_unlock(); + goto unlock_transaction; + } + mk_cpu_ownership_unlock(); + /* * The CPU is parked on the host slot, where the spawn kernel's * secondary wakeup cannot reach it. Point it at the instance's @@ -1188,22 +1299,23 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl if (ret < 0) { pr_err("Multikernel hotplug: Failed to repark CPU %llu to instance %d: %d\n", cpu_id, instance_id, ret); - goto out; + goto unlock_transaction; } pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_ADD, cpu_id); if (!pending) { mk_repark_cpu_to_host(target_instance, cpu_id); ret = -ENOMEM; - goto out; + goto unlock_transaction; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_ADD, - &payload, sizeof(payload)); + ret = mk_send_message_to_instance(target_instance, MK_MSG_RESOURCE, + MK_RES_CPU_ADD, &payload, + sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ mk_repark_cpu_to_host(target_instance, cpu_id); - goto out; + goto unlock_transaction; } ret = mk_msg_pending_wait(pending, 10000); @@ -1215,20 +1327,51 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl * watching the context and this times out harmlessly. */ mk_repark_cpu_to_host(target_instance, cpu_id); - goto out; + goto unlock_transaction; } - if (mk_cpu_set_add(target_instance->cpus, cpu_id)) + down_write(&target_instance->control_route_sem); + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(mk_cpu_pool, cpu_id)) { + ret = -ESTALE; + goto unlock_ownership; + } + ret = mk_cpu_set_add(target_instance->cpus, cpu_id); + if (ret) { pr_warn("Multikernel hotplug: Failed to track CPU %llu in instance %d\n", cpu_id, instance_id); + goto unlock_ownership; + } mk_cpu_set_del(mk_cpu_pool, cpu_id); + if (mk_instance_irq_route_load(target_instance) == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(target_instance, cpu_id); ret = 0; +unlock_ownership: + mk_cpu_ownership_unlock(); + up_write(&target_instance->control_route_sem); +unlock_transaction: + mk_cpu_transaction_unlock(); out: mk_instance_put(target_instance); return ret; } +static int mk_memory_change_allowed(struct mk_instance *instance) +{ + bool iommu_active; + + mutex_lock(&instance->resource_mutex); + iommu_active = mk_pci_iommu_lease_active_locked(instance); + mutex_unlock(&instance->resource_mutex); + if (!iommu_active) + return 0; + + pr_err("Cannot change memory for instance %d while an IOMMU lease is active\n", + instance->id); + return -EBUSY; +} + /** * mk_send_mem_add - Add memory to instance * @instance_id: Target instance ID @@ -1265,6 +1408,11 @@ int mk_send_mem_add(int instance_id, u64 start_pfn, u64 nr_pages, target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; + ret = mk_memory_change_allowed(target_instance); + if (ret) { + mk_instance_put(target_instance); + return ret; + } /* For non-running instances, allocate memory from pool and add to instance */ if (target_instance->state != MK_STATE_ACTIVE) { @@ -1331,6 +1479,11 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; + ret = mk_memory_change_allowed(target_instance); + if (ret) { + mk_instance_put(target_instance); + return ret; + } /* For non-running instances, just remove the memory region from the instance */ if (target_instance->state != MK_STATE_ACTIVE) { @@ -1370,117 +1523,66 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) } /** - * mk_send_device_add - Add PCI device to instance and wait for completion + * mk_send_device_add - Assign a PCI device to an instance * @instance_id: Target instance ID * @domain: PCI domain * @bus: PCI bus * @devfn: PCI device and function (combined) - * @driver_override: Target driver name for binding (can be NULL) - * @flags: Additional flags + * @driver_override: Target driver name for a root-kernel add + * @flags: Additional root-kernel add flags * - * For local instance, executes addition synchronously. - * For remote instance, sends IPI and waits for ACK response. - * For instances that are not yet running (MK_STATE_READY/LOADED), - * adds device to instance's device list. + * Remote assignment changes are permitted only while the target instance is + * ready. Active instances must be stopped and returned to ready state first. * * Returns: 0 on success, negative error code on failure */ int mk_send_device_add(int instance_id, u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { - struct mk_device_resource_payload payload = { - .domain = domain, - .bus = bus, - .devfn = devfn, - .flags = flags, - .sender_instance_id = root_instance->id - }; - struct mk_pending_msg *pending; struct mk_instance *target_instance; int ret; - u32 resource_id; - - if (driver_override) - strscpy(payload.driver_override, driver_override, sizeof(payload.driver_override)); - else - payload.driver_override[0] = '\0'; - - resource_id = (domain << 16) | (bus << 8) | devfn; + if (!root_instance) + return -ENODEV; if (instance_id == root_instance->id) - return mk_do_device_add(domain, bus, devfn, driver_override, flags); + return mk_do_device_add(domain, bus, devfn, driver_override, + flags); target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; - if (target_instance->state != MK_STATE_ACTIVE) { - ret = mk_instance_add_pci_device(target_instance, domain, bus, devfn); - goto out; - } - - pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_DEVICE_ADD, resource_id); - if (!pending) { - ret = -ENOMEM; - goto out; - } - - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_DEVICE_ADD, - &payload, sizeof(payload)); - if (ret < 0) { - mk_msg_pending_wait(pending, 0); - goto out; - } - - ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) - goto out; - - ret = mk_instance_add_pci_device(target_instance, domain, bus, devfn); - if (ret < 0) { - pr_warn("Device added to target but failed to update tracking: %d\n", ret); + if (target_instance->state != MK_STATE_READY) { + pr_err("PCI assignment changes require instance %d to be ready\n", + instance_id); + ret = -EBUSY; + } else { + ret = mk_instance_add_pci_device(target_instance, domain, bus, + devfn); } - - pr_info("Multikernel hotplug: Device %04x:%02x:%02x.%x successfully added to instance %d\n", - domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), instance_id); - - ret = 0; -out: mk_instance_put(target_instance); return ret; } /** - * mk_send_device_remove - Remove PCI device from instance and wait for completion + * mk_send_device_remove - Release a PCI device from an instance * @instance_id: Target instance ID * @domain: PCI domain * @bus: PCI bus * @devfn: PCI device and function (combined) * - * For local instance, executes removal synchronously. - * For remote instance, sends IPI and waits for ACK response. - * For instances that are not yet running (MK_STATE_READY/LOADED), - * removes device from instance's device list. + * Remote assignment changes are permitted only while the target instance is + * ready. Active instances must be stopped and returned to ready state first. * * Returns: 0 on success, negative error code on failure */ int mk_send_device_remove(int instance_id, u16 domain, u8 bus, u8 devfn) { - struct mk_device_resource_payload payload = { - .domain = domain, - .bus = bus, - .devfn = devfn, - .flags = 0, - .sender_instance_id = root_instance->id - }; - struct mk_pending_msg *pending; struct mk_instance *target_instance; int ret; - u32 resource_id; - - payload.driver_override[0] = '\0'; - resource_id = (domain << 16) | (bus << 8) | devfn; + if (!root_instance) + return -ENODEV; if (instance_id == root_instance->id) return mk_do_device_remove(domain, bus, devfn); @@ -1488,38 +1590,14 @@ int mk_send_device_remove(int instance_id, u16 domain, u8 bus, u8 devfn) if (!target_instance) return -ENODEV; - if (target_instance->state != MK_STATE_ACTIVE) { - ret = mk_instance_remove_pci_device(target_instance, domain, bus, devfn); - goto out; - } - - pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_DEVICE_REMOVE, resource_id); - if (!pending) { - ret = -ENOMEM; - goto out; - } - - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_DEVICE_REMOVE, - &payload, sizeof(payload)); - if (ret < 0) { - mk_msg_pending_wait(pending, 0); - goto out; - } - - ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) - goto out; - - ret = mk_instance_remove_pci_device(target_instance, domain, bus, devfn); - if (ret < 0) { - pr_warn("Device removed from target but failed to update tracking: %d\n", ret); + if (target_instance->state != MK_STATE_READY) { + pr_err("PCI assignment changes require instance %d to be ready\n", + instance_id); + ret = -EBUSY; + } else { + ret = mk_instance_remove_pci_device(target_instance, domain, bus, + devfn); } - - pr_info("Multikernel hotplug: Device %04x:%02x:%02x.%x successfully removed from instance %d\n", - domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), instance_id); - - ret = 0; -out: mk_instance_put(target_instance); return ret; } diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index f21cc947149f10..df804639af6028 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "internal.h" #define PROP_SUB_FDT "fdt" @@ -34,6 +35,23 @@ struct mk_instance *root_instance = NULL; EXPORT_SYMBOL_GPL(root_instance); +static void __init __noreturn mk_manifest_reject_and_park(int error) +{ + int ret; + + ret = mk_arch_prepare_park(); + if (ret || !mk_arch_park_ready()) + panic("multikernel: rejected manifest before park path became ready"); + ret = mk_register_stop_nmi_handler(); + if (ret) + pr_emerg("multikernel: stop-NMI registration failed while rejecting manifest: %d\n", + ret); + pr_emerg("multikernel: parking CPUs after rejecting supplied manifest: %d\n", + error); + smp_call_function(mk_enter_pool_state, NULL, 0); + mk_enter_pool_state(NULL); +} + /* * Collect every CPU the instance might receive through hotplug later: * the unassigned pool plus every other kernel's CPUs (the host's and @@ -168,6 +186,7 @@ int mk_manifest_add_instance_dtb(struct kimage *image, void *fdt, int mk_id) */ int mk_manifest_add_host_ipi(struct kimage *image, void *fdt) { + mk_phys_cpu_t target_cpu = arch_cpu_physical_id(0); int ret = 0; if (!root_instance->ipi_data) { @@ -175,12 +194,15 @@ int mk_manifest_add_host_ipi(struct kimage *image, void *fdt) return 0; } - pr_info("Preserving host IPI buffer: phys=0x%llx, pages=%u\n", - (unsigned long long)root_instance->ipi_phys, root_instance->ipi_pages); + pr_info("Preserving host IPI buffer: phys=0x%llx, pages=%u, target CPU=%llu\n", + (unsigned long long)root_instance->ipi_phys, + root_instance->ipi_pages, + (unsigned long long)target_cpu); ret |= fdt_begin_node(fdt, "host-ipi-buffer"); ret |= fdt_property_u64(fdt, "phys-addr", root_instance->ipi_phys); ret |= fdt_property_u32(fdt, "pages", root_instance->ipi_pages); + ret |= fdt_property_u64(fdt, "target-cpu", target_cpu); ret |= fdt_end_node(fdt); if (ret) { @@ -386,6 +408,7 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char pr_err("Failed to allocate IPI buffer for instance %d\n", instance_id); goto err_free_name; } + mk_shared_data_reset(instance->ipi_data); instance->ipi_phys = virt_to_phys(instance->ipi_data); instance->ipi_pages = (sizeof(struct mk_shared_data) + PAGE_SIZE - 1) / PAGE_SIZE; @@ -399,10 +422,13 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char goto err_free_ipi; instance->state = MK_STATE_READY; + init_rwsem(&instance->control_route_sem); + instance->irq_route_cpu = MK_PHYS_CPU_INVALID; INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); kref_init(&instance->refcount); INIT_LIST_HEAD(&instance->pci_devices); + mk_pci_lease_instance_init(instance); instance->pci_devices_valid = false; instance->pci_device_count = 0; INIT_LIST_HEAD(&instance->platform_devices); @@ -452,19 +478,13 @@ static int __init mk_copy_pci_devices(const struct mk_dt_config *config, instance->pci_devices_valid = true; list_for_each_entry(src_dev, &config->pci_devices, list) { - dst_dev = kzalloc(sizeof(*dst_dev), GFP_KERNEL); + dst_dev = kmemdup(src_dev, sizeof(*dst_dev), GFP_KERNEL); if (!dst_dev) { pr_err("Failed to allocate PCI device entry\n"); return -ENOMEM; } - dst_dev->vendor = src_dev->vendor; - dst_dev->device = src_dev->device; - dst_dev->domain = src_dev->domain; - dst_dev->bus = src_dev->bus; - dst_dev->slot = src_dev->slot; - dst_dev->func = src_dev->func; - + INIT_LIST_HEAD(&dst_dev->list); list_add_tail(&dst_dev->list, &instance->pci_devices); instance->pci_device_count++; } @@ -543,6 +563,12 @@ static int __init mk_restore_instance_ipi(const void *manifest, struct mk_instan (unsigned long long)ipi_phys, ipi_pages); return 0; } + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small for ABI %u: %zu < %zu\n", + MK_IPI_ABI_VERSION, ipi_size, + sizeof(struct mk_shared_data)); + return -EPROTO; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -564,8 +590,10 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest struct mk_instance *host_instance; int host_ipi_node; const fdt64_t *phys_prop; + const fdt64_t *cpu_prop; const fdt32_t *pages_prop; phys_addr_t host_ipi_phys = 0; + mk_phys_cpu_t host_ipi_cpu = MK_PHYS_CPU_INVALID; u32 host_ipi_pages = 0; size_t host_ipi_size = 0; int len; @@ -585,10 +613,21 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest host_ipi_pages = fdt32_to_cpu(*pages_prop); host_ipi_size = (size_t)host_ipi_pages << PAGE_SHIFT; } + cpu_prop = fdt_getprop(manifest, host_ipi_node, "target-cpu", &len); + if (cpu_prop && len == sizeof(*cpu_prop)) + host_ipi_cpu = fdt64_to_cpu(*cpu_prop); - if (!host_ipi_phys || !host_ipi_pages) { - pr_warn("Incomplete host IPI buffer info (phys=0x%llx, pages=%u)\n", - (unsigned long long)host_ipi_phys, host_ipi_pages); + if (!host_ipi_phys || !host_ipi_pages || + host_ipi_cpu == MK_PHYS_CPU_INVALID) { + pr_warn("Incomplete host IPI buffer info (phys=0x%llx, pages=%u, target CPU=%llu)\n", + (unsigned long long)host_ipi_phys, host_ipi_pages, + (unsigned long long)host_ipi_cpu); + return NULL; + } + if (host_ipi_size < sizeof(struct mk_shared_data)) { + pr_err("Host IPI buffer is too small for ABI %u: %zu < %zu\n", + MK_IPI_ABI_VERSION, host_ipi_size, + sizeof(struct mk_shared_data)); return NULL; } @@ -596,8 +635,7 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest if (!host_instance) return NULL; - /* Set physical CPU 0 as default target for host IPIs */ - if (mk_cpu_set_add(host_instance->cpus, 0)) { + if (mk_cpu_set_add(host_instance->cpus, host_ipi_cpu)) { kfree(host_instance->name); mk_cpu_set_free(host_instance->cpus); kfree(host_instance); @@ -615,9 +653,9 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest } host_instance->ipi_phys = host_ipi_phys; host_instance->ipi_pages = host_ipi_pages; - pr_info("Restored host IPI buffer: phys=0x%llx, virt=%px, pages=%u\n", + pr_info("Restored host IPI buffer: phys=0x%llx, virt=%p, pages=%u, target CPU=%llu\n", (unsigned long long)host_ipi_phys, host_instance->ipi_data, - host_ipi_pages); + host_ipi_pages, (unsigned long long)host_ipi_cpu); pr_info("Registered host instance (ID 0) for spawn→host communication\n"); return host_instance; @@ -645,6 +683,9 @@ int __init mk_instance_restore_from_manifest(void) const void *manifest = NULL; phys_addr_t fdt_phys; + if (mk_manifest_rejected()) + mk_manifest_reject_and_park(-EPROTO); + fdt_phys = mk_manifest_phys(); if (!fdt_phys) { pr_info("No manifest available for multikernel DTB restoration\n"); @@ -682,15 +723,15 @@ int __init mk_instance_restore_from_manifest(void) int mk_node = fdt_subnode_offset(manifest, 0, "multikernel"); if (mk_node < 0) { - pr_info("No multikernel node found in manifest\n"); - ret = 0; + pr_err("No multikernel node found in supplied manifest\n"); + ret = -EINVAL; goto cleanup_fdt; } const void *dtb_data = fdt_getprop(manifest, mk_node, "dtb-data", &dtb_len); if (!dtb_data || dtb_len <= 0) { - pr_info("No dtb-data property found in multikernel node\n"); - ret = 0; + pr_err("No dtb-data property found in multikernel node\n"); + ret = -EINVAL; goto cleanup_fdt; } @@ -787,8 +828,25 @@ int __init mk_instance_restore_from_manifest(void) host_instance = mk_restore_host_instance(manifest); if (!host_instance) - pr_warn("Failed to restore host instance (spawn→host communication unavailable)\n"); - + mk_manifest_reject_and_park(-ENODEV); + + ret = mk_ipi_shared_validate(instance->ipi_data); + if (ret) + mk_manifest_reject_and_park(ret); + ret = mk_ipi_shared_validate(host_instance->ipi_data); + if (ret) + mk_manifest_reject_and_park(ret); + if (!atomic_read_acquire(&host_instance->ipi_data->ready) || + READ_ONCE(host_instance->ipi_data->ready_instance_id) != 0) + mk_manifest_reject_and_park(-EHOSTDOWN); + ret = mk_arch_prepare_park(); + if (ret) + mk_manifest_reject_and_park(ret); + if (!mk_arch_park_ready()) + mk_manifest_reject_and_park(-EIO); + ret = mk_register_stop_nmi_handler(); + if (ret) + mk_manifest_reject_and_park(ret); pr_info("Successfully restored multikernel root instance %d ('%s') from manifest (%d bytes)\n", instance_id, instance_name, dtb_len); mk_dt_config_free(&config); @@ -822,95 +880,14 @@ int __init mk_instance_restore_from_manifest(void) kfree(dtb_virt); cleanup_fdt: early_memunmap((void *)manifest, PAGE_SIZE); + if (ret) + mk_manifest_reject_and_park(ret); return ret; } /* Run at early_initcall to enforce CPU restrictions before per-CPU allocations */ early_initcall(mk_instance_restore_from_manifest); -/** - * mk_pci_should_probe - Check if PCI probing should occur at all - * @bus: PCI bus - * @devfn: device/function number - * - * Called BEFORE any PCI config space reads to determine if probing - * should proceed. This prevents config space accesses to devices - * that are not in the whitelist. - * - * Returns: true if probing should proceed, false to skip entirely - */ -bool mk_pci_should_probe(struct pci_bus *bus, int devfn) -{ - struct mk_pci_device *pci_dev; - u16 domain = pci_domain_nr(bus); - u8 bus_num = bus->number; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - u8 hdr_type; - - if (!root_instance) - return true; - - if (!root_instance->dtb_data) - return true; - - if (!root_instance->pci_devices_valid || root_instance->pci_device_count == 0) - return false; - - list_for_each_entry(pci_dev, &root_instance->pci_devices, list) { - if (pci_dev->domain != domain) - continue; - - /* Exact location match - always allow */ - if (pci_dev->bus == bus_num && - pci_dev->slot == slot && - pci_dev->func == func) - return true; - } - - /* - * Check if any whitelisted device is on a downstream bus. - * If so, this might be a bridge in the path to that device. - */ - list_for_each_entry(pci_dev, &root_instance->pci_devices, list) { - if (pci_dev->domain == domain && pci_dev->bus > bus_num) - goto check_bridge; - } - return false; - -check_bridge: - /* - * There's a whitelisted device on a downstream bus. Check if this - * is a bridge that serves it. - */ - if (pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type) == 0) { - bool is_bridge = ((hdr_type & PCI_HEADER_TYPE_MASK) == PCI_HEADER_TYPE_BRIDGE); - - if (is_bridge) { - u8 secondary_bus = 0, subordinate_bus = 0; - - pci_bus_read_config_byte(bus, devfn, PCI_SECONDARY_BUS, &secondary_bus); - pci_bus_read_config_byte(bus, devfn, PCI_SUBORDINATE_BUS, &subordinate_bus); - - /* - * Allow bridge if there's a whitelisted device on any bus - * between secondary and subordinate (inclusive). - */ - if (secondary_bus > 0 && subordinate_bus >= secondary_bus) { - list_for_each_entry(pci_dev, &root_instance->pci_devices, list) { - if (pci_dev->domain == domain && - pci_dev->bus >= secondary_bus && - pci_dev->bus <= subordinate_bus) - return true; - } - } - } - } - - return false; -} -EXPORT_SYMBOL_GPL(mk_pci_should_probe); - bool mk_platform_device_allowed(const char *name, const char *hid) { struct mk_platform_device *plat_dev; diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 75e4b8d25eb939..bb51152d7ef2f4 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -5,6 +5,19 @@ extern struct idr mk_instance_idr; extern struct list_head mk_instance_list; extern struct mk_instance *root_instance; +/* ipi.c */ +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); +int mk_send_ipi_data_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, void *data, + size_t data_size, unsigned long type); +/* messaging.c */ +int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, + u32 subtype, void *payload, u32 payload_len); +int mk_send_message_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, u32 msg_type, u32 subtype, + void *payload, u32 payload_len); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; @@ -12,12 +25,111 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, int resources_node, size_t dtb_size); struct mk_instance *mk_instance_find_by_name(const char *name); int mk_instance_destroy(struct mk_instance *instance); +int mk_instance_release_resources(struct mk_instance *instance); /* dts.c */ int mk_dt_parse_resources(const void *fdt, int resources_node, const char *instance_name, struct mk_dt_config *config); int mk_dt_generate_instance_dtb(struct mk_instance *instance, void **out_dtb, size_t *out_size); +int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, + u8 *slot, u8 *func); + +/* CPU ownership serialization: transaction must be acquired before ownership. */ +void mk_cpu_transaction_lock(void); +void mk_cpu_transaction_unlock(void); +void mk_cpu_ownership_lock(void); +void mk_cpu_ownership_unlock(void); +void mk_cpu_ownership_assert_held(void); +/* Caller serializes CPU ownership changes with mk_cpu_transaction_lock(). */ +int mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing); + +/* pci.c */ +#ifdef CONFIG_PCI +int mk_pci_lease_system_init(void); +void mk_pci_lease_system_cleanup(void); +void mk_pci_lease_instance_init(struct mk_instance *instance); +bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance); +int mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count); +int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn); +int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn); +int mk_pci_release_assignments(struct mk_instance *instance); + +/* Caller must hold instance->resource_mutex. */ +int mk_pci_quiesce_instance_irqs(struct mk_instance *instance, + bool parked_force); +unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance); +#else +static inline int mk_pci_lease_system_init(void) +{ + return 0; +} + +static inline void mk_pci_lease_system_cleanup(void) +{ +} + +static inline void mk_pci_lease_instance_init(struct mk_instance *instance) +{ + mutex_init(&instance->resource_mutex); + INIT_LIST_HEAD(&instance->pci_assignments); +} + +static inline bool +mk_pci_iommu_lease_active_locked(struct mk_instance *instance) +{ + return false; +} + +static inline int +mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count) +{ + if (requested_count < 0) + return -EINVAL; + + return requested_count ? -EOPNOTSUPP : 0; +} + +static inline int mk_pci_assign_device(struct mk_instance *instance, + u16 domain, u8 bus, u8 devfn) +{ + return -EOPNOTSUPP; +} + +static inline int mk_pci_unassign_device(struct mk_instance *instance, + u16 domain, u8 bus, u8 devfn) +{ + return -EOPNOTSUPP; +} + +static inline int mk_pci_release_assignments(struct mk_instance *instance) +{ + return 0; +} + +/* Caller must hold instance->resource_mutex. */ +static inline int +mk_pci_quiesce_instance_irqs(struct mk_instance *instance, bool parked_force) +{ + return 0; +} + +static inline unsigned int +mk_pci_sync_instance_irq_route(struct mk_instance *instance) +{ + return 0; +} + +#endif + +int mk_instance_force_halt(struct mk_instance *instance); /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 7f4ba78b39b06a..786505dba0e47f 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -12,6 +12,9 @@ #include #include #include +#include +#include +#include #include "internal.h" /* Callback management */ @@ -19,42 +22,728 @@ static struct mk_ipi_handler *mk_handlers; static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); static void mk_ipi_drain_ring(void); +static DECLARE_WAIT_QUEUE_HEAD(mk_reply_waitq); + +#define MK_IPI_PRODUCER_RETRIES 10000 +#define MK_IPI_GATE_INDEX_BITS 6 +#define MK_IPI_GATE_INDEX_MASK (MK_IPI_RING_SIZE - 1) +#define MK_REPLY_STATE_MASK (BIT(MK_REPLY_STATE_BITS) - 1) +#define MK_REPLY_GENERATION_MAX (U64_MAX >> MK_REPLY_STATE_BITS) + +static u64 mk_reply_token(u64 generation, enum mk_reply_state state) +{ + return generation << MK_REPLY_STATE_BITS | state; +} + +static u64 mk_reply_generation(u64 token) +{ + return token >> MK_REPLY_STATE_BITS; +} + +static enum mk_reply_state mk_reply_state(u64 token) +{ + return token & MK_REPLY_STATE_MASK; +} + +/* + * A nonzero gate records both the physical producer CPU and the slot at head. + * This makes the serialization recoverable after that exact CPU is confirmed + * parked. A boolean shared lock would be unsafe because the force-stop NMI may + * prevent its owner from ever returning to release it. + */ +static u64 mk_ipi_gate_token(mk_phys_cpu_t owner, unsigned int idx) +{ + BUILD_BUG_ON(BIT(MK_IPI_GATE_INDEX_BITS) != MK_IPI_RING_SIZE); + if (owner >= (U64_MAX >> MK_IPI_GATE_INDEX_BITS)) + return 0; + + return ((owner + 1) << MK_IPI_GATE_INDEX_BITS) | idx; +} + +static mk_phys_cpu_t mk_ipi_gate_owner(u64 token) +{ + return (token >> MK_IPI_GATE_INDEX_BITS) - 1; +} + +static unsigned int mk_ipi_gate_index(u64 token) +{ + return token & MK_IPI_GATE_INDEX_MASK; +} /* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. + * Publish the complete message while local IRQs and preemption are disabled. + * No NMI path sends general messages; force halt uses emergency_shutdown. + * Advancing head before READY lets recovery distinguish both interruption + * windows without allowing another producer to pass the gate. */ -static inline unsigned int mk_ring_idx(unsigned int i) +static int mk_ipi_ring_publish(struct mk_shared_data *shared, int instance_id, + const void *data, size_t data_size, + unsigned long type) +{ + struct mk_ipi_ring *ring = &shared->ring; + struct mk_ipi_data *slot; + mk_phys_cpu_t owner; + unsigned long flags; + bool contended = false; + unsigned int retry; + unsigned int idx; + u64 token, old; + int state; + int head; + int ret; + + preempt_disable(); + local_irq_save(flags); + owner = arch_cpu_physical_id(smp_processor_id()); + + for (retry = 0; retry < MK_IPI_PRODUCER_RETRIES; retry++) { + head = atomic_read(&ring->head); + idx = head & MK_IPI_GATE_INDEX_MASK; + token = mk_ipi_gate_token(owner, idx); + if (!token) { + ret = -EOVERFLOW; + goto out_restore; + } + + old = atomic64_cmpxchg_acquire(&ring->producer_gate, 0, token); + if (!old) { + if ((atomic_read(&ring->head) & + MK_IPI_GATE_INDEX_MASK) != idx) { + atomic64_set_release(&ring->producer_gate, 0); + contended = true; + cpu_relax(); + continue; + } + break; + } + contended = true; + if (mk_ipi_gate_owner(old) == owner) { + ret = -EDEADLK; + goto out_count_contention; + } + cpu_relax(); + } + + if (retry == MK_IPI_PRODUCER_RETRIES) { + ret = -EAGAIN; + goto out_count_contention; + } + if (contended) + atomic_inc(&ring->producer_contention); + if (!atomic_read_acquire(&shared->ready) || + READ_ONCE(shared->ready_instance_id) != instance_id) { + ret = -ESHUTDOWN; + goto out_release_gate; + } + + slot = &ring->entries[idx]; + state = atomic_cmpxchg(&slot->state, MK_IPI_SLOT_EMPTY, + MK_IPI_SLOT_WRITING); + if (state != MK_IPI_SLOT_EMPTY) { + if (state == MK_IPI_SLOT_READY || + state == MK_IPI_SLOT_CONSUMING || + state == MK_IPI_SLOT_CANCELLED) { + atomic_inc(&ring->full_failures); + ret = -ENOSPC; + } else { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + goto out_release_gate; + } + + WRITE_ONCE(slot->data_size, 0); + WRITE_ONCE(slot->sender_cpu, owner); + WRITE_ONCE(slot->type, type); + if (data_size) + memcpy(slot->buffer, data, data_size); + WRITE_ONCE(slot->data_size, data_size); + atomic_set(&ring->head, (idx + 1) & MK_IPI_GATE_INDEX_MASK); + atomic_set_release(&slot->state, MK_IPI_SLOT_READY); + ret = 0; + +out_release_gate: + atomic64_set_release(&ring->producer_gate, 0); +out_restore: + local_irq_restore(flags); + preempt_enable(); + return ret; + +out_count_contention: + atomic_inc(&ring->producer_contention); + goto out_restore; +} + +static bool mk_ipi_slot_is_pending(int state) +{ + return state == MK_IPI_SLOT_READY || + state == MK_IPI_SLOT_CANCELLED; +} + +static void mk_ipi_slot_release(struct mk_ipi_data *slot) +{ + WRITE_ONCE(slot->data_size, 0); + atomic_set_release(&slot->state, MK_IPI_SLOT_EMPTY); +} + +int mk_ipi_shared_validate(const struct mk_shared_data *shared) +{ + if (!shared) + return -ENODEV; + if (READ_ONCE(shared->abi_magic) != MK_IPI_ABI_MAGIC || + READ_ONCE(shared->abi_version) != MK_IPI_ABI_VERSION || + READ_ONCE(shared->abi_size) != sizeof(*shared)) + return -EPROTO; + + return 0; +} + +int mk_ipi_shared_mark_ready(struct mk_shared_data *shared, int instance_id) +{ + int ret; + + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + WRITE_ONCE(shared->ready_instance_id, instance_id); + atomic_set_release(&shared->ready, 1); + return 0; +} + +int mk_reply_reserve(struct mk_shared_data *shared, u32 kind, u64 request_id, + struct mk_reply_handle *reply) +{ + struct mk_reply_table *table; + unsigned int i; + int ret; + + if (!reply || !request_id || !kind) + return -EINVAL; + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + table = &shared->replies; + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &table->slots[i]; + u64 generation; + u64 claim; + u64 old; + + old = atomic64_read(&slot->state_generation); + if (mk_reply_state(old) != MK_REPLY_FREE) + continue; + generation = mk_reply_generation(old) + 1; + if (!generation || generation > MK_REPLY_GENERATION_MAX) + generation = 1; + claim = mk_reply_token(generation, MK_REPLY_WRITING); + if (atomic64_cmpxchg_acquire(&slot->state_generation, old, + claim) != old) + continue; + + WRITE_ONCE(slot->request_id, request_id); + WRITE_ONCE(slot->kind, kind); + WRITE_ONCE(slot->status, -ETIMEDOUT); + WRITE_ONCE(slot->value, ~0U); + atomic64_set_release(&slot->state_generation, + mk_reply_token(generation, + MK_REPLY_RESERVED)); + reply->slot = i; + reply->kind = kind; + reply->request_id = request_id; + reply->generation = generation; + return 0; + } + + atomic_inc(&table->occupied_failures); + return -ENOSPC; +} + +static int mk_reply_take_ready(struct mk_shared_data *shared, + struct mk_reply_handle *reply, + s32 *status, u32 *value) +{ + struct mk_reply_slot *slot = &shared->replies.slots[reply->slot]; + u64 ready = mk_reply_token(reply->generation, MK_REPLY_READY); + u64 free = mk_reply_token(reply->generation, MK_REPLY_FREE); + + if (atomic64_read_acquire(&slot->state_generation) != ready) + return -EAGAIN; + if (READ_ONCE(slot->request_id) != reply->request_id || + READ_ONCE(slot->kind) != reply->kind) + return -EPROTO; + if (status) + *status = READ_ONCE(slot->status); + if (value) + *value = READ_ONCE(slot->value); + if (atomic64_cmpxchg_release(&slot->state_generation, ready, free) != + ready) + return -EAGAIN; + return 0; +} + +static int mk_reply_cancel(struct mk_shared_data *shared, + struct mk_reply_handle *reply, bool atomic_timeout) +{ + struct mk_reply_table *table = &shared->replies; + struct mk_reply_slot *slot = &table->slots[reply->slot]; + u64 reserved = mk_reply_token(reply->generation, MK_REPLY_RESERVED); + u64 writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + u64 executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + u64 committed = mk_reply_token(reply->generation, MK_REPLY_COMMITTED); + u64 abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + u64 ready = mk_reply_token(reply->generation, MK_REPLY_READY); + u64 free = mk_reply_token(reply->generation, MK_REPLY_FREE); + u64 token; + + if (atomic64_cmpxchg_release(&slot->state_generation, reserved, free) == + reserved) + goto cancelled; + + for (;;) { + token = atomic64_read_acquire(&slot->state_generation); + if (token == ready) + return 1; + if (token == writing && + atomic64_cmpxchg_release(&slot->state_generation, writing, + abandoned) == writing) + break; + if (token == executing && + atomic64_cmpxchg_release(&slot->state_generation, executing, + committed) == executing) { + atomic_inc(&table->indeterminate_timeouts); + if (atomic_timeout) + atomic_inc(&table->atomic_timeouts); + return -EINPROGRESS; + } + if (token != writing && token != executing) + break; + } + +cancelled: + atomic_inc(&table->cancelled_slots); + if (atomic_timeout) + atomic_inc(&table->atomic_timeouts); + return 0; +} + +static bool mk_reply_wait_done(struct mk_shared_data *shared, + const struct mk_reply_handle *reply) { - return i & (MK_IPI_RING_SIZE - 1); + struct mk_reply_slot *slot = &shared->replies.slots[reply->slot]; + u64 token; + + token = atomic64_read_acquire(&slot->state_generation); + return token == mk_reply_token(reply->generation, MK_REPLY_READY) || + mk_reply_generation(token) != reply->generation || + mk_reply_state(token) == MK_REPLY_FREE; +} + +int mk_reply_wait_atomic(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_us, + s32 *status, u32 *value) +{ + u64 deadline; + int cancelled; + + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + deadline = ktime_get_mono_fast_ns() + (u64)timeout_us * NSEC_PER_USEC; + for (;;) { + int ret = mk_reply_take_ready(shared, reply, status, value); + + if (!ret) + return 0; + if (ret != -EAGAIN) + return ret; + if (ktime_get_mono_fast_ns() >= deadline) + break; + cpu_relax(); + } + + cancelled = mk_reply_cancel(shared, reply, true); + if (cancelled > 0) + return mk_reply_take_ready(shared, reply, status, value); + if (cancelled < 0) + return cancelled; + return -ETIMEDOUT; +} + +int mk_reply_wait(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_ms, + s32 *status, u32 *value) +{ + long waited; + int cancelled; + int ret; + + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + waited = wait_event_timeout(mk_reply_waitq, + mk_reply_wait_done(shared, reply), + msecs_to_jiffies(timeout_ms)); + if (!waited) { + cancelled = mk_reply_cancel(shared, reply, false); + if (cancelled > 0) + return mk_reply_take_ready(shared, reply, status, value); + if (cancelled < 0) + return cancelled; + return -ETIMEDOUT; + } + ret = mk_reply_take_ready(shared, reply, status, value); + if (ret == -EAGAIN) + ret = -ESTALE; + return ret; +} + +void mk_reply_release(struct mk_shared_data *shared, + struct mk_reply_handle *reply) +{ + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return; + mk_reply_cancel(shared, reply, false); +} + +int mk_reply_claim(struct mk_instance *instance, + const struct mk_reply_handle *reply) +{ + struct mk_shared_data *shared; + struct mk_reply_slot *slot; + u64 writing; + u64 abandoned; + u64 reserved; + u64 free; + int ret; + + if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + shared = instance->ipi_data; + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + if (!atomic_read_acquire(&shared->ready) || + READ_ONCE(shared->ready_instance_id) != instance->id) + return -ESHUTDOWN; + + slot = &shared->replies.slots[reply->slot]; + reserved = mk_reply_token(reply->generation, MK_REPLY_RESERVED); + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + free = mk_reply_token(reply->generation, MK_REPLY_FREE); + if (atomic64_cmpxchg_acquire(&slot->state_generation, reserved, + writing) != reserved) { + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (READ_ONCE(slot->request_id) != reply->request_id || + READ_ONCE(slot->kind) != reply->kind) { + u64 old; + + old = atomic64_cmpxchg_release(&slot->state_generation, writing, + free); + if (old == abandoned) + old = atomic64_cmpxchg_release(&slot->state_generation, + abandoned, free); + if (old == writing || old == abandoned) + wake_up_all(&mk_reply_waitq); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + return 0; +} + +int mk_reply_begin_execute(struct mk_instance *instance, + const struct mk_reply_handle *reply) +{ + struct mk_reply_slot *slot; + u64 writing; + u64 executing; + u64 old; + + if (!instance || !instance->ipi_data || !reply || + reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + slot = &instance->ipi_data->replies.slots[reply->slot]; + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + old = atomic64_cmpxchg_acquire(&slot->state_generation, writing, + executing); + if (old == writing) + return 0; + if (old == mk_reply_token(reply->generation, MK_REPLY_ABANDONED)) + return -ECANCELED; + return -ESTALE; +} + +int mk_reply_publish(struct mk_instance *instance, + const struct mk_reply_handle *reply, s32 status, u32 value) +{ + struct mk_shared_data *shared; + struct mk_reply_slot *slot; + mk_phys_cpu_t target; + u64 writing; + u64 executing; + u64 committed; + u64 abandoned; + u64 ready; + u64 free; + u64 old; + int ret; + + if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + shared = instance->ipi_data; + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + slot = &shared->replies.slots[reply->slot]; + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + committed = mk_reply_token(reply->generation, MK_REPLY_COMMITTED); + abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + ready = mk_reply_token(reply->generation, MK_REPLY_READY); + free = mk_reply_token(reply->generation, MK_REPLY_FREE); + + WRITE_ONCE(slot->status, status); + WRITE_ONCE(slot->value, value); + old = atomic64_cmpxchg_release(&slot->state_generation, executing, ready); + if (old == writing) + old = atomic64_cmpxchg_release(&slot->state_generation, writing, + ready); + if (old == abandoned) { + atomic64_set_release(&slot->state_generation, free); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (old == committed) { + atomic64_set_release(&slot->state_generation, free); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (old != executing && old != writing) { + atomic_inc(&shared->replies.late_replies); + return -EIO; + } + + target = mk_cpu_set_first(instance->cpus); + if (target == MK_PHYS_CPU_INVALID) + return -ENODEV; + mk_arch_send_ipi(target); + return 0; +} + +void mk_reply_scan(struct mk_shared_data *shared) +{ + unsigned int i; + + if (!shared) + return; + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &shared->replies.slots[i]; + u64 token = atomic64_read_acquire(&slot->state_generation); + + if (mk_reply_state(token) == MK_REPLY_READY) { + wake_up_all(&mk_reply_waitq); + return; + } + } +} + +int mk_ipi_shared_wait_ready(struct mk_shared_data *shared, int instance_id, + unsigned int timeout_ms) +{ + unsigned long deadline; + int ret; + + ret = mk_ipi_shared_validate(shared); + if (ret) + return ret; + + deadline = jiffies + msecs_to_jiffies(timeout_ms); + do { + if (atomic_read_acquire(&shared->ready)) + return READ_ONCE(shared->ready_instance_id) == instance_id ? + 0 : -EPROTO; + msleep(20); + } while (time_before(jiffies, deadline)); + + return -ETIMEDOUT; +} + +int mk_ipi_shared_reset_downlink(struct mk_shared_data *shared) +{ + struct mk_ipi_ring *ring; + mk_phys_cpu_t owner; + unsigned long flags; + unsigned int retry; + unsigned int idx; + u64 epoch; + u64 token, old; + int head; + int ret = 0; + + if (!shared) + return -EINVAL; + + /* Exclude new publishers before waiting for an in-flight one. */ + atomic_set_release(&shared->ready, 0); + /* Pair exclusion with the readiness recheck after gate acquisition. */ + smp_mb(); + ring = &shared->ring; + preempt_disable(); + local_irq_save(flags); + owner = arch_cpu_physical_id(smp_processor_id()); + for (retry = 0; retry < MK_IPI_PRODUCER_RETRIES; retry++) { + head = atomic_read(&ring->head); + idx = head & MK_IPI_GATE_INDEX_MASK; + token = mk_ipi_gate_token(owner, idx); + if (!token) { + ret = -EOVERFLOW; + goto out_restore; + } + old = atomic64_cmpxchg_acquire(&ring->producer_gate, 0, token); + if (!old) + break; + if (mk_ipi_gate_owner(old) == owner) { + ret = -EDEADLK; + goto out_restore; + } + cpu_relax(); + } + if (retry == MK_IPI_PRODUCER_RETRIES) { + ret = -EAGAIN; + goto out_restore; + } + + /* The old receiver is parked and every pre-existing publisher drained. */ + atomic_set(&shared->emergency_shutdown, 0); + mk_ipi_ring_reset_contents(ring); + mk_reply_table_reset(&shared->replies); + mk_irq_mailbox_reset(&shared->irq_mailbox); + epoch = READ_ONCE(shared->spawn_epoch) + 1; + if (!epoch) + epoch = 1; + WRITE_ONCE(shared->spawn_epoch, epoch); + WRITE_ONCE(shared->abi_magic, MK_IPI_ABI_MAGIC); + WRITE_ONCE(shared->abi_version, MK_IPI_ABI_VERSION); + WRITE_ONCE(shared->abi_size, sizeof(*shared)); + WRITE_ONCE(shared->ready_instance_id, -1); + atomic_set(&shared->ready, 0); + atomic64_set_release(&ring->producer_gate, 0); + +out_restore: + local_irq_restore(flags); + preempt_enable(); + return ret; } /** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring + * mk_ipi_ring_recover_halted - Recover a ring producer after it is parked + * @halted_cpus: Exact set of CPUs confirmed parked by the caller * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. + * A force-stop NMI can park a producer while it owns the shared gate. Only its + * receiver may recover the write, and only after the owner CPU is proven not + * to be executing it. Other instances may still publish into the host ring, + * so this function never resets the ring or touches another owner's gate. */ -void mk_ipi_ring_drop_pending(void) +int mk_ipi_ring_recover_halted(const struct mk_cpu_set *halted_cpus) { + struct mk_ipi_data *slot; struct mk_ipi_ring *ring; - unsigned int head, tail; + mk_phys_cpu_t owner; + mk_phys_cpu_t target; + unsigned int idx; + unsigned int next; + u64 token; + int state; + int head; + int ret = 0; - if (!root_instance || !root_instance->ipi_data) - return; + if (!halted_cpus || !root_instance || !root_instance->ipi_data) + return -EINVAL; ring = &root_instance->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); + token = atomic64_read_acquire(&ring->producer_gate); + if (!token) + goto kick; + if (!(token >> MK_IPI_GATE_INDEX_BITS)) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + goto kick; + } - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; + owner = mk_ipi_gate_owner(token); + if (!mk_cpu_set_contains(halted_cpus, owner)) + goto kick; + + idx = mk_ipi_gate_index(token); + next = (idx + 1) & MK_IPI_GATE_INDEX_MASK; + slot = &ring->entries[idx]; + state = atomic_read_acquire(&slot->state); + head = atomic_read(&ring->head) & MK_IPI_GATE_INDEX_MASK; + switch (state) { + case MK_IPI_SLOT_EMPTY: + if (head != idx && head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + case MK_IPI_SLOT_WRITING: + if (head == idx) { + atomic_set(&ring->head, next); + } else if (head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + break; + } + atomic_set_release(&slot->state, MK_IPI_SLOT_CANCELLED); + atomic_inc(&ring->cancelled_writes); + break; + case MK_IPI_SLOT_READY: + case MK_IPI_SLOT_CANCELLED: + if (head == idx) { + /* Repair an interrupted publication before releasing its gate. */ + atomic_set(&ring->head, next); + atomic_inc(&ring->invalid_state); + } else if (head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + case MK_IPI_SLOT_CONSUMING: + /* + * The consumer may claim a full-ring tail while this producer + * waits to test it, leaving head at idx with nothing published. + * Head at next means the previous publication was claimed before + * its now-stale gate could be released. Both cursors are valid. + */ + if (head != idx && head != next) { + atomic_inc(&ring->invalid_state); + ret = -EIO; + } + break; + default: + atomic_inc(&ring->invalid_state); + ret = -EIO; + break; + } + + /* Keep the gate closed when cursor repair cannot make the FIFO safe. */ + if (ret) + goto kick; + if (atomic64_cmpxchg_release(&ring->producer_gate, token, 0) != token) { + atomic_inc(&ring->invalid_state); + ret = -EAGAIN; + } - atomic_set(&ring->tail, head); +kick: + /* The producer may have published and parked before ringing the bell. */ + target = mk_cpu_set_first(root_instance->cpus); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + + return ret; } /** @@ -119,7 +808,7 @@ EXPORT_SYMBOL(multikernel_unregister_handler); /** * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID + * @instance: Target multikernel instance * @data: Pointer to data to send * @data_size: Size of data * @type: User-defined type identifier @@ -129,26 +818,18 @@ EXPORT_SYMBOL(multikernel_unregister_handler); * * Returns 0 on success, negative error code on failure */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +static int __mk_send_ipi_data(struct mk_instance *instance, + mk_phys_cpu_t target, void *data, + size_t data_size, unsigned long type) { - struct mk_ipi_data *slot; - struct mk_instance *instance = mk_instance_find(instance_id); - unsigned int head, next_head, tail; - mk_phys_cpu_t target; + int instance_id; + int ret; - if (!instance) + if (!instance || target == MK_PHYS_CPU_INVALID) return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); + instance_id = instance->id; + if (data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; - } - - target = mk_cpu_set_first(instance->cpus); - if (target == MK_PHYS_CPU_INVALID) { - pr_err("Instance %d has no CPUs to receive the IPI\n", instance_id); - mk_instance_put(instance); - return -ENODEV; - } if (!instance->ipi_data) { struct mk_shared_data *ipi_data = NULL; @@ -165,98 +846,121 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns if (!instance->ipi_data) { pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); return -ENODEV; } + ret = mk_ipi_shared_validate(instance->ipi_data); + if (ret) + return ret; + if (!atomic_read_acquire(&instance->ipi_data->ready)) + return -EAGAIN; + if (READ_ONCE(instance->ipi_data->ready_instance_id) != instance_id) + return -EPROTO; - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&instance->ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&instance->ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - mk_instance_put(instance); - return -ENOSPC; - } - - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&instance->ipi_data->ring.head, head, next_head) != head); + ret = mk_ipi_ring_publish(instance->ipi_data, instance_id, data, + data_size, type); + if (ret) { + /* + * A doorbell can be coalesced while the target is draining this + * ring. Kick it again before reporting backpressure so READY + * entries cannot remain stranded without another notification. + */ + mk_arch_send_ipi(target); + if (ret == -ENOSPC) + pr_warn_ratelimited("multikernel: IPI ring full for instance %d\n", + instance_id); + else if (ret == -EAGAIN) + pr_warn_ratelimited("multikernel: IPI producer busy for instance %d\n", + instance_id); + else if (ret != -EDEADLK) + pr_err_ratelimited("multikernel: IPI publish failed for instance %d: %d\n", + instance_id, ret); + return ret; + } + mk_arch_send_ipi(target); - /* We've claimed slot 'head', now fill it */ - slot = &instance->ipi_data->ring.entries[head]; + return 0; +} - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; +int mk_send_ipi_data_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, void *data, + size_t data_size, unsigned long type) +{ + return __mk_send_ipi_data(instance, target, data, data_size, type); +} - if (data && data_size > 0) - memcpy(slot->buffer, data, data_size); +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) +{ + mk_phys_cpu_t target; - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); + if (!instance) + return -EINVAL; + target = mk_cpu_set_first(instance->cpus); + if (target == MK_PHYS_CPU_INVALID) { + pr_err("Instance %d has no CPUs to receive the IPI\n", + instance->id); + return -ENODEV; + } + return __mk_send_ipi_data(instance, target, data, data_size, type); +} - mk_arch_send_ipi(target); +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + struct mk_instance *instance; + int ret; + instance = mk_instance_find(instance_id); + if (!instance) + return -EINVAL; + ret = mk_send_ipi_data(instance, data, data_size, type); mk_instance_put(instance); - return 0; + return ret; } static void mk_ipi_drain_ring(void) { struct mk_ipi_data *slot; struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; + struct mk_ipi_ring *ring; + unsigned int tail, idx; size_t data_size; + int state; int messages_processed = 0; if (!root_instance || !root_instance->ipi_data) return; - while (1) { - tail = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.head)); + ring = &root_instance->ipi_data->ring; + while (messages_processed < MK_IPI_RING_SIZE) { + tail = atomic_read(&ring->tail); + idx = tail & (MK_IPI_RING_SIZE - 1); + slot = &ring->entries[idx]; - if (tail == head) + state = atomic_read_acquire(&slot->state); + if (!mk_ipi_slot_is_pending(state)) { + if (state != MK_IPI_SLOT_EMPTY && + state != MK_IPI_SLOT_WRITING && + state != MK_IPI_SLOT_CONSUMING) { + atomic_inc(&ring->invalid_state); + pr_warn_once("Multikernel IPI slot %u has bad state %d\n", + idx, state); + } break; + } - slot = &root_instance->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + if (atomic_cmpxchg_acquire(&slot->state, state, + MK_IPI_SLOT_CONSUMING) != state) break; + if (state == MK_IPI_SLOT_CANCELLED) + goto advance_tail; + + data_size = READ_ONCE(slot->data_size); if (data_size > MK_MAX_DATA_SIZE) { pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); - continue; + idx, data_size); + goto advance_tail; } /* Dispatch to registered handler */ @@ -274,14 +978,9 @@ static void mk_ipi_drain_ring(void) raw_spin_unlock(&mk_handlers_lock); advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); + mk_ipi_slot_release(slot); + atomic_set(&ring->tail, (idx + 1) & (MK_IPI_RING_SIZE - 1)); messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; } } @@ -295,6 +994,8 @@ static void multikernel_interrupt_handler(void) { if (!root_instance || !root_instance->ipi_data) return; + mk_reply_scan(root_instance->ipi_data); + mk_pci_irq_mailbox_drain(root_instance->ipi_data); /* * Drain here rather than from irq_work. We are already in interrupt @@ -320,8 +1021,9 @@ void generic_multikernel_interrupt(void) /** * mk_has_pending_shutdown - Check if there's a pending shutdown message * - * Peeks at the IPI ring buffer to check for a MK_SYS_SHUTDOWN message - * with MK_SHUTDOWN_IMMEDIATE flag. Used by NMI handler for force halt. + * Checks the dedicated emergency flag published before force-halt NMIs. + * This path must not depend on claiming a normal IPI ring slot because a + * crashed producer can leave slots unavailable. * * Safe to call from NMI context (no locks, read-only peek). * @@ -329,41 +1031,8 @@ void generic_multikernel_interrupt(void) */ bool mk_has_pending_shutdown(void) { - struct mk_ipi_data *slot; - struct mk_message *msg; - struct mk_shutdown_payload *payload; - unsigned int head, tail, idx, scanned; - if (!root_instance || !root_instance->ipi_data) return false; - tail = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.head)); - - /* - * Scan pending messages without consuming them. The trip count is - * bounded by the ring size rather than by the indices alone: this - * runs in NMI context, where a never-terminating loop takes the CPU - * out permanently with NMIs latched. - */ - for (scanned = 0, idx = tail; - idx != head && scanned < MK_IPI_RING_SIZE; - idx = mk_ring_idx(idx + 1), scanned++) { - slot = &root_instance->ipi_data->ring.entries[idx]; - - if (slot->data_size < sizeof(struct mk_message)) - continue; - - msg = (struct mk_message *)slot->buffer; - if (msg->msg_type != MK_MSG_SYSTEM || msg->msg_subtype != MK_SYS_SHUTDOWN) - continue; - - if (msg->payload_len >= sizeof(struct mk_shutdown_payload)) { - payload = (struct mk_shutdown_payload *)msg->payload; - if (payload->flags & MK_SHUTDOWN_IMMEDIATE) - return true; - } - } - - return false; + return atomic_read_acquire(&root_instance->ipi_data->emergency_shutdown); } diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index a0573fc23a1c99..43ad19a4a6edaf 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -126,6 +126,156 @@ static int status_seq_show(struct seq_file *sf, void *v) return 0; } +/* + * This is an observational snapshot, not a transaction: producers may update + * fields while the file is read. Shared transport counters are u32 values + * and restart with a new spawn_epoch, so consumers compare modulo-u32 deltas + * only between samples from the same epoch. + */ +static int stats_seq_show(struct seq_file *sf, void *v) +{ + struct kernfs_open_file *of = sf->private; + struct mk_instance *instance = of->kn->priv; + struct mk_shared_data *shared; + struct { + u64 spawn_epoch; + u64 irq_pending_total; + u32 ipi_producer_contention; + u32 ipi_full_failures; + u32 ipi_invalid_state; + u32 ipi_cancelled_writes; + u32 reply_late_replies; + u32 reply_cancelled_slots; + u32 reply_atomic_timeouts; + u32 reply_indeterminate_timeouts; + u32 reply_occupied_failures; + u32 irq_recorded; + u32 irq_coalesced; + u32 irq_masked_deferred; + u32 irq_stale; + u32 irq_dispatch_failed; + u32 irq_saturated; + u32 ipi_slots[4]; + u32 reply_states[MK_REPLY_ABANDONED + 1]; + u32 irq_active; + u32 irq_pending; + u32 irq_masked; + u32 irq_consuming; + u32 reply_busy; + u32 ipi_gate_busy; + bool available; + } stats = {}; + unsigned int i; + + down_read(&instance->control_route_sem); + shared = READ_ONCE(instance->ipi_data); + if (!shared) + goto unlock; + stats.available = true; + stats.spawn_epoch = READ_ONCE(shared->spawn_epoch); + stats.ipi_producer_contention = + atomic_read(&shared->ring.producer_contention); + stats.ipi_full_failures = atomic_read(&shared->ring.full_failures); + stats.ipi_invalid_state = atomic_read(&shared->ring.invalid_state); + stats.ipi_cancelled_writes = atomic_read(&shared->ring.cancelled_writes); + stats.ipi_gate_busy = !!atomic64_read_acquire(&shared->ring.producer_gate); + stats.reply_late_replies = atomic_read(&shared->replies.late_replies); + stats.reply_cancelled_slots = atomic_read(&shared->replies.cancelled_slots); + stats.reply_atomic_timeouts = atomic_read(&shared->replies.atomic_timeouts); + stats.reply_indeterminate_timeouts = + atomic_read(&shared->replies.indeterminate_timeouts); + stats.reply_occupied_failures = + atomic_read(&shared->replies.occupied_failures); + stats.irq_recorded = atomic_read(&shared->irq_mailbox.recorded); + stats.irq_coalesced = atomic_read(&shared->irq_mailbox.coalesced); + stats.irq_masked_deferred = + atomic_read(&shared->irq_mailbox.masked_deferred); + stats.irq_stale = atomic_read(&shared->irq_mailbox.stale); + stats.irq_dispatch_failed = + atomic_read(&shared->irq_mailbox.dispatch_failed); + stats.irq_saturated = atomic_read(&shared->irq_mailbox.saturated); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + u32 state = atomic_read_acquire(&shared->ring.entries[i].state); + + if (state >= MK_IPI_SLOT_WRITING && state <= MK_IPI_SLOT_CANCELLED) + stats.ipi_slots[state - MK_IPI_SLOT_WRITING]++; + } + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &shared->replies.slots[i]; + u64 token = atomic64_read_acquire(&slot->state_generation); + u32 state = token & (BIT(MK_REPLY_STATE_BITS) - 1); + + if (state <= MK_REPLY_ABANDONED) + stats.reply_states[state]++; + if (state != MK_REPLY_FREE) + stats.reply_busy++; + } + for (i = 0; i < MK_IRQ_MAILBOX_SLOTS; i++) { + struct mk_irq_mailbox_entry *entry = + &shared->irq_mailbox.entries[i]; + u64 token = atomic64_read_acquire(&entry->pending_generation); + + if (!mk_irq_mailbox_generation(token)) + continue; + stats.irq_active++; + if (mk_irq_mailbox_pending(token)) { + stats.irq_pending++; + stats.irq_pending_total += mk_irq_mailbox_pending(token); + } + if (mk_irq_mailbox_masked(token)) + stats.irq_masked++; + if (mk_irq_mailbox_consuming(token)) + stats.irq_consuming++; + } +unlock: + up_read(&instance->control_route_sem); + + seq_puts(sf, "stats_version 1\n"); + seq_printf(sf, "abi_version %u\n", MK_IPI_ABI_VERSION); + seq_puts(sf, "snapshot_atomic 0\n"); + seq_puts(sf, "transport_counter_bits 32\n"); + seq_printf(sf, "transport_available %u\n", stats.available); + seq_printf(sf, "spawn_epoch %llu\n", + (unsigned long long)stats.spawn_epoch); +#define MK_STATS_VALUE(name, value) seq_printf(sf, name " %u\n", value) + MK_STATS_VALUE("ipi.producer_contention", stats.ipi_producer_contention); + MK_STATS_VALUE("ipi.full_failures", stats.ipi_full_failures); + MK_STATS_VALUE("ipi.invalid_state", stats.ipi_invalid_state); + MK_STATS_VALUE("ipi.cancelled_writes", stats.ipi_cancelled_writes); + MK_STATS_VALUE("ipi.gate_busy", stats.ipi_gate_busy); + MK_STATS_VALUE("ipi.slot_writing", stats.ipi_slots[0]); + MK_STATS_VALUE("ipi.slot_ready", stats.ipi_slots[1]); + MK_STATS_VALUE("ipi.slot_consuming", stats.ipi_slots[2]); + MK_STATS_VALUE("ipi.slot_cancelled", stats.ipi_slots[3]); + MK_STATS_VALUE("reply.late_replies", stats.reply_late_replies); + MK_STATS_VALUE("reply.cancelled_slots", stats.reply_cancelled_slots); + MK_STATS_VALUE("reply.atomic_timeouts", stats.reply_atomic_timeouts); + MK_STATS_VALUE("reply.indeterminate_timeouts", + stats.reply_indeterminate_timeouts); + MK_STATS_VALUE("reply.occupied_failures", stats.reply_occupied_failures); + MK_STATS_VALUE("reply.slot_busy", stats.reply_busy); + MK_STATS_VALUE("reply.slot_reserved", stats.reply_states[MK_REPLY_RESERVED]); + MK_STATS_VALUE("reply.slot_writing", stats.reply_states[MK_REPLY_WRITING]); + MK_STATS_VALUE("reply.slot_executing", stats.reply_states[MK_REPLY_EXECUTING]); + MK_STATS_VALUE("reply.slot_committed", stats.reply_states[MK_REPLY_COMMITTED]); + MK_STATS_VALUE("reply.slot_ready", stats.reply_states[MK_REPLY_READY]); + MK_STATS_VALUE("reply.slot_abandoned", stats.reply_states[MK_REPLY_ABANDONED]); + MK_STATS_VALUE("irq.recorded", stats.irq_recorded); + MK_STATS_VALUE("irq.coalesced", stats.irq_coalesced); + MK_STATS_VALUE("irq.masked_deferred", stats.irq_masked_deferred); + MK_STATS_VALUE("irq.stale", stats.irq_stale); + MK_STATS_VALUE("irq.dispatch_failed", stats.irq_dispatch_failed); + MK_STATS_VALUE("irq.saturated", stats.irq_saturated); + MK_STATS_VALUE("irq.slot_active", stats.irq_active); + MK_STATS_VALUE("irq.slot_pending", stats.irq_pending); + MK_STATS_VALUE("irq.slot_masked", stats.irq_masked); + MK_STATS_VALUE("irq.slot_consuming", stats.irq_consuming); +#undef MK_STATS_VALUE + seq_printf(sf, "irq.pending_total %llu\n", + (unsigned long long)stats.irq_pending_total); + return 0; +} + /* Root-level device_tree attribute - shows current baseline pool from kernel structures */ static int root_device_tree_seq_show(struct seq_file *sf, void *v) { @@ -239,6 +389,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, struct kernfs_node *kn; struct mk_dt_config config; void *dtb_copy; + int release_ret; int ret; int allocated_id; @@ -268,7 +419,10 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); + init_rwsem(&instance->control_route_sem); + instance->irq_route_cpu = MK_PHYS_CPU_INVALID; INIT_LIST_HEAD(&instance->pci_devices); + mk_pci_lease_instance_init(instance); INIT_LIST_HEAD(&instance->platform_devices); kref_init(&instance->refcount); @@ -333,7 +487,16 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, kfree(instance->dtb_data); instance->dtb_data = NULL; err_free_resources: - mk_instance_free_memory(instance); + release_ret = mk_instance_release_resources(instance); + if (release_ret) { + pr_crit("Retaining failed instance '%s' because PCI cleanup failed: %d\n", + name, release_ret); + list_add_tail(&instance->list, &mk_instance_list); + kernfs_activate(kn); + mk_instance_set_state(instance, MK_STATE_FAILED); + mk_dt_config_free(&config); + return release_ret; + } err_free_idr: idr_remove(&mk_instance_idr, instance->id); err_remove_dir: @@ -360,6 +523,10 @@ static const struct kernfs_ops mk_status_ops = { .seq_show = status_seq_show, }; +static const struct kernfs_ops mk_stats_ops = { + .seq_show = stats_seq_show, +}; + /* Root-level device_tree operations */ static const struct kernfs_ops mk_root_device_tree_ops = { .seq_show = root_device_tree_seq_show, @@ -394,6 +561,13 @@ static int mk_create_instance_files(struct mk_instance *instance) pr_err("Failed to create status file for instance %s\n", instance->name); return PTR_ERR(kn); } + kn = __kernfs_create_file(instance->kn, "stats", 0444, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, + &mk_stats_ops, instance, NULL, NULL); + if (IS_ERR(kn)) { + pr_err("Failed to create stats file for instance %s\n", instance->name); + return PTR_ERR(kn); + } kn = __kernfs_create_file(instance->kn, "device_tree", 0444, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, @@ -435,6 +609,8 @@ static int mk_kernfs_rmdir(struct kernfs_node *kn) */ int mk_instance_destroy(struct mk_instance *instance) { + int ret; + lockdep_assert_held(&mk_instance_mutex); if (!instance) { @@ -454,6 +630,14 @@ int mk_instance_destroy(struct mk_instance *instance) return -EBUSY; } + ret = mk_instance_release_resources(instance); + if (ret) { + pr_crit("Cannot remove instance '%s' while PCI cleanup is unsafe: %d\n", + instance->name, ret); + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; + } + list_del(&instance->list); idr_remove(&mk_instance_idr, instance->id); if (instance->kn) { diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index 89a8ee0d2c59b3..19cdc625e9e04d 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -23,12 +23,18 @@ /* Physical address of the manifest this kernel booted with, 0 if none */ static phys_addr_t mk_manifest_fdt_phys; +static bool mk_manifest_fdt_rejected; phys_addr_t mk_manifest_phys(void) { return mk_manifest_fdt_phys; } +bool mk_manifest_rejected(void) +{ + return READ_ONCE(mk_manifest_fdt_rejected); +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT @@ -50,6 +56,7 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) if (!fdt) { pr_warn("multikernel: failed to memremap manifest (0x%llx)\n", fdt_phys); + err = -ENOMEM; goto out; } @@ -68,14 +75,17 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) } mk_manifest_fdt_phys = fdt_phys; + mk_manifest_fdt_rejected = false; pr_info("multikernel: manifest accepted\n"); out: if (fdt) early_memunmap(fdt, fdt_len); - if (err) - pr_warn("multikernel: ignoring invalid manifest\n"); + if (err) { + mk_manifest_fdt_rejected = true; + pr_warn("multikernel: supplied manifest rejected: %d\n", err); + } } /** diff --git a/kernel/multikernel/mem.c b/kernel/multikernel/mem.c index 5c2da8dadb12ed..001d75bcc0368d 100644 --- a/kernel/multikernel/mem.c +++ b/kernel/multikernel/mem.c @@ -285,17 +285,30 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) phys_addr_t phys_addr; int ret; + if (!instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + if (mk_pci_iommu_lease_active_locked(instance)) { + pr_err("Cannot add memory to instance %d while an IOMMU lease is active\n", + instance->id); + ret = -EBUSY; + goto out_unlock; + } + phys_addr = multikernel_alloc(size); if (!phys_addr) { pr_err("Failed to allocate %zu bytes from multikernel pool for instance %d\n", size, instance->id); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region = kzalloc(sizeof(*region), GFP_KERNEL); if (!region) { multikernel_free(phys_addr, size); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region->res.name = kasprintf(GFP_KERNEL, "mk-instance-%d-%s-region-%d", @@ -303,7 +316,8 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) if (!region->res.name) { kfree(region); multikernel_free(phys_addr, size); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region->res.start = phys_addr; @@ -318,7 +332,7 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) kfree(region->res.name); kfree(region); multikernel_free(phys_addr, size); - return ret; + goto out_unlock; } INIT_LIST_HEAD(®ion->list); @@ -329,7 +343,10 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size) (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), size >> 20, instance->id, instance->name); - return 0; + ret = 0; +out_unlock: + mutex_unlock(&instance->resource_mutex); + return ret; } /** @@ -350,10 +367,19 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, { struct mk_memory_region *region, *tmp; bool found = false; + int ret; if (!instance) return -EINVAL; + mutex_lock(&instance->resource_mutex); + if (mk_pci_iommu_lease_active_locked(instance)) { + pr_err("Cannot remove memory from instance %d while an IOMMU lease is active\n", + instance->id); + ret = -EBUSY; + goto out_unlock; + } + list_for_each_entry_safe(region, tmp, &instance->memory_regions, list) { if (region->res.start == phys_addr && resource_size(®ion->res) == size) { @@ -381,10 +407,14 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), instance->id, instance->name); - return -ENOENT; + ret = -ENOENT; + } else { + ret = 0; } - return 0; +out_unlock: + mutex_unlock(&instance->resource_mutex); + return ret; } /** diff --git a/kernel/multikernel/messaging.c b/kernel/multikernel/messaging.c index 2cd2b7682f8d3e..1963613325ac0c 100644 --- a/kernel/multikernel/messaging.c +++ b/kernel/multikernel/messaging.c @@ -11,6 +11,7 @@ #include #include #include +#include "internal.h" /* Pending message tracking for request-response pattern */ struct mk_pending_msg { @@ -86,7 +87,8 @@ static void mk_message_type_ipi_callback(struct mk_ipi_data *data, void *ctx) msg_type, msg_subtype, payload_len, data->sender_cpu); /* Call the registered handler for this message type */ - type_handler->msg_handler(msg_type, msg_subtype, payload, payload_len, type_handler->context); + type_handler->msg_handler(msg_type, msg_subtype, payload, payload_len, + data->sender_cpu, type_handler->context); } /* @@ -190,8 +192,10 @@ int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms * * Returns 0 on success, negative error code on failure */ -int mk_send_message(int instance_id, u32 msg_type, u32 subtype, - void *payload, u32 payload_len) +static int __mk_send_message(struct mk_instance *instance, int instance_id, + mk_phys_cpu_t target, + u32 msg_type, u32 subtype, void *payload, + u32 payload_len) { struct mk_message *msg; size_t total_size; @@ -223,7 +227,14 @@ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, memcpy(msg->payload, payload, payload_len); /* Send via IPI using the message type as IPI type */ - ret = multikernel_send_ipi_data(instance_id, msg, total_size, msg_type); + if (instance && target != MK_PHYS_CPU_INVALID) + ret = mk_send_ipi_data_to_cpu(instance, target, msg, total_size, + msg_type); + else if (instance) + ret = mk_send_ipi_data(instance, msg, total_size, msg_type); + else + ret = multikernel_send_ipi_data(instance_id, msg, total_size, + msg_type); /* Clean up temporary buffer */ kfree(msg); @@ -238,6 +249,32 @@ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, return 0; } + +int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, + u32 subtype, void *payload, u32 payload_len) +{ + if (!instance) + return -EINVAL; + return __mk_send_message(instance, instance->id, MK_PHYS_CPU_INVALID, + msg_type, subtype, payload, payload_len); +} + +int mk_send_message_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, u32 msg_type, u32 subtype, + void *payload, u32 payload_len) +{ + if (!instance || target == MK_PHYS_CPU_INVALID) + return -EINVAL; + return __mk_send_message(instance, instance->id, target, msg_type, + subtype, payload, payload_len); +} + +int mk_send_message(int instance_id, u32 msg_type, u32 subtype, + void *payload, u32 payload_len) +{ + return __mk_send_message(NULL, instance_id, MK_PHYS_CPU_INVALID, + msg_type, subtype, payload, payload_len); +} EXPORT_SYMBOL(mk_send_message); /** diff --git a/kernel/multikernel/overlay.c b/kernel/multikernel/overlay.c index 5098918fee048a..1fafe915b0ecaf 100644 --- a/kernel/multikernel/overlay.c +++ b/kernel/multikernel/overlay.c @@ -581,7 +581,8 @@ static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, fdt_for_each_subnode(item_node, fdt, op_node) { const char *name = fdt_get_name(fdt, item_node, NULL); const char *pci_id_str; - unsigned int domain, bus, slot, func; + u16 domain; + u8 bus, slot, func; if (strncmp(name, "pci@", 4) != 0) continue; @@ -593,9 +594,12 @@ static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s'\n", pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s'\n", + len, pci_id_str); + return ret; } pr_info("Overlay tx%d: -device %04x:%02x:%02x.%x from %s\n", @@ -624,7 +628,8 @@ static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, const char *name = fdt_get_name(fdt, item_node, NULL); const char *pci_id_str; const char *driver_name = NULL; - unsigned int domain, bus, slot, func; + u16 domain; + u8 bus, slot, func; u32 flags = 0; if (strncmp(name, "pci@", 4) != 0) @@ -637,9 +642,12 @@ static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s'\n", pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s'\n", + len, pci_id_str); + return ret; } /* Get optional driver override */ @@ -799,7 +807,8 @@ static int mk_overlay_parse_and_rollback(struct mk_overlay_tx *tx, fdt_for_each_subnode(item_node, fdt, op_node) { const char *name = fdt_get_name(fdt, item_node, NULL); const char *pci_id_str; - unsigned int domain, bus, slot, func; + u16 domain; + u8 bus, slot, func; if (strncmp(name, "pci@", 4) != 0) continue; @@ -810,9 +819,12 @@ static int mk_overlay_parse_and_rollback(struct mk_overlay_tx *tx, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s'\n", pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s'\n", + len, pci_id_str); + return ret; } pr_info("Rollback tx%d: -device %04x:%02x:%02x.%x from %s\n", @@ -845,7 +857,8 @@ static int mk_overlay_parse_and_rollback(struct mk_overlay_tx *tx, const char *name = fdt_get_name(fdt, item_node, NULL); const char *pci_id_str; const char *driver_name = NULL; - unsigned int domain, bus, slot, func; + u16 domain; + u8 bus, slot, func; u32 flags = 0; if (strncmp(name, "pci@", 4) != 0) @@ -857,9 +870,12 @@ static int mk_overlay_parse_and_rollback(struct mk_overlay_tx *tx, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s'\n", pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s'\n", + len, pci_id_str); + return ret; } /* Get optional driver override */ diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c new file mode 100644 index 00000000000000..e31fd04b6b2b95 --- /dev/null +++ b/kernel/multikernel/pci.c @@ -0,0 +1,2578 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Multikernel PCI assignment policy + * + * Keeps assigned-device discovery, identity presentation, bridge traversal, + * and resource restoration independent from the manifest that populated + * the current instance. + * + * Configuration-space filtering constrains normal PCI access by a cooperative + * spawn kernel. It is not a security boundary: a privileged kernel can issue + * configuration cycles or map physical configuration windows directly. The + * host-owned IOMMU domain separately constrains DMA initiated by an assigned + * device. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +struct mk_pci_assignment; + +#define MK_PCI_FLR_SETTLE_MS 100 +#define MK_PCI_MAILBOX_QUIESCE_MS 1000 + +static unsigned long mk_pci_mailbox_deadline(void) +{ + return jiffies + msecs_to_jiffies(MK_PCI_MAILBOX_QUIESCE_MS); +} + +struct mk_pci_irq_vector { + struct mk_pci_assignment *assignment; + unsigned int host_irq; + u32 local_irq; + u32 mailbox_slot; + u32 mailbox_generation; + atomic64_t forwarded; + bool requested; + bool disabled; +}; + +struct mk_pci_assignment { + struct list_head instance_node; + struct list_head active_node; + struct list_head transaction_node; + struct mk_instance *instance; + struct mk_pci_device *inventory; + struct pci_dev *vf; + struct pci_dev *pf; + const struct device_driver *host_driver; + struct iommu_group *iommu_group; + struct iommu_domain *iommu_domain; + char *host_driver_override; + struct mutex iommu_mutex; /* Serializes IOMMU activation and teardown. */ + unsigned int iommu_mapped_regions; + bool iommu_dma_owner; + bool iommu_attached; + bool iommu_override_active; + struct mk_pci_irq_vector *irq_vectors; + unsigned long *irq_bound_map; + unsigned int nr_irq_vectors; + u64 irq_epoch; + u32 irq_generation; + u32 reset_generation; + u8 irq_state; + bool irq_msix; + bool irq_needs_reprogram; + bool irq_mailbox_failed; + unsigned long irq_flr_deadline; + bool device_enabled; + struct work_struct failure_work; + atomic_t failure_pending; + bool assigned; + bool inventory_moved; + bool expected_unbind; +}; + +static DEFINE_MUTEX(mk_pci_lease_mutex); +static DEFINE_SPINLOCK(mk_pci_active_lock); +static LIST_HEAD(mk_pci_active_assignments); +static bool mk_pci_notifier_registered; +static bool mk_pci_control_registered; +static mempool_t *mk_pci_control_pool; +static struct workqueue_struct *mk_pci_control_wq; +static DEFINE_SPINLOCK(mk_pci_control_lock); +static DECLARE_WAIT_QUEUE_HEAD(mk_pci_control_waitq); +static unsigned int mk_pci_control_active; +static bool mk_pci_control_shutdown = true; +static atomic64_t mk_pci_control_pool_exhausted = ATOMIC64_INIT(0); + +struct mk_pci_control_work { + struct work_struct work; + mk_phys_cpu_t sender_cpu; + union { + struct mk_pci_cfg_request cfg; + struct mk_pci_irq_request irq; + struct mk_pci_reset_request reset; + } request; +}; + +static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment); + +static bool mk_pci_control_handler_get(void) +{ + unsigned long flags; + bool acquired = false; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + if (!mk_pci_control_shutdown) { + mk_pci_control_active++; + acquired = true; + } + spin_unlock_irqrestore(&mk_pci_control_lock, flags); + return acquired; +} + +static void mk_pci_control_handler_put(void) +{ + unsigned long flags; + bool drained; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + WARN_ON_ONCE(!mk_pci_control_active); + if (mk_pci_control_active) + mk_pci_control_active--; + drained = !mk_pci_control_active; + spin_unlock_irqrestore(&mk_pci_control_lock, flags); + if (drained) + wake_up_all(&mk_pci_control_waitq); +} + +static void mk_pci_control_shutdown_begin(void) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + mk_pci_control_shutdown = true; + spin_unlock_irqrestore(&mk_pci_control_lock, flags); +} + +static void mk_pci_control_shutdown_end(void) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + mk_pci_control_active = 0; + mk_pci_control_shutdown = false; + spin_unlock_irqrestore(&mk_pci_control_lock, flags); +} + +static bool mk_pci_device_live(struct pci_dev *pdev) +{ + return device_is_registered(&pdev->dev) && + !pci_dev_is_disconnected(pdev) && + pci_device_is_present(pdev); +} + +static int mk_pci_forwarding_cpu(void) +{ + /* The host IPI manifest publishes logical CPU 0's physical ID. */ + if (!cpu_online(0)) + return -ENODEV; + + return 0; +} + +static bool mk_pci_device_matches_bdf(const struct mk_pci_device *device, + u16 domain, u8 bus, u8 devfn) +{ + return device->domain == domain && device->bus == bus && + device->slot == PCI_SLOT(devfn) && + device->func == PCI_FUNC(devfn); +} + +static struct mk_pci_device * +mk_pci_find_device_bdf(struct list_head *devices, u16 domain, u8 bus, u8 devfn) +{ + struct mk_pci_device *device; + + list_for_each_entry(device, devices, list) { + if (mk_pci_device_matches_bdf(device, domain, bus, devfn)) + return device; + } + + return NULL; +} + +static bool mk_pci_inventory_matches(const struct mk_pci_device *left, + const struct mk_pci_device *right) +{ + return left->vendor == right->vendor && + left->device == right->device && + mk_pci_device_matches_bdf(left, right->domain, right->bus, + PCI_DEVFN(right->slot, right->func)); +} + +static struct mk_pci_device * +mk_pci_find_root_inventory(const struct mk_pci_device *requested) +{ + struct mk_pci_device *device; + + list_for_each_entry(device, &root_instance->pci_devices, list) { + if (mk_pci_inventory_matches(device, requested)) + return device; + } + + return NULL; +} + +static struct mk_pci_device * +mk_pci_find_root_bdf(u16 domain, u8 bus, u8 devfn) +{ + return mk_pci_find_device_bdf(&root_instance->pci_devices, domain, bus, + devfn); +} + +static struct mk_pci_assignment * +mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_assignment *assignment; + + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + if (mk_pci_device_matches_bdf(assignment->inventory, domain, bus, + devfn)) + return assignment; + } + + return NULL; +} + +static void mk_pci_mailbox_clear_bit(struct mk_irq_mailbox *mailbox, + unsigned int slot) +{ + atomic64_andnot(BIT_ULL(slot & 63), + &mailbox->pending_bitmap[slot / 64]); +} + +static bool mk_pci_mailbox_quiesce(struct mk_pci_irq_vector *vector, + bool parked_force, + unsigned long deadline) +{ + struct mk_shared_data *shared = vector->assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u64 base, token; + + if (!shared || vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return true; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + base = mk_irq_mailbox_token(vector->mailbox_generation, 0) | + MK_IRQ_MAILBOX_MASKED; + for (;;) { + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return false; + } + if (mk_irq_mailbox_consuming(token)) { + if (parked_force) { + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, base) == token) + break; + continue; + } + if (!mk_irq_mailbox_masked(token)) + atomic64_cmpxchg(&entry->pending_generation, + token, + token | MK_IRQ_MAILBOX_MASKED); + if (time_after_eq(jiffies, deadline)) + return false; + usleep_range(50, 100); + continue; + } + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, base) == token) + break; + cpu_relax(); + } + mk_pci_mailbox_clear_bit(mailbox, vector->mailbox_slot); + WRITE_ONCE(entry->local_irq, 0); + return true; +} + +static void mk_pci_mailbox_release(struct mk_pci_irq_vector *vector) +{ + struct mk_shared_data *shared = vector->assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u64 token; + + if (!shared || vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + for (;;) { + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return; + } + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, 0) == token) + break; + cpu_relax(); + } + mk_pci_mailbox_clear_bit(mailbox, vector->mailbox_slot); + WRITE_ONCE(entry->lifecycle_epoch, 0); + WRITE_ONCE(entry->lifecycle_generation, 0); + WRITE_ONCE(entry->device_id, 0); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, 0); + vector->mailbox_slot = MK_IRQ_MAILBOX_SLOT_INVALID; + vector->mailbox_generation = 0; +} + +static int mk_pci_mailbox_reserve(struct mk_pci_irq_vector *vector, + unsigned int vector_index) +{ + struct mk_pci_assignment *assignment = vector->assignment; + struct mk_shared_data *shared = assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u32 generation; + u32 device_id; + unsigned int slot; + u64 base; + + if (!shared) + return -ENODEV; + mailbox = &shared->irq_mailbox; + generation = (u32)atomic_inc_return(&mailbox->next_generation); + if (!generation) + generation = (u32)atomic_inc_return(&mailbox->next_generation); + if (!generation) + return -EOVERFLOW; + device_id = MK_PCI_IRQ_ID(pci_domain_nr(assignment->vf->bus), + assignment->vf->bus->number, + assignment->vf->devfn); + base = mk_irq_mailbox_token(generation, 0) | MK_IRQ_MAILBOX_MASKED; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + entry = &mailbox->entries[slot]; + if (atomic64_read(&entry->pending_generation)) + continue; + if (atomic64_cmpxchg_release(&entry->pending_generation, 0, + base)) + continue; + WRITE_ONCE(entry->lifecycle_epoch, assignment->irq_epoch); + WRITE_ONCE(entry->lifecycle_generation, + assignment->irq_generation); + WRITE_ONCE(entry->device_id, device_id); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, vector_index); + vector->mailbox_slot = slot; + vector->mailbox_generation = generation; + return 0; + } + return -ENOSPC; +} + +static void mk_pci_irq_retry_workfn(struct work_struct *work) +{ + struct mk_instance *instance = container_of(to_delayed_work(work), + struct mk_instance, + irq_retry_work); + struct mk_shared_data *shared = READ_ONCE(instance->ipi_data); + bool any_pending = false; + bool unmasked_pending = false; + mk_phys_cpu_t target; + unsigned int slot; + + if (!shared) + return; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry; + u64 token; + + entry = &shared->irq_mailbox.entries[slot]; + token = atomic64_read_acquire(&entry->pending_generation); + + if (!mk_irq_mailbox_generation(token)) + continue; + if (mk_irq_mailbox_pending(token)) { + any_pending = true; + atomic64_or(BIT_ULL(slot & 63), + &shared->irq_mailbox.pending_bitmap[slot / 64]); + if (!mk_irq_mailbox_masked(token)) + unmasked_pending = true; + } else if (mk_irq_mailbox_consuming(token)) { + any_pending = true; + } + } + if (unmasked_pending) { + target = mk_instance_irq_route_load(instance); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } + if (any_pending) + mod_delayed_work(system_wq, &instance->irq_retry_work, + msecs_to_jiffies(unmasked_pending ? 10 : 100)); +} + +static irqreturn_t mk_pci_forward_irq(int irq, void *data) +{ + struct mk_pci_irq_vector *vector = data; + struct mk_pci_assignment *assignment = vector->assignment; + struct mk_shared_data *shared = assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + mk_phys_cpu_t target; + u64 old, new; + u32 pending; + u32 local_irq; + + /* Pair with BIND's publication of the shared mailbox identity. */ + local_irq = smp_load_acquire(&vector->local_irq); + + if (READ_ONCE(assignment->instance->state) != MK_STATE_ACTIVE || + READ_ONCE(assignment->irq_state) != MK_PCI_MSI_ACTIVE || + !local_irq || !shared || + vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return IRQ_HANDLED; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + for (;;) { + old = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(old) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return IRQ_HANDLED; + } + pending = mk_irq_mailbox_pending(old); + if (pending == MK_IRQ_MAILBOX_PENDING_MASK) { + atomic_inc(&mailbox->saturated); + break; + } + new = mk_irq_mailbox_token(vector->mailbox_generation, + pending + 1) | + (old & (MK_IRQ_MAILBOX_MASKED | + MK_IRQ_MAILBOX_CONSUMING)); + if (atomic64_cmpxchg(&entry->pending_generation, old, new) == old) + break; + cpu_relax(); + } + atomic64_or(BIT_ULL(vector->mailbox_slot & 63), + &mailbox->pending_bitmap[vector->mailbox_slot / 64]); + atomic_inc(&mailbox->recorded); + if (!pending) + mod_delayed_work(system_wq, &assignment->instance->irq_retry_work, + msecs_to_jiffies(10)); + if (atomic64_inc_return(&vector->forwarded) == 1) + pr_info("Forwarding host IRQ %u as instance IRQ %u for %s vector %u\n", + irq, local_irq, pci_name(assignment->vf), + (unsigned int)(vector - assignment->irq_vectors)); + target = mk_instance_irq_route_load(assignment->instance); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + return IRQ_HANDLED; +} + +static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment, + bool parked_force, + unsigned long deadline) +{ + unsigned int disabled = 0; + unsigned int i; + + assignment->irq_mailbox_failed = false; + for (i = 0; i < assignment->nr_irq_vectors; i++) + WRITE_ONCE(assignment->irq_vectors[i].local_irq, 0); + + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->requested && !vector->disabled) { + disable_irq(vector->host_irq); + vector->disabled = true; + disabled++; + } + } + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!mk_pci_mailbox_quiesce(vector, parked_force, deadline)) + assignment->irq_mailbox_failed = true; + } + if (assignment->irq_bound_map) + bitmap_zero(assignment->irq_bound_map, + assignment->nr_irq_vectors); + if (assignment->irq_state != MK_PCI_MSI_IDLE && + assignment->irq_state != MK_PCI_MSI_FAILED) + assignment->irq_state = MK_PCI_MSI_PREPARED; + + return disabled; +} + +static int mk_pci_release_irqs(struct mk_pci_assignment *assignment, + bool parked_force) +{ + unsigned long deadline = mk_pci_mailbox_deadline(); + unsigned int i; + + mk_pci_quiesce_irqs(assignment, parked_force, deadline); + if (assignment->irq_mailbox_failed) + return -ETIMEDOUT; + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->requested) + free_irq(vector->host_irq, vector); + mk_pci_mailbox_release(vector); + } + if (assignment->nr_irq_vectors) + pci_free_irq_vectors(assignment->vf); + kfree(assignment->irq_vectors); + bitmap_free(assignment->irq_bound_map); + assignment->irq_vectors = NULL; + assignment->irq_bound_map = NULL; + assignment->nr_irq_vectors = 0; + assignment->irq_msix = false; + assignment->irq_needs_reprogram = false; + assignment->irq_flr_deadline = 0; + assignment->irq_mailbox_failed = false; + return 0; +} + +int mk_pci_quiesce_instance_irqs(struct mk_instance *instance, + bool parked_force) +{ + struct mk_pci_assignment *assignment; + unsigned long deadline = mk_pci_mailbox_deadline(); + unsigned int disabled = 0; + int ret = 0; + + if (!instance || instance == root_instance) + return 0; + + lockdep_assert_held(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + disabled += mk_pci_quiesce_irqs(assignment, parked_force, + deadline); + if (assignment->irq_mailbox_failed) + ret = -ETIMEDOUT; + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + if (disabled) + pr_info("Quiesced %u host-owned PCI IRQ vectors for instance %d\n", + disabled, instance->id); + return ret; +} + +unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + unsigned int requested = 0; + unsigned int i; + + lockdep_assert_held(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = + &assignment->irq_vectors[i]; + + if (vector->requested) { + requested++; + synchronize_irq(vector->host_irq); + } + } + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + if (requested) { + mk_phys_cpu_t target = mk_instance_irq_route_load(instance); + + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } + return requested; +} + +static bool mk_pci_generation_after(u32 generation, u32 last) +{ + return !last || (s32)(generation - last) > 0; +} + +static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + struct mk_pci_irq_vector *vectors; + unsigned long *bound_map; + unsigned int flags; + int forwarding_cpu; + int i; + int nvec; + int ret; + + if (!request->nr_vectors || + request->nr_vectors > MK_IRQ_MAILBOX_SLOTS) + return -EINVAL; + if (!request->lifecycle_generation) + return -EINVAL; + if (assignment->irq_epoch && + assignment->irq_epoch != request->lifecycle_epoch) + return -ESTALE; + forwarding_cpu = mk_pci_forwarding_cpu(); + if (forwarding_cpu < 0) { + pr_err("Host control CPU is unavailable for %s MSI forwarding\n", + pci_name(assignment->vf)); + return forwarding_cpu; + } + if (assignment->irq_generation == request->lifecycle_generation) { + if ((assignment->irq_state == MK_PCI_MSI_PREPARED || + assignment->irq_state == MK_PCI_MSI_COMMITTED || + assignment->irq_state == MK_PCI_MSI_ACTIVE) && + assignment->nr_irq_vectors == request->nr_vectors && + assignment->irq_msix == request->msix) + return 0; + return -ESTALE; + } + if (!mk_pci_generation_after(request->lifecycle_generation, + assignment->irq_generation)) + return -ESTALE; + if (assignment->nr_irq_vectors) { + ret = mk_pci_release_irqs(assignment, false); + if (ret) + return ret; + } + assignment->irq_epoch = request->lifecycle_epoch; + assignment->irq_generation = request->lifecycle_generation; + assignment->irq_state = MK_PCI_MSI_FAILED; + + vectors = kcalloc(request->nr_vectors, sizeof(*vectors), GFP_KERNEL); + if (!vectors) + return -ENOMEM; + bound_map = bitmap_zalloc(request->nr_vectors, GFP_KERNEL); + if (!bound_map) { + kfree(vectors); + return -ENOMEM; + } + flags = request->msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI; + nvec = pci_alloc_irq_vectors(assignment->vf, request->nr_vectors, + request->nr_vectors, flags); + if (nvec < 0) { + bitmap_free(bound_map); + kfree(vectors); + return nvec; + } + + assignment->irq_vectors = vectors; + assignment->irq_bound_map = bound_map; + assignment->nr_irq_vectors = nvec; + assignment->irq_msix = request->msix; + for (i = 0; i < nvec; i++) { + vectors[i].assignment = assignment; + vectors[i].host_irq = pci_irq_vector(assignment->vf, i); + vectors[i].mailbox_slot = MK_IRQ_MAILBOX_SLOT_INVALID; + ret = mk_pci_mailbox_reserve(&vectors[i], i); + if (ret) + goto err_release; + } + for (i = 0; i < nvec; i++) { + ret = request_irq(vectors[i].host_irq, mk_pci_forward_irq, + IRQF_NO_AUTOEN | IRQF_NOBALANCING, + "multikernel-pci-forward", &vectors[i]); + if (ret) + goto err_release; + vectors[i].requested = true; + vectors[i].disabled = true; + ret = irq_set_affinity(vectors[i].host_irq, + cpumask_of(forwarding_cpu)); + if (ret) + goto err_release; + } + assignment->irq_state = MK_PCI_MSI_PREPARED; + pr_info("Allocated %d host-owned %s vectors for %s (instance %d, CPU %d)\n", + nvec, request->msix ? "MSI-X" : "MSI", + pci_name(assignment->vf), assignment->instance->id, + forwarding_cpu); + return 0; + +err_release: + pr_err("Failed to configure host-owned IRQ for %s vector %d: %d\n", + pci_name(assignment->vf), i, ret); + if (mk_pci_release_irqs(assignment, false)) + return -ETIMEDOUT; + return ret; +} + +static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + struct mk_shared_data *shared = assignment->instance->ipi_data; + struct mk_pci_irq_vector *vector; + unsigned long delay; + unsigned int count; + unsigned int i; + u32 last_irq; + u32 local_irq; + + if (mk_instance_irq_route_load(assignment->instance) == + MK_PHYS_CPU_INVALID) + return -ENODEV; + if (!request->lifecycle_generation || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_epoch != request->lifecycle_epoch || + (assignment->irq_state != MK_PCI_MSI_PREPARED && + assignment->irq_state != MK_PCI_MSI_ACTIVE)) + return -ESTALE; + + if (!assignment->irq_vectors || + assignment->irq_msix != request->msix) + return -EINVAL; + count = request->msix ? 1 : request->nr_vectors; + if (!count || request->vector >= assignment->nr_irq_vectors || + count > assignment->nr_irq_vectors - request->vector) + return -EINVAL; + last_irq = request->local_irq + count - 1; + if (!request->local_irq || last_irq < request->local_irq) + return -EINVAL; + + for (i = 0; i < count; i++) { + struct mk_irq_mailbox_entry *entry; + u64 token; + + vector = &assignment->irq_vectors[request->vector + i]; + local_irq = READ_ONCE(vector->local_irq); + if (!vector->requested || + vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return -EINVAL; + entry = &shared->irq_mailbox.entries[vector->mailbox_slot]; + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation || + READ_ONCE(entry->lifecycle_epoch) != assignment->irq_epoch || + READ_ONCE(entry->lifecycle_generation) != + assignment->irq_generation || + READ_ONCE(entry->vector) != request->vector + i) + return -ESTALE; + if (local_irq && local_irq != request->local_irq + i) + return -EBUSY; + } + + if (assignment->irq_needs_reprogram) { + if (time_before(jiffies, assignment->irq_flr_deadline)) { + delay = assignment->irq_flr_deadline - jiffies; + msleep(jiffies_to_msecs(delay) + 1); + } + pci_restore_msi_state(assignment->vf); + assignment->irq_needs_reprogram = false; + assignment->irq_flr_deadline = 0; + pr_info("Reprogrammed host-owned MSI state for %s during restore\n", + pci_name(assignment->vf)); + } + + for (i = 0; i < count; i++) { + struct mk_irq_mailbox_entry *entry; + + vector = &assignment->irq_vectors[request->vector + i]; + entry = &shared->irq_mailbox.entries[vector->mailbox_slot]; + WRITE_ONCE(entry->local_irq, request->local_irq + i); + /* Make the shared identity visible before routing this vector. */ + smp_store_release(&vector->local_irq, request->local_irq + i); + } + bitmap_set(assignment->irq_bound_map, request->vector, count); + return 0; +} + +static int +mk_pci_restore_irqs_begin(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_state == MK_PCI_MSI_PREPARED && + assignment->irq_needs_reprogram) + return 0; + if (assignment->irq_state != MK_PCI_MSI_ACTIVE) + return -ESTALE; + + mk_pci_quiesce_irqs(assignment, false, mk_pci_mailbox_deadline()); + if (assignment->irq_mailbox_failed) + return -ETIMEDOUT; + assignment->irq_needs_reprogram = true; + return 0; +} + +static int mk_pci_commit_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + unsigned int i; + + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation || + (assignment->irq_state != MK_PCI_MSI_PREPARED && + assignment->irq_state != MK_PCI_MSI_ACTIVE) || + !assignment->irq_bound_map || + !bitmap_full(assignment->irq_bound_map, + assignment->nr_irq_vectors)) + return -ESTALE; + + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!READ_ONCE(vector->local_irq)) + return -EINVAL; + } + if (assignment->irq_state == MK_PCI_MSI_ACTIVE) + return 0; + assignment->irq_state = MK_PCI_MSI_COMMITTED; + return 0; +} + +static int mk_pci_activate_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + unsigned int i; + + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_state == MK_PCI_MSI_ACTIVE) + return 0; + if (assignment->irq_state != MK_PCI_MSI_COMMITTED || + !assignment->nr_irq_vectors || !assignment->irq_bound_map || + !bitmap_full(assignment->irq_bound_map, + assignment->nr_irq_vectors)) + return -ESTALE; + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!vector->requested || !READ_ONCE(vector->local_irq)) + return -EINVAL; + } + assignment->irq_state = MK_PCI_MSI_ACTIVE; + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->disabled) { + enable_irq(vector->host_irq); + vector->disabled = false; + } + } + return 0; +} + +static int mk_pci_teardown_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + if (assignment->irq_epoch && + assignment->irq_epoch != request->lifecycle_epoch) + return -ESTALE; + if (assignment->irq_generation != request->lifecycle_generation && + !mk_pci_generation_after(request->lifecycle_generation, + assignment->irq_generation)) + return -ESTALE; + + if (assignment->nr_irq_vectors) { + int ret = mk_pci_release_irqs(assignment, false); + + if (ret) + return ret; + } + assignment->irq_epoch = request->lifecycle_epoch; + assignment->irq_generation = request->lifecycle_generation; + assignment->irq_state = MK_PCI_MSI_IDLE; + return 0; +} + +static bool mk_pci_is_flr_write(struct pci_dev *vf, + const struct mk_pci_cfg_request *request) +{ + u16 flr_byte; + unsigned int bit; + + if (!request->write || !pci_is_pcie(vf)) + return false; + flr_byte = pci_pcie_cap(vf) + PCI_EXP_DEVCTL + 1; + if (request->reg > flr_byte || + request->reg + request->len <= flr_byte) + return false; + bit = (flr_byte - request->reg) * 8 + 7; + return request->value & BIT(bit); +} + +static bool mk_pci_request_route_stale(struct mk_instance *instance, + mk_phys_cpu_t sender_cpu) +{ + if (!instance || instance == root_instance) + return true; + if (READ_ONCE(instance->state) != MK_STATE_ACTIVE) + return true; + + mk_cpu_ownership_assert_held(); + return !mk_cpu_set_contains(instance->cpus, sender_cpu); +} + +static int mk_pci_config_access(struct mk_instance *instance, + const struct mk_pci_cfg_request *request, + u32 *value, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + struct pci_dev *vf; + bool flr; + int ret; + + if (request->len != 1 && request->len != 2 && request->len != 4) + return PCIBIOS_BAD_REGISTER_NUMBER; + if (request->reg > PCI_CFG_SPACE_EXP_SIZE - request->len) + return PCIBIOS_BAD_REGISTER_NUMBER; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf)) { + ret = PCIBIOS_DEVICE_NOT_FOUND; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + /* Committed: complete and let publish reclaim an indeterminate timeout. */ + + vf = assignment->vf; + flr = mk_pci_is_flr_write(vf, request); + if (flr) { + ret = PCIBIOS_SET_FAILED; + goto out; + } + if (request->write) { + switch (request->len) { + case 1: + ret = pci_write_config_byte(vf, request->reg, + request->value); + break; + case 2: + ret = pci_write_config_word(vf, request->reg, + request->value); + break; + default: + ret = pci_write_config_dword(vf, request->reg, + request->value); + break; + } + } else { + switch (request->len) { + case 1: { + u8 data; + + ret = pci_read_config_byte(vf, request->reg, &data); + *value = data; + break; + } + case 2: { + u16 data; + + ret = pci_read_config_word(vf, request->reg, &data); + *value = data; + break; + } + default: + ret = pci_read_config_dword(vf, request->reg, value); + break; + } + } +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_cfg_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_pci_cfg_request *request = &control_work->request.cfg; + struct mk_reply_handle reply = { + .slot = request->reply_slot, + .kind = MK_REPLY_PCI_CFG, + .request_id = request->request_id, + .generation = request->reply_generation, + }; + struct mk_instance *instance; + u32 value = ~0U; + s32 status; + + instance = mk_instance_find(request->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + mk_cpu_ownership_lock(); + status = mk_pci_request_route_stale(instance, control_work->sender_cpu) ? + -ESTALE : 0; + mk_cpu_ownership_unlock(); + /* An untrusted payload ID must never select another instance's slot. */ + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_config_access(instance, request, &value, &reply); + if (mk_reply_publish(instance, &reply, status, value)) + pr_warn_ratelimited("Failed to return PCI config response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + mempool_free(control_work, mk_pci_control_pool); +} + +static int mk_pci_irq_access(struct mk_instance *instance, + const struct mk_pci_irq_request *request, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + int ret; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + if (request->operation != MK_PCI_IRQ_TEARDOWN && + READ_ONCE(instance->state) != MK_STATE_ACTIVE) { + ret = -ESHUTDOWN; + goto out; + } + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf)) { + ret = -ENODEV; + goto out; + } + if (!instance->ipi_data || !request->lifecycle_epoch || + request->lifecycle_epoch != + READ_ONCE(instance->ipi_data->spawn_epoch)) { + ret = -ESTALE; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + /* + * This CAS is the sole cancellation boundary. IRQ programming is + * non-cancellable once EXECUTING; a timed-out waiter moves the exact + * generation to COMMITTED for mk_reply_publish() to reclaim afterwards. + */ + + switch (request->operation) { + case MK_PCI_IRQ_SETUP: + ret = mk_pci_setup_irqs(assignment, request); + break; + case MK_PCI_IRQ_RESTORE_BEGIN: + ret = mk_pci_restore_irqs_begin(assignment, request); + break; + case MK_PCI_IRQ_BIND: + ret = mk_pci_bind_irq(assignment, request); + break; + case MK_PCI_IRQ_COMMIT: + ret = mk_pci_commit_irqs(assignment, request); + break; + case MK_PCI_IRQ_ACTIVATE: + ret = mk_pci_activate_irqs(assignment, request); + break; + case MK_PCI_IRQ_TEARDOWN: + ret = mk_pci_teardown_irqs(assignment, request); + break; + default: + ret = -EINVAL; + break; + } +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_irq_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_pci_irq_request *request = &control_work->request.irq; + struct mk_reply_handle reply = { + .slot = request->reply_slot, + .kind = MK_REPLY_PCI_IRQ, + .request_id = request->request_id, + .generation = request->reply_generation, + }; + struct mk_instance *instance; + s32 status; + + instance = mk_instance_find(request->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + mk_cpu_ownership_lock(); + status = mk_pci_request_route_stale(instance, control_work->sender_cpu) ? + -ESTALE : 0; + mk_cpu_ownership_unlock(); + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_irq_access(instance, request, &reply); + if (mk_reply_publish(instance, &reply, status, 0)) + pr_warn_ratelimited("Failed to return PCI IRQ response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + mempool_free(control_work, mk_pci_control_pool); +} + +static int mk_pci_reset_access(struct mk_instance *instance, + const struct mk_pci_reset_request *request, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + int ret; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + if (READ_ONCE(instance->state) != MK_STATE_ACTIVE) { + ret = -ESHUTDOWN; + goto out; + } + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf) || !assignment->vf->is_virtfn) { + ret = -ENODEV; + goto out; + } + if (!instance->ipi_data || !request->lifecycle_epoch || + request->lifecycle_epoch != + READ_ONCE(instance->ipi_data->spawn_epoch) || + !request->reset_generation || + !mk_pci_generation_after(request->reset_generation, + assignment->reset_generation)) { + ret = -ESTALE; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + + /* Tombstone this serial before side effects so delayed replays reject. */ + assignment->reset_generation = request->reset_generation; + mk_pci_quiesce_irqs(assignment, false, mk_pci_mailbox_deadline()); + if (assignment->irq_mailbox_failed) { + assignment->irq_state = MK_PCI_MSI_FAILED; + mk_pci_schedule_failure(assignment); + ret = -ETIMEDOUT; + goto out; + } + ret = pcie_reset_flr(assignment->vf, false); + if (ret) { + assignment->irq_state = MK_PCI_MSI_FAILED; + mk_pci_schedule_failure(assignment); + goto out; + } + if (assignment->nr_irq_vectors) { + assignment->irq_needs_reprogram = true; + assignment->irq_flr_deadline = + jiffies + msecs_to_jiffies(MK_PCI_FLR_SETTLE_MS); + } + pr_info("Invalidated host-owned MSI bindings for spawn-triggered FLR of %s\n", + pci_name(assignment->vf)); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_reset_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_pci_reset_request *request = &control_work->request.reset; + struct mk_reply_handle reply = { + .slot = request->reply_slot, + .kind = MK_REPLY_PCI_RESET, + .request_id = request->request_id, + .generation = request->reply_generation, + }; + struct mk_instance *instance; + s32 status; + + instance = mk_instance_find(request->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + mk_cpu_ownership_lock(); + status = mk_pci_request_route_stale(instance, control_work->sender_cpu) ? + -ESTALE : 0; + mk_cpu_ownership_unlock(); + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_reset_access(instance, request, &reply); + if (mk_reply_publish(instance, &reply, status, 0)) + pr_warn_ratelimited("Failed to return PCI reset response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + mempool_free(control_work, mk_pci_control_pool); +} + +static void mk_pci_control_msg_handler(u32 msg_type, u32 subtype, + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx) +{ + struct mk_pci_control_work *control_work; + size_t request_size; + work_func_t work_fn; + + if (msg_type != MK_MSG_PCI) + return; + if (!mk_pci_control_handler_get()) + return; + + switch (subtype) { + case MK_PCI_CFG_REQUEST: + request_size = sizeof(control_work->request.cfg); + work_fn = mk_pci_cfg_work_fn; + break; + case MK_PCI_IRQ_REQUEST: + request_size = sizeof(control_work->request.irq); + work_fn = mk_pci_irq_work_fn; + break; + case MK_PCI_RESET_REQUEST: + request_size = sizeof(control_work->request.reset); + work_fn = mk_pci_reset_work_fn; + break; + default: + goto out; + } + if (payload_len != request_size) + goto out; + /* + * Valid senders reserve one of MK_REPLY_SLOTS before publishing. Every + * active instance owns at least one disjoint possible CPU, so the pool + * covers the maximum number of valid requests across all instances. This + * keeps the hardirq receive path allocation-safe without changing reply + * or route validation. Duplicate traffic is outside the cooperative ABI. + */ + control_work = mempool_alloc(mk_pci_control_pool, GFP_ATOMIC); + if (!control_work) { + atomic64_inc(&mk_pci_control_pool_exhausted); + pr_warn_ratelimited("Multikernel PCI control work pool exhausted\n"); + goto out; + } + INIT_WORK(&control_work->work, work_fn); + memcpy(&control_work->request, payload, request_size); + control_work->sender_cpu = sender_cpu; + if (!queue_work(mk_pci_control_wq, &control_work->work)) + mempool_free(control_work, mk_pci_control_pool); +out: + mk_pci_control_handler_put(); +} + +static void mk_pci_assignment_failure_work(struct work_struct *work) +{ + struct mk_pci_assignment *assignment = + container_of(work, struct mk_pci_assignment, failure_work); + struct mk_instance *instance = assignment->instance; + int ret; + + pr_err("PCI assignment lease for %s was lost by instance %d (%s)\n", + pci_name(assignment->vf), instance->id, instance->name); + + if (READ_ONCE(instance->state) == MK_STATE_ACTIVE) { + ret = mk_instance_force_halt(instance); + if (ret) + pr_err("Failed to force halt instance %d after PCI lease loss: %d\n", + instance->id, ret); + } + + mk_instance_mark_failed(instance); +} + +static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment) +{ + if (atomic_cmpxchg(&assignment->failure_pending, 0, 1)) + return; + + if (!schedule_work(&assignment->failure_work)) + atomic_set(&assignment->failure_pending, 0); +} + +static int mk_pci_bus_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct pci_dev *pdev = to_pci_dev(data); + struct mk_pci_assignment *assignment; + unsigned long flags; + + if (action != BUS_NOTIFY_DEL_DEVICE && + action != BUS_NOTIFY_REMOVED_DEVICE && + action != BUS_NOTIFY_UNBOUND_DRIVER) + return NOTIFY_DONE; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + list_for_each_entry(assignment, &mk_pci_active_assignments, + active_node) { + if (pdev != assignment->vf && pdev != assignment->pf) + continue; + if (pdev == assignment->vf && + action == BUS_NOTIFY_UNBOUND_DRIVER && + assignment->expected_unbind) + continue; + mk_pci_schedule_failure(assignment); + } + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + return NOTIFY_OK; +} + +static struct notifier_block mk_pci_bus_notifier = { + .notifier_call = mk_pci_bus_notify, +}; + +static void +mk_pci_release_bound_driver(struct mk_pci_assignment *assignment) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = true; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + device_release_driver(&assignment->vf->dev); + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); +} + +#if IS_ENABLED(CONFIG_IOMMU_API) +#define MK_PCI_ASSIGNMENT_DRIVER_NAME "multikernel-pci-assignment" + +static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, + const struct pci_device_id *id); +static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev); + +static struct pci_driver mk_pci_assignment_driver = { + .name = MK_PCI_ASSIGNMENT_DRIVER_NAME, + .probe = mk_pci_iommu_assignment_probe, + .remove = mk_pci_iommu_assignment_remove, + .driver_managed_dma = true, +}; + +struct mk_pci_iommu_group_check { + struct device *vf; + unsigned int count; +}; + +static int mk_pci_iommu_check_group_device(struct device *dev, void *data) +{ + struct mk_pci_iommu_group_check *check = data; + + check->count++; + return dev == check->vf ? 0 : -EXDEV; +} + +static void mk_pci_iommu_free_resv_regions(struct list_head *regions) +{ + struct iommu_resv_region *region, *tmp; + + list_for_each_entry_safe(region, tmp, regions, list) { + list_del(®ion->list); + kfree(region); + } +} + +static int mk_pci_iommu_validate_group(struct mk_pci_assignment *assignment) +{ + struct mk_pci_iommu_group_check check = { + .vf = &assignment->vf->dev, + }; + int ret; + + ret = iommu_group_for_each_dev(assignment->iommu_group, &check, + mk_pci_iommu_check_group_device); + if (ret || check.count != 1) { + pr_err("IOMMU group %d for %s is not an isolated singleton group\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf)); + return ret ?: -EXDEV; + } + + if (!iommu_group_has_isolated_msi(assignment->iommu_group)) { + pr_err("IOMMU group %d for %s lacks isolated MSI delivery\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf)); + return -EPERM; + } + + return 0; +} + +static int +mk_pci_iommu_validate_resv_regions(struct mk_pci_assignment *assignment) +{ + struct iommu_resv_region *region; + struct mk_memory_region *memory; + LIST_HEAD(resv_regions); + u64 memory_end, resv_end; + int ret; + + ret = iommu_get_group_resv_regions(assignment->iommu_group, + &resv_regions); + if (ret) + goto out; + + list_for_each_entry(region, &resv_regions, list) { + if (region->type == IOMMU_RESV_DIRECT_RELAXABLE) + continue; + if (!region->length || + check_add_overflow((u64)region->start, + (u64)region->length - 1, &resv_end)) { + pr_err("IOMMU group %d for %s has invalid reserved region at %#llx\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf), + (unsigned long long)region->start); + ret = -EOVERFLOW; + goto out; + } + + list_for_each_entry(memory, + &assignment->instance->memory_regions, list) { + resource_size_t size = resource_size(&memory->res); + + if (!size || + check_add_overflow((u64)memory->res.start, + (u64)size - 1, &memory_end)) { + ret = -EOVERFLOW; + goto out; + } + if ((u64)memory->res.start > resv_end || + memory_end < (u64)region->start) + continue; + + pr_err("Instance %d IOVA %#llx-%#llx overlaps IOMMU reserved region %#llx-%#llx type %u for %s\n", + assignment->instance->id, + (unsigned long long)memory->res.start, + (unsigned long long)memory_end, + (unsigned long long)region->start, + (unsigned long long)resv_end, region->type, + pci_name(assignment->vf)); + ret = -EPERM; + goto out; + } + } + + ret = 0; +out: + mk_pci_iommu_free_resv_regions(&resv_regions); + return ret; +} + +static int +mk_pci_iommu_validate_region(struct mk_pci_assignment *assignment, + const struct mk_memory_region *region) +{ + struct iommu_domain *domain = assignment->iommu_domain; + resource_size_t start = region->res.start; + resource_size_t size = resource_size(®ion->res); + u64 dma_mask = dma_get_mask(&assignment->vf->dev); + u64 end; + unsigned long min_page_size; + + if (!size || check_add_overflow((u64)start, (u64)size - 1, &end)) + return -EOVERFLOW; + if (!domain->pgsize_bitmap) + return -EOPNOTSUPP; + + min_page_size = 1UL << __ffs(domain->pgsize_bitmap); + if (!IS_ALIGNED(start, min_page_size) || + !IS_ALIGNED(size, min_page_size)) { + pr_err("Instance %d memory %#llx-%#llx is not aligned to IOMMU page size %#lx\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, min_page_size); + return -EINVAL; + } + if (start > ULONG_MAX || end > ULONG_MAX || end > dma_mask) { + pr_err("Instance %d memory %#llx-%#llx exceeds DMA addressability of %s\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, pci_name(assignment->vf)); + return -ERANGE; + } + if (domain->geometry.force_aperture && + (start < domain->geometry.aperture_start || + end > domain->geometry.aperture_end)) { + pr_err("Instance %d memory %#llx-%#llx is outside the IOMMU aperture for %s\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, pci_name(assignment->vf)); + return -ERANGE; + } + + return 0; +} + +static void mk_pci_iommu_unmap_regions(struct mk_pci_assignment *assignment) +{ + struct mk_memory_region *region; + unsigned int remaining = assignment->iommu_mapped_regions; + + list_for_each_entry(region, &assignment->instance->memory_regions, list) { + resource_size_t size; + size_t unmapped; + + if (!remaining) + break; + size = resource_size(®ion->res); + unmapped = iommu_unmap(assignment->iommu_domain, + region->res.start, size); + if (unmapped != size) + pr_err("IOMMU unmapped only %#zx of %#llx bytes for instance %d at %#llx\n", + unmapped, (unsigned long long)size, + assignment->instance->id, + (unsigned long long)region->res.start); + remaining--; + } + if (remaining) + pr_err("IOMMU lease for %s lost %u mapped instance regions\n", + pci_name(assignment->vf), remaining); + assignment->iommu_mapped_regions = 0; +} + +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + bool transactions_drained; + int ret; + + ret = mk_pci_release_irqs(assignment, false); + if (ret) + return ret; + if (!mk_pci_device_live(vf)) + return 0; + + /* + * Releasing DMA ownership restores the group's default domain. Stop new + * DMA first, drain requests already issued, and reset the VF while the + * assignment domain still contains any stragglers. + */ + pci_clear_master(vf); + transactions_drained = pci_wait_for_pending_transaction(vf); + ret = pcie_reset_flr(vf, false); + if (!ret) + return 0; + if (ret == -ENOTTY && transactions_drained) + return 0; + + if (!transactions_drained) + pr_err("Timed out draining DMA from assigned VF %s\n", + pci_name(vf)); + if (ret != -ENOTTY) + pr_err("Failed to reset assigned VF %s: %d\n", + pci_name(vf), ret); + + /* + * Keep the assignment domain attached when the device cannot be made + * safe. The lease owner can retry teardown after the instance halts. + */ + return ret == -ENOTTY ? -ETIMEDOUT : ret; +} + +static int +mk_pci_reset_assignment_for_start(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + int ret; + + ret = mk_pci_release_irqs(assignment, true); + if (ret) + return ret; + assignment->irq_epoch = 0; + assignment->irq_generation = 0; + assignment->reset_generation = 0; + assignment->irq_state = MK_PCI_MSI_IDLE; + if (!assignment->assigned || !assignment->iommu_attached) + return -EINVAL; + if (!mk_pci_device_live(vf)) + return -ENODEV; + + /* + * A stopped instance may have left DMA active. Keep its restrictive + * domain attached while stopping new requests, draining old ones, and + * resetting device state before the instance image is reused. + */ + pci_clear_master(vf); + if (!pci_wait_for_pending_transaction(vf)) { + pr_err("Timed out draining assigned VF %s before instance restart\n", + pci_name(vf)); + return -ETIMEDOUT; + } + + ret = pcie_reset_flr(vf, false); + if (ret) { + pr_err("Failed to reset assigned VF %s before instance restart: %d\n", + pci_name(vf), ret); + return ret == -ENOTTY ? -EOPNOTSUPP : ret; + } + return 0; +} + +static void +__mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ + if (assignment->iommu_attached) { + iommu_detach_group(assignment->iommu_domain, + assignment->iommu_group); + assignment->iommu_attached = false; + } + if (assignment->iommu_dma_owner) { + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + } +} + +static void +mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ + mutex_lock(&assignment->iommu_mutex); + __mk_pci_iommu_deactivate_assignment(assignment); + mutex_unlock(&assignment->iommu_mutex); +} + +static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment) +{ + mutex_lock(&assignment->iommu_mutex); + __mk_pci_iommu_deactivate_assignment(assignment); + + if (assignment->iommu_domain) { + mk_pci_iommu_unmap_regions(assignment); + iommu_domain_free(assignment->iommu_domain); + assignment->iommu_domain = NULL; + } + if (assignment->iommu_group) { + iommu_group_put(assignment->iommu_group); + assignment->iommu_group = NULL; + } + mutex_unlock(&assignment->iommu_mutex); +} + +static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) +{ + struct mk_memory_region *region; + int ret; + + if (!device_iommu_mapped(&assignment->vf->dev)) { + pr_err("Cannot assign %s without an active hardware IOMMU\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; + } + if (!device_iommu_capable(&assignment->vf->dev, + IOMMU_CAP_CACHE_COHERENCY)) { + pr_err("Cannot assign %s without coherent IOMMU mappings\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; + } + if (!assignment->instance->region_count || + list_empty(&assignment->instance->memory_regions)) + return -EINVAL; + + assignment->iommu_group = iommu_group_get(&assignment->vf->dev); + if (!assignment->iommu_group) + return -ENODEV; + + ret = mk_pci_iommu_validate_group(assignment); + if (ret) + goto err_release; + ret = mk_pci_iommu_validate_resv_regions(assignment); + if (ret) + goto err_release; + + assignment->iommu_domain = + iommu_paging_domain_alloc(&assignment->vf->dev); + if (IS_ERR(assignment->iommu_domain)) { + ret = PTR_ERR(assignment->iommu_domain); + assignment->iommu_domain = NULL; + goto err_release; + } + + list_for_each_entry(region, &assignment->instance->memory_regions, list) { + resource_size_t size = resource_size(®ion->res); + + ret = mk_pci_iommu_validate_region(assignment, region); + if (ret) + goto err_release; + ret = iommu_map(assignment->iommu_domain, region->res.start, + region->res.start, size, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE, GFP_KERNEL); + if (ret) + goto err_release; + assignment->iommu_mapped_regions++; + } + + /* + * The domain blocks DMA outside these mappings, but translation-fault + * notification is not portable. In particular, Intel VT-d reports primary + * faults through dmar_fault() without invoking a legacy domain handler. + * Do not claim automatic instance failure on an IOMMU fault here. + */ + pr_info("Prepared host IOMMU domain for %s with %u instance regions\n", + pci_name(assignment->vf), assignment->iommu_mapped_regions); + return 0; + +err_release: + mk_pci_iommu_release_assignment(assignment); + return ret; +} + +static int mk_pci_iommu_commit_assignment(struct mk_pci_assignment *assignment) +{ + int ret; + + if (!assignment->iommu_domain) + return 0; + + if (assignment->vf->driver_override) { + assignment->host_driver_override = + kstrdup(assignment->vf->driver_override, GFP_KERNEL); + if (!assignment->host_driver_override) + return -ENOMEM; + } + + ret = driver_set_override(&assignment->vf->dev, + &assignment->vf->driver_override, + MK_PCI_ASSIGNMENT_DRIVER_NAME, + strlen(MK_PCI_ASSIGNMENT_DRIVER_NAME)); + if (ret) + return ret; + assignment->iommu_override_active = true; + pci_set_drvdata(assignment->vf, assignment); + ret = device_driver_attach(&mk_pci_assignment_driver.driver, + &assignment->vf->dev); + if (ret) + return ret; + if (assignment->vf->dev.driver != &mk_pci_assignment_driver.driver) + return -ENODEV; + return 0; +} + +static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + int ret; + + if (!assignment || assignment->vf != pdev || !assignment->iommu_domain) + return -ENODEV; + ret = iommu_device_claim_dma_owner(&assignment->vf->dev, assignment); + if (ret) + return ret; + assignment->iommu_dma_owner = true; + + ret = iommu_attach_group(assignment->iommu_domain, + assignment->iommu_group); + if (ret) { + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + return ret; + } + assignment->iommu_attached = true; + ret = pci_enable_device(pdev); + if (ret) { + iommu_detach_group(assignment->iommu_domain, + assignment->iommu_group); + assignment->iommu_attached = false; + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + return ret; + } + assignment->device_enabled = true; + pr_info("Attached %s to host-owned IOMMU domain for instance %d\n", + pci_name(assignment->vf), assignment->instance->id); + return 0; +} + +static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) +{ + struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + int ret; + + if (!assignment || assignment->vf != pdev) + return; + + if (READ_ONCE(assignment->expected_unbind)) { + if (assignment->device_enabled) { + pci_disable_device(pdev); + assignment->device_enabled = false; + } + pci_set_drvdata(pdev, NULL); + return; + } + + ret = mk_pci_quiesce_assignment(assignment); + if (ret) { + pr_crit("Keeping IOMMU containment for %s after unsafe driver removal: %d\n", + pci_name(pdev), ret); + mk_pci_schedule_failure(assignment); + } else { + mk_pci_iommu_deactivate_assignment(assignment); + } + if (assignment->device_enabled) { + pci_disable_device(pdev); + assignment->device_enabled = false; + } + pci_set_drvdata(pdev, NULL); +} + +static int mk_pci_restore_host_binding(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + const char *override = assignment->host_driver_override ?: ""; + int ret = 0; + + if (vf->dev.driver == &mk_pci_assignment_driver.driver) { + mk_pci_release_bound_driver(assignment); + } else if (vf->dev.driver) { + pr_err("Cannot release assignment driver from %s: device is bound to %s\n", + pci_name(vf), vf->dev.driver->name); + return -EBUSY; + } + + pci_set_drvdata(vf, NULL); + if (assignment->iommu_override_active) { + ret = driver_set_override(&vf->dev, &vf->driver_override, + override, strlen(override)); + if (ret) + return ret; + assignment->iommu_override_active = false; + } + + mk_pci_iommu_deactivate_assignment(assignment); + if (assignment->host_driver && mk_pci_device_live(vf)) { + if (!vf->dev.driver) { + ret = device_driver_attach(assignment->host_driver, &vf->dev); + if (ret) { + pr_err("Failed to restore driver %s to %s: %d\n", + assignment->host_driver->name, + pci_name(vf), ret); + return ret; + } + } else if (vf->dev.driver != assignment->host_driver) { + pr_err("Cannot restore driver %s to %s: device is bound to %s\n", + assignment->host_driver->name, pci_name(vf), + vf->dev.driver->name); + return -EBUSY; + } + } + return 0; +} + +static int mk_pci_iommu_system_init(void) +{ + return pci_register_driver(&mk_pci_assignment_driver); +} + +static void mk_pci_iommu_system_cleanup(void) +{ + pci_unregister_driver(&mk_pci_assignment_driver); +} +#else +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static void +mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ +} + +static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) +{ + pr_err("Cannot assign %s without CONFIG_IOMMU_API\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; +} + +static int mk_pci_iommu_commit_assignment(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment) +{ +} + +static int mk_pci_restore_host_binding(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static int mk_pci_iommu_system_init(void) +{ + return 0; +} + +static void mk_pci_iommu_system_cleanup(void) +{ +} +#endif + +static int +mk_pci_prepare_assignment(struct mk_instance *instance, + const struct mk_pci_device *requested, + struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + struct mk_pci_device *inventory; + struct pci_dev *vf; + struct pci_dev *pf; + struct pci_dev *physfn; + int ret; + + inventory = mk_pci_find_root_inventory(requested); + if (!inventory) { + pr_err("PCI device %04x:%04x@%04x:%02x:%02x.%x is not available in the root pool\n", + requested->vendor, requested->device, requested->domain, + requested->bus, requested->slot, requested->func); + return -ENOENT; + } + + vf = pci_get_domain_bus_and_slot(inventory->domain, + inventory->bus, + PCI_DEVFN(inventory->slot, + inventory->func)); + if (!vf) + return -ENODEV; + + if (vf->vendor != inventory->vendor || + vf->device != inventory->device) { + pr_err("PCI identity changed for %s: expected %04x:%04x, found %04x:%04x\n", + pci_name(vf), inventory->vendor, inventory->device, + vf->vendor, vf->device); + pci_dev_put(vf); + return -ENODEV; + } + + physfn = pci_physfn(vf); + if (!vf->is_virtfn || physfn == vf) { + pr_err("PCI assignment only supports SR-IOV VFs, rejecting %s\n", + pci_name(vf)); + pci_dev_put(vf); + return -EOPNOTSUPP; + } + + if (!mk_pci_device_live(vf) || !mk_pci_device_live(physfn)) { + pci_dev_put(vf); + return -ENODEV; + } + ret = pcie_reset_flr(vf, true); + if (ret) { + pr_err("PCI assignment requires FLR for safe instance restart, rejecting %s\n", + pci_name(vf)); + pci_dev_put(vf); + return -EOPNOTSUPP; + } + + if (pci_is_dev_assigned(vf) || + mk_pci_find_assignment(instance, inventory->domain, + inventory->bus, + PCI_DEVFN(inventory->slot, + inventory->func))) { + pci_dev_put(vf); + return -EBUSY; + } + + pf = pci_dev_get(physfn); + assignment = kzalloc_obj(*assignment, GFP_KERNEL); + if (!assignment) { + pci_dev_put(pf); + pci_dev_put(vf); + return -ENOMEM; + } + + assignment->instance = instance; + assignment->inventory = inventory; + assignment->vf = vf; + assignment->pf = pf; + assignment->host_driver = vf->dev.driver; + if (assignment->host_driver && assignment->host_driver->owner && + !try_module_get(assignment->host_driver->owner)) { + kfree(assignment); + pci_dev_put(pf); + pci_dev_put(vf); + return -ENODEV; + } + + INIT_LIST_HEAD(&assignment->instance_node); + INIT_LIST_HEAD(&assignment->active_node); + INIT_LIST_HEAD(&assignment->transaction_node); + mutex_init(&assignment->iommu_mutex); + INIT_WORK(&assignment->failure_work, mk_pci_assignment_failure_work); + atomic_set(&assignment->failure_pending, 0); + + ret = mk_pci_iommu_prepare_assignment(assignment); + if (ret) + goto err_module; + + list_add_tail(&assignment->instance_node, &instance->pci_assignments); + list_add_tail(&assignment->transaction_node, transaction); + + return 0; + +err_module: + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + kfree(assignment); + pci_dev_put(pf); + pci_dev_put(vf); + return ret; +} + +static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + unsigned long flags; + int ret; + int i; + + if (!mk_pci_device_live(vf) || !mk_pci_device_live(assignment->pf)) + return -ENODEV; + + if (vf->dev.driver != assignment->host_driver || + pci_is_dev_assigned(vf)) + return -EBUSY; + + pci_set_dev_assigned(vf); + assignment->assigned = true; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + list_add_tail(&assignment->active_node, &mk_pci_active_assignments); + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + if (assignment->host_driver) + mk_pci_release_bound_driver(assignment); + + if (vf->dev.driver) + return -EBUSY; + + ret = mk_pci_iommu_commit_assignment(assignment); + if (ret) + return ret; + + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + assignment->inventory->resources[i].start = + vf->resource[i].start; + assignment->inventory->resources[i].end = + vf->resource[i].end; + assignment->inventory->resources[i].flags = + vf->resource[i].flags; + } + assignment->inventory->resources_valid = true; + + list_move_tail(&assignment->inventory->list, + &assignment->instance->pci_devices); + root_instance->pci_device_count--; + assignment->instance->pci_device_count++; + assignment->instance->pci_devices_valid = true; + assignment->inventory_moved = true; + + pr_info("Leased SR-IOV VF %s to instance %d (%s)\n", + pci_name(vf), assignment->instance->id, + assignment->instance->name); + return 0; +} + +static int mk_pci_release_assignment(struct mk_pci_assignment *assignment, + struct list_head *released) +{ + struct mk_instance *instance = assignment->instance; + struct pci_dev *vf = assignment->vf; + unsigned long flags; + int ret; + + if (!assignment->assigned) + goto release_resources; + + ret = mk_pci_quiesce_assignment(assignment); + if (ret) + return ret; + + ret = mk_pci_restore_host_binding(assignment); + if (ret) + return ret; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + if (!list_empty(&assignment->active_node)) + list_del_init(&assignment->active_node); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + if (assignment->assigned) { + pci_clear_dev_assigned(vf); + assignment->assigned = false; + } + +release_resources: + mk_pci_iommu_release_assignment(assignment); + + if (assignment->inventory_moved && root_instance) { + assignment->inventory->resources_valid = false; + list_move_tail(&assignment->inventory->list, + &root_instance->pci_devices); + instance->pci_device_count--; + root_instance->pci_device_count++; + root_instance->pci_devices_valid = true; + assignment->inventory_moved = false; + } + + if (!list_empty(&assignment->transaction_node)) + list_del_init(&assignment->transaction_node); + list_del_init(&assignment->instance_node); + list_add_tail(&assignment->transaction_node, released); + + return 0; +} + +static void mk_pci_finalize_releases(struct list_head *released) +{ + struct mk_pci_assignment *assignment, *tmp; + + list_for_each_entry_safe(assignment, tmp, released, transaction_node) { + list_del_init(&assignment->transaction_node); + cancel_work_sync(&assignment->failure_work); + kfree(assignment->host_driver_override); + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + pci_dev_put(assignment->pf); + pci_dev_put(assignment->vf); + kfree(assignment); + } +} + +static int mk_pci_rollback_transaction(struct list_head *transaction, + struct list_head *released) +{ + struct mk_pci_assignment *assignment, *tmp; + int rollback_ret = 0; + int ret; + + list_for_each_entry_safe_reverse(assignment, tmp, transaction, + transaction_node) { + ret = mk_pci_release_assignment(assignment, released); + if (!ret) + continue; + pr_crit("Failed to roll back PCI assignment for %s: %d\n", + pci_name(assignment->vf), ret); + list_del_init(&assignment->transaction_node); + if (!rollback_ret) + rollback_ret = ret; + } + + return rollback_ret; +} + +static int mk_pci_commit_transaction(struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + int ret; + + list_for_each_entry(assignment, transaction, transaction_node) { + ret = mk_pci_commit_assignment(assignment); + if (ret) + return ret; + } + + while (!list_empty(transaction)) { + assignment = list_first_entry(transaction, + struct mk_pci_assignment, + transaction_node); + list_del_init(&assignment->transaction_node); + } + + return 0; +} + +void mk_pci_lease_instance_init(struct mk_instance *instance) +{ + mutex_init(&instance->resource_mutex); + INIT_LIST_HEAD(&instance->pci_assignments); + INIT_DELAYED_WORK(&instance->irq_retry_work, mk_pci_irq_retry_workfn); +} + +bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance) +{ + if (!instance) + return false; + + lockdep_assert_held(&instance->resource_mutex); + return !list_empty(&instance->pci_assignments); +} + +int mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count) +{ + struct mk_pci_device *requested; + LIST_HEAD(released); + LIST_HEAD(transaction); + int prepared = 0; + int ret = 0; + int rollback_ret; + + if (!instance || instance == root_instance || !requested_devices || + requested_count < 0) + return -EINVAL; + if (!root_instance || !root_instance->pci_devices_valid) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + list_for_each_entry(requested, requested_devices, list) { + ret = mk_pci_prepare_assignment(instance, requested, + &transaction); + if (ret) + goto rollback; + prepared++; + } + + if (prepared != requested_count) { + ret = -EINVAL; + goto rollback; + } + + ret = mk_pci_commit_transaction(&transaction); + if (ret) + goto rollback; + goto out; + +rollback: + rollback_ret = mk_pci_rollback_transaction(&transaction, &released); + if (rollback_ret) + ret = rollback_ret; +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); + return ret; +} + +int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_device *inventory; + LIST_HEAD(released); + LIST_HEAD(transaction); + int ret; + int rollback_ret; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + if (instance->state != MK_STATE_READY) { + ret = -EBUSY; + goto out; + } + if (!root_instance || !root_instance->pci_devices_valid) { + ret = -EINVAL; + goto out; + } + + inventory = mk_pci_find_root_bdf(domain, bus, devfn); + if (!inventory) { + ret = -ENOENT; + goto out; + } + + ret = mk_pci_prepare_assignment(instance, inventory, &transaction); + if (ret) + goto rollback; + ret = mk_pci_commit_transaction(&transaction); + if (ret) + goto rollback; + goto out; + +rollback: + rollback_ret = mk_pci_rollback_transaction(&transaction, &released); + if (rollback_ret) + ret = rollback_ret; +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); + return ret; +} + +int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_assignment *assignment; + LIST_HEAD(released); + int ret; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + if (instance->state != MK_STATE_READY) { + ret = -EBUSY; + goto out; + } + + assignment = mk_pci_find_assignment(instance, domain, bus, devfn); + if (!assignment) { + ret = -ENOENT; + goto out; + } + + ret = mk_pci_release_assignment(assignment, &released); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); + return ret; +} + +int mk_pci_release_assignments(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + LIST_HEAD(released); + int ret = 0; + + if (!instance || instance == root_instance) + return 0; + + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + while (!list_empty(&instance->pci_assignments)) { + assignment = list_last_entry(&instance->pci_assignments, + struct mk_pci_assignment, + instance_node); + ret = mk_pci_release_assignment(assignment, &released); + if (ret) { + pr_crit("Instance %d retains unsafe PCI lease for %s: %d\n", + instance->id, pci_name(assignment->vf), ret); + break; + } + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + /* + * The assignment is no longer reachable by routed requests. Cancel its + * failure work after dropping the route and transaction locks because the + * worker may itself force-halt the instance and take both locks. + */ + mk_pci_finalize_releases(&released); + if (!ret) + cancel_delayed_work_sync(&instance->irq_retry_work); + return ret; +} + +int mk_pci_prepare_instance_start(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + int ret = 0; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + ret = mk_pci_reset_assignment_for_start(assignment); + if (ret) + break; + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +int mk_pci_lease_system_init(void) +{ + unsigned int pool_size; + size_t work_size = sizeof(struct mk_pci_control_work); + int ret; + + ret = mk_pci_iommu_system_init(); + if (ret) + return ret; + ret = bus_register_notifier(&pci_bus_type, &mk_pci_bus_notifier); + if (ret) { + mk_pci_iommu_system_cleanup(); + return ret; + } + mk_pci_notifier_registered = true; + if (root_instance && root_instance->id == 0) { + pool_size = max_t(unsigned int, num_possible_cpus(), 1) * + MK_REPLY_SLOTS; + mk_pci_control_pool = mempool_create_kmalloc_pool(pool_size, work_size); + if (!mk_pci_control_pool) { + ret = -ENOMEM; + goto unregister_notifier; + } + mk_pci_control_wq = + alloc_workqueue("mk-pci-control", + WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!mk_pci_control_wq) { + ret = -ENOMEM; + goto destroy_pool; + } + mk_pci_control_shutdown_end(); + ret = mk_register_msg_handler(MK_MSG_PCI, + mk_pci_control_msg_handler, NULL); + if (ret) + goto destroy_workqueue; + mk_pci_control_registered = true; + } + return 0; + +destroy_workqueue: + mk_pci_control_shutdown_begin(); + destroy_workqueue(mk_pci_control_wq); + mk_pci_control_wq = NULL; +destroy_pool: + mempool_destroy(mk_pci_control_pool); + mk_pci_control_pool = NULL; +unregister_notifier: + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; + mk_pci_iommu_system_cleanup(); + return ret; +} + +void mk_pci_lease_system_cleanup(void) +{ + if (mk_pci_control_registered) { + mk_pci_control_shutdown_begin(); + mk_unregister_msg_handler(MK_MSG_PCI, + mk_pci_control_msg_handler); + mk_pci_control_registered = false; + wait_event(mk_pci_control_waitq, !READ_ONCE(mk_pci_control_active)); + } + if (mk_pci_control_wq) { + destroy_workqueue(mk_pci_control_wq); + mk_pci_control_wq = NULL; + } + mempool_destroy(mk_pci_control_pool); + mk_pci_control_pool = NULL; + if (mk_pci_notifier_registered) { + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; + } + mk_pci_iommu_system_cleanup(); +} + +static struct mk_pci_device * +mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) +{ + if (!root_instance || root_instance->id == 0 || + !root_instance->dtb_data || + !root_instance->pci_devices_valid) + return NULL; + + return mk_pci_find_device_bdf(&root_instance->pci_devices, + domain, bus, devfn); +} + +static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) +{ + return mk_pci_find_assigned_bdf(pci_domain_nr(bus), bus->number, devfn); +} + +/** + * mk_pci_get_assigned_identity_bdf - Get an assigned function's identity + * @domain: PCI domain number + * @bus: PCI bus number + * @devfn: device/function number + * @vendor: assigned Vendor ID + * @device_id: assigned Device ID + * + * Returns: true when assignment metadata contains an exact location match. + */ +bool mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, + u16 *device_id) +{ + struct mk_pci_device *device; + + if (domain != (u16)domain || bus != (u8)bus || devfn != (u8)devfn) + return false; + device = mk_pci_find_assigned_bdf(domain, bus, devfn); + + if (!device) + return false; + + if (vendor) + *vendor = device->vendor; + if (device_id) + *device_id = device->device; + return true; +} + +static void mk_pci_restore_resources(struct pci_dev *dev) +{ + struct mk_pci_device *device; + int i; + + device = mk_pci_find_assigned(dev->bus, dev->devfn); + if (!device || !device->resources_valid) + return; + + dev->non_compliant_bars = true; + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + dev->resource[i].start = device->resources[i].start; + dev->resource[i].end = device->resources[i].end; + dev->resource[i].flags = device->resources[i].flags; + } + pr_info("Restored PCI BAR resources for %s\n", pci_name(dev)); +} + +DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, mk_pci_restore_resources); diff --git a/net/vmw_vsock/mk_transport.c b/net/vmw_vsock/mk_transport.c index f75dbf3f2c8209..e7594eaee1393f 100644 --- a/net/vmw_vsock/mk_transport.c +++ b/net/vmw_vsock/mk_transport.c @@ -279,7 +279,8 @@ static void mk_vsock_rx_work(struct work_struct *work) } static void mk_vsock_ipi_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + mk_phys_cpu_t sender_cpu, void *ctx) { struct sk_buff *skb; struct virtio_vsock_hdr *hdr;