zephyr: ll_user: carve out separate heap for LL data that kernel needs to access

src/include/sof/schedule/ll_schedule_domain.h

@@ -227,6 +227,41 @@ bool zephyr_ll_user_heap_verify(struct k_heap *heap);
  */
 void zephyr_ll_user_resources_init(void);
 
+/** @brief Kernel-managed areas of the LL user memory region */
+enum zephyr_ll_karea {
+	ZEPHYR_LL_KAREA_RW,	/**< read-write for LL user threads */
+	ZEPHYR_LL_KAREA_RO,	/**< read-only for LL user threads */
+	ZEPHYR_LL_KAREA_COUNT,
+};
+
+/**
+ * @brief Allocate from a kernel-managed area of the LL user memory region.
+ *
+ * For objects that kernel code creates and LL user threads access. The
+ * allocator state is kernel-only and memory is never freed, so use this
+ * for objects that live until the firmware is reset. Objects in
+ * ZEPHYR_LL_KAREA_RO can only be written by kernel threads, not by kernel
+ * code running in the LL memory domain, such as system calls from LL
+ * threads. Kernel context only.
+ *
+ * @param[in] area The area to allocate from.
+ * @param[in] flags SOF_MEM_FLAG_COHERENT for an uncached pointer.
+ * @param[in] bytes Size in bytes.
+ * @param[in] alignment Required alignment in bytes, 0 for the cache line size.
+ * @return Pointer to the memory, or NULL if the area is full.
+ */
+void *zephyr_ll_karea_alloc(enum zephyr_ll_karea area, uint32_t flags, size_t bytes,
+			    size_t alignment);
+
+/**
+ * @brief Check that memory lies in a kernel-managed area of the LL region.
+ * @param[in] area The area to check.
+ * @param[in] ptr Start of the memory, cached or uncached alias.
+ * @param[in] bytes Size in bytes.
+ * @return true if [ptr, ptr + bytes) lies in the area.
+ */
+bool zephyr_ll_karea_contains(enum zephyr_ll_karea area, const void *ptr, size_t bytes);
+
 /**
  * @brief Grant thread access to per-core Low-Latency mutexes.
  * @param[in,out] thread Target thread.

src/include/sof/userspace/README.md

@@ -155,6 +155,28 @@ The `zephyr/syscall/*.c` files hold the validators in one place per subsystem; t
 
 Most syscall declarations are gated by `#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION)`; outside that guard (non-Zephyr host-test and ztest builds) the same name is a plain macro or inline calling `z_impl_*` directly. Non-userspace Zephyr builds (no `CONFIG_USERSPACE`) still use the `__syscall` declaration path, which Zephyr compiles to a direct `z_impl_*` call without the privilege-elevation trap.
 
+## LL memory region
+
+With `CONFIG_SOF_USERSPACE_LL`, the memory of the LL and IPC user threads is one region of
+`CONFIG_SOF_ZEPHYR_SYS_USER_HEAP_SIZE` bytes, allocated from the kernel heap at boot
+(`src/kernel/schedule/zephyr_ll_user.c`) and split into page-aligned parts:
+
+| Part | Mapping in the LL memory domain | Allocator |
+|------|---------------------------------|-----------|
+| Kernel RO area (`CONFIG_SOF_ZEPHYR_LL_KAREA_RO_SIZE`) | read-only | `zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RO, ...)` |
+| Kernel RW area (`CONFIG_SOF_ZEPHYR_LL_KAREA_RW_SIZE`) | read-write | `zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, ...)` |
+| LL user heap (the rest) | read-write | `sof_heap_alloc(sof_sys_user_heap_get(), ...)` |
+
+The kernel areas are for objects that kernel code creates and LL user threads access. Their
+allocator is a bump allocator with kernel-only state and no free, so use it for objects that
+live until the firmware is reset. The Xtensa MMU has one write permission for both rings, so
+the RO area is also read-only to kernel code running in the LL memory domain, such as system
+calls from LL threads: only kernel threads can write objects there.
+
+The LL user heap keeps its `sys_heap` metadata in user-writable memory. Kernel code is being
+moved off it; `CONFIG_SOF_ZEPHYR_LL_HEAP_KERNEL_TRACE` logs the remaining uses by kernel
+threads.
+
 ## Memory-partition markers
 
 User threads can only touch memory that is mapped into their memory domain. SOF exposes shared

src/kernel/schedule/zephyr_ll_user.c

@@ -6,12 +6,44 @@
 #include <zephyr/kernel.h>
 #include <zephyr/logging/log.h>
 
+#include <rtos/alloc.h>
+#include <rtos/cache.h>
 #include <rtos/userspace_helper.h>
 
+#include <sof/common.h>
 #include <sof/schedule/ll_schedule_domain.h>
 
 LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL);
 
+/*
+ * Layout of the LL user memory region, each part page aligned:
+ *
+ *   | kernel RO area | kernel RW area | LL user heap |
+ *
+ * The heap and the kernel RW area are mapped read-write into the LL memory
+ * domain, the kernel RO area read-only. Kernel code allocates objects that
+ * LL user threads must access from the kernel areas, see
+ * zephyr_ll_karea_alloc(). The allocator state of those areas is kept in
+ * ll_mem_resources, which is kernel-only.
+ */
+#define LL_REGION_SIZE		CONFIG_SOF_ZEPHYR_SYS_USER_HEAP_SIZE
+#define LL_KAREA_RO_SIZE	CONFIG_SOF_ZEPHYR_LL_KAREA_RO_SIZE
+#define LL_KAREA_RW_SIZE	CONFIG_SOF_ZEPHYR_LL_KAREA_RW_SIZE
+#define LL_HEAP_SIZE		(LL_REGION_SIZE - LL_KAREA_RO_SIZE - LL_KAREA_RW_SIZE)
+
+BUILD_ASSERT(LL_REGION_SIZE % CONFIG_MM_DRV_PAGE_SIZE == 0);
+BUILD_ASSERT(LL_KAREA_RO_SIZE % CONFIG_MM_DRV_PAGE_SIZE == 0);
+BUILD_ASSERT(LL_KAREA_RW_SIZE % CONFIG_MM_DRV_PAGE_SIZE == 0);
+BUILD_ASSERT(LL_KAREA_RO_SIZE + LL_KAREA_RW_SIZE < LL_REGION_SIZE,
+	     "kernel areas do not leave room for the LL user heap");
+
+/* A kernel-managed area, addresses are the uncached alias */
+struct zephyr_ll_karea_state {
+	uint8_t *base;
+	size_t size;
+	size_t used;
+};
+
 /**
  * Memory resources for userspace LL scheduler
  *
@@ -22,64 +54,140 @@ LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL);
 struct zephyr_ll_mem_resources {
 	struct k_mem_domain mem_domain; /**< Memory domain for LL thread isolation */
 	struct k_heap heap; /**< Heap allocator for LL scheduler memory */
+	struct zephyr_ll_karea_state karea[ZEPHYR_LL_KAREA_COUNT]; /**< Kernel-managed areas */
 };
 
 static struct zephyr_ll_mem_resources ll_mem_resources;
 
-static struct k_heap *ll_heap_alloc(void)
+static void ll_region_map(uint8_t *start, size_t size, k_mem_partition_attr_t attr)
 {
-	const size_t alloc_size = CONFIG_SOF_ZEPHYR_SYS_USER_HEAP_SIZE;
+	struct k_mem_partition part;
+	int ret;
 
-	BUILD_ASSERT(CONFIG_SOF_ZEPHYR_SYS_USER_HEAP_SIZE % CONFIG_MM_DRV_PAGE_SIZE == 0);
+	part.start = (uintptr_t)sys_cache_cached_ptr_get(start);
+	part.size = size;
+	part.attr = attr | XTENSA_MMU_CACHED_WB;
 
-	void *mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, alloc_size,
-				  CONFIG_MM_DRV_PAGE_SIZE);
-	if (!mem)
-		return NULL;
+	ret = k_mem_domain_add_partition(&ll_mem_resources.mem_domain, &part);
+	tr_dbg(&ll_tr, "map %p size %#zx (cached), ret %d", (void *)part.start, size, ret);
+	if (ret)
+		k_panic();
 
-	k_heap_init(&ll_mem_resources.heap, mem, alloc_size);
+#ifdef CONFIG_CACHE_HAS_MIRRORED_MEMORY_REGIONS
+	part.start = (uintptr_t)sys_cache_uncached_ptr_get(start);
+	part.attr = attr;
+
+	ret = k_mem_domain_add_partition(&ll_mem_resources.mem_domain, &part);
+	tr_dbg(&ll_tr, "map %p size %#zx (uncached), ret %d", (void *)part.start, size, ret);
+	if (ret)
+		k_panic();
+#endif
+}
+
+static void ll_region_init(void)
+{
+	struct zephyr_ll_karea_state *ro = &ll_mem_resources.karea[ZEPHYR_LL_KAREA_RO];
+	struct zephyr_ll_karea_state *rw = &ll_mem_resources.karea[ZEPHYR_LL_KAREA_RW];
+	uint8_t *mem;
+
+	mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, LL_REGION_SIZE,
+			    CONFIG_MM_DRV_PAGE_SIZE);
+	if (!mem) {
+		tr_err(&ll_tr, "region alloc fail");
+		k_panic();
+	}
+
+	ro->base = mem;
+	ro->size = LL_KAREA_RO_SIZE;
+	rw->base = mem + LL_KAREA_RO_SIZE;
+	rw->size = LL_KAREA_RW_SIZE;
+
+	mem += LL_KAREA_RO_SIZE + LL_KAREA_RW_SIZE;
+	k_heap_init(&ll_mem_resources.heap, mem, LL_HEAP_SIZE);
 
 	/*
 	 * k_heap_init() does not set these, so set the values
 	 * manually here
 	 */
 	ll_mem_resources.heap.heap.init_mem = mem;
-	ll_mem_resources.heap.heap.init_bytes = alloc_size;
+	ll_mem_resources.heap.heap.init_bytes = LL_HEAP_SIZE;
 
-	return &ll_mem_resources.heap;
+	if (ro->size)
+		ll_region_map(ro->base, ro->size, K_MEM_PARTITION_P_RO_U_RO);
+
+	/* the kernel RW area and the heap are adjacent, map them together */
+	ll_region_map(rw->base, rw->size + LL_HEAP_SIZE, K_MEM_PARTITION_P_RW_U_RW);
+
+	tr_info(&ll_tr, "LL region: RO area %p %#zx, RW area %p %#zx, heap %p %#zx",
+		(void *)ro->base, ro->size, (void *)rw->base, rw->size, (void *)mem,
+		(size_t)LL_HEAP_SIZE);
 }
 
-static void ll_heap_init(void)
+/* serialises allocations from the kernel-managed areas */
+static struct k_spinlock ll_karea_lock;
+
+void *zephyr_ll_karea_alloc(enum zephyr_ll_karea area, uint32_t flags, size_t bytes,
+			    size_t alignment)
 {
-	struct k_heap *heap = ll_heap_alloc();
-	struct k_mem_partition mem_partition;
-	int ret;
+	struct zephyr_ll_karea_state *ka;
+	k_spinlock_key_t key;
+	uintptr_t start;
+	void *ptr = NULL;
 
-	if (!heap) {
-		tr_err(&ll_tr, "heap alloc fail");
-		k_panic();
+	__ASSERT(!k_is_user_context(), "kernel context only");
+
+	if (area >= ZEPHYR_LL_KAREA_COUNT || !bytes)
+		return NULL;
+
+	ka = &ll_mem_resources.karea[area];
+
+	/* never share a cache line between objects with different aliases */
+	alignment = MAX(alignment, PLATFORM_DCACHE_ALIGN);
+	/* area bases are page aligned, so offsets keep alignments up to a page */
+	if (!is_power_of_2(alignment) || alignment > CONFIG_MM_DRV_PAGE_SIZE ||
+	    bytes > ka->size)
+		return NULL;
+	bytes = ROUND_UP(bytes, PLATFORM_DCACHE_ALIGN);
+
+	key = k_spin_lock(&ll_karea_lock);
+
+	/* offsets from the area base, all at most ka->size, cannot overflow */
+	start = ROUND_UP(ka->used, alignment);
+	if (start <= ka->size && bytes <= ka->size - start) {
+		ka->used = start + bytes;
+		ptr = ka->base + start;
 	}
 
-	/* Create memory partition for sch_data array */
-	mem_partition.start = (uintptr_t)sys_cache_cached_ptr_get(heap->heap.init_mem);
-	mem_partition.size = heap->heap.init_bytes;
-	mem_partition.attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB;
+	k_spin_unlock(&ll_karea_lock, key);
 
-	ret = k_mem_domain_add_partition(&ll_mem_resources.mem_domain, &mem_partition);
-	tr_dbg(&ll_tr, "init ll heap %p, size %u (cached), ret %d",
-	       (void *)mem_partition.start, heap->heap.init_bytes, ret);
-	if (ret)
-		k_panic();
+	if (!ptr) {
+		tr_err(&ll_tr, "kernel area %d full: %zu bytes requested, %zu of %zu used",
+		       area, bytes, ka->used, ka->size);
+		return NULL;
+	}
 
-#ifdef CONFIG_CACHE_HAS_MIRRORED_MEMORY_REGIONS
-	mem_partition.start = (uintptr_t)sys_cache_uncached_ptr_get(heap->heap.init_mem);
-	mem_partition.attr = K_MEM_PARTITION_P_RW_U_RW;
-	ret = k_mem_domain_add_partition(&ll_mem_resources.mem_domain, &mem_partition);
-	tr_dbg(&ll_tr, "init ll heap %p, size %u (uncached), ret %d",
-	       (void *)mem_partition.start, heap->heap.init_bytes, ret);
-	if (ret)
-		k_panic();
-#endif
+	tr_dbg(&ll_tr, "kernel area %d: %zu bytes at %p, %zu of %zu used", area, bytes, ptr,
+	       ka->used, ka->size);
+
+	if (!(flags & SOF_MEM_FLAG_COHERENT))
+		ptr = (__sparse_force void *)sys_cache_cached_ptr_get(ptr);
+
+	return ptr;
+}
+
+bool zephyr_ll_karea_contains(enum zephyr_ll_karea area, const void *ptr, size_t bytes)
+{
+	const struct zephyr_ll_karea_state *ka;
+	uintptr_t start;
+
+	if (area >= ZEPHYR_LL_KAREA_COUNT)
+		return false;
+
+	ka = &ll_mem_resources.karea[area];
+	start = (uintptr_t)sys_cache_uncached_ptr_get((__sparse_force void __sparse_cache *)ptr);
+
+	return start >= (uintptr_t)ka->base && bytes <= ka->size &&
+	       start - (uintptr_t)ka->base <= ka->size - bytes;
 }
 
 void zephyr_ll_user_resources_init(void)
@@ -88,7 +196,7 @@ void zephyr_ll_user_resources_init(void)
 
 	k_mem_domain_init(&ll_mem_resources.mem_domain, 0, NULL);
 
-	ll_heap_init();
+	ll_region_init();
 
 	/* attach common partition to LL domain */
 	user_memory_attach_common_partition(zephyr_ll_mem_domain());

zephyr/Kconfig

@@ -166,16 +166,53 @@ config SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE
 	 shared between instances of the same module.
 
 config SOF_ZEPHYR_SYS_USER_HEAP_SIZE
-	hex "Size of the shared LL user-space heap"
+	hex "Size of the LL user-space memory region"
 	default 0x20000
 	depends on SOF_USERSPACE_LL
 	help
-	 The size of the shared heap used by the low-latency (LL) scheduler
-	 user-space thread. This heap is shared across all pipelines running
+	 The size of the memory region of the low-latency (LL) scheduler
+	 user-space threads: the shared LL user heap and the kernel-managed
+	 areas SOF_ZEPHYR_LL_KAREA_RW_SIZE and SOF_ZEPHYR_LL_KAREA_RO_SIZE.
+	 The heap is shared across all pipelines running
 	 on the LL thread and must be large enough to hold all host DMA
 	 buffers, chain DMA data, SG elements, and module adapter buffers
 	 for all simultaneously active pipelines.
 
+config SOF_ZEPHYR_LL_KAREA_RW_SIZE
+	hex "Size of the kernel-managed read-write area for LL user threads"
+	default 0x4000
+	depends on SOF_USERSPACE_LL
+	help
+	 Size of the area in the LL user memory region from which kernel
+	 code allocates objects that the LL and IPC user threads read and
+	 write. The kernel keeps the allocator state for this area in its
+	 own memory, so user threads cannot affect kernel allocations.
+	 The area is taken from SOF_ZEPHYR_SYS_USER_HEAP_SIZE. Must be a
+	 multiple of the MMU page size.
+
+config SOF_ZEPHYR_LL_KAREA_RO_SIZE
+	hex "Size of the kernel-managed read-only area for LL user threads"
+	default 0x1000
+	depends on SOF_USERSPACE_LL
+	help
+	 Size of the area in the LL user memory region from which kernel
+	 code allocates objects that the LL and IPC user threads may only
+	 read. The area is mapped read-only into the LL memory domain, so
+	 kernel code running in that domain, for example in a system call,
+	 cannot write it either. The area is taken from
+	 SOF_ZEPHYR_SYS_USER_HEAP_SIZE. Must be a multiple of the MMU page
+	 size.
+
+config SOF_ZEPHYR_LL_HEAP_KERNEL_TRACE
+	bool "Log LL user heap allocations made by kernel threads"
+	depends on SOF_USERSPACE_LL
+	help
+	 Log every allocation from and free to the LL user heap that a
+	 kernel thread makes, with its size and caller address. Kernel
+	 code is being moved off this heap, whose metadata user threads
+	 can write. Use this to find the remaining callers. Calls made by
+	 user threads, including through system calls, are not logged.
+
 config SOF_USERSPACE_PROXY
 	bool "Use userspace proxy to support userspace modules"
 	select SOF_USERSPACE_USE_DRIVER_HEAP