fast-get: enable sharing between userspace threads

If a module fast-gets memory for a specific its instance, running in
userspace and then another usuerspace instance wants to share it,
access has to be granted to that object by that thread too. This
patch does that while also restricting object sharing to "large"
objects (currently larger than half a page), and smaller objects are
just copied.

Signed-off-by: Guennadi Liakhovetski <guennadi.liakhovetski@linux.intel.com>

Author: lyakh

src/audio/module_adapter/module_adapter.c

@@ -53,7 +53,7 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv,
 #if CONFIG_MM_DRV
 #define PAGE_SZ CONFIG_MM_DRV_PAGE_SIZE
 #else
-#include <platform/platform.h>
+#include <sof/platform.h>
 #define PAGE_SZ HOST_PAGE_SIZE
 #endif
 

zephyr/lib/fast-get.c

@@ -28,6 +28,9 @@
 struct sof_fast_get_entry {
 	const void *dram_ptr;
 	void *sram_ptr;
+#if CONFIG_USERSPACE
+	struct k_thread *thread;
+#endif
 	size_t size;
 	unsigned int refcount;
 };
@@ -90,16 +93,71 @@ static struct sof_fast_get_entry *fast_get_find_entry(struct sof_fast_get_data *
 	return NULL;
 }
 
+#if CONFIG_MM_DRV
+#define PAGE_SZ CONFIG_MM_DRV_PAGE_SIZE
+#define FAST_GET_MAX_COPY_SIZE (PAGE_SZ / 2)
+#else
+#include <sof/platform.h>
+#define PAGE_SZ HOST_PAGE_SIZE
+#define FAST_GET_MAX_COPY_SIZE 0
+#endif
+
+#if CONFIG_USERSPACE
+static bool fast_get_domain_exists(struct k_thread *thread, void *start, size_t size)
+{
+	struct k_mem_domain *domain = thread->mem_domain_info.mem_domain;
+
+	for (unsigned int i = 0; i < domain->num_partitions; i++) {
+		struct k_mem_partition *dpart = &domain->partitions[i];
+
+		if (dpart->start == (uintptr_t)start && dpart->size == size)
+			return true;
+	}
+
+	return false;
+}
+
+static int fast_get_access_grant(k_tid_t thread, void *addr, size_t size)
+{
+	struct k_mem_partition part = {
+		.start = (uintptr_t)addr,
+		.size = ALIGN_UP(size, CONFIG_MM_DRV_PAGE_SIZE),
+		.attr = K_MEM_PARTITION_P_RO_U_RO | XTENSA_MMU_CACHED_WB,
+	};
+
+	LOG_DBG("add %#zx @ %p", part.size, addr);
+	return k_mem_domain_add_partition(thread->mem_domain_info.mem_domain, &part);
+}
+#endif /* CONFIG_USERSPACE */
+
 const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size)
 {
 	struct sof_fast_get_data *data = &fast_get_data;
+	uint32_t alloc_flags = SOF_MEM_FLAG_USER;
 	struct sof_fast_get_entry *entry;
+	size_t alloc_size, alloc_align;
+	const void *alloc_ptr;
 	k_spinlock_key_t key;
 	void *ret;
 
 	key = k_spin_lock(&data->lock);
+	if (IS_ENABLED(CONFIG_USERSPACE) && size > FAST_GET_MAX_COPY_SIZE) {
+		alloc_size = ALIGN_UP(size, PAGE_SZ);
+		alloc_align = PAGE_SZ;
+		alloc_flags |= SOF_MEM_FLAG_LARGE_BUFFER;
+	} else {
+		alloc_size = size;
+		alloc_align = PLATFORM_DCACHE_ALIGN;
+	}
+
+	if (size > FAST_GET_MAX_COPY_SIZE || !IS_ENABLED(CONFIG_USERSPACE))
+		alloc_ptr = dram_ptr;
+	else
+		/* When userspace is enabled only share large buffers */
+		alloc_ptr = NULL;
+
 	do {
-		entry = fast_get_find_entry(data, dram_ptr);
+		entry = fast_get_find_entry(data, alloc_ptr);
 		if (!entry) {
 			if (fast_get_realloc(data)) {
 				ret = NULL;
@@ -108,6 +166,12 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size)
 		}
 	} while (!entry);
 
+#if CONFIG_USERSPACE
+	LOG_DBG("userspace %u part %#zx bytes alloc %p entry %p DRAM %p",
+		k_current_get()->mem_domain_info.mem_domain->num_partitions, size,
+		alloc_ptr, entry->sram_ptr, dram_ptr);
+#endif
+
 	if (entry->sram_ptr) {
 		if (entry->size != size || entry->dram_ptr != dram_ptr) {
 			LOG_ERR("size %u != %u or ptr %p != %p mismatch",
@@ -117,22 +181,65 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size)
 		}
 
 		ret = entry->sram_ptr;
+
+#if CONFIG_USERSPACE
+		/* We only get there for large buffers */
+		if (k_current_get()->mem_domain_info.mem_domain->num_partitions > 1) {
+			/* A userspace thread makes the request */
+			if (k_current_get() != entry->thread &&
+			    !fast_get_domain_exists(k_current_get(), ret,
+						    ALIGN_UP(size, CONFIG_MM_DRV_PAGE_SIZE))) {
+				LOG_DBG("grant access to thread %p first was %p", k_current_get(),
+					entry->thread);
+				int err = fast_get_access_grant(k_current_get(), ret, size);
+
+				if (err < 0) {
+					ret = NULL;
+					goto out;
+				}
+				/*
+				 * The data is constant, so it's safe to use cached access to
+				 * it, but initially we have to invalidate caches
+				 */
+				dcache_invalidate_region((__sparse_force void __sparse_cache *)ret,
+							 size);
+			} else {
+				LOG_WRN("Repeated access request by thread");
+			}
+		}
+#endif
+
 		entry->refcount++;
-		/*
-		 * The data is constant, so it's safe to use cached access to
-		 * it, but initially we have to invalidate cached
-		 */
-		dcache_invalidate_region((__sparse_force void __sparse_cache *)ret, size);
 		goto out;
 	}
 
-	ret = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, size, PLATFORM_DCACHE_ALIGN);
+	/*
+	 * If a userspace threads is the first user to fast-get the buffer, an
+	 * SRAM copy will be allocated on its own heap, so it will have access
+	 * to it
+	 */
+	ret = sof_heap_alloc(heap, alloc_flags, alloc_size, alloc_align);
 	if (!ret)
 		goto out;
 	entry->size = size;
 	entry->sram_ptr = ret;
 	memcpy_s(entry->sram_ptr, entry->size, dram_ptr, size);
 	dcache_writeback_region((__sparse_force void __sparse_cache *)entry->sram_ptr, size);
+
+#if CONFIG_USERSPACE
+	entry->thread = k_current_get();
+	if (size > FAST_GET_MAX_COPY_SIZE) {
+		/* Otherwise we've allocated on thread's heap, so it already has access */
+		int err = fast_get_access_grant(entry->thread, ret, size);
+
+		if (err < 0) {
+			sof_heap_free(NULL, ret);
+			ret = NULL;
+			goto out;
+		}
+	}
+#endif /* CONFIG_USERSPACE */
+
 	entry->dram_ptr = dram_ptr;
 	entry->refcount = 1;
 out:
@@ -169,6 +276,20 @@ void fast_put(struct k_heap *heap, const void *sram_ptr)
 		goto out;
 	}
 	entry->refcount--;
+
+#if CONFIG_USERSPACE
+	if (entry->size > FAST_GET_MAX_COPY_SIZE && entry->thread) {
+		struct k_mem_partition part = {
+			.start = (uintptr_t)entry->sram_ptr,
+			.size = ALIGN_UP(entry->size, CONFIG_MM_DRV_PAGE_SIZE),
+			.attr = K_MEM_PARTITION_P_RO_U_RO | XTENSA_MMU_CACHED_WB,
+		};
+
+		LOG_DBG("remove %#zx @ %p", part.size, entry->sram_ptr);
+		k_mem_domain_remove_partition(entry->thread->mem_domain_info.mem_domain, &part);
+	}
+#endif
+
 	if (!entry->refcount) {
 		sof_heap_free(heap, entry->sram_ptr);
 		memset(entry, 0, sizeof(*entry));