alloc.c

// SPDX-License-Identifier: BSD-3-Clause
/*
 * Copyright(c) 2022 Intel Corporation. All rights reserved.
 *
 */

#include <sof/init.h>
#include <rtos/alloc.h>
#include <rtos/idc.h>
#include <rtos/interrupt.h>
#include <sof/drivers/interrupt-map.h>
#include <sof/schedule/schedule.h>
#include <sof/lib/notifier.h>
#include <sof/lib/pm_runtime.h>
#include <sof/audio/pipeline.h>
#include <sof/trace/trace.h>
#include <rtos/symbol.h>
#include <rtos/wait.h>

#define SHARED_BUFFER_HEAP_MEM_SIZE	0

#if CONFIG_L3_HEAP && CONFIG_MMU
#include <kernel_arch_interface.h>
#endif

#if CONFIG_VIRTUAL_HEAP
#include <sof/lib/regions_mm.h>
#include <zephyr/drivers/mm/mm_drv_intel_adsp_mtl_tlb.h>

struct vmh_heap;
struct vmh_heap *virtual_buffers_heap;
#endif /* CONFIG_VIRTUAL_HEAP */


/* Zephyr includes */
#include <zephyr/init.h>
#include <zephyr/kernel.h>
#include <zephyr/pm/policy.h>
#include <version.h>
#include <zephyr/sys/__assert.h>
#include <zephyr/cache.h>

#if CONFIG_SYS_HEAP_RUNTIME_STATS && CONFIG_IPC_MAJOR_4
#include <zephyr/sys/sys_heap.h>
#endif

LOG_MODULE_REGISTER(mem_allocator, CONFIG_SOF_LOG_LEVEL);

extern struct tr_ctx zephyr_tr;

/*
 * Memory - Create Zephyr HEAP for SOF.
 *
 * Currently functional but some items still WIP.
 */

#ifndef HEAP_RUNTIME_SIZE
#define HEAP_RUNTIME_SIZE	0
#endif

/* system size not declared on some platforms */
#ifndef HEAP_SYSTEM_SIZE
#define HEAP_SYSTEM_SIZE	0
#endif

/* The Zephyr heap */

#if defined(CONFIG_IMX) || defined(CONFIG_AMD)

#ifdef CONFIG_XTENSA
#define HEAPMEM_SIZE		(HEAP_SYSTEM_SIZE + HEAP_RUNTIME_SIZE + HEAP_BUFFER_SIZE)

/*
 * Include heapmem variable in .heap_mem section, otherwise the HEAPMEM_SIZE is
 * duplicated in two sections and the sdram0 region overflows.
 */
__section(".heap_mem") static uint8_t __aligned(64) heapmem[HEAPMEM_SIZE];

#else

/* for ARM64 the heap is placed inside the .bss section.
 *
 * This is because we want to avoid introducing new sections in
 * the arm64 linker script. Also, is there really a need to place
 * it inside a special section?
 *
 * i.MX93 is the only ARM64-based platform so defining the heap this way
 * for all ARM64-based platforms should be safe.
 */
static uint8_t __aligned(PLATFORM_DCACHE_ALIGN) heapmem[HEAPMEM_SIZE];
#endif /* CONFIG_XTENSA */

#elif CONFIG_ACE

/*
 * System heap definition for ACE is defined below.
 * It needs to be explicitly packed into dedicated section
 * to allow memory management driver to control unused
 * memory pages.
 */
#if CONFIG_USERSPACE
#undef SHARED_BUFFER_HEAP_MEM_SIZE
#define SHARED_BUFFER_HEAP_MEM_SIZE	ROUND_UP(CONFIG_SOF_ZEPHYR_SHARED_BUFFER_HEAP_SIZE, \
						 HOST_PAGE_SIZE)
#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP
__section(".shared_heap_mem")
static uint8_t __aligned(HOST_PAGE_SIZE) shared_heapmem[SHARED_BUFFER_HEAP_MEM_SIZE];
#endif
#endif /* CONFIG_USERSPACE */
__section(".heap_mem")
static uint8_t __aligned(HOST_PAGE_SIZE) heapmem[HEAPMEM_SIZE - SHARED_BUFFER_HEAP_MEM_SIZE];

#elif defined(CONFIG_ARCH_POSIX)

/* Zephyr native_posix links as a host binary and lacks the automated heap markers */
#define HEAPMEM_SIZE (256 * 1024)
char __aligned(8) heapmem[HEAPMEM_SIZE];

#elif defined(CONFIG_SOC_FAMILY_MTK)

extern char _mtk_adsp_sram_end;
#if defined(CONFIG_SOC_MT8365)
#define SRAM_START DT_REG_ADDR(DT_NODELABEL(sram1))
#define SRAM_SIZE  DT_REG_SIZE(DT_NODELABEL(sram1))
#define heapmem ((uint8_t *)SRAM_START)
#else
#define SRAM_START DT_REG_ADDR(DT_NODELABEL(sram0))
#define SRAM_SIZE  DT_REG_SIZE(DT_NODELABEL(sram0))
#define heapmem ((uint8_t *)ALIGN_UP((uintptr_t)&_mtk_adsp_sram_end, PLATFORM_DCACHE_ALIGN))
#endif /* CONFIG_SOC_MT8365 */

#define SRAM_END   (SRAM_START + SRAM_SIZE)

/* Heap size is limited to 0x7fffU chunk units when CONFIG_SYS_HEAP_SMALL_ONLY is set */
#if defined(CONFIG_SYS_HEAP_SMALL_ONLY)
#define HEAPMEM_SIZE MIN(((uint8_t *)SRAM_END - heapmem), 0x7fff * 8)
#else
#define HEAPMEM_SIZE ((uint8_t *)SRAM_END - heapmem)
#endif /* CONFIG_SYS_HEAP_SMALL_ONLY */

#else

extern char _end[], _heap_sentry[];
#define heapmem ((uint8_t *)ALIGN_UP((uintptr_t)_end, PLATFORM_DCACHE_ALIGN))
#define HEAPMEM_SIZE ((uint8_t *)_heap_sentry - heapmem)

#endif

static struct k_heap sof_heap;

/**
 * Checks whether pointer is from a given heap memory.
 * @param heap Pointer to a heap.
 * @param ptr Pointer to memory being checked.
 * @return True if pointer falls into heap memory region, false otherwise.
 */
static bool is_heap_pointer(const struct k_heap *heap, void *ptr)
{
	uintptr_t heap_start =
		POINTER_TO_UINT(sys_cache_cached_ptr_get(heap->heap.init_mem));
	uintptr_t heap_end = heap_start + heap->heap.init_bytes;

	if (!sys_cache_is_ptr_cached(ptr))
		ptr = (__sparse_force void *)sys_cache_cached_ptr_get(ptr);

	return ((POINTER_TO_UINT(ptr) >= heap_start) &&
		(POINTER_TO_UINT(ptr) < heap_end));
}

#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP
static struct k_heap shared_buffer_heap;

/**
 * Returns the start of HPSRAM Shared memory heap.
 * @return Pointer to the HPSRAM Shared memory location which can be used
 * for HPSRAM Shared heap.
 */
uintptr_t get_shared_buffer_heap_start(void)
{
	return ROUND_UP(POINTER_TO_UINT(shared_heapmem), HOST_PAGE_SIZE);
}

/**
 * Returns the size of HPSRAM Shared memory heap.
 * @return Size of the HPSRAM Shared memory region which can be used for HPSRAM Shared heap.
 */
size_t get_shared_buffer_heap_size(void)
{
	return ROUND_DOWN(SHARED_BUFFER_HEAP_MEM_SIZE, HOST_PAGE_SIZE);
}
#endif /* CONFIG_SOF_USERSPACE_USE_SHARED_HEAP */

#if CONFIG_L3_HEAP
static struct k_heap l3_heap;
static struct k_heap l3_heap_copy __imrdata;

/**
 * Returns the start of L3 memory heap.
 * @return Pointer to the L3 memory location which can be used for L3 heap.
 */
static inline uintptr_t get_l3_heap_start(void)
{
	/*
	 * TODO: parse the actual offset using:
	 * - HfIMRIA1 register
	 * - rom_ext_load_offset
	 * - main_fw_load_offset
	 * - main fw size in manifest
	 */
	return (uintptr_t)(ROUND_UP(IMR_L3_HEAP_BASE, L3_MEM_PAGE_SIZE));
}

/**
 * Returns the size of L3 memory heap.
 * @return Size of the L3 memory region which can be used for L3 heap.
 */
static inline size_t get_l3_heap_size(void)
{
	 /*
	  * Calculate the IMR heap size using:
	  * - total IMR size
	  * - IMR base address
	  * - actual IMR heap start
	  */
	size_t offset = IMR_L3_HEAP_BASE - L3_MEM_BASE_ADDR;

	return ROUND_DOWN(ace_imr_get_mem_size() - offset, L3_MEM_PAGE_SIZE);
}

void l3_heap_save(void)
{
	l3_heap_copy = l3_heap;
	LOG_DBG("L3 heap copy: %p", (void *)l3_heap_copy.heap.heap);
	dcache_writeback_region((__sparse_force void __sparse_cache *)&l3_heap_copy,
				sizeof(l3_heap_copy));
	dcache_writeback_region((__sparse_force void __sparse_cache *)get_l3_heap_start(),
				get_l3_heap_size());
}

static void *l3_heap_alloc_aligned(struct k_heap *h, size_t min_align, size_t bytes)
{
	k_spinlock_key_t key;
	void *ret;
#if CONFIG_SYS_HEAP_RUNTIME_STATS && CONFIG_IPC_MAJOR_4
	struct sys_memory_stats stats;
#endif
	if (!cpu_is_primary(arch_proc_id())) {
		tr_err(&zephyr_tr, "L3_HEAP available only for primary core!");
		return NULL;
	}

	key = k_spin_lock(&h->lock);
	ret = sys_heap_aligned_alloc(&h->heap, min_align, bytes);
	k_spin_unlock(&h->lock, key);

#if CONFIG_SYS_HEAP_RUNTIME_STATS && CONFIG_IPC_MAJOR_4
	sys_heap_runtime_stats_get(&h->heap, &stats);
	tr_info(&zephyr_tr, "heap allocated: %u free: %u max allocated: %u",
		stats.allocated_bytes, stats.free_bytes, stats.max_allocated_bytes);
#endif

	return ret;
}

static void l3_heap_free(struct k_heap *h, void *mem)
{
	if (!cpu_is_primary(arch_proc_id())) {
		tr_err(&zephyr_tr, "L3_HEAP available only for primary core!");
		return;
	}

	k_spinlock_key_t key = k_spin_lock(&h->lock);

	sys_heap_free(&h->heap, mem);
	k_spin_unlock(&h->lock, key);
}

#endif

#if CONFIG_VIRTUAL_HEAP
static void *virtual_heap_alloc(struct vmh_heap *heap, uint32_t flags, size_t bytes,
				uint32_t align)
{
	void *mem = vmh_alloc(heap, bytes);

	if (!mem) {
#ifdef CONFIG_SYS_MEM_BLOCKS_RUNTIME_STATS
		vmh_log_stats(heap);
#endif
		return NULL;
	}

	assert(align == 0 || IS_ALIGNED(mem, align));

	if (flags & SOF_MEM_FLAG_COHERENT)
		return sys_cache_uncached_ptr_get((__sparse_force void __sparse_cache *)mem);

	return mem;
}

extern int _unused_ram_start_marker;

/**
 * Checks whether pointer is from virtual memory range.
 * @param ptr Pointer to memory being checked.
 * @return True if pointer falls into virtual memory region, false otherwise.
 */
static bool is_virtual_heap_pointer(void *ptr)
{
	uintptr_t virtual_heap_start =
		POINTER_TO_UINT(sys_cache_cached_ptr_get(&_unused_ram_start_marker));
	uintptr_t virtual_heap_end = CONFIG_KERNEL_VM_BASE + CONFIG_KERNEL_VM_SIZE;

	if (!sys_cache_is_ptr_cached(ptr))
		ptr = (__sparse_force void *)sys_cache_cached_ptr_get(ptr);

	return ((POINTER_TO_UINT(ptr) >= virtual_heap_start) &&
		(POINTER_TO_UINT(ptr) < virtual_heap_end));
}

static void virtual_heap_free(void *ptr)
{
	int ret;

	ptr = (__sparse_force void *)sys_cache_cached_ptr_get(ptr);

	ret = vmh_free(virtual_buffers_heap, ptr);
	if (ret) {
		tr_err(&zephyr_tr, "Unable to free %p! %d", ptr, ret);
		k_panic();
	}
}

static const struct vmh_heap_config static_hp_buffers = {
	{
		{ 128, 32},
		{ 512, 8},
		{ 1024, 44},
		{ 2048, 8},
		{ 4096, 11},
		{ 8192, 10},
		{ 65536, 3},
		{ 131072, 1},
		{ 524288, 1} /* buffer for kpb */
	},
};

static int virtual_heap_init(void)
{
	int ret;

	if (virtual_buffers_heap)
		return -EEXIST;

	/* add a virtual memory region */
	ret = adsp_add_virtual_memory_region(adsp_mm_get_unused_l2_start_aligned(),
					     CONFIG_SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE,
					     VIRTUAL_REGION_SHARED_HEAP_ATTR);
	if (ret)
		return ret;

	virtual_buffers_heap = vmh_init_heap(&static_hp_buffers, false);
	if (!virtual_buffers_heap) {
		tr_err(&zephyr_tr, "Unable to init virtual heap");
		return -ENOMEM;
	}

	return 0;
}

SYS_INIT(virtual_heap_init, POST_KERNEL, 1);

#endif /* CONFIG_VIRTUAL_HEAP */

struct k_heap *sof_sys_heap_get(void)
{
	return &sof_heap;
}

static void *heap_alloc_aligned(struct k_heap *h, size_t min_align, size_t bytes)
{
	k_spinlock_key_t key;
	void *ret;
#if CONFIG_SYS_HEAP_RUNTIME_STATS && CONFIG_IPC_MAJOR_4
	struct sys_memory_stats stats;
#endif

	/*
	 * Zephyr sys_heap stores metadata at start of each
	 * heap allocation. To ensure no allocated cached buffer
	 * overlaps the same cacheline with the metadata chunk,
	 * align both allocation start and size of allocation
	 * to cacheline. As cached and non-cached allocations are
	 * mixed, same rules need to be followed for both type of
	 * allocations.
	 */
#ifdef CONFIG_SOF_ZEPHYR_HEAP_CACHED
	min_align = MAX(PLATFORM_DCACHE_ALIGN, min_align);
	bytes = ALIGN_UP(bytes, PLATFORM_DCACHE_ALIGN);
#endif

	key = k_spin_lock(&h->lock);
	ret = sys_heap_aligned_alloc(&h->heap, min_align, bytes);
	k_spin_unlock(&h->lock, key);

#if CONFIG_SYS_HEAP_RUNTIME_STATS && CONFIG_IPC_MAJOR_4
	sys_heap_runtime_stats_get(&h->heap, &stats);
	tr_info(&zephyr_tr, "heap allocated: %u free: %u max allocated: %u",
		stats.allocated_bytes, stats.free_bytes, stats.max_allocated_bytes);
#endif

	return ret;
}

static void __sparse_cache *heap_alloc_aligned_cached(struct k_heap *h,
						      size_t min_align, size_t bytes)
{
	void __sparse_cache *ptr;

	ptr = (__sparse_force void __sparse_cache *)heap_alloc_aligned(h, min_align, bytes);

#ifdef CONFIG_SOF_ZEPHYR_HEAP_CACHED
	if (ptr)
		ptr = sys_cache_cached_ptr_get((__sparse_force void *)ptr);
#endif

	return ptr;
}

static void heap_free(struct k_heap *h, void *mem)
{
	k_spinlock_key_t key = k_spin_lock(&h->lock);
#ifdef CONFIG_SOF_ZEPHYR_HEAP_CACHED
	void *mem_uncached;

	if (sys_cache_is_ptr_cached(mem)) {
		mem_uncached = sys_cache_uncached_ptr_get((__sparse_force void __sparse_cache *)mem);
		sys_cache_data_flush_and_invd_range(mem,
				sys_heap_usable_size(&h->heap, mem_uncached));

		mem = mem_uncached;
	}
#endif

	sys_heap_free(&h->heap, mem);

	k_spin_unlock(&h->lock, key);
}


void *rmalloc_align(uint32_t flags, size_t bytes, uint32_t alignment)
{
	void *ptr;
	struct k_heap *heap;

	/* choose a heap */
	if (flags & SOF_MEM_FLAG_L3) {
#if CONFIG_L3_HEAP
		heap = &l3_heap;
		/* Uncached L3_HEAP should not be used */
		if (flags & SOF_MEM_FLAG_COHERENT) {
			tr_err(&zephyr_tr, "L3_HEAP available for cached addresses only!");
			return NULL;
		}
		ptr = (__sparse_force void *)l3_heap_alloc_aligned(heap, alignment, bytes);

		return ptr;
#else
		k_panic();
#endif
#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP
	} else if (flags & SOF_MEM_FLAG_USER_SHARED_BUFFER) {
		heap = &shared_buffer_heap;
#endif
	} else {
		heap = &sof_heap;
	}

	if (!(flags & SOF_MEM_FLAG_COHERENT)) {
		ptr = (__sparse_force void *)heap_alloc_aligned_cached(heap, alignment, bytes);
	} else {
		ptr = heap_alloc_aligned(heap, alignment, bytes);
	}

	return ptr;
}
EXPORT_SYMBOL(rmalloc_align);

void *rmalloc(uint32_t flags, size_t bytes)
{
	return rmalloc_align(flags, bytes, 0);
}
EXPORT_SYMBOL(rmalloc);

void *rbrealloc_align(void *ptr, uint32_t flags, size_t bytes,
		      size_t old_bytes, uint32_t alignment)
{
	void *new_ptr;

	if (!ptr) {
		return rballoc_align(flags, bytes, alignment);
	}

	/* Original version returns NULL without freeing this memory */
	if (!bytes) {
		/* TODO: Should we call rfree(ptr); */
		tr_err(&zephyr_tr, "realloc failed for 0 bytes");
		return NULL;
	}

	new_ptr = rballoc_align(flags, bytes, alignment);
	if (!new_ptr)
		return NULL;

	if (!(flags & SOF_MEM_FLAG_NO_COPY))
		memcpy_s(new_ptr, bytes, ptr, MIN(bytes, old_bytes));

	rfree(ptr);

	tr_info(&zephyr_tr, "rbealloc: new ptr %p", new_ptr);

	return new_ptr;
}

/**
 * Similar to rmalloc(), guarantees that returned block is zeroed.
 */
void *rzalloc(uint32_t flags, size_t bytes)
{
	void *ptr = rmalloc(flags, bytes);

	if (ptr)
		memset(ptr, 0, bytes);

	return ptr;
}
EXPORT_SYMBOL(rzalloc);

/**
 * Allocates memory block.
 * @param flags see SOF_MEM_FLAG_...
 * @param bytes Size in bytes.
 * @param align Alignment in bytes.
 * @return Pointer to the allocated memory or NULL if failed.
 */
void *rballoc_align(uint32_t flags, size_t bytes, uint32_t align)
{
	struct k_heap *heap;

	/* choose a heap */
	if (flags & SOF_MEM_FLAG_L3) {
#if CONFIG_L3_HEAP
		heap = &l3_heap;
		return (__sparse_force void *)l3_heap_alloc_aligned(heap, align, bytes);
#else
		tr_err(&zephyr_tr, "L3_HEAP not available.");
		return NULL;
#endif
#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP
	} else if (flags & SOF_MEM_FLAG_USER_SHARED_BUFFER) {
		heap = &shared_buffer_heap;
#endif /* CONFIG_USERSPACE */
	} else {
#if CONFIG_VIRTUAL_HEAP
		/* Use virtual heap if it is available */
		if (virtual_buffers_heap)
			return virtual_heap_alloc(virtual_buffers_heap, flags, bytes, align);
#endif /* CONFIG_VIRTUAL_HEAP */

		heap = &sof_heap;
	}

	if (flags & SOF_MEM_FLAG_COHERENT)
		return heap_alloc_aligned(heap, align, bytes);

	return (__sparse_force void *)heap_alloc_aligned_cached(heap, align, bytes);
}
EXPORT_SYMBOL(rballoc_align);

/*
 * Free's memory allocated by above alloc calls.
 */
void rfree(void *ptr)
{
	if (!ptr)
		return;

#if CONFIG_L3_HEAP
	if (is_heap_pointer(&l3_heap, ptr)) {
		l3_heap_free(&l3_heap, ptr);
		return;
	}
#endif

#if CONFIG_VIRTUAL_HEAP
	if (is_virtual_heap_pointer(ptr)) {
		virtual_heap_free(ptr);
		return;
	}
#endif

#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP
	if (is_heap_pointer(&shared_buffer_heap, ptr)) {
		heap_free(&shared_buffer_heap, ptr);
		return;
	}
#endif

	heap_free(&sof_heap, ptr);
}
EXPORT_SYMBOL(rfree);

/*
 * To match the fall-back SOF main heap all private heaps should also be in the
 * uncached address range.
 */
void *sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes,
		     size_t alignment)
{
	if (flags & (SOF_MEM_FLAG_LARGE_BUFFER | SOF_MEM_FLAG_USER_SHARED_BUFFER))
		return rballoc_align(flags, bytes, alignment);

	if (!heap)
		heap = &sof_heap;

	if (flags & SOF_MEM_FLAG_COHERENT)
		return heap_alloc_aligned(heap, alignment, bytes);

	return (__sparse_force void *)heap_alloc_aligned_cached(heap, alignment, bytes);
}

void sof_heap_free(struct k_heap *heap, void *addr)
{
	if (heap && addr && is_heap_pointer(heap, addr))
		heap_free(heap, addr);
	else
		rfree(addr);
}

static int heap_init(void)
{
	sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE - SHARED_BUFFER_HEAP_MEM_SIZE);

#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP
	shared_buffer_heap.heap.init_mem = shared_heapmem;
	shared_buffer_heap.heap.init_bytes = SHARED_BUFFER_HEAP_MEM_SIZE;
	sys_heap_init(&shared_buffer_heap.heap, shared_heapmem, SHARED_BUFFER_HEAP_MEM_SIZE);
#endif

#if CONFIG_L3_HEAP
	if (ace_imr_used()) {
		void *l3_heap_start = UINT_TO_POINTER(get_l3_heap_start());
		size_t l3_heap_size = get_l3_heap_size();
#if CONFIG_MMU
		void *cached_ptr = sys_cache_cached_ptr_get(l3_heap_start);
		uintptr_t va = POINTER_TO_UINT(cached_ptr);

		arch_mem_map(l3_heap_start, va, l3_heap_size, K_MEM_PERM_RW | K_MEM_CACHE_WB);
#endif
		if (l3_heap_copy.heap.heap) {
			l3_heap = l3_heap_copy;
		} else {
			l3_heap.heap.init_mem = l3_heap_start;
			l3_heap.heap.init_bytes = l3_heap_size;
			sys_heap_init(&l3_heap.heap, l3_heap_start, l3_heap_size);
		}
	}
#endif

	return 0;
}

/* This is a weak stub for the Cadence libc's allocator (which is just
 * a newlib build).  It's traditionally been provided like this in SOF
 * for the benefit of C++ code where the standard library needs to
 * link to a working malloc() even if it will never call it.
 */
struct _reent;
__weak void *_sbrk_r(struct _reent *ptr, ptrdiff_t incr)
{
	k_panic();
	return NULL;
}

SYS_INIT(heap_init, PRE_KERNEL_1, CONFIG_KERNEL_INIT_PRIORITY_OBJECTS);