diff --git a/posix/include/rtos/alloc.h b/posix/include/rtos/alloc.h index b8efac0d3112..5ba40b258512 100644 --- a/posix/include/rtos/alloc.h +++ b/posix/include/rtos/alloc.h @@ -97,29 +97,6 @@ static inline void *rballoc(uint32_t flags, size_t bytes) return rballoc_align(flags, bytes, PLATFORM_DCACHE_ALIGN); } -/** - * Changes size of the memory block allocated. - * @param ptr Address of the block to resize. - * @param flags Flags, see SOF_MEM_FLAG_... - * @param bytes New size in bytes. - * @param old_bytes Old size in bytes. - * @param alignment Alignment in bytes. - * @return Pointer to the resized memory of NULL if failed. - */ -void *rbrealloc_align(void *ptr, uint32_t flags, size_t bytes, - size_t old_bytes, uint32_t alignment); - -/** - * Similar to rballoc_align(), returns resized buffer aligned to - * PLATFORM_DCACHE_ALIGN. - */ -static inline void *rbrealloc(void *ptr, uint32_t flags, - size_t bytes, size_t old_bytes) -{ - return rbrealloc_align(ptr, flags, bytes, old_bytes, - PLATFORM_DCACHE_ALIGN); -} - /** * Frees the memory block. * @param ptr Pointer to the memory block. diff --git a/posix/include/sof/lib/dma.h b/posix/include/sof/lib/dma.h index 960cfd469215..cb829dcaa21a 100644 --- a/posix/include/sof/lib/dma.h +++ b/posix/include/sof/lib/dma.h @@ -35,6 +35,7 @@ struct comp_buffer; struct comp_dev; +struct k_heap; /** \addtogroup sof_dma_drivers DMA Drivers * DMA Drivers API specification. @@ -511,13 +512,14 @@ static inline void dma_sg_init(struct dma_sg_elem_array *ea) ea->elems = NULL; } -int dma_sg_alloc(struct dma_sg_elem_array *ea, +int dma_sg_alloc(struct k_heap *heap, + struct dma_sg_elem_array *ea, uint32_t flags, uint32_t direction, uint32_t buffer_count, uint32_t buffer_bytes, uintptr_t dma_buffer_addr, uintptr_t external_addr); -void dma_sg_free(struct dma_sg_elem_array *ea); +void dma_sg_free(struct k_heap *heap, struct dma_sg_elem_array *ea); /** * \brief Get the total size of SG buffer diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index d6562b8d821f..328b4ba571f6 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -153,22 +153,31 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) buf_dbg(buffer, "buffer_free()"); +#ifndef CONFIG_SOF_USERSPACE_LL + /* TODO: to be replaced with direct function calls */ notifier_event(buffer, NOTIFIER_ID_BUFFER_FREE, NOTIFIER_TARGET_CORE_LOCAL, &cb_data, sizeof(cb_data)); /* In case some listeners didn't unregister from buffer's callbacks */ notifier_unregister_all(NULL, buffer); +#endif struct k_heap *heap = buffer->audio_buffer.heap; - rfree(buffer->stream.addr); + sof_heap_free(heap, buffer->stream.addr); sof_heap_free(heap, buffer); +#ifndef CONFIG_SOF_USERSPACE_LL + /* + * TODO: this is not the correct place to free up + * mod_heap_user + */ if (heap) { struct dp_heap_user *mod_heap_user = container_of(heap, struct dp_heap_user, heap); if (!--mod_heap_user->client_count) rfree(mod_heap_user); } +#endif } APP_TASK_DATA static const struct source_ops comp_buffer_source_ops = { @@ -218,6 +227,7 @@ static struct comp_buffer *buffer_alloc_struct(struct k_heap *heap, memset(buffer, 0, sizeof(*buffer)); + buffer->heap = heap; buffer->flags = flags; /* Force channels to 2 for init to prevent bad call to clz in buffer_init_stream */ buffer->stream.runtime_stream_params.channels = 2; @@ -254,7 +264,7 @@ struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flag return NULL; } - stream_addr = rballoc_align(flags, size, align); + stream_addr = sof_heap_alloc(heap, flags, size, align); if (!stream_addr) { tr_err(&buffer_tr, "could not alloc size = %zu bytes of flags = 0x%x", size, flags); @@ -264,9 +274,11 @@ struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flag buffer = buffer_alloc_struct(heap, stream_addr, size, flags, is_shared); if (!buffer) { tr_err(&buffer_tr, "could not alloc buffer structure"); - rfree(stream_addr); + sof_heap_free(heap, stream_addr); } + buffer->heap = heap; + return buffer; } @@ -292,7 +304,7 @@ struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_siz preferred_size += minimum_size - preferred_size % minimum_size; for (size = preferred_size; size >= minimum_size; size -= minimum_size) { - stream_addr = rballoc_align(flags, size, align); + stream_addr = sof_heap_alloc(heap, flags, size, align); if (stream_addr) break; } @@ -308,9 +320,11 @@ struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_siz buffer = buffer_alloc_struct(heap, stream_addr, size, flags, is_shared); if (!buffer) { tr_err(&buffer_tr, "could not alloc buffer structure"); - rfree(stream_addr); + sof_heap_free(heap, stream_addr); } + buffer->heap = heap; + return buffer; } @@ -341,14 +355,8 @@ int buffer_set_size(struct comp_buffer *buffer, uint32_t size, uint32_t alignmen if (size == audio_stream_get_size(&buffer->stream)) return 0; - if (!alignment) - new_ptr = rbrealloc(audio_stream_get_addr(&buffer->stream), - buffer->flags | SOF_MEM_FLAG_NO_COPY, - size, audio_stream_get_size(&buffer->stream)); - else - new_ptr = rbrealloc_align(audio_stream_get_addr(&buffer->stream), - buffer->flags | SOF_MEM_FLAG_NO_COPY, size, - audio_stream_get_size(&buffer->stream), alignment); + new_ptr = sof_heap_alloc(buffer->heap, buffer->flags, size, alignment); + /* we couldn't allocate bigger chunk */ if (!new_ptr && size > audio_stream_get_size(&buffer->stream)) { buf_err(buffer, "resize can't alloc %u bytes of flags 0x%x", @@ -357,8 +365,10 @@ int buffer_set_size(struct comp_buffer *buffer, uint32_t size, uint32_t alignmen } /* use bigger chunk, else just use the old chunk but set smaller */ - if (new_ptr) + if (new_ptr) { + sof_heap_free(buffer->heap, audio_stream_get_addr(&buffer->stream)); buffer->stream.addr = new_ptr; + } buffer_init_stream(buffer, size); @@ -389,22 +399,11 @@ int buffer_set_size_range(struct comp_buffer *buffer, size_t preferred_size, siz if (preferred_size == actual_size) return 0; - if (!alignment) { - for (new_size = preferred_size; new_size >= minimum_size; - new_size -= minimum_size) { - new_ptr = rbrealloc(ptr, buffer->flags | SOF_MEM_FLAG_NO_COPY, - new_size, actual_size); - if (new_ptr) - break; - } - } else { - for (new_size = preferred_size; new_size >= minimum_size; - new_size -= minimum_size) { - new_ptr = rbrealloc_align(ptr, buffer->flags | SOF_MEM_FLAG_NO_COPY, - new_size, actual_size, alignment); - if (new_ptr) - break; - } + for (new_size = preferred_size; new_size >= minimum_size; + new_size -= minimum_size) { + new_ptr = sof_heap_alloc(buffer->heap, buffer->flags, new_size, alignment); + if (new_ptr) + break; } /* we couldn't allocate bigger chunk */ @@ -415,8 +414,10 @@ int buffer_set_size_range(struct comp_buffer *buffer, size_t preferred_size, siz } /* use bigger chunk, else just use the old chunk but set smaller */ - if (new_ptr) + if (new_ptr) { + sof_heap_free(buffer->heap, audio_stream_get_addr(&buffer->stream)); buffer->stream.addr = new_ptr; + } buffer_init_stream(buffer, new_size); diff --git a/src/audio/component.c b/src/audio/component.c index 941be20f8534..83d35bc18a1c 100644 --- a/src/audio/component.c +++ b/src/audio/component.c @@ -699,3 +699,11 @@ void comp_update_ibs_obs_cpc(struct comp_dev *dev) #endif } +#ifdef CONFIG_SOF_USERSPACE_LL +void comp_grant_access_to_thread(const struct comp_dev *dev, struct k_thread *th) +{ + assert(dev->list_mutex); + tr_dbg(&ipc_tr, "grant access to mutex %p for thread %p", dev->list_mutex, th); + k_thread_access_grant(th, dev->list_mutex); +} +#endif diff --git a/src/audio/copier/copier_dai.c b/src/audio/copier/copier_dai.c index 821aa59d7640..43e731814300 100644 --- a/src/audio/copier/copier_dai.c +++ b/src/audio/copier/copier_dai.c @@ -208,6 +208,8 @@ __cold static int copier_dai_init(struct comp_dev *dev, if (!dd) return -ENOMEM; memset(dd, 0, sizeof(*dd)); + dd->chan_index = -1; + comp_info(dev, "dd %p initialized, index %d", dd, dd->chan_index); ret = dai_common_new(dd, dev, dai); if (ret < 0) diff --git a/src/audio/copier/host_copier.h b/src/audio/copier/host_copier.h index 28605ed9c3ab..5192506da42f 100644 --- a/src/audio/copier/host_copier.h +++ b/src/audio/copier/host_copier.h @@ -50,10 +50,11 @@ struct host_data { /* local DMA config */ #if CONFIG_ZEPHYR_NATIVE_DRIVERS struct sof_dma *dma; + int chan_index; #else struct dma *dma; -#endif struct dma_chan_data *chan; +#endif struct dma_sg_config config; #ifdef __ZEPHYR__ struct dma_config z_config; @@ -112,6 +113,7 @@ struct host_data { uint64_t next_sync; uint64_t period_in_cycles; #endif + struct k_heap *heap; }; int host_common_new(struct host_data *hd, struct comp_dev *dev, diff --git a/src/audio/dai-legacy.c b/src/audio/dai-legacy.c index 11179334e6fe..655fa94afb86 100644 --- a/src/audio/dai-legacy.c +++ b/src/audio/dai-legacy.c @@ -379,7 +379,7 @@ static int dai_playback_params(struct comp_dev *dev, uint32_t period_bytes, comp_info(dev, "fifo 0x%x", fifo); - err = dma_sg_alloc(&config->elem_array, SOF_MEM_FLAG_USER, + err = dma_sg_alloc(NULL, &config->elem_array, SOF_MEM_FLAG_USER, config->direction, period_count, period_bytes, @@ -444,7 +444,7 @@ static int dai_capture_params(struct comp_dev *dev, uint32_t period_bytes, comp_info(dev, "fifo 0x%x", fifo); - err = dma_sg_alloc(&config->elem_array, SOF_MEM_FLAG_USER, + err = dma_sg_alloc(NULL, &config->elem_array, SOF_MEM_FLAG_USER, config->direction, period_count, period_bytes, @@ -709,7 +709,7 @@ void dai_common_reset(struct dai_data *dd, struct comp_dev *dev) if (!dd->delayed_dma_stop) dai_dma_release(dd, dev); - dma_sg_free(&config->elem_array); + dma_sg_free(NULL, &config->elem_array); if (dd->dma_buffer) { buffer_free(dd->dma_buffer); diff --git a/src/audio/dai-zephyr.c b/src/audio/dai-zephyr.c index e653e93ae082..95605ed07dd4 100644 --- a/src/audio/dai-zephyr.c +++ b/src/audio/dai-zephyr.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -202,55 +203,62 @@ __cold int dai_set_config(struct dai *dai, struct ipc_config_dai *common_config, /* called from ipc/ipc3/dai.c */ int dai_get_handshake(struct dai *dai, int direction, int stream_id) { - k_spinlock_key_t key = k_spin_lock(&dai->lock); - const struct dai_properties *props = dai_get_properties(dai->dev, direction, - stream_id); - int hs_id = props->dma_hs_id; + struct dai_properties props; + int ret; - k_spin_unlock(&dai->lock, key); + k_mutex_lock(dai->lock, K_FOREVER); + ret = dai_get_properties_copy(dai->dev, direction, stream_id, &props); + k_mutex_unlock(dai->lock); + if (ret < 0) + return ret; - return hs_id; + return props.dma_hs_id; } /* called from ipc/ipc3/dai.c and ipc/ipc4/dai.c */ int dai_get_fifo_depth(struct dai *dai, int direction) { - const struct dai_properties *props; - k_spinlock_key_t key; - int fifo_depth; + struct dai_properties props; + int ret; if (!dai) return 0; - key = k_spin_lock(&dai->lock); - props = dai_get_properties(dai->dev, direction, 0); - fifo_depth = props->fifo_depth; - k_spin_unlock(&dai->lock, key); + k_mutex_lock(dai->lock, K_FOREVER); + ret = dai_get_properties_copy(dai->dev, direction, 0, &props); + k_mutex_unlock(dai->lock); + if (ret < 0) + return 0; - return fifo_depth; + return props.fifo_depth; } int dai_get_stream_id(struct dai *dai, int direction) { - k_spinlock_key_t key = k_spin_lock(&dai->lock); - const struct dai_properties *props = dai_get_properties(dai->dev, direction, 0); - int stream_id = props->stream_id; + struct dai_properties props; + int ret; - k_spin_unlock(&dai->lock, key); + k_mutex_lock(dai->lock, K_FOREVER); + ret = dai_get_properties_copy(dai->dev, direction, 0, &props); + k_mutex_unlock(dai->lock); + if (ret < 0) + return ret; - return stream_id; + return props.stream_id; } static int dai_get_fifo(struct dai *dai, int direction, int stream_id) { - k_spinlock_key_t key = k_spin_lock(&dai->lock); - const struct dai_properties *props = dai_get_properties(dai->dev, direction, - stream_id); - int fifo_address = props->fifo_address; + struct dai_properties props; + int ret; - k_spin_unlock(&dai->lock, key); + k_mutex_lock(dai->lock, K_FOREVER); + ret = dai_get_properties_copy(dai->dev, direction, stream_id, &props); + k_mutex_unlock(dai->lock); + if (ret < 0) + return ret; - return fifo_address; + return props.fifo_address; } /* this is called by DMA driver every time descriptor has completed */ @@ -500,11 +508,25 @@ __cold int dai_common_new(struct dai_data *dd, struct comp_dev *dev, return -ENODEV; } - k_spinlock_init(&dd->dai->lock); +#ifdef CONFIG_SOF_USERSPACE_LL + dd->dai->lock = k_object_alloc(K_OBJ_MUTEX); + comp_set_drvdata(dev, dd); +#else + dd->dai->lock = &dd->dai->lock_obj; +#endif + k_mutex_init(dd->dai->lock); dma_sg_init(&dd->config.elem_array); dd->xrun = 0; - dd->chan = NULL; + +#ifdef CONFIG_SOF_USERSPACE_LL + /* + * copier_dai_create() uses mod_zalloc() to allocate + * the 'dd' dai data object and does not set dd->heap. + * If LL is run in user-space, assign the 'heap' here. + */ + dd->heap = zephyr_ll_user_heap(); +#endif /* I/O performance init, keep it last so the function does not reach this in case * of return on error, so that we do not waste a slot @@ -558,6 +580,7 @@ __cold static struct comp_dev *dai_new(const struct comp_driver *drv, struct comp_dev *dev; const struct ipc_config_dai *dai_cfg = spec; struct dai_data *dd; + struct k_heap *heap = NULL; int ret; assert_can_be_cold(); @@ -570,10 +593,19 @@ __cold static struct comp_dev *dai_new(const struct comp_driver *drv, dev->ipc_config = *config; - dd = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*dd)); +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); +#endif + + dd = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*dd), 0); if (!dd) goto e_data; + memset(dd, 0, sizeof(*dd)); + dd->heap = heap; + dd->chan_index = -1; + comp_info(dev, "dd %p initialized, index %d", dd, dd->chan_index); + comp_set_drvdata(dev, dd); ret = dai_common_new(dd, dev, dai_cfg); @@ -587,7 +619,7 @@ __cold static struct comp_dev *dai_new(const struct comp_driver *drv, return dev; error: - rfree(dd); + sof_heap_free(dd->heap, dd); e_data: comp_free_device(dev); return NULL; @@ -604,18 +636,20 @@ __cold void dai_common_free(struct dai_data *dd) if (dd->group) dai_group_put(dd->group); - if (dd->chan) { - sof_dma_release_channel(dd->dma, dd->chan->index); - dd->chan->dev_data = NULL; - } + if (dd->chan_index != -1) + sof_dma_release_channel(dd->dma, dd->chan_index); sof_dma_put(dd->dma); dai_release_llp_slot(dd); +#ifdef CONFIG_SOF_USERSPACE_LL + k_object_free(dd->dai->lock); +#endif + dai_put(dd->dai); - rfree(dd->dai_spec_config); + sof_heap_free(dd->heap, dd->dai_spec_config); } __cold static void dai_free(struct comp_dev *dev) @@ -629,7 +663,7 @@ __cold static void dai_free(struct comp_dev *dev) dai_common_free(dd); - rfree(dd); + sof_heap_free(dd->heap, dd); comp_free_device(dev); } @@ -824,7 +858,7 @@ static int dai_set_sg_config(struct dai_data *dd, struct comp_dev *dev, uint32_t } while (--max_block_count > 0); } - err = dma_sg_alloc(&config->elem_array, SOF_MEM_FLAG_USER, + err = dma_sg_alloc(dd->heap, &config->elem_array, SOF_MEM_FLAG_USER, config->direction, period_count, period_bytes, @@ -850,8 +884,9 @@ static int dai_set_dma_config(struct dai_data *dd, struct comp_dev *dev) comp_dbg(dev, "entry"); - dma_cfg = rballoc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT | SOF_MEM_FLAG_DMA, - sizeof(struct dma_config)); + dma_cfg = sof_heap_alloc(dd->heap, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT | SOF_MEM_FLAG_DMA, + sizeof(struct dma_config), 0); if (!dma_cfg) { comp_err(dev, "dma_cfg allocation failed"); return -ENOMEM; @@ -880,10 +915,11 @@ static int dai_set_dma_config(struct dai_data *dd, struct comp_dev *dev) else dma_cfg->dma_slot = config->src_dev; - dma_block_cfg = rballoc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT | SOF_MEM_FLAG_DMA, - sizeof(struct dma_block_config) * dma_cfg->block_count); + dma_block_cfg = sof_heap_alloc(dd->heap, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT | SOF_MEM_FLAG_DMA, + sizeof(struct dma_block_config) * dma_cfg->block_count, 0); if (!dma_block_cfg) { - rfree(dma_cfg); + sof_heap_free(dd->heap, dma_cfg); comp_err(dev, "dma_block_config allocation failed"); return -ENOMEM; } @@ -1017,7 +1053,7 @@ static int dai_set_dma_buffer(struct dai_data *dd, struct comp_dev *dev, return err; } } else { - dd->dma_buffer = buffer_alloc_range(NULL, buffer_size_preferred, buffer_size, + dd->dma_buffer = buffer_alloc_range(dd->heap, buffer_size_preferred, buffer_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, BUFFER_USAGE_NOT_SHARED); if (!dd->dma_buffer) { @@ -1105,8 +1141,8 @@ int dai_common_params(struct dai_data *dd, struct comp_dev *dev, if (err < 0) { buffer_free(dd->dma_buffer); dd->dma_buffer = NULL; - dma_sg_free(&config->elem_array); - rfree(dd->z_config); + dma_sg_free(dd->heap, &config->elem_array); + sof_heap_free(dd->heap, dd->z_config); dd->z_config = NULL; } @@ -1137,9 +1173,9 @@ int dai_common_config_prepare(struct dai_data *dd, struct comp_dev *dev) return -EINVAL; } - if (dd->chan) { + if (dd->chan_index != -1) { comp_info(dev, "dma channel index %d already configured", - dd->chan->index); + dd->chan_index); return 0; } @@ -1153,18 +1189,14 @@ int dai_common_config_prepare(struct dai_data *dd, struct comp_dev *dev) } /* get DMA channel */ - channel = sof_dma_request_channel(dd->dma, channel); - if (channel < 0) { + dd->chan_index = sof_dma_request_channel(dd->dma, channel); + if (dd->chan_index < 0) { comp_err(dev, "dma_request_channel() failed"); - dd->chan = NULL; return -EIO; } - dd->chan = &dd->dma->chan[channel]; - dd->chan->dev_data = dd; - comp_dbg(dev, "new configured dma channel index %d", - dd->chan->index); + dd->chan_index); return 0; } @@ -1175,8 +1207,8 @@ int dai_common_prepare(struct dai_data *dd, struct comp_dev *dev) dd->total_data_processed = 0; - if (!dd->chan) { - comp_err(dev, "Missing dd->chan."); + if (dd->chan_index == -1) { + comp_err(dev, "Missing dd->chan_index."); comp_set_state(dev, COMP_TRIGGER_RESET); return -EINVAL; } @@ -1197,7 +1229,7 @@ int dai_common_prepare(struct dai_data *dd, struct comp_dev *dev) return 0; } - ret = sof_dma_config(dd->chan->dma, dd->chan->index, dd->z_config); + ret = sof_dma_config(dd->dma, dd->chan_index, dd->z_config); if (ret < 0) comp_set_state(dev, COMP_TRIGGER_RESET); @@ -1236,10 +1268,10 @@ void dai_common_reset(struct dai_data *dd, struct comp_dev *dev) if (!dd->delayed_dma_stop) dai_dma_release(dd, dev); - dma_sg_free(&config->elem_array); + dma_sg_free(dd->heap, &config->elem_array); if (dd->z_config) { - rfree(dd->z_config->head_block); - rfree(dd->z_config); + sof_heap_free(dd->heap, dd->z_config->head_block); + sof_heap_free(dd->heap, dd->z_config); dd->z_config = NULL; } @@ -1284,7 +1316,7 @@ static int dai_comp_trigger_internal(struct dai_data *dd, struct comp_dev *dev, /* only start the DAI if we are not XRUN handling */ if (dd->xrun == 0) { - ret = sof_dma_start(dd->chan->dma, dd->chan->index); + ret = sof_dma_start(dd->dma, dd->chan_index); if (ret < 0) return ret; @@ -1322,16 +1354,16 @@ static int dai_comp_trigger_internal(struct dai_data *dd, struct comp_dev *dev, /* only start the DAI if we are not XRUN handling */ if (dd->xrun == 0) { /* recover valid start position */ - ret = sof_dma_stop(dd->chan->dma, dd->chan->index); + ret = sof_dma_stop(dd->dma, dd->chan_index); if (ret < 0) return ret; /* dma_config needed after stop */ - ret = sof_dma_config(dd->chan->dma, dd->chan->index, dd->z_config); + ret = sof_dma_config(dd->dma, dd->chan_index, dd->z_config); if (ret < 0) return ret; - ret = sof_dma_start(dd->chan->dma, dd->chan->index); + ret = sof_dma_start(dd->dma, dd->chan_index); if (ret < 0) return ret; @@ -1359,11 +1391,11 @@ static int dai_comp_trigger_internal(struct dai_data *dd, struct comp_dev *dev, * as soon as possible. */ #if CONFIG_COMP_DAI_STOP_TRIGGER_ORDER_REVERSE - ret = sof_dma_stop(dd->chan->dma, dd->chan->index); + ret = sof_dma_stop(dd->dma, dd->chan_index); dai_trigger_op(dd->dai, cmd, dev->direction); #else dai_trigger_op(dd->dai, cmd, dev->direction); - ret = sof_dma_stop(dd->chan->dma, dd->chan->index); + ret = sof_dma_stop(dd->dma, dd->chan_index); if (ret) { comp_warn(dev, "dma was stopped earlier"); ret = 0; @@ -1373,11 +1405,11 @@ static int dai_comp_trigger_internal(struct dai_data *dd, struct comp_dev *dev, case COMP_TRIGGER_PAUSE: comp_dbg(dev, "PAUSE"); #if CONFIG_COMP_DAI_STOP_TRIGGER_ORDER_REVERSE - ret = sof_dma_suspend(dd->chan->dma, dd->chan->index); + ret = sof_dma_suspend(dd->dma, dd->chan_index); dai_trigger_op(dd->dai, cmd, dev->direction); #else dai_trigger_op(dd->dai, cmd, dev->direction); - ret = sof_dma_suspend(dd->chan->dma, dd->chan->index); + ret = sof_dma_suspend(dd->dma, dd->chan_index); #endif break; case COMP_TRIGGER_PRE_START: @@ -1475,7 +1507,7 @@ static int dai_comp_trigger(struct comp_dev *dev, int cmd) */ static int dai_get_status(struct comp_dev *dev, struct dai_data *dd, struct dma_status *stat) { - int ret = sof_dma_get_status(dd->chan->dma, dd->chan->index, stat); + int ret = sof_dma_get_status(dd->dma, dd->chan_index, stat); #if CONFIG_XRUN_NOTIFICATIONS_ENABLE if (ret == -EPIPE && !dd->xrun_notification_sent) { struct ipc_msg *notify = ipc_notification_pool_get(IPC4_RESOURCE_EVENT_SIZE); @@ -1591,7 +1623,7 @@ int dai_zephyr_multi_endpoint_copy(struct dai_data **dd, struct comp_dev *dev, #endif for (i = 0; i < num_endpoints; i++) { - ret = sof_dma_reload(dd[i]->chan->dma, dd[i]->chan->index, 0); + ret = sof_dma_reload(dd[i]->dma, dd[i]->chan_index, 0); if (ret < 0) { dai_report_reload_xrun(dd[i], dev, 0); return ret; @@ -1617,10 +1649,10 @@ int dai_zephyr_multi_endpoint_copy(struct dai_data **dd, struct comp_dev *dev, status = dai_dma_multi_endpoint_cb(dd[i], dev, frames, multi_endpoint_buffer); if (status == SOF_DMA_CB_STATUS_END) - sof_dma_stop(dd[i]->chan->dma, dd[i]->chan->index); + sof_dma_stop(dd[i]->dma, dd[i]->chan_index); copy_bytes = frames * audio_stream_frame_bytes(&dd[i]->dma_buffer->stream); - ret = sof_dma_reload(dd[i]->chan->dma, dd[i]->chan->index, copy_bytes); + ret = sof_dma_reload(dd[i]->dma, dd[i]->chan_index, copy_bytes); if (ret < 0) { dai_report_reload_xrun(dd[i], dev, copy_bytes); return ret; @@ -1809,7 +1841,7 @@ int dai_common_copy(struct dai_data *dd, struct comp_dev *dev, pcm_converter_fun comp_warn(dev, "nothing to copy, src_frames: %u, sink_frames: %u", src_frames, sink_frames); #endif - sof_dma_reload(dd->chan->dma, dd->chan->index, 0); + sof_dma_reload(dd->dma, dd->chan_index, 0); return 0; } @@ -1819,9 +1851,9 @@ int dai_common_copy(struct dai_data *dd, struct comp_dev *dev, pcm_converter_fun comp_warn(dev, "dai trigger copy failed"); if (dai_dma_cb(dd, dev, copy_bytes, converter) == SOF_DMA_CB_STATUS_END) - sof_dma_stop(dd->chan->dma, dd->chan->index); + sof_dma_stop(dd->dma, dd->chan_index); - ret = sof_dma_reload(dd->chan->dma, dd->chan->index, copy_bytes); + ret = sof_dma_reload(dd->dma, dd->chan_index, copy_bytes); if (ret < 0) { dai_report_reload_xrun(dd, dev, copy_bytes); return ret; @@ -1859,7 +1891,7 @@ int dai_common_ts_config_op(struct dai_data *dd, struct comp_dev *dev) struct dai_ts_cfg *cfg = &dd->ts_config; comp_dbg(dev, "dai_ts_config()"); - if (!dd->chan) { + if (dd->chan_index == -1) { comp_err(dev, "No DMA channel information"); return -EINVAL; } @@ -1882,7 +1914,7 @@ int dai_common_ts_config_op(struct dai_data *dd, struct comp_dev *dev) cfg->direction = dai->direction; cfg->index = dd->dai->index; cfg->dma_id = dd->dma->plat_data.id; - cfg->dma_chan_index = dd->chan->index; + cfg->dma_chan_index = dd->chan_index; cfg->dma_chan_count = dd->dma->plat_data.channels; return dai_ts_config(dd->dai->dev, cfg); @@ -1940,17 +1972,18 @@ static int dai_ts_stop_op(struct comp_dev *dev) uint32_t dai_get_init_delay_ms(struct dai *dai) { - const struct dai_properties *props; - k_spinlock_key_t key; - uint32_t init_delay; + struct dai_properties props; + uint32_t init_delay = 0; + int ret; if (!dai) return 0; - key = k_spin_lock(&dai->lock); - props = dai_get_properties(dai->dev, 0, 0); - init_delay = props->reg_init_delay; - k_spin_unlock(&dai->lock, key); + k_mutex_lock(dai->lock, K_FOREVER); + ret = dai_get_properties_copy(dai->dev, 0, 0, &props); + if (!ret) + init_delay = props.reg_init_delay; + k_mutex_unlock(dai->lock); return init_delay; } diff --git a/src/audio/host-legacy.c b/src/audio/host-legacy.c index a16b3f74e1c3..36c259b7f23c 100644 --- a/src/audio/host-legacy.c +++ b/src/audio/host-legacy.c @@ -440,7 +440,7 @@ static int create_local_elems(struct host_data *hd, struct comp_dev *dev, uint32 elem_array = &hd->local.elem_array; /* config buffer will be used as proxy */ - err = dma_sg_alloc(&hd->config.elem_array, SOF_MEM_FLAG_USER, + err = dma_sg_alloc(NULL, &hd->config.elem_array, SOF_MEM_FLAG_USER, dir, 1, 0, 0, 0); if (err < 0) { comp_err(dev, "dma_sg_alloc() failed"); @@ -450,7 +450,7 @@ static int create_local_elems(struct host_data *hd, struct comp_dev *dev, uint32 elem_array = &hd->config.elem_array; } - err = dma_sg_alloc(elem_array, SOF_MEM_FLAG_USER, dir, buffer_count, + err = dma_sg_alloc(NULL, elem_array, SOF_MEM_FLAG_USER, dir, buffer_count, buffer_bytes, (uintptr_t)(audio_stream_get_addr(&hd->dma_buffer->stream)), 0); if (err < 0) { @@ -602,7 +602,7 @@ void host_common_free(struct host_data *hd) dma_put(hd->dma); ipc_msg_free(hd->msg); - dma_sg_free(&hd->config.elem_array); + dma_sg_free(NULL, &hd->config.elem_array); } static void host_free(struct comp_dev *dev) @@ -905,9 +905,9 @@ void host_common_reset(struct host_data *hd, uint16_t state) } /* free all DMA elements */ - dma_sg_free(&hd->host.elem_array); - dma_sg_free(&hd->local.elem_array); - dma_sg_free(&hd->config.elem_array); + dma_sg_free(NULL, &hd->host.elem_array); + dma_sg_free(NULL, &hd->local.elem_array); + dma_sg_free(NULL, &hd->config.elem_array); /* It's safe that cleaning out `hd->config` after `dma_sg_free` for config.elem_array */ memset(&hd->config, 0, sizeof(hd->config)); diff --git a/src/audio/host-zephyr.c b/src/audio/host-zephyr.c index 30cb465897ba..eebe7c4e3416 100644 --- a/src/audio/host-zephyr.c +++ b/src/audio/host-zephyr.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -84,7 +85,7 @@ static int host_dma_set_config_and_copy(struct host_data *hd, struct comp_dev *d local_elem->size = bytes; /* reconfigure transfer */ - ret = sof_dma_config(hd->chan->dma, hd->chan->index, &hd->z_config); + ret = sof_dma_config(hd->dma, hd->chan_index, &hd->z_config); if (ret < 0) { comp_err(dev, "dma_config() failed, ret = %d", ret); @@ -93,7 +94,7 @@ static int host_dma_set_config_and_copy(struct host_data *hd, struct comp_dev *d cb(dev, bytes); - ret = sof_dma_reload(hd->chan->dma, hd->chan->index, bytes); + ret = sof_dma_reload(hd->dma, hd->chan_index, bytes); if (ret < 0) { comp_err(dev, "dma_copy() failed, ret = %d", ret); @@ -223,7 +224,7 @@ static int host_copy_one_shot(struct host_data *hd, struct comp_dev *dev, copy_c hd->z_config.head_block->block_size = local_elem->size; /* reconfigure transfer */ - ret = sof_dma_config(hd->chan->dma, hd->chan->index, &hd->z_config); + ret = sof_dma_config(hd->dma, hd->chan_index, &hd->z_config); if (ret < 0) { comp_err(dev, "dma_config() failed, ret = %u", ret); return ret; @@ -231,7 +232,7 @@ static int host_copy_one_shot(struct host_data *hd, struct comp_dev *dev, copy_c cb(dev, copy_bytes); - ret = sof_dma_reload(hd->chan->dma, hd->chan->index, copy_bytes); + ret = sof_dma_reload(hd->dma, hd->chan_index, copy_bytes); if (ret < 0) comp_err(dev, "dma_copy() failed, ret = %u", ret); @@ -365,7 +366,7 @@ static void host_dma_cb(struct comp_dev *dev, size_t bytes) /* get status from dma and check for xrun */ static int host_get_status(struct comp_dev *dev, struct host_data *hd, struct dma_status *stat) { - int ret = sof_dma_get_status(hd->chan->dma, hd->chan->index, stat); + int ret = sof_dma_get_status(hd->dma, hd->chan_index, stat); #if CONFIG_XRUN_NOTIFICATIONS_ENABLE if (ret == -EPIPE && !hd->xrun_notification_sent) { struct ipc_msg *notify = ipc_notification_pool_get(IPC4_RESOURCE_EVENT_SIZE); @@ -563,7 +564,7 @@ static int host_copy_normal(struct host_data *hd, struct comp_dev *dev, copy_cal if (!copy_bytes) { if (hd->partial_size != 0) { if (stream_sync(hd, dev)) { - ret = sof_dma_reload(hd->chan->dma, hd->chan->index, + ret = sof_dma_reload(hd->dma, hd->chan_index, hd->partial_size); if (ret < 0) comp_err(dev, "dma_reload() failed, ret = %u", ret); @@ -590,7 +591,7 @@ static int host_copy_normal(struct host_data *hd, struct comp_dev *dev, copy_cal hd->dma_buffer_size - hd->partial_size <= (2 + threshold) * hd->period_bytes) { if (stream_sync(hd, dev)) { - ret = sof_dma_reload(hd->chan->dma, hd->chan->index, + ret = sof_dma_reload(hd->dma, hd->chan_index, hd->partial_size); if (ret < 0) comp_err(dev, "dma_reload() failed, ret = %u", ret); @@ -617,7 +618,7 @@ static int create_local_elems(struct host_data *hd, struct comp_dev *dev, elem_array = &hd->local.elem_array; /* config buffer will be used as proxy */ - err = dma_sg_alloc(&hd->config.elem_array, SOF_MEM_FLAG_USER, + err = dma_sg_alloc(hd->heap, &hd->config.elem_array, SOF_MEM_FLAG_USER, dir, 1, 0, 0, 0); if (err < 0) { comp_err(dev, "dma_sg_alloc() failed"); @@ -627,7 +628,7 @@ static int create_local_elems(struct host_data *hd, struct comp_dev *dev, elem_array = &hd->config.elem_array; } - err = dma_sg_alloc(elem_array, SOF_MEM_FLAG_USER, dir, buffer_count, + err = dma_sg_alloc(hd->heap, elem_array, SOF_MEM_FLAG_USER, dir, buffer_count, buffer_bytes, (uintptr_t)audio_stream_get_addr(&hd->dma_buffer->stream), 0); if (err < 0) { @@ -658,7 +659,7 @@ int host_common_trigger(struct host_data *hd, struct comp_dev *dev, int cmd) if (cmd != COMP_TRIGGER_START && hd->copy_type == COMP_COPY_ONE_SHOT) return ret; - if (!hd->chan) { + if (hd->chan_index == -1) { comp_err(dev, "no dma channel configured"); return -EINVAL; } @@ -666,14 +667,14 @@ int host_common_trigger(struct host_data *hd, struct comp_dev *dev, int cmd) switch (cmd) { case COMP_TRIGGER_START: hd->partial_size = 0; - ret = sof_dma_start(hd->chan->dma, hd->chan->index); + ret = sof_dma_start(hd->dma, hd->chan_index); if (ret < 0) comp_err(dev, "dma_start() failed, ret = %u", ret); break; case COMP_TRIGGER_STOP: case COMP_TRIGGER_XRUN: - ret = sof_dma_stop(hd->chan->dma, hd->chan->index); + ret = sof_dma_stop(hd->dma, hd->chan_index); if (ret < 0) comp_err(dev, "dma stop failed: %d", ret); @@ -733,9 +734,18 @@ __cold int host_common_new(struct host_data *hd, struct comp_dev *dev, sof_dma_put(hd->dma); return -ENOMEM; } - hd->chan = NULL; + hd->chan_index = -1; hd->copy_type = COMP_COPY_NORMAL; +#ifdef CONFIG_SOF_USERSPACE_LL + /* + * copier_host_create() uses mod_zalloc() to allocate + * the 'hd' host data object and does not set hd->heap. + * If LL is run in user-space, assign the 'heap' here. + */ + hd->heap = zephyr_ll_user_heap(); +#endif + return 0; } @@ -746,6 +756,7 @@ __cold static struct comp_dev *host_new(const struct comp_driver *drv, struct comp_dev *dev; struct host_data *hd; const struct ipc_config_host *ipc_host = spec; + struct k_heap *heap = NULL; int ret; assert_can_be_cold(); @@ -757,10 +768,17 @@ __cold static struct comp_dev *host_new(const struct comp_driver *drv, return NULL; dev->ipc_config = *config; - hd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*hd)); +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); +#endif + + hd = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, sizeof(*hd), 0); if (!hd) goto e_data; + memset(hd, 0, sizeof(*hd)); + hd->heap = heap; + hd->nobytes_last_logged = k_uptime_get(); comp_set_drvdata(dev, hd); @@ -773,7 +791,7 @@ __cold static struct comp_dev *host_new(const struct comp_driver *drv, return dev; e_dev: - rfree(hd); + sof_heap_free(heap, hd); e_data: comp_free_device(dev); return NULL; @@ -786,7 +804,7 @@ __cold void host_common_free(struct host_data *hd) sof_dma_put(hd->dma); ipc_msg_free(hd->msg); - dma_sg_free(&hd->config.elem_array); + dma_sg_free(hd->heap, &hd->config.elem_array); } __cold static void host_free(struct comp_dev *dev) @@ -797,7 +815,7 @@ __cold static void host_free(struct comp_dev *dev) comp_dbg(dev, "entry"); host_common_free(hd); - rfree(hd); + sof_heap_free(hd->heap, hd); comp_free_device(dev); } @@ -864,9 +882,14 @@ int host_common_params(struct host_data *hd, struct comp_dev *dev, uint32_t buffer_size_preferred; uint32_t addr_align; uint32_t align; - int i, channel, err; + int i, err; bool is_scheduling_source = dev == dev->pipeline->sched_comp; uint32_t round_up_size; +#ifdef CONFIG_SOF_USERSPACE_LL + struct k_heap *heap = zephyr_ll_user_heap(); +#else + struct k_heap *heap = NULL; +#endif /* host params always installed by pipeline IPC */ hd->host_size = params->buffer.size; @@ -955,7 +978,7 @@ int host_common_params(struct host_data *hd, struct comp_dev *dev, } } else { /* allocate not shared buffer */ - hd->dma_buffer = buffer_alloc_range(NULL, buffer_size_preferred, buffer_size, + hd->dma_buffer = buffer_alloc_range(hd->heap, buffer_size_preferred, buffer_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, BUFFER_USAGE_NOT_SHARED); if (!hd->dma_buffer) { @@ -1000,26 +1023,20 @@ int host_common_params(struct host_data *hd, struct comp_dev *dev, /* get DMA channel from DMAC * note: stream_tag is ignored by dw-dma */ - channel = sof_dma_request_channel(hd->dma, hda_chan); - if (channel < 0) { + hd->chan_index = sof_dma_request_channel(hd->dma, hda_chan); + if (hd->chan_index < 0) { comp_err(dev, "requested channel %d is busy", hda_chan); return -ENODEV; } - hd->chan = &hd->dma->chan[channel]; uint32_t buffer_addr = 0; uint32_t buffer_bytes = 0; uint32_t addr; - hd->chan->direction = config->direction; - hd->chan->desc_count = config->elem_array.count; - hd->chan->is_scheduling_source = config->is_scheduling_source; - hd->chan->period = config->period; - memset(dma_cfg, 0, sizeof(*dma_cfg)); - dma_block_cfg = rzalloc(SOF_MEM_FLAG_USER, - sizeof(*dma_block_cfg)); + dma_block_cfg = sof_heap_alloc(hd->heap, SOF_MEM_FLAG_USER, + sizeof(*dma_block_cfg), 0); if (!dma_block_cfg) { comp_err(dev, "dma_block_config allocation failed"); @@ -1027,6 +1044,8 @@ int host_common_params(struct host_data *hd, struct comp_dev *dev, goto err_release_channel; } + memset(dma_block_cfg, 0, sizeof(*dma_block_cfg)); + dma_cfg->block_count = 1; dma_cfg->source_data_size = config->src_width; dma_cfg->dest_data_size = config->dest_width; @@ -1062,7 +1081,7 @@ int host_common_params(struct host_data *hd, struct comp_dev *dev, break; } - err = sof_dma_config(hd->chan->dma, hd->chan->index, dma_cfg); + err = sof_dma_config(hd->dma, hd->chan_index, dma_cfg); if (err < 0) { comp_err(dev, "dma_config() failed"); goto err_free_block_cfg; @@ -1090,10 +1109,10 @@ int host_common_params(struct host_data *hd, struct comp_dev *dev, err_free_block_cfg: dma_cfg->head_block = NULL; - rfree(dma_block_cfg); + sof_heap_free(hd->heap, dma_block_cfg); err_release_channel: - sof_dma_release_channel(hd->dma, hd->chan->index); - hd->chan = NULL; + sof_dma_release_channel(hd->dma, hd->chan_index); + hd->chan_index = -1; return err; } @@ -1151,16 +1170,16 @@ static int host_position(struct comp_dev *dev, void host_common_reset(struct host_data *hd, uint16_t state) { - if (hd->chan) { - sof_dma_stop(hd->chan->dma, hd->chan->index); - sof_dma_release_channel(hd->dma, hd->chan->index); - hd->chan = NULL; + if (hd->chan_index != -1) { + sof_dma_stop(hd->dma, hd->chan_index); + sof_dma_release_channel(hd->dma, hd->chan_index); + hd->chan_index = -1; } /* free all DMA elements */ - dma_sg_free(&hd->host.elem_array); - dma_sg_free(&hd->local.elem_array); - dma_sg_free(&hd->config.elem_array); + dma_sg_free(hd->heap, &hd->host.elem_array); + dma_sg_free(hd->heap, &hd->local.elem_array); + dma_sg_free(hd->heap, &hd->config.elem_array); /* free DMA buffer */ if (hd->dma_buffer) { @@ -1170,7 +1189,7 @@ void host_common_reset(struct host_data *hd, uint16_t state) /* free DMA block configuration */ if (hd->z_config.head_block) - rfree(hd->z_config.head_block); + sof_heap_free(hd->heap, hd->z_config.head_block); /* reset buffer pointers */ hd->local_pos = 0; diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 048eda0d47e0..fde9894317f1 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -111,7 +111,12 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv } mod_heap = &mod_heap_user->heap; } else { +#ifdef CONFIG_SOF_USERSPACE_LL + mod_heap = zephyr_ll_user_heap(); + comp_cl_dbg(drv, "using ll user heap for module"); +#else mod_heap = drv->user_heap; +#endif mod_heap_user = NULL; heap_size = 0; } diff --git a/src/audio/pcm_converter/pcm_converter_generic.c b/src/audio/pcm_converter/pcm_converter_generic.c index fe96e6d1f124..4938be438dbc 100644 --- a/src/audio/pcm_converter/pcm_converter_generic.c +++ b/src/audio/pcm_converter/pcm_converter_generic.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -669,7 +670,7 @@ static int pcm_convert_f_to_s32(const struct audio_stream *source, } #endif /* CONFIG_PCM_CONVERTER_FORMAT_FLOAT && CONFIG_PCM_CONVERTER_FORMAT_S32LE */ -const struct pcm_func_map pcm_func_map[] = { +APP_TASK_DATA const struct pcm_func_map pcm_func_map[] = { #if CONFIG_PCM_CONVERTER_FORMAT_U8 { SOF_IPC_FRAME_U8, SOF_IPC_FRAME_U8, just_copy }, #endif /* CONFIG_PCM_CONVERTER_FORMAT_U8 */ @@ -732,7 +733,7 @@ const struct pcm_func_map pcm_func_map[] = { #endif /* CONFIG_PCM_CONVERTER_FORMAT_FLOAT && CONFIG_PCM_CONVERTER_FORMAT_S32LE */ }; -const size_t pcm_func_count = ARRAY_SIZE(pcm_func_map); +APP_TASK_DATA const size_t pcm_func_count = ARRAY_SIZE(pcm_func_map); #if CONFIG_PCM_CONVERTER_FORMAT_S16_C16_AND_S16_C32 static int pcm_convert_s16_c16_to_s16_c32(const struct audio_stream *source, @@ -1020,7 +1021,7 @@ static int pcm_convert_s24_c32_to_s24_c24_link_gtw(const struct audio_stream *so #endif -const struct pcm_func_vc_map pcm_func_vc_map[] = { +APP_TASK_DATA const struct pcm_func_vc_map pcm_func_vc_map[] = { #if CONFIG_PCM_CONVERTER_FORMAT_S16_C16_AND_S16_C32 { SOF_IPC_FRAME_S16_LE, SOF_IPC_FRAME_S16_LE, SOF_IPC_FRAME_S32_LE, SOF_IPC_FRAME_S16_LE, pcm_convert_s16_c16_to_s16_c32 }, @@ -1101,6 +1102,6 @@ const struct pcm_func_vc_map pcm_func_vc_map[] = { #endif }; -const size_t pcm_func_vc_count = ARRAY_SIZE(pcm_func_vc_map); +APP_TASK_DATA const size_t pcm_func_vc_count = ARRAY_SIZE(pcm_func_vc_map); #endif diff --git a/src/audio/pcm_converter/pcm_converter_hifi3.c b/src/audio/pcm_converter/pcm_converter_hifi3.c index 2b6ca607415d..7c75e326b1c0 100644 --- a/src/audio/pcm_converter/pcm_converter_hifi3.c +++ b/src/audio/pcm_converter/pcm_converter_hifi3.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -764,7 +765,7 @@ static int pcm_convert_f_to_s32(const struct audio_stream *source, #endif /* CONFIG_PCM_CONVERTER_FORMAT_FLOAT && CONFIG_PCM_CONVERTER_FORMAT_32LE */ #endif /* XCHAL_HAVE_FP */ -const struct pcm_func_map pcm_func_map[] = { +APP_TASK_DATA const struct pcm_func_map pcm_func_map[] = { #if CONFIG_PCM_CONVERTER_FORMAT_S16LE { SOF_IPC_FRAME_S16_LE, SOF_IPC_FRAME_S16_LE, just_copy }, #endif /* CONFIG_PCM_CONVERTER_FORMAT_S16LE */ @@ -807,7 +808,7 @@ const struct pcm_func_map pcm_func_map[] = { #endif /* CONFIG_PCM_CONVERTER_FORMAT_FLOAT && CONFIG_PCM_CONVERTER_FORMAT_S32LE */ #endif /* XCHAL_HAVE_FP */ }; -const size_t pcm_func_count = ARRAY_SIZE(pcm_func_map); +APP_TASK_DATA const size_t pcm_func_count = ARRAY_SIZE(pcm_func_map); #if CONFIG_PCM_CONVERTER_FORMAT_S16_C16_AND_S16_C32 static int pcm_convert_s16_c16_to_s16_c32(const struct audio_stream *source, @@ -1206,7 +1207,7 @@ static int pcm_convert_s24_c32_to_s24_c24(const struct audio_stream *source, */ #endif -const struct pcm_func_vc_map pcm_func_vc_map[] = { +APP_TASK_DATA const struct pcm_func_vc_map pcm_func_vc_map[] = { #if CONFIG_PCM_CONVERTER_FORMAT_S16_C16_AND_S16_C32 { SOF_IPC_FRAME_S16_LE, SOF_IPC_FRAME_S16_LE, SOF_IPC_FRAME_S32_LE, SOF_IPC_FRAME_S16_LE, pcm_convert_s16_c16_to_s16_c32 }, @@ -1283,6 +1284,6 @@ const struct pcm_func_vc_map pcm_func_vc_map[] = { #endif }; -const size_t pcm_func_vc_count = ARRAY_SIZE(pcm_func_vc_map); +APP_TASK_DATA const size_t pcm_func_vc_count = ARRAY_SIZE(pcm_func_vc_map); #endif diff --git a/src/audio/pcm_converter/pcm_remap.c b/src/audio/pcm_converter/pcm_remap.c index 9204b21ee8ab..4ae300e195c9 100644 --- a/src/audio/pcm_converter/pcm_remap.c +++ b/src/audio/pcm_converter/pcm_remap.c @@ -5,6 +5,7 @@ #include #include +#include static void mute_channel_c16(struct audio_stream *stream, int channel, int frames) { @@ -423,7 +424,7 @@ static int remap_c16_to_c32_no_shift(const struct audio_stream *source, uint32_t /* Unfortunately, all these nice "if"s were commented out to suppress * CI "defined but not used" warnings. */ -const struct pcm_func_map pcm_remap_func_map[] = { +APP_TASK_DATA const struct pcm_func_map pcm_remap_func_map[] = { /* #if CONFIG_PCM_CONVERTER_FORMAT_S16LE */ { SOF_IPC_FRAME_S16_LE, SOF_IPC_FRAME_S16_LE, remap_c16}, /* #endif */ @@ -474,4 +475,4 @@ const struct pcm_func_map pcm_remap_func_map[] = { /* #endif */ }; -const size_t pcm_remap_func_count = ARRAY_SIZE(pcm_remap_func_map); +APP_TASK_DATA const size_t pcm_remap_func_count = ARRAY_SIZE(pcm_remap_func_map); diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 89bb3574289b..39125b6346e3 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -178,24 +178,34 @@ static void buffer_set_comp(struct comp_buffer *buffer, struct comp_dev *comp, comp_buffer_set_sink_component(buffer, comp); } +#ifdef CONFIG_SOF_USERSPACE_LL +#define PPL_LOCK_DECLARE +#define PPL_LOCK(x) k_mutex_lock(comp->list_mutex, K_FOREVER) +#define PPL_UNLOCK(x) k_mutex_unlock(comp->list_mutex) +#else +#define PPL_LOCK_DECLARE uint32_t flags +#define PPL_LOCK(x) irq_local_disable(flags) +#define PPL_UNLOCK(x) irq_local_enable(flags) +#endif + int pipeline_connect(struct comp_dev *comp, struct comp_buffer *buffer, int dir) { struct list_item *comp_list; - uint32_t flags; + PPL_LOCK_DECLARE; if (dir == PPL_CONN_DIR_COMP_TO_BUFFER) comp_info(comp, "connect buffer %d as sink", buf_get_id(buffer)); else comp_info(comp, "connect buffer %d as source", buf_get_id(buffer)); - irq_local_disable(flags); + PPL_LOCK(); comp_list = comp_buffer_list(comp, dir); buffer_attach(buffer, comp_list, dir); buffer_set_comp(buffer, comp, dir); - irq_local_enable(flags); + PPL_UNLOCK(); return 0; } @@ -203,20 +213,20 @@ int pipeline_connect(struct comp_dev *comp, struct comp_buffer *buffer, void pipeline_disconnect(struct comp_dev *comp, struct comp_buffer *buffer, int dir) { struct list_item *comp_list; - uint32_t flags; + PPL_LOCK_DECLARE; if (dir == PPL_CONN_DIR_COMP_TO_BUFFER) comp_dbg(comp, "disconnect buffer %d as sink", buf_get_id(buffer)); else comp_dbg(comp, "disconnect buffer %d as source", buf_get_id(buffer)); - irq_local_disable(flags); + PPL_LOCK(); comp_list = comp_buffer_list(comp, dir); buffer_detach(buffer, comp_list, dir); buffer_set_comp(buffer, NULL, dir); - irq_local_enable(flags); + PPL_UNLOCK(); } /* pipelines must be inactive */ diff --git a/src/include/sof/audio/buffer.h b/src/include/sof/audio/buffer.h index 91c09ef2e510..5d5e64e8043d 100644 --- a/src/include/sof/audio/buffer.h +++ b/src/include/sof/audio/buffer.h @@ -33,6 +33,7 @@ #include struct comp_dev; +struct k_heap; /** \name Trace macros * @{ @@ -148,6 +149,8 @@ struct comp_buffer { /* list of buffers, to be used i.e. in raw data processing mode*/ struct list_item buffers_list; + + struct k_heap *heap; }; /* diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index b6695b9cd312..a69be34b15a1 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -679,6 +680,10 @@ struct comp_dev { struct list_item bsource_list; /**< list of source buffers */ struct list_item bsink_list; /**< list of sink buffers */ +#ifdef CONFIG_SOF_USERSPACE_LL + struct k_mutex *list_mutex; /**< protect lists of source/sinks */ +#endif + /* performance data*/ struct comp_perf_data perf_data; /* Input Buffer Size for pin 0, add array for other pins if needed */ @@ -863,6 +868,10 @@ static inline void comp_init(const struct comp_driver *drv, dev->state = COMP_STATE_INIT; list_init(&dev->bsink_list); list_init(&dev->bsource_list); +#ifdef CONFIG_SOF_USERSPACE_LL + dev->list_mutex = k_object_alloc(K_OBJ_MUTEX); + k_mutex_init(dev->list_mutex); +#endif memcpy_s(&dev->tctx, sizeof(dev->tctx), trace_comp_drv_get_tr_ctx(dev->drv), sizeof(struct tr_ctx)); } @@ -1228,6 +1237,16 @@ void comp_init_performance_data(struct comp_dev *dev); */ bool comp_update_performance_data(struct comp_dev *dev, uint32_t cycles_used); +/** + * Grant access to component to a thread. + * + * Must be called from kernel context. + * + * @param dev Component to update. + * @param th thread to give access to + */ +void comp_grant_access_to_thread(const struct comp_dev *dev, struct k_thread *th); + static inline int user_get_buffer_memory_region(const struct comp_driver *drv) { #if CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index d2bbf87a7764..d0afe4be3ad8 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -53,6 +53,10 @@ static inline void comp_free(struct comp_dev *dev) * be freed after this. */ drv->ops.free(dev); + +#ifdef CONFIG_SOF_USERSPACE_LL + k_object_free(dev->list_mutex); +#endif } /** diff --git a/src/include/sof/lib/dai-zephyr.h b/src/include/sof/lib/dai-zephyr.h index 8ff739fa42d5..76e9de96c325 100644 --- a/src/include/sof/lib/dai-zephyr.h +++ b/src/include/sof/lib/dai-zephyr.h @@ -52,7 +52,10 @@ struct dai { uint32_t dma_dev; const struct device *dev; const struct dai_data *dd; - struct k_spinlock lock; /* protect properties */ + struct k_mutex *lock; /* protect properties */ +#ifndef CONFIG_SOF_USERSPACE_LL + struct k_mutex lock_obj; +#endif }; union hdalink_cfg { @@ -117,7 +120,7 @@ typedef int (*channel_copy_func)(const struct audio_stream *src, unsigned int sr */ struct dai_data { /* local DMA config */ - struct dma_chan_data *chan; + int chan_index; uint32_t stream_id; struct dma_sg_config config; struct dma_config *z_config; @@ -168,6 +171,7 @@ struct dai_data { #endif /* Copier gain params */ struct copier_gain_params *gain_data; + struct k_heap *heap; }; /* these 3 are here to satisfy clk.c and ssp.h interconnection, will be removed leter */ diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 10f241784625..a62223dc97a2 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -294,11 +294,6 @@ __cold int ipc_init(struct sof *sof) tr_dbg(&ipc_tr, "entry"); -#if CONFIG_SOF_BOOT_TEST_STANDALONE - LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); - return 0; -#endif - /* init ipc data */ sof->ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*sof->ipc)); if (!sof->ipc) { @@ -329,6 +324,11 @@ __cold int ipc_init(struct sof *sof) io_perf_monitor_init_data(&sof->ipc->io_perf_out_msg_count, &init_data); #endif +#if CONFIG_SOF_BOOT_TEST_STANDALONE + LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); + return 0; +#endif + #ifdef __ZEPHYR__ struct k_thread *thread = &sof->ipc->ipc_send_wq.thread; diff --git a/src/ipc/ipc3/handler.c b/src/ipc/ipc3/handler.c index 80ddb2225e43..5bd5293c61b7 100644 --- a/src/ipc/ipc3/handler.c +++ b/src/ipc/ipc3/handler.c @@ -878,7 +878,7 @@ static int ipc_dma_trace_config(uint32_t header) error: #if CONFIG_HOST_PTABLE - dma_sg_free(&elem_array); + dma_sg_free(NULL, &elem_array); processing_error: #endif diff --git a/src/ipc/ipc3/host-page-table.c b/src/ipc/ipc3/host-page-table.c index 7a3da31caa82..997ee6683352 100644 --- a/src/ipc/ipc3/host-page-table.c +++ b/src/ipc/ipc3/host-page-table.c @@ -239,6 +239,6 @@ int ipc_process_host_buffer(struct ipc *ipc, return 0; error: - dma_sg_free(elem_array); + dma_sg_free(NULL, elem_array); return err; } diff --git a/src/ipc/ipc4/dai.c b/src/ipc/ipc4/dai.c index dd55236f0917..1b26a52e467f 100644 --- a/src/ipc/ipc4/dai.c +++ b/src/ipc/ipc4/dai.c @@ -202,7 +202,7 @@ void dai_dma_release(struct dai_data *dd, struct comp_dev *dev) } /* put the allocated DMA channel first */ - if (dd->chan) { + if (dd->chan_index != -1) { struct ipc4_llp_reading_slot slot; if (dd->slot_info.node_id) { @@ -224,15 +224,16 @@ void dai_dma_release(struct dai_data *dd, struct comp_dev *dev) */ #if CONFIG_ZEPHYR_NATIVE_DRIVERS /* if reset is after pause dma has already been stopped */ - dma_stop(dd->chan->dma->z_dev, dd->chan->index); + dma_stop(dd->dma->z_dev, dd->chan_index); - dma_release_channel(dd->chan->dma->z_dev, dd->chan->index); + dma_release_channel(dd->dma->z_dev, dd->chan_index); #else + /* TODO: to remove this, no longer works! */ dma_stop_legacy(dd->chan); dma_channel_put_legacy(dd->chan); -#endif - dd->chan->dev_data = NULL; dd->chan = NULL; +#endif + } } @@ -351,9 +352,9 @@ __cold int dai_config(struct dai_data *dd, struct comp_dev *dev, return 0; } - if (dd->chan) { + if (dd->chan_index != -1) { comp_info(dev, "Configured. dma channel index %d, ignore...", - dd->chan->index); + dd->chan_index); return 0; } @@ -374,15 +375,17 @@ __cold int dai_config(struct dai_data *dd, struct comp_dev *dev, /* allocated dai_config if not yet */ if (!dd->dai_spec_config) { size = sizeof(*copier_cfg); - dd->dai_spec_config = rzalloc(SOF_MEM_FLAG_USER, size); + dd->dai_spec_config = sof_heap_alloc(dd->heap, SOF_MEM_FLAG_USER, size, 0); if (!dd->dai_spec_config) { comp_err(dev, "No memory for size %d", size); return -ENOMEM; } + memset(dd->dai_spec_config, 0, size); + ret = memcpy_s(dd->dai_spec_config, size, copier_cfg, size); if (ret < 0) { - rfree(dd->dai_spec_config); + sof_heap_free(dd->heap, dd->dai_spec_config); dd->dai_spec_config = NULL; return -EINVAL; } @@ -412,7 +415,7 @@ int dai_common_position(struct dai_data *dd, struct comp_dev *dev, platform_dai_wallclock(dev, &dd->wallclock); posn->wallclock = dd->wallclock; - ret = dma_get_status(dd->dma->z_dev, dd->chan->index, &status); + ret = dma_get_status(dd->dma->z_dev, dd->chan_index, &status); if (ret < 0) return ret; @@ -437,7 +440,7 @@ void dai_dma_position_update(struct dai_data *dd, struct comp_dev *dev) if (!dd->slot_info.node_id) return; - ret = dma_get_status(dd->dma->z_dev, dd->chan->index, &status); + ret = dma_get_status(dd->dma->z_dev, dd->chan_index, &status); if (ret < 0) return; diff --git a/src/lib/dai.c b/src/lib/dai.c index 8e45f8a4bb97..3f77fe0a4b20 100644 --- a/src/lib/dai.c +++ b/src/lib/dai.c @@ -11,6 +11,7 @@ #include #include #include +#include /* for zephyr_ll_user_heap() */ #include #include #include @@ -298,6 +299,11 @@ struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) { const struct device *dev; struct dai *d; + struct k_heap *heap = NULL; + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); +#endif dev = dai_get_device(type, index); if (!dev) { @@ -306,10 +312,12 @@ struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) return NULL; } - d = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(struct dai)); + d = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(struct dai), 0); if (!d) return NULL; + memset(d, 0, sizeof(struct dai)); + d->index = index; d->type = type; d->dev = dev; @@ -319,7 +327,7 @@ struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) if (dai_probe(d->dev)) { tr_err(&dai_tr, "dai_get: failed to probe dai with index %d type %d", index, type); - rfree(d); + sof_heap_free(heap, d); return NULL; } @@ -330,6 +338,11 @@ struct dai *dai_get(uint32_t type, uint32_t index, uint32_t flags) void dai_put(struct dai *dai) { int ret; + struct k_heap *heap = NULL; + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); +#endif ret = dai_remove(dai->dev); if (ret < 0) { @@ -337,7 +350,7 @@ void dai_put(struct dai *dai) dai->index, ret); } - rfree(dai); + sof_heap_free(heap, dai); } #else static inline const struct dai_type_info *dai_find_type(uint32_t type) diff --git a/src/lib/dma.c b/src/lib/dma.c index 608ee091ac25..70ac5c975058 100644 --- a/src/lib/dma.c +++ b/src/lib/dma.c @@ -285,7 +285,8 @@ void dma_put(struct dma *dma) } #endif -int dma_sg_alloc(struct dma_sg_elem_array *elem_array, +int dma_sg_alloc(struct k_heap *heap, + struct dma_sg_elem_array *elem_array, uint32_t flags, uint32_t direction, uint32_t buffer_count, uint32_t buffer_bytes, @@ -293,11 +294,13 @@ int dma_sg_alloc(struct dma_sg_elem_array *elem_array, { int i; - elem_array->elems = rzalloc(SOF_MEM_FLAG_USER, - sizeof(struct dma_sg_elem) * buffer_count); + elem_array->elems = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, + sizeof(struct dma_sg_elem) * buffer_count, 0); if (!elem_array->elems) return -ENOMEM; + memset(elem_array->elems, 0, sizeof(struct dma_sg_elem) * buffer_count); + for (i = 0; i < buffer_count; i++) { elem_array->elems[i].size = buffer_bytes; // TODO: may count offsets once @@ -319,9 +322,9 @@ int dma_sg_alloc(struct dma_sg_elem_array *elem_array, return 0; } -void dma_sg_free(struct dma_sg_elem_array *elem_array) +void dma_sg_free(struct k_heap *heap, struct dma_sg_elem_array *elem_array) { - rfree(elem_array->elems); + sof_heap_free(heap, elem_array->elems); dma_sg_init(elem_array); } diff --git a/src/probe/probe.c b/src/probe/probe.c index f15ee84f7daf..088a2eb1773b 100644 --- a/src/probe/probe.c +++ b/src/probe/probe.c @@ -176,7 +176,7 @@ static int probe_dma_init(struct probe_dma_ext *dma, uint32_t direction) dma->config.dest_width = sizeof(uint32_t); dma->config.cyclic = 0; - err = dma_sg_alloc(&dma->config.elem_array, SOF_MEM_FLAG_USER, + err = dma_sg_alloc(NULL, &dma->config.elem_array, SOF_MEM_FLAG_USER, dma->config.direction, elem_num, elem_size, elem_addr, 0); if (err < 0) return err; @@ -255,7 +255,7 @@ static int probe_dma_init(struct probe_dma_ext *dma, uint32_t direction) static int probe_dma_deinit(struct probe_dma_ext *dma) { int err = 0; - dma_sg_free(&dma->config.elem_array); + dma_sg_free(NULL, &dma->config.elem_array); #if CONFIG_ZEPHYR_NATIVE_DRIVERS err = dma_stop(dma->dc.dmac->z_dev, dma->dc.chan->index); #else diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 575a82d91dda..e2d02f12926b 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -40,7 +40,10 @@ struct zephyr_ll { struct zephyr_ll_pdata { bool run; bool freeing; - struct k_sem sem; +#ifndef CONFIG_SOF_USERSPACE_LL + struct k_sem sem_obj; +#endif + struct k_sem *sem; }; static void zephyr_ll_lock(struct zephyr_ll *sch, uint32_t *flags) @@ -87,7 +90,7 @@ static void zephyr_ll_task_done(struct zephyr_ll *sch, * zephyr_ll_task_free() is trying to free this task. Complete * it and signal the semaphore to let the function proceed */ - k_sem_give(&pdata->sem); + k_sem_give(pdata->sem); tr_info(&ll_tr, "task complete %p %pU", task, task->uid); tr_info(&ll_tr, "num_tasks %d total_num_tasks %ld", @@ -454,7 +457,11 @@ static int zephyr_ll_task_free(void *data, struct task *task) if (must_wait) /* Wait for up to 100 periods */ - k_sem_take(&pdata->sem, K_USEC(LL_TIMER_PERIOD_US * 100)); + k_sem_take(pdata->sem, K_USEC(LL_TIMER_PERIOD_US * 100)); + +#ifdef CONFIG_SOF_USERSPACE_LL + k_object_free(pdata->sem); +#endif /* Protect against racing with schedule_task() */ zephyr_ll_lock(sch, &flags); @@ -560,7 +567,12 @@ int zephyr_ll_task_init(struct task *task, memset(pdata, 0, sizeof(*pdata)); - k_sem_init(&pdata->sem, 0, 1); +#ifdef CONFIG_SOF_USERSPACE_LL + pdata->sem = k_object_alloc(K_OBJ_SEM); +#else + pdata->sem = &pdata->sem_obj; +#endif + k_sem_init(pdata->sem, 0, 1); task->priv_data = pdata; diff --git a/src/schedule/zephyr_ll_user.c b/src/schedule/zephyr_ll_user.c index aa33807b4aa3..e64d65ab8ae5 100644 --- a/src/schedule/zephyr_ll_user.c +++ b/src/schedule/zephyr_ll_user.c @@ -17,15 +17,17 @@ LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); * * This structure encapsulates the memory management resources required for the * low-latency (LL) scheduler in userspace mode. It provides memory isolation - * and heap management for LL scheduler threads. + * and heap management for LL scheduler threads. Only kernel accessible. */ struct zephyr_ll_mem_resources { struct k_mem_domain mem_domain; /**< Memory domain for LL thread isolation */ - struct k_heap *heap; /**< Heap allocator for LL scheduler memory */ }; static struct zephyr_ll_mem_resources ll_mem_resources; +/* Heap allocator for LL scheduler memory (user accessible pointer) */ +APP_TASK_DATA static struct k_heap *zephyr_ll_heap; + static struct k_heap *zephyr_ll_heap_init(void) { struct k_heap *heap = module_driver_heap_init(); @@ -68,7 +70,7 @@ void zephyr_ll_user_resources_init(void) { k_mem_domain_init(&ll_mem_resources.mem_domain, 0, NULL); - ll_mem_resources.heap = zephyr_ll_heap_init(); + zephyr_ll_heap = zephyr_ll_heap_init(); /* attach common partition to LL domain */ user_memory_attach_common_partition(zephyr_ll_mem_domain()); @@ -76,7 +78,7 @@ void zephyr_ll_user_resources_init(void) struct k_heap *zephyr_ll_user_heap(void) { - return ll_mem_resources.heap; + return zephyr_ll_heap; } struct k_mem_domain *zephyr_ll_mem_domain(void) diff --git a/src/trace/dma-trace.c b/src/trace/dma-trace.c index 0523a362e397..554204ac5c70 100644 --- a/src/trace/dma-trace.c +++ b/src/trace/dma-trace.c @@ -407,7 +407,7 @@ void dma_trace_disable(struct dma_trace_data *d) #if (CONFIG_HOST_PTABLE) /* Free up the host SG if it is set */ if (d->host_size) { - dma_sg_free(&d->config.elem_array); + dma_sg_free(NULL, &d->config.elem_array); d->host_size = 0; } #endif diff --git a/test/cmocka/src/common_mocks.c b/test/cmocka/src/common_mocks.c index 60b6215c4cd5..16fee8f2ff48 100644 --- a/test/cmocka/src/common_mocks.c +++ b/test/cmocka/src/common_mocks.c @@ -59,16 +59,6 @@ void WEAK *rzalloc(uint32_t flags, return calloc(bytes, 1); } -void WEAK *rbrealloc_align(void *ptr, uint32_t flags, - size_t bytes, size_t old_bytes, uint32_t alignment) -{ - (void)flags; - (void)old_bytes; - (void)alignment; - - return realloc(ptr, bytes); -} - void WEAK *rmalloc_align(uint32_t flags, size_t bytes, uint32_t alignment) { (void)flags; diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 61ed49a3d15f..642bf5dc60c1 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -536,6 +536,8 @@ zephyr_library_sources_ifdef(CONFIG_SHELL zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/audio/module_adapter/module/generic.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/fast-get.h) +zephyr_syscall_header(include/rtos/alloc.h) +zephyr_library_sources(syscall/alloc.c) zephyr_library_link_libraries(SOF) target_link_libraries(SOF INTERFACE zephyr_interface) diff --git a/zephyr/Kconfig b/zephyr/Kconfig index a642ee52471c..fe7fe8b234c7 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -29,6 +29,13 @@ config SOF_USERSPACE_INTERFACE_DMA help Allow user-space threads to use the SOF DMA interface. +config SOF_USERSPACE_INTERFACE_ALLOC + bool "Enable SOF heap alloc interface to userspace threads" + depends on USERSPACE + help + Allow user-space threads to use sof_heap_alloc/sof_heap_free + as Zephyr system calls. + config SOF_USERSPACE_LL bool "Run Low-Latency pipelines in userspace threads" depends on USERSPACE diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index 116789ea6ead..6a7fd17aa474 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -88,29 +88,6 @@ static inline void *rballoc(uint32_t flags, size_t bytes) return rballoc_align(flags, bytes, PLATFORM_DCACHE_ALIGN); } -/** - * Changes size of the memory block allocated. - * @param ptr Address of the block to resize. - * @param flags Flags, see SOF_MEM_FLAG_... - * @param bytes New size in bytes. - * @param old_bytes Old size in bytes. - * @param alignment Alignment in bytes. - * @return Pointer to the resized memory of NULL if failed. - */ -void *rbrealloc_align(void *ptr, uint32_t flags, size_t bytes, - size_t old_bytes, uint32_t alignment); - -/** - * Similar to rballoc_align(), returns resized buffer aligned to - * PLATFORM_DCACHE_ALIGN. - */ -static inline void *rbrealloc(void *ptr, uint32_t flags, - size_t bytes, size_t old_bytes) -{ - return rbrealloc_align(ptr, flags, bytes, old_bytes, - PLATFORM_DCACHE_ALIGN); -} - /** * Frees the memory block. * @param ptr Pointer to the memory block. @@ -122,9 +99,16 @@ void rfree(void *ptr); */ void l3_heap_save(void); -void *sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, - size_t alignment); -void sof_heap_free(struct k_heap *heap, void *addr); +__syscall void *sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, + size_t alignment); + +void *z_impl_sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, + size_t alignment); + +__syscall void sof_heap_free(struct k_heap *heap, void *addr); + +void z_impl_sof_heap_free(struct k_heap *heap, void *addr); + struct k_heap *sof_sys_heap_get(void); /* TODO: remove - debug only - only needed for linking */ @@ -149,4 +133,6 @@ size_t get_shared_buffer_heap_size(void); #endif +#include + #endif /* __ZEPHYR_RTOS_ALLOC_H__ */ diff --git a/zephyr/include/sof/lib/dma.h b/zephyr/include/sof/lib/dma.h index b13f3c25221b..e85f4b4d2e80 100644 --- a/zephyr/include/sof/lib/dma.h +++ b/zephyr/include/sof/lib/dma.h @@ -34,6 +34,7 @@ struct comp_buffer; struct comp_dev; +struct k_heap; /** \addtogroup sof_dma_drivers DMA Drivers * SOF DMA Drivers API specification (deprecated interface, to be @@ -291,13 +292,14 @@ static inline void dma_sg_init(struct dma_sg_elem_array *ea) ea->elems = NULL; } -int dma_sg_alloc(struct dma_sg_elem_array *ea, +int dma_sg_alloc(struct k_heap *heap, + struct dma_sg_elem_array *ea, uint32_t flags, uint32_t direction, uint32_t buffer_count, uint32_t buffer_bytes, uintptr_t dma_buffer_addr, uintptr_t external_addr); -void dma_sg_free(struct dma_sg_elem_array *ea); +void dma_sg_free(struct k_heap *heap, struct dma_sg_elem_array *ea); /** * \brief Get the total size of SG buffer diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index f95f66fbdb7f..a0dfb465f33a 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -619,8 +619,8 @@ EXPORT_SYMBOL(rfree); * To match the fall-back SOF main heap all private heaps should also be in the * uncached address range. */ -void *sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, - size_t alignment) +void *z_impl_sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, + size_t alignment) { if (flags & (SOF_MEM_FLAG_LARGE_BUFFER | SOF_MEM_FLAG_USER_SHARED_BUFFER)) return rballoc_align(flags, bytes, alignment); @@ -634,7 +634,7 @@ void *sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, return (__sparse_force void *)heap_alloc_aligned_cached(heap, alignment, bytes); } -void sof_heap_free(struct k_heap *heap, void *addr) +void z_impl_sof_heap_free(struct k_heap *heap, void *addr) { if (heap && addr && is_heap_pointer(heap, addr)) heap_free(heap, addr); diff --git a/zephyr/lib/dma.c b/zephyr/lib/dma.c index 7452459f8b0a..0b8987ed65d0 100644 --- a/zephyr/lib/dma.c +++ b/zephyr/lib/dma.c @@ -15,12 +15,13 @@ #include #include #include +#include #include #define DW_DMA_BUFFER_PERIOD_COUNT 0x4 #define HDA_DMA_BUFFER_PERIOD_COUNT 4 -SHARED_DATA struct sof_dma dma[] = { +APP_TASK_DATA SHARED_DATA struct sof_dma dma[] = { #if DT_NODE_HAS_STATUS(DT_NODELABEL(lpgpdma0), okay) { /* Low Power GP DMAC 0 */ .plat_data = { diff --git a/zephyr/syscall/alloc.c b/zephyr/syscall/alloc.c new file mode 100644 index 000000000000..fad39865b9d2 --- /dev/null +++ b/zephyr/syscall/alloc.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. + +#include +#include +#include + +static inline void *z_vrfy_sof_heap_alloc(struct k_heap *heap, uint32_t flags, + size_t bytes, size_t alignment) +{ + return z_impl_sof_heap_alloc(heap, flags, bytes, alignment); +} +#include + +static inline void z_vrfy_sof_heap_free(struct k_heap *heap, void *addr) +{ + z_impl_sof_heap_free(heap, addr); +} +#include diff --git a/zephyr/syscall/sof_dma.c b/zephyr/syscall/sof_dma.c index ed69ffc78423..11ee8156f7ff 100644 --- a/zephyr/syscall/sof_dma.c +++ b/zephyr/syscall/sof_dma.c @@ -119,9 +119,15 @@ static inline struct dma_block_config *deep_copy_dma_blk_cfg_list(struct dma_con for (user_next = cfg->head_block, kern_next = kern_cfg; user_next; - user_next = user_next->next_block, kern_next++) { - if (++i > cfg->block_count) - goto err; + user_next = user_next->next_block, kern_next++, i++) { + if (i == cfg->block_count) { + /* last block can point to first one */ + if (user_next != cfg->head_block) + goto err; + + kern_prev->next_block = kern_cfg; + break; + } if (k_usermode_from_copy(kern_next, user_next, sizeof(*kern_next))) goto err; diff --git a/zephyr/test/CMakeLists.txt b/zephyr/test/CMakeLists.txt index c5b66c83bbaa..b276bb307259 100644 --- a/zephyr/test/CMakeLists.txt +++ b/zephyr/test/CMakeLists.txt @@ -5,6 +5,9 @@ if(CONFIG_SOF_BOOT_TEST) zephyr_library_sources_ifdef(CONFIG_USERSPACE userspace/ksem.c ) + if(CONFIG_USERSPACE AND CONFIG_SOF_USERSPACE_INTERFACE_ALLOC) + zephyr_library_sources(userspace/test_heap_alloc.c) + endif() endif() if(CONFIG_SOF_BOOT_TEST_STANDALONE AND CONFIG_SOF_USERSPACE_INTERFACE_DMA) diff --git a/zephyr/test/userspace/test_heap_alloc.c b/zephyr/test/userspace/test_heap_alloc.c new file mode 100644 index 000000000000..8018e4b27b9d --- /dev/null +++ b/zephyr/test/userspace/test_heap_alloc.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2026 Intel Corporation. + */ + +/* + * Test case for sof_heap_alloc() / sof_heap_free() use from a Zephyr + * user-space thread. + */ + +#include +#include +#include + +#include +#include +#include + +LOG_MODULE_DECLARE(sof_boot_test, LOG_LEVEL_DBG); + +#define USER_STACKSIZE 2048 + +static struct k_thread user_thread; +static K_THREAD_STACK_DEFINE(user_stack, USER_STACKSIZE); + +static void user_function(void *p1, void *p2, void *p3) +{ + struct k_heap *heap = (struct k_heap *)p1; + void *ptr; + + __ASSERT(k_is_user_context(), "isn't user"); + + LOG_INF("SOF thread %s (%s)", + k_is_user_context() ? "UserSpace!" : "privileged mode.", + CONFIG_BOARD_TARGET); + + /* allocate a block from the user heap */ + ptr = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, 128, 0); + zassert_not_null(ptr, "sof_heap_alloc returned NULL"); + + LOG_INF("sof_heap_alloc returned %p", ptr); + + /* free the block */ + sof_heap_free(heap, ptr); + + LOG_INF("sof_heap_free done"); +} + +static void test_user_thread_heap_alloc(void) +{ + struct k_heap *heap; + + heap = zephyr_ll_user_heap(); + zassert_not_null(heap, "user heap not found"); + + k_thread_create(&user_thread, user_stack, USER_STACKSIZE, + user_function, heap, NULL, NULL, + -1, K_USER, K_FOREVER); + + /* Add thread to LL memory domain so it can access the user heap */ + k_mem_domain_add_thread(zephyr_ll_mem_domain(), &user_thread); + + k_thread_start(&user_thread); + k_thread_join(&user_thread, K_FOREVER); +} + +ZTEST(sof_boot, user_space_heap_alloc) +{ + test_user_thread_heap_alloc(); + + ztest_test_pass(); +} diff --git a/zephyr/test/userspace/test_ll_task.c b/zephyr/test/userspace/test_ll_task.c index 234423defc60..958bbf104f9b 100644 --- a/zephyr/test/userspace/test_ll_task.c +++ b/zephyr/test/userspace/test_ll_task.c @@ -14,9 +14,19 @@ #include #include #include +#include +#include +#include +#include #include #include #include +#include +#include +#include +#include +#include +#include #include #include @@ -24,6 +34,7 @@ #include #include /* offsetof() */ +#include LOG_MODULE_DECLARE(sof_boot_test, LOG_LEVEL_DBG); @@ -36,6 +47,12 @@ K_APPMEM_PARTITION_DEFINE(userspace_ll_part); /* Global variable for test runs counter, accessible from user-space */ K_APP_BMEM(userspace_ll_part) static int test_runs; +/* User-space thread for pipeline_two_components test */ +#define PPL_USER_STACKSIZE 4096 + +static struct k_thread ppl_user_thread; +static K_THREAD_STACK_DEFINE(ppl_user_stack, PPL_USER_STACKSIZE); + static enum task_state task_callback(void *arg) { LOG_INF("entry"); @@ -129,6 +146,365 @@ ZTEST(userspace_ll, pipeline_check) pipeline_check(); } +/* Copier UUID: 9ba00c83-ca12-4a83-943c-1fa2e82f9dda */ +static const uint8_t copier_uuid[16] = { + 0x83, 0x0c, 0xa0, 0x9b, 0x12, 0xca, 0x83, 0x4a, + 0x94, 0x3c, 0x1f, 0xa2, 0xe8, 0x2f, 0x9d, 0xda +}; + +/** + * Find the module_id (manifest entry index) for the copier module + * by iterating the firmware manifest and matching the copier UUID. + */ +static int find_copier_module_id(void) +{ + const struct sof_man_fw_desc *desc = basefw_vendor_get_manifest(); + const struct sof_man_module *mod; + uint32_t i; + + if (!desc) + return -1; + + for (i = 0; i < desc->header.num_module_entries; i++) { + mod = (const struct sof_man_module *)((const char *)desc + + SOF_MAN_MODULE_OFFSET(i)); + if (!memcmp(&mod->uuid, copier_uuid, sizeof(copier_uuid))) + return (int)i; + } + + return -1; +} + +/** + * IPC4 copier module config - used as payload for comp_new_ipc4(). + * Placed at MAILBOX_HOSTBOX_BASE before calling comp_new_ipc4(). + * Layout matches struct ipc4_copier_module_cfg from copier.h. + */ +struct copier_init_data { + struct ipc4_base_module_cfg base; + struct ipc4_audio_format out_fmt; + uint32_t copier_feature_mask; + /* Gateway config (matches struct ipc4_copier_gateway_cfg) */ + union ipc4_connector_node_id node_id; + uint32_t dma_buffer_size; + uint32_t config_length; +} __packed __aligned(4); + +static void fill_audio_format(struct ipc4_audio_format *fmt) +{ + memset(fmt, 0, sizeof(*fmt)); + fmt->sampling_frequency = IPC4_FS_48000HZ; + fmt->depth = IPC4_DEPTH_32BIT; + fmt->ch_cfg = IPC4_CHANNEL_CONFIG_STEREO; + fmt->channels_count = 2; + fmt->valid_bit_depth = 32; + fmt->s_type = IPC4_TYPE_MSB_INTEGER; + fmt->interleaving_style = IPC4_CHANNELS_INTERLEAVED; +} + +/** + * Create a copier component via IPC4. + * + * @param module_id Copier module_id from manifest + * @param instance_id Instance ID for this component + * @param pipeline_id Parent pipeline ID + * @param node_id Gateway node ID (type + virtual DMA index) + */ +static struct comp_dev *create_copier(int module_id, int instance_id, + int pipeline_id, + union ipc4_connector_node_id node_id) +{ + struct ipc4_module_init_instance module_init; + struct copier_init_data cfg; + struct comp_dev *dev; + + /* Prepare copier config payload */ + memset(&cfg, 0, sizeof(cfg)); + fill_audio_format(&cfg.base.audio_fmt); + /* 2 channels * 4 bytes * 48 frames = 384 bytes */ + cfg.base.ibs = 384; + cfg.base.obs = 384; + cfg.base.is_pages = 0; + cfg.base.cpc = 0; + cfg.out_fmt = cfg.base.audio_fmt; + cfg.copier_feature_mask = 0; + cfg.node_id = node_id; + cfg.dma_buffer_size = 768; + cfg.config_length = 0; + + /* Write config data to mailbox hostbox (where comp_new_ipc4 reads it). + * Flush cache so that data is visible in SRAM before comp_new_ipc4() + * invalidates the cache line (in normal IPC flow, host writes via DMA + * directly to SRAM, so the invalidation reads fresh data; here the DSP + * core itself writes, so an explicit flush is needed). + */ + memcpy((void *)MAILBOX_HOSTBOX_BASE, &cfg, sizeof(cfg)); + sys_cache_data_flush_range((void *)MAILBOX_HOSTBOX_BASE, sizeof(cfg)); + + /* Prepare IPC4 module init header */ + memset(&module_init, 0, sizeof(module_init)); + module_init.primary.r.module_id = module_id; + module_init.primary.r.instance_id = instance_id; + module_init.primary.r.type = SOF_IPC4_MOD_INIT_INSTANCE; + module_init.primary.r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_MODULE_MSG; + module_init.primary.r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; + + module_init.extension.r.param_block_size = sizeof(cfg) / sizeof(uint32_t); + module_init.extension.r.ppl_instance_id = pipeline_id; + module_init.extension.r.core_id = 0; + module_init.extension.r.proc_domain = 0; /* LL */ + + dev = comp_new_ipc4(&module_init); + + /* + * We use the IPC code to create the components. This code runs + * in kernel space, so we need to separately assign thecreated + * components to the user LL and IPC threads before it can be used. + */ + comp_grant_access_to_thread(dev, zephyr_domain_thread_tid(zephyr_ll_domain())); + comp_grant_access_to_thread(dev, &ppl_user_thread); + + return dev; +} + +/** + * Context shared between kernel setup and the user-space pipeline thread. + */ +struct ppl_test_ctx { + struct pipeline *p; + struct k_heap *heap; + struct comp_dev *host_comp; + struct comp_dev *dai_comp; + struct comp_buffer *buf; + struct ipc *ipc; + struct ipc_comp_dev *ipc_pipe; +}; + +/** + * Pipeline operations: connect, complete, prepare, copy, verify, and clean up. + * This function is called either directly (kernel mode) or from a user-space + * thread, exercising pipeline_*() calls from the requested context. + */ +static void pipeline_ops(struct ppl_test_ctx *ctx) +{ + struct pipeline *p = ctx->p; + struct comp_dev *host_comp = ctx->host_comp; + struct comp_dev *dai_comp = ctx->dai_comp; + struct comp_buffer *buf = ctx->buf; + int ret; + + LOG_INF("pipeline_ops: user_context=%d", k_is_user_context()); + + /* Step 6: Connect host -> buffer -> DAI */ + ret = pipeline_connect(host_comp, buf, PPL_CONN_DIR_COMP_TO_BUFFER); + zassert_equal(ret, 0, "connect host to buffer failed"); + + ret = pipeline_connect(dai_comp, buf, PPL_CONN_DIR_BUFFER_TO_COMP); + zassert_equal(ret, 0, "connect buffer to DAI failed"); + + LOG_INF("host -> buffer -> DAI connected"); + + /* Step 7: Complete the pipeline */ + ret = pipeline_complete(p, host_comp, dai_comp); + zassert_equal(ret, 0, "pipeline complete failed"); + + /* Step 8: Prepare the pipeline */ + p->sched_comp = host_comp; + k_sleep(K_MSEC(10)); + + ret = pipeline_prepare(p, host_comp); + zassert_equal(ret, 0, "pipeline prepare failed"); + + LOG_INF("pipeline complete, status = %d", p->status); + + /* Step 9: Run copies */ + pipeline_copy(p); + pipeline_copy(p); + + /* Verify pipeline source and sink assignments */ + zassert_equal(p->source_comp, host_comp, "source comp mismatch"); + zassert_equal(p->sink_comp, dai_comp, "sink comp mismatch"); + + LOG_INF("pipeline_ops done"); +} + +/** + * User-space thread entry point for pipeline_two_components test. + * p1 points to the ppl_test_ctx shared with the kernel launcher. + */ +static void pipeline_user_thread(void *p1, void *p2, void *p3) +{ + struct ppl_test_ctx *ctx = (struct ppl_test_ctx *)p1; + + zassert_true(k_is_user_context(), "expected user context"); + pipeline_ops(ctx); +} + +/** + * Test creating a pipeline with a host copier and a DAI (link) copier, + * connected through a shared buffer. + * + * When run_in_user is true, all pipeline_*() calls are made from a + * separate user-space thread. + */ +static void pipeline_two_components(bool run_in_user) +{ + struct ppl_test_ctx *ctx; + struct k_heap *heap = NULL; + uint32_t pipeline_id = 2; + uint32_t priority = 0; + uint32_t comp_id; + int copier_module_id; + int host_instance_id = 0; + int dai_instance_id = 1; + int ret; + + /* Step: Find the copier module_id from the firmware manifest */ + copier_module_id = find_copier_module_id(); + zassert_true(copier_module_id >= 0, "copier module not found in manifest"); + LOG_INF("copier module_id = %d", copier_module_id); + + /* Step: Create pipeline */ + if (run_in_user) { + LOG_INF("running test with user memory domain"); + heap = zephyr_ll_user_heap(); + zassert_not_null(heap, "user heap not found"); + } else { + LOG_INF("running test with kernel memory domain"); + } + + ctx = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, sizeof(*ctx), 0); + ctx->heap = heap; + ctx->ipc = ipc_get(); + + comp_id = IPC4_COMP_ID(copier_module_id, host_instance_id); + ctx->p = pipeline_new(ctx->heap, pipeline_id, priority, comp_id, NULL); + zassert_not_null(ctx->p, "pipeline creation failed"); + + /* Set pipeline period so components get correct dev->period and dev->frames. + * This mirrors what ipc4_create_pipeline() does in normal IPC flow. + */ + ctx->p->time_domain = SOF_TIME_DOMAIN_TIMER; + ctx->p->period = LL_TIMER_PERIOD_US; + + /* Register pipeline in IPC component list so comp_new_ipc4() can + * find it via ipc_get_comp_by_ppl_id() and set dev->period. + */ + ctx->ipc_pipe = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(struct ipc_comp_dev)); + zassert_not_null(ctx->ipc_pipe, "ipc_comp_dev alloc failed"); + ctx->ipc_pipe->pipeline = ctx->p; + ctx->ipc_pipe->type = COMP_TYPE_PIPELINE; + ctx->ipc_pipe->id = pipeline_id; + ctx->ipc_pipe->core = 0; + list_item_append(&ctx->ipc_pipe->list, &ctx->ipc->comp_list); + + /* Step: Create host copier with HDA host output gateway */ + union ipc4_connector_node_id host_node_id = { .f = { + .dma_type = ipc4_hda_host_output_class, + .v_index = 0 + }}; + ctx->host_comp = create_copier(copier_module_id, host_instance_id, pipeline_id, + host_node_id); + zassert_not_null(ctx->host_comp, "host copier creation failed"); + + /* Assign pipeline to host component */ + ctx->host_comp->pipeline = ctx->p; + ctx->host_comp->ipc_config.type = SOF_COMP_HOST; + + LOG_INF("host copier created, comp_id = 0x%x", ctx->host_comp->ipc_config.id); + + /* Step: Create link copier with HDA link output gateway */ + union ipc4_connector_node_id link_node_id = { .f = { + .dma_type = ipc4_hda_link_output_class, + .v_index = 0 + }}; + ctx->dai_comp = create_copier(copier_module_id, dai_instance_id, pipeline_id, + link_node_id); + zassert_not_null(ctx->dai_comp, "DAI copier creation failed"); + + /* Assign pipeline to DAI component */ + ctx->dai_comp->pipeline = ctx->p; + ctx->dai_comp->ipc_config.type = SOF_COMP_DAI; + + LOG_INF("DAI copier created, comp_id = 0x%x", ctx->dai_comp->ipc_config.id); + + /* Step: Allocate a buffer to connect host -> DAI */ + ctx->buf = buffer_alloc(ctx->heap, 384, 0, 0, false); + zassert_not_null(ctx->buf, "buffer allocation failed"); + + if (run_in_user) { + /* Create a user-space thread to execute pipeline operations */ + k_thread_create(&ppl_user_thread, ppl_user_stack, PPL_USER_STACKSIZE, + pipeline_user_thread, ctx, NULL, NULL, + -1, K_USER, K_FOREVER); + + /* Add thread to LL memory domain so it can access pipeline memory */ + k_mem_domain_add_thread(zephyr_ll_mem_domain(), &ppl_user_thread); + + user_grant_dai_access_all(&ppl_user_thread); + user_grant_dma_access_all(&ppl_user_thread); + user_access_to_mailbox(zephyr_ll_mem_domain(), &ppl_user_thread); + + /* + * A hack for testing purposes, normally DAI module + * is created in user-space so it gets access + * automatically. Until that works, use dai_dd directly. + */ + struct dai_data *dai_dd = comp_get_drvdata(ctx->dai_comp); + struct k_mutex *dai_lock = dai_dd->dai->lock; + k_thread_access_grant(&ppl_user_thread, dai_lock); + + k_thread_start(&ppl_user_thread); + + LOG_INF("user thread started, waiting for completion"); + + k_thread_join(&ppl_user_thread, K_FOREVER); + } else { + /* Run pipeline operations directly in kernel context */ + pipeline_ops(ctx); + } + + /* Step: Clean up - reset, disconnect, free buffer, free components, free pipeline */ + /* Reset pipeline to bring components back to COMP_STATE_READY, + * required before ipc_comp_free() which rejects non-READY components. + */ + ret = pipeline_reset(ctx->p, ctx->host_comp); + zassert_equal(ret, 0, "pipeline reset failed"); + + pipeline_disconnect(ctx->host_comp, ctx->buf, PPL_CONN_DIR_COMP_TO_BUFFER); + pipeline_disconnect(ctx->dai_comp, ctx->buf, PPL_CONN_DIR_BUFFER_TO_COMP); + + buffer_free(ctx->buf); + + /* Free components through IPC to properly remove from IPC device list */ + ret = ipc_comp_free(ctx->ipc, ctx->host_comp->ipc_config.id); + zassert_equal(ret, 0, "host comp free failed"); + + ret = ipc_comp_free(ctx->ipc, ctx->dai_comp->ipc_config.id); + zassert_equal(ret, 0, "DAI comp free failed"); + + /* Unregister pipeline from IPC component list */ + list_item_del(&ctx->ipc_pipe->list); + rfree(ctx->ipc_pipe); + + ret = pipeline_free(ctx->p); + zassert_equal(ret, 0, "pipeline free failed"); + + sof_heap_free(heap, ctx); + LOG_INF("two component pipeline test complete"); +} + +ZTEST(userspace_ll, pipeline_two_components_kernel) +{ + pipeline_two_components(false); +} + +ZTEST(userspace_ll, pipeline_two_components_user) +{ + pipeline_two_components(true); +} + ZTEST_SUITE(userspace_ll, NULL, NULL, NULL, NULL, NULL); /**