| // SPDX-License-Identifier: GPL-2.0-only |
| // Copyright(c) 2017-2021 Intel Corporation |
| |
| #include <linux/atomic.h> |
| #include <linux/device.h> |
| #include <linux/idr.h> |
| #include <linux/mm.h> |
| #include <linux/mutex.h> |
| #include <linux/pm_runtime.h> |
| #include <linux/uaccess.h> |
| #include <linux/slab.h> |
| |
| #include "device.h" |
| #include "hw.h" |
| #include "mem.h" |
| #include "request.h" |
| #include "score.h" |
| |
| static void gna_request_update_status(struct gna_request *score_request) |
| { |
| struct gna_private *gna_priv = score_request->gna_priv; |
| /* The gna_priv's hw_status should be updated first */ |
| u32 hw_status = gna_priv->hw_status; |
| u32 stall_cycles; |
| u32 total_cycles; |
| |
| /* Technically, the time stamp can be a bit later than |
| * when the hw actually completed scoring. Here we just |
| * do our best in a deferred work, unless we want to |
| * tax isr for a more accurate record. |
| */ |
| score_request->drv_perf.hw_completed = ktime_get_ns(); |
| |
| score_request->hw_status = hw_status; |
| |
| score_request->status = gna_parse_hw_status(gna_priv, hw_status); |
| |
| if (gna_hw_perf_enabled(gna_priv)) { |
| if (hw_status & GNA_STS_STATISTICS_VALID) { |
| total_cycles = gna_reg_read(gna_priv, GNA_MMIO_PTC); |
| stall_cycles = gna_reg_read(gna_priv, GNA_MMIO_PSC); |
| score_request->hw_perf.total = total_cycles; |
| score_request->hw_perf.stall = stall_cycles; |
| } else |
| dev_warn(gna_dev(gna_priv), "GNA statistics missing\n"); |
| } |
| if (unlikely(hw_status & GNA_ERROR)) |
| gna_print_error_status(gna_priv, hw_status); |
| } |
| |
| static void gna_request_process(struct work_struct *work) |
| { |
| struct gna_request *score_request; |
| struct gna_memory_object *mo; |
| struct gna_private *gna_priv; |
| struct gna_buffer *buffer; |
| unsigned long hw_timeout; |
| int ret; |
| u64 i; |
| |
| score_request = container_of(work, struct gna_request, work); |
| gna_priv = score_request->gna_priv; |
| dev_dbg(gna_dev(gna_priv), "processing request %llu\n", score_request->request_id); |
| |
| score_request->state = ACTIVE; |
| |
| score_request->drv_perf.pre_processing = ktime_get_ns(); |
| |
| ret = pm_runtime_get_sync(gna_parent(gna_priv)); |
| if (ret < 0 && ret != -EACCES) { |
| dev_warn(gna_dev(gna_priv), "pm_runtime_get_sync() failed: %d\n", ret); |
| score_request->status = -ENODEV; |
| pm_runtime_put_noidle(gna_parent(gna_priv)); |
| goto end; |
| } |
| |
| /* Set busy flag before kicking off HW. The isr will clear it and wake up us. There is |
| * no difference if isr is missed in a timeout situation of the last request. We just |
| * always set it busy and let the wait_event_timeout check the reset. |
| * wq: X -> true |
| * isr: X -> false |
| */ |
| gna_priv->dev_busy = true; |
| |
| ret = gna_score(score_request); |
| if (ret) { |
| if (pm_runtime_put(gna_parent(gna_priv)) < 0) |
| dev_warn(gna_dev(gna_priv), "pm_runtime_put() failed: %d\n", ret); |
| score_request->status = ret; |
| goto end; |
| } |
| |
| score_request->drv_perf.processing = ktime_get_ns(); |
| |
| hw_timeout = gna_priv->recovery_timeout_jiffies; |
| |
| hw_timeout = wait_event_timeout(gna_priv->dev_busy_waitq, |
| !gna_priv->dev_busy, hw_timeout); |
| |
| if (!hw_timeout) |
| dev_warn(gna_dev(gna_priv), "hardware timeout occurred\n"); |
| |
| gna_priv->hw_status = gna_reg_read(gna_priv, GNA_MMIO_STS); |
| |
| gna_request_update_status(score_request); |
| gna_abort_hw(gna_priv); |
| |
| ret = pm_runtime_put(gna_parent(gna_priv)); |
| if (ret < 0) |
| dev_warn(gna_dev(gna_priv), "pm_runtime_put() failed: %d\n", ret); |
| |
| buffer = score_request->buffer_list; |
| for (i = 0; i < score_request->buffer_count; i++, buffer++) { |
| mutex_lock(&gna_priv->memidr_lock); |
| mo = idr_find(&gna_priv->memory_idr, buffer->memory_id); |
| mutex_unlock(&gna_priv->memidr_lock); |
| if (mo) { |
| mutex_lock(&mo->page_lock); |
| mo->ops->put_pages(mo); |
| mutex_unlock(&mo->page_lock); |
| } else { |
| dev_warn(gna_dev(gna_priv), "mo not found %llu\n", buffer->memory_id); |
| } |
| } |
| |
| /* patches_ptr's are already freed by ops->score() function */ |
| kvfree(score_request->buffer_list); |
| score_request->buffer_list = NULL; |
| score_request->buffer_count = 0; |
| |
| gna_mmu_clear(gna_priv); |
| |
| end: |
| score_request->drv_perf.completion = ktime_get_ns(); |
| dev_dbg(gna_dev(gna_priv), "request %llu done, waking processes\n", |
| score_request->request_id); |
| score_request->state = DONE; |
| wake_up_interruptible_all(&score_request->waitq); |
| } |
| |
| static struct gna_request *gna_request_create(struct gna_file_private *file_priv, |
| struct gna_compute_cfg *compute_cfg) |
| { |
| struct gna_request *score_request; |
| struct gna_private *gna_priv; |
| |
| gna_priv = file_priv->gna_priv; |
| if (IS_ERR(gna_priv)) |
| return NULL; |
| |
| score_request = kzalloc(sizeof(*score_request), GFP_KERNEL); |
| if (!score_request) |
| return NULL; |
| kref_init(&score_request->refcount); |
| |
| dev_dbg(gna_dev(gna_priv), "layer_base %d layer_count %d\n", |
| compute_cfg->layer_base, compute_cfg->layer_count); |
| |
| score_request->request_id = atomic_inc_return(&gna_priv->request_count); |
| score_request->compute_cfg = *compute_cfg; |
| score_request->fd = file_priv->fd; |
| score_request->gna_priv = gna_priv; |
| score_request->state = NEW; |
| init_waitqueue_head(&score_request->waitq); |
| INIT_WORK(&score_request->work, gna_request_process); |
| |
| return score_request; |
| } |
| |
| /* |
| * returns true if [inner_offset, inner_size) is embraced by [0, outer_size). False otherwise. |
| */ |
| static bool gna_validate_ranges(u64 outer_size, u64 inner_offset, u64 inner_size) |
| { |
| return inner_offset < outer_size && |
| inner_size <= (outer_size - inner_offset); |
| } |
| |
| static int gna_validate_patches(struct gna_private *gna_priv, __u64 buffer_size, |
| struct gna_memory_patch *patches, u64 count) |
| { |
| u64 idx; |
| |
| for (idx = 0; idx < count; ++idx) { |
| if (patches[idx].size > 8) { |
| dev_err(gna_dev(gna_priv), "invalid patch size: %llu\n", patches[idx].size); |
| return -EINVAL; |
| } |
| |
| if (!gna_validate_ranges(buffer_size, patches[idx].offset, patches[idx].size)) { |
| dev_err(gna_dev(gna_priv), |
| "patch out of bounds. buffer size: %llu, patch offset/size:%llu/%llu\n", |
| buffer_size, patches[idx].offset, patches[idx].size); |
| return -EINVAL; |
| } |
| } |
| |
| return 0; |
| } |
| |
| static int gna_buffer_fill_patches(struct gna_buffer *buffer, struct gna_private *gna_priv) |
| { |
| __u64 patches_user = buffer->patches_ptr; |
| struct gna_memory_patch *patches; |
| /* At this point, the buffer points to a memory region in kernel space where the copied |
| * patches_ptr also lives, but the value of it is still an address from user space. This |
| * function will set patches_ptr to either an address in kernel space or null before it |
| * exits. |
| */ |
| u64 patch_count; |
| int ret; |
| |
| buffer->patches_ptr = 0; |
| patch_count = buffer->patch_count; |
| if (!patch_count) |
| return 0; |
| |
| patches = kvmalloc_array(patch_count, sizeof(struct gna_memory_patch), GFP_KERNEL); |
| if (!patches) |
| return -ENOMEM; |
| |
| if (copy_from_user(patches, u64_to_user_ptr(patches_user), |
| sizeof(struct gna_memory_patch) * patch_count)) { |
| dev_err(gna_dev(gna_priv), "copy %llu patches from user failed\n", patch_count); |
| ret = -EFAULT; |
| goto err_fill_patches; |
| } |
| |
| ret = gna_validate_patches(gna_priv, buffer->size, patches, patch_count); |
| if (ret) { |
| dev_err(gna_dev(gna_priv), "patches failed validation\n"); |
| goto err_fill_patches; |
| } |
| |
| buffer->patches_ptr = (uintptr_t)patches; |
| |
| return 0; |
| |
| err_fill_patches: |
| kvfree(patches); |
| return ret; |
| } |
| |
| static int gna_request_fill_buffers(struct gna_request *score_request, |
| struct gna_compute_cfg *compute_cfg) |
| { |
| struct gna_buffer *buffer_list; |
| struct gna_memory_object *mo; |
| struct gna_private *gna_priv; |
| u64 buffers_total_size = 0; |
| struct gna_buffer *buffer; |
| u64 buffer_count; |
| u64 memory_id; |
| u64 i, j; |
| int ret; |
| |
| gna_priv = score_request->gna_priv; |
| |
| buffer_count = compute_cfg->buffer_count; |
| buffer_list = kvmalloc_array(buffer_count, sizeof(struct gna_buffer), GFP_KERNEL); |
| if (!buffer_list) |
| return -ENOMEM; |
| |
| if (copy_from_user(buffer_list, u64_to_user_ptr(compute_cfg->buffers_ptr), |
| sizeof(*buffer_list) * buffer_count)) { |
| dev_err(gna_dev(gna_priv), "copying %llu buffers failed\n", buffer_count); |
| ret = -EFAULT; |
| goto err_free_buffers; |
| } |
| |
| for (i = 0; i < buffer_count; i++) { |
| buffer = &buffer_list[i]; |
| memory_id = buffer->memory_id; |
| |
| for (j = 0; j < i; j++) { |
| if (buffer_list[j].memory_id == memory_id) { |
| dev_err(gna_dev(gna_priv), |
| "doubled memory id in score config. id:%llu\n", memory_id); |
| ret = -EINVAL; |
| goto err_zero_patch_ptr; |
| } |
| } |
| |
| buffers_total_size += |
| gna_buffer_get_size(buffer->offset, buffer->size); |
| if (buffers_total_size > gna_priv->info.max_hw_mem) { |
| dev_err(gna_dev(gna_priv), "buffers' total size too big\n"); |
| ret = -EINVAL; |
| goto err_zero_patch_ptr; |
| } |
| |
| mutex_lock(&gna_priv->memidr_lock); |
| mo = idr_find(&gna_priv->memory_idr, memory_id); |
| if (!mo) { |
| mutex_unlock(&gna_priv->memidr_lock); |
| dev_err(gna_dev(gna_priv), "memory object %llu not found\n", memory_id); |
| ret = -EINVAL; |
| goto err_zero_patch_ptr; |
| } |
| mutex_unlock(&gna_priv->memidr_lock); |
| |
| if (mo->fd != score_request->fd) { |
| dev_err(gna_dev(gna_priv), |
| "memory object from another file. %p != %p\n", |
| mo->fd, score_request->fd); |
| ret = -EINVAL; |
| goto err_zero_patch_ptr; |
| } |
| |
| if (!gna_validate_ranges(mo->memory_size, buffer->offset, buffer->size)) { |
| dev_err(gna_dev(gna_priv), |
| "buffer out of bounds. mo size: %llu, buffer offset/size:%llu/%llu\n", |
| mo->memory_size, buffer->offset, buffer->size); |
| ret = -EINVAL; |
| goto err_zero_patch_ptr; |
| } |
| |
| ret = gna_buffer_fill_patches(buffer, gna_priv); |
| if (ret) |
| goto err_free_patches; |
| } |
| |
| score_request->buffer_list = buffer_list; |
| score_request->buffer_count = buffer_count; |
| |
| return 0; |
| |
| err_zero_patch_ptr: |
| /* patches_ptr may still hold an address in userspace. |
| * Don't pass it to kvfree(). |
| */ |
| buffer->patches_ptr = 0; |
| |
| err_free_patches: |
| /* patches_ptr of each processed buffer should be either |
| * null or pointing to an allocated memory block in the |
| * kernel at this point. |
| */ |
| for (j = 0; j <= i; j++) |
| kvfree((void *)(uintptr_t)buffer_list[j].patches_ptr); |
| |
| err_free_buffers: |
| kvfree(buffer_list); |
| return ret; |
| } |
| |
| int gna_enqueue_request(struct gna_compute_cfg *compute_cfg, |
| struct gna_file_private *file_priv, u64 *request_id) |
| { |
| struct gna_request *score_request; |
| struct gna_private *gna_priv; |
| int ret; |
| |
| if (!file_priv) |
| return -EINVAL; |
| |
| gna_priv = file_priv->gna_priv; |
| |
| score_request = gna_request_create(file_priv, compute_cfg); |
| if (!score_request) |
| return -ENOMEM; |
| |
| ret = gna_request_fill_buffers(score_request, compute_cfg); |
| if (ret) { |
| kref_put(&score_request->refcount, gna_request_release); |
| return ret; |
| } |
| |
| kref_get(&score_request->refcount); |
| mutex_lock(&gna_priv->reqlist_lock); |
| list_add_tail(&score_request->node, &gna_priv->request_list); |
| mutex_unlock(&gna_priv->reqlist_lock); |
| |
| queue_work(gna_priv->request_wq, &score_request->work); |
| kref_put(&score_request->refcount, gna_request_release); |
| |
| *request_id = score_request->request_id; |
| |
| return 0; |
| } |
| |
| void gna_request_release(struct kref *ref) |
| { |
| struct gna_request *score_request = |
| container_of(ref, struct gna_request, refcount); |
| kfree(score_request); |
| } |
| |
| struct gna_request *gna_find_request_by_id(u64 req_id, struct gna_private *gna_priv) |
| { |
| struct gna_request *req, *found_req; |
| struct list_head *reqs_list; |
| |
| mutex_lock(&gna_priv->reqlist_lock); |
| |
| reqs_list = &gna_priv->request_list; |
| found_req = NULL; |
| if (!list_empty(reqs_list)) { |
| list_for_each_entry(req, reqs_list, node) { |
| if (req_id == req->request_id) { |
| found_req = req; |
| kref_get(&found_req->refcount); |
| break; |
| } |
| } |
| } |
| |
| mutex_unlock(&gna_priv->reqlist_lock); |
| |
| return found_req; |
| } |
| |
| void gna_delete_request_by_id(u64 req_id, struct gna_private *gna_priv) |
| { |
| struct gna_request *req, *temp_req; |
| struct list_head *reqs_list; |
| |
| mutex_lock(&gna_priv->reqlist_lock); |
| |
| reqs_list = &gna_priv->request_list; |
| if (!list_empty(reqs_list)) { |
| list_for_each_entry_safe(req, temp_req, reqs_list, node) { |
| if (req->request_id == req_id) { |
| list_del(&req->node); |
| cancel_work_sync(&req->work); |
| kref_put(&req->refcount, gna_request_release); |
| break; |
| } |
| } |
| } |
| |
| mutex_unlock(&gna_priv->reqlist_lock); |
| } |
| |
| void gna_delete_file_requests(struct file *fd, struct gna_private *gna_priv) |
| { |
| struct gna_request *req, *temp_req; |
| struct list_head *reqs_list; |
| |
| mutex_lock(&gna_priv->reqlist_lock); |
| |
| reqs_list = &gna_priv->request_list; |
| if (!list_empty(reqs_list)) { |
| list_for_each_entry_safe(req, temp_req, reqs_list, node) { |
| if (req->fd == fd) { |
| list_del(&req->node); |
| cancel_work_sync(&req->work); |
| kref_put(&req->refcount, gna_request_release); |
| break; |
| } |
| } |
| } |
| |
| mutex_unlock(&gna_priv->reqlist_lock); |
| } |
| |
| void gna_delete_memory_requests(u64 memory_id, struct gna_private *gna_priv) |
| { |
| struct gna_request *req, *temp_req; |
| struct list_head *reqs_list; |
| int i; |
| |
| mutex_lock(&gna_priv->reqlist_lock); |
| |
| reqs_list = &gna_priv->request_list; |
| if (!list_empty(reqs_list)) { |
| list_for_each_entry_safe(req, temp_req, reqs_list, node) { |
| for (i = 0; i < req->buffer_count; ++i) { |
| if (req->buffer_list[i].memory_id == memory_id) { |
| list_del(&req->node); |
| cancel_work_sync(&req->work); |
| kref_put(&req->refcount, gna_request_release); |
| break; |
| } |
| } |
| } |
| } |
| |
| mutex_unlock(&gna_priv->reqlist_lock); |
| } |