/* * Android/Easel coprocessor communication. * * Copyright 2016 Google Inc. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ /* #define DEBUG */ #include #include "google-easel-comm-shared.h" #include "google-easel-comm-private.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* retry delay in msec for retrying flush of a message being processed */ #define MSG_FLUSH_RETRY_DELAY 10 /* Service for link status update */ static struct easelcomm_service link_service; /* Mutex to guard easelcomm_service_list */ struct mutex service_mutex; /* Per-Easel-service state */ static struct easelcomm_service *easelcomm_service_list[ EASELCOMM_SERVICE_COUNT]; /* Local command channel local state */ struct easelcomm_cmd_channel_local { char *buffer; /* start of command buffer */ char *readp; /* ptr to next entry to read */ uint64_t consumer_seqnbr_next; /* next cmd seq# to be consumed */ }; static struct easelcomm_cmd_channel_local cmd_channel_local; /* Remote command channel local state */ struct easelcomm_cmd_channel_remote { /* Protects access to all mutable fields below */ struct mutex mutex; /* offset of next entry to write */ uint64_t write_offset; /* next command sequence number to write */ uint64_t write_seqnbr; /* remote consumer has caught up, ready to wrap channel buffer */ struct completion wrap_ready; /* remote channel is initialized */ bool initialized; /* channel initialization or link shutdown received from remote */ struct completion init_state_changed; }; static struct easelcomm_cmd_channel_remote cmd_channel_remote; /* Per-file-descriptor (user) state */ struct easelcomm_user_state { /* Which Easel service is registered for this fd, or NULL if none */ struct easelcomm_service *service; }; /* max delay in msec waiting for remote to wrap command channel */ #define CMDCHAN_WRAP_DONE_TIMEOUT_MS 1000 /* max delay in msec waiting for remote to ack link shutdown */ #define LINK_SHUTDOWN_ACK_TIMEOUT 500 /* max delay in msec waiting for remote to return flush done */ #define SERVICE_FLUSH_DONE_TIMEOUT 5000 static bool easelcomm_up; /* is easelcomm up and running? */ /* used to wait for remote peer to ack link shutdown */ static DECLARE_COMPLETION(easelcomm_link_peer_shutdown); static int easelcomm_open(struct inode *inode, struct file *file); static int easelcomm_release(struct inode *inode, struct file *file); static long easelcomm_ioctl( struct file *file, unsigned int cmd, unsigned long arg); #ifdef CONFIG_COMPAT /* 32-bit userspace on 64-bit OS conversion defines */ struct easelcomm_compat_kbuf_desc { easelcomm_msgid_t message_id; /* ID of message for this transfer */ compat_uptr_t __user buf; /* local buffer source or dest */ int dma_buf_fd; /* fd of local dma_buf */ int buf_type; /* e.g. EASELCOMM_DMA_BUFFER_DMA_BUF */ uint32_t buf_size; /* size of the local buffer */ }; static long easelcomm_compat_ioctl( struct file *file, unsigned int cmd, unsigned long arg); #else #define easelcomm_compat_ioctl NULL #endif static const struct file_operations easelcomm_fops = { .owner = THIS_MODULE, .open = easelcomm_open, .release = easelcomm_release, .unlocked_ioctl = easelcomm_ioctl, .compat_ioctl = easelcomm_compat_ioctl, .llseek = no_llseek, }; struct miscdevice easelcomm_miscdev = { .minor = MISC_DYNAMIC_MINOR, #ifdef CONFIG_GOOGLE_EASEL_AP .name = "easelcomm-client", #else .name = "easelcomm-server", #endif .fops = &easelcomm_fops, }; EXPORT_SYMBOL(easelcomm_miscdev); /* Dump message info for debugging. */ static void easelcomm_dump_message( struct easelcomm_message_metadata *msg_metadata) { struct easelcomm_kmsg *msg = msg_metadata->msg; dev_dbg(easelcomm_miscdev.this_device, "msg: id=%s%llu msgsize=%u dmasize=%u needreply=%d inreplyto=%llu replycode=%d\n", easelcomm_msgid_prefix(msg_metadata), msg->desc.message_id, msg->desc.message_size, msg->desc.dma_buf_size, msg->desc.need_reply, msg->desc.in_reply_to, msg->desc.replycode); } /* * Add an additional refcount bump on a message for which a refcount is * already held. This function is for use by callers that hold a valid * message pointer for which a refcount is already held, in order to reflect * another reference being created, which will be dropped by another context * later. At present this is used only to hand off incoming reply messages to * waiters, where the additional refcount reflects the fact that the original * message contains a pointer to the reply message. */ static void easelcomm_grab_reference( struct easelcomm_service *service, struct easelcomm_message_metadata *msg_metadata) { if (!msg_metadata) return; mutex_lock(&service->lock); msg_metadata->reference_count++; mutex_unlock(&service->lock); } /* * Find a local message (for the specified service) by ID. * Returns a pointer to the metadata with the reference count bumped. */ struct easelcomm_message_metadata *easelcomm_find_local_message( struct easelcomm_service *service, easelcomm_msgid_t message_id) { struct easelcomm_message_metadata *msg_metadata = NULL; struct easelcomm_message_metadata *msg_cursor; mutex_lock(&service->lock); list_for_each_entry(msg_cursor, &service->local_list, list) { if (msg_cursor->msg->desc.message_id == message_id) { msg_metadata = msg_cursor; msg_metadata->reference_count++; break; } } mutex_unlock(&service->lock); return msg_metadata; } EXPORT_SYMBOL(easelcomm_find_local_message); /* * Find a remote message (for the specified service) by ID. * Returns a pointer to the metadata with the reference count bumped. */ struct easelcomm_message_metadata *easelcomm_find_remote_message( struct easelcomm_service *service, easelcomm_msgid_t message_id) { struct easelcomm_message_metadata *msg_metadata = NULL; struct easelcomm_message_metadata *msg_cursor; mutex_lock(&service->lock); list_for_each_entry(msg_cursor, &service->remote_list, list) { if (msg_cursor->msg->desc.message_id == message_id) { msg_metadata = msg_cursor; msg_metadata->reference_count++; break; } } mutex_unlock(&service->lock); return msg_metadata; } EXPORT_SYMBOL(easelcomm_find_remote_message); /* * Return a pointer to the reply message for the specified original message. * Responsibility for the existing reference count held for the reply message * (due to the linkage from the original message) now transfers to the caller, * which must drop the reference when appropriate. */ static struct easelcomm_message_metadata *easelcomm_grab_reply_message( struct easelcomm_service *service, struct easelcomm_message_metadata *orig_msg_metadata) { struct easelcomm_message_metadata *reply_msg_metadata; if (!orig_msg_metadata) return NULL; mutex_lock(&service->lock); reply_msg_metadata = orig_msg_metadata->reply_metadata; orig_msg_metadata->reply_metadata = NULL; mutex_unlock(&service->lock); return reply_msg_metadata; } /* * Add a new message and metadata for the specified service. Depending on the * message type, the message will be placed in the local or remote list and * potentially the receiveMessage queue. The reference count for the message * starts at 1 due to the returned pointer. */ static struct easelcomm_message_metadata *easelcomm_add_metadata( struct easelcomm_service *service, enum easelcomm_msg_type msg_type, struct easelcomm_kmsg *msg) { struct easelcomm_message_metadata *msg_metadata = kmalloc(sizeof(struct easelcomm_message_metadata), GFP_KERNEL); if (!msg_metadata) return NULL; msg_metadata->msg_type = msg_type; INIT_LIST_HEAD(&msg_metadata->list); msg_metadata->dma_xfer.sg_local = NULL; msg_metadata->dma_xfer.sg_local_size = 0; msg_metadata->dma_xfer.sg_remote = NULL; msg_metadata->dma_xfer.sg_remote_size = 0; init_completion(&msg_metadata->dma_xfer.sg_remote_ready); init_completion(&msg_metadata->dma_xfer.xfer_ready); init_completion(&msg_metadata->dma_xfer.xfer_done); msg_metadata->dma_xfer.aborting = false; msg_metadata->msg = msg; msg_metadata->reference_count = 1; msg_metadata->queued = false; msg_metadata->flushing = false; msg_metadata->free_message = false; init_completion(&msg_metadata->reply_received); msg_metadata->reply_metadata = NULL; mutex_lock(&service->lock); switch (msg_metadata->msg_type) { case TYPE_LOCAL: /* add to the local list */ list_add_tail(&msg_metadata->list, &service->local_list); break; case TYPE_REMOTE_REPLY: case TYPE_REMOTE_NONREPLY: /* add to the remote list */ list_add_tail(&msg_metadata->list, &service->remote_list); if (msg_metadata->msg_type == TYPE_REMOTE_NONREPLY) { /* * Also add to the receiveMessage queue and wakeup * waiter. */ list_add_tail(&msg_metadata->rcvq_list, &service->receivemsg_queue); msg_metadata->queued = true; complete(&service->receivemsg_queue_new); } break; default: WARN_ON(1); } mutex_unlock(&service->lock); return msg_metadata; } /* * Free message, remove from lists and receiveMessage queue. Called when * message reference count drops to zero, and either: the message is marked * for freeing when no longer referenced, or: the message is being flushed. * The lock for the containing service must be held. This method is only for * use by reference counting and message flushing routines, not for general use. */ static void easelcomm_free_message( struct easelcomm_service *service, struct easelcomm_message_metadata *msg_metadata) { if (!msg_metadata) return; WARN_ON(msg_metadata->reference_count); /* remove from local or remote list */ list_del(&msg_metadata->list); /* If queued remove from receive queue */ if (msg_metadata->queued) list_del(&msg_metadata->rcvq_list); kfree(msg_metadata->msg); vfree(msg_metadata->dma_xfer.sg_local); /* was allocated by vmalloc */ kfree(msg_metadata->dma_xfer.sg_remote); kfree(msg_metadata); } /* * Decrement message refcount, optionally mark for freeing. The message is * freed if it is now, or has previously been, marked for freeing when * unreferenced and the reference count is dropped to zero. Once this method * is called the supplied message pointer is no longer valid and must not be * dereferenced. Any code that needs to reference the same message again * needs to find it by message ID. * * A reference count must be held whenever a pointer to the message is created * and valid (apart from local or remote message list linkage, which is always * present until the message is freed). (And except briefly while maintaining * message state and the service lock is held.) A reference count of zero * indicates the message is not actively being manipulated by a kernel code * path and no pointers to it existing apart from the local or remote message * list for its associated service -- if not in the process of being freed then * it is expected something will later grab a reference to the message and * resume handling it. */ void easelcomm_drop_reference( struct easelcomm_service *service, struct easelcomm_message_metadata *msg_metadata, bool mark_free) { if (!msg_metadata) return; mutex_lock(&service->lock); if (!WARN_ON(!msg_metadata->reference_count)) msg_metadata->reference_count--; if (mark_free) msg_metadata->free_message = true; if (!msg_metadata->reference_count && msg_metadata->free_message) easelcomm_free_message(service, msg_metadata); mutex_unlock(&service->lock); } EXPORT_SYMBOL(easelcomm_drop_reference); /* * Attempt to flush a message. If a reference count is held on the message or * the DMA transfer for the message is in progress then the message is not * flushed. If a DMA transfer is active then a DMA abort is initiated. * Function returns true if need to retry flushing the message later due to * the above, or false if now flushed. */ static bool easelcomm_flush_message( struct easelcomm_service *service, struct easelcomm_message_metadata *msg_metadata) { easelcomm_msgid_t msgid; bool ret = false; if (!msg_metadata || !msg_metadata->msg) return false; msgid = msg_metadata->msg->desc.message_id; /* Mark message for flushing in case message is still in progress */ msg_metadata->flushing = true; /* * If we may be working on a DMA transfer then trigger an abort. If * a local process is working on it then a refcount will be held, and * we'll need to retry after dropping the mutex, letting the * process handle the DMA abort and then drop the refcount. */ if (msg_metadata->msg->desc.dma_buf_size && !msg_metadata->dma_xfer.aborting) { dev_dbg(easelcomm_miscdev.this_device, "flush abort DMA msg %u:%s%llu\n", service->service_id, easelcomm_msgid_prefix(msg_metadata), msgid); msg_metadata->dma_xfer.aborting = true; complete(&msg_metadata->dma_xfer.sg_remote_ready); complete(&msg_metadata->dma_xfer.xfer_ready); complete(&msg_metadata->dma_xfer.xfer_done); } if (msg_metadata->reference_count) { dev_dbg(easelcomm_miscdev.this_device, "not flushing msg %u:%s%llu refcnt=%d\n", service->service_id, easelcomm_msgid_prefix(msg_metadata), msgid, msg_metadata->reference_count); ret = true; } else { easelcomm_free_message(service, msg_metadata); dev_dbg(easelcomm_miscdev.this_device, "flushed msg %u:%s%llu\n", service->service_id, easelcomm_msgid_prefix(msg_metadata), msgid); ret = false; } return ret; } /* Flush all messages (local or remote) kept locally for the service. */ static void easelcomm_flush_local_service(struct easelcomm_service *service) { struct easelcomm_message_metadata *msg_cursor; struct easelcomm_message_metadata *msg_temp; int i; bool need_retry = true; #define MAX_FLUSH_TRIES 4 for (i = 0; need_retry && i < MAX_FLUSH_TRIES; i++) { mutex_lock(&service->lock); need_retry = false; list_for_each_entry_safe( msg_cursor, msg_temp, &service->local_list, list) { need_retry |= easelcomm_flush_message( service, msg_cursor); } list_for_each_entry_safe( msg_cursor, msg_temp, &service->remote_list, list) { need_retry |= easelcomm_flush_message( service, msg_cursor); } mutex_unlock(&service->lock); if (!need_retry) break; /* sleep, let refcount holders abort DMA/reply wait */ msleep(MSG_FLUSH_RETRY_DELAY); } if (need_retry) dev_err(easelcomm_miscdev.this_device, "svc %u failed to flush messages after %d tries\n", service->service_id, MAX_FLUSH_TRIES); } /* * Wake up any message of a service waiting for reply. */ static void __force_complete_reply_waiter(struct easelcomm_service *service) { struct easelcomm_message_metadata *msg_cursor; mutex_lock(&service->lock); list_for_each_entry(msg_cursor, &service->local_list, list) { if (msg_cursor->msg->desc.need_reply) complete(&msg_cursor->reply_received); } mutex_unlock(&service->lock); } /* * Handle local-side processing of shutting down an Easel service. Easel * services are shutdown when the local side handling process closes its fd * or issues the shutdown ioctl, or the remote side sends a CLOSE_SERVICE * command (meaning the remote side is doing one of those things). * * Flush local state if local side initiated the shutdown, wakeup the * receiveMessage() waiter. */ static void easelcomm_handle_service_shutdown( struct easelcomm_service *service, bool shutdown_local) { dev_dbg(easelcomm_miscdev.this_device, "svc %u shutdown from %s\n", service->service_id, shutdown_local ? "local" : "remote"); /* * If local side shutting down then flush local state, just in case. * If remote is closing then local side may still have messages in * progress, just signal the receivemessage() caller to return * shutdown status. */ if (shutdown_local) easelcomm_flush_local_service(service); mutex_lock(&service->lock); if (shutdown_local) service->shutdown_local = true; else service->shutdown_remote = true; mutex_unlock(&service->lock); /* Wakeup any receiveMessage() waiter so they can return to user */ complete(&service->receivemsg_queue_new); /* Wakeup flush_done waiter so they can return to user */ complete(&service->flush_done); /* Wakeup any sendMessageReceiveReply() waiter to return to user */ __force_complete_reply_waiter(service); } /* * Shutdown easelcomm local activity, mark link down. */ static void easelcomm_stop_local(void) { int i; if (!easelcomm_up) return; dev_dbg(easelcomm_miscdev.this_device, "stopping\n"); /* Set local shutdown flag, disallow further activity */ easelcomm_up = false; for (i = 0; i < EASELCOMM_SERVICE_COUNT; i++) { struct easelcomm_service *service = easelcomm_service_list[i]; if (service != NULL) easelcomm_handle_service_shutdown(service, true); } } /* * Shutdown easelcomm communications. async is true if emergency shutdown (on * panic), don't wait for sync from other side. */ static void easelcomm_stop(bool async) { int ret; if (!easelcomm_up) return; easelcomm_send_cmd_noargs( &link_service, EASELCOMM_CMD_LINK_SHUTDOWN); if (!async) { ret = wait_for_completion_interruptible_timeout( &easelcomm_link_peer_shutdown, msecs_to_jiffies(LINK_SHUTDOWN_ACK_TIMEOUT)); if (ret <= 0) dev_warn(easelcomm_miscdev.this_device, "error or timeout on peer link shutdown ack\n"); } easelcomm_stop_local(); } /* * LINK_SHUTDOWN command received from remote, stop local side and send * ACK_SHUTDOWN back to remote. */ static void easelcomm_handle_cmd_link_shutdown(void) { dev_dbg(easelcomm_miscdev.this_device, "recv cmd LINK_SHUTDOWN\n"); easelcomm_stop_local(); dev_dbg(easelcomm_miscdev.this_device, "send cmd ACK_SHUTDOWN\n"); easelcomm_send_cmd_noargs( &link_service, EASELCOMM_CMD_ACK_SHUTDOWN); } /* ACK_SHUTDOWN command received from remote, wakeup waiter. */ static void easelcomm_handle_cmd_ack_shutdown(void) { dev_dbg(easelcomm_miscdev.this_device, "recv cmd ACK_SHUTDOWN\n"); complete(&easelcomm_link_peer_shutdown); } /* * Handle SEND_MSG command from remote. Add a remote message, link a reply * to original message and wakeup waiter. */ static void easelcomm_handle_cmd_send_msg( struct easelcomm_service *service, char *command_args, size_t command_arg_len) { struct easelcomm_kmsg *cmd_msg; struct easelcomm_kmsg *new_msg; enum easelcomm_msg_type msg_type; struct easelcomm_message_metadata *msg_metadata; bool discard_message = false; if (WARN_ON(command_arg_len < sizeof(struct easelcomm_kmsg_desc))) return; cmd_msg = (struct easelcomm_kmsg *) command_args; if (WARN_ON(command_arg_len != sizeof(struct easelcomm_kmsg_desc) + cmd_msg->desc.message_size)) return; dev_dbg(easelcomm_miscdev.this_device, "recv cmd SEND_MSG msg %u:r%llu\n", service->service_id, cmd_msg->desc.message_id); new_msg = kmalloc(command_arg_len, GFP_KERNEL); if (WARN_ON(!new_msg)) return; memcpy(new_msg, cmd_msg, command_arg_len); msg_type = new_msg->desc.in_reply_to ? TYPE_REMOTE_REPLY : TYPE_REMOTE_NONREPLY; msg_metadata = easelcomm_add_metadata(service, msg_type, new_msg); if (WARN_ON(!msg_metadata)) { kfree(new_msg); return; } easelcomm_dump_message(msg_metadata); /* If a reply then link to the original message and wakeup waiter. */ if (msg_type == TYPE_REMOTE_REPLY) { struct easelcomm_message_metadata *orig_msg_metadata = easelcomm_find_local_message( service, msg_metadata->msg->desc.in_reply_to); if (orig_msg_metadata) { orig_msg_metadata->reply_metadata = msg_metadata; /* * Bump reference count for this new ref, waiter will * drop. */ easelcomm_grab_reference(service, msg_metadata); complete(&orig_msg_metadata->reply_received); easelcomm_drop_reference( service, orig_msg_metadata, false); } else { dev_dbg(easelcomm_miscdev.this_device, "reply msg %u:r%llu reply-to msg %u:l%llu not found\n", service->service_id, msg_metadata->msg->desc.message_id, service->service_id, msg_metadata->msg->desc.in_reply_to); discard_message = true; } } easelcomm_drop_reference(service, msg_metadata, discard_message); } /* * Remote side of link indicates command channel ready. Both server and * client call this when handling any command. * In addition, server calls this when a LINK_INIT command is received from * the client. * * Init stuff and wakeup anybody waiting for the channel to be ready (to send * a command). */ static void easelcomm_cmd_channel_remote_set_ready(void) { struct easelcomm_cmd_channel_remote *channel = &cmd_channel_remote; mutex_lock(&channel->mutex); /* Return right away if channel is already initialized */ if (channel->initialized) { mutex_unlock(&channel->mutex); return; } /* Next write offset starts at top after header, cmd seq# zero */ channel->write_offset = (sizeof(struct easelcomm_cmd_channel_header) + 7) & ~0x7; channel->write_seqnbr = 0; /* mark channel initialized, wakeup waiters. */ channel->initialized = true; complete_all(&channel->init_state_changed); easelcomm_up = true; mutex_unlock(&channel->mutex); } /* * LINK_INIT command received from client, init local state for remote command * channel. */ static void easelcomm_handle_cmd_link_init( char *command_args, size_t command_arg_len) { dev_dbg(easelcomm_miscdev.this_device, "recv cmd LINK_INIT\n"); easelcomm_cmd_channel_remote_set_ready(); } /* * Wait for remote command channel ready and initialized, prior to sending a * command. Returns 0 if command channel ready and channel mutex is held, or * non-zero if signal received while waiting, channel mutex not held. */ static int easelcomm_wait_channel_initialized( struct easelcomm_cmd_channel_remote *channel) { int ret = 0; mutex_lock(&channel->mutex); while (!channel->initialized) { mutex_unlock(&channel->mutex); ret = wait_for_completion_interruptible( &channel->init_state_changed); if (ret) return ret; mutex_lock(&channel->mutex); } return ret; } /* CLOSE_SERVICE command received from remote, shut down service. */ static void easelcomm_handle_cmd_close_service( struct easelcomm_service *service) { dev_dbg(easelcomm_miscdev.this_device, "recv cmd CLOSE_SERVICE svc %u\n", service->service_id); easelcomm_handle_service_shutdown(service, false); } /* * FLUSH_SERVICE command received from remote, flush local state for service * and send FLUSH_SERVICE_DONE back to remote. */ static void easelcomm_handle_cmd_flush_service( struct easelcomm_service *service) { dev_dbg(easelcomm_miscdev.this_device, "recv cmd FLUSH_SERVICE svc %u\n", service->service_id); easelcomm_flush_local_service(service); dev_dbg(easelcomm_miscdev.this_device, "send cmd FLUSH_SERVICE_DONE svc %u\n", service->service_id); easelcomm_send_cmd_noargs(service, EASELCOMM_CMD_FLUSH_SERVICE_DONE); } /* FLUSH_SERVICE_DONE command received from remote, wakeup waiter. */ static void easelcomm_handle_cmd_flush_service_done( struct easelcomm_service *service) { dev_dbg(easelcomm_miscdev.this_device, "recv cmd FLUSH_SERVICE_DONE svc %u\n", service->service_id); complete(&service->flush_done); } static void easelcomm_init_service(struct easelcomm_service *service, int id) { if (service == NULL) { dev_err(easelcomm_miscdev.this_device, "service is null\n"); return; } service->service_id = id; service->user = NULL; service->shutdown_local = false; service->shutdown_remote = false; mutex_init(&service->lock); INIT_LIST_HEAD(&service->local_list); INIT_LIST_HEAD(&service->receivemsg_queue); init_completion(&service->receivemsg_queue_new); INIT_LIST_HEAD(&service->remote_list); init_completion(&service->flush_done); } /* Caller of this function must hold service_mutex. */ static struct easelcomm_service *easelcomm_create_service(int id) { struct easelcomm_service *service; if (easelcomm_service_list[id] != NULL) return easelcomm_service_list[id]; service = kzalloc(sizeof(struct easelcomm_service), GFP_KERNEL); if (service == NULL) { dev_err(easelcomm_miscdev.this_device, "could not allocate service %u\n", id); return NULL; } easelcomm_init_service(service, id); easelcomm_service_list[id] = service; return service; } /* Command received from remote, dispatch. */ static void easelcomm_handle_command(struct easelcomm_cmd_header *cmdhdr) { struct easelcomm_service *service; char *cmdargs = (char *)cmdhdr + sizeof(struct easelcomm_cmd_header); /* * Any command can inform the server that remote is ready, not just * LINK_INIT. */ if (!easelcomm_is_client()) easelcomm_cmd_channel_remote_set_ready(); switch(cmdhdr->command_code) { case EASELCOMM_CMD_LINK_INIT: easelcomm_handle_cmd_link_init( cmdargs, cmdhdr->command_arg_len); return; case EASELCOMM_CMD_LINK_SHUTDOWN: easelcomm_handle_cmd_link_shutdown(); return; case EASELCOMM_CMD_ACK_SHUTDOWN: easelcomm_handle_cmd_ack_shutdown(); return; default: break; } if (cmdhdr->service_id >= EASELCOMM_SERVICE_COUNT) { dev_err(easelcomm_miscdev.this_device, "invalid service ID %u received\n", cmdhdr->service_id); return; } service = easelcomm_service_list[cmdhdr->service_id]; /* * If service has not been initialized, * initialize the service here without userspace ownership. * Userspace ownership could be claimed later. */ if (service == NULL) { mutex_lock(&service_mutex); service = easelcomm_create_service(cmdhdr->service_id); mutex_unlock(&service_mutex); if (service == NULL) { dev_err(easelcomm_miscdev.this_device, "could not handle cmd %d for service %u\n", cmdhdr->command_code, cmdhdr->service_id); return; } } switch (cmdhdr->command_code) { case EASELCOMM_CMD_SEND_MSG: easelcomm_handle_cmd_send_msg( service, cmdargs, cmdhdr->command_arg_len); break; case EASELCOMM_CMD_DMA_SG: easelcomm_handle_cmd_dma_sg( service, cmdargs, cmdhdr->command_arg_len); break; case EASELCOMM_CMD_DMA_XFER: easelcomm_handle_cmd_dma_xfer( service, cmdargs, cmdhdr->command_arg_len); break; case EASELCOMM_CMD_DMA_DONE: easelcomm_handle_cmd_dma_done( service, cmdargs, cmdhdr->command_arg_len); break; case EASELCOMM_CMD_FLUSH_SERVICE: easelcomm_handle_cmd_flush_service(service); break; case EASELCOMM_CMD_FLUSH_SERVICE_DONE: easelcomm_handle_cmd_flush_service_done(service); break; case EASELCOMM_CMD_CLOSE_SERVICE: easelcomm_handle_cmd_close_service(service); break; default: dev_err(easelcomm_miscdev.this_device, "svc %u invalid command code %u received\n", cmdhdr->service_id, cmdhdr->command_code); } } /* * Bump the consumer sequence number in the local command channel state, * indicating that we've consumed messages prior to the new number in * sequence. The sequence number cannot be the reserved "command buffer * wrapped" marker; bump twice if needed to avoid. * * Not locked, but there's only one thread handling local command channel * data, in worker context. */ static void easelcomm_cmd_channel_bump_consumer_seqnbr( struct easelcomm_cmd_channel_local *channel) { channel->consumer_seqnbr_next++; if (channel->consumer_seqnbr_next == CMD_BUFFER_WRAP_MARKER) channel->consumer_seqnbr_next++; } /* * Interrupt for new local command channel data ready was received. This * function is called on a workqueue worker. Process any new command channel * data found. * * Not locked, but there's only one thread handling local command channel * data, in worker context. */ void easelcomm_cmd_channel_data_handler(void) { struct easelcomm_cmd_channel_local *channel = &cmd_channel_local; struct easelcomm_cmd_channel_header *channel_buf_hdr = (struct easelcomm_cmd_channel_header *) channel->buffer; /* While we haven't caught up to producer in sequence #s processed. */ while (channel->consumer_seqnbr_next != channel_buf_hdr->producer_seqnbr_next) { struct easelcomm_cmd_header *cmdhdr = (struct easelcomm_cmd_header *)channel->readp; uint32_t saved_cmd_len; dev_dbg(easelcomm_miscdev.this_device, "cmdchan consumer loop prodseq=%llu consseq=%llu off=%lx\n", channel_buf_hdr->producer_seqnbr_next, channel->consumer_seqnbr_next, channel->readp - channel->buffer); /* * If producer dropped a wrap marker at the current position * then wrap our read pointer and let the producer know we * wrapped and both sides are ready to continue at the top of * the buffer. */ if (cmdhdr->sequence_nbr == CMD_BUFFER_WRAP_MARKER) { dev_dbg(easelcomm_miscdev.this_device, "cmdchan consumer wrap at off=%lx\n", channel->readp - channel->buffer); channel->readp = channel->buffer + sizeof(struct easelcomm_cmd_channel_header); /* Send consumer wrapped IRQ to remote */ easelcomm_hw_send_wrap_interrupt(); /* Wrapping consumes a seqnbr */ easelcomm_cmd_channel_bump_consumer_seqnbr(channel); continue; } saved_cmd_len = READ_ONCE(cmdhdr->command_arg_len); dev_dbg(easelcomm_miscdev.this_device, "cmdchan recv cmd seq=%llu svc=%u cmd=%u len=%u off=%lx\n", cmdhdr->sequence_nbr, cmdhdr->service_id, cmdhdr->command_code, saved_cmd_len, channel->readp - channel->buffer); if (sizeof(struct easelcomm_cmd_header) + saved_cmd_len > EASELCOMM_CMD_CHANNEL_SIZE) { dev_err(easelcomm_miscdev.this_device, "command channel corruption detected: seq=%llu svc=%u cmd=%u len=%u off=%lx\n", cmdhdr->sequence_nbr, cmdhdr->service_id, cmdhdr->command_code, saved_cmd_len, channel->readp - channel->buffer); break; } if (cmdhdr->sequence_nbr != channel->consumer_seqnbr_next) { dev_err(easelcomm_miscdev.this_device, "command channel corruption detected: expected seq# %llu, got %llu\n", channel->consumer_seqnbr_next, cmdhdr->sequence_nbr); break; } /* Process the command. */ easelcomm_handle_command(cmdhdr); /* Post-process double check */ if (saved_cmd_len != cmdhdr->command_arg_len) { dev_err(easelcomm_miscdev.this_device, "command channel corruption detected: off=%lx expected len %u got %u\n", channel->readp - channel->buffer, saved_cmd_len, cmdhdr->command_arg_len); break; } channel->readp += sizeof(struct easelcomm_cmd_header) + saved_cmd_len; /* 8-byte-align next entry pointer */ if ((uintptr_t)channel->readp & 0x7) channel->readp += 8 - ((uintptr_t)channel->readp & 0x7); easelcomm_cmd_channel_bump_consumer_seqnbr(channel); } dev_dbg(easelcomm_miscdev.this_device, "cmdchan consumer exiting prodseq=%llu consseq=%llu off=%lx\n", channel_buf_hdr->producer_seqnbr_next, channel->consumer_seqnbr_next, channel->readp - channel->buffer); } EXPORT_SYMBOL(easelcomm_cmd_channel_data_handler); /* * Interrupt from remote indicating remote has followed our channel buffer * wrap received. This function called in workqueue worker context. Wakeup * the command producer that is waiting on the remote to follow the wrap. */ void easelcomm_cmd_channel_wrap_handler(void) { struct easelcomm_cmd_channel_remote *channel = &cmd_channel_remote; dev_dbg(easelcomm_miscdev.this_device, "cmdchan remote wrap IRQ received\n"); complete(&channel->wrap_ready); } EXPORT_SYMBOL(easelcomm_cmd_channel_wrap_handler); /* Device file open. Allocate a user state structure. */ static int easelcomm_open(struct inode *inode, struct file *file) { struct easelcomm_user_state *user_state; if (!easelcomm_up) return -ENODEV; user_state = kzalloc(sizeof(struct easelcomm_user_state), GFP_KERNEL); if (!user_state) return -ENOMEM; file->private_data = user_state; return 0; } /* Initiate service shutdown from fd release or ioctl. */ static void easelcomm_initiate_service_shutdown( struct easelcomm_service *service) { bool already_shutdown; mutex_lock(&service->lock); already_shutdown = service->shutdown_local; mutex_unlock(&service->lock); if (already_shutdown) { dev_dbg(easelcomm_miscdev.this_device, "svc %u already shutdown\n", service->service_id); return; } dev_dbg(easelcomm_miscdev.this_device, "svc %u initiate shutdown\n", service->service_id); easelcomm_handle_service_shutdown(service, true); dev_dbg(easelcomm_miscdev.this_device, "svc %u send cmd CLOSE_SERVICE\n", service->service_id); easelcomm_send_cmd_noargs(service, EASELCOMM_CMD_CLOSE_SERVICE); } /* Device file descriptor release, initiate service shutdown and cleanup */ static int easelcomm_release(struct inode *inode, struct file *file) { struct easelcomm_user_state *user_state = file->private_data; if (user_state) { struct easelcomm_service *service = user_state->service; if (service) { dev_dbg(easelcomm_miscdev.this_device, "svc %u release\n", service->service_id); if (easelcomm_up) easelcomm_initiate_service_shutdown(service); mutex_lock(&service->lock); service->user = NULL; mutex_unlock(&service->lock); } kfree(file->private_data); file->private_data = NULL; } return 0; } /* * Bump the producer next sequence number in the remote command channel buffer * header. After this, the remote side can then see that entries prior to * that sequence number are present. The sequence number cannot be the * reserved "command buffer wrapped" marker; bump twice to avoid if needed. * * Call with remote channel producer mutex held. */ static int easelcomm_cmd_channel_bump_producer_seqnbr( struct easelcomm_cmd_channel_remote *channel) { int ret; channel->write_seqnbr++; if (channel->write_seqnbr == CMD_BUFFER_WRAP_MARKER) channel->write_seqnbr++; ret = easelcomm_hw_remote_write( &channel->write_seqnbr, sizeof(uint64_t), (uint64_t)EASELCOMM_CMDCHAN_PRODUCER_SEQNBR); return ret; } /* * Wrap around remote's buffer. If producer does not receive a signal that * consumer has wrapped around within predefined time, it will retry sending * wrap command if enabled. * * Note: this retry mechanism is done by writing the wrap marker to the top * of remote's cmd channel, which could, in the worst case, overwrite the * unconsumed command at the top of cmd channel, causing all unconsumed * commands behind it to be discarded. * * Called with remote channel mutex held. * Caller is responsible for releasing the mutex. * * @channel: the pointer to remote channel * * @Return: 0 - sent wrap and remote completed wrapping * negative - on error */ static int easelcomm_cmd_channel_send_wrap( struct easelcomm_cmd_channel_remote *channel) { int ret = 0; int attempted = 0; long remaining; uint64_t wrap_marker = CMD_BUFFER_WRAP_MARKER; { if (attempted > 0) { dev_warn(easelcomm_miscdev.this_device, "%s: retrying after %d attempts", __func__, attempted); } attempted++; /* Write the "buffer wrapped" marker in place of sequence # */ dev_dbg(easelcomm_miscdev.this_device, "cmdchan producer wrap at off=%llx seq=%llu\n", channel->write_offset, channel->write_seqnbr); ret = easelcomm_hw_remote_write( &wrap_marker, sizeof(wrap_marker), channel->write_offset); if (ret) goto exit; /* * Bump the producer seqnbr for the wrap marker (so consumer * knows its there) and send IRQ to remote to consume the new * data. * * Consumer will process entries up to and including the wrap * marker and then send a "wrapped" interrupt that wakes up * the wrap_ready completion. */ ret = easelcomm_cmd_channel_bump_producer_seqnbr(channel); if (ret) goto exit; ret = easelcomm_hw_send_data_interrupt(); if (ret) goto exit; channel->write_offset = (sizeof(struct easelcomm_cmd_channel_header) + 7) & ~0x7; /* Wait for remote to catch up. IRQ from remote wakes this. */ remaining = wait_for_completion_interruptible_timeout( &channel->wrap_ready, msecs_to_jiffies(CMDCHAN_WRAP_DONE_TIMEOUT_MS)); if (remaining > 0) { ret = 0; dev_dbg(easelcomm_miscdev.this_device, "cmdchan wrap completed, new off=%llx\n", channel->write_offset); goto exit; } if (remaining < 0) { /* waiting was interrupted */ dev_err(easelcomm_miscdev.this_device, "remote channel did not catch up: reason interrupted off=%llx seq=%llu\n", channel->write_offset, channel->write_seqnbr); ret = remaining; goto exit; } /* remaining jiffies is 0; timed out */ dev_err(easelcomm_miscdev.this_device, "remote channel did not catch up: reason timeout off=%llx seq=%llu\n", channel->write_offset, channel->write_seqnbr); ret = -ETIMEDOUT; } exit: return ret; } /* * Start the process of writing a new command to the remote command channel. * Wait for channel init'ed and grab the mutex. Wraparound the buffer if * needed. Write the command header. * If no error then return zero and channel mutex is locked on return. * If error then return non-zero and channel mutex is unlocked. */ int easelcomm_start_cmd( struct easelcomm_service *service, int command_code, size_t command_arg_len) { struct easelcomm_cmd_channel_remote *channel = &cmd_channel_remote; struct easelcomm_cmd_header cmdhdr; size_t cmdbuf_size = sizeof(struct easelcomm_cmd_header) + command_arg_len; int ret; /* * If never enough room for the command plus an 8-byte wrap marker, * bail. */ if (cmdbuf_size > EASELCOMM_CMD_CHANNEL_SIZE - 8 - sizeof(struct easelcomm_cmd_channel_header)) return -EINVAL; ret = easelcomm_wait_channel_initialized(channel); /* If no error then channel->mutex is locked */ if (ret) return ret; /* Choose a spot for the new entry, wrap around if needed. */ if (channel->write_offset + cmdbuf_size > EASELCOMM_CMD_CHANNEL_SIZE - 8 - sizeof(struct easelcomm_cmd_channel_header)) { ret = easelcomm_cmd_channel_send_wrap(channel); if (ret) goto error; } dev_dbg(easelcomm_miscdev.this_device, "cmdchan producer sending cmd seq#%llu svc=%u cmd=%u arglen=%zu off=%llx\n", channel->write_seqnbr, service->service_id, command_code, command_arg_len, channel->write_offset); cmdhdr.service_id = service->service_id; cmdhdr.sequence_nbr = channel->write_seqnbr; cmdhdr.command_code = command_code; cmdhdr.command_arg_len = (uint32_t)command_arg_len; /* * Send the command header. Subsequent calls to * easelcomm_append_cmd_args() and easelcomm_send_cmd() will finish * the in-progress command. */ ret = easelcomm_hw_remote_write( &cmdhdr, sizeof(cmdhdr), channel->write_offset); if (ret) goto error; /* Advance write offset */ channel->write_offset += sizeof(cmdhdr); return 0; error: mutex_unlock(&channel->mutex); return ret; } EXPORT_SYMBOL(easelcomm_start_cmd); /* * Append arguments to the in-progress command being sent. * Call with remote channel producer mutex held. */ int easelcomm_append_cmd_args( struct easelcomm_service *service, void *cmd_args, size_t cmd_args_len) { struct easelcomm_cmd_channel_remote *channel = &cmd_channel_remote; int ret = easelcomm_hw_remote_write( cmd_args, cmd_args_len, channel->write_offset); if (ret) { mutex_unlock(&channel->mutex); return ret; } channel->write_offset += cmd_args_len; return 0; } EXPORT_SYMBOL(easelcomm_append_cmd_args); /* Start and send a command with no arguments. */ int easelcomm_send_cmd_noargs( struct easelcomm_service *service, int command_code) { int ret; ret = easelcomm_start_cmd(service, command_code, 0); if (ret) return ret; return easelcomm_send_cmd(service); } /* * Finish sending an in-progress command to the remote. */ int easelcomm_send_cmd(struct easelcomm_service *service) { struct easelcomm_cmd_channel_remote *channel = &cmd_channel_remote; int ret; /* * Bump the producer sequence number. Remote may now see and process * the new entry as soon as this value is visible to it. */ ret = easelcomm_cmd_channel_bump_producer_seqnbr(channel); if (ret) goto out; /* Send data ready IRQ to remote */ ret = easelcomm_hw_send_data_interrupt(); if (ret) goto out; /* Bump write offset for next command, 8-byte-integer-aligned */ if (channel->write_offset & 0x7) channel->write_offset += 8 - (channel->write_offset & 0x7); out: mutex_unlock(&channel->mutex); return ret; } EXPORT_SYMBOL(easelcomm_send_cmd); /* * Return next message ID in sequence for this service. Message ID zero is * not used, bump twice to avoid if needed. */ static easelcomm_msgid_t easelcomm_next_msgid( struct easelcomm_service *service) { easelcomm_msgid_t next_id; mutex_lock(&service->lock); next_id = ++service->next_id; if (!next_id) next_id = ++service->next_id; mutex_unlock(&service->lock); return next_id; } /* * Handle SENDMSG ioctl, the message descriptor send part of easelcomm * sendMessage*() and sendReply() calls. */ static int easelcomm_send_message_ioctl( struct easelcomm_service *service, struct easelcomm_kmsg_desc __user *pmsg_desc) { struct easelcomm_kmsg_desc *msg_desc; struct easelcomm_message_metadata *msg_metadata = NULL; int ret = 0; msg_desc = kmalloc(sizeof(struct easelcomm_kmsg_desc), GFP_KERNEL); if (!msg_desc) return -ENOMEM; if (copy_from_user(msg_desc, (void __user *) pmsg_desc, sizeof(struct easelcomm_kmsg_desc))) { ret = -EFAULT; goto out_freedesc; } if (msg_desc->message_size > EASELCOMM_MAX_MESSAGE_SIZE) { dev_err(easelcomm_miscdev.this_device, "%s: svc:%u msg size %d is larger than max size %d\n", __func__, service->service_id, msg_desc->message_size, EASELCOMM_MAX_MESSAGE_SIZE); ret = -EINVAL; goto out_freedesc; } /* Assign a message ID to the new message and add to local list. */ msg_desc->message_id = easelcomm_next_msgid(service); msg_metadata = easelcomm_add_metadata(service, TYPE_LOCAL, (struct easelcomm_kmsg *)msg_desc); if (!msg_metadata) { ret = -ENOMEM; goto out_freedesc; } /* * After a successful call to the above, msg_desc is pointed to by * msg_metadata and the memory is managed along with the other * metadata. */ dev_dbg(easelcomm_miscdev.this_device, "SENDMSG msg %u:l%llu\n", service->service_id, msg_desc->message_id); easelcomm_dump_message(msg_metadata); /* Copy the updated msg desc with message ID to user. */ if (copy_to_user((void __user *) pmsg_desc, msg_desc, sizeof(struct easelcomm_kmsg_desc))) { easelcomm_drop_reference(service, msg_metadata, true); return -EFAULT; } /* * If this is a reply then we are done with the replied-to remote * message. */ if (msg_metadata->msg->desc.in_reply_to) { struct easelcomm_message_metadata *orig_msg_metadata = easelcomm_find_remote_message( service, msg_metadata->msg->desc.in_reply_to); if (!orig_msg_metadata) { dev_dbg(easelcomm_miscdev.this_device, "msg %u:l%llu replied-to msg r%llu not found\n", service->service_id, msg_desc->message_id, msg_metadata->msg->desc.in_reply_to); } else { dev_dbg(easelcomm_miscdev.this_device, "msg %u:l%llu is reply to msg r%llu\n", service->service_id, msg_desc->message_id, msg_metadata->msg->desc.in_reply_to); easelcomm_drop_reference( service, orig_msg_metadata, true); } } easelcomm_drop_reference(service, msg_metadata, false); return 0; out_freedesc: kfree(msg_desc); return ret; } /* Handle RECVMSG ioctl / easelcomm receiveMessage() call. */ static int easelcomm_wait_message( struct easelcomm_service *service, struct easelcomm_kmsg_desc __user *msgp) { struct easelcomm_message_metadata *msg_metadata = NULL; struct easelcomm_kmsg_desc msg_temp; int ret = 0; bool has_timeout = false; long remaining = 0; dev_dbg(easelcomm_miscdev.this_device, "WAITMSG svc %u\n", service->service_id); if (copy_from_user(&msg_temp, (void __user *) msgp, sizeof(struct easelcomm_kmsg_desc))) return -EFAULT; dev_dbg(easelcomm_miscdev.this_device, "%s: timeout_ms=%d\n", __func__, msg_temp.wait.timeout_ms); if (msg_temp.wait.timeout_ms >= 0) { has_timeout = true; remaining = (long)msecs_to_jiffies( (unsigned int)msg_temp.wait.timeout_ms); dev_dbg(easelcomm_miscdev.this_device, "%s: remaining=%ld\n", __func__, remaining); } while (!msg_metadata) { mutex_lock(&service->lock); if (service->shutdown_local || (easelcomm_is_client() && service->shutdown_remote)) { /* Service shutdown initiated locally or remotely. */ dev_dbg(easelcomm_miscdev.this_device, "WAITMSG svc %u returning shutdown local=%d remote=%d\n", service->service_id, service->shutdown_local, service->shutdown_remote); service->shutdown_remote = false; mutex_unlock(&service->lock); return -ESHUTDOWN; } /* grab first entry off receiveMessage queue. */ msg_metadata = list_first_entry_or_null( &service->receivemsg_queue, struct easelcomm_message_metadata, rcvq_list); if (msg_metadata) break; mutex_unlock(&service->lock); if (has_timeout) { remaining = wait_for_completion_interruptible_timeout( &service->receivemsg_queue_new, (unsigned long)remaining); if (remaining == 0) ret = -ETIMEDOUT; if (remaining < 0) ret = remaining; } else { ret = wait_for_completion_interruptible( &service->receivemsg_queue_new); } if (ret) return ret; } dev_dbg(easelcomm_miscdev.this_device, "WAITMSG svc %u returning msg %u:r%llu\n", service->service_id, service->service_id, msg_metadata->msg->desc.message_id); /* remove from receive queue, grab a ref while copy to user */ list_del(&msg_metadata->rcvq_list); msg_metadata->queued = false; msg_metadata->reference_count++; mutex_unlock(&service->lock); /* Copy the message desc to the caller */ if (copy_to_user(msgp, msg_metadata->msg, sizeof(struct easelcomm_kmsg_desc))) ret = -EFAULT; easelcomm_drop_reference(service, msg_metadata, false); return ret; } /* * Handle WAITREPLY ioctl / reply waiting part of easelcomm * sendMessageReceiveReply() call. */ static int easelcomm_wait_reply( struct easelcomm_service *service, struct easelcomm_kmsg_desc __user *pmsg_desc) { struct easelcomm_kmsg_desc orig_msg_desc; struct easelcomm_message_metadata *orig_msg_metadata; struct easelcomm_message_metadata *msg_metadata = NULL; int ret = 0; bool has_timeout = false; long remaining = 0; if (copy_from_user(&orig_msg_desc, (void __user *) pmsg_desc, sizeof(struct easelcomm_kmsg_desc))) return -EFAULT; dev_dbg(easelcomm_miscdev.this_device, "%s: timeout_ms=%d\n", __func__, orig_msg_desc.wait.timeout_ms); if (orig_msg_desc.wait.timeout_ms >= 0) { has_timeout = true; remaining = (long)msecs_to_jiffies( (unsigned int)orig_msg_desc.wait.timeout_ms); dev_dbg(easelcomm_miscdev.this_device, "%s: remaining=%ld\n", __func__, remaining); } /* Grab a reference to the original message. */ orig_msg_metadata = easelcomm_find_local_message( service, orig_msg_desc.message_id); if (!orig_msg_metadata) { dev_dbg(easelcomm_miscdev.this_device, "WAITREPLY msg %u:l%llu not found\n", service->service_id, orig_msg_desc.message_id); return -EINVAL; } dev_dbg(easelcomm_miscdev.this_device, "WAITREPLY msg %u:l%llu\n", service->service_id, orig_msg_desc.message_id); if (has_timeout) { remaining = wait_for_completion_interruptible_timeout( &orig_msg_metadata->reply_received, (unsigned long)remaining); if (remaining == 0) ret = -ETIMEDOUT; if (remaining < 0) ret = remaining; } else { ret = wait_for_completion_interruptible( &orig_msg_metadata->reply_received); } if (ret) goto out; if (orig_msg_metadata->flushing) { dev_dbg(easelcomm_miscdev.this_device, "WAITREPLY msg %u:l%llu flushed\n", service->service_id, orig_msg_desc.message_id); goto out; } msg_metadata = easelcomm_grab_reply_message( service, orig_msg_metadata); /* * If completion was waken up on receiving the reply, * msg_metadata really shouldn't be null. * If completion was waken up on receiving shutdown, we should * return to user immediately. */ if (!msg_metadata) { dev_err(easelcomm_miscdev.this_device, "WAITREPLY msg %u:l%llu no matching reply\n", service->service_id, orig_msg_desc.message_id); /* * Here we return -ESHUTDOWN. See above comment and * easelcomm_handle_service_shutdown(). */ ret = -ESHUTDOWN; goto out; } /* Copy the reply message descriptor to the caller */ if (copy_to_user(pmsg_desc, &msg_metadata->msg->desc, sizeof(struct easelcomm_kmsg_desc))) { ret = -EFAULT; goto out; } dev_dbg(easelcomm_miscdev.this_device, "WAITREPLY msg %u:l%llu returning msg r%llu\n", service->service_id, orig_msg_desc.message_id, msg_metadata->msg->desc.message_id); ret = 0; out: /* Discard reply message if error retrieving it. */ if (msg_metadata) easelcomm_drop_reference(service, msg_metadata, ret); /* Always discard the original message, we're done with it. */ easelcomm_drop_reference(service, orig_msg_metadata, true); return ret; } /* Initiate a flush of service on both sides of the link. */ static void easelcomm_flush_service(struct easelcomm_service *service) { int ret; easelcomm_flush_local_service(service); dev_dbg(easelcomm_miscdev.this_device, "send cmd FLUSH_SERVICE svc %u\n", service->service_id); ret = easelcomm_send_cmd_noargs(service, EASELCOMM_CMD_FLUSH_SERVICE); if (ret) return; ret = wait_for_completion_interruptible_timeout( &service->flush_done, msecs_to_jiffies(SERVICE_FLUSH_DONE_TIMEOUT)); if (ret <= 0) dev_err(easelcomm_miscdev.this_device, "service flush done wait aborted svc %u\n", service->service_id); } /* Handle REGISTER ioctl, register an open fd for an Easel service. */ static int easelcomm_register(struct easelcomm_user_state *user_state, unsigned int service_id) { struct easelcomm_service *service; if (service_id >= EASELCOMM_SERVICE_COUNT) return -EINVAL; mutex_lock(&service_mutex); service = easelcomm_service_list[service_id]; if (service == NULL) { service = easelcomm_create_service(service_id); if (service == NULL) { mutex_unlock(&service_mutex); dev_err(easelcomm_miscdev.this_device, "could not create service %u\n", service_id); return -ENOMEM; } } mutex_unlock(&service_mutex); mutex_lock(&service->lock); if (service->user) { mutex_unlock(&service->lock); return -EBUSY; } user_state->service = service; service->user = user_state; service->shutdown_local = false; service->shutdown_remote = false; mutex_unlock(&service->lock); dev_dbg(easelcomm_miscdev.this_device, "REGISTER svc %u\n", service_id); /* * New client flushes any messages generated from activity by previous * clients. * * New server handles any client messages that were waiting for the * server to startup and process, does not flush at start/restart. * Server can explicitly call the flush ioctl if wanted, but with the * understanding any clients starting at the same time may have their * traffic dropped. */ if (easelcomm_is_client()) easelcomm_flush_service(service); return 0; } /* * Handle READDATA ioctl, the read message data part of easelcomm * receiveMessage() and sendMessageReceiveReply() calls. */ static int easelcomm_read_msgdata( struct easelcomm_service *service, struct easelcomm_kbuf_desc *buf_desc) { struct easelcomm_message_metadata *msg_metadata; int ret = 0; dev_dbg(easelcomm_miscdev.this_device, "READDATA msg %u:r%llu buf=%p\n", service->service_id, buf_desc->message_id, buf_desc->buf); msg_metadata = easelcomm_find_remote_message(service, buf_desc->message_id); if (!msg_metadata) { dev_err(easelcomm_miscdev.this_device, "READDATA msg %u:r%llu not found\n", service->service_id, buf_desc->message_id); return -EINVAL; } if (buf_desc->buf_size && buf_desc->buf_size != msg_metadata->msg->desc.message_size) { dev_err(easelcomm_miscdev.this_device, "READDATA descriptor buffer size %u doesn't match message %u:r%llu size %u\n", buf_desc->buf_size, service->service_id, buf_desc->message_id, msg_metadata->msg->desc.message_size); ret = -EINVAL; goto out; } if (buf_desc->buf_size) { if (copy_to_user(buf_desc->buf, &msg_metadata->msg->message_data, buf_desc->buf_size)) { ret = -EFAULT; goto out; } } /* * If no DMA transfer and no reply needed then we're done with this * message. */ if (!msg_metadata->msg->desc.dma_buf_size && !msg_metadata->msg->desc.need_reply) { easelcomm_drop_reference(service, msg_metadata, true); return 0; } out: easelcomm_drop_reference(service, msg_metadata, false); return ret; } /* * Handle WRITEDATA ioctl, the message data write portion of easelcomm * sendMessage*() and sendReply() calls. */ static int easelcomm_write_msgdata( struct easelcomm_service *service, struct easelcomm_kbuf_desc *buf_desc) { struct easelcomm_message_metadata *msg_metadata; char *databuf = NULL; bool discard_message = true; int ret; dev_dbg(easelcomm_miscdev.this_device, "WRITEDATA msg %u:l%llu buf=%p\n", service->service_id, buf_desc->message_id, buf_desc->buf); msg_metadata = easelcomm_find_local_message(service, buf_desc->message_id); if (!msg_metadata) { dev_err(easelcomm_miscdev.this_device, "WRITEDATA msg %u:l%llu not found\n", service->service_id, buf_desc->message_id); return -EINVAL; } if (buf_desc->buf_size && buf_desc->buf_size != msg_metadata->msg->desc.message_size) { dev_err(easelcomm_miscdev.this_device, "WRITEDATA descriptor buffer size %u doesn't match message %u:l%llu size %u\n", buf_desc->buf_size, service->service_id, buf_desc->message_id, msg_metadata->msg->desc.message_size); ret = -EINVAL; goto out; } if (buf_desc->buf_size) { databuf = kmalloc(buf_desc->buf_size, GFP_KERNEL); if (!databuf) { ret = -ENOMEM; goto out; } if (copy_from_user( databuf, buf_desc->buf, buf_desc->buf_size)) { ret = -EFAULT; goto out; } } dev_dbg(easelcomm_miscdev.this_device, "send cmd SEND_MSG msg %u:l%llu\n", service->service_id, msg_metadata->msg->desc.message_id); ret = easelcomm_start_cmd( service, EASELCOMM_CMD_SEND_MSG, sizeof(struct easelcomm_kmsg_desc) + buf_desc->buf_size); if (ret) goto out; ret = easelcomm_append_cmd_args( service, &msg_metadata->msg->desc, sizeof(struct easelcomm_kmsg_desc)); if (ret) goto out; if (buf_desc->buf_size) { ret = easelcomm_append_cmd_args( service, databuf, buf_desc->buf_size); if (ret) goto out; } ret = easelcomm_send_cmd(service); if (ret) goto out; /* * If no DMA transfer and no reply needed then we're done with this * message. */ discard_message = !msg_metadata->msg->desc.dma_buf_size && !msg_metadata->msg->desc.need_reply; out: /* * If success and no DMA transfer and no reply needed, or if error, * then free the message. */ easelcomm_drop_reference(service, msg_metadata, discard_message); kfree(databuf); return ret; } /* * Client calls this upon receipt of BOOTSTRAP_READY IRQ * from server. Call MNH host driver to setup iATU mappings for the command * channels and send LINK_INIT. */ int easelcomm_client_remote_cmdchan_ready_handler(void) { int ret; /* Sync up client and server sides using MNH host driver APIs */ ret = easelcomm_hw_ap_setup_cmdchans(); if (ret) return ret; easelcomm_cmd_channel_remote_set_ready(); /* Client can now send link init to server */ dev_dbg(easelcomm_miscdev.this_device, "send cmd LINK_INIT\n"); return easelcomm_send_cmd_noargs( &link_service, EASELCOMM_CMD_LINK_INIT); } EXPORT_SYMBOL(easelcomm_client_remote_cmdchan_ready_handler); /* * Handle ioctls that take a struct easelcomm_kbuf_desc * argument, convert * field layout and buf field pointer for 32-bit compat if needed. */ static int easelcomm_ioctl_kbuf_desc( struct easelcomm_service *service, unsigned int cmd, unsigned long arg, bool compat) { struct easelcomm_kbuf_desc kbuf_desc; int ret; if (compat) { #ifdef CONFIG_COMPAT struct easelcomm_compat_kbuf_desc compat_kbuf_desc; if (copy_from_user( &compat_kbuf_desc, (void __user *) arg, sizeof(struct easelcomm_compat_kbuf_desc))) return -EFAULT; kbuf_desc.message_id = compat_kbuf_desc.message_id; kbuf_desc.buf = compat_ptr(compat_kbuf_desc.buf); kbuf_desc.buf_size = compat_kbuf_desc.buf_size; kbuf_desc.buf_type = compat_kbuf_desc.buf_type; kbuf_desc.dma_buf_fd = compat_kbuf_desc.dma_buf_fd; #else return -EINVAL; #endif } else { if (copy_from_user( &kbuf_desc, (void __user *) arg, sizeof(struct easelcomm_kbuf_desc))) return -EFAULT; } switch (cmd) { case EASELCOMM_IOC_WRITEDATA: ret = easelcomm_write_msgdata(service, &kbuf_desc); break; case EASELCOMM_IOC_READDATA: ret = easelcomm_read_msgdata(service, &kbuf_desc); break; case EASELCOMM_IOC_SENDDMA: ret = easelcomm_send_dma(service, &kbuf_desc); break; case EASELCOMM_IOC_RECVDMA: ret = easelcomm_receive_dma(service, &kbuf_desc); break; default: ret = -EINVAL; } return ret; } /* 32-bit and 64-bit userspace ioctl handling, dispatch. */ static long easelcomm_ioctl_common( struct file *file, unsigned int cmd, unsigned long arg, bool compat) { int ret = 0; struct easelcomm_user_state *user_state = file->private_data; struct easelcomm_service *service; if (_IOC_TYPE(cmd) != EASELCOMM_IOC_MAGIC) return -EINVAL; if (WARN_ON(!user_state)) return -EINVAL; if (!easelcomm_up) return -ESHUTDOWN; /* REGISTER is the only ioctl that doesn't need a service registered. */ if (cmd == EASELCOMM_IOC_REGISTER) return easelcomm_register(user_state, (unsigned int) arg); service = user_state->service; if (!service) { dev_err(easelcomm_miscdev.this_device, "user has not registered a service for the file\n"); return -EINVAL; } switch (cmd) { case EASELCOMM_IOC_SENDMSG: ret = easelcomm_send_message_ioctl( service, (struct easelcomm_kmsg_desc *)arg); break; case EASELCOMM_IOC_WRITEDATA: case EASELCOMM_IOC_READDATA: case EASELCOMM_IOC_SENDDMA: case EASELCOMM_IOC_RECVDMA: /* Convert 32-bit kbuf_desc argument if needed and dispatch. */ ret = easelcomm_ioctl_kbuf_desc(service, cmd, arg, compat); break; case EASELCOMM_IOC_WAITMSG: ret = easelcomm_wait_message( service, (struct easelcomm_kmsg_desc *)arg); break; case EASELCOMM_IOC_WAITREPLY: ret = easelcomm_wait_reply( service, (struct easelcomm_kmsg_desc *)arg); break; case EASELCOMM_IOC_SHUTDOWN: easelcomm_initiate_service_shutdown(service); break; case EASELCOMM_IOC_FLUSH: easelcomm_flush_service(service); break; default: ret = -ENOIOCTLCMD; break; } return ret; } /* 64-bit ioctl handling. */ static long easelcomm_ioctl( struct file *file, unsigned int cmd, unsigned long arg) { return easelcomm_ioctl_common(file, cmd, arg, false); } #ifdef CONFIG_COMPAT /* 32-bit ioctl handling. */ static long easelcomm_compat_ioctl( struct file *file, unsigned int cmd, unsigned long arg) { unsigned int cmd_nr = _IOC_NR(cmd); int ret; /* Fixup pointer argument size for ioctls 1..7 */ if ((cmd_nr >= 1 && cmd_nr <= 7) && _IOC_SIZE(cmd) == sizeof(compat_uptr_t)) { cmd &= ~(_IOC_SIZEMASK << _IOC_SIZESHIFT); cmd |= sizeof(void *) << _IOC_SIZESHIFT; } switch (cmd) { case EASELCOMM_IOC_SENDMSG: case EASELCOMM_IOC_WAITMSG: case EASELCOMM_IOC_WAITREPLY: case EASELCOMM_IOC_WRITEDATA: case EASELCOMM_IOC_READDATA: case EASELCOMM_IOC_SENDDMA: case EASELCOMM_IOC_RECVDMA: /* pointer argument, fixup */ ret = easelcomm_ioctl_common(file, cmd, (unsigned long)compat_ptr(arg), true); break; case EASELCOMM_IOC_REGISTER: case EASELCOMM_IOC_SHUTDOWN: case EASELCOMM_IOC_FLUSH: /* no ioctl argument conversion needed */ ret = easelcomm_ioctl_common(file, cmd, arg, true); break; default: ret = -ENOIOCTLCMD; break; } return ret; } #endif /* CONFIG_COMPAT */ /* Init local state for remote command channel. */ static void easelcomm_init_cmd_channel_remote( struct easelcomm_cmd_channel_remote *channel) { mutex_init(&channel->mutex); channel->initialized = false; init_completion(&channel->init_state_changed); init_completion(&channel->wrap_ready); /* Write pointer starts after the channel header */ channel->write_offset = (sizeof(struct easelcomm_cmd_channel_header) + 7) & ~0x7; channel->write_seqnbr = 0; } /* Init local command channel. */ static int easelcomm_init_cmd_channel_local( struct easelcomm_cmd_channel_local *channel, void *cmdchan_buffer) { channel->buffer = cmdchan_buffer; if (!channel->buffer) return -ENOMEM; channel->readp = channel->buffer + sizeof(struct easelcomm_cmd_channel_header); /* next sequence # to be produced/consumed starts at zero */ channel->consumer_seqnbr_next = 0; ((struct easelcomm_cmd_channel_header *) channel->buffer)->producer_seqnbr_next = 0; return 0; } /* * Callback from h/w layer when PCIe ready (which is immediate for server, * happens on first PCIe host driver probe on client). The h/w layer has * allocated the local command channel buffer (which needs the host PCIe * driver on client), the pointer to which is passed as an argument. Can now * init our local command channel state, and do remote too. */ int easelcomm_init_pcie_ready(void *local_cmdchan_buffer) { int ret; ret = easelcomm_init_cmd_channel_local( &cmd_channel_local, local_cmdchan_buffer); easelcomm_init_cmd_channel_remote(&cmd_channel_remote); return ret; } EXPORT_SYMBOL(easelcomm_init_pcie_ready); /* * Callback for AP/client from h/w layer when PCIe EP is off. */ int easelcomm_pcie_hotplug_out(void) { struct easelcomm_cmd_channel_remote *channel = &cmd_channel_remote; /* * PCIe link is already down, so no need to call easelcomm_stop() to * notify remote, but only call easelcomm_stop_local(). */ easelcomm_stop_local(); mutex_lock(&channel->mutex); channel->initialized = false; mutex_unlock(&channel->mutex); return 0; } EXPORT_SYMBOL(easelcomm_pcie_hotplug_out); static int easelcomm_notify_sys( struct notifier_block *this, unsigned long code, void *unused) { if (code == SYS_RESTART || code == SYS_HALT || code == SYS_POWER_OFF) easelcomm_stop(false); return NOTIFY_DONE; } static struct notifier_block easelcomm_notifier = { .notifier_call = easelcomm_notify_sys, }; static int __init easelcomm_init(void) { int ret; int i; ret = misc_register(&easelcomm_miscdev); if (ret) { dev_err(easelcomm_miscdev.this_device, "misc_register failed\n"); return ret; } dev_info(easelcomm_miscdev.this_device, "registered at misc device minor %d\n", easelcomm_miscdev.minor); mutex_init(&service_mutex); for (i = 0; i < EASELCOMM_SERVICE_COUNT; i++) { easelcomm_service_list[i] = NULL; } /* Initializes link_service. */ easelcomm_init_service(&link_service, EASELCOMM_SERVICE_COUNT); easelcomm_hw_init(); /* Shut down easelcomm on reboot */ ret = register_reboot_notifier(&easelcomm_notifier); if (ret) dev_warn(easelcomm_miscdev.this_device, "register reboot notifier failed\n"); return 0; } static void __exit easelcomm_exit(void) { int i; easelcomm_stop(false); kfree(cmd_channel_local.buffer); misc_deregister(&easelcomm_miscdev); /* * TODO(cjluo): Some of these service struct * could be freed when unregistered. */ for (i = 0; i < EASELCOMM_SERVICE_COUNT; i++) { if (easelcomm_service_list[i] != NULL) { kfree(easelcomm_service_list[i]); easelcomm_service_list[i] = NULL; } } } module_init(easelcomm_init); module_exit(easelcomm_exit); MODULE_AUTHOR("Google Inc."); MODULE_DESCRIPTION("Easel coprocessor communication driver"); MODULE_LICENSE("GPL");