path: root/drivers/misc/google-easel-comm-hw.c

/*
 * Android coprocessor communication hardware access functions
 *
 * Copyright 2016 Google Inc.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

/* #define DEBUG */

#ifdef CONFIG_GOOGLE_EASEL_AP
#define EASELCOMM_AP 1
#else
#define EASELCOMM_EASEL 1
#endif

#ifdef EASELCOMM_EASEL
#include <linux/mnh_pcie_ep.h>
#else
#include "mnh/mnh-pcie.h"
#include "mnh/mnh-sm.h"
#include "mnh/hw-mnh-regs.h"
#endif

#include "google-easel-comm-shared.h"
#include "google-easel-comm-private.h"

#include <linux/io.h>
#include <linux/compiler.h>
#include <linux/completion.h>
#include <linux/dma-mapping.h>
#include <linux/io.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/workqueue.h>

#ifdef EASELCOMM_AP
/* Use only one DMA channel for app requests */
#define APP_DMA_CHAN 1

/* timeout for waiting for bootstrap MSI after hotplug */
#define BOOTSTRAP_TIMEOUT_MS 2000

/* Signaled when server sents bootstrap MSI */
static DECLARE_COMPLETION(bootstrap_done);

/*
 * Mutex serializes thread context access to app_dma* data structures below.
 * Client thread performing the app DMA transfer holds this for the duration
 * of the h/w-layer transfer operation.
 * The DMA IRQ callback accesses these without locking.
 */
static struct mutex app_dma_mutex;
/* ISR signals app DMA done */
static DECLARE_COMPLETION(app_dma_done);
/* Status of last DMA transfer */
static enum mnh_dma_trans_status_t app_dma_status;

/* MNH BAR4 window size (and addressing granularity) 4MB */
#define MNH_BAR4_MASK ((4 * 1024 * 1024) - 1)

/* DMA address of Easel cmdchan buffer */
static uint64_t easel_cmdchan_dma_addr;
/* Offset between iATU inbound 4MB-aligned window and actual buffer start */
static uint32_t easel_cmdchan_pcie_offset;
#endif

/* Common AP/Easel defines */
/* Local cmdchan buffer cpu virtual address */
static void *local_cmdchan_cpu_addr;
/* Local cmdchan buffer DMA address */
static dma_addr_t local_cmdchan_dma_addr;

/* Command channel data ready, call up to generic layer. */
static void easelcomm_hw_cmdchan_data_ready(struct work_struct *work)
{
	easelcomm_cmd_channel_data_handler();
}
static DECLARE_WORK(cmdchan_data, easelcomm_hw_cmdchan_data_ready);

/* Command channel remote wrapped received, call up to generic layer. */
static void easelcomm_hw_cmdchan_wrap(struct work_struct *work)
{
	easelcomm_cmd_channel_wrap_handler();
}
static DECLARE_WORK(cmdchan_wrap, easelcomm_hw_cmdchan_wrap);

/* AP/client cmdchan setup. */
int easelcomm_hw_ap_setup_cmdchans(void)
{
#ifdef EASELCOMM_AP
	struct mnh_inb_window inbound;
	struct mnh_outb_region outbound;
	int ret;

	/* Command channel size shall not exceed the translation region size */
	BUILD_BUG_ON_MSG(
		EASELCOMM_CMD_CHANNEL_SIZE > HW_MNH_PCIE_OUTBOUND_SIZE,
	       "Easelcomm command channel size exceeds translation region");

	ret = mnh_get_rb_base(&easel_cmdchan_dma_addr);
	if (WARN_ON(ret))
		return ret;
	pr_debug("easelcomm: cmdchans: ap virt %p dma %pad easel dma %llx",
		 local_cmdchan_cpu_addr, &local_cmdchan_dma_addr,
		 easel_cmdchan_dma_addr);

	/* Inbound region 0 BAR4 match map to Easel cmdchan phys addr */
	inbound.region = 0;
	inbound.mode = BAR_MATCH;
	inbound.bar = 4;
	/* Align target address for 4MB BAR size */
	/*
	 * TODO: Want a carveout dedicated for this so no chance of 4MB
	 * boundary crossed.  b/32782918.
	 */
	inbound.target_mth_address =
		easel_cmdchan_dma_addr & ~(MNH_BAR4_MASK);
	easel_cmdchan_pcie_offset = easel_cmdchan_dma_addr & (MNH_BAR4_MASK);
	ret = mnh_set_inbound_iatu(&inbound);
	if (WARN_ON(ret))
		return ret;

	/* Outbound region 1 map to AP cmdchan DMA addr */
	outbound.region = 1;
	outbound.base_address = HW_MNH_PCIE_OUTBOUND_BASE;
	outbound.limit_address = HW_MNH_PCIE_OUTBOUND_BASE +
		EASELCOMM_CMD_CHANNEL_SIZE - 1;
	outbound.target_pcie_address = local_cmdchan_dma_addr;
	ret = mnh_set_outbound_iatu(&outbound);
	if (WARN_ON(ret))
		return ret;
	return 0;
#else
	return -EIO;
#endif
}
EXPORT_SYMBOL(easelcomm_hw_ap_setup_cmdchans);

/*
 * Send "remote producer wrote new data to the local command channel ring
 * buffer" interrupt.
 */
int easelcomm_hw_send_data_interrupt(void)
{
	int ret;

#ifdef EASELCOMM_AP
	ret = mnh_send_irq(IRQ_MSG_SENT);
#else
	ret = mnh_send_msi(MSG_SEND_M);
#endif
	WARN_ON(ret);
	return ret;
}
EXPORT_SYMBOL(easelcomm_hw_send_data_interrupt);

/*
 * Send "remote consumer caught up with local producer and is ready to wrap the
 * remote command channel ring buffer" interrupt.
 */
int easelcomm_hw_send_wrap_interrupt(void)
{
	int ret;
#ifdef EASELCOMM_AP
	ret = mnh_send_irq(IRQ_APPDEFINED_1);
#else
	ret = mnh_send_msi(APPDEFINED_1_M);
#endif
	WARN_ON(ret);
	return ret;
}
EXPORT_SYMBOL(easelcomm_hw_send_wrap_interrupt);

#ifdef EASELCOMM_EASEL
/* EP/server MNH API IRQ callback */
static int easelcomm_hw_easel_irq_callback(struct mnh_pcie_irq *irq)
{
	if (irq->msi_irq == MSG_SEND_I)
		schedule_work(&cmdchan_data);
	else if (irq->msi_irq == APPDEFINED_1_I)
		schedule_work(&cmdchan_wrap);
	else
		pr_info("easelcomm: mnh msi %u ignored\n",
			irq->msi_irq);
	return 0;
}

/* EP/server MNH DMA IRQ callback, not currently used */
static int easelcomm_hw_easel_dma_callback(struct mnh_dma_irq *irq)
{
	return 0;
}

/*
 * Set the command channel ring buffer address in the MNH cluster register
 * that the AP/client reads from, and send the MSI informing the ap that
 * the value is setup.
 */
static int easelcomm_hw_easel_advertise_cmdchan(uint64_t buffer_dma_addr)
{
	int ret;

	pr_debug("easelcomm: advertising buffer dma=%llx\n", buffer_dma_addr);
	ret = mnh_set_rb_base(buffer_dma_addr);
	WARN_ON(ret);
	return ret;
}

/* Module init time actions for EP/server */
int easelcomm_hw_init(void)
{
	int ret;

	/* dma alloc a buffer for now, may want dedicated carveout */
	/* temporary: alloc twice as much, align to boundary */
	local_cmdchan_cpu_addr =
	    mnh_alloc_coherent(EASELCOMM_CMD_CHANNEL_SIZE,
			       &local_cmdchan_dma_addr);
	pr_debug("easelcomm: cmdchan v=%p d=%pad\n",
		 local_cmdchan_cpu_addr, &local_cmdchan_dma_addr);

	/* "PCIe" is immediately ready for server, no hotplug in/probe wait */
	ret = easelcomm_init_pcie_ready(local_cmdchan_cpu_addr);
	WARN_ON(ret);

	/* Register EP-specific IRQ callbacks */
	ret = mnh_reg_irq_callback(
		&easelcomm_hw_easel_irq_callback,
		&easelcomm_hw_easel_dma_callback);
	WARN_ON(ret);

	ret = easelcomm_hw_easel_advertise_cmdchan(
		(uint64_t)local_cmdchan_dma_addr);
	return ret;
}
EXPORT_SYMBOL(easelcomm_hw_init);
#endif

#ifdef EASELCOMM_AP
/* AP/client MNH API MSI callback */
static int easelcomm_hw_ap_msi_callback(uint32_t msi)
{
	pr_debug("easelcomm: msi %u\n", msi);

	switch (msi) {
	case MSI_BOOTSTRAP_SET:
		complete(&bootstrap_done);
		break;
	case MSI_MSG_SEND:
		schedule_work(&cmdchan_data);
		break;
	case MSI_PET_WATCHDOG:
		pr_debug("easelcomm: ignore MSI_PET_WATCHDOG\n");
		break;
	case MSI_APPDEFINED_1:
		schedule_work(&cmdchan_wrap);
		break;
	default:
		pr_warn("easelcomm: MSI %u ignored\n", msi);
	}

	return 0;
}

/* AP/client MNH API DMA IRQ callback */
static int easelcomm_hw_ap_dma_callback(
	uint8_t chan, enum mnh_dma_chan_dir_t dir,
	enum mnh_dma_trans_status_t status)
{
	pr_debug("easelcomm: DMA done chan=%u dir=%u status=%u\n",
		chan, dir, status);
	if (chan == APP_DMA_CHAN) {
		app_dma_status = status;
		complete(&app_dma_done);
	}
	return 0;
}

/* AP/client PCIe ready, EP enumerated, can now use MNH host driver. */
static int easelcomm_hw_ap_pcie_ready(unsigned long bootstrap_timeout_jiffies)
{
	int ret = 0;
	uint64_t temp_rb_base_val;

	/* Only allocate ringbuffer once for AP */
	if (local_cmdchan_cpu_addr == NULL) {
		local_cmdchan_cpu_addr = mnh_alloc_coherent(
						EASELCOMM_CMD_CHANNEL_SIZE,
						&local_cmdchan_dma_addr);
		if (IS_ERR_OR_NULL(local_cmdchan_cpu_addr)) {
			pr_warn("%s: failed to alloc ringbuffer\n", __func__);
			return -ENOMEM;
		}
	}

	ret = easelcomm_init_pcie_ready(local_cmdchan_cpu_addr);
	WARN_ON(ret);

	/* Register AP-specific IRQ callbacks */
	ret = mnh_reg_irq_callback(
		&easelcomm_hw_ap_msi_callback, NULL,
		&easelcomm_hw_ap_dma_callback);
	WARN_ON(ret);

	/*
	 * Wasel is booting in parallel, poll whether it already sent
	 * bootstrap done MSI with ringbuffer base setup, prior to us
	 * being ready to handle the IRQ.
	 */
	ret = mnh_get_rb_base(&temp_rb_base_val);
	if (WARN_ON(ret)) {
		pr_err("%s: mnh_get_rb_base failed (%d)\n", __func__, ret);
		mnh_reg_irq_callback(NULL, NULL, NULL);
		return ret;
	} else if (!temp_rb_base_val) {
		/* wait for bootstrap completion */
		ret = wait_for_completion_timeout(&bootstrap_done,
					bootstrap_timeout_jiffies);
		if (!ret) {
			pr_err("%s: timeout waiting for bootstrap msi\n",
			       __func__);
			mnh_reg_irq_callback(NULL, NULL, NULL);
			return -ETIMEDOUT;
		}
	}

	ret = easelcomm_client_remote_cmdchan_ready_handler();
	if (ret)
		pr_warn("%s: remote_cmdchan_ready_handler returns %d\n",
			__func__, ret);

	return ret;
}

/* Callback on MNH host driver hotplug in/out events. */
static int easelcomm_hw_ap_hotplug_callback(enum mnh_hotplug_event_t event,
					    void *param)
{
	int ret = 0;
	unsigned long timeout_ms = (unsigned long)param;
	static enum mnh_hotplug_event_t state = MNH_HOTPLUG_OUT;

	if (state == event)
		return 0;

	switch (event) {
	case MNH_HOTPLUG_IN:
		pr_debug("%s: mnh hotplug in\n", __func__);
		if (!timeout_ms)
			timeout_ms = BOOTSTRAP_TIMEOUT_MS;
		ret = easelcomm_hw_ap_pcie_ready(msecs_to_jiffies(timeout_ms));
		break;
	case MNH_HOTPLUG_OUT:
		pr_debug("%s: mnh hotplug out\n", __func__);
		reinit_completion(&bootstrap_done);
		/* Unregister IRQ callbacks first */
		ret = mnh_reg_irq_callback(NULL, NULL, NULL);
		WARN_ON(ret);
		/* Call hotplug out callback implemented by easelcomm layer */
		easelcomm_pcie_hotplug_out();
		break;
	default:
		ret = -EINVAL;
		break;
	}

	if (!ret)
		state = event;

	return ret;
}

/* Module init time actions for AP/client */
int easelcomm_hw_init(void)
{
	int ret;

	mutex_init(&app_dma_mutex);

	local_cmdchan_cpu_addr = NULL;

	ret = mnh_sm_reg_hotplug_callback(&easelcomm_hw_ap_hotplug_callback);
	if (WARN_ON(ret))
		return ret;

	return 0;
}
EXPORT_SYMBOL(easelcomm_hw_init);
#endif

/* Read remote ringbuffer memory */
int easelcomm_hw_remote_read(
	void *local_addr, size_t len, uint64_t remote_offset)
{
	int ret;

#ifdef EASELCOMM_AP
	ret = mnh_ddr_read(easel_cmdchan_pcie_offset + (uint32_t)remote_offset,
			   len, local_addr);
#else
	ret = mnh_pcie_read(local_addr, len, remote_offset);
#endif
	WARN_ON(ret);
	return ret;
}
EXPORT_SYMBOL(easelcomm_hw_remote_read);

/* Write remote ringbuffer memory. */
int easelcomm_hw_remote_write(
	void *local_addr, size_t len, uint64_t remote_offset)
{
	int ret;

#ifdef EASELCOMM_AP
	ret = mnh_ddr_write(easel_cmdchan_pcie_offset + (uint32_t)remote_offset,
			    len, local_addr);
#else
	ret = mnh_pcie_write(local_addr, len, remote_offset);
#endif
	WARN_ON(ret);
	return ret;
}
EXPORT_SYMBOL(easelcomm_hw_remote_write);

/* Build an MNH scatter-gather list */
void *easelcomm_hw_build_scatterlist(struct easelcomm_kbuf_desc *buf_desc,
	uint32_t *scatterlist_size,
	void **sglocaldata, enum easelcomm_dma_direction dma_dir)
{
	int n_ents_used = 0;
	struct mnh_sg_entry *sg_ents;
	struct mnh_sg_list *local_sg_info;
	int ret;
	bool to_easel = (dma_dir == EASELCOMM_DMA_DIR_TO_SERVER);

	local_sg_info = kmalloc(sizeof(struct mnh_sg_list), GFP_KERNEL);
	*sglocaldata = local_sg_info;
	if (!local_sg_info) {
		*scatterlist_size = 0;
		return NULL;
	}
	/*
	 * Initialize dma_buf related pointers to NULL; if dma_buf is used,
	 * they will become non-zero by mnh_sg_retrieve_from_dma_buf().
	 * easelcomm_hw_destroy_scatterlist() will use this information
	 * to decide how to release the scatterlist.
	 */
	local_sg_info->dma_buf = NULL;
	local_sg_info->attach = NULL;
	local_sg_info->sg_table = NULL;

	local_sg_info->dir = to_easel ? DMA_AP2EP : DMA_EP2AP;

	switch (buf_desc->buf_type) {
	case EASELCOMM_DMA_BUFFER_UNUSED:
		pr_err("%s: DMA buffer not used.\n", __func__);
		ret = -EINVAL;
		break;
	case EASELCOMM_DMA_BUFFER_USER:
		ret = mnh_sg_build(buf_desc->buf, buf_desc->buf_size,
					&sg_ents, local_sg_info);
		break;
	case EASELCOMM_DMA_BUFFER_DMA_BUF:
		ret = mnh_sg_retrieve_from_dma_buf(buf_desc->dma_buf_fd,
					&sg_ents, local_sg_info);
		break;
	default:
		pr_err("%s: Unknown DMA buffer type %d.\n",
					__func__, buf_desc->buf_type);
		ret = -EINVAL;
		break;
	}

	if (ret < 0) {
		kfree(local_sg_info);
		*sglocaldata = NULL;
		*scatterlist_size = 0;
		return NULL;
	}

	n_ents_used = local_sg_info->length;
	*scatterlist_size = sizeof(struct mnh_sg_entry) * n_ents_used;
	return sg_ents;
}
EXPORT_SYMBOL(easelcomm_hw_build_scatterlist);

int easelcomm_hw_verify_scatterlist(struct easelcomm_dma_xfer_info *xfer)
{
	return mnh_sg_verify(xfer->sg_local, xfer->sg_local_size,
		xfer->sg_local_localdata);
}

/*
 * Return the number of scatter-gather entries in the MNH SG list.  Used to
 * determine whether both sides require only 1 block and can use single-block
 * DMA for the transfer.
 */
int easelcomm_hw_scatterlist_block_count(uint32_t scatterlist_size)
{
	if (scatterlist_size == 0)
		return 0;
	scatterlist_size /= sizeof(struct mnh_sg_entry);
	return scatterlist_size - 1; /* subtract the terminator entry */
}
EXPORT_SYMBOL(easelcomm_hw_scatterlist_block_count);

/*
 * Return the physical/DMA address of the (server-side) single entry in the
 * SG list.  Used to determine the address needed by the client side for the
 * single-block DMA transfer.  It must alreayd have been determined that the
 * SG list has only one entry via easelcomm_hw_scatterlist_block_count()
 * above.
 */
uint64_t easelcomm_hw_scatterlist_sblk_addr(void *sgent)
{
	return (uint64_t)(((struct mnh_sg_entry *)sgent)->paddr);
}
EXPORT_SYMBOL(easelcomm_hw_scatterlist_sblk_addr);

/* Destroy the MNH SG local mapping data */
void easelcomm_hw_destroy_scatterlist(void *sglocaldata)
{
	struct mnh_sg_list *sg_local_data = (struct mnh_sg_list *)sglocaldata;

	if (sglocaldata) {
		if (sg_local_data->dma_buf == NULL) {
			/* Destroy sgl created by mnh_sg_build() */
			mnh_sg_destroy(sg_local_data);
		} else {
			/* Release sgl retrieved from dma_buf framework */
			mnh_sg_release_from_dma_buf(sg_local_data);
		}
	}
}
EXPORT_SYMBOL(easelcomm_hw_destroy_scatterlist);

/* Server builds an MNH DMA linked list for a multi-block transfer */
int easelcomm_hw_easel_build_ll(
	void *src_sg, void *dest_sg, void **ll_data)
{
#ifdef EASELCOMM_EASEL
	struct mnh_dma_ll *mnh_ll;
	int ret;

	*ll_data = NULL;
	mnh_ll = kmalloc(sizeof(struct mnh_dma_ll), GFP_KERNEL);
	if (!mnh_ll)
		return -ENOMEM;
	ret = mnh_ll_build(src_sg, dest_sg, mnh_ll);
	if (ret) {
		kfree(mnh_ll);
		return ret;
	}

	*ll_data = mnh_ll;
	return 0;
#else
	return -EIO;
#endif
}
EXPORT_SYMBOL(easelcomm_hw_easel_build_ll);

/*
 * Server returns Linked List DMA start address to send to client to
 * initiate a multi-block DMA transfer.
 */
uint64_t easelcomm_hw_easel_ll_addr(void *ll_data)
{
#ifdef EASELCOMM_EASEL
	return (uint64_t)((struct mnh_dma_ll *)ll_data)->dma[0];
#else
	return 0;
#endif
}
EXPORT_SYMBOL(easelcomm_hw_easel_ll_addr);

/* Server destroys an MNH DMA Linked List. */
int easelcomm_hw_easel_destroy_ll(void *ll_data)
{
#ifdef EASELCOMM_EASEL
	return mnh_ll_destroy((struct mnh_dma_ll *)ll_data);
#else
	return -EIO;
#endif
}
EXPORT_SYMBOL(easelcomm_hw_easel_destroy_ll);

/* Client performs a single-block DMA transfer */
int easelcomm_hw_ap_dma_sblk_transfer(
	uint64_t ap_daddr, uint64_t easel_daddr, uint32_t xfer_len,
	bool to_easel)
{
#ifdef EASELCOMM_AP
	enum mnh_dma_chan_dir_t dir = to_easel ? DMA_AP2EP : DMA_EP2AP;
	struct mnh_dma_element_t blk;
	int ret;

	blk.src_addr = to_easel ? ap_daddr : easel_daddr;
	blk.dst_addr = to_easel ? easel_daddr : ap_daddr;
	blk.len = xfer_len;

	mutex_lock(&app_dma_mutex);
	reinit_completion(&app_dma_done);
	ret = mnh_dma_sblk_start(APP_DMA_CHAN, dir, &blk);
	if (WARN_ON(ret)) {
		mutex_unlock(&app_dma_mutex);
		return -EIO;
	}

	ret = wait_for_completion_interruptible(&app_dma_done);
	if (WARN_ON(ret)) {
		/* Ensure DMA aborted before returning */
		ret = mnh_dma_abort(APP_DMA_CHAN, dir);
		WARN_ON(ret);
		mutex_unlock(&app_dma_mutex);
		return ret;
	}

	mutex_unlock(&app_dma_mutex);
	return app_dma_status == DMA_DONE ? 0 : -EIO;
#else
	return -EIO;
#endif
}
EXPORT_SYMBOL(easelcomm_hw_ap_dma_sblk_transfer);

/* Client performs a multi-block DMA transfer */
int easelcomm_hw_ap_dma_mblk_transfer(uint64_t ll_paddr, bool to_easel)
{
#ifdef EASELCOMM_AP
	enum mnh_dma_chan_dir_t dir = to_easel ? DMA_AP2EP : DMA_EP2AP;
	int ret;

	mutex_lock(&app_dma_mutex);
	reinit_completion(&app_dma_done);
	ret = mnh_dma_mblk_start(APP_DMA_CHAN, dir, &ll_paddr);
	if (WARN_ON(ret)) {
		mutex_unlock(&app_dma_mutex);
		return -EIO;
	}
	ret = wait_for_completion_interruptible(&app_dma_done);
	if (WARN_ON(ret)) {
		/* Ensure DMA aborted before returning */
		ret = mnh_dma_abort(APP_DMA_CHAN, dir);
		WARN_ON(ret);
		mutex_unlock(&app_dma_mutex);
		return ret;
	}
	mutex_unlock(&app_dma_mutex);
	return app_dma_status == DMA_DONE ? 0 : -EIO;
#else
	return -EIO;
#endif
}
EXPORT_SYMBOL(easelcomm_hw_ap_dma_mblk_transfer);