path: root/drivers/cenalloc/cenalloc.c

/*
 * Allocator helper framework for constraints-aware dma-buf backing storage
 * allocation.
 * This allows constraint-sharing devices to deferred-allocate buffers shared
 * via dma-buf.
 *
 * Copyright(C) 2014 Linaro Limited. All rights reserved.
 * Author: Sumit Semwal <sumit.semwal@linaro.org>
 *
 * Structure for management of clients, buffers etc heavily derived from
 * Android's ION framework.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published by
 * the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/device.h>
#include <linux/err.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/miscdevice.h>
#include <linux/rbtree.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/vmalloc.h>
#include <linux/debugfs.h>
#include <linux/module.h>

#include "cenalloc.h"
#include "cenalloc_priv.h"

extern struct cenalloc_allocator system_contig;

/*
 * Constraints-aware allocator framework helper is meant to facilitate
 * deferred allocation of backing storage for dma-buf buffers.
 * It works for devices that can share their constraints via dma_params.
 * These dma_params are then used by dma_buf_attach() to create a mask of
 * common constraints. The cenalloc constraint helpers then allocate
 * for the preferred allocator according to the constraint mask.
 * Allocators and their corresponding constraint masks are pre-populated
 * for a given system - likely at the time of platform initialization.
 */
/**
 * struct cenalloc_device - the metadata of the cenalloc device node
 * @dev:			the actual misc device
 * @buffers:		an rb tree of all the existing buffers
 * @buffer_lock:	lock protecting the tree of buffers & handles
 * @lock:			rwsem protecting the tree of allocators
 * @clients:		list of all the clients created
 * @allocators:		list of all the allocators in the system
 */
struct cenalloc_device {
	struct miscdevice dev;
	struct rb_root buffers;

	struct mutex buffer_lock;
	struct rw_semaphore lock;

	struct plist_head allocators;
};

static struct cenalloc_device cenalloc_dev;

/* this function should only be called while dev->buffer_lock is held */
static void cenalloc_buffer_add(struct cenalloc_device *dev,
			   struct cenalloc_buffer *buffer)
{
	struct rb_node **p = &dev->buffers.rb_node;
	struct rb_node *parent = NULL;
	struct cenalloc_buffer *entry;

	while (*p) {
		parent = *p;
		entry = rb_entry(parent, struct cenalloc_buffer, node);

		if (buffer < entry) {
			p = &(*p)->rb_left;
		} else if (buffer > entry) {
			p = &(*p)->rb_right;
		} else {
			pr_err("%s: buffer already found.", __func__);
			BUG();
		}
	}

	rb_link_node(&buffer->node, parent, p);
	rb_insert_color(&buffer->node, &dev->buffers);
}

static struct dma_buf_ops ca_dma_buf_ops;

static bool is_cenalloc_buffer(struct dma_buf *dmabuf);

/*
 * cenalloc_buffer_create creates a buffer, exports the dma-buf handle and
 * associates a dma-buf handle with it.
 * Returns:
 *	on success, pointer to the associated dma_buf;
 *	error if dma-buf cannot be exported or if it is out of memory.
 */
struct dma_buf *cenalloc_buffer_create(unsigned long len,
				     unsigned long align,
				     unsigned long flags)
{
	struct cenalloc_buffer *buffer;
	struct dma_buf *dmabuf;

	buffer = kzalloc(sizeof(struct cenalloc_buffer), GFP_KERNEL);
	if (!buffer)
		return ERR_PTR(-ENOMEM);

	buffer->flags = flags;
	kref_init(&buffer->ref);

	buffer->dev = &cenalloc_dev;
	buffer->size = len;
	buffer->align = align;

	dmabuf = dma_buf_export(buffer, &ca_dma_buf_ops, buffer->size, O_RDWR,
							NULL);
	if (IS_ERR(dmabuf))
		goto err;

	buffer->dmabuf = dmabuf;
	dmabuf->priv = buffer;

	mutex_init(&buffer->lock);

	mutex_lock(&cenalloc_dev.buffer_lock);
	cenalloc_buffer_add(&cenalloc_dev, buffer);
	mutex_unlock(&cenalloc_dev.buffer_lock);

	return dmabuf;

err:
	kfree(buffer);
	return ERR_CAST(dmabuf);
}
EXPORT_SYMBOL_GPL(cenalloc_buffer_create);


static void cenalloc_buffer_destroy(struct cenalloc_buffer *buffer)
{
	if (!buffer->allocator)
		goto no_allocator;

	if (WARN_ON(buffer->kmap_cnt > 0))
		buffer->allocator->ops->unmap_kernel(buffer->allocator, buffer);
	buffer->allocator->ops->unmap_dma(buffer->allocator, buffer);
	buffer->allocator->ops->free(buffer);

no_allocator:
	kfree(buffer);
}

static void _cenalloc_buffer_destroy(struct cenalloc_buffer *buffer)
{
	struct cenalloc_device *dev = buffer->dev;

	mutex_lock(&dev->buffer_lock);
	rb_erase(&buffer->node, &dev->buffers);
	mutex_unlock(&dev->buffer_lock);

	cenalloc_buffer_destroy(buffer);
}

void cenalloc_buffer_free(struct dma_buf *dmabuf)
{
	if (is_cenalloc_buffer(dmabuf))
			dma_buf_put(dmabuf);
}
EXPORT_SYMBOL_GPL(cenalloc_buffer_free);

int cenalloc_phys(struct dma_buf *dmabuf,
	     phys_addr_t *addr, size_t *len)
{
	struct cenalloc_buffer *buffer;
	int ret;

	if (is_cenalloc_buffer(dmabuf))
		buffer = (struct cenalloc_buffer *)dmabuf->priv;
	else
		return -EINVAL;

	if (!buffer->allocator->ops->phys) {
		pr_err("%s: cenalloc_phys is not implemented by this allocator.\n",
		       __func__);
		return -ENODEV;
	}
	mutex_lock(&buffer->lock);
	ret = buffer->allocator->ops->phys(buffer->allocator, buffer, addr,
						len);
	mutex_lock(&buffer->lock);
	return ret;
}
EXPORT_SYMBOL_GPL(cenalloc_phys);

static void *cenalloc_buffer_kmap_get(struct cenalloc_buffer *buffer)
{
	void *vaddr;

	if (buffer->kmap_cnt) {
		buffer->kmap_cnt++;
		return buffer->vaddr;
	}
	vaddr = buffer->allocator->ops->map_kernel(buffer->allocator, buffer);
	if (WARN_ONCE(vaddr == NULL,
		  "allocator->ops->map_kernel should return ERR_PTR on error"))
		return ERR_PTR(-EINVAL);
	if (IS_ERR(vaddr))
		return vaddr;
	buffer->vaddr = vaddr;
	buffer->kmap_cnt++;
	return vaddr;
}

static void cenalloc_buffer_kmap_put(struct cenalloc_buffer *buffer)
{
	buffer->kmap_cnt--;
	if (!buffer->kmap_cnt) {
		buffer->allocator->ops->unmap_kernel(buffer->allocator, buffer);
		buffer->vaddr = NULL;
	}
}

struct sg_table *cenalloc_sg_table(struct dma_buf *dmabuf)
{
	struct sg_table *table;
	struct cenalloc_buffer *buffer;

	if (is_cenalloc_buffer(dmabuf))
		buffer = (struct cenalloc_buffer *)dmabuf->priv;
	else {
		pr_err("%s: invalid buffer passed to sg_table.\n",
		       __func__);
		return ERR_PTR(-EINVAL);
	}

	mutex_lock(&buffer->lock);
	table = buffer->sg_table;
	mutex_unlock(&buffer->lock);
	return table;
}
EXPORT_SYMBOL_GPL(cenalloc_sg_table);

/*
 * dma_buf ops implementation
 */

static void cenalloc_buffer_sync_for_device(struct cenalloc_buffer *buffer,
				       struct device *dev,
				       enum dma_data_direction direction);

/*
 * This will find the right allocator for the buffer passed;
 * assumption is, that all the interested importers have called dma_buf_attach()
 * with their right constraint masks before the first call to
 * dma_buf_map_attachment().
 * At present, using the same priority based mechanism as ION.
 */
static int cenalloc_find_allocator(struct cenalloc_device *dev,
				struct cenalloc_buffer *buf)
{
	struct cenalloc_allocator *allocator;
	unsigned long constraints_mask = buf->dmabuf->access_constraints_mask;
	size_t len = buf->size;

	/*
	 * traverse the list of allocators available in this system in priority
	 * order.  If the allocator type is supported by the client, and matches
	 * the request of the caller allocate from it.  Repeat until allocate
	 * has succeeded or all allocators have been tried.
	 */
	len = PAGE_ALIGN(len);

	if (!len)
		return -EINVAL;

	plist_for_each_entry(allocator, &dev->allocators, node) {
		/* if the caller didn't specify this allocator id */
		if (!((1 << allocator->id) & constraints_mask))
			continue;
		buf->allocator = allocator;
	}

	if (buf->allocator == NULL)
		return -ENODEV;

	return 0;
}

static struct sg_table *cenalloc_buffer_first_alloc(
					struct cenalloc_buffer *buffer)
{
	struct cenalloc_allocator *allocator;
	struct sg_table *table;

	int num_pages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;
	struct scatterlist *sg;
	int ret, i, j, k = 0;

	mutex_lock(&buffer->lock);
	ret = cenalloc_find_allocator(buffer->dev, buffer);

	if (ret) {
		mutex_unlock(&buffer->lock);
		return ERR_PTR(-ENODEV);
	}

	allocator = buffer->allocator;

	ret = allocator->ops->allocate(allocator, buffer, buffer->size,
						buffer->align, buffer->flags);
	if (ret)
		goto cannot_allocate;

	table = allocator->ops->map_dma(allocator, buffer);
	if (WARN_ONCE(table == NULL,
		    "allocator->ops->map_dma should return ERR_PTR on error"))
		table = ERR_PTR(-EINVAL);

	if (IS_ERR(table)) {
		/* TODO: find the right way to handle an error here? */
		allocator->ops->free(buffer);

		mutex_unlock(&buffer->lock);
		return ERR_CAST(table);
	}

	buffer->pages = vmalloc(sizeof(struct page *) * num_pages);
	if (!buffer->pages) {
		ret = -ENOMEM;
		goto cannot_allocate;
	}

	for_each_sg(table->sgl, sg, table->nents, i) {
		struct page *page = sg_page(sg);

		for (j = 0; j < sg->length / PAGE_SIZE; j++)
			buffer->pages[k++] = page++;
	}

	mutex_unlock(&buffer->lock);

	return table;

cannot_allocate:
	mutex_unlock(&buffer->lock);
	return ERR_PTR(ret);
}

static void cenalloc_buffer_sync_for_device(struct cenalloc_buffer *buffer,
				       struct device *dev,
				       enum dma_data_direction dir)
{

	if (!buffer->allocator->ops->sync_for_device) {
		pr_err("%s: this allocator does not define a method for sync-to-device\n",
					__func__);
		return;
	}

	buffer->allocator->ops->sync_for_device(buffer->allocator, buffer, dev,
						dir);

}

/*
 * cenalloc_map_dma_buf() models delayed allocation; so if the buffer is not
 * backed up by storage, the allocation shall happen here for the first time,
 * based on the constraint_mask of the dma_buf, which is set based on devices
 * currently attached to the dma_buf.
 *
 * IMP: Assumption is that all participating devices call dma_buf_attach()
 * before the first dma_buf_map_attachment() is called.
 *
 * Migration is not supported at this time.
 */
static struct sg_table *cenalloc_map_dma_buf(struct dma_buf_attachment *attach,
					enum dma_data_direction direction)
{
	struct dma_buf *dmabuf = attach->dmabuf;
	struct cenalloc_buffer *buffer = dmabuf->priv;

	if (!buffer->sg_table) {
		down_read(&(buffer->dev->lock));
		buffer->sg_table = cenalloc_buffer_first_alloc(buffer);
		up_read(&(buffer->dev->lock));
		if (IS_ERR(buffer->sg_table))
			return buffer->sg_table;
	}

	mutex_lock(&buffer->lock);
	cenalloc_buffer_sync_for_device(buffer, attach->dev, direction);
	mutex_unlock(&buffer->lock);

	return buffer->sg_table;
}

static void cenalloc_unmap_dma_buf(struct dma_buf_attachment *attachment,
			      struct sg_table *table,
			      enum dma_data_direction direction)
{
	struct dma_buf *dmabuf = attachment->dmabuf;
	struct cenalloc_buffer *buffer = dmabuf->priv;

	if (buffer->allocator)
		buffer->allocator->ops->unmap_dma(buffer->allocator, buffer);
}

static int cenalloc_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
{
	struct cenalloc_buffer *buffer = dmabuf->priv;
	int ret = 0;

	if (!buffer->sg_table) {
		pr_err(
			"%s: buffer needs to be mapped first before userspace mapping\n",
			__func__);
		return -EINVAL;
	}

	if (!buffer->allocator->ops->map_user) {
		pr_err(
			"%s: allocator does not define a method for userspace mapping\n",
			__func__);
		return -EINVAL;
	}

	/*
	 * NOTE: For now, assume that the allocators will take care of any VMA
	 * management for the buffer - includes providing vma_ops, and / or
	 * managing mmap faults.
	 * struct cenalloc_buffer will still provide struct **pages,
	 * and also the vma area list for the buffer.
	 * This also allows for device-specific vma operations.
	 *
	 */

	mutex_lock(&buffer->lock);
	/* now map it to userspace */
	ret = buffer->allocator->ops->map_user(buffer->allocator, buffer, vma);
	mutex_unlock(&buffer->lock);

	if (ret)
		pr_err("%s: failure mapping buffer to userspace\n",
		       __func__);

	return ret;
}

static void cenalloc_dma_buf_release(struct dma_buf *dmabuf)
{
	struct cenalloc_buffer *buffer = dmabuf->priv;

	_cenalloc_buffer_destroy(buffer);
}

static void *cenalloc_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset)
{
	struct cenalloc_buffer *buffer = dmabuf->priv;

	return buffer->vaddr + offset * PAGE_SIZE;
}

static void cenalloc_dma_buf_kunmap(struct dma_buf *dmabuf,
					unsigned long offset,
					void *ptr)
{
	/* TODO */
}

static int cenalloc_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
						size_t start, size_t len,
						enum dma_data_direction dir)
{
	struct cenalloc_buffer *buffer = dmabuf->priv;
	void *vaddr;

	if (!buffer->allocator->ops->map_kernel) {
		pr_err("%s: map kernel is not implemented by this allocator.\n",
		       __func__);
		return -ENODEV;
	}

	mutex_lock(&buffer->lock);
	vaddr = cenalloc_buffer_kmap_get(buffer);
	mutex_unlock(&buffer->lock);
	return PTR_ERR_OR_ZERO(vaddr);
}

static void cenalloc_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
						size_t start, size_t len,
						enum dma_data_direction dir)
{
	struct cenalloc_buffer *buffer = dmabuf->priv;

	mutex_lock(&buffer->lock);
	cenalloc_buffer_kmap_put(buffer);
	mutex_unlock(&buffer->lock);
}

static struct dma_buf_ops ca_dma_buf_ops = {
	.map_dma_buf = cenalloc_map_dma_buf,
	.unmap_dma_buf = cenalloc_unmap_dma_buf,
	.mmap = cenalloc_mmap,
	.release = cenalloc_dma_buf_release,
	.begin_cpu_access = cenalloc_dma_buf_begin_cpu_access,
	.end_cpu_access = cenalloc_dma_buf_end_cpu_access,
	.kmap = cenalloc_dma_buf_kmap,
	.kunmap = cenalloc_dma_buf_kunmap,
	.kmap_atomic = cenalloc_dma_buf_kmap,
	.kunmap_atomic = cenalloc_dma_buf_kunmap,
};

static bool is_cenalloc_buffer(struct dma_buf *dmabuf)
{
	return(dmabuf->ops == &ca_dma_buf_ops);
}

static int cenalloc_buffer_getfd(unsigned long len,
	     unsigned long align,
	     unsigned long flags)
{
	struct dma_buf *dmabuf;
	int fd;

	dmabuf = cenalloc_buffer_create(len, align, flags);
	if (IS_ERR(dmabuf))
		return PTR_ERR(dmabuf);

	fd = dma_buf_fd(dmabuf, O_CLOEXEC);
	if (fd < 0)
		dma_buf_put(dmabuf);

	return fd;
}


static long cenalloc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
	int ret = 0;
	int fd = 0;

	struct cenalloc_fd_data data;

	if (_IOC_SIZE(cmd) > sizeof(data))
		return -EINVAL;

	if (copy_from_user(&data, (void __user *)arg, _IOC_SIZE(cmd)))
			return -EFAULT;

	switch (cmd) {
	case CA_IOC_CREATE:
	{
		fd = cenalloc_buffer_getfd(data.len, data.align, data.flags);
		if (fd < 0)
			return -ENOMEM;

		data.fd = fd;

		break;
	}

	default:
		return -ENOTTY;
	}

	if (copy_to_user((void __user *)arg, &data, _IOC_SIZE(cmd)))
			return -EFAULT;

	return ret;
}

static int cenalloc_release(struct inode *inode, struct file *file)
{
	struct miscdevice *dev = file->private_data;

	pr_debug("%s: %d\n", __func__, __LINE__);
	pr_debug("devname: %s\n", dev->name);
	return 0;
}

static int cenalloc_open(struct inode *inode, struct file *file)
{
	struct miscdevice *miscdev = file->private_data;
	struct cenalloc_device *dev = container_of(miscdev, struct cenalloc_device, dev);

	pr_debug("%s: %d\n", __func__, __LINE__);
	pr_debug("task pid: %u\n", task_pid_nr(current->group_leader));
	file->private_data = dev;

	return 0;
}

static const struct file_operations cenalloc_fops = {
	.owner          = THIS_MODULE,
	.open           = cenalloc_open,
	.release        = cenalloc_release,
	.unlocked_ioctl = cenalloc_ioctl,
};

/**
 * cenalloc_device_add_allocator:
 *	Adds an allocator to the cenalloc_device; This should be called at
 *	platform initialization time, for all allocators for the platform.
 */
void cenalloc_device_add_allocator(struct cenalloc_allocator *allocator)
{

	if (!allocator->ops->allocate || !allocator->ops->free ||
		!allocator->ops->map_dma || !allocator->ops->unmap_dma)
		pr_err("%s: can not add allocator with invalid ops struct.\n",
		       __func__);

	allocator->dev = &cenalloc_dev;
	down_write(&cenalloc_dev.lock);
	/* use negative allocator->id to reverse the priority -- when traversing
	   the list later attempt higher id numbers first */
	plist_node_init(&allocator->node, -allocator->id);
	plist_add(&allocator->node, &cenalloc_dev.allocators);

	up_write(&cenalloc_dev.lock);
}
EXPORT_SYMBOL_GPL(cenalloc_device_add_allocator);

/*
 * Cenalloc Device Init and Remove
 */
static struct cenalloc_device cenalloc_dev = {
		.dev.minor = MISC_DYNAMIC_MINOR,
		.dev.name = "cenalloc",
		.dev.fops = &cenalloc_fops,
		.dev.parent = NULL,
};

#if 0
/*
 * TODO: this mechanism of getting a cenalloc device isn't the best,
 * Need to have a better way of getting handle to device.
 */
struct cenalloc_device *cenalloc_get_device(void)
{
	return &cenalloc_dev;
}
EXPORT_SYMBOL_GPL(cenalloc_get_device);
#endif

static int __init cenalloc_device_init(void)
{
	int ret;

	ret = misc_register(&cenalloc_dev.dev);
	if (ret) {
		pr_err("cenalloc: failed to register misc device.\n");
		return ret;
	}

	cenalloc_dev.buffers = RB_ROOT;
	mutex_init(&cenalloc_dev.buffer_lock);
	init_rwsem(&cenalloc_dev.lock);
	plist_head_init(&cenalloc_dev.allocators);

	cenalloc_device_add_allocator(&system_contig);

	return ret;
}

static void __exit cenalloc_device_remove(void)
{
	misc_deregister(&cenalloc_dev.dev);

	/* XXX need to free the allocators? */
}

module_init(cenalloc_device_init);
module_exit(cenalloc_device_remove);

MODULE_AUTHOR("Sumit Semwal <sumit.semwal@linaro.org>");
MODULE_DESCRIPTION("Constraint Aware Central Allocation Helper");
MODULE_LICENSE("GPL");