From 4f1ab4600ad72dc8e8f5a714e08766dfdf5e9faa Mon Sep 17 00:00:00 2001 From: Sultan Alsawaf Date: Wed, 9 Dec 2020 00:04:57 -0800 Subject: [PATCH] iommu: msm: Rewrite to improve clarity and performance This scope of this driver's lock usage is extremely wide, leading to excessively long lock hold times. Additionally, there is lots of excessive linked-list traversal and unnecessary dynamic memory allocation in a critical path, causing poor performance across the board. Fix all of this by greatly reducing the scope of the locks used and by significantly reducing the amount of operations performed when msm_dma_map_sg_attrs() is called. The entire driver's code is overhauled for better cleanliness and performance. Note that ION must be modified to pass a known structure via the private dma_buf pointer, so that the IOMMU driver can prevent races when operating on the same buffer concurrently. This is the only way to eliminate said buffer races without hurting the IOMMU driver's performance. Some additional members are added to the device struct as well to make these various performance improvements possible. This also removes the manual cache maintenance since ION already handles it. Signed-off-by: Sultan Alsawaf --- drivers/base/core.c | 2 + drivers/iommu/msm_dma_iommu_mapping.c | 538 ++++++-------------------- drivers/staging/android/ion/ion.c | 45 ++- drivers/staging/android/ion/ion.h | 2 + include/linux/device.h | 3 + include/linux/msm_dma_iommu_mapping.h | 42 +- 6 files changed, 159 insertions(+), 473 deletions(-) diff --git a/drivers/base/core.c b/drivers/base/core.c index 63e1e8f41426..1415b913ecae 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2134,6 +2134,8 @@ void device_initialize(struct device *dev) INIT_LIST_HEAD(&dev->links.needs_suppliers); INIT_LIST_HEAD(&dev->links.defer_hook); dev->links.status = DL_DEV_NO_DRIVER; + INIT_LIST_HEAD(&dev->iommu_map_list); + mutex_init(&dev->iommu_map_lock); } EXPORT_SYMBOL_GPL(device_initialize); diff --git a/drivers/iommu/msm_dma_iommu_mapping.c b/drivers/iommu/msm_dma_iommu_mapping.c index 76eb30a7473e..e318f1b865ba 100644 --- a/drivers/iommu/msm_dma_iommu_mapping.c +++ b/drivers/iommu/msm_dma_iommu_mapping.c @@ -1,473 +1,165 @@ -// SPDX-License-Identifier: GPL-2.0-only +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. + * Copyright (C) 2019-2020 Sultan Alsawaf . */ -#include -#include +#include +#include +#include #include -#include -#include -#include #include -#include - -/** - * struct msm_iommu_map - represents a mapping of an ion buffer to an iommu - * @lnode - list node to exist in the buffer's list of iommu mappings - * @dev - Device this is mapped to. Used as key - * @sgl - The scatterlist for this mapping - * @nents - Number of entries in sgl - * @dir - The direction for the map. - * @meta - Backpointer to the meta this guy belongs to. - * @ref - for reference counting this mapping - * @attrs - dma mapping attributes - * @buf_start_addr - address of start of buffer - * - * Represents a mapping of one dma_buf buffer to a particular device - * and address range. There may exist other mappings of this buffer in - * different devices. All mappings will have the same cacheability and security. - */ struct msm_iommu_map { - struct list_head lnode; - struct rb_node node; struct device *dev; + struct msm_iommu_data *data; + struct list_head data_node; + struct list_head dev_node; struct scatterlist *sgl; - unsigned int nents; enum dma_data_direction dir; - struct msm_iommu_meta *meta; - struct kref ref; unsigned long attrs; - dma_addr_t buf_start_addr; + int nents; + int refcount; }; -struct msm_iommu_meta { - struct rb_node node; - struct list_head iommu_maps; - struct kref ref; - struct mutex lock; - void *buffer; -}; - -static struct rb_root iommu_root; -static DEFINE_MUTEX(msm_iommu_map_mutex); - -static void msm_iommu_meta_add(struct msm_iommu_meta *meta) +static struct msm_iommu_map *msm_iommu_map_lookup(struct msm_iommu_data *data, + struct device *dev) { - struct rb_root *root = &iommu_root; - struct rb_node **p = &root->rb_node; - struct rb_node *parent = NULL; - struct msm_iommu_meta *entry; + struct msm_iommu_map *map; - while (*p) { - parent = *p; - entry = rb_entry(parent, struct msm_iommu_meta, node); - - if (meta->buffer < entry->buffer) - p = &(*p)->rb_left; - else if (meta->buffer > entry->buffer) - p = &(*p)->rb_right; - else - pr_err("%s: dma_buf %p already exists\n", __func__, - entry->buffer); - } - - rb_link_node(&meta->node, parent, p); - rb_insert_color(&meta->node, root); -} - -static struct msm_iommu_meta *msm_iommu_meta_lookup(void *buffer) -{ - struct rb_root *root = &iommu_root; - struct rb_node **p = &root->rb_node; - struct rb_node *parent = NULL; - struct msm_iommu_meta *entry = NULL; - - while (*p) { - parent = *p; - entry = rb_entry(parent, struct msm_iommu_meta, node); - - if (buffer < entry->buffer) - p = &(*p)->rb_left; - else if (buffer > entry->buffer) - p = &(*p)->rb_right; - else - return entry; + list_for_each_entry(map, &data->map_list, data_node) { + if (map->dev == dev) + return map; } return NULL; } -static void msm_iommu_add(struct msm_iommu_meta *meta, - struct msm_iommu_map *iommu) +static void msm_iommu_map_free(struct msm_iommu_map *map) { - INIT_LIST_HEAD(&iommu->lnode); - list_add(&iommu->lnode, &meta->iommu_maps); + struct sg_table table = { + .sgl = map->sgl, + .nents = map->nents, + .orig_nents = map->nents + }; + + dma_unmap_sg_attrs(map->dev, map->sgl, map->nents, map->dir, + map->attrs | DMA_ATTR_SKIP_CPU_SYNC); + sg_free_table(&table); + list_del(&map->data_node); + list_del(&map->dev_node); + kfree(map); } - -static struct msm_iommu_map *msm_iommu_lookup(struct msm_iommu_meta *meta, - struct device *dev) +static struct scatterlist *clone_sgl(struct scatterlist *sgl, int nents) { - struct msm_iommu_map *entry; - - list_for_each_entry(entry, &meta->iommu_maps, lnode) { - if (entry->dev == dev) - return entry; - } - - return NULL; -} - -static struct msm_iommu_meta *msm_iommu_meta_create(struct dma_buf *dma_buf) -{ - struct msm_iommu_meta *meta; - - meta = kzalloc(sizeof(*meta), GFP_KERNEL); - - if (!meta) - return ERR_PTR(-ENOMEM); - - INIT_LIST_HEAD(&meta->iommu_maps); - meta->buffer = dma_buf->priv; - kref_init(&meta->ref); - mutex_init(&meta->lock); - msm_iommu_meta_add(meta); - - return meta; -} - -static void msm_iommu_meta_put(struct msm_iommu_meta *meta); - -static struct scatterlist *clone_sgl(struct scatterlist *sg, int nents) -{ - struct scatterlist *next, *s; - int i; + struct scatterlist *d, *s; struct sg_table table; - if (sg_alloc_table(&table, nents, GFP_KERNEL)) - return NULL; - next = table.sgl; - for_each_sg(sg, s, nents, i) { - *next = *s; - next = sg_next(next); - } + sg_alloc_table(&table, nents, GFP_KERNEL | __GFP_NOFAIL); + for (d = table.sgl, s = sgl; + nents > SG_MAX_SINGLE_ALLOC; nents -= SG_MAX_SINGLE_ALLOC - 1, + d = sg_chain_ptr(&d[SG_MAX_SINGLE_ALLOC - 1]), + s = sg_chain_ptr(&s[SG_MAX_SINGLE_ALLOC - 1])) + memcpy(d, s, (SG_MAX_SINGLE_ALLOC - 1) * sizeof(*d)); + + if (nents) + memcpy(d, s, nents * sizeof(*d)); + return table.sgl; } -static inline int __msm_dma_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, - struct dma_buf *dma_buf, - unsigned long attrs) +int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, + enum dma_data_direction dir, struct dma_buf *dmabuf, + unsigned long attrs) { - struct msm_iommu_map *iommu_map; - struct msm_iommu_meta *iommu_meta = NULL; - int ret = 0; - bool extra_meta_ref_taken = false; - int late_unmap = !(attrs & DMA_ATTR_NO_DELAYED_UNMAP); + struct msm_iommu_data *data = dmabuf->priv; + struct msm_iommu_map *map; - mutex_lock(&msm_iommu_map_mutex); - iommu_meta = msm_iommu_meta_lookup(dma_buf->priv); + mutex_lock(&dev->iommu_map_lock); + mutex_lock(&data->lock); + map = msm_iommu_map_lookup(data, dev); + if (map) { + struct scatterlist *d = sgl, *s = map->sgl; - if (!iommu_meta) { - iommu_meta = msm_iommu_meta_create(dma_buf); - - if (IS_ERR(iommu_meta)) { - mutex_unlock(&msm_iommu_map_mutex); - ret = PTR_ERR(iommu_meta); - goto out; - } - if (late_unmap) { - kref_get(&iommu_meta->ref); - extra_meta_ref_taken = true; - } + map->refcount++; + do { + d->dma_address = s->dma_address; + d->dma_length = s->dma_length; + } while ((s = sg_next(s)) && s->dma_length && (d = sg_next(d))); + if (is_device_dma_coherent(dev)) + dmb(ish); } else { - kref_get(&iommu_meta->ref); - } - - mutex_unlock(&msm_iommu_map_mutex); - - mutex_lock(&iommu_meta->lock); - iommu_map = msm_iommu_lookup(iommu_meta, dev); - if (!iommu_map) { - iommu_map = kmalloc(sizeof(*iommu_map), GFP_KERNEL); - - if (!iommu_map) { - ret = -ENOMEM; - goto out_unlock; - } - - ret = dma_map_sg_attrs(dev, sg, nents, dir, attrs); - if (!ret) { - kfree(iommu_map); - goto out_unlock; - } - - iommu_map->sgl = clone_sgl(sg, nents); - if (!iommu_map->sgl) { - kfree(iommu_map); - ret = -ENOMEM; - goto out_unlock; - } - iommu_map->nents = nents; - iommu_map->dev = dev; - iommu_map->dir = dir; - iommu_map->attrs = attrs; - iommu_map->buf_start_addr = sg_phys(sg); - - kref_init(&iommu_map->ref); - if (late_unmap) - kref_get(&iommu_map->ref); - iommu_map->meta = iommu_meta; - msm_iommu_add(iommu_meta, iommu_map); - - } else { - if (nents == iommu_map->nents && - dir == iommu_map->dir && - (attrs & ~DMA_ATTR_SKIP_CPU_SYNC) == - (iommu_map->attrs & ~DMA_ATTR_SKIP_CPU_SYNC) && - sg_phys(sg) == iommu_map->buf_start_addr) { - struct scatterlist *sg_tmp = sg; - struct scatterlist *map_sg; - int i; - - for_each_sg(iommu_map->sgl, map_sg, nents, i) { - sg_dma_address(sg_tmp) = sg_dma_address(map_sg); - sg_dma_len(sg_tmp) = sg_dma_len(map_sg); - if (sg_dma_len(map_sg) == 0) - break; - - sg_tmp = sg_next(sg_tmp); - if (sg_tmp == NULL) - break; - } - - kref_get(&iommu_map->ref); - - if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) - dma_sync_sg_for_device(dev, iommu_map->sgl, - iommu_map->nents, iommu_map->dir); - - if (is_device_dma_coherent(dev)) - /* - * Ensure all outstanding changes for coherent - * buffers are applied to the cache before any - * DMA occurs. - */ - dmb(ish); - ret = nents; + if (dma_map_sg_attrs(dev, sgl, nents, dir, attrs)) { + map = kmalloc(sizeof(*map), GFP_KERNEL | __GFP_NOFAIL); + map->sgl = clone_sgl(sgl, nents); + map->data = data; + map->dev = dev; + map->dir = dir; + map->nents = nents; + map->refcount = 2; + map->attrs = attrs; + list_add(&map->data_node, &data->map_list); + list_add(&map->dev_node, &dev->iommu_map_list); } else { - bool start_diff = (sg_phys(sg) != - iommu_map->buf_start_addr); - - dev_err(dev, "lazy map request differs:\n" - "req dir:%d, original dir:%d\n" - "req nents:%d, original nents:%d\n" - "req map attrs:%lu, original map attrs:%lu\n" - "req buffer start address differs:%d\n", - dir, iommu_map->dir, nents, - iommu_map->nents, attrs, iommu_map->attrs, - start_diff); - ret = -EINVAL; + nents = 0; } } - mutex_unlock(&iommu_meta->lock); - return ret; + mutex_unlock(&data->lock); + mutex_unlock(&dev->iommu_map_lock); -out_unlock: - mutex_unlock(&iommu_meta->lock); -out: - if (!IS_ERR(iommu_meta)) { - if (extra_meta_ref_taken) - msm_iommu_meta_put(iommu_meta); - msm_iommu_meta_put(iommu_meta); - } - return ret; - -} - -/* - * We are not taking a reference to the dma_buf here. It is expected that - * clients hold reference to the dma_buf until they are done with mapping and - * unmapping. - */ -int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, struct dma_buf *dma_buf, - unsigned long attrs) -{ - int ret; - - if (IS_ERR_OR_NULL(dev)) { - pr_err("%s: dev pointer is invalid\n", __func__); - return -EINVAL; - } - - if (IS_ERR_OR_NULL(sg)) { - pr_err("%s: sg table pointer is invalid\n", __func__); - return -EINVAL; - } - - if (IS_ERR_OR_NULL(dma_buf)) { - pr_err("%s: dma_buf pointer is invalid\n", __func__); - return -EINVAL; - } - - ret = __msm_dma_map_sg(dev, sg, nents, dir, dma_buf, attrs); - - return ret; -} -EXPORT_SYMBOL(msm_dma_map_sg_attrs); - -static void msm_iommu_meta_destroy(struct kref *kref) -{ - struct msm_iommu_meta *meta = container_of(kref, struct msm_iommu_meta, - ref); - - if (!list_empty(&meta->iommu_maps)) { - WARN(1, "%s: DMA Buffer %p being destroyed with outstanding iommu mappins!\n", - __func__, meta->buffer); - } - rb_erase(&meta->node, &iommu_root); - kfree(meta); -} - -static void msm_iommu_meta_put(struct msm_iommu_meta *meta) -{ - /* - * Need to lock here to prevent race against map/unmap - */ - mutex_lock(&msm_iommu_map_mutex); - kref_put(&meta->ref, msm_iommu_meta_destroy); - mutex_unlock(&msm_iommu_map_mutex); -} - -static void msm_iommu_map_release(struct kref *kref) -{ - struct msm_iommu_map *map = container_of(kref, struct msm_iommu_map, - ref); - struct sg_table table; - - table.nents = table.orig_nents = map->nents; - table.sgl = map->sgl; - list_del(&map->lnode); - - /* Skip an additional cache maintenance on the dma unmap path */ - if (!(map->attrs & DMA_ATTR_SKIP_CPU_SYNC)) - map->attrs |= DMA_ATTR_SKIP_CPU_SYNC; - dma_unmap_sg_attrs(map->dev, map->sgl, map->nents, map->dir, - map->attrs); - sg_free_table(&table); - kfree(map); + return nents; } void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, - struct dma_buf *dma_buf, unsigned long attrs) + struct dma_buf *dmabuf, unsigned long attrs) { - struct msm_iommu_map *iommu_map; - struct msm_iommu_meta *meta; + struct msm_iommu_data *data = dmabuf->priv; + struct msm_iommu_map *map; - mutex_lock(&msm_iommu_map_mutex); - meta = msm_iommu_meta_lookup(dma_buf->priv); - if (!meta) { - WARN(1, "%s: (%p) was never mapped\n", __func__, dma_buf); - mutex_unlock(&msm_iommu_map_mutex); - goto out; - - } - mutex_unlock(&msm_iommu_map_mutex); - - mutex_lock(&meta->lock); - iommu_map = msm_iommu_lookup(meta, dev); - - if (!iommu_map) { - WARN(1, "%s: (%p) was never mapped for device %p\n", __func__, - dma_buf, dev); - mutex_unlock(&meta->lock); - goto out; - } - - if (dir != iommu_map->dir) - WARN(1, "%s: (%pK) dir:%d differs from original dir:%d\n", - __func__, dma_buf, dir, iommu_map->dir); - - if (attrs && ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)) - dma_sync_sg_for_cpu(dev, iommu_map->sgl, iommu_map->nents, dir); - - iommu_map->attrs = attrs; - kref_put(&iommu_map->ref, msm_iommu_map_release); - mutex_unlock(&meta->lock); - - msm_iommu_meta_put(meta); - -out: - return; -} -EXPORT_SYMBOL(msm_dma_unmap_sg_attrs); - -int msm_dma_unmap_all_for_dev(struct device *dev) -{ - int ret = 0; - struct msm_iommu_meta *meta; - struct rb_root *root; - struct rb_node *meta_node; - - mutex_lock(&msm_iommu_map_mutex); - root = &iommu_root; - meta_node = rb_first(root); - while (meta_node) { - struct msm_iommu_map *iommu_map; - struct msm_iommu_map *iommu_map_next; - - meta = rb_entry(meta_node, struct msm_iommu_meta, node); - mutex_lock(&meta->lock); - list_for_each_entry_safe(iommu_map, iommu_map_next, - &meta->iommu_maps, lnode) - if (iommu_map->dev == dev) - if (!kref_put(&iommu_map->ref, - msm_iommu_map_release)) - ret = -EINVAL; - - mutex_unlock(&meta->lock); - meta_node = rb_next(meta_node); - } - mutex_unlock(&msm_iommu_map_mutex); - - return ret; + mutex_lock(&dev->iommu_map_lock); + mutex_lock(&data->lock); + map = msm_iommu_map_lookup(data, dev); + if (map && !--map->refcount) + msm_iommu_map_free(map); + mutex_unlock(&data->lock); + mutex_unlock(&dev->iommu_map_lock); } -/* - * Only to be called by ION code when a buffer is freed - */ -void msm_dma_buf_freed(void *buffer) +void msm_dma_unmap_all_for_dev(struct device *dev) { - struct msm_iommu_map *iommu_map; - struct msm_iommu_map *iommu_map_next; - struct msm_iommu_meta *meta; + struct msm_iommu_map *map, *tmp; - mutex_lock(&msm_iommu_map_mutex); - meta = msm_iommu_meta_lookup(buffer); - if (!meta) { - /* Already unmapped (assuming no late unmapping) */ - mutex_unlock(&msm_iommu_map_mutex); - return; + mutex_lock(&dev->iommu_map_lock); + list_for_each_entry_safe(map, tmp, &dev->iommu_map_list, dev_node) { + struct msm_iommu_data *data = map->data; + + mutex_lock(&data->lock); + msm_iommu_map_free(map); + mutex_unlock(&data->lock); } - mutex_unlock(&msm_iommu_map_mutex); - - mutex_lock(&meta->lock); - - list_for_each_entry_safe(iommu_map, iommu_map_next, &meta->iommu_maps, - lnode) - kref_put(&iommu_map->ref, msm_iommu_map_release); - - if (!list_empty(&meta->iommu_maps)) { - WARN(1, "%s: DMA buffer %p destroyed with outstanding iommu mappings\n", - __func__, meta->buffer); - } - - INIT_LIST_HEAD(&meta->iommu_maps); - mutex_unlock(&meta->lock); - - msm_iommu_meta_put(meta); + mutex_unlock(&dev->iommu_map_lock); +} + +void msm_dma_buf_freed(struct msm_iommu_data *data) +{ + struct msm_iommu_map *map, *tmp; + int retry = 0; + + do { + mutex_lock(&data->lock); + list_for_each_entry_safe(map, tmp, &data->map_list, data_node) { + struct device *dev = map->dev; + + if (!mutex_trylock(&dev->iommu_map_lock)) { + retry = 1; + break; + } + + msm_iommu_map_free(map); + mutex_unlock(&dev->iommu_map_lock); + } + mutex_unlock(&data->lock); + } while (retry--); } diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c index 8ae8ff83d107..03660dd9e4bc 100644 --- a/drivers/staging/android/ion/ion.c +++ b/drivers/staging/android/ion/ion.c @@ -109,6 +109,8 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, if (!buffer) return ERR_PTR(-ENOMEM); + INIT_LIST_HEAD(&buffer->iommu_data.map_list); + mutex_init(&buffer->iommu_data.lock); buffer->heap = heap; buffer->flags = flags; buffer->dev = dev; @@ -186,7 +188,7 @@ static void _ion_buffer_destroy(struct ion_buffer *buffer) struct ion_heap *heap = buffer->heap; struct ion_device *dev = buffer->dev; - msm_dma_buf_freed(buffer); + msm_dma_buf_freed(&buffer->iommu_data); mutex_lock(&dev->buffer_lock); rb_erase(&buffer->node, &dev->buffers); @@ -282,7 +284,8 @@ static int ion_dma_buf_attach(struct dma_buf *dmabuf, { struct ion_dma_buf_attachment *a; struct sg_table *table; - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); a = kzalloc(sizeof(*a), GFP_KERNEL); if (!a) @@ -312,7 +315,8 @@ static void ion_dma_buf_detatch(struct dma_buf *dmabuf, struct dma_buf_attachment *attachment) { struct ion_dma_buf_attachment *a = attachment->priv; - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); mutex_lock(&buffer->lock); list_del(&a->list); @@ -328,7 +332,8 @@ static struct sg_table *ion_map_dma_buf(struct dma_buf_attachment *attachment, struct ion_dma_buf_attachment *a = attachment->priv; struct sg_table *table; int count, map_attrs; - struct ion_buffer *buffer = attachment->dmabuf->priv; + struct ion_buffer *buffer = container_of(attachment->dmabuf->priv, + typeof(*buffer), iommu_data); table = a->table; @@ -378,7 +383,8 @@ static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment, enum dma_data_direction direction) { int map_attrs; - struct ion_buffer *buffer = attachment->dmabuf->priv; + struct ion_buffer *buffer = container_of(attachment->dmabuf->priv, + typeof(*buffer), iommu_data); struct ion_dma_buf_attachment *a = attachment->priv; map_attrs = attachment->dma_map_attrs; @@ -467,7 +473,8 @@ static const struct vm_operations_struct ion_vma_ops = { static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) { - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); int ret = 0; if (!buffer->heap->ops->map_user) { @@ -497,7 +504,8 @@ static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) static void ion_dma_buf_release(struct dma_buf *dmabuf) { - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); _ion_buffer_destroy(buffer); kfree(dmabuf->exp_name); @@ -505,7 +513,8 @@ static void ion_dma_buf_release(struct dma_buf *dmabuf) static void *ion_dma_buf_vmap(struct dma_buf *dmabuf) { - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); void *vaddr = ERR_PTR(-EINVAL); if (buffer->heap->ops->map_kernel) { @@ -522,7 +531,8 @@ static void *ion_dma_buf_vmap(struct dma_buf *dmabuf) static void ion_dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) { - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); if (buffer->heap->ops->map_kernel) { mutex_lock(&buffer->lock); @@ -636,7 +646,8 @@ static int __ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, enum dma_data_direction direction, bool sync_only_mapped) { - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); struct ion_dma_buf_attachment *a; int ret = 0; @@ -732,7 +743,8 @@ static int __ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, enum dma_data_direction direction, bool sync_only_mapped) { - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); struct ion_dma_buf_attachment *a; int ret = 0; @@ -849,7 +861,8 @@ static int ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, unsigned int offset, unsigned int len) { - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); struct ion_dma_buf_attachment *a; int ret = 0; @@ -930,7 +943,8 @@ static int ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, unsigned int offset, unsigned int len) { - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); struct ion_dma_buf_attachment *a; int ret = 0; @@ -1010,7 +1024,8 @@ out: static int ion_dma_buf_get_flags(struct dma_buf *dmabuf, unsigned long *flags) { - struct ion_buffer *buffer = dmabuf->priv; + struct ion_buffer *buffer = container_of(dmabuf->priv, typeof(*buffer), + iommu_data); *flags = buffer->flags; return 0; @@ -1081,7 +1096,7 @@ struct dma_buf *ion_alloc_dmabuf(size_t len, unsigned int heap_id_mask, exp_info.ops = &dma_buf_ops; exp_info.size = buffer->size; exp_info.flags = O_RDWR; - exp_info.priv = buffer; + exp_info.priv = &buffer->iommu_data; exp_info.exp_name = kasprintf(GFP_KERNEL, "%s-%s-%d-%s", KBUILD_MODNAME, heap->name, current->tgid, task_comm); diff --git a/drivers/staging/android/ion/ion.h b/drivers/staging/android/ion/ion.h index da143d3e49fb..ccce81cdb58c 100644 --- a/drivers/staging/android/ion/ion.h +++ b/drivers/staging/android/ion/ion.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "ion_kernel.h" #include "../uapi/ion.h" #include "../uapi/msm_ion.h" @@ -144,6 +145,7 @@ struct ion_buffer { struct sg_table *sg_table; struct list_head attachments; struct list_head vmas; + struct msm_iommu_data iommu_data; }; void ion_buffer_destroy(struct ion_buffer *buffer); diff --git a/include/linux/device.h b/include/linux/device.h index 2d9e155fcc40..ee6149cd4734 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1113,6 +1113,9 @@ struct device { bool of_node_reused:1; bool state_synced:1; + struct list_head iommu_map_list; + struct mutex iommu_map_lock; + ANDROID_KABI_RESERVE(1); ANDROID_KABI_RESERVE(2); ANDROID_KABI_RESERVE(3); diff --git a/include/linux/msm_dma_iommu_mapping.h b/include/linux/msm_dma_iommu_mapping.h index 6495c2b6d473..d46ca465ec31 100644 --- a/include/linux/msm_dma_iommu_mapping.h +++ b/include/linux/msm_dma_iommu_mapping.h @@ -11,6 +11,11 @@ #include #include +struct msm_iommu_data { + struct list_head map_list; + struct mutex lock; +}; + #ifdef CONFIG_QCOM_LAZY_MAPPING /* * This function is not taking a reference to the dma_buf here. It is expected @@ -21,50 +26,17 @@ int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, struct dma_buf *dma_buf, unsigned long attrs); -/* - * This function takes an extra reference to the dma_buf. - * What this means is that calling msm_dma_unmap_sg will not result in buffer's - * iommu mapping being removed, which means that subsequent calls to lazy map - * will simply re-use the existing iommu mapping. - * The iommu unmapping of the buffer will occur when the ION buffer is - * destroyed. - * Using lazy mapping can provide a performance benefit because subsequent - * mappings are faster. - * - * The limitation of using this API are that all subsequent iommu mappings - * must be the same as the original mapping, ie they must map the same part of - * the buffer with the same dma data direction. Also there can't be multiple - * mappings of different parts of the buffer. - */ -static inline int msm_dma_map_sg_lazy(struct device *dev, - struct scatterlist *sg, int nents, - enum dma_data_direction dir, - struct dma_buf *dma_buf) -{ - return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, 0); -} - -static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, - struct dma_buf *dma_buf) -{ - unsigned long attrs; - - attrs = DMA_ATTR_NO_DELAYED_UNMAP; - return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, attrs); -} - void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, struct dma_buf *dma_buf, unsigned long attrs); -int msm_dma_unmap_all_for_dev(struct device *dev); +void msm_dma_unmap_all_for_dev(struct device *dev); /* * Below is private function only to be called by framework (ION) and not by * clients. */ -void msm_dma_buf_freed(void *buffer); +void msm_dma_buf_freed(struct msm_iommu_data *data); #else /*CONFIG_QCOM_LAZY_MAPPING*/