iommu/amd: Add per-domain timer to flush per-cpu queues

Add a timer to each dma_ops domain so that we flush unused
IOTLB entries regularily, even if the queues don't get full
all the time.

Signed-off-by: Joerg Roedel <jroedel@suse.de>

1 files changed, 67 insertions, 17 deletions

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 795208bd39bd..00c1796e07bf 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -182,6 +182,13 @@ struct dma_ops_domain {
 	 * flushed out of the IOTLB of the domain.
 	 */
 	atomic64_t flush_finish_cnt;
+
+	/*
+	 * Timer to make sure we don't keep IOVAs around unflushed
+	 * for too long
+	 */
+	struct timer_list flush_timer;
+	atomic_t flush_timer_on;
 };
 
 static struct iova_domain reserved_iova_ranges;
@@ -1834,6 +1841,14 @@ static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom)
 	return 0;
 }
 
+static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom)
+{
+	atomic64_inc(&dom->flush_start_cnt);
+	domain_flush_tlb(&dom->domain);
+	domain_flush_complete(&dom->domain);
+	atomic64_inc(&dom->flush_finish_cnt);
+}
+
 static inline bool queue_ring_full(struct flush_queue *queue)
 {
 	assert_spin_locked(&queue->lock);
@@ -1875,22 +1890,12 @@ static inline void queue_ring_remove_head(struct flush_queue *queue)
 	queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;
 }
 
-static void queue_add(struct dma_ops_domain *dom,
-		      unsigned long address, unsigned long pages)
+static void queue_ring_free_flushed(struct dma_ops_domain *dom,
+				    struct flush_queue *queue)
 {
-	struct flush_queue *queue;
-	unsigned long flags;
-	u64 counter;
+	u64 counter = atomic64_read(&dom->flush_finish_cnt);
 	int idx;
 
-	pages     = __roundup_pow_of_two(pages);
-	address >>= PAGE_SHIFT;
-
-	counter = atomic64_read(&dom->flush_finish_cnt);
-
-	queue = get_cpu_ptr(dom->flush_queue);
-	spin_lock_irqsave(&queue->lock, flags);
-
 	queue_ring_for_each(idx, queue) {
 		/*
 		 * This assumes that counter values in the ring-buffer are
@@ -1905,12 +1910,25 @@ static void queue_add(struct dma_ops_domain *dom,
 
 		queue_ring_remove_head(queue);
 	}
+}
+
+static void queue_add(struct dma_ops_domain *dom,
+		      unsigned long address, unsigned long pages)
+{
+	struct flush_queue *queue;
+	unsigned long flags;
+	int idx;
+
+	pages     = __roundup_pow_of_two(pages);
+	address >>= PAGE_SHIFT;
+
+	queue = get_cpu_ptr(dom->flush_queue);
+	spin_lock_irqsave(&queue->lock, flags);
+
+	queue_ring_free_flushed(dom, queue);
 
 	if (queue_ring_full(queue)) {
-		atomic64_inc(&dom->flush_start_cnt);
-		domain_flush_tlb(&dom->domain);
-		domain_flush_complete(&dom->domain);
-		atomic64_inc(&dom->flush_finish_cnt);
+		dma_ops_domain_flush_tlb(dom);
 		queue_release(dom, queue);
 	}
 
@@ -1921,9 +1939,33 @@ static void queue_add(struct dma_ops_domain *dom,
 	queue->entries[idx].counter  = atomic64_read(&dom->flush_start_cnt);
 
 	spin_unlock_irqrestore(&queue->lock, flags);
+
+	if (atomic_cmpxchg(&dom->flush_timer_on, 0, 1) == 0)
+		mod_timer(&dom->flush_timer, jiffies + msecs_to_jiffies(10));
+
 	put_cpu_ptr(dom->flush_queue);
 }
 
+static void queue_flush_timeout(unsigned long data)
+{
+	struct dma_ops_domain *dom = (struct dma_ops_domain *)data;
+	int cpu;
+
+	atomic_set(&dom->flush_timer_on, 0);
+
+	dma_ops_domain_flush_tlb(dom);
+
+	for_each_possible_cpu(cpu) {
+		struct flush_queue *queue;
+		unsigned long flags;
+
+		queue = per_cpu_ptr(dom->flush_queue, cpu);
+		spin_lock_irqsave(&queue->lock, flags);
+		queue_ring_free_flushed(dom, queue);
+		spin_unlock_irqrestore(&queue->lock, flags);
+	}
+}
+
 /*
  * Free a domain, only used if something went wrong in the
  * allocation path and we need to free an already allocated page table
@@ -1935,6 +1977,9 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
 
 	del_domain_from_list(&dom->domain);
 
+	if (timer_pending(&dom->flush_timer))
+		del_timer(&dom->flush_timer);
+
 	dma_ops_domain_free_flush_queue(dom);
 
 	put_iova_domain(&dom->iovad);
@@ -1978,6 +2023,11 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
 	if (dma_ops_domain_alloc_flush_queue(dma_dom))
 		goto free_dma_dom;
 
+	setup_timer(&dma_dom->flush_timer, queue_flush_timeout,
+		    (unsigned long)dma_dom);
+
+	atomic_set(&dma_dom->flush_timer_on, 0);
+
 	add_domain_to_list(&dma_dom->domain);
 
 	return dma_dom;