@@ -58,7 +58,7 @@ DEFINE_WD_CLASS(reservation_ww_class);
EXPORT_SYMBOL(reservation_ww_class);
/* Mask for the lower fence pointer bits */
-#define DMA_RESV_LIST_MASK 0x3
+#define DMA_RESV_LIST_MASK 0x7
struct dma_resv_list {
struct rcu_head rcu;
@@ -288,6 +288,10 @@ void dma_resv_add_fence(struct dma_resv *obj, struct dma_fence *fence,
*/
WARN_ON(dma_fence_is_container(fence));
+ /* User fences must be added using DMA_RESV_USAGE_USER */
+ WARN_ON(test_bit(DMA_FENCE_FLAG_USER, &fence->flags) !=
+ (usage == DMA_RESV_USAGE_USER));
+
fobj = dma_resv_fences_list(obj);
count = fobj->num_fences;
@@ -349,6 +353,15 @@ void dma_resv_replace_fences(struct dma_resv *obj, uint64_t context,
}
EXPORT_SYMBOL(dma_resv_replace_fences);
+/* Matches requested usage with the fence usage for iterators */
+static bool dma_resv_iter_match_usage(struct dma_resv_iter *cursor)
+{
+ if (cursor->usage == DMA_RESV_USAGE_USER)
+ return cursor->fence_usage == DMA_RESV_USAGE_USER;
+
+ return cursor->usage >= cursor->fence_usage;
+}
+
/* Restart the unlocked iteration by initializing the cursor object. */
static void dma_resv_iter_restart_unlocked(struct dma_resv_iter *cursor)
{
@@ -385,8 +398,7 @@ static void dma_resv_iter_walk_unlocked(struct dma_resv_iter *cursor)
continue;
}
- if (!dma_fence_is_signaled(cursor->fence) &&
- cursor->usage >= cursor->fence_usage)
+ if (dma_resv_iter_match_usage(cursor))
break;
} while (true);
}
@@ -405,14 +417,9 @@ static void dma_resv_iter_walk_unlocked(struct dma_resv_iter *cursor)
*/
struct dma_fence *dma_resv_iter_first_unlocked(struct dma_resv_iter *cursor)
{
- rcu_read_lock();
- do {
- dma_resv_iter_restart_unlocked(cursor);
- dma_resv_iter_walk_unlocked(cursor);
- } while (dma_resv_fences_list(cursor->obj) != cursor->fences);
- rcu_read_unlock();
-
- return cursor->fence;
+ /* Force a restart */
+ cursor->fences = NULL;
+ return dma_resv_iter_next_unlocked(cursor);
}
EXPORT_SYMBOL(dma_resv_iter_first_unlocked);
@@ -428,18 +435,21 @@ EXPORT_SYMBOL(dma_resv_iter_first_unlocked);
*/
struct dma_fence *dma_resv_iter_next_unlocked(struct dma_resv_iter *cursor)
{
- bool restart;
-
- rcu_read_lock();
cursor->is_restarted = false;
- restart = dma_resv_fences_list(cursor->obj) != cursor->fences;
do {
- if (restart)
- dma_resv_iter_restart_unlocked(cursor);
- dma_resv_iter_walk_unlocked(cursor);
- restart = true;
- } while (dma_resv_fences_list(cursor->obj) != cursor->fences);
- rcu_read_unlock();
+ bool restart;
+
+ rcu_read_lock();
+ restart = dma_resv_fences_list(cursor->obj) != cursor->fences;
+ do {
+ if (restart)
+ dma_resv_iter_restart_unlocked(cursor);
+ dma_resv_iter_walk_unlocked(cursor);
+ restart = true;
+ } while (dma_resv_fences_list(cursor->obj) != cursor->fences);
+ rcu_read_unlock();
+
+ } while (cursor->fence && dma_fence_is_signaled(cursor->fence));
return cursor->fence;
}
@@ -491,7 +501,7 @@ struct dma_fence *dma_resv_iter_next(struct dma_resv_iter *cursor)
dma_resv_list_entry(cursor->fences, cursor->index++,
cursor->obj, &fence, &cursor->fence_usage);
- } while (cursor->fence_usage > cursor->usage);
+ } while (!dma_resv_iter_match_usage(cursor));
return fence;
}
@@ -663,6 +673,9 @@ long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage,
struct dma_resv_iter cursor;
struct dma_fence *fence;
+ if (usage == DMA_RESV_USAGE_USER)
+ lockdep_assert_none_held_once();
+
dma_resv_iter_begin(&cursor, obj, usage);
dma_resv_for_each_fence_unlocked(&cursor, fence) {
@@ -678,7 +691,6 @@ long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage,
}
EXPORT_SYMBOL_GPL(dma_resv_wait_timeout);
-
/**
* dma_resv_test_signaled - Test if a reservation object's fences have been
* signaled.
@@ -717,7 +729,9 @@ EXPORT_SYMBOL_GPL(dma_resv_test_signaled);
*/
void dma_resv_describe(struct dma_resv *obj, struct seq_file *seq)
{
- static const char *usage[] = { "kernel", "write", "read", "bookkeep" };
+ static const char *usage[] = {
+ "kernel", "write", "read", "bookkeep", "user"
+ };
struct dma_resv_iter cursor;
struct dma_fence *fence;
@@ -42,7 +42,6 @@
#include <linux/ww_mutex.h>
#include <linux/dma-fence.h>
#include <linux/slab.h>
-#include <linux/seqlock.h>
#include <linux/rcupdate.h>
extern struct ww_class reservation_ww_class;
@@ -57,11 +56,15 @@ struct dma_resv_list;
*
* An important fact is that there is the order KERNEL<WRITE<READ<BOOKKEEP and
* when the dma_resv object is asked for fences for one use case the fences
- * for the lower use case are returned as well.
+ * for the lower use case are returned as well. The exception are USER fences
+ * which only return USER fences and nothing else.
*
* For example when asking for WRITE fences then the KERNEL fences are returned
* as well. Similar when asked for READ fences then both WRITE and KERNEL
* fences are returned as well.
+ *
+ * But when asked for USER fences only USER fences are returned and not WRITE
+ * nor any other fences.
*/
enum dma_resv_usage {
/**
@@ -103,7 +106,18 @@ enum dma_resv_usage {
* The most common case are preemption fences as well as page table
* updates and their TLB flushes.
*/
- DMA_RESV_USAGE_BOOKKEEP
+ DMA_RESV_USAGE_BOOKKEEP,
+
+ /**
+ * @DMA_RESV_USAGE_USER: Special usage for user fences.
+ *
+ * This must only be used with fences which have DMA_FENCE_FLAG_USER
+ * set so that memory mangement completely ignores those fences.
+ *
+ * A warning is raised if a fence with DMA_FENCE_FLAG USER is added with
+ * any other usage than DMA_RESV_USAGE_USER.
+ */
+ DMA_RESV_USAGE_USER
};
/**
@@ -221,6 +235,9 @@ static inline void dma_resv_iter_begin(struct dma_resv_iter *cursor,
struct dma_resv *obj,
enum dma_resv_usage usage)
{
+ if (usage == DMA_RESV_USAGE_USER)
+ lockdep_assert_not_held(&(obj)->lock.base);
+
cursor->obj = obj;
cursor->usage = usage;
cursor->fence = NULL;