diff mbox series

[1/2] dma-buf: add dma_fence_unwrap

Message ID 20220311110244.1245-1-christian.koenig@amd.com (mailing list archive)
State Not Applicable
Headers
Series [1/2] dma-buf: add dma_fence_unwrap |

Commit Message

Christian König March 11, 2022, 11:02 a.m. UTC 
  Add a general purpose helper to deep dive into dma_fence_chain/dma_fence_array
structures and iterate over all the fences in them.

This is useful when we need to flatten out all fences in those structures.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 Documentation/driver-api/dma-buf.rst  |   6 +
 drivers/dma-buf/Makefile              |   1 +
 drivers/dma-buf/selftests.h           |   1 +
 drivers/dma-buf/st-dma-fence-unwrap.c | 279 ++++++++++++++++++++++++++
 include/linux/dma-fence-unwrap.h      |  99 +++++++++
 5 files changed, 386 insertions(+)
 create mode 100644 drivers/dma-buf/st-dma-fence-unwrap.c
 create mode 100644 include/linux/dma-fence-unwrap.h

Comments

kernel test robot March 11, 2022, 3:45 p.m. UTC | #1 
Hi "Christian,

I love your patch! Perhaps something to improve:

[auto build test WARNING on linus/master]
[also build test WARNING on v5.17-rc7 next-20220310]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Christian-K-nig/dma-buf-add-dma_fence_unwrap/20220311-190352
base:   https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git 79b00034e9dcd2b065c1665c8b42f62b6b80a9be
config: microblaze-randconfig-r011-20220310 (https://download.01.org/0day-ci/archive/20220311/202203112305.WcEordi2-lkp@intel.com/config)
compiler: microblaze-linux-gcc (GCC) 11.2.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/0day-ci/linux/commit/088aa14c0f5cad378854823fa661ee145dd2c01b
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Christian-K-nig/dma-buf-add-dma_fence_unwrap/20220311-190352
        git checkout 088aa14c0f5cad378854823fa661ee145dd2c01b
        # save the config file to linux build tree
        mkdir build_dir
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-11.2.0 make.cross O=build_dir ARCH=microblaze SHELL=/bin/bash drivers/dma-buf/

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All warnings (new ones prefixed by >>):

   In file included from drivers/dma-buf/st-dma-fence-unwrap.c:7:
   include/linux/dma-fence-unwrap.h: In function 'dma_fence_unwrap_array':
   include/linux/dma-fence-unwrap.h:44:25: error: implicit declaration of function 'dma_fence_chain_contained'; did you mean 'dma_fence_chain_init'? [-Werror=implicit-function-declaration]
      44 |         cursor->array = dma_fence_chain_contained(cursor->chain);
         |                         ^~~~~~~~~~~~~~~~~~~~~~~~~
         |                         dma_fence_chain_init
   include/linux/dma-fence-unwrap.h:44:23: warning: assignment to 'struct dma_fence *' from 'int' makes pointer from integer without a cast [-Wint-conversion]
      44 |         cursor->array = dma_fence_chain_contained(cursor->chain);
         |                       ^
   include/linux/dma-fence-unwrap.h:46:16: error: implicit declaration of function 'dma_fence_array_first'; did you mean 'dma_fence_array_create'? [-Werror=implicit-function-declaration]
      46 |         return dma_fence_array_first(cursor->array);
         |                ^~~~~~~~~~~~~~~~~~~~~
         |                dma_fence_array_create
   include/linux/dma-fence-unwrap.h:46:16: warning: returning 'int' from a function with return type 'struct dma_fence *' makes pointer from integer without a cast [-Wint-conversion]
      46 |         return dma_fence_array_first(cursor->array);
         |                ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/dma-fence-unwrap.h: In function 'dma_fence_unwrap_next':
   include/linux/dma-fence-unwrap.h:77:15: error: implicit declaration of function 'dma_fence_array_next'; did you mean 'dma_fence_unwrap_next'? [-Werror=implicit-function-declaration]
      77 |         tmp = dma_fence_array_next(cursor->array, cursor->index);
         |               ^~~~~~~~~~~~~~~~~~~~
         |               dma_fence_unwrap_next
   include/linux/dma-fence-unwrap.h:77:13: warning: assignment to 'struct dma_fence *' from 'int' makes pointer from integer without a cast [-Wint-conversion]
      77 |         tmp = dma_fence_array_next(cursor->array, cursor->index);
         |             ^
   drivers/dma-buf/st-dma-fence-unwrap.c: In function 'unwrap_array':
>> drivers/dma-buf/st-dma-fence-unwrap.c:133:13: warning: variable 'err' set but not used [-Wunused-but-set-variable]
     133 |         int err = 0;
         |             ^~~
   drivers/dma-buf/st-dma-fence-unwrap.c: In function 'unwrap_chain':
   drivers/dma-buf/st-dma-fence-unwrap.c:175:13: warning: variable 'err' set but not used [-Wunused-but-set-variable]
     175 |         int err = 0;
         |             ^~~
   drivers/dma-buf/st-dma-fence-unwrap.c: In function 'unwrap_chain_array':
   drivers/dma-buf/st-dma-fence-unwrap.c:217:13: warning: variable 'err' set but not used [-Wunused-but-set-variable]
     217 |         int err = 0;
         |             ^~~
   cc1: some warnings being treated as errors


vim +/err +133 drivers/dma-buf/st-dma-fence-unwrap.c

   128	
   129	static int unwrap_array(void *arg)
   130	{
   131		struct dma_fence *fence, *f1, *f2, *array;
   132		struct dma_fence_unwrap iter;
 > 133		int err = 0;
   134	
   135		f1 = mock_fence();
   136		if (!f1)
   137			return -ENOMEM;
   138	
   139		f2 = mock_fence();
   140		if (!f2) {
   141			dma_fence_put(f1);
   142			return -ENOMEM;
   143		}
   144	
   145		array = mock_array(2, f1, f2);
   146		if (!array)
   147			return -ENOMEM;
   148	
   149		dma_fence_unwrap_for_each(fence, &iter, array) {
   150			if (fence == f1) {
   151				f1 = NULL;
   152			} else if (fence == f2) {
   153				f2 = NULL;
   154			} else {
   155				pr_err("Unexpected fence!\n");
   156				err = -EINVAL;
   157			}
   158		}
   159	
   160		if (f1 || f2) {
   161			pr_err("Not all fences seen!\n");
   162			err = -EINVAL;
   163		}
   164	
   165		dma_fence_signal(f1);
   166		dma_fence_signal(f2);
   167		dma_fence_put(array);
   168		return 0;
   169	}
   170	

---
0-DAY CI Kernel Test Service
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
kernel test robot March 11, 2022, 5:28 p.m. UTC | #2 
Hi "Christian,

I love your patch! Perhaps something to improve:

[auto build test WARNING on linus/master]
[also build test WARNING on v5.17-rc7 next-20220310]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Christian-K-nig/dma-buf-add-dma_fence_unwrap/20220311-190352
base:   https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git 79b00034e9dcd2b065c1665c8b42f62b6b80a9be
config: hexagon-randconfig-r045-20220310 (https://download.01.org/0day-ci/archive/20220312/202203120114.jpyQmShC-lkp@intel.com/config)
compiler: clang version 15.0.0 (https://github.com/llvm/llvm-project 276ca87382b8f16a65bddac700202924228982f6)
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/0day-ci/linux/commit/088aa14c0f5cad378854823fa661ee145dd2c01b
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Christian-K-nig/dma-buf-add-dma_fence_unwrap/20220311-190352
        git checkout 088aa14c0f5cad378854823fa661ee145dd2c01b
        # save the config file to linux build tree
        mkdir build_dir
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=hexagon SHELL=/bin/bash drivers/dma-buf/

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All warnings (new ones prefixed by >>):

   In file included from drivers/dma-buf/st-dma-fence-unwrap.c:7:
   include/linux/dma-fence-unwrap.h:44:18: error: implicit declaration of function 'dma_fence_chain_contained' [-Werror,-Wimplicit-function-declaration]
           cursor->array = dma_fence_chain_contained(cursor->chain);
                           ^
   include/linux/dma-fence-unwrap.h:44:18: note: did you mean 'dma_fence_chain_init'?
   include/linux/dma-fence-chain.h:108:6: note: 'dma_fence_chain_init' declared here
   void dma_fence_chain_init(struct dma_fence_chain *chain,
        ^
   In file included from drivers/dma-buf/st-dma-fence-unwrap.c:7:
>> include/linux/dma-fence-unwrap.h:44:16: warning: incompatible integer to pointer conversion assigning to 'struct dma_fence *' from 'int' [-Wint-conversion]
           cursor->array = dma_fence_chain_contained(cursor->chain);
                         ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/dma-fence-unwrap.h:46:9: error: implicit declaration of function 'dma_fence_array_first' [-Werror,-Wimplicit-function-declaration]
           return dma_fence_array_first(cursor->array);
                  ^
   include/linux/dma-fence-unwrap.h:46:9: note: did you mean 'dma_fence_array_create'?
   include/linux/dma-fence-array.h:77:25: note: 'dma_fence_array_create' declared here
   struct dma_fence_array *dma_fence_array_create(int num_fences,
                           ^
   In file included from drivers/dma-buf/st-dma-fence-unwrap.c:7:
>> include/linux/dma-fence-unwrap.h:46:9: warning: incompatible integer to pointer conversion returning 'int' from a function with result type 'struct dma_fence *' [-Wint-conversion]
           return dma_fence_array_first(cursor->array);
                  ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/dma-fence-unwrap.h:77:8: error: implicit declaration of function 'dma_fence_array_next' [-Werror,-Wimplicit-function-declaration]
           tmp = dma_fence_array_next(cursor->array, cursor->index);
                 ^
   include/linux/dma-fence-unwrap.h:77:6: warning: incompatible integer to pointer conversion assigning to 'struct dma_fence *' from 'int' [-Wint-conversion]
           tmp = dma_fence_array_next(cursor->array, cursor->index);
               ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> drivers/dma-buf/st-dma-fence-unwrap.c:133:6: warning: variable 'err' set but not used [-Wunused-but-set-variable]
           int err = 0;
               ^
   drivers/dma-buf/st-dma-fence-unwrap.c:175:6: warning: variable 'err' set but not used [-Wunused-but-set-variable]
           int err = 0;
               ^
   drivers/dma-buf/st-dma-fence-unwrap.c:217:6: warning: variable 'err' set but not used [-Wunused-but-set-variable]
           int err = 0;
               ^
   6 warnings and 3 errors generated.


vim +44 include/linux/dma-fence-unwrap.h

    33	
    34	/**
    35	 * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
    36	 * @cursor: cursor to initialize
    37	 *
    38	 * Helper function to unwrap dma_fence_array containers, don't touch directly.
    39	 * Use dma_fence_unwrap_first/next instead.
    40	 */
    41	static inline struct dma_fence *
    42	dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)
    43	{
  > 44		cursor->array = dma_fence_chain_contained(cursor->chain);
    45		cursor->index = 0;
  > 46		return dma_fence_array_first(cursor->array);
    47	}
    48	

---
0-DAY CI Kernel Test Service
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
Christian König March 14, 2022, 11:14 a.m. UTC | #3 
Just a gentle ping.

This series is an important fix for drm-misc-next-fixes.

Regards,
Christian.

Am 11.03.22 um 12:02 schrieb Christian König:
> Add a general purpose helper to deep dive into dma_fence_chain/dma_fence_array
> structures and iterate over all the fences in them.
>
> This is useful when we need to flatten out all fences in those structures.
>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>   Documentation/driver-api/dma-buf.rst  |   6 +
>   drivers/dma-buf/Makefile              |   1 +
>   drivers/dma-buf/selftests.h           |   1 +
>   drivers/dma-buf/st-dma-fence-unwrap.c | 279 ++++++++++++++++++++++++++
>   include/linux/dma-fence-unwrap.h      |  99 +++++++++
>   5 files changed, 386 insertions(+)
>   create mode 100644 drivers/dma-buf/st-dma-fence-unwrap.c
>   create mode 100644 include/linux/dma-fence-unwrap.h
>
> diff --git a/Documentation/driver-api/dma-buf.rst b/Documentation/driver-api/dma-buf.rst
> index 2cd7db82d9fe..7209500f08c8 100644
> --- a/Documentation/driver-api/dma-buf.rst
> +++ b/Documentation/driver-api/dma-buf.rst
> @@ -194,6 +194,12 @@ DMA Fence Chain
>   .. kernel-doc:: include/linux/dma-fence-chain.h
>      :internal:
>   
> +DMA Fence unwrap
> +~~~~~~~~~~~~~~~~
> +
> +.. kernel-doc:: include/linux/dma-fence-unwrap.h
> +   :internal:
> +
>   DMA Fence uABI/Sync File
>   ~~~~~~~~~~~~~~~~~~~~~~~~
>   
> diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
> index 511805dbeb75..4c9eb53ba3f8 100644
> --- a/drivers/dma-buf/Makefile
> +++ b/drivers/dma-buf/Makefile
> @@ -12,6 +12,7 @@ dmabuf_selftests-y := \
>   	selftest.o \
>   	st-dma-fence.o \
>   	st-dma-fence-chain.o \
> +	st-dma-fence-unwrap.o \
>   	st-dma-resv.o
>   
>   obj-$(CONFIG_DMABUF_SELFTESTS)	+= dmabuf_selftests.o
> diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
> index 97d73aaa31da..851965867d9c 100644
> --- a/drivers/dma-buf/selftests.h
> +++ b/drivers/dma-buf/selftests.h
> @@ -12,4 +12,5 @@
>   selftest(sanitycheck, __sanitycheck__) /* keep first (igt selfcheck) */
>   selftest(dma_fence, dma_fence)
>   selftest(dma_fence_chain, dma_fence_chain)
> +selftest(dma_fence_unwrap, dma_fence_unwrap)
>   selftest(dma_resv, dma_resv)
> diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
> new file mode 100644
> index 000000000000..d821faaebe93
> --- /dev/null
> +++ b/drivers/dma-buf/st-dma-fence-unwrap.c
> @@ -0,0 +1,279 @@
> +// SPDX-License-Identifier: MIT
> +
> +/*
> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
> + */
> +
> +#include <linux/dma-fence-unwrap.h>
> +#if 0
> +#include <linux/kernel.h>
> +#include <linux/kthread.h>
> +#include <linux/mm.h>
> +#include <linux/sched/signal.h>
> +#include <linux/slab.h>
> +#include <linux/spinlock.h>
> +#include <linux/random.h>
> +#endif
> +
> +#include "selftest.h"
> +
> +#define CHAIN_SZ (4 << 10)
> +
> +static struct kmem_cache *slab_fences;
> +
> +static inline struct mock_fence {
> +	struct dma_fence base;
> +	spinlock_t lock;
> +} *to_mock_fence(struct dma_fence *f) {
> +	return container_of(f, struct mock_fence, base);
> +}
> +
> +static const char *mock_name(struct dma_fence *f)
> +{
> +	return "mock";
> +}
> +
> +static void mock_fence_release(struct dma_fence *f)
> +{
> +	kmem_cache_free(slab_fences, to_mock_fence(f));
> +}
> +
> +static const struct dma_fence_ops mock_ops = {
> +	.get_driver_name = mock_name,
> +	.get_timeline_name = mock_name,
> +	.release = mock_fence_release,
> +};
> +
> +static struct dma_fence *mock_fence(void)
> +{
> +	struct mock_fence *f;
> +
> +	f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
> +	if (!f)
> +		return NULL;
> +
> +	spin_lock_init(&f->lock);
> +	dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
> +
> +	return &f->base;
> +}
> +
> +static struct dma_fence *mock_array(unsigned int num_fences, ...)
> +{
> +	struct dma_fence_array *array;
> +	struct dma_fence **fences;
> +	va_list valist;
> +	int i;
> +
> +	fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
> +	if (!fences)
> +		return NULL;
> +
> +	va_start(valist, num_fences);
> +	for (i = 0; i < num_fences; ++i)
> +		fences[i] = va_arg(valist, typeof(*fences));
> +	va_end(valist);
> +
> +	array = dma_fence_array_create(num_fences, fences,
> +				       dma_fence_context_alloc(1),
> +				       1, false);
> +	if (!array)
> +		goto cleanup;
> +	return &array->base;
> +
> +cleanup:
> +	for (i = 0; i < num_fences; ++i)
> +		dma_fence_put(fences[i]);
> +	kfree(fences);
> +	return NULL;
> +}
> +
> +static struct dma_fence *mock_chain(struct dma_fence *prev,
> +				    struct dma_fence *fence)
> +{
> +	struct dma_fence_chain *f;
> +
> +	f = dma_fence_chain_alloc();
> +	if (!f) {
> +		dma_fence_put(prev);
> +		dma_fence_put(fence);
> +		return NULL;
> +	}
> +
> +	dma_fence_chain_init(f, prev, fence, 1);
> +	return &f->base;
> +}
> +
> +static int sanitycheck(void *arg)
> +{
> +	struct dma_fence *f, *chain, *array;
> +	int err = 0;
> +
> +	f = mock_fence();
> +	if (!f)
> +		return -ENOMEM;
> +
> +	array = mock_array(1, f);
> +	if (!array)
> +		return -ENOMEM;
> +
> +	chain = mock_chain(NULL, array);
> +	if (!chain)
> +		return -ENOMEM;
> +
> +	dma_fence_signal(f);
> +	dma_fence_put(chain);
> +	return err;
> +}
> +
> +static int unwrap_array(void *arg)
> +{
> +	struct dma_fence *fence, *f1, *f2, *array;
> +	struct dma_fence_unwrap iter;
> +	int err = 0;
> +
> +	f1 = mock_fence();
> +	if (!f1)
> +		return -ENOMEM;
> +
> +	f2 = mock_fence();
> +	if (!f2) {
> +		dma_fence_put(f1);
> +		return -ENOMEM;
> +	}
> +
> +	array = mock_array(2, f1, f2);
> +	if (!array)
> +		return -ENOMEM;
> +
> +	dma_fence_unwrap_for_each(fence, &iter, array) {
> +		if (fence == f1) {
> +			f1 = NULL;
> +		} else if (fence == f2) {
> +			f2 = NULL;
> +		} else {
> +			pr_err("Unexpected fence!\n");
> +			err = -EINVAL;
> +		}
> +	}
> +
> +	if (f1 || f2) {
> +		pr_err("Not all fences seen!\n");
> +		err = -EINVAL;
> +	}
> +
> +	dma_fence_signal(f1);
> +	dma_fence_signal(f2);
> +	dma_fence_put(array);
> +	return 0;
> +}
> +
> +static int unwrap_chain(void *arg)
> +{
> +	struct dma_fence *fence, *f1, *f2, *chain;
> +	struct dma_fence_unwrap iter;
> +	int err = 0;
> +
> +	f1 = mock_fence();
> +	if (!f1)
> +		return -ENOMEM;
> +
> +	f2 = mock_fence();
> +	if (!f2) {
> +		dma_fence_put(f1);
> +		return -ENOMEM;
> +	}
> +
> +	chain = mock_chain(f1, f2);
> +	if (!chain)
> +		return -ENOMEM;
> +
> +	dma_fence_unwrap_for_each(fence, &iter, chain) {
> +		if (fence == f1) {
> +			f1 = NULL;
> +		} else if (fence == f2) {
> +			f2 = NULL;
> +		} else {
> +			pr_err("Unexpected fence!\n");
> +			err = -EINVAL;
> +		}
> +	}
> +
> +	if (f1 || f2) {
> +		pr_err("Not all fences seen!\n");
> +		err = -EINVAL;
> +	}
> +
> +	dma_fence_signal(f1);
> +	dma_fence_signal(f2);
> +	dma_fence_put(chain);
> +	return 0;
> +}
> +
> +static int unwrap_chain_array(void *arg)
> +{
> +	struct dma_fence *fence, *f1, *f2, *array, *chain;
> +	struct dma_fence_unwrap iter;
> +	int err = 0;
> +
> +	f1 = mock_fence();
> +	if (!f1)
> +		return -ENOMEM;
> +
> +	f2 = mock_fence();
> +	if (!f2) {
> +		dma_fence_put(f1);
> +		return -ENOMEM;
> +	}
> +
> +	array = mock_array(2, f1, f2);
> +	if (!array)
> +		return -ENOMEM;
> +
> +	chain = mock_chain(NULL, array);
> +	if (!chain)
> +		return -ENOMEM;
> +
> +	dma_fence_unwrap_for_each(fence, &iter, chain) {
> +		if (fence == f1) {
> +			f1 = NULL;
> +		} else if (fence == f2) {
> +			f2 = NULL;
> +		} else {
> +			pr_err("Unexpected fence!\n");
> +			err = -EINVAL;
> +		}
> +	}
> +
> +	if (f1 || f2) {
> +		pr_err("Not all fences seen!\n");
> +		err = -EINVAL;
> +	}
> +
> +	dma_fence_signal(f1);
> +	dma_fence_signal(f2);
> +	dma_fence_put(chain);
> +	return 0;
> +}
> +
> +int dma_fence_unwrap(void)
> +{
> +	static const struct subtest tests[] = {
> +		SUBTEST(sanitycheck),
> +		SUBTEST(unwrap_array),
> +		SUBTEST(unwrap_chain),
> +		SUBTEST(unwrap_chain_array),
> +	};
> +	int ret;
> +
> +	slab_fences = KMEM_CACHE(mock_fence,
> +				 SLAB_TYPESAFE_BY_RCU |
> +				 SLAB_HWCACHE_ALIGN);
> +	if (!slab_fences)
> +		return -ENOMEM;
> +
> +	ret = subtests(tests, NULL);
> +
> +	kmem_cache_destroy(slab_fences);
> +	return ret;
> +}
> diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h
> new file mode 100644
> index 000000000000..54963df00c98
> --- /dev/null
> +++ b/include/linux/dma-fence-unwrap.h
> @@ -0,0 +1,99 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * fence-chain: chain fences together in a timeline
> + *
> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
> + * Authors:
> + *	Christian König <christian.koenig@amd.com>
> + */
> +
> +#ifndef __LINUX_DMA_FENCE_UNWRAP_H
> +#define __LINUX_DMA_FENCE_UNWRAP_H
> +
> +#include <linux/dma-fence-chain.h>
> +#include <linux/dma-fence-array.h>
> +
> +/**
> + * struct dma_fence_unwrap - cursor into the container structure
> + */
> +struct dma_fence_unwrap {
> +	/**
> +	 * @chain: potential dma_fence_chain, but can be other fence as well
> +	 */
> +	struct dma_fence *chain;
> +	/**
> +	 * @array: potential dma_fence_array, but can be other fence as well
> +	 */
> +	struct dma_fence *array;
> +	/**
> +	 * @index: last returned index if @array is really a dma_fence_array
> +	 */
> +	unsigned int index;
> +};
> +
> +/**
> + * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
> + * @cursor: cursor to initialize
> + *
> + * Helper function to unwrap dma_fence_array containers, don't touch directly.
> + * Use dma_fence_unwrap_first/next instead.
> + */
> +static inline struct dma_fence *
> +dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)
> +{
> +	cursor->array = dma_fence_chain_contained(cursor->chain);
> +	cursor->index = 0;
> +	return dma_fence_array_first(cursor->array);
> +}
> +
> +/**
> + * dma_fence_unwrap_first - return the first fence from fence containers
> + * @head: the entrypoint into the containers
> + * @cursor: current position inside the containers
> + *
> + * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
> + * first fence.
> + */
> +static inline struct dma_fence *
> +dma_fence_unwrap_first(struct dma_fence *head, struct dma_fence_unwrap *cursor)
> +{
> +	cursor->chain = dma_fence_get(head);
> +	return dma_fence_unwrap_array(cursor);
> +}
> +
> +/**
> + * dma_fence_unwrap_next - return the next fence from a fence containers
> + * @cursor: current position inside the containers
> + *
> + * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
> + * the next fence from them.
> + */
> +static inline struct dma_fence *
> +dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
> +{
> +	struct dma_fence *tmp;
> +
> +	++cursor->index;
> +	tmp = dma_fence_array_next(cursor->array, cursor->index);
> +	if (tmp)
> +		return tmp;
> +
> +	cursor->chain = dma_fence_chain_walk(cursor->chain);
> +	return dma_fence_unwrap_array(cursor);
> +}
> +
> +/**
> + * dma_fence_unwrap_for_each - iterate over all fences in containers
> + * @fence: current fence
> + * @cursor: current position inside the containers
> + * @head: starting point for the iterator
> + *
> + * Unwrap dma_fence_chain and dma_fence_array containers and deep dive into all
> + * potential fences in them. If @head is just a normal fence only that one is
> + * returned.
> + */
> +#define dma_fence_unwrap_for_each(fence, cursor, head)			\
> +	for (fence = dma_fence_unwrap_first(head, cursor); fence;	\
> +	     fence = dma_fence_unwrap_next(cursor))
> +
> +#endif
Christian König March 21, 2022, 10:31 a.m. UTC | #4 
[Adding Daniel]

Just once more a ping for this.

It's an important bug fix and the end user already reported that it works.

Regards,
Christian.

Am 14.03.22 um 12:14 schrieb Christian König:
> Just a gentle ping.
>
> This series is an important fix for drm-misc-next-fixes.
>
> Regards,
> Christian.
>
> Am 11.03.22 um 12:02 schrieb Christian König:
>> Add a general purpose helper to deep dive into 
>> dma_fence_chain/dma_fence_array
>> structures and iterate over all the fences in them.
>>
>> This is useful when we need to flatten out all fences in those 
>> structures.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> ---
>>   Documentation/driver-api/dma-buf.rst  |   6 +
>>   drivers/dma-buf/Makefile              |   1 +
>>   drivers/dma-buf/selftests.h           |   1 +
>>   drivers/dma-buf/st-dma-fence-unwrap.c | 279 ++++++++++++++++++++++++++
>>   include/linux/dma-fence-unwrap.h      |  99 +++++++++
>>   5 files changed, 386 insertions(+)
>>   create mode 100644 drivers/dma-buf/st-dma-fence-unwrap.c
>>   create mode 100644 include/linux/dma-fence-unwrap.h
>>
>> diff --git a/Documentation/driver-api/dma-buf.rst 
>> b/Documentation/driver-api/dma-buf.rst
>> index 2cd7db82d9fe..7209500f08c8 100644
>> --- a/Documentation/driver-api/dma-buf.rst
>> +++ b/Documentation/driver-api/dma-buf.rst
>> @@ -194,6 +194,12 @@ DMA Fence Chain
>>   .. kernel-doc:: include/linux/dma-fence-chain.h
>>      :internal:
>>   +DMA Fence unwrap
>> +~~~~~~~~~~~~~~~~
>> +
>> +.. kernel-doc:: include/linux/dma-fence-unwrap.h
>> +   :internal:
>> +
>>   DMA Fence uABI/Sync File
>>   ~~~~~~~~~~~~~~~~~~~~~~~~
>>   diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
>> index 511805dbeb75..4c9eb53ba3f8 100644
>> --- a/drivers/dma-buf/Makefile
>> +++ b/drivers/dma-buf/Makefile
>> @@ -12,6 +12,7 @@ dmabuf_selftests-y := \
>>       selftest.o \
>>       st-dma-fence.o \
>>       st-dma-fence-chain.o \
>> +    st-dma-fence-unwrap.o \
>>       st-dma-resv.o
>>     obj-$(CONFIG_DMABUF_SELFTESTS)    += dmabuf_selftests.o
>> diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
>> index 97d73aaa31da..851965867d9c 100644
>> --- a/drivers/dma-buf/selftests.h
>> +++ b/drivers/dma-buf/selftests.h
>> @@ -12,4 +12,5 @@
>>   selftest(sanitycheck, __sanitycheck__) /* keep first (igt 
>> selfcheck) */
>>   selftest(dma_fence, dma_fence)
>>   selftest(dma_fence_chain, dma_fence_chain)
>> +selftest(dma_fence_unwrap, dma_fence_unwrap)
>>   selftest(dma_resv, dma_resv)
>> diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c 
>> b/drivers/dma-buf/st-dma-fence-unwrap.c
>> new file mode 100644
>> index 000000000000..d821faaebe93
>> --- /dev/null
>> +++ b/drivers/dma-buf/st-dma-fence-unwrap.c
>> @@ -0,0 +1,279 @@
>> +// SPDX-License-Identifier: MIT
>> +
>> +/*
>> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
>> + */
>> +
>> +#include <linux/dma-fence-unwrap.h>
>> +#if 0
>> +#include <linux/kernel.h>
>> +#include <linux/kthread.h>
>> +#include <linux/mm.h>
>> +#include <linux/sched/signal.h>
>> +#include <linux/slab.h>
>> +#include <linux/spinlock.h>
>> +#include <linux/random.h>
>> +#endif
>> +
>> +#include "selftest.h"
>> +
>> +#define CHAIN_SZ (4 << 10)
>> +
>> +static struct kmem_cache *slab_fences;
>> +
>> +static inline struct mock_fence {
>> +    struct dma_fence base;
>> +    spinlock_t lock;
>> +} *to_mock_fence(struct dma_fence *f) {
>> +    return container_of(f, struct mock_fence, base);
>> +}
>> +
>> +static const char *mock_name(struct dma_fence *f)
>> +{
>> +    return "mock";
>> +}
>> +
>> +static void mock_fence_release(struct dma_fence *f)
>> +{
>> +    kmem_cache_free(slab_fences, to_mock_fence(f));
>> +}
>> +
>> +static const struct dma_fence_ops mock_ops = {
>> +    .get_driver_name = mock_name,
>> +    .get_timeline_name = mock_name,
>> +    .release = mock_fence_release,
>> +};
>> +
>> +static struct dma_fence *mock_fence(void)
>> +{
>> +    struct mock_fence *f;
>> +
>> +    f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
>> +    if (!f)
>> +        return NULL;
>> +
>> +    spin_lock_init(&f->lock);
>> +    dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
>> +
>> +    return &f->base;
>> +}
>> +
>> +static struct dma_fence *mock_array(unsigned int num_fences, ...)
>> +{
>> +    struct dma_fence_array *array;
>> +    struct dma_fence **fences;
>> +    va_list valist;
>> +    int i;
>> +
>> +    fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
>> +    if (!fences)
>> +        return NULL;
>> +
>> +    va_start(valist, num_fences);
>> +    for (i = 0; i < num_fences; ++i)
>> +        fences[i] = va_arg(valist, typeof(*fences));
>> +    va_end(valist);
>> +
>> +    array = dma_fence_array_create(num_fences, fences,
>> +                       dma_fence_context_alloc(1),
>> +                       1, false);
>> +    if (!array)
>> +        goto cleanup;
>> +    return &array->base;
>> +
>> +cleanup:
>> +    for (i = 0; i < num_fences; ++i)
>> +        dma_fence_put(fences[i]);
>> +    kfree(fences);
>> +    return NULL;
>> +}
>> +
>> +static struct dma_fence *mock_chain(struct dma_fence *prev,
>> +                    struct dma_fence *fence)
>> +{
>> +    struct dma_fence_chain *f;
>> +
>> +    f = dma_fence_chain_alloc();
>> +    if (!f) {
>> +        dma_fence_put(prev);
>> +        dma_fence_put(fence);
>> +        return NULL;
>> +    }
>> +
>> +    dma_fence_chain_init(f, prev, fence, 1);
>> +    return &f->base;
>> +}
>> +
>> +static int sanitycheck(void *arg)
>> +{
>> +    struct dma_fence *f, *chain, *array;
>> +    int err = 0;
>> +
>> +    f = mock_fence();
>> +    if (!f)
>> +        return -ENOMEM;
>> +
>> +    array = mock_array(1, f);
>> +    if (!array)
>> +        return -ENOMEM;
>> +
>> +    chain = mock_chain(NULL, array);
>> +    if (!chain)
>> +        return -ENOMEM;
>> +
>> +    dma_fence_signal(f);
>> +    dma_fence_put(chain);
>> +    return err;
>> +}
>> +
>> +static int unwrap_array(void *arg)
>> +{
>> +    struct dma_fence *fence, *f1, *f2, *array;
>> +    struct dma_fence_unwrap iter;
>> +    int err = 0;
>> +
>> +    f1 = mock_fence();
>> +    if (!f1)
>> +        return -ENOMEM;
>> +
>> +    f2 = mock_fence();
>> +    if (!f2) {
>> +        dma_fence_put(f1);
>> +        return -ENOMEM;
>> +    }
>> +
>> +    array = mock_array(2, f1, f2);
>> +    if (!array)
>> +        return -ENOMEM;
>> +
>> +    dma_fence_unwrap_for_each(fence, &iter, array) {
>> +        if (fence == f1) {
>> +            f1 = NULL;
>> +        } else if (fence == f2) {
>> +            f2 = NULL;
>> +        } else {
>> +            pr_err("Unexpected fence!\n");
>> +            err = -EINVAL;
>> +        }
>> +    }
>> +
>> +    if (f1 || f2) {
>> +        pr_err("Not all fences seen!\n");
>> +        err = -EINVAL;
>> +    }
>> +
>> +    dma_fence_signal(f1);
>> +    dma_fence_signal(f2);
>> +    dma_fence_put(array);
>> +    return 0;
>> +}
>> +
>> +static int unwrap_chain(void *arg)
>> +{
>> +    struct dma_fence *fence, *f1, *f2, *chain;
>> +    struct dma_fence_unwrap iter;
>> +    int err = 0;
>> +
>> +    f1 = mock_fence();
>> +    if (!f1)
>> +        return -ENOMEM;
>> +
>> +    f2 = mock_fence();
>> +    if (!f2) {
>> +        dma_fence_put(f1);
>> +        return -ENOMEM;
>> +    }
>> +
>> +    chain = mock_chain(f1, f2);
>> +    if (!chain)
>> +        return -ENOMEM;
>> +
>> +    dma_fence_unwrap_for_each(fence, &iter, chain) {
>> +        if (fence == f1) {
>> +            f1 = NULL;
>> +        } else if (fence == f2) {
>> +            f2 = NULL;
>> +        } else {
>> +            pr_err("Unexpected fence!\n");
>> +            err = -EINVAL;
>> +        }
>> +    }
>> +
>> +    if (f1 || f2) {
>> +        pr_err("Not all fences seen!\n");
>> +        err = -EINVAL;
>> +    }
>> +
>> +    dma_fence_signal(f1);
>> +    dma_fence_signal(f2);
>> +    dma_fence_put(chain);
>> +    return 0;
>> +}
>> +
>> +static int unwrap_chain_array(void *arg)
>> +{
>> +    struct dma_fence *fence, *f1, *f2, *array, *chain;
>> +    struct dma_fence_unwrap iter;
>> +    int err = 0;
>> +
>> +    f1 = mock_fence();
>> +    if (!f1)
>> +        return -ENOMEM;
>> +
>> +    f2 = mock_fence();
>> +    if (!f2) {
>> +        dma_fence_put(f1);
>> +        return -ENOMEM;
>> +    }
>> +
>> +    array = mock_array(2, f1, f2);
>> +    if (!array)
>> +        return -ENOMEM;
>> +
>> +    chain = mock_chain(NULL, array);
>> +    if (!chain)
>> +        return -ENOMEM;
>> +
>> +    dma_fence_unwrap_for_each(fence, &iter, chain) {
>> +        if (fence == f1) {
>> +            f1 = NULL;
>> +        } else if (fence == f2) {
>> +            f2 = NULL;
>> +        } else {
>> +            pr_err("Unexpected fence!\n");
>> +            err = -EINVAL;
>> +        }
>> +    }
>> +
>> +    if (f1 || f2) {
>> +        pr_err("Not all fences seen!\n");
>> +        err = -EINVAL;
>> +    }
>> +
>> +    dma_fence_signal(f1);
>> +    dma_fence_signal(f2);
>> +    dma_fence_put(chain);
>> +    return 0;
>> +}
>> +
>> +int dma_fence_unwrap(void)
>> +{
>> +    static const struct subtest tests[] = {
>> +        SUBTEST(sanitycheck),
>> +        SUBTEST(unwrap_array),
>> +        SUBTEST(unwrap_chain),
>> +        SUBTEST(unwrap_chain_array),
>> +    };
>> +    int ret;
>> +
>> +    slab_fences = KMEM_CACHE(mock_fence,
>> +                 SLAB_TYPESAFE_BY_RCU |
>> +                 SLAB_HWCACHE_ALIGN);
>> +    if (!slab_fences)
>> +        return -ENOMEM;
>> +
>> +    ret = subtests(tests, NULL);
>> +
>> +    kmem_cache_destroy(slab_fences);
>> +    return ret;
>> +}
>> diff --git a/include/linux/dma-fence-unwrap.h 
>> b/include/linux/dma-fence-unwrap.h
>> new file mode 100644
>> index 000000000000..54963df00c98
>> --- /dev/null
>> +++ b/include/linux/dma-fence-unwrap.h
>> @@ -0,0 +1,99 @@
>> +/* SPDX-License-Identifier: GPL-2.0-only */
>> +/*
>> + * fence-chain: chain fences together in a timeline
>> + *
>> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
>> + * Authors:
>> + *    Christian König <christian.koenig@amd.com>
>> + */
>> +
>> +#ifndef __LINUX_DMA_FENCE_UNWRAP_H
>> +#define __LINUX_DMA_FENCE_UNWRAP_H
>> +
>> +#include <linux/dma-fence-chain.h>
>> +#include <linux/dma-fence-array.h>
>> +
>> +/**
>> + * struct dma_fence_unwrap - cursor into the container structure
>> + */
>> +struct dma_fence_unwrap {
>> +    /**
>> +     * @chain: potential dma_fence_chain, but can be other fence as 
>> well
>> +     */
>> +    struct dma_fence *chain;
>> +    /**
>> +     * @array: potential dma_fence_array, but can be other fence as 
>> well
>> +     */
>> +    struct dma_fence *array;
>> +    /**
>> +     * @index: last returned index if @array is really a 
>> dma_fence_array
>> +     */
>> +    unsigned int index;
>> +};
>> +
>> +/**
>> + * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
>> + * @cursor: cursor to initialize
>> + *
>> + * Helper function to unwrap dma_fence_array containers, don't touch 
>> directly.
>> + * Use dma_fence_unwrap_first/next instead.
>> + */
>> +static inline struct dma_fence *
>> +dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)
>> +{
>> +    cursor->array = dma_fence_chain_contained(cursor->chain);
>> +    cursor->index = 0;
>> +    return dma_fence_array_first(cursor->array);
>> +}
>> +
>> +/**
>> + * dma_fence_unwrap_first - return the first fence from fence 
>> containers
>> + * @head: the entrypoint into the containers
>> + * @cursor: current position inside the containers
>> + *
>> + * Unwraps potential dma_fence_chain/dma_fence_array containers and 
>> return the
>> + * first fence.
>> + */
>> +static inline struct dma_fence *
>> +dma_fence_unwrap_first(struct dma_fence *head, struct 
>> dma_fence_unwrap *cursor)
>> +{
>> +    cursor->chain = dma_fence_get(head);
>> +    return dma_fence_unwrap_array(cursor);
>> +}
>> +
>> +/**
>> + * dma_fence_unwrap_next - return the next fence from a fence 
>> containers
>> + * @cursor: current position inside the containers
>> + *
>> + * Continue unwrapping the dma_fence_chain/dma_fence_array 
>> containers and return
>> + * the next fence from them.
>> + */
>> +static inline struct dma_fence *
>> +dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
>> +{
>> +    struct dma_fence *tmp;
>> +
>> +    ++cursor->index;
>> +    tmp = dma_fence_array_next(cursor->array, cursor->index);
>> +    if (tmp)
>> +        return tmp;
>> +
>> +    cursor->chain = dma_fence_chain_walk(cursor->chain);
>> +    return dma_fence_unwrap_array(cursor);
>> +}
>> +
>> +/**
>> + * dma_fence_unwrap_for_each - iterate over all fences in containers
>> + * @fence: current fence
>> + * @cursor: current position inside the containers
>> + * @head: starting point for the iterator
>> + *
>> + * Unwrap dma_fence_chain and dma_fence_array containers and deep 
>> dive into all
>> + * potential fences in them. If @head is just a normal fence only 
>> that one is
>> + * returned.
>> + */
>> +#define dma_fence_unwrap_for_each(fence, cursor, head)            \
>> +    for (fence = dma_fence_unwrap_first(head, cursor); fence;    \
>> +         fence = dma_fence_unwrap_next(cursor))
>> +
>> +#endif
>
Christian König March 25, 2022, 7:42 a.m. UTC | #5 
Once more a ping on this here.

It's fixing a warning with sync_files and is already tested.

Regards,
Christian.

Am 21.03.22 um 11:31 schrieb Christian König:
> [Adding Daniel]
>
> Just once more a ping for this.
>
> It's an important bug fix and the end user already reported that it 
> works.
>
> Regards,
> Christian.
>
> Am 14.03.22 um 12:14 schrieb Christian König:
>> Just a gentle ping.
>>
>> This series is an important fix for drm-misc-next-fixes.
>>
>> Regards,
>> Christian.
>>
>> Am 11.03.22 um 12:02 schrieb Christian König:
>>> Add a general purpose helper to deep dive into 
>>> dma_fence_chain/dma_fence_array
>>> structures and iterate over all the fences in them.
>>>
>>> This is useful when we need to flatten out all fences in those 
>>> structures.
>>>
>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>> ---
>>>   Documentation/driver-api/dma-buf.rst  |   6 +
>>>   drivers/dma-buf/Makefile              |   1 +
>>>   drivers/dma-buf/selftests.h           |   1 +
>>>   drivers/dma-buf/st-dma-fence-unwrap.c | 279 
>>> ++++++++++++++++++++++++++
>>>   include/linux/dma-fence-unwrap.h      |  99 +++++++++
>>>   5 files changed, 386 insertions(+)
>>>   create mode 100644 drivers/dma-buf/st-dma-fence-unwrap.c
>>>   create mode 100644 include/linux/dma-fence-unwrap.h
>>>
>>> diff --git a/Documentation/driver-api/dma-buf.rst 
>>> b/Documentation/driver-api/dma-buf.rst
>>> index 2cd7db82d9fe..7209500f08c8 100644
>>> --- a/Documentation/driver-api/dma-buf.rst
>>> +++ b/Documentation/driver-api/dma-buf.rst
>>> @@ -194,6 +194,12 @@ DMA Fence Chain
>>>   .. kernel-doc:: include/linux/dma-fence-chain.h
>>>      :internal:
>>>   +DMA Fence unwrap
>>> +~~~~~~~~~~~~~~~~
>>> +
>>> +.. kernel-doc:: include/linux/dma-fence-unwrap.h
>>> +   :internal:
>>> +
>>>   DMA Fence uABI/Sync File
>>>   ~~~~~~~~~~~~~~~~~~~~~~~~
>>>   diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
>>> index 511805dbeb75..4c9eb53ba3f8 100644
>>> --- a/drivers/dma-buf/Makefile
>>> +++ b/drivers/dma-buf/Makefile
>>> @@ -12,6 +12,7 @@ dmabuf_selftests-y := \
>>>       selftest.o \
>>>       st-dma-fence.o \
>>>       st-dma-fence-chain.o \
>>> +    st-dma-fence-unwrap.o \
>>>       st-dma-resv.o
>>>     obj-$(CONFIG_DMABUF_SELFTESTS)    += dmabuf_selftests.o
>>> diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
>>> index 97d73aaa31da..851965867d9c 100644
>>> --- a/drivers/dma-buf/selftests.h
>>> +++ b/drivers/dma-buf/selftests.h
>>> @@ -12,4 +12,5 @@
>>>   selftest(sanitycheck, __sanitycheck__) /* keep first (igt 
>>> selfcheck) */
>>>   selftest(dma_fence, dma_fence)
>>>   selftest(dma_fence_chain, dma_fence_chain)
>>> +selftest(dma_fence_unwrap, dma_fence_unwrap)
>>>   selftest(dma_resv, dma_resv)
>>> diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c 
>>> b/drivers/dma-buf/st-dma-fence-unwrap.c
>>> new file mode 100644
>>> index 000000000000..d821faaebe93
>>> --- /dev/null
>>> +++ b/drivers/dma-buf/st-dma-fence-unwrap.c
>>> @@ -0,0 +1,279 @@
>>> +// SPDX-License-Identifier: MIT
>>> +
>>> +/*
>>> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
>>> + */
>>> +
>>> +#include <linux/dma-fence-unwrap.h>
>>> +#if 0
>>> +#include <linux/kernel.h>
>>> +#include <linux/kthread.h>
>>> +#include <linux/mm.h>
>>> +#include <linux/sched/signal.h>
>>> +#include <linux/slab.h>
>>> +#include <linux/spinlock.h>
>>> +#include <linux/random.h>
>>> +#endif
>>> +
>>> +#include "selftest.h"
>>> +
>>> +#define CHAIN_SZ (4 << 10)
>>> +
>>> +static struct kmem_cache *slab_fences;
>>> +
>>> +static inline struct mock_fence {
>>> +    struct dma_fence base;
>>> +    spinlock_t lock;
>>> +} *to_mock_fence(struct dma_fence *f) {
>>> +    return container_of(f, struct mock_fence, base);
>>> +}
>>> +
>>> +static const char *mock_name(struct dma_fence *f)
>>> +{
>>> +    return "mock";
>>> +}
>>> +
>>> +static void mock_fence_release(struct dma_fence *f)
>>> +{
>>> +    kmem_cache_free(slab_fences, to_mock_fence(f));
>>> +}
>>> +
>>> +static const struct dma_fence_ops mock_ops = {
>>> +    .get_driver_name = mock_name,
>>> +    .get_timeline_name = mock_name,
>>> +    .release = mock_fence_release,
>>> +};
>>> +
>>> +static struct dma_fence *mock_fence(void)
>>> +{
>>> +    struct mock_fence *f;
>>> +
>>> +    f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
>>> +    if (!f)
>>> +        return NULL;
>>> +
>>> +    spin_lock_init(&f->lock);
>>> +    dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
>>> +
>>> +    return &f->base;
>>> +}
>>> +
>>> +static struct dma_fence *mock_array(unsigned int num_fences, ...)
>>> +{
>>> +    struct dma_fence_array *array;
>>> +    struct dma_fence **fences;
>>> +    va_list valist;
>>> +    int i;
>>> +
>>> +    fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
>>> +    if (!fences)
>>> +        return NULL;
>>> +
>>> +    va_start(valist, num_fences);
>>> +    for (i = 0; i < num_fences; ++i)
>>> +        fences[i] = va_arg(valist, typeof(*fences));
>>> +    va_end(valist);
>>> +
>>> +    array = dma_fence_array_create(num_fences, fences,
>>> +                       dma_fence_context_alloc(1),
>>> +                       1, false);
>>> +    if (!array)
>>> +        goto cleanup;
>>> +    return &array->base;
>>> +
>>> +cleanup:
>>> +    for (i = 0; i < num_fences; ++i)
>>> +        dma_fence_put(fences[i]);
>>> +    kfree(fences);
>>> +    return NULL;
>>> +}
>>> +
>>> +static struct dma_fence *mock_chain(struct dma_fence *prev,
>>> +                    struct dma_fence *fence)
>>> +{
>>> +    struct dma_fence_chain *f;
>>> +
>>> +    f = dma_fence_chain_alloc();
>>> +    if (!f) {
>>> +        dma_fence_put(prev);
>>> +        dma_fence_put(fence);
>>> +        return NULL;
>>> +    }
>>> +
>>> +    dma_fence_chain_init(f, prev, fence, 1);
>>> +    return &f->base;
>>> +}
>>> +
>>> +static int sanitycheck(void *arg)
>>> +{
>>> +    struct dma_fence *f, *chain, *array;
>>> +    int err = 0;
>>> +
>>> +    f = mock_fence();
>>> +    if (!f)
>>> +        return -ENOMEM;
>>> +
>>> +    array = mock_array(1, f);
>>> +    if (!array)
>>> +        return -ENOMEM;
>>> +
>>> +    chain = mock_chain(NULL, array);
>>> +    if (!chain)
>>> +        return -ENOMEM;
>>> +
>>> +    dma_fence_signal(f);
>>> +    dma_fence_put(chain);
>>> +    return err;
>>> +}
>>> +
>>> +static int unwrap_array(void *arg)
>>> +{
>>> +    struct dma_fence *fence, *f1, *f2, *array;
>>> +    struct dma_fence_unwrap iter;
>>> +    int err = 0;
>>> +
>>> +    f1 = mock_fence();
>>> +    if (!f1)
>>> +        return -ENOMEM;
>>> +
>>> +    f2 = mock_fence();
>>> +    if (!f2) {
>>> +        dma_fence_put(f1);
>>> +        return -ENOMEM;
>>> +    }
>>> +
>>> +    array = mock_array(2, f1, f2);
>>> +    if (!array)
>>> +        return -ENOMEM;
>>> +
>>> +    dma_fence_unwrap_for_each(fence, &iter, array) {
>>> +        if (fence == f1) {
>>> +            f1 = NULL;
>>> +        } else if (fence == f2) {
>>> +            f2 = NULL;
>>> +        } else {
>>> +            pr_err("Unexpected fence!\n");
>>> +            err = -EINVAL;
>>> +        }
>>> +    }
>>> +
>>> +    if (f1 || f2) {
>>> +        pr_err("Not all fences seen!\n");
>>> +        err = -EINVAL;
>>> +    }
>>> +
>>> +    dma_fence_signal(f1);
>>> +    dma_fence_signal(f2);
>>> +    dma_fence_put(array);
>>> +    return 0;
>>> +}
>>> +
>>> +static int unwrap_chain(void *arg)
>>> +{
>>> +    struct dma_fence *fence, *f1, *f2, *chain;
>>> +    struct dma_fence_unwrap iter;
>>> +    int err = 0;
>>> +
>>> +    f1 = mock_fence();
>>> +    if (!f1)
>>> +        return -ENOMEM;
>>> +
>>> +    f2 = mock_fence();
>>> +    if (!f2) {
>>> +        dma_fence_put(f1);
>>> +        return -ENOMEM;
>>> +    }
>>> +
>>> +    chain = mock_chain(f1, f2);
>>> +    if (!chain)
>>> +        return -ENOMEM;
>>> +
>>> +    dma_fence_unwrap_for_each(fence, &iter, chain) {
>>> +        if (fence == f1) {
>>> +            f1 = NULL;
>>> +        } else if (fence == f2) {
>>> +            f2 = NULL;
>>> +        } else {
>>> +            pr_err("Unexpected fence!\n");
>>> +            err = -EINVAL;
>>> +        }
>>> +    }
>>> +
>>> +    if (f1 || f2) {
>>> +        pr_err("Not all fences seen!\n");
>>> +        err = -EINVAL;
>>> +    }
>>> +
>>> +    dma_fence_signal(f1);
>>> +    dma_fence_signal(f2);
>>> +    dma_fence_put(chain);
>>> +    return 0;
>>> +}
>>> +
>>> +static int unwrap_chain_array(void *arg)
>>> +{
>>> +    struct dma_fence *fence, *f1, *f2, *array, *chain;
>>> +    struct dma_fence_unwrap iter;
>>> +    int err = 0;
>>> +
>>> +    f1 = mock_fence();
>>> +    if (!f1)
>>> +        return -ENOMEM;
>>> +
>>> +    f2 = mock_fence();
>>> +    if (!f2) {
>>> +        dma_fence_put(f1);
>>> +        return -ENOMEM;
>>> +    }
>>> +
>>> +    array = mock_array(2, f1, f2);
>>> +    if (!array)
>>> +        return -ENOMEM;
>>> +
>>> +    chain = mock_chain(NULL, array);
>>> +    if (!chain)
>>> +        return -ENOMEM;
>>> +
>>> +    dma_fence_unwrap_for_each(fence, &iter, chain) {
>>> +        if (fence == f1) {
>>> +            f1 = NULL;
>>> +        } else if (fence == f2) {
>>> +            f2 = NULL;
>>> +        } else {
>>> +            pr_err("Unexpected fence!\n");
>>> +            err = -EINVAL;
>>> +        }
>>> +    }
>>> +
>>> +    if (f1 || f2) {
>>> +        pr_err("Not all fences seen!\n");
>>> +        err = -EINVAL;
>>> +    }
>>> +
>>> +    dma_fence_signal(f1);
>>> +    dma_fence_signal(f2);
>>> +    dma_fence_put(chain);
>>> +    return 0;
>>> +}
>>> +
>>> +int dma_fence_unwrap(void)
>>> +{
>>> +    static const struct subtest tests[] = {
>>> +        SUBTEST(sanitycheck),
>>> +        SUBTEST(unwrap_array),
>>> +        SUBTEST(unwrap_chain),
>>> +        SUBTEST(unwrap_chain_array),
>>> +    };
>>> +    int ret;
>>> +
>>> +    slab_fences = KMEM_CACHE(mock_fence,
>>> +                 SLAB_TYPESAFE_BY_RCU |
>>> +                 SLAB_HWCACHE_ALIGN);
>>> +    if (!slab_fences)
>>> +        return -ENOMEM;
>>> +
>>> +    ret = subtests(tests, NULL);
>>> +
>>> +    kmem_cache_destroy(slab_fences);
>>> +    return ret;
>>> +}
>>> diff --git a/include/linux/dma-fence-unwrap.h 
>>> b/include/linux/dma-fence-unwrap.h
>>> new file mode 100644
>>> index 000000000000..54963df00c98
>>> --- /dev/null
>>> +++ b/include/linux/dma-fence-unwrap.h
>>> @@ -0,0 +1,99 @@
>>> +/* SPDX-License-Identifier: GPL-2.0-only */
>>> +/*
>>> + * fence-chain: chain fences together in a timeline
>>> + *
>>> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
>>> + * Authors:
>>> + *    Christian König <christian.koenig@amd.com>
>>> + */
>>> +
>>> +#ifndef __LINUX_DMA_FENCE_UNWRAP_H
>>> +#define __LINUX_DMA_FENCE_UNWRAP_H
>>> +
>>> +#include <linux/dma-fence-chain.h>
>>> +#include <linux/dma-fence-array.h>
>>> +
>>> +/**
>>> + * struct dma_fence_unwrap - cursor into the container structure
>>> + */
>>> +struct dma_fence_unwrap {
>>> +    /**
>>> +     * @chain: potential dma_fence_chain, but can be other fence as 
>>> well
>>> +     */
>>> +    struct dma_fence *chain;
>>> +    /**
>>> +     * @array: potential dma_fence_array, but can be other fence as 
>>> well
>>> +     */
>>> +    struct dma_fence *array;
>>> +    /**
>>> +     * @index: last returned index if @array is really a 
>>> dma_fence_array
>>> +     */
>>> +    unsigned int index;
>>> +};
>>> +
>>> +/**
>>> + * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
>>> + * @cursor: cursor to initialize
>>> + *
>>> + * Helper function to unwrap dma_fence_array containers, don't 
>>> touch directly.
>>> + * Use dma_fence_unwrap_first/next instead.
>>> + */
>>> +static inline struct dma_fence *
>>> +dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)
>>> +{
>>> +    cursor->array = dma_fence_chain_contained(cursor->chain);
>>> +    cursor->index = 0;
>>> +    return dma_fence_array_first(cursor->array);
>>> +}
>>> +
>>> +/**
>>> + * dma_fence_unwrap_first - return the first fence from fence 
>>> containers
>>> + * @head: the entrypoint into the containers
>>> + * @cursor: current position inside the containers
>>> + *
>>> + * Unwraps potential dma_fence_chain/dma_fence_array containers and 
>>> return the
>>> + * first fence.
>>> + */
>>> +static inline struct dma_fence *
>>> +dma_fence_unwrap_first(struct dma_fence *head, struct 
>>> dma_fence_unwrap *cursor)
>>> +{
>>> +    cursor->chain = dma_fence_get(head);
>>> +    return dma_fence_unwrap_array(cursor);
>>> +}
>>> +
>>> +/**
>>> + * dma_fence_unwrap_next - return the next fence from a fence 
>>> containers
>>> + * @cursor: current position inside the containers
>>> + *
>>> + * Continue unwrapping the dma_fence_chain/dma_fence_array 
>>> containers and return
>>> + * the next fence from them.
>>> + */
>>> +static inline struct dma_fence *
>>> +dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
>>> +{
>>> +    struct dma_fence *tmp;
>>> +
>>> +    ++cursor->index;
>>> +    tmp = dma_fence_array_next(cursor->array, cursor->index);
>>> +    if (tmp)
>>> +        return tmp;
>>> +
>>> +    cursor->chain = dma_fence_chain_walk(cursor->chain);
>>> +    return dma_fence_unwrap_array(cursor);
>>> +}
>>> +
>>> +/**
>>> + * dma_fence_unwrap_for_each - iterate over all fences in containers
>>> + * @fence: current fence
>>> + * @cursor: current position inside the containers
>>> + * @head: starting point for the iterator
>>> + *
>>> + * Unwrap dma_fence_chain and dma_fence_array containers and deep 
>>> dive into all
>>> + * potential fences in them. If @head is just a normal fence only 
>>> that one is
>>> + * returned.
>>> + */
>>> +#define dma_fence_unwrap_for_each(fence, cursor, head)            \
>>> +    for (fence = dma_fence_unwrap_first(head, cursor); fence;    \
>>> +         fence = dma_fence_unwrap_next(cursor))
>>> +
>>> +#endif
>>
>
Daniel Vetter March 25, 2022, 10:03 a.m. UTC | #6 
On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
> Add a general purpose helper to deep dive into dma_fence_chain/dma_fence_array
> structures and iterate over all the fences in them.
> 
> This is useful when we need to flatten out all fences in those structures.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  Documentation/driver-api/dma-buf.rst  |   6 +
>  drivers/dma-buf/Makefile              |   1 +
>  drivers/dma-buf/selftests.h           |   1 +
>  drivers/dma-buf/st-dma-fence-unwrap.c | 279 ++++++++++++++++++++++++++
>  include/linux/dma-fence-unwrap.h      |  99 +++++++++
>  5 files changed, 386 insertions(+)
>  create mode 100644 drivers/dma-buf/st-dma-fence-unwrap.c
>  create mode 100644 include/linux/dma-fence-unwrap.h
> 
> diff --git a/Documentation/driver-api/dma-buf.rst b/Documentation/driver-api/dma-buf.rst
> index 2cd7db82d9fe..7209500f08c8 100644
> --- a/Documentation/driver-api/dma-buf.rst
> +++ b/Documentation/driver-api/dma-buf.rst
> @@ -194,6 +194,12 @@ DMA Fence Chain
>  .. kernel-doc:: include/linux/dma-fence-chain.h
>     :internal:
>  
> +DMA Fence unwrap
> +~~~~~~~~~~~~~~~~
> +
> +.. kernel-doc:: include/linux/dma-fence-unwrap.h
> +   :internal:
> +
>  DMA Fence uABI/Sync File
>  ~~~~~~~~~~~~~~~~~~~~~~~~
>  
> diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
> index 511805dbeb75..4c9eb53ba3f8 100644
> --- a/drivers/dma-buf/Makefile
> +++ b/drivers/dma-buf/Makefile
> @@ -12,6 +12,7 @@ dmabuf_selftests-y := \
>  	selftest.o \
>  	st-dma-fence.o \
>  	st-dma-fence-chain.o \
> +	st-dma-fence-unwrap.o \
>  	st-dma-resv.o
>  
>  obj-$(CONFIG_DMABUF_SELFTESTS)	+= dmabuf_selftests.o
> diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
> index 97d73aaa31da..851965867d9c 100644
> --- a/drivers/dma-buf/selftests.h
> +++ b/drivers/dma-buf/selftests.h
> @@ -12,4 +12,5 @@
>  selftest(sanitycheck, __sanitycheck__) /* keep first (igt selfcheck) */
>  selftest(dma_fence, dma_fence)
>  selftest(dma_fence_chain, dma_fence_chain)
> +selftest(dma_fence_unwrap, dma_fence_unwrap)
>  selftest(dma_resv, dma_resv)
> diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
> new file mode 100644
> index 000000000000..d821faaebe93
> --- /dev/null
> +++ b/drivers/dma-buf/st-dma-fence-unwrap.c
> @@ -0,0 +1,279 @@
> +// SPDX-License-Identifier: MIT
> +
> +/*
> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
> + */
> +
> +#include <linux/dma-fence-unwrap.h>
> +#if 0
> +#include <linux/kernel.h>
> +#include <linux/kthread.h>
> +#include <linux/mm.h>
> +#include <linux/sched/signal.h>
> +#include <linux/slab.h>
> +#include <linux/spinlock.h>
> +#include <linux/random.h>
> +#endif
> +
> +#include "selftest.h"
> +
> +#define CHAIN_SZ (4 << 10)
> +
> +static struct kmem_cache *slab_fences;

Your own slab feels a bit like overkill. kmalloc/kfree not good enough?

> +
> +static inline struct mock_fence {
> +	struct dma_fence base;
> +	spinlock_t lock;
> +} *to_mock_fence(struct dma_fence *f) {
> +	return container_of(f, struct mock_fence, base);
> +}
> +
> +static const char *mock_name(struct dma_fence *f)
> +{
> +	return "mock";
> +}
> +
> +static void mock_fence_release(struct dma_fence *f)
> +{
> +	kmem_cache_free(slab_fences, to_mock_fence(f));
> +}
> +
> +static const struct dma_fence_ops mock_ops = {
> +	.get_driver_name = mock_name,
> +	.get_timeline_name = mock_name,
> +	.release = mock_fence_release,
> +};
> +
> +static struct dma_fence *mock_fence(void)
> +{
> +	struct mock_fence *f;
> +
> +	f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
> +	if (!f)
> +		return NULL;
> +
> +	spin_lock_init(&f->lock);
> +	dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
> +
> +	return &f->base;
> +}
> +
> +static struct dma_fence *mock_array(unsigned int num_fences, ...)
> +{
> +	struct dma_fence_array *array;
> +	struct dma_fence **fences;
> +	va_list valist;
> +	int i;
> +
> +	fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
> +	if (!fences)
> +		return NULL;
> +
> +	va_start(valist, num_fences);
> +	for (i = 0; i < num_fences; ++i)
> +		fences[i] = va_arg(valist, typeof(*fences));
> +	va_end(valist);
> +
> +	array = dma_fence_array_create(num_fences, fences,
> +				       dma_fence_context_alloc(1),
> +				       1, false);
> +	if (!array)
> +		goto cleanup;
> +	return &array->base;
> +
> +cleanup:
> +	for (i = 0; i < num_fences; ++i)
> +		dma_fence_put(fences[i]);
> +	kfree(fences);
> +	return NULL;
> +}
> +
> +static struct dma_fence *mock_chain(struct dma_fence *prev,
> +				    struct dma_fence *fence)
> +{
> +	struct dma_fence_chain *f;
> +
> +	f = dma_fence_chain_alloc();
> +	if (!f) {
> +		dma_fence_put(prev);
> +		dma_fence_put(fence);
> +		return NULL;
> +	}
> +
> +	dma_fence_chain_init(f, prev, fence, 1);
> +	return &f->base;
> +}
> +
> +static int sanitycheck(void *arg)
> +{
> +	struct dma_fence *f, *chain, *array;
> +	int err = 0;
> +
> +	f = mock_fence();
> +	if (!f)
> +		return -ENOMEM;
> +
> +	array = mock_array(1, f);
> +	if (!array)
> +		return -ENOMEM;
> +
> +	chain = mock_chain(NULL, array);
> +	if (!chain)
> +		return -ENOMEM;
> +
> +	dma_fence_signal(f);
> +	dma_fence_put(chain);
> +	return err;
> +}
> +
> +static int unwrap_array(void *arg)
> +{
> +	struct dma_fence *fence, *f1, *f2, *array;
> +	struct dma_fence_unwrap iter;
> +	int err = 0;
> +
> +	f1 = mock_fence();
> +	if (!f1)
> +		return -ENOMEM;
> +
> +	f2 = mock_fence();
> +	if (!f2) {
> +		dma_fence_put(f1);
> +		return -ENOMEM;
> +	}
> +
> +	array = mock_array(2, f1, f2);
> +	if (!array)
> +		return -ENOMEM;
> +
> +	dma_fence_unwrap_for_each(fence, &iter, array) {
> +		if (fence == f1) {
> +			f1 = NULL;
> +		} else if (fence == f2) {
> +			f2 = NULL;
> +		} else {
> +			pr_err("Unexpected fence!\n");
> +			err = -EINVAL;
> +		}
> +	}
> +
> +	if (f1 || f2) {
> +		pr_err("Not all fences seen!\n");
> +		err = -EINVAL;
> +	}
> +
> +	dma_fence_signal(f1);
> +	dma_fence_signal(f2);
> +	dma_fence_put(array);
> +	return 0;
> +}
> +
> +static int unwrap_chain(void *arg)
> +{
> +	struct dma_fence *fence, *f1, *f2, *chain;
> +	struct dma_fence_unwrap iter;
> +	int err = 0;
> +
> +	f1 = mock_fence();
> +	if (!f1)
> +		return -ENOMEM;
> +
> +	f2 = mock_fence();
> +	if (!f2) {
> +		dma_fence_put(f1);
> +		return -ENOMEM;
> +	}
> +
> +	chain = mock_chain(f1, f2);
> +	if (!chain)
> +		return -ENOMEM;
> +
> +	dma_fence_unwrap_for_each(fence, &iter, chain) {
> +		if (fence == f1) {
> +			f1 = NULL;
> +		} else if (fence == f2) {
> +			f2 = NULL;
> +		} else {
> +			pr_err("Unexpected fence!\n");
> +			err = -EINVAL;
> +		}
> +	}
> +
> +	if (f1 || f2) {
> +		pr_err("Not all fences seen!\n");
> +		err = -EINVAL;
> +	}
> +
> +	dma_fence_signal(f1);
> +	dma_fence_signal(f2);
> +	dma_fence_put(chain);
> +	return 0;
> +}
> +
> +static int unwrap_chain_array(void *arg)
> +{
> +	struct dma_fence *fence, *f1, *f2, *array, *chain;
> +	struct dma_fence_unwrap iter;
> +	int err = 0;
> +
> +	f1 = mock_fence();
> +	if (!f1)
> +		return -ENOMEM;
> +
> +	f2 = mock_fence();
> +	if (!f2) {
> +		dma_fence_put(f1);
> +		return -ENOMEM;
> +	}
> +
> +	array = mock_array(2, f1, f2);
> +	if (!array)
> +		return -ENOMEM;
> +
> +	chain = mock_chain(NULL, array);
> +	if (!chain)
> +		return -ENOMEM;
> +
> +	dma_fence_unwrap_for_each(fence, &iter, chain) {
> +		if (fence == f1) {
> +			f1 = NULL;
> +		} else if (fence == f2) {
> +			f2 = NULL;
> +		} else {
> +			pr_err("Unexpected fence!\n");
> +			err = -EINVAL;
> +		}
> +	}
> +
> +	if (f1 || f2) {
> +		pr_err("Not all fences seen!\n");
> +		err = -EINVAL;
> +	}
> +
> +	dma_fence_signal(f1);
> +	dma_fence_signal(f2);
> +	dma_fence_put(chain);
> +	return 0;
> +}
> +
> +int dma_fence_unwrap(void)
> +{
> +	static const struct subtest tests[] = {
> +		SUBTEST(sanitycheck),
> +		SUBTEST(unwrap_array),
> +		SUBTEST(unwrap_chain),
> +		SUBTEST(unwrap_chain_array),
> +	};
> +	int ret;
> +
> +	slab_fences = KMEM_CACHE(mock_fence,
> +				 SLAB_TYPESAFE_BY_RCU |
> +				 SLAB_HWCACHE_ALIGN);
> +	if (!slab_fences)
> +		return -ENOMEM;
> +
> +	ret = subtests(tests, NULL);
> +
> +	kmem_cache_destroy(slab_fences);
> +	return ret;
> +}
> diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h
> new file mode 100644
> index 000000000000..54963df00c98
> --- /dev/null
> +++ b/include/linux/dma-fence-unwrap.h
> @@ -0,0 +1,99 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * fence-chain: chain fences together in a timeline
> + *
> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
> + * Authors:
> + *	Christian König <christian.koenig@amd.com>
> + */
> +
> +#ifndef __LINUX_DMA_FENCE_UNWRAP_H
> +#define __LINUX_DMA_FENCE_UNWRAP_H
> +
> +#include <linux/dma-fence-chain.h>
> +#include <linux/dma-fence-array.h>
> +
> +/**
> + * struct dma_fence_unwrap - cursor into the container structure

I think adding "This should be used together with
dma_fence_unwrap_for_each() iterator macro." would be nice here. I just
like links :-)

> + */
> +struct dma_fence_unwrap {
> +	/**
> +	 * @chain: potential dma_fence_chain, but can be other fence as well
> +	 */
> +	struct dma_fence *chain;
> +	/**
> +	 * @array: potential dma_fence_array, but can be other fence as well
> +	 */
> +	struct dma_fence *array;
> +	/**
> +	 * @index: last returned index if @array is really a dma_fence_array
> +	 */
> +	unsigned int index;
> +};
> +
> +/**
> + * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
> + * @cursor: cursor to initialize
> + *
> + * Helper function to unwrap dma_fence_array containers, don't touch directly.
> + * Use dma_fence_unwrap_first/next instead.
> + */
> +static inline struct dma_fence *
> +dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)

Since this is a helper that no one should call I'd give it a __ prefix and
drop the kerneldoc. Documenting stuff that people shouldn't use is
confusing :-)

> +{
> +	cursor->array = dma_fence_chain_contained(cursor->chain);
> +	cursor->index = 0;
> +	return dma_fence_array_first(cursor->array);
> +}
> +
> +/**
> + * dma_fence_unwrap_first - return the first fence from fence containers
> + * @head: the entrypoint into the containers
> + * @cursor: current position inside the containers
> + *
> + * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
> + * first fence.
> + */
> +static inline struct dma_fence *
> +dma_fence_unwrap_first(struct dma_fence *head, struct dma_fence_unwrap *cursor)
> +{
> +	cursor->chain = dma_fence_get(head);
> +	return dma_fence_unwrap_array(cursor);
> +}
> +
> +/**
> + * dma_fence_unwrap_next - return the next fence from a fence containers
> + * @cursor: current position inside the containers
> + *
> + * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
> + * the next fence from them.
> + */
> +static inline struct dma_fence *
> +dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
> +{
> +	struct dma_fence *tmp;
> +
> +	++cursor->index;
> +	tmp = dma_fence_array_next(cursor->array, cursor->index);
> +	if (tmp)
> +		return tmp;
> +
> +	cursor->chain = dma_fence_chain_walk(cursor->chain);
> +	return dma_fence_unwrap_array(cursor);
> +}
> +
> +/**
> + * dma_fence_unwrap_for_each - iterate over all fences in containers
> + * @fence: current fence
> + * @cursor: current position inside the containers
> + * @head: starting point for the iterator
> + *
> + * Unwrap dma_fence_chain and dma_fence_array containers and deep dive into all
> + * potential fences in them. If @head is just a normal fence only that one is
> + * returned.
> + */
> +#define dma_fence_unwrap_for_each(fence, cursor, head)			\
> +	for (fence = dma_fence_unwrap_first(head, cursor); fence;	\
> +	     fence = dma_fence_unwrap_next(cursor))
> +
> +#endif

I think it'd be really good to add a small paragraph to struct
dma_fence_chain that this macro and iterator should be used for walking
over all fences in a chain, including any fence arrays or anything like
that.

With the bikesheds addressed:

Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>

> -- 
> 2.25.1
>
Daniel Vetter March 25, 2022, 10:07 a.m. UTC | #7 
On Fri, Mar 25, 2022 at 11:03:54AM +0100, Daniel Vetter wrote:
> On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
> > Add a general purpose helper to deep dive into dma_fence_chain/dma_fence_array
> > structures and iterate over all the fences in them.
> > 
> > This is useful when we need to flatten out all fences in those structures.
> > 
> > Signed-off-by: Christian König <christian.koenig@amd.com>
> > ---
> >  Documentation/driver-api/dma-buf.rst  |   6 +
> >  drivers/dma-buf/Makefile              |   1 +
> >  drivers/dma-buf/selftests.h           |   1 +
> >  drivers/dma-buf/st-dma-fence-unwrap.c | 279 ++++++++++++++++++++++++++
> >  include/linux/dma-fence-unwrap.h      |  99 +++++++++
> >  5 files changed, 386 insertions(+)
> >  create mode 100644 drivers/dma-buf/st-dma-fence-unwrap.c
> >  create mode 100644 include/linux/dma-fence-unwrap.h
> > 
> > diff --git a/Documentation/driver-api/dma-buf.rst b/Documentation/driver-api/dma-buf.rst
> > index 2cd7db82d9fe..7209500f08c8 100644
> > --- a/Documentation/driver-api/dma-buf.rst
> > +++ b/Documentation/driver-api/dma-buf.rst
> > @@ -194,6 +194,12 @@ DMA Fence Chain
> >  .. kernel-doc:: include/linux/dma-fence-chain.h
> >     :internal:
> >  
> > +DMA Fence unwrap
> > +~~~~~~~~~~~~~~~~
> > +
> > +.. kernel-doc:: include/linux/dma-fence-unwrap.h
> > +   :internal:

Ok I forgot one bikeshed: I'd just include this in dma-fence-chain.h and
maybe go with the dma_fence_chain_unwrap_ prefix for everything. That
makes it even more clearer that the two are meant to go together. Plus ofc
the link from struct dma_fence_chain to this iterator in the docs too.

Or I'm just not understanding why you made this a separate thing?
-Daniel

> > +
> >  DMA Fence uABI/Sync File
> >  ~~~~~~~~~~~~~~~~~~~~~~~~
> >  
> > diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
> > index 511805dbeb75..4c9eb53ba3f8 100644
> > --- a/drivers/dma-buf/Makefile
> > +++ b/drivers/dma-buf/Makefile
> > @@ -12,6 +12,7 @@ dmabuf_selftests-y := \
> >  	selftest.o \
> >  	st-dma-fence.o \
> >  	st-dma-fence-chain.o \
> > +	st-dma-fence-unwrap.o \
> >  	st-dma-resv.o
> >  
> >  obj-$(CONFIG_DMABUF_SELFTESTS)	+= dmabuf_selftests.o
> > diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
> > index 97d73aaa31da..851965867d9c 100644
> > --- a/drivers/dma-buf/selftests.h
> > +++ b/drivers/dma-buf/selftests.h
> > @@ -12,4 +12,5 @@
> >  selftest(sanitycheck, __sanitycheck__) /* keep first (igt selfcheck) */
> >  selftest(dma_fence, dma_fence)
> >  selftest(dma_fence_chain, dma_fence_chain)
> > +selftest(dma_fence_unwrap, dma_fence_unwrap)
> >  selftest(dma_resv, dma_resv)
> > diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
> > new file mode 100644
> > index 000000000000..d821faaebe93
> > --- /dev/null
> > +++ b/drivers/dma-buf/st-dma-fence-unwrap.c
> > @@ -0,0 +1,279 @@
> > +// SPDX-License-Identifier: MIT
> > +
> > +/*
> > + * Copyright (C) 2022 Advanced Micro Devices, Inc.
> > + */
> > +
> > +#include <linux/dma-fence-unwrap.h>
> > +#if 0
> > +#include <linux/kernel.h>
> > +#include <linux/kthread.h>
> > +#include <linux/mm.h>
> > +#include <linux/sched/signal.h>
> > +#include <linux/slab.h>
> > +#include <linux/spinlock.h>
> > +#include <linux/random.h>
> > +#endif
> > +
> > +#include "selftest.h"
> > +
> > +#define CHAIN_SZ (4 << 10)
> > +
> > +static struct kmem_cache *slab_fences;
> 
> Your own slab feels a bit like overkill. kmalloc/kfree not good enough?
> 
> > +
> > +static inline struct mock_fence {
> > +	struct dma_fence base;
> > +	spinlock_t lock;
> > +} *to_mock_fence(struct dma_fence *f) {
> > +	return container_of(f, struct mock_fence, base);
> > +}
> > +
> > +static const char *mock_name(struct dma_fence *f)
> > +{
> > +	return "mock";
> > +}
> > +
> > +static void mock_fence_release(struct dma_fence *f)
> > +{
> > +	kmem_cache_free(slab_fences, to_mock_fence(f));
> > +}
> > +
> > +static const struct dma_fence_ops mock_ops = {
> > +	.get_driver_name = mock_name,
> > +	.get_timeline_name = mock_name,
> > +	.release = mock_fence_release,
> > +};
> > +
> > +static struct dma_fence *mock_fence(void)
> > +{
> > +	struct mock_fence *f;
> > +
> > +	f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
> > +	if (!f)
> > +		return NULL;
> > +
> > +	spin_lock_init(&f->lock);
> > +	dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
> > +
> > +	return &f->base;
> > +}
> > +
> > +static struct dma_fence *mock_array(unsigned int num_fences, ...)
> > +{
> > +	struct dma_fence_array *array;
> > +	struct dma_fence **fences;
> > +	va_list valist;
> > +	int i;
> > +
> > +	fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
> > +	if (!fences)
> > +		return NULL;
> > +
> > +	va_start(valist, num_fences);
> > +	for (i = 0; i < num_fences; ++i)
> > +		fences[i] = va_arg(valist, typeof(*fences));
> > +	va_end(valist);
> > +
> > +	array = dma_fence_array_create(num_fences, fences,
> > +				       dma_fence_context_alloc(1),
> > +				       1, false);
> > +	if (!array)
> > +		goto cleanup;
> > +	return &array->base;
> > +
> > +cleanup:
> > +	for (i = 0; i < num_fences; ++i)
> > +		dma_fence_put(fences[i]);
> > +	kfree(fences);
> > +	return NULL;
> > +}
> > +
> > +static struct dma_fence *mock_chain(struct dma_fence *prev,
> > +				    struct dma_fence *fence)
> > +{
> > +	struct dma_fence_chain *f;
> > +
> > +	f = dma_fence_chain_alloc();
> > +	if (!f) {
> > +		dma_fence_put(prev);
> > +		dma_fence_put(fence);
> > +		return NULL;
> > +	}
> > +
> > +	dma_fence_chain_init(f, prev, fence, 1);
> > +	return &f->base;
> > +}
> > +
> > +static int sanitycheck(void *arg)
> > +{
> > +	struct dma_fence *f, *chain, *array;
> > +	int err = 0;
> > +
> > +	f = mock_fence();
> > +	if (!f)
> > +		return -ENOMEM;
> > +
> > +	array = mock_array(1, f);
> > +	if (!array)
> > +		return -ENOMEM;
> > +
> > +	chain = mock_chain(NULL, array);
> > +	if (!chain)
> > +		return -ENOMEM;
> > +
> > +	dma_fence_signal(f);
> > +	dma_fence_put(chain);
> > +	return err;
> > +}
> > +
> > +static int unwrap_array(void *arg)
> > +{
> > +	struct dma_fence *fence, *f1, *f2, *array;
> > +	struct dma_fence_unwrap iter;
> > +	int err = 0;
> > +
> > +	f1 = mock_fence();
> > +	if (!f1)
> > +		return -ENOMEM;
> > +
> > +	f2 = mock_fence();
> > +	if (!f2) {
> > +		dma_fence_put(f1);
> > +		return -ENOMEM;
> > +	}
> > +
> > +	array = mock_array(2, f1, f2);
> > +	if (!array)
> > +		return -ENOMEM;
> > +
> > +	dma_fence_unwrap_for_each(fence, &iter, array) {
> > +		if (fence == f1) {
> > +			f1 = NULL;
> > +		} else if (fence == f2) {
> > +			f2 = NULL;
> > +		} else {
> > +			pr_err("Unexpected fence!\n");
> > +			err = -EINVAL;
> > +		}
> > +	}
> > +
> > +	if (f1 || f2) {
> > +		pr_err("Not all fences seen!\n");
> > +		err = -EINVAL;
> > +	}
> > +
> > +	dma_fence_signal(f1);
> > +	dma_fence_signal(f2);
> > +	dma_fence_put(array);
> > +	return 0;
> > +}
> > +
> > +static int unwrap_chain(void *arg)
> > +{
> > +	struct dma_fence *fence, *f1, *f2, *chain;
> > +	struct dma_fence_unwrap iter;
> > +	int err = 0;
> > +
> > +	f1 = mock_fence();
> > +	if (!f1)
> > +		return -ENOMEM;
> > +
> > +	f2 = mock_fence();
> > +	if (!f2) {
> > +		dma_fence_put(f1);
> > +		return -ENOMEM;
> > +	}
> > +
> > +	chain = mock_chain(f1, f2);
> > +	if (!chain)
> > +		return -ENOMEM;
> > +
> > +	dma_fence_unwrap_for_each(fence, &iter, chain) {
> > +		if (fence == f1) {
> > +			f1 = NULL;
> > +		} else if (fence == f2) {
> > +			f2 = NULL;
> > +		} else {
> > +			pr_err("Unexpected fence!\n");
> > +			err = -EINVAL;
> > +		}
> > +	}
> > +
> > +	if (f1 || f2) {
> > +		pr_err("Not all fences seen!\n");
> > +		err = -EINVAL;
> > +	}
> > +
> > +	dma_fence_signal(f1);
> > +	dma_fence_signal(f2);
> > +	dma_fence_put(chain);
> > +	return 0;
> > +}
> > +
> > +static int unwrap_chain_array(void *arg)
> > +{
> > +	struct dma_fence *fence, *f1, *f2, *array, *chain;
> > +	struct dma_fence_unwrap iter;
> > +	int err = 0;
> > +
> > +	f1 = mock_fence();
> > +	if (!f1)
> > +		return -ENOMEM;
> > +
> > +	f2 = mock_fence();
> > +	if (!f2) {
> > +		dma_fence_put(f1);
> > +		return -ENOMEM;
> > +	}
> > +
> > +	array = mock_array(2, f1, f2);
> > +	if (!array)
> > +		return -ENOMEM;
> > +
> > +	chain = mock_chain(NULL, array);
> > +	if (!chain)
> > +		return -ENOMEM;
> > +
> > +	dma_fence_unwrap_for_each(fence, &iter, chain) {
> > +		if (fence == f1) {
> > +			f1 = NULL;
> > +		} else if (fence == f2) {
> > +			f2 = NULL;
> > +		} else {
> > +			pr_err("Unexpected fence!\n");
> > +			err = -EINVAL;
> > +		}
> > +	}
> > +
> > +	if (f1 || f2) {
> > +		pr_err("Not all fences seen!\n");
> > +		err = -EINVAL;
> > +	}
> > +
> > +	dma_fence_signal(f1);
> > +	dma_fence_signal(f2);
> > +	dma_fence_put(chain);
> > +	return 0;
> > +}
> > +
> > +int dma_fence_unwrap(void)
> > +{
> > +	static const struct subtest tests[] = {
> > +		SUBTEST(sanitycheck),
> > +		SUBTEST(unwrap_array),
> > +		SUBTEST(unwrap_chain),
> > +		SUBTEST(unwrap_chain_array),
> > +	};
> > +	int ret;
> > +
> > +	slab_fences = KMEM_CACHE(mock_fence,
> > +				 SLAB_TYPESAFE_BY_RCU |
> > +				 SLAB_HWCACHE_ALIGN);
> > +	if (!slab_fences)
> > +		return -ENOMEM;
> > +
> > +	ret = subtests(tests, NULL);
> > +
> > +	kmem_cache_destroy(slab_fences);
> > +	return ret;
> > +}
> > diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h
> > new file mode 100644
> > index 000000000000..54963df00c98
> > --- /dev/null
> > +++ b/include/linux/dma-fence-unwrap.h
> > @@ -0,0 +1,99 @@
> > +/* SPDX-License-Identifier: GPL-2.0-only */
> > +/*
> > + * fence-chain: chain fences together in a timeline
> > + *
> > + * Copyright (C) 2022 Advanced Micro Devices, Inc.
> > + * Authors:
> > + *	Christian König <christian.koenig@amd.com>
> > + */
> > +
> > +#ifndef __LINUX_DMA_FENCE_UNWRAP_H
> > +#define __LINUX_DMA_FENCE_UNWRAP_H
> > +
> > +#include <linux/dma-fence-chain.h>
> > +#include <linux/dma-fence-array.h>
> > +
> > +/**
> > + * struct dma_fence_unwrap - cursor into the container structure
> 
> I think adding "This should be used together with
> dma_fence_unwrap_for_each() iterator macro." would be nice here. I just
> like links :-)
> 
> > + */
> > +struct dma_fence_unwrap {
> > +	/**
> > +	 * @chain: potential dma_fence_chain, but can be other fence as well
> > +	 */
> > +	struct dma_fence *chain;
> > +	/**
> > +	 * @array: potential dma_fence_array, but can be other fence as well
> > +	 */
> > +	struct dma_fence *array;
> > +	/**
> > +	 * @index: last returned index if @array is really a dma_fence_array
> > +	 */
> > +	unsigned int index;
> > +};
> > +
> > +/**
> > + * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
> > + * @cursor: cursor to initialize
> > + *
> > + * Helper function to unwrap dma_fence_array containers, don't touch directly.
> > + * Use dma_fence_unwrap_first/next instead.
> > + */
> > +static inline struct dma_fence *
> > +dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)
> 
> Since this is a helper that no one should call I'd give it a __ prefix and
> drop the kerneldoc. Documenting stuff that people shouldn't use is
> confusing :-)
> 
> > +{
> > +	cursor->array = dma_fence_chain_contained(cursor->chain);
> > +	cursor->index = 0;
> > +	return dma_fence_array_first(cursor->array);
> > +}
> > +
> > +/**
> > + * dma_fence_unwrap_first - return the first fence from fence containers
> > + * @head: the entrypoint into the containers
> > + * @cursor: current position inside the containers
> > + *
> > + * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
> > + * first fence.
> > + */
> > +static inline struct dma_fence *
> > +dma_fence_unwrap_first(struct dma_fence *head, struct dma_fence_unwrap *cursor)
> > +{
> > +	cursor->chain = dma_fence_get(head);
> > +	return dma_fence_unwrap_array(cursor);
> > +}
> > +
> > +/**
> > + * dma_fence_unwrap_next - return the next fence from a fence containers
> > + * @cursor: current position inside the containers
> > + *
> > + * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
> > + * the next fence from them.
> > + */
> > +static inline struct dma_fence *
> > +dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
> > +{
> > +	struct dma_fence *tmp;
> > +
> > +	++cursor->index;
> > +	tmp = dma_fence_array_next(cursor->array, cursor->index);
> > +	if (tmp)
> > +		return tmp;
> > +
> > +	cursor->chain = dma_fence_chain_walk(cursor->chain);
> > +	return dma_fence_unwrap_array(cursor);
> > +}
> > +
> > +/**
> > + * dma_fence_unwrap_for_each - iterate over all fences in containers
> > + * @fence: current fence
> > + * @cursor: current position inside the containers
> > + * @head: starting point for the iterator
> > + *
> > + * Unwrap dma_fence_chain and dma_fence_array containers and deep dive into all
> > + * potential fences in them. If @head is just a normal fence only that one is
> > + * returned.
> > + */
> > +#define dma_fence_unwrap_for_each(fence, cursor, head)			\
> > +	for (fence = dma_fence_unwrap_first(head, cursor); fence;	\
> > +	     fence = dma_fence_unwrap_next(cursor))
> > +
> > +#endif
> 
> I think it'd be really good to add a small paragraph to struct
> dma_fence_chain that this macro and iterator should be used for walking
> over all fences in a chain, including any fence arrays or anything like
> that.
> 
> With the bikesheds addressed:
> 
> Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
> 
> > -- 
> > 2.25.1
> > 
> 
> -- 
> Daniel Vetter
> Software Engineer, Intel Corporation
> http://blog.ffwll.ch
Christian König March 25, 2022, 10:10 a.m. UTC | #8 
Am 25.03.22 um 11:07 schrieb Daniel Vetter:
> On Fri, Mar 25, 2022 at 11:03:54AM +0100, Daniel Vetter wrote:
>> On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
>>> Add a general purpose helper to deep dive into dma_fence_chain/dma_fence_array
>>> structures and iterate over all the fences in them.
>>>
>>> This is useful when we need to flatten out all fences in those structures.
>>>
>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>> ---
>>>   Documentation/driver-api/dma-buf.rst  |   6 +
>>>   drivers/dma-buf/Makefile              |   1 +
>>>   drivers/dma-buf/selftests.h           |   1 +
>>>   drivers/dma-buf/st-dma-fence-unwrap.c | 279 ++++++++++++++++++++++++++
>>>   include/linux/dma-fence-unwrap.h      |  99 +++++++++
>>>   5 files changed, 386 insertions(+)
>>>   create mode 100644 drivers/dma-buf/st-dma-fence-unwrap.c
>>>   create mode 100644 include/linux/dma-fence-unwrap.h
>>>
>>> diff --git a/Documentation/driver-api/dma-buf.rst b/Documentation/driver-api/dma-buf.rst
>>> index 2cd7db82d9fe..7209500f08c8 100644
>>> --- a/Documentation/driver-api/dma-buf.rst
>>> +++ b/Documentation/driver-api/dma-buf.rst
>>> @@ -194,6 +194,12 @@ DMA Fence Chain
>>>   .. kernel-doc:: include/linux/dma-fence-chain.h
>>>      :internal:
>>>   
>>> +DMA Fence unwrap
>>> +~~~~~~~~~~~~~~~~
>>> +
>>> +.. kernel-doc:: include/linux/dma-fence-unwrap.h
>>> +   :internal:
> Ok I forgot one bikeshed: I'd just include this in dma-fence-chain.h and
> maybe go with the dma_fence_chain_unwrap_ prefix for everything. That
> makes it even more clearer that the two are meant to go together. Plus ofc
> the link from struct dma_fence_chain to this iterator in the docs too.
>
> Or I'm just not understanding why you made this a separate thing?

Well it should be used to unwrap dma_fence_array containers as well and 
I don't really want to add a dependency between dma_fence_chain and 
dma_fence_array.

I've spend quite some work to keep the two containers separated and also 
describe the separate use cases for each.

I can of course add some kerneldoc to let the chain and array 
documentation point to this one here.

Thanks,
Christian.

> -Daniel
>
>>> +
>>>   DMA Fence uABI/Sync File
>>>   ~~~~~~~~~~~~~~~~~~~~~~~~
>>>   
>>> diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
>>> index 511805dbeb75..4c9eb53ba3f8 100644
>>> --- a/drivers/dma-buf/Makefile
>>> +++ b/drivers/dma-buf/Makefile
>>> @@ -12,6 +12,7 @@ dmabuf_selftests-y := \
>>>   	selftest.o \
>>>   	st-dma-fence.o \
>>>   	st-dma-fence-chain.o \
>>> +	st-dma-fence-unwrap.o \
>>>   	st-dma-resv.o
>>>   
>>>   obj-$(CONFIG_DMABUF_SELFTESTS)	+= dmabuf_selftests.o
>>> diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
>>> index 97d73aaa31da..851965867d9c 100644
>>> --- a/drivers/dma-buf/selftests.h
>>> +++ b/drivers/dma-buf/selftests.h
>>> @@ -12,4 +12,5 @@
>>>   selftest(sanitycheck, __sanitycheck__) /* keep first (igt selfcheck) */
>>>   selftest(dma_fence, dma_fence)
>>>   selftest(dma_fence_chain, dma_fence_chain)
>>> +selftest(dma_fence_unwrap, dma_fence_unwrap)
>>>   selftest(dma_resv, dma_resv)
>>> diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
>>> new file mode 100644
>>> index 000000000000..d821faaebe93
>>> --- /dev/null
>>> +++ b/drivers/dma-buf/st-dma-fence-unwrap.c
>>> @@ -0,0 +1,279 @@
>>> +// SPDX-License-Identifier: MIT
>>> +
>>> +/*
>>> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
>>> + */
>>> +
>>> +#include <linux/dma-fence-unwrap.h>
>>> +#if 0
>>> +#include <linux/kernel.h>
>>> +#include <linux/kthread.h>
>>> +#include <linux/mm.h>
>>> +#include <linux/sched/signal.h>
>>> +#include <linux/slab.h>
>>> +#include <linux/spinlock.h>
>>> +#include <linux/random.h>
>>> +#endif
>>> +
>>> +#include "selftest.h"
>>> +
>>> +#define CHAIN_SZ (4 << 10)
>>> +
>>> +static struct kmem_cache *slab_fences;
>> Your own slab feels a bit like overkill. kmalloc/kfree not good enough?
>>
>>> +
>>> +static inline struct mock_fence {
>>> +	struct dma_fence base;
>>> +	spinlock_t lock;
>>> +} *to_mock_fence(struct dma_fence *f) {
>>> +	return container_of(f, struct mock_fence, base);
>>> +}
>>> +
>>> +static const char *mock_name(struct dma_fence *f)
>>> +{
>>> +	return "mock";
>>> +}
>>> +
>>> +static void mock_fence_release(struct dma_fence *f)
>>> +{
>>> +	kmem_cache_free(slab_fences, to_mock_fence(f));
>>> +}
>>> +
>>> +static const struct dma_fence_ops mock_ops = {
>>> +	.get_driver_name = mock_name,
>>> +	.get_timeline_name = mock_name,
>>> +	.release = mock_fence_release,
>>> +};
>>> +
>>> +static struct dma_fence *mock_fence(void)
>>> +{
>>> +	struct mock_fence *f;
>>> +
>>> +	f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
>>> +	if (!f)
>>> +		return NULL;
>>> +
>>> +	spin_lock_init(&f->lock);
>>> +	dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
>>> +
>>> +	return &f->base;
>>> +}
>>> +
>>> +static struct dma_fence *mock_array(unsigned int num_fences, ...)
>>> +{
>>> +	struct dma_fence_array *array;
>>> +	struct dma_fence **fences;
>>> +	va_list valist;
>>> +	int i;
>>> +
>>> +	fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
>>> +	if (!fences)
>>> +		return NULL;
>>> +
>>> +	va_start(valist, num_fences);
>>> +	for (i = 0; i < num_fences; ++i)
>>> +		fences[i] = va_arg(valist, typeof(*fences));
>>> +	va_end(valist);
>>> +
>>> +	array = dma_fence_array_create(num_fences, fences,
>>> +				       dma_fence_context_alloc(1),
>>> +				       1, false);
>>> +	if (!array)
>>> +		goto cleanup;
>>> +	return &array->base;
>>> +
>>> +cleanup:
>>> +	for (i = 0; i < num_fences; ++i)
>>> +		dma_fence_put(fences[i]);
>>> +	kfree(fences);
>>> +	return NULL;
>>> +}
>>> +
>>> +static struct dma_fence *mock_chain(struct dma_fence *prev,
>>> +				    struct dma_fence *fence)
>>> +{
>>> +	struct dma_fence_chain *f;
>>> +
>>> +	f = dma_fence_chain_alloc();
>>> +	if (!f) {
>>> +		dma_fence_put(prev);
>>> +		dma_fence_put(fence);
>>> +		return NULL;
>>> +	}
>>> +
>>> +	dma_fence_chain_init(f, prev, fence, 1);
>>> +	return &f->base;
>>> +}
>>> +
>>> +static int sanitycheck(void *arg)
>>> +{
>>> +	struct dma_fence *f, *chain, *array;
>>> +	int err = 0;
>>> +
>>> +	f = mock_fence();
>>> +	if (!f)
>>> +		return -ENOMEM;
>>> +
>>> +	array = mock_array(1, f);
>>> +	if (!array)
>>> +		return -ENOMEM;
>>> +
>>> +	chain = mock_chain(NULL, array);
>>> +	if (!chain)
>>> +		return -ENOMEM;
>>> +
>>> +	dma_fence_signal(f);
>>> +	dma_fence_put(chain);
>>> +	return err;
>>> +}
>>> +
>>> +static int unwrap_array(void *arg)
>>> +{
>>> +	struct dma_fence *fence, *f1, *f2, *array;
>>> +	struct dma_fence_unwrap iter;
>>> +	int err = 0;
>>> +
>>> +	f1 = mock_fence();
>>> +	if (!f1)
>>> +		return -ENOMEM;
>>> +
>>> +	f2 = mock_fence();
>>> +	if (!f2) {
>>> +		dma_fence_put(f1);
>>> +		return -ENOMEM;
>>> +	}
>>> +
>>> +	array = mock_array(2, f1, f2);
>>> +	if (!array)
>>> +		return -ENOMEM;
>>> +
>>> +	dma_fence_unwrap_for_each(fence, &iter, array) {
>>> +		if (fence == f1) {
>>> +			f1 = NULL;
>>> +		} else if (fence == f2) {
>>> +			f2 = NULL;
>>> +		} else {
>>> +			pr_err("Unexpected fence!\n");
>>> +			err = -EINVAL;
>>> +		}
>>> +	}
>>> +
>>> +	if (f1 || f2) {
>>> +		pr_err("Not all fences seen!\n");
>>> +		err = -EINVAL;
>>> +	}
>>> +
>>> +	dma_fence_signal(f1);
>>> +	dma_fence_signal(f2);
>>> +	dma_fence_put(array);
>>> +	return 0;
>>> +}
>>> +
>>> +static int unwrap_chain(void *arg)
>>> +{
>>> +	struct dma_fence *fence, *f1, *f2, *chain;
>>> +	struct dma_fence_unwrap iter;
>>> +	int err = 0;
>>> +
>>> +	f1 = mock_fence();
>>> +	if (!f1)
>>> +		return -ENOMEM;
>>> +
>>> +	f2 = mock_fence();
>>> +	if (!f2) {
>>> +		dma_fence_put(f1);
>>> +		return -ENOMEM;
>>> +	}
>>> +
>>> +	chain = mock_chain(f1, f2);
>>> +	if (!chain)
>>> +		return -ENOMEM;
>>> +
>>> +	dma_fence_unwrap_for_each(fence, &iter, chain) {
>>> +		if (fence == f1) {
>>> +			f1 = NULL;
>>> +		} else if (fence == f2) {
>>> +			f2 = NULL;
>>> +		} else {
>>> +			pr_err("Unexpected fence!\n");
>>> +			err = -EINVAL;
>>> +		}
>>> +	}
>>> +
>>> +	if (f1 || f2) {
>>> +		pr_err("Not all fences seen!\n");
>>> +		err = -EINVAL;
>>> +	}
>>> +
>>> +	dma_fence_signal(f1);
>>> +	dma_fence_signal(f2);
>>> +	dma_fence_put(chain);
>>> +	return 0;
>>> +}
>>> +
>>> +static int unwrap_chain_array(void *arg)
>>> +{
>>> +	struct dma_fence *fence, *f1, *f2, *array, *chain;
>>> +	struct dma_fence_unwrap iter;
>>> +	int err = 0;
>>> +
>>> +	f1 = mock_fence();
>>> +	if (!f1)
>>> +		return -ENOMEM;
>>> +
>>> +	f2 = mock_fence();
>>> +	if (!f2) {
>>> +		dma_fence_put(f1);
>>> +		return -ENOMEM;
>>> +	}
>>> +
>>> +	array = mock_array(2, f1, f2);
>>> +	if (!array)
>>> +		return -ENOMEM;
>>> +
>>> +	chain = mock_chain(NULL, array);
>>> +	if (!chain)
>>> +		return -ENOMEM;
>>> +
>>> +	dma_fence_unwrap_for_each(fence, &iter, chain) {
>>> +		if (fence == f1) {
>>> +			f1 = NULL;
>>> +		} else if (fence == f2) {
>>> +			f2 = NULL;
>>> +		} else {
>>> +			pr_err("Unexpected fence!\n");
>>> +			err = -EINVAL;
>>> +		}
>>> +	}
>>> +
>>> +	if (f1 || f2) {
>>> +		pr_err("Not all fences seen!\n");
>>> +		err = -EINVAL;
>>> +	}
>>> +
>>> +	dma_fence_signal(f1);
>>> +	dma_fence_signal(f2);
>>> +	dma_fence_put(chain);
>>> +	return 0;
>>> +}
>>> +
>>> +int dma_fence_unwrap(void)
>>> +{
>>> +	static const struct subtest tests[] = {
>>> +		SUBTEST(sanitycheck),
>>> +		SUBTEST(unwrap_array),
>>> +		SUBTEST(unwrap_chain),
>>> +		SUBTEST(unwrap_chain_array),
>>> +	};
>>> +	int ret;
>>> +
>>> +	slab_fences = KMEM_CACHE(mock_fence,
>>> +				 SLAB_TYPESAFE_BY_RCU |
>>> +				 SLAB_HWCACHE_ALIGN);
>>> +	if (!slab_fences)
>>> +		return -ENOMEM;
>>> +
>>> +	ret = subtests(tests, NULL);
>>> +
>>> +	kmem_cache_destroy(slab_fences);
>>> +	return ret;
>>> +}
>>> diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h
>>> new file mode 100644
>>> index 000000000000..54963df00c98
>>> --- /dev/null
>>> +++ b/include/linux/dma-fence-unwrap.h
>>> @@ -0,0 +1,99 @@
>>> +/* SPDX-License-Identifier: GPL-2.0-only */
>>> +/*
>>> + * fence-chain: chain fences together in a timeline
>>> + *
>>> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
>>> + * Authors:
>>> + *	Christian König <christian.koenig@amd.com>
>>> + */
>>> +
>>> +#ifndef __LINUX_DMA_FENCE_UNWRAP_H
>>> +#define __LINUX_DMA_FENCE_UNWRAP_H
>>> +
>>> +#include <linux/dma-fence-chain.h>
>>> +#include <linux/dma-fence-array.h>
>>> +
>>> +/**
>>> + * struct dma_fence_unwrap - cursor into the container structure
>> I think adding "This should be used together with
>> dma_fence_unwrap_for_each() iterator macro." would be nice here. I just
>> like links :-)
>>
>>> + */
>>> +struct dma_fence_unwrap {
>>> +	/**
>>> +	 * @chain: potential dma_fence_chain, but can be other fence as well
>>> +	 */
>>> +	struct dma_fence *chain;
>>> +	/**
>>> +	 * @array: potential dma_fence_array, but can be other fence as well
>>> +	 */
>>> +	struct dma_fence *array;
>>> +	/**
>>> +	 * @index: last returned index if @array is really a dma_fence_array
>>> +	 */
>>> +	unsigned int index;
>>> +};
>>> +
>>> +/**
>>> + * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
>>> + * @cursor: cursor to initialize
>>> + *
>>> + * Helper function to unwrap dma_fence_array containers, don't touch directly.
>>> + * Use dma_fence_unwrap_first/next instead.
>>> + */
>>> +static inline struct dma_fence *
>>> +dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)
>> Since this is a helper that no one should call I'd give it a __ prefix and
>> drop the kerneldoc. Documenting stuff that people shouldn't use is
>> confusing :-)
>>
>>> +{
>>> +	cursor->array = dma_fence_chain_contained(cursor->chain);
>>> +	cursor->index = 0;
>>> +	return dma_fence_array_first(cursor->array);
>>> +}
>>> +
>>> +/**
>>> + * dma_fence_unwrap_first - return the first fence from fence containers
>>> + * @head: the entrypoint into the containers
>>> + * @cursor: current position inside the containers
>>> + *
>>> + * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
>>> + * first fence.
>>> + */
>>> +static inline struct dma_fence *
>>> +dma_fence_unwrap_first(struct dma_fence *head, struct dma_fence_unwrap *cursor)
>>> +{
>>> +	cursor->chain = dma_fence_get(head);
>>> +	return dma_fence_unwrap_array(cursor);
>>> +}
>>> +
>>> +/**
>>> + * dma_fence_unwrap_next - return the next fence from a fence containers
>>> + * @cursor: current position inside the containers
>>> + *
>>> + * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
>>> + * the next fence from them.
>>> + */
>>> +static inline struct dma_fence *
>>> +dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
>>> +{
>>> +	struct dma_fence *tmp;
>>> +
>>> +	++cursor->index;
>>> +	tmp = dma_fence_array_next(cursor->array, cursor->index);
>>> +	if (tmp)
>>> +		return tmp;
>>> +
>>> +	cursor->chain = dma_fence_chain_walk(cursor->chain);
>>> +	return dma_fence_unwrap_array(cursor);
>>> +}
>>> +
>>> +/**
>>> + * dma_fence_unwrap_for_each - iterate over all fences in containers
>>> + * @fence: current fence
>>> + * @cursor: current position inside the containers
>>> + * @head: starting point for the iterator
>>> + *
>>> + * Unwrap dma_fence_chain and dma_fence_array containers and deep dive into all
>>> + * potential fences in them. If @head is just a normal fence only that one is
>>> + * returned.
>>> + */
>>> +#define dma_fence_unwrap_for_each(fence, cursor, head)			\
>>> +	for (fence = dma_fence_unwrap_first(head, cursor); fence;	\
>>> +	     fence = dma_fence_unwrap_next(cursor))
>>> +
>>> +#endif
>> I think it'd be really good to add a small paragraph to struct
>> dma_fence_chain that this macro and iterator should be used for walking
>> over all fences in a chain, including any fence arrays or anything like
>> that.
>>
>> With the bikesheds addressed:
>>
>> Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
>>
>>> -- 
>>> 2.25.1
>>>
>> -- 
>> Daniel Vetter
>> Software Engineer, Intel Corporation
>> https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7Cchristian.koenig%40amd.com%7Caa747083900b451d359308da0e4745e3%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637837996532802687%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&amp;sdata=Jts5%2BObWJHeUG4oy2biwj5Bf3PKkMrYU%2F0EihvQRNuY%3D&amp;reserved=0
Daniel Vetter March 25, 2022, 10:17 a.m. UTC | #9 
On Fri, Mar 25, 2022 at 11:10:15AM +0100, Christian König wrote:
> Am 25.03.22 um 11:07 schrieb Daniel Vetter:
> > On Fri, Mar 25, 2022 at 11:03:54AM +0100, Daniel Vetter wrote:
> > > On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
> > > > Add a general purpose helper to deep dive into dma_fence_chain/dma_fence_array
> > > > structures and iterate over all the fences in them.
> > > > 
> > > > This is useful when we need to flatten out all fences in those structures.
> > > > 
> > > > Signed-off-by: Christian König <christian.koenig@amd.com>
> > > > ---
> > > >   Documentation/driver-api/dma-buf.rst  |   6 +
> > > >   drivers/dma-buf/Makefile              |   1 +
> > > >   drivers/dma-buf/selftests.h           |   1 +
> > > >   drivers/dma-buf/st-dma-fence-unwrap.c | 279 ++++++++++++++++++++++++++
> > > >   include/linux/dma-fence-unwrap.h      |  99 +++++++++
> > > >   5 files changed, 386 insertions(+)
> > > >   create mode 100644 drivers/dma-buf/st-dma-fence-unwrap.c
> > > >   create mode 100644 include/linux/dma-fence-unwrap.h
> > > > 
> > > > diff --git a/Documentation/driver-api/dma-buf.rst b/Documentation/driver-api/dma-buf.rst
> > > > index 2cd7db82d9fe..7209500f08c8 100644
> > > > --- a/Documentation/driver-api/dma-buf.rst
> > > > +++ b/Documentation/driver-api/dma-buf.rst
> > > > @@ -194,6 +194,12 @@ DMA Fence Chain
> > > >   .. kernel-doc:: include/linux/dma-fence-chain.h
> > > >      :internal:
> > > > +DMA Fence unwrap
> > > > +~~~~~~~~~~~~~~~~
> > > > +
> > > > +.. kernel-doc:: include/linux/dma-fence-unwrap.h
> > > > +   :internal:
> > Ok I forgot one bikeshed: I'd just include this in dma-fence-chain.h and
> > maybe go with the dma_fence_chain_unwrap_ prefix for everything. That
> > makes it even more clearer that the two are meant to go together. Plus ofc
> > the link from struct dma_fence_chain to this iterator in the docs too.
> > 
> > Or I'm just not understanding why you made this a separate thing?
> 
> Well it should be used to unwrap dma_fence_array containers as well and I
> don't really want to add a dependency between dma_fence_chain and
> dma_fence_array.
> 
> I've spend quite some work to keep the two containers separated and also
> describe the separate use cases for each.
> 
> I can of course add some kerneldoc to let the chain and array documentation
> point to this one here.

Yeah I think as a general iterator they should be fine as a separate
thing. Also just realized that we'd need links from both array and chain
to this since it's for both.

The other thing I noticed is that we have dma_fence_chain_for_each()
already. Should we replace all users of that outside of dma-fence-chain.c
with this new thing, and move the chain specific iterator into
dma-fence-chain.c so that it's hidden and people don't make funny
accidents? Just for more safety in this maze, also ofc that's all
follow-up.
-Daniel



> 
> Thanks,
> Christian.
> 
> > -Daniel
> > 
> > > > +
> > > >   DMA Fence uABI/Sync File
> > > >   ~~~~~~~~~~~~~~~~~~~~~~~~
> > > > diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
> > > > index 511805dbeb75..4c9eb53ba3f8 100644
> > > > --- a/drivers/dma-buf/Makefile
> > > > +++ b/drivers/dma-buf/Makefile
> > > > @@ -12,6 +12,7 @@ dmabuf_selftests-y := \
> > > >   	selftest.o \
> > > >   	st-dma-fence.o \
> > > >   	st-dma-fence-chain.o \
> > > > +	st-dma-fence-unwrap.o \
> > > >   	st-dma-resv.o
> > > >   obj-$(CONFIG_DMABUF_SELFTESTS)	+= dmabuf_selftests.o
> > > > diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
> > > > index 97d73aaa31da..851965867d9c 100644
> > > > --- a/drivers/dma-buf/selftests.h
> > > > +++ b/drivers/dma-buf/selftests.h
> > > > @@ -12,4 +12,5 @@
> > > >   selftest(sanitycheck, __sanitycheck__) /* keep first (igt selfcheck) */
> > > >   selftest(dma_fence, dma_fence)
> > > >   selftest(dma_fence_chain, dma_fence_chain)
> > > > +selftest(dma_fence_unwrap, dma_fence_unwrap)
> > > >   selftest(dma_resv, dma_resv)
> > > > diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
> > > > new file mode 100644
> > > > index 000000000000..d821faaebe93
> > > > --- /dev/null
> > > > +++ b/drivers/dma-buf/st-dma-fence-unwrap.c
> > > > @@ -0,0 +1,279 @@
> > > > +// SPDX-License-Identifier: MIT
> > > > +
> > > > +/*
> > > > + * Copyright (C) 2022 Advanced Micro Devices, Inc.
> > > > + */
> > > > +
> > > > +#include <linux/dma-fence-unwrap.h>
> > > > +#if 0
> > > > +#include <linux/kernel.h>
> > > > +#include <linux/kthread.h>
> > > > +#include <linux/mm.h>
> > > > +#include <linux/sched/signal.h>
> > > > +#include <linux/slab.h>
> > > > +#include <linux/spinlock.h>
> > > > +#include <linux/random.h>
> > > > +#endif
> > > > +
> > > > +#include "selftest.h"
> > > > +
> > > > +#define CHAIN_SZ (4 << 10)
> > > > +
> > > > +static struct kmem_cache *slab_fences;
> > > Your own slab feels a bit like overkill. kmalloc/kfree not good enough?
> > > 
> > > > +
> > > > +static inline struct mock_fence {
> > > > +	struct dma_fence base;
> > > > +	spinlock_t lock;
> > > > +} *to_mock_fence(struct dma_fence *f) {
> > > > +	return container_of(f, struct mock_fence, base);
> > > > +}
> > > > +
> > > > +static const char *mock_name(struct dma_fence *f)
> > > > +{
> > > > +	return "mock";
> > > > +}
> > > > +
> > > > +static void mock_fence_release(struct dma_fence *f)
> > > > +{
> > > > +	kmem_cache_free(slab_fences, to_mock_fence(f));
> > > > +}
> > > > +
> > > > +static const struct dma_fence_ops mock_ops = {
> > > > +	.get_driver_name = mock_name,
> > > > +	.get_timeline_name = mock_name,
> > > > +	.release = mock_fence_release,
> > > > +};
> > > > +
> > > > +static struct dma_fence *mock_fence(void)
> > > > +{
> > > > +	struct mock_fence *f;
> > > > +
> > > > +	f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
> > > > +	if (!f)
> > > > +		return NULL;
> > > > +
> > > > +	spin_lock_init(&f->lock);
> > > > +	dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
> > > > +
> > > > +	return &f->base;
> > > > +}
> > > > +
> > > > +static struct dma_fence *mock_array(unsigned int num_fences, ...)
> > > > +{
> > > > +	struct dma_fence_array *array;
> > > > +	struct dma_fence **fences;
> > > > +	va_list valist;
> > > > +	int i;
> > > > +
> > > > +	fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
> > > > +	if (!fences)
> > > > +		return NULL;
> > > > +
> > > > +	va_start(valist, num_fences);
> > > > +	for (i = 0; i < num_fences; ++i)
> > > > +		fences[i] = va_arg(valist, typeof(*fences));
> > > > +	va_end(valist);
> > > > +
> > > > +	array = dma_fence_array_create(num_fences, fences,
> > > > +				       dma_fence_context_alloc(1),
> > > > +				       1, false);
> > > > +	if (!array)
> > > > +		goto cleanup;
> > > > +	return &array->base;
> > > > +
> > > > +cleanup:
> > > > +	for (i = 0; i < num_fences; ++i)
> > > > +		dma_fence_put(fences[i]);
> > > > +	kfree(fences);
> > > > +	return NULL;
> > > > +}
> > > > +
> > > > +static struct dma_fence *mock_chain(struct dma_fence *prev,
> > > > +				    struct dma_fence *fence)
> > > > +{
> > > > +	struct dma_fence_chain *f;
> > > > +
> > > > +	f = dma_fence_chain_alloc();
> > > > +	if (!f) {
> > > > +		dma_fence_put(prev);
> > > > +		dma_fence_put(fence);
> > > > +		return NULL;
> > > > +	}
> > > > +
> > > > +	dma_fence_chain_init(f, prev, fence, 1);
> > > > +	return &f->base;
> > > > +}
> > > > +
> > > > +static int sanitycheck(void *arg)
> > > > +{
> > > > +	struct dma_fence *f, *chain, *array;
> > > > +	int err = 0;
> > > > +
> > > > +	f = mock_fence();
> > > > +	if (!f)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	array = mock_array(1, f);
> > > > +	if (!array)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	chain = mock_chain(NULL, array);
> > > > +	if (!chain)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	dma_fence_signal(f);
> > > > +	dma_fence_put(chain);
> > > > +	return err;
> > > > +}
> > > > +
> > > > +static int unwrap_array(void *arg)
> > > > +{
> > > > +	struct dma_fence *fence, *f1, *f2, *array;
> > > > +	struct dma_fence_unwrap iter;
> > > > +	int err = 0;
> > > > +
> > > > +	f1 = mock_fence();
> > > > +	if (!f1)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	f2 = mock_fence();
> > > > +	if (!f2) {
> > > > +		dma_fence_put(f1);
> > > > +		return -ENOMEM;
> > > > +	}
> > > > +
> > > > +	array = mock_array(2, f1, f2);
> > > > +	if (!array)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	dma_fence_unwrap_for_each(fence, &iter, array) {
> > > > +		if (fence == f1) {
> > > > +			f1 = NULL;
> > > > +		} else if (fence == f2) {
> > > > +			f2 = NULL;
> > > > +		} else {
> > > > +			pr_err("Unexpected fence!\n");
> > > > +			err = -EINVAL;
> > > > +		}
> > > > +	}
> > > > +
> > > > +	if (f1 || f2) {
> > > > +		pr_err("Not all fences seen!\n");
> > > > +		err = -EINVAL;
> > > > +	}
> > > > +
> > > > +	dma_fence_signal(f1);
> > > > +	dma_fence_signal(f2);
> > > > +	dma_fence_put(array);
> > > > +	return 0;
> > > > +}
> > > > +
> > > > +static int unwrap_chain(void *arg)
> > > > +{
> > > > +	struct dma_fence *fence, *f1, *f2, *chain;
> > > > +	struct dma_fence_unwrap iter;
> > > > +	int err = 0;
> > > > +
> > > > +	f1 = mock_fence();
> > > > +	if (!f1)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	f2 = mock_fence();
> > > > +	if (!f2) {
> > > > +		dma_fence_put(f1);
> > > > +		return -ENOMEM;
> > > > +	}
> > > > +
> > > > +	chain = mock_chain(f1, f2);
> > > > +	if (!chain)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	dma_fence_unwrap_for_each(fence, &iter, chain) {
> > > > +		if (fence == f1) {
> > > > +			f1 = NULL;
> > > > +		} else if (fence == f2) {
> > > > +			f2 = NULL;
> > > > +		} else {
> > > > +			pr_err("Unexpected fence!\n");
> > > > +			err = -EINVAL;
> > > > +		}
> > > > +	}
> > > > +
> > > > +	if (f1 || f2) {
> > > > +		pr_err("Not all fences seen!\n");
> > > > +		err = -EINVAL;
> > > > +	}
> > > > +
> > > > +	dma_fence_signal(f1);
> > > > +	dma_fence_signal(f2);
> > > > +	dma_fence_put(chain);
> > > > +	return 0;
> > > > +}
> > > > +
> > > > +static int unwrap_chain_array(void *arg)
> > > > +{
> > > > +	struct dma_fence *fence, *f1, *f2, *array, *chain;
> > > > +	struct dma_fence_unwrap iter;
> > > > +	int err = 0;
> > > > +
> > > > +	f1 = mock_fence();
> > > > +	if (!f1)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	f2 = mock_fence();
> > > > +	if (!f2) {
> > > > +		dma_fence_put(f1);
> > > > +		return -ENOMEM;
> > > > +	}
> > > > +
> > > > +	array = mock_array(2, f1, f2);
> > > > +	if (!array)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	chain = mock_chain(NULL, array);
> > > > +	if (!chain)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	dma_fence_unwrap_for_each(fence, &iter, chain) {
> > > > +		if (fence == f1) {
> > > > +			f1 = NULL;
> > > > +		} else if (fence == f2) {
> > > > +			f2 = NULL;
> > > > +		} else {
> > > > +			pr_err("Unexpected fence!\n");
> > > > +			err = -EINVAL;
> > > > +		}
> > > > +	}
> > > > +
> > > > +	if (f1 || f2) {
> > > > +		pr_err("Not all fences seen!\n");
> > > > +		err = -EINVAL;
> > > > +	}
> > > > +
> > > > +	dma_fence_signal(f1);
> > > > +	dma_fence_signal(f2);
> > > > +	dma_fence_put(chain);
> > > > +	return 0;
> > > > +}
> > > > +
> > > > +int dma_fence_unwrap(void)
> > > > +{
> > > > +	static const struct subtest tests[] = {
> > > > +		SUBTEST(sanitycheck),
> > > > +		SUBTEST(unwrap_array),
> > > > +		SUBTEST(unwrap_chain),
> > > > +		SUBTEST(unwrap_chain_array),
> > > > +	};
> > > > +	int ret;
> > > > +
> > > > +	slab_fences = KMEM_CACHE(mock_fence,
> > > > +				 SLAB_TYPESAFE_BY_RCU |
> > > > +				 SLAB_HWCACHE_ALIGN);
> > > > +	if (!slab_fences)
> > > > +		return -ENOMEM;
> > > > +
> > > > +	ret = subtests(tests, NULL);
> > > > +
> > > > +	kmem_cache_destroy(slab_fences);
> > > > +	return ret;
> > > > +}
> > > > diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h
> > > > new file mode 100644
> > > > index 000000000000..54963df00c98
> > > > --- /dev/null
> > > > +++ b/include/linux/dma-fence-unwrap.h
> > > > @@ -0,0 +1,99 @@
> > > > +/* SPDX-License-Identifier: GPL-2.0-only */
> > > > +/*
> > > > + * fence-chain: chain fences together in a timeline
> > > > + *
> > > > + * Copyright (C) 2022 Advanced Micro Devices, Inc.
> > > > + * Authors:
> > > > + *	Christian König <christian.koenig@amd.com>
> > > > + */
> > > > +
> > > > +#ifndef __LINUX_DMA_FENCE_UNWRAP_H
> > > > +#define __LINUX_DMA_FENCE_UNWRAP_H
> > > > +
> > > > +#include <linux/dma-fence-chain.h>
> > > > +#include <linux/dma-fence-array.h>
> > > > +
> > > > +/**
> > > > + * struct dma_fence_unwrap - cursor into the container structure
> > > I think adding "This should be used together with
> > > dma_fence_unwrap_for_each() iterator macro." would be nice here. I just
> > > like links :-)
> > > 
> > > > + */
> > > > +struct dma_fence_unwrap {
> > > > +	/**
> > > > +	 * @chain: potential dma_fence_chain, but can be other fence as well
> > > > +	 */
> > > > +	struct dma_fence *chain;
> > > > +	/**
> > > > +	 * @array: potential dma_fence_array, but can be other fence as well
> > > > +	 */
> > > > +	struct dma_fence *array;
> > > > +	/**
> > > > +	 * @index: last returned index if @array is really a dma_fence_array
> > > > +	 */
> > > > +	unsigned int index;
> > > > +};
> > > > +
> > > > +/**
> > > > + * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
> > > > + * @cursor: cursor to initialize
> > > > + *
> > > > + * Helper function to unwrap dma_fence_array containers, don't touch directly.
> > > > + * Use dma_fence_unwrap_first/next instead.
> > > > + */
> > > > +static inline struct dma_fence *
> > > > +dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)
> > > Since this is a helper that no one should call I'd give it a __ prefix and
> > > drop the kerneldoc. Documenting stuff that people shouldn't use is
> > > confusing :-)
> > > 
> > > > +{
> > > > +	cursor->array = dma_fence_chain_contained(cursor->chain);
> > > > +	cursor->index = 0;
> > > > +	return dma_fence_array_first(cursor->array);
> > > > +}
> > > > +
> > > > +/**
> > > > + * dma_fence_unwrap_first - return the first fence from fence containers
> > > > + * @head: the entrypoint into the containers
> > > > + * @cursor: current position inside the containers
> > > > + *
> > > > + * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
> > > > + * first fence.
> > > > + */
> > > > +static inline struct dma_fence *
> > > > +dma_fence_unwrap_first(struct dma_fence *head, struct dma_fence_unwrap *cursor)
> > > > +{
> > > > +	cursor->chain = dma_fence_get(head);
> > > > +	return dma_fence_unwrap_array(cursor);
> > > > +}
> > > > +
> > > > +/**
> > > > + * dma_fence_unwrap_next - return the next fence from a fence containers
> > > > + * @cursor: current position inside the containers
> > > > + *
> > > > + * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
> > > > + * the next fence from them.
> > > > + */
> > > > +static inline struct dma_fence *
> > > > +dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
> > > > +{
> > > > +	struct dma_fence *tmp;
> > > > +
> > > > +	++cursor->index;
> > > > +	tmp = dma_fence_array_next(cursor->array, cursor->index);
> > > > +	if (tmp)
> > > > +		return tmp;
> > > > +
> > > > +	cursor->chain = dma_fence_chain_walk(cursor->chain);
> > > > +	return dma_fence_unwrap_array(cursor);
> > > > +}
> > > > +
> > > > +/**
> > > > + * dma_fence_unwrap_for_each - iterate over all fences in containers
> > > > + * @fence: current fence
> > > > + * @cursor: current position inside the containers
> > > > + * @head: starting point for the iterator
> > > > + *
> > > > + * Unwrap dma_fence_chain and dma_fence_array containers and deep dive into all
> > > > + * potential fences in them. If @head is just a normal fence only that one is
> > > > + * returned.
> > > > + */
> > > > +#define dma_fence_unwrap_for_each(fence, cursor, head)			\
> > > > +	for (fence = dma_fence_unwrap_first(head, cursor); fence;	\
> > > > +	     fence = dma_fence_unwrap_next(cursor))
> > > > +
> > > > +#endif
> > > I think it'd be really good to add a small paragraph to struct
> > > dma_fence_chain that this macro and iterator should be used for walking
> > > over all fences in a chain, including any fence arrays or anything like
> > > that.
> > > 
> > > With the bikesheds addressed:
> > > 
> > > Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
> > > 
> > > > -- 
> > > > 2.25.1
> > > > 
> > > -- 
> > > Daniel Vetter
> > > Software Engineer, Intel Corporation
> > > https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7Cchristian.koenig%40amd.com%7Caa747083900b451d359308da0e4745e3%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637837996532802687%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&amp;sdata=Jts5%2BObWJHeUG4oy2biwj5Bf3PKkMrYU%2F0EihvQRNuY%3D&amp;reserved=0
>
Christian König March 25, 2022, 12:56 p.m. UTC | #10 
Am 25.03.22 um 11:17 schrieb Daniel Vetter:
> On Fri, Mar 25, 2022 at 11:10:15AM +0100, Christian König wrote:
>> Am 25.03.22 um 11:07 schrieb Daniel Vetter:
>>> On Fri, Mar 25, 2022 at 11:03:54AM +0100, Daniel Vetter wrote:
>>>> On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
>>>>> Add a general purpose helper to deep dive into dma_fence_chain/dma_fence_array
>>>>> structures and iterate over all the fences in them.
>>>>>
>>>>> This is useful when we need to flatten out all fences in those structures.
>>>>>
>>>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>>>> ---
>>>>>    Documentation/driver-api/dma-buf.rst  |   6 +
>>>>>    drivers/dma-buf/Makefile              |   1 +
>>>>>    drivers/dma-buf/selftests.h           |   1 +
>>>>>    drivers/dma-buf/st-dma-fence-unwrap.c | 279 ++++++++++++++++++++++++++
>>>>>    include/linux/dma-fence-unwrap.h      |  99 +++++++++
>>>>>    5 files changed, 386 insertions(+)
>>>>>    create mode 100644 drivers/dma-buf/st-dma-fence-unwrap.c
>>>>>    create mode 100644 include/linux/dma-fence-unwrap.h
>>>>>
>>>>> diff --git a/Documentation/driver-api/dma-buf.rst b/Documentation/driver-api/dma-buf.rst
>>>>> index 2cd7db82d9fe..7209500f08c8 100644
>>>>> --- a/Documentation/driver-api/dma-buf.rst
>>>>> +++ b/Documentation/driver-api/dma-buf.rst
>>>>> @@ -194,6 +194,12 @@ DMA Fence Chain
>>>>>    .. kernel-doc:: include/linux/dma-fence-chain.h
>>>>>       :internal:
>>>>> +DMA Fence unwrap
>>>>> +~~~~~~~~~~~~~~~~
>>>>> +
>>>>> +.. kernel-doc:: include/linux/dma-fence-unwrap.h
>>>>> +   :internal:
>>> Ok I forgot one bikeshed: I'd just include this in dma-fence-chain.h and
>>> maybe go with the dma_fence_chain_unwrap_ prefix for everything. That
>>> makes it even more clearer that the two are meant to go together. Plus ofc
>>> the link from struct dma_fence_chain to this iterator in the docs too.
>>>
>>> Or I'm just not understanding why you made this a separate thing?
>> Well it should be used to unwrap dma_fence_array containers as well and I
>> don't really want to add a dependency between dma_fence_chain and
>> dma_fence_array.
>>
>> I've spend quite some work to keep the two containers separated and also
>> describe the separate use cases for each.
>>
>> I can of course add some kerneldoc to let the chain and array documentation
>> point to this one here.
> Yeah I think as a general iterator they should be fine as a separate
> thing. Also just realized that we'd need links from both array and chain
> to this since it's for both.

Done.

>
> The other thing I noticed is that we have dma_fence_chain_for_each()
> already. Should we replace all users of that outside of dma-fence-chain.c
> with this new thing, and move the chain specific iterator into
> dma-fence-chain.c so that it's hidden and people don't make funny
> accidents? Just for more safety in this maze, also ofc that's all
> follow-up.

Uff, good question. There are some valid use cases for it I think.

Especially the timeline syncobj handling only want to work with chain 
structures and not with the eventually contained array.

It could just be that we might want to wrap those use cases into 
dma_fence_chain helpers.

Anyway, not stuff for drm-misc-next-fixes, but going to keep that in mind.

Christian.

> -Daniel
>
>
>
>> Thanks,
>> Christian.
>>
>>> -Daniel
>>>
>>>>> +
>>>>>    DMA Fence uABI/Sync File
>>>>>    ~~~~~~~~~~~~~~~~~~~~~~~~
>>>>> diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
>>>>> index 511805dbeb75..4c9eb53ba3f8 100644
>>>>> --- a/drivers/dma-buf/Makefile
>>>>> +++ b/drivers/dma-buf/Makefile
>>>>> @@ -12,6 +12,7 @@ dmabuf_selftests-y := \
>>>>>    	selftest.o \
>>>>>    	st-dma-fence.o \
>>>>>    	st-dma-fence-chain.o \
>>>>> +	st-dma-fence-unwrap.o \
>>>>>    	st-dma-resv.o
>>>>>    obj-$(CONFIG_DMABUF_SELFTESTS)	+= dmabuf_selftests.o
>>>>> diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
>>>>> index 97d73aaa31da..851965867d9c 100644
>>>>> --- a/drivers/dma-buf/selftests.h
>>>>> +++ b/drivers/dma-buf/selftests.h
>>>>> @@ -12,4 +12,5 @@
>>>>>    selftest(sanitycheck, __sanitycheck__) /* keep first (igt selfcheck) */
>>>>>    selftest(dma_fence, dma_fence)
>>>>>    selftest(dma_fence_chain, dma_fence_chain)
>>>>> +selftest(dma_fence_unwrap, dma_fence_unwrap)
>>>>>    selftest(dma_resv, dma_resv)
>>>>> diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
>>>>> new file mode 100644
>>>>> index 000000000000..d821faaebe93
>>>>> --- /dev/null
>>>>> +++ b/drivers/dma-buf/st-dma-fence-unwrap.c
>>>>> @@ -0,0 +1,279 @@
>>>>> +// SPDX-License-Identifier: MIT
>>>>> +
>>>>> +/*
>>>>> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
>>>>> + */
>>>>> +
>>>>> +#include <linux/dma-fence-unwrap.h>
>>>>> +#if 0
>>>>> +#include <linux/kernel.h>
>>>>> +#include <linux/kthread.h>
>>>>> +#include <linux/mm.h>
>>>>> +#include <linux/sched/signal.h>
>>>>> +#include <linux/slab.h>
>>>>> +#include <linux/spinlock.h>
>>>>> +#include <linux/random.h>
>>>>> +#endif
>>>>> +
>>>>> +#include "selftest.h"
>>>>> +
>>>>> +#define CHAIN_SZ (4 << 10)
>>>>> +
>>>>> +static struct kmem_cache *slab_fences;
>>>> Your own slab feels a bit like overkill. kmalloc/kfree not good enough?
>>>>
>>>>> +
>>>>> +static inline struct mock_fence {
>>>>> +	struct dma_fence base;
>>>>> +	spinlock_t lock;
>>>>> +} *to_mock_fence(struct dma_fence *f) {
>>>>> +	return container_of(f, struct mock_fence, base);
>>>>> +}
>>>>> +
>>>>> +static const char *mock_name(struct dma_fence *f)
>>>>> +{
>>>>> +	return "mock";
>>>>> +}
>>>>> +
>>>>> +static void mock_fence_release(struct dma_fence *f)
>>>>> +{
>>>>> +	kmem_cache_free(slab_fences, to_mock_fence(f));
>>>>> +}
>>>>> +
>>>>> +static const struct dma_fence_ops mock_ops = {
>>>>> +	.get_driver_name = mock_name,
>>>>> +	.get_timeline_name = mock_name,
>>>>> +	.release = mock_fence_release,
>>>>> +};
>>>>> +
>>>>> +static struct dma_fence *mock_fence(void)
>>>>> +{
>>>>> +	struct mock_fence *f;
>>>>> +
>>>>> +	f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
>>>>> +	if (!f)
>>>>> +		return NULL;
>>>>> +
>>>>> +	spin_lock_init(&f->lock);
>>>>> +	dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
>>>>> +
>>>>> +	return &f->base;
>>>>> +}
>>>>> +
>>>>> +static struct dma_fence *mock_array(unsigned int num_fences, ...)
>>>>> +{
>>>>> +	struct dma_fence_array *array;
>>>>> +	struct dma_fence **fences;
>>>>> +	va_list valist;
>>>>> +	int i;
>>>>> +
>>>>> +	fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
>>>>> +	if (!fences)
>>>>> +		return NULL;
>>>>> +
>>>>> +	va_start(valist, num_fences);
>>>>> +	for (i = 0; i < num_fences; ++i)
>>>>> +		fences[i] = va_arg(valist, typeof(*fences));
>>>>> +	va_end(valist);
>>>>> +
>>>>> +	array = dma_fence_array_create(num_fences, fences,
>>>>> +				       dma_fence_context_alloc(1),
>>>>> +				       1, false);
>>>>> +	if (!array)
>>>>> +		goto cleanup;
>>>>> +	return &array->base;
>>>>> +
>>>>> +cleanup:
>>>>> +	for (i = 0; i < num_fences; ++i)
>>>>> +		dma_fence_put(fences[i]);
>>>>> +	kfree(fences);
>>>>> +	return NULL;
>>>>> +}
>>>>> +
>>>>> +static struct dma_fence *mock_chain(struct dma_fence *prev,
>>>>> +				    struct dma_fence *fence)
>>>>> +{
>>>>> +	struct dma_fence_chain *f;
>>>>> +
>>>>> +	f = dma_fence_chain_alloc();
>>>>> +	if (!f) {
>>>>> +		dma_fence_put(prev);
>>>>> +		dma_fence_put(fence);
>>>>> +		return NULL;
>>>>> +	}
>>>>> +
>>>>> +	dma_fence_chain_init(f, prev, fence, 1);
>>>>> +	return &f->base;
>>>>> +}
>>>>> +
>>>>> +static int sanitycheck(void *arg)
>>>>> +{
>>>>> +	struct dma_fence *f, *chain, *array;
>>>>> +	int err = 0;
>>>>> +
>>>>> +	f = mock_fence();
>>>>> +	if (!f)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	array = mock_array(1, f);
>>>>> +	if (!array)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	chain = mock_chain(NULL, array);
>>>>> +	if (!chain)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	dma_fence_signal(f);
>>>>> +	dma_fence_put(chain);
>>>>> +	return err;
>>>>> +}
>>>>> +
>>>>> +static int unwrap_array(void *arg)
>>>>> +{
>>>>> +	struct dma_fence *fence, *f1, *f2, *array;
>>>>> +	struct dma_fence_unwrap iter;
>>>>> +	int err = 0;
>>>>> +
>>>>> +	f1 = mock_fence();
>>>>> +	if (!f1)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	f2 = mock_fence();
>>>>> +	if (!f2) {
>>>>> +		dma_fence_put(f1);
>>>>> +		return -ENOMEM;
>>>>> +	}
>>>>> +
>>>>> +	array = mock_array(2, f1, f2);
>>>>> +	if (!array)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	dma_fence_unwrap_for_each(fence, &iter, array) {
>>>>> +		if (fence == f1) {
>>>>> +			f1 = NULL;
>>>>> +		} else if (fence == f2) {
>>>>> +			f2 = NULL;
>>>>> +		} else {
>>>>> +			pr_err("Unexpected fence!\n");
>>>>> +			err = -EINVAL;
>>>>> +		}
>>>>> +	}
>>>>> +
>>>>> +	if (f1 || f2) {
>>>>> +		pr_err("Not all fences seen!\n");
>>>>> +		err = -EINVAL;
>>>>> +	}
>>>>> +
>>>>> +	dma_fence_signal(f1);
>>>>> +	dma_fence_signal(f2);
>>>>> +	dma_fence_put(array);
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +static int unwrap_chain(void *arg)
>>>>> +{
>>>>> +	struct dma_fence *fence, *f1, *f2, *chain;
>>>>> +	struct dma_fence_unwrap iter;
>>>>> +	int err = 0;
>>>>> +
>>>>> +	f1 = mock_fence();
>>>>> +	if (!f1)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	f2 = mock_fence();
>>>>> +	if (!f2) {
>>>>> +		dma_fence_put(f1);
>>>>> +		return -ENOMEM;
>>>>> +	}
>>>>> +
>>>>> +	chain = mock_chain(f1, f2);
>>>>> +	if (!chain)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	dma_fence_unwrap_for_each(fence, &iter, chain) {
>>>>> +		if (fence == f1) {
>>>>> +			f1 = NULL;
>>>>> +		} else if (fence == f2) {
>>>>> +			f2 = NULL;
>>>>> +		} else {
>>>>> +			pr_err("Unexpected fence!\n");
>>>>> +			err = -EINVAL;
>>>>> +		}
>>>>> +	}
>>>>> +
>>>>> +	if (f1 || f2) {
>>>>> +		pr_err("Not all fences seen!\n");
>>>>> +		err = -EINVAL;
>>>>> +	}
>>>>> +
>>>>> +	dma_fence_signal(f1);
>>>>> +	dma_fence_signal(f2);
>>>>> +	dma_fence_put(chain);
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +static int unwrap_chain_array(void *arg)
>>>>> +{
>>>>> +	struct dma_fence *fence, *f1, *f2, *array, *chain;
>>>>> +	struct dma_fence_unwrap iter;
>>>>> +	int err = 0;
>>>>> +
>>>>> +	f1 = mock_fence();
>>>>> +	if (!f1)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	f2 = mock_fence();
>>>>> +	if (!f2) {
>>>>> +		dma_fence_put(f1);
>>>>> +		return -ENOMEM;
>>>>> +	}
>>>>> +
>>>>> +	array = mock_array(2, f1, f2);
>>>>> +	if (!array)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	chain = mock_chain(NULL, array);
>>>>> +	if (!chain)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	dma_fence_unwrap_for_each(fence, &iter, chain) {
>>>>> +		if (fence == f1) {
>>>>> +			f1 = NULL;
>>>>> +		} else if (fence == f2) {
>>>>> +			f2 = NULL;
>>>>> +		} else {
>>>>> +			pr_err("Unexpected fence!\n");
>>>>> +			err = -EINVAL;
>>>>> +		}
>>>>> +	}
>>>>> +
>>>>> +	if (f1 || f2) {
>>>>> +		pr_err("Not all fences seen!\n");
>>>>> +		err = -EINVAL;
>>>>> +	}
>>>>> +
>>>>> +	dma_fence_signal(f1);
>>>>> +	dma_fence_signal(f2);
>>>>> +	dma_fence_put(chain);
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +int dma_fence_unwrap(void)
>>>>> +{
>>>>> +	static const struct subtest tests[] = {
>>>>> +		SUBTEST(sanitycheck),
>>>>> +		SUBTEST(unwrap_array),
>>>>> +		SUBTEST(unwrap_chain),
>>>>> +		SUBTEST(unwrap_chain_array),
>>>>> +	};
>>>>> +	int ret;
>>>>> +
>>>>> +	slab_fences = KMEM_CACHE(mock_fence,
>>>>> +				 SLAB_TYPESAFE_BY_RCU |
>>>>> +				 SLAB_HWCACHE_ALIGN);
>>>>> +	if (!slab_fences)
>>>>> +		return -ENOMEM;
>>>>> +
>>>>> +	ret = subtests(tests, NULL);
>>>>> +
>>>>> +	kmem_cache_destroy(slab_fences);
>>>>> +	return ret;
>>>>> +}
>>>>> diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h
>>>>> new file mode 100644
>>>>> index 000000000000..54963df00c98
>>>>> --- /dev/null
>>>>> +++ b/include/linux/dma-fence-unwrap.h
>>>>> @@ -0,0 +1,99 @@
>>>>> +/* SPDX-License-Identifier: GPL-2.0-only */
>>>>> +/*
>>>>> + * fence-chain: chain fences together in a timeline
>>>>> + *
>>>>> + * Copyright (C) 2022 Advanced Micro Devices, Inc.
>>>>> + * Authors:
>>>>> + *	Christian König <christian.koenig@amd.com>
>>>>> + */
>>>>> +
>>>>> +#ifndef __LINUX_DMA_FENCE_UNWRAP_H
>>>>> +#define __LINUX_DMA_FENCE_UNWRAP_H
>>>>> +
>>>>> +#include <linux/dma-fence-chain.h>
>>>>> +#include <linux/dma-fence-array.h>
>>>>> +
>>>>> +/**
>>>>> + * struct dma_fence_unwrap - cursor into the container structure
>>>> I think adding "This should be used together with
>>>> dma_fence_unwrap_for_each() iterator macro." would be nice here. I just
>>>> like links :-)
>>>>
>>>>> + */
>>>>> +struct dma_fence_unwrap {
>>>>> +	/**
>>>>> +	 * @chain: potential dma_fence_chain, but can be other fence as well
>>>>> +	 */
>>>>> +	struct dma_fence *chain;
>>>>> +	/**
>>>>> +	 * @array: potential dma_fence_array, but can be other fence as well
>>>>> +	 */
>>>>> +	struct dma_fence *array;
>>>>> +	/**
>>>>> +	 * @index: last returned index if @array is really a dma_fence_array
>>>>> +	 */
>>>>> +	unsigned int index;
>>>>> +};
>>>>> +
>>>>> +/**
>>>>> + * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
>>>>> + * @cursor: cursor to initialize
>>>>> + *
>>>>> + * Helper function to unwrap dma_fence_array containers, don't touch directly.
>>>>> + * Use dma_fence_unwrap_first/next instead.
>>>>> + */
>>>>> +static inline struct dma_fence *
>>>>> +dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)
>>>> Since this is a helper that no one should call I'd give it a __ prefix and
>>>> drop the kerneldoc. Documenting stuff that people shouldn't use is
>>>> confusing :-)
>>>>
>>>>> +{
>>>>> +	cursor->array = dma_fence_chain_contained(cursor->chain);
>>>>> +	cursor->index = 0;
>>>>> +	return dma_fence_array_first(cursor->array);
>>>>> +}
>>>>> +
>>>>> +/**
>>>>> + * dma_fence_unwrap_first - return the first fence from fence containers
>>>>> + * @head: the entrypoint into the containers
>>>>> + * @cursor: current position inside the containers
>>>>> + *
>>>>> + * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
>>>>> + * first fence.
>>>>> + */
>>>>> +static inline struct dma_fence *
>>>>> +dma_fence_unwrap_first(struct dma_fence *head, struct dma_fence_unwrap *cursor)
>>>>> +{
>>>>> +	cursor->chain = dma_fence_get(head);
>>>>> +	return dma_fence_unwrap_array(cursor);
>>>>> +}
>>>>> +
>>>>> +/**
>>>>> + * dma_fence_unwrap_next - return the next fence from a fence containers
>>>>> + * @cursor: current position inside the containers
>>>>> + *
>>>>> + * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
>>>>> + * the next fence from them.
>>>>> + */
>>>>> +static inline struct dma_fence *
>>>>> +dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
>>>>> +{
>>>>> +	struct dma_fence *tmp;
>>>>> +
>>>>> +	++cursor->index;
>>>>> +	tmp = dma_fence_array_next(cursor->array, cursor->index);
>>>>> +	if (tmp)
>>>>> +		return tmp;
>>>>> +
>>>>> +	cursor->chain = dma_fence_chain_walk(cursor->chain);
>>>>> +	return dma_fence_unwrap_array(cursor);
>>>>> +}
>>>>> +
>>>>> +/**
>>>>> + * dma_fence_unwrap_for_each - iterate over all fences in containers
>>>>> + * @fence: current fence
>>>>> + * @cursor: current position inside the containers
>>>>> + * @head: starting point for the iterator
>>>>> + *
>>>>> + * Unwrap dma_fence_chain and dma_fence_array containers and deep dive into all
>>>>> + * potential fences in them. If @head is just a normal fence only that one is
>>>>> + * returned.
>>>>> + */
>>>>> +#define dma_fence_unwrap_for_each(fence, cursor, head)			\
>>>>> +	for (fence = dma_fence_unwrap_first(head, cursor); fence;	\
>>>>> +	     fence = dma_fence_unwrap_next(cursor))
>>>>> +
>>>>> +#endif
>>>> I think it'd be really good to add a small paragraph to struct
>>>> dma_fence_chain that this macro and iterator should be used for walking
>>>> over all fences in a chain, including any fence arrays or anything like
>>>> that.
>>>>
>>>> With the bikesheds addressed:
>>>>
>>>> Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
>>>>
>>>>> -- 
>>>>> 2.25.1
>>>>>
>>>> -- 
>>>> Daniel Vetter
>>>> Software Engineer, Intel Corporation
>>>> https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7Cchristian.koenig%40amd.com%7Caa747083900b451d359308da0e4745e3%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637837996532802687%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&amp;sdata=Jts5%2BObWJHeUG4oy2biwj5Bf3PKkMrYU%2F0EihvQRNuY%3D&amp;reserved=0
Ville Syrjälä March 25, 2022, 3:25 p.m. UTC | #11 
On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
> Add a general purpose helper to deep dive into dma_fence_chain/dma_fence_array
> structures and iterate over all the fences in them.
> 
> This is useful when we need to flatten out all fences in those structures.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>

One of the dma-buf patches took down Intel CI. Looks like every
machine oopses in some sync_file thing now:
<1>[  260.470008] BUG: kernel NULL pointer dereference, address: 0000000000000010
<1>[  260.470020] #PF: supervisor read access in kernel mode
<1>[  260.470025] #PF: error_code(0x0000) - not-present page
<6>[  260.470030] PGD 0 P4D 0 
<4>[  260.470035] Oops: 0000 [#1] PREEMPT SMP NOPTI
<4>[  260.470040] CPU: 0 PID: 5306 Comm: core_hotunplug Not tainted 5.17.0-CI-CI_DRM_11405+ #1
<4>[  260.470049] Hardware name: Intel Corporation Jasper Lake Client Platform/Jasperlake DDR4 SODIMM RVP, BIOS JSLSFWI1.R00.2385.D02.2010160831 10/16/2020
<4>[  260.470058] RIP: 0010:dma_fence_array_first+0x19/0x20
<4>[  260.470068] Code: c3 31 c0 c3 0f 1f 00 66 2e 0f 1f 84 00 00 00 00 00 48 85 ff 74 18 48 89 f8 48 81 7f 08 60 7f 0a 82 75 0d 48 8b 87 88 00 00 00 <48> 8b 00 c3 31 c0 c3 31 c0 48 85 ff 74 20 48 81 7f 08 60 7f 0a 82
<4>[  260.470080] RSP: 0018:ffffc900009a7e40 EFLAGS: 00010246
<4>[  260.470086] RAX: 0000000000000010 RBX: 000000000000000b RCX: 0000000000000000
<4>[  260.470100] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff888109a796b8
<4>[  260.470106] RBP: ffff888109a796b8 R08: 0000000000000002 R09: 0000000000000000
<4>[  260.470112] R10: 0000000000000001 R11: 0000000000000000 R12: ffff888109a796b8
<4>[  260.470118] R13: 00007ffc5d21f750 R14: ffff8881065be340 R15: 0000000000000000
<4>[  260.470124] FS:  00007f08bd6d04c0(0000) GS:ffff88885fc00000(0000) knlGS:0000000000000000
<4>[  260.470131] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
<4>[  260.470137] CR2: 0000000000000010 CR3: 000000010e690000 CR4: 0000000000350ef0
<4>[  260.470143] Call Trace:
<4>[  260.470146]  <TASK>
<4>[  260.470149]  sync_file_ioctl+0x2eb/0xc90
<4>[  260.470157]  __x64_sys_ioctl+0x6a/0xa0
<4>[  260.470164]  do_syscall_64+0x37/0xb0
<4>[  260.470170]  entry_SYSCALL_64_after_hwframe+0x44/0xae
<4>[  260.470176] RIP: 0033:0x7f08c047750b
<4>[  260.470182] Code: 0f 1e fa 48 8b 05 85 39 0d 00 64 c7 00 26 00 00 00 48 c7 c0 ff ff ff ff c3 66 0f 1f 44 00 00 f3 0f 1e fa b8 10 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 55 39 0d 00 f7 d8 64 89 01 48
<4>[  260.470195] RSP: 002b:00007ffc5d21f748 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
<4>[  260.470202] RAX: ffffffffffffffda RBX: 000000000000000a RCX: 00007f08c047750b
<4>[  260.470208] RDX: 00007ffc5d21f750 RSI: 00000000c0303e03 RDI: 000000000000000a
<4>[  260.470214] RBP: 0000000000000005 R08: 0000000000000000 R09: 00007ffc5d21ebb0
<4>[  260.470219] R10: 00007f08c0a6170e R11: 0000000000000246 R12: 0000000000000000
<4>[  260.470225] R13: 00007ffc5d21f840 R14: 000055e80d13d1b0 R15: 0000000000000009
<4>[  260.470233]  </TASK>
<4>[  260.470239] Modules linked in: vgem drm_shmem_helper fuse snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_codec_generic ledtrig_audio i915 snd_hda_intel snd_intel_dspcfg snd_hda_codec x86_pkg_temp_thermal coretemp crct10dif_pclmul crc32_pclmul snd_hwdep ghash_clmulni_intel snd_hda_core ttm drm_buddy drm_dp_helper snd_pcm drm_kms_helper i2c_i801 r8169 i2c_smbus syscopyarea realtek sysfillrect sysimgblt fb_sys_fops prime_numbers intel_lpss_pci
<4>[  260.470289] CR2: 0000000000000010
<4>[  260.470312] ---[ end trace 0000000000000000 ]---

Can we please start running dma-buf stuff through CI so this
kind of stuff doesn't keep happening so often?
Christian König March 25, 2022, 3:28 p.m. UTC | #12 
Am 25.03.22 um 16:25 schrieb Ville Syrjälä:
> On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
>> Add a general purpose helper to deep dive into dma_fence_chain/dma_fence_array
>> structures and iterate over all the fences in them.
>>
>> This is useful when we need to flatten out all fences in those structures.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
> One of the dma-buf patches took down Intel CI. Looks like every
> machine oopses in some sync_file thing now:
> <1>[  260.470008] BUG: kernel NULL pointer dereference, address: 0000000000000010
> <1>[  260.470020] #PF: supervisor read access in kernel mode
> <1>[  260.470025] #PF: error_code(0x0000) - not-present page
> <6>[  260.470030] PGD 0 P4D 0
> <4>[  260.470035] Oops: 0000 [#1] PREEMPT SMP NOPTI
> <4>[  260.470040] CPU: 0 PID: 5306 Comm: core_hotunplug Not tainted 5.17.0-CI-CI_DRM_11405+ #1
> <4>[  260.470049] Hardware name: Intel Corporation Jasper Lake Client Platform/Jasperlake DDR4 SODIMM RVP, BIOS JSLSFWI1.R00.2385.D02.2010160831 10/16/2020
> <4>[  260.470058] RIP: 0010:dma_fence_array_first+0x19/0x20
> <4>[  260.470068] Code: c3 31 c0 c3 0f 1f 00 66 2e 0f 1f 84 00 00 00 00 00 48 85 ff 74 18 48 89 f8 48 81 7f 08 60 7f 0a 82 75 0d 48 8b 87 88 00 00 00 <48> 8b 00 c3 31 c0 c3 31 c0 48 85 ff 74 20 48 81 7f 08 60 7f 0a 82
> <4>[  260.470080] RSP: 0018:ffffc900009a7e40 EFLAGS: 00010246
> <4>[  260.470086] RAX: 0000000000000010 RBX: 000000000000000b RCX: 0000000000000000
> <4>[  260.470100] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff888109a796b8
> <4>[  260.470106] RBP: ffff888109a796b8 R08: 0000000000000002 R09: 0000000000000000
> <4>[  260.470112] R10: 0000000000000001 R11: 0000000000000000 R12: ffff888109a796b8
> <4>[  260.470118] R13: 00007ffc5d21f750 R14: ffff8881065be340 R15: 0000000000000000
> <4>[  260.470124] FS:  00007f08bd6d04c0(0000) GS:ffff88885fc00000(0000) knlGS:0000000000000000
> <4>[  260.470131] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> <4>[  260.470137] CR2: 0000000000000010 CR3: 000000010e690000 CR4: 0000000000350ef0
> <4>[  260.470143] Call Trace:
> <4>[  260.470146]  <TASK>
> <4>[  260.470149]  sync_file_ioctl+0x2eb/0xc90
> <4>[  260.470157]  __x64_sys_ioctl+0x6a/0xa0
> <4>[  260.470164]  do_syscall_64+0x37/0xb0
> <4>[  260.470170]  entry_SYSCALL_64_after_hwframe+0x44/0xae
> <4>[  260.470176] RIP: 0033:0x7f08c047750b
> <4>[  260.470182] Code: 0f 1e fa 48 8b 05 85 39 0d 00 64 c7 00 26 00 00 00 48 c7 c0 ff ff ff ff c3 66 0f 1f 44 00 00 f3 0f 1e fa b8 10 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 55 39 0d 00 f7 d8 64 89 01 48
> <4>[  260.470195] RSP: 002b:00007ffc5d21f748 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
> <4>[  260.470202] RAX: ffffffffffffffda RBX: 000000000000000a RCX: 00007f08c047750b
> <4>[  260.470208] RDX: 00007ffc5d21f750 RSI: 00000000c0303e03 RDI: 000000000000000a
> <4>[  260.470214] RBP: 0000000000000005 R08: 0000000000000000 R09: 00007ffc5d21ebb0
> <4>[  260.470219] R10: 00007f08c0a6170e R11: 0000000000000246 R12: 0000000000000000
> <4>[  260.470225] R13: 00007ffc5d21f840 R14: 000055e80d13d1b0 R15: 0000000000000009
> <4>[  260.470233]  </TASK>
> <4>[  260.470239] Modules linked in: vgem drm_shmem_helper fuse snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_codec_generic ledtrig_audio i915 snd_hda_intel snd_intel_dspcfg snd_hda_codec x86_pkg_temp_thermal coretemp crct10dif_pclmul crc32_pclmul snd_hwdep ghash_clmulni_intel snd_hda_core ttm drm_buddy drm_dp_helper snd_pcm drm_kms_helper i2c_i801 r8169 i2c_smbus syscopyarea realtek sysfillrect sysimgblt fb_sys_fops prime_numbers intel_lpss_pci
> <4>[  260.470289] CR2: 0000000000000010
> <4>[  260.470312] ---[ end trace 0000000000000000 ]---
>
> Can we please start running dma-buf stuff through CI so this
> kind of stuff doesn't keep happening so often?

I've did ran it through CI, but since the most recent patches didn't 
applied on top of drm-tip for some reason the result was just garbage.

Going to take a look into this immediately, looks like something which 
I've seen during development maybe the wrong patch was pushed to 
drm-misc-next-fixes or something like this.

Thanks for letting me know,
Christian.
Daniel Vetter March 25, 2022, 4 p.m. UTC | #13 
On Fri, Mar 25, 2022 at 01:56:23PM +0100, Christian König wrote:
> Am 25.03.22 um 11:17 schrieb Daniel Vetter:
> > On Fri, Mar 25, 2022 at 11:10:15AM +0100, Christian König wrote:
> > > Am 25.03.22 um 11:07 schrieb Daniel Vetter:
> > > > On Fri, Mar 25, 2022 at 11:03:54AM +0100, Daniel Vetter wrote:
> > > > > On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
> > > > > > Add a general purpose helper to deep dive into dma_fence_chain/dma_fence_array
> > > > > > structures and iterate over all the fences in them.
> > > > > > 
> > > > > > This is useful when we need to flatten out all fences in those structures.
> > > > > > 
> > > > > > Signed-off-by: Christian König <christian.koenig@amd.com>
> > > > > > ---
> > > > > >    Documentation/driver-api/dma-buf.rst  |   6 +
> > > > > >    drivers/dma-buf/Makefile              |   1 +
> > > > > >    drivers/dma-buf/selftests.h           |   1 +
> > > > > >    drivers/dma-buf/st-dma-fence-unwrap.c | 279 ++++++++++++++++++++++++++
> > > > > >    include/linux/dma-fence-unwrap.h      |  99 +++++++++
> > > > > >    5 files changed, 386 insertions(+)
> > > > > >    create mode 100644 drivers/dma-buf/st-dma-fence-unwrap.c
> > > > > >    create mode 100644 include/linux/dma-fence-unwrap.h
> > > > > > 
> > > > > > diff --git a/Documentation/driver-api/dma-buf.rst b/Documentation/driver-api/dma-buf.rst
> > > > > > index 2cd7db82d9fe..7209500f08c8 100644
> > > > > > --- a/Documentation/driver-api/dma-buf.rst
> > > > > > +++ b/Documentation/driver-api/dma-buf.rst
> > > > > > @@ -194,6 +194,12 @@ DMA Fence Chain
> > > > > >    .. kernel-doc:: include/linux/dma-fence-chain.h
> > > > > >       :internal:
> > > > > > +DMA Fence unwrap
> > > > > > +~~~~~~~~~~~~~~~~
> > > > > > +
> > > > > > +.. kernel-doc:: include/linux/dma-fence-unwrap.h
> > > > > > +   :internal:
> > > > Ok I forgot one bikeshed: I'd just include this in dma-fence-chain.h and
> > > > maybe go with the dma_fence_chain_unwrap_ prefix for everything. That
> > > > makes it even more clearer that the two are meant to go together. Plus ofc
> > > > the link from struct dma_fence_chain to this iterator in the docs too.
> > > > 
> > > > Or I'm just not understanding why you made this a separate thing?
> > > Well it should be used to unwrap dma_fence_array containers as well and I
> > > don't really want to add a dependency between dma_fence_chain and
> > > dma_fence_array.
> > > 
> > > I've spend quite some work to keep the two containers separated and also
> > > describe the separate use cases for each.
> > > 
> > > I can of course add some kerneldoc to let the chain and array documentation
> > > point to this one here.
> > Yeah I think as a general iterator they should be fine as a separate
> > thing. Also just realized that we'd need links from both array and chain
> > to this since it's for both.
> 
> Done.
> 
> > 
> > The other thing I noticed is that we have dma_fence_chain_for_each()
> > already. Should we replace all users of that outside of dma-fence-chain.c
> > with this new thing, and move the chain specific iterator into
> > dma-fence-chain.c so that it's hidden and people don't make funny
> > accidents? Just for more safety in this maze, also ofc that's all
> > follow-up.
> 
> Uff, good question. There are some valid use cases for it I think.
> 
> Especially the timeline syncobj handling only want to work with chain
> structures and not with the eventually contained array.
> 
> It could just be that we might want to wrap those use cases into
> dma_fence_chain helpers.

Yeah one of the drm_syncobj ones looked somewhat legit, the other is just
a dma_fence_chain_do_gc() in a trechcoat and should be wrapped in a proper
interface I guess. The driver ones looked more like they really want to
just walk fences (some iirc even hand-roll the array unwrapping further
down on their own).

> Anyway, not stuff for drm-misc-next-fixes, but going to keep that in mind.

Agreed.
-Daniel

> 
> Christian.
> 
> > -Daniel
> > 
> > 
> > 
> > > Thanks,
> > > Christian.
> > > 
> > > > -Daniel
> > > > 
> > > > > > +
> > > > > >    DMA Fence uABI/Sync File
> > > > > >    ~~~~~~~~~~~~~~~~~~~~~~~~
> > > > > > diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
> > > > > > index 511805dbeb75..4c9eb53ba3f8 100644
> > > > > > --- a/drivers/dma-buf/Makefile
> > > > > > +++ b/drivers/dma-buf/Makefile
> > > > > > @@ -12,6 +12,7 @@ dmabuf_selftests-y := \
> > > > > >    	selftest.o \
> > > > > >    	st-dma-fence.o \
> > > > > >    	st-dma-fence-chain.o \
> > > > > > +	st-dma-fence-unwrap.o \
> > > > > >    	st-dma-resv.o
> > > > > >    obj-$(CONFIG_DMABUF_SELFTESTS)	+= dmabuf_selftests.o
> > > > > > diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
> > > > > > index 97d73aaa31da..851965867d9c 100644
> > > > > > --- a/drivers/dma-buf/selftests.h
> > > > > > +++ b/drivers/dma-buf/selftests.h
> > > > > > @@ -12,4 +12,5 @@
> > > > > >    selftest(sanitycheck, __sanitycheck__) /* keep first (igt selfcheck) */
> > > > > >    selftest(dma_fence, dma_fence)
> > > > > >    selftest(dma_fence_chain, dma_fence_chain)
> > > > > > +selftest(dma_fence_unwrap, dma_fence_unwrap)
> > > > > >    selftest(dma_resv, dma_resv)
> > > > > > diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
> > > > > > new file mode 100644
> > > > > > index 000000000000..d821faaebe93
> > > > > > --- /dev/null
> > > > > > +++ b/drivers/dma-buf/st-dma-fence-unwrap.c
> > > > > > @@ -0,0 +1,279 @@
> > > > > > +// SPDX-License-Identifier: MIT
> > > > > > +
> > > > > > +/*
> > > > > > + * Copyright (C) 2022 Advanced Micro Devices, Inc.
> > > > > > + */
> > > > > > +
> > > > > > +#include <linux/dma-fence-unwrap.h>
> > > > > > +#if 0
> > > > > > +#include <linux/kernel.h>
> > > > > > +#include <linux/kthread.h>
> > > > > > +#include <linux/mm.h>
> > > > > > +#include <linux/sched/signal.h>
> > > > > > +#include <linux/slab.h>
> > > > > > +#include <linux/spinlock.h>
> > > > > > +#include <linux/random.h>
> > > > > > +#endif
> > > > > > +
> > > > > > +#include "selftest.h"
> > > > > > +
> > > > > > +#define CHAIN_SZ (4 << 10)
> > > > > > +
> > > > > > +static struct kmem_cache *slab_fences;
> > > > > Your own slab feels a bit like overkill. kmalloc/kfree not good enough?
> > > > > 
> > > > > > +
> > > > > > +static inline struct mock_fence {
> > > > > > +	struct dma_fence base;
> > > > > > +	spinlock_t lock;
> > > > > > +} *to_mock_fence(struct dma_fence *f) {
> > > > > > +	return container_of(f, struct mock_fence, base);
> > > > > > +}
> > > > > > +
> > > > > > +static const char *mock_name(struct dma_fence *f)
> > > > > > +{
> > > > > > +	return "mock";
> > > > > > +}
> > > > > > +
> > > > > > +static void mock_fence_release(struct dma_fence *f)
> > > > > > +{
> > > > > > +	kmem_cache_free(slab_fences, to_mock_fence(f));
> > > > > > +}
> > > > > > +
> > > > > > +static const struct dma_fence_ops mock_ops = {
> > > > > > +	.get_driver_name = mock_name,
> > > > > > +	.get_timeline_name = mock_name,
> > > > > > +	.release = mock_fence_release,
> > > > > > +};
> > > > > > +
> > > > > > +static struct dma_fence *mock_fence(void)
> > > > > > +{
> > > > > > +	struct mock_fence *f;
> > > > > > +
> > > > > > +	f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
> > > > > > +	if (!f)
> > > > > > +		return NULL;
> > > > > > +
> > > > > > +	spin_lock_init(&f->lock);
> > > > > > +	dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
> > > > > > +
> > > > > > +	return &f->base;
> > > > > > +}
> > > > > > +
> > > > > > +static struct dma_fence *mock_array(unsigned int num_fences, ...)
> > > > > > +{
> > > > > > +	struct dma_fence_array *array;
> > > > > > +	struct dma_fence **fences;
> > > > > > +	va_list valist;
> > > > > > +	int i;
> > > > > > +
> > > > > > +	fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
> > > > > > +	if (!fences)
> > > > > > +		return NULL;
> > > > > > +
> > > > > > +	va_start(valist, num_fences);
> > > > > > +	for (i = 0; i < num_fences; ++i)
> > > > > > +		fences[i] = va_arg(valist, typeof(*fences));
> > > > > > +	va_end(valist);
> > > > > > +
> > > > > > +	array = dma_fence_array_create(num_fences, fences,
> > > > > > +				       dma_fence_context_alloc(1),
> > > > > > +				       1, false);
> > > > > > +	if (!array)
> > > > > > +		goto cleanup;
> > > > > > +	return &array->base;
> > > > > > +
> > > > > > +cleanup:
> > > > > > +	for (i = 0; i < num_fences; ++i)
> > > > > > +		dma_fence_put(fences[i]);
> > > > > > +	kfree(fences);
> > > > > > +	return NULL;
> > > > > > +}
> > > > > > +
> > > > > > +static struct dma_fence *mock_chain(struct dma_fence *prev,
> > > > > > +				    struct dma_fence *fence)
> > > > > > +{
> > > > > > +	struct dma_fence_chain *f;
> > > > > > +
> > > > > > +	f = dma_fence_chain_alloc();
> > > > > > +	if (!f) {
> > > > > > +		dma_fence_put(prev);
> > > > > > +		dma_fence_put(fence);
> > > > > > +		return NULL;
> > > > > > +	}
> > > > > > +
> > > > > > +	dma_fence_chain_init(f, prev, fence, 1);
> > > > > > +	return &f->base;
> > > > > > +}
> > > > > > +
> > > > > > +static int sanitycheck(void *arg)
> > > > > > +{
> > > > > > +	struct dma_fence *f, *chain, *array;
> > > > > > +	int err = 0;
> > > > > > +
> > > > > > +	f = mock_fence();
> > > > > > +	if (!f)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	array = mock_array(1, f);
> > > > > > +	if (!array)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	chain = mock_chain(NULL, array);
> > > > > > +	if (!chain)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	dma_fence_signal(f);
> > > > > > +	dma_fence_put(chain);
> > > > > > +	return err;
> > > > > > +}
> > > > > > +
> > > > > > +static int unwrap_array(void *arg)
> > > > > > +{
> > > > > > +	struct dma_fence *fence, *f1, *f2, *array;
> > > > > > +	struct dma_fence_unwrap iter;
> > > > > > +	int err = 0;
> > > > > > +
> > > > > > +	f1 = mock_fence();
> > > > > > +	if (!f1)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	f2 = mock_fence();
> > > > > > +	if (!f2) {
> > > > > > +		dma_fence_put(f1);
> > > > > > +		return -ENOMEM;
> > > > > > +	}
> > > > > > +
> > > > > > +	array = mock_array(2, f1, f2);
> > > > > > +	if (!array)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	dma_fence_unwrap_for_each(fence, &iter, array) {
> > > > > > +		if (fence == f1) {
> > > > > > +			f1 = NULL;
> > > > > > +		} else if (fence == f2) {
> > > > > > +			f2 = NULL;
> > > > > > +		} else {
> > > > > > +			pr_err("Unexpected fence!\n");
> > > > > > +			err = -EINVAL;
> > > > > > +		}
> > > > > > +	}
> > > > > > +
> > > > > > +	if (f1 || f2) {
> > > > > > +		pr_err("Not all fences seen!\n");
> > > > > > +		err = -EINVAL;
> > > > > > +	}
> > > > > > +
> > > > > > +	dma_fence_signal(f1);
> > > > > > +	dma_fence_signal(f2);
> > > > > > +	dma_fence_put(array);
> > > > > > +	return 0;
> > > > > > +}
> > > > > > +
> > > > > > +static int unwrap_chain(void *arg)
> > > > > > +{
> > > > > > +	struct dma_fence *fence, *f1, *f2, *chain;
> > > > > > +	struct dma_fence_unwrap iter;
> > > > > > +	int err = 0;
> > > > > > +
> > > > > > +	f1 = mock_fence();
> > > > > > +	if (!f1)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	f2 = mock_fence();
> > > > > > +	if (!f2) {
> > > > > > +		dma_fence_put(f1);
> > > > > > +		return -ENOMEM;
> > > > > > +	}
> > > > > > +
> > > > > > +	chain = mock_chain(f1, f2);
> > > > > > +	if (!chain)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	dma_fence_unwrap_for_each(fence, &iter, chain) {
> > > > > > +		if (fence == f1) {
> > > > > > +			f1 = NULL;
> > > > > > +		} else if (fence == f2) {
> > > > > > +			f2 = NULL;
> > > > > > +		} else {
> > > > > > +			pr_err("Unexpected fence!\n");
> > > > > > +			err = -EINVAL;
> > > > > > +		}
> > > > > > +	}
> > > > > > +
> > > > > > +	if (f1 || f2) {
> > > > > > +		pr_err("Not all fences seen!\n");
> > > > > > +		err = -EINVAL;
> > > > > > +	}
> > > > > > +
> > > > > > +	dma_fence_signal(f1);
> > > > > > +	dma_fence_signal(f2);
> > > > > > +	dma_fence_put(chain);
> > > > > > +	return 0;
> > > > > > +}
> > > > > > +
> > > > > > +static int unwrap_chain_array(void *arg)
> > > > > > +{
> > > > > > +	struct dma_fence *fence, *f1, *f2, *array, *chain;
> > > > > > +	struct dma_fence_unwrap iter;
> > > > > > +	int err = 0;
> > > > > > +
> > > > > > +	f1 = mock_fence();
> > > > > > +	if (!f1)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	f2 = mock_fence();
> > > > > > +	if (!f2) {
> > > > > > +		dma_fence_put(f1);
> > > > > > +		return -ENOMEM;
> > > > > > +	}
> > > > > > +
> > > > > > +	array = mock_array(2, f1, f2);
> > > > > > +	if (!array)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	chain = mock_chain(NULL, array);
> > > > > > +	if (!chain)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	dma_fence_unwrap_for_each(fence, &iter, chain) {
> > > > > > +		if (fence == f1) {
> > > > > > +			f1 = NULL;
> > > > > > +		} else if (fence == f2) {
> > > > > > +			f2 = NULL;
> > > > > > +		} else {
> > > > > > +			pr_err("Unexpected fence!\n");
> > > > > > +			err = -EINVAL;
> > > > > > +		}
> > > > > > +	}
> > > > > > +
> > > > > > +	if (f1 || f2) {
> > > > > > +		pr_err("Not all fences seen!\n");
> > > > > > +		err = -EINVAL;
> > > > > > +	}
> > > > > > +
> > > > > > +	dma_fence_signal(f1);
> > > > > > +	dma_fence_signal(f2);
> > > > > > +	dma_fence_put(chain);
> > > > > > +	return 0;
> > > > > > +}
> > > > > > +
> > > > > > +int dma_fence_unwrap(void)
> > > > > > +{
> > > > > > +	static const struct subtest tests[] = {
> > > > > > +		SUBTEST(sanitycheck),
> > > > > > +		SUBTEST(unwrap_array),
> > > > > > +		SUBTEST(unwrap_chain),
> > > > > > +		SUBTEST(unwrap_chain_array),
> > > > > > +	};
> > > > > > +	int ret;
> > > > > > +
> > > > > > +	slab_fences = KMEM_CACHE(mock_fence,
> > > > > > +				 SLAB_TYPESAFE_BY_RCU |
> > > > > > +				 SLAB_HWCACHE_ALIGN);
> > > > > > +	if (!slab_fences)
> > > > > > +		return -ENOMEM;
> > > > > > +
> > > > > > +	ret = subtests(tests, NULL);
> > > > > > +
> > > > > > +	kmem_cache_destroy(slab_fences);
> > > > > > +	return ret;
> > > > > > +}
> > > > > > diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h
> > > > > > new file mode 100644
> > > > > > index 000000000000..54963df00c98
> > > > > > --- /dev/null
> > > > > > +++ b/include/linux/dma-fence-unwrap.h
> > > > > > @@ -0,0 +1,99 @@
> > > > > > +/* SPDX-License-Identifier: GPL-2.0-only */
> > > > > > +/*
> > > > > > + * fence-chain: chain fences together in a timeline
> > > > > > + *
> > > > > > + * Copyright (C) 2022 Advanced Micro Devices, Inc.
> > > > > > + * Authors:
> > > > > > + *	Christian König <christian.koenig@amd.com>
> > > > > > + */
> > > > > > +
> > > > > > +#ifndef __LINUX_DMA_FENCE_UNWRAP_H
> > > > > > +#define __LINUX_DMA_FENCE_UNWRAP_H
> > > > > > +
> > > > > > +#include <linux/dma-fence-chain.h>
> > > > > > +#include <linux/dma-fence-array.h>
> > > > > > +
> > > > > > +/**
> > > > > > + * struct dma_fence_unwrap - cursor into the container structure
> > > > > I think adding "This should be used together with
> > > > > dma_fence_unwrap_for_each() iterator macro." would be nice here. I just
> > > > > like links :-)
> > > > > 
> > > > > > + */
> > > > > > +struct dma_fence_unwrap {
> > > > > > +	/**
> > > > > > +	 * @chain: potential dma_fence_chain, but can be other fence as well
> > > > > > +	 */
> > > > > > +	struct dma_fence *chain;
> > > > > > +	/**
> > > > > > +	 * @array: potential dma_fence_array, but can be other fence as well
> > > > > > +	 */
> > > > > > +	struct dma_fence *array;
> > > > > > +	/**
> > > > > > +	 * @index: last returned index if @array is really a dma_fence_array
> > > > > > +	 */
> > > > > > +	unsigned int index;
> > > > > > +};
> > > > > > +
> > > > > > +/**
> > > > > > + * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
> > > > > > + * @cursor: cursor to initialize
> > > > > > + *
> > > > > > + * Helper function to unwrap dma_fence_array containers, don't touch directly.
> > > > > > + * Use dma_fence_unwrap_first/next instead.
> > > > > > + */
> > > > > > +static inline struct dma_fence *
> > > > > > +dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)
> > > > > Since this is a helper that no one should call I'd give it a __ prefix and
> > > > > drop the kerneldoc. Documenting stuff that people shouldn't use is
> > > > > confusing :-)
> > > > > 
> > > > > > +{
> > > > > > +	cursor->array = dma_fence_chain_contained(cursor->chain);
> > > > > > +	cursor->index = 0;
> > > > > > +	return dma_fence_array_first(cursor->array);
> > > > > > +}
> > > > > > +
> > > > > > +/**
> > > > > > + * dma_fence_unwrap_first - return the first fence from fence containers
> > > > > > + * @head: the entrypoint into the containers
> > > > > > + * @cursor: current position inside the containers
> > > > > > + *
> > > > > > + * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
> > > > > > + * first fence.
> > > > > > + */
> > > > > > +static inline struct dma_fence *
> > > > > > +dma_fence_unwrap_first(struct dma_fence *head, struct dma_fence_unwrap *cursor)
> > > > > > +{
> > > > > > +	cursor->chain = dma_fence_get(head);
> > > > > > +	return dma_fence_unwrap_array(cursor);
> > > > > > +}
> > > > > > +
> > > > > > +/**
> > > > > > + * dma_fence_unwrap_next - return the next fence from a fence containers
> > > > > > + * @cursor: current position inside the containers
> > > > > > + *
> > > > > > + * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
> > > > > > + * the next fence from them.
> > > > > > + */
> > > > > > +static inline struct dma_fence *
> > > > > > +dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
> > > > > > +{
> > > > > > +	struct dma_fence *tmp;
> > > > > > +
> > > > > > +	++cursor->index;
> > > > > > +	tmp = dma_fence_array_next(cursor->array, cursor->index);
> > > > > > +	if (tmp)
> > > > > > +		return tmp;
> > > > > > +
> > > > > > +	cursor->chain = dma_fence_chain_walk(cursor->chain);
> > > > > > +	return dma_fence_unwrap_array(cursor);
> > > > > > +}
> > > > > > +
> > > > > > +/**
> > > > > > + * dma_fence_unwrap_for_each - iterate over all fences in containers
> > > > > > + * @fence: current fence
> > > > > > + * @cursor: current position inside the containers
> > > > > > + * @head: starting point for the iterator
> > > > > > + *
> > > > > > + * Unwrap dma_fence_chain and dma_fence_array containers and deep dive into all
> > > > > > + * potential fences in them. If @head is just a normal fence only that one is
> > > > > > + * returned.
> > > > > > + */
> > > > > > +#define dma_fence_unwrap_for_each(fence, cursor, head)			\
> > > > > > +	for (fence = dma_fence_unwrap_first(head, cursor); fence;	\
> > > > > > +	     fence = dma_fence_unwrap_next(cursor))
> > > > > > +
> > > > > > +#endif
> > > > > I think it'd be really good to add a small paragraph to struct
> > > > > dma_fence_chain that this macro and iterator should be used for walking
> > > > > over all fences in a chain, including any fence arrays or anything like
> > > > > that.
> > > > > 
> > > > > With the bikesheds addressed:
> > > > > 
> > > > > Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
> > > > > 
> > > > > > -- 
> > > > > > 2.25.1
> > > > > > 
> > > > > -- 
> > > > > Daniel Vetter
> > > > > Software Engineer, Intel Corporation
> > > > > https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7Cchristian.koenig%40amd.com%7Caa747083900b451d359308da0e4745e3%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637837996532802687%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&amp;sdata=Jts5%2BObWJHeUG4oy2biwj5Bf3PKkMrYU%2F0EihvQRNuY%3D&amp;reserved=0
>
Christian König March 28, 2022, 10:28 a.m. UTC | #14 
Hi Ville & Daniel,

Am 25.03.22 um 16:28 schrieb Christian König:
> Am 25.03.22 um 16:25 schrieb Ville Syrjälä:
>> On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
>>> Add a general purpose helper to deep dive into 
>>> dma_fence_chain/dma_fence_array
>>> structures and iterate over all the fences in them.
>>>
>>> This is useful when we need to flatten out all fences in those 
>>> structures.
>>>
>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> One of the dma-buf patches took down Intel CI. Looks like every
>> machine oopses in some sync_file thing now:
>> <1>[  260.470008] BUG: kernel NULL pointer dereference, address: 
>> 0000000000000010
>> <1>[  260.470020] #PF: supervisor read access in kernel mode
>> <1>[  260.470025] #PF: error_code(0x0000) - not-present page
>> <6>[  260.470030] PGD 0 P4D 0
>> <4>[  260.470035] Oops: 0000 [#1] PREEMPT SMP NOPTI
>> <4>[  260.470040] CPU: 0 PID: 5306 Comm: core_hotunplug Not tainted 
>> 5.17.0-CI-CI_DRM_11405+ #1
>> <4>[  260.470049] Hardware name: Intel Corporation Jasper Lake Client 
>> Platform/Jasperlake DDR4 SODIMM RVP, BIOS 
>> JSLSFWI1.R00.2385.D02.2010160831 10/16/2020
>> <4>[  260.470058] RIP: 0010:dma_fence_array_first+0x19/0x20

I've looked into this and the root cause seems to be that the new code 
doesn't handle dma_fence_arrays with zero elements.

That is rather easy to fix, but a dma_fence_array with zero number of 
elements is most likely a bug because under the wrong circumstances it 
can create a dma_fence instance which will never signal.

I've send out a patch on Frinday ([PATCH] dma-buf: WIP 
dma_fence_array_first fix) which avoids the crash and prints a warning 
if anybody tries to create a dma_fence_array with zero length.

Can you test this?

Thanks in advance,
Christian.
Daniel Vetter March 28, 2022, 2:22 p.m. UTC | #15 
On Mon, Mar 28, 2022 at 12:28:31PM +0200, Christian König wrote:
> Hi Ville & Daniel,
> 
> Am 25.03.22 um 16:28 schrieb Christian König:
> > Am 25.03.22 um 16:25 schrieb Ville Syrjälä:
> > > On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
> > > > Add a general purpose helper to deep dive into
> > > > dma_fence_chain/dma_fence_array
> > > > structures and iterate over all the fences in them.
> > > > 
> > > > This is useful when we need to flatten out all fences in those
> > > > structures.
> > > > 
> > > > Signed-off-by: Christian König <christian.koenig@amd.com>
> > > One of the dma-buf patches took down Intel CI. Looks like every
> > > machine oopses in some sync_file thing now:
> > > <1>[  260.470008] BUG: kernel NULL pointer dereference, address:
> > > 0000000000000010
> > > <1>[  260.470020] #PF: supervisor read access in kernel mode
> > > <1>[  260.470025] #PF: error_code(0x0000) - not-present page
> > > <6>[  260.470030] PGD 0 P4D 0
> > > <4>[  260.470035] Oops: 0000 [#1] PREEMPT SMP NOPTI
> > > <4>[  260.470040] CPU: 0 PID: 5306 Comm: core_hotunplug Not tainted
> > > 5.17.0-CI-CI_DRM_11405+ #1
> > > <4>[  260.470049] Hardware name: Intel Corporation Jasper Lake
> > > Client Platform/Jasperlake DDR4 SODIMM RVP, BIOS
> > > JSLSFWI1.R00.2385.D02.2010160831 10/16/2020
> > > <4>[  260.470058] RIP: 0010:dma_fence_array_first+0x19/0x20
> 
> I've looked into this and the root cause seems to be that the new code
> doesn't handle dma_fence_arrays with zero elements.
> 
> That is rather easy to fix, but a dma_fence_array with zero number of
> elements is most likely a bug because under the wrong circumstances it can
> create a dma_fence instance which will never signal.
> 
> I've send out a patch on Frinday ([PATCH] dma-buf: WIP dma_fence_array_first
> fix) which avoids the crash and prints a warning if anybody tries to create
> a dma_fence_array with zero length.
> 
> Can you test this?

It's in drm-tip now (in the fixup branch) so drm-tip results should have
the result soonish:

https://intel-gfx-ci.01.org/tree/drm-tip/index.html?

Cheers, Daniel
Christian König March 28, 2022, 2:26 p.m. UTC | #16 
Am 28.03.22 um 16:22 schrieb Daniel Vetter:
> On Mon, Mar 28, 2022 at 12:28:31PM +0200, Christian König wrote:
>> Hi Ville & Daniel,
>>
>> Am 25.03.22 um 16:28 schrieb Christian König:
>>> Am 25.03.22 um 16:25 schrieb Ville Syrjälä:
>>>> On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
>>>>> Add a general purpose helper to deep dive into
>>>>> dma_fence_chain/dma_fence_array
>>>>> structures and iterate over all the fences in them.
>>>>>
>>>>> This is useful when we need to flatten out all fences in those
>>>>> structures.
>>>>>
>>>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>>> One of the dma-buf patches took down Intel CI. Looks like every
>>>> machine oopses in some sync_file thing now:
>>>> <1>[  260.470008] BUG: kernel NULL pointer dereference, address:
>>>> 0000000000000010
>>>> <1>[  260.470020] #PF: supervisor read access in kernel mode
>>>> <1>[  260.470025] #PF: error_code(0x0000) - not-present page
>>>> <6>[  260.470030] PGD 0 P4D 0
>>>> <4>[  260.470035] Oops: 0000 [#1] PREEMPT SMP NOPTI
>>>> <4>[  260.470040] CPU: 0 PID: 5306 Comm: core_hotunplug Not tainted
>>>> 5.17.0-CI-CI_DRM_11405+ #1
>>>> <4>[  260.470049] Hardware name: Intel Corporation Jasper Lake
>>>> Client Platform/Jasperlake DDR4 SODIMM RVP, BIOS
>>>> JSLSFWI1.R00.2385.D02.2010160831 10/16/2020
>>>> <4>[  260.470058] RIP: 0010:dma_fence_array_first+0x19/0x20
>> I've looked into this and the root cause seems to be that the new code
>> doesn't handle dma_fence_arrays with zero elements.
>>
>> That is rather easy to fix, but a dma_fence_array with zero number of
>> elements is most likely a bug because under the wrong circumstances it can
>> create a dma_fence instance which will never signal.
>>
>> I've send out a patch on Frinday ([PATCH] dma-buf: WIP dma_fence_array_first
>> fix) which avoids the crash and prints a warning if anybody tries to create
>> a dma_fence_array with zero length.
>>
>> Can you test this?
> It's in drm-tip now (in the fixup branch) so drm-tip results should have
> the result soonish:
>
> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fintel-gfx-ci.01.org%2Ftree%2Fdrm-tip%2Findex.html&amp;data=04%7C01%7Cchristian.koenig%40amd.com%7C0afc74b5df0c4ea384af08da10c672fa%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637840742273792356%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&amp;sdata=SL2CuMgM6lCSOhDTvs%2FaFg6zRlc7F3X%2BNkf6GuGMGXI%3D&amp;reserved=0?

How do I find something in there? Is there a search function over all 
the test results?

Thanks,
Christian.

>
> Cheers, Daniel
Daniel Vetter March 28, 2022, 2:28 p.m. UTC | #17 
On Mon, Mar 28, 2022 at 04:26:44PM +0200, Christian König wrote:
> Am 28.03.22 um 16:22 schrieb Daniel Vetter:
> > On Mon, Mar 28, 2022 at 12:28:31PM +0200, Christian König wrote:
> > > Hi Ville & Daniel,
> > > 
> > > Am 25.03.22 um 16:28 schrieb Christian König:
> > > > Am 25.03.22 um 16:25 schrieb Ville Syrjälä:
> > > > > On Fri, Mar 11, 2022 at 12:02:43PM +0100, Christian König wrote:
> > > > > > Add a general purpose helper to deep dive into
> > > > > > dma_fence_chain/dma_fence_array
> > > > > > structures and iterate over all the fences in them.
> > > > > > 
> > > > > > This is useful when we need to flatten out all fences in those
> > > > > > structures.
> > > > > > 
> > > > > > Signed-off-by: Christian König <christian.koenig@amd.com>
> > > > > One of the dma-buf patches took down Intel CI. Looks like every
> > > > > machine oopses in some sync_file thing now:
> > > > > <1>[  260.470008] BUG: kernel NULL pointer dereference, address:
> > > > > 0000000000000010
> > > > > <1>[  260.470020] #PF: supervisor read access in kernel mode
> > > > > <1>[  260.470025] #PF: error_code(0x0000) - not-present page
> > > > > <6>[  260.470030] PGD 0 P4D 0
> > > > > <4>[  260.470035] Oops: 0000 [#1] PREEMPT SMP NOPTI
> > > > > <4>[  260.470040] CPU: 0 PID: 5306 Comm: core_hotunplug Not tainted
> > > > > 5.17.0-CI-CI_DRM_11405+ #1
> > > > > <4>[  260.470049] Hardware name: Intel Corporation Jasper Lake
> > > > > Client Platform/Jasperlake DDR4 SODIMM RVP, BIOS
> > > > > JSLSFWI1.R00.2385.D02.2010160831 10/16/2020
> > > > > <4>[  260.470058] RIP: 0010:dma_fence_array_first+0x19/0x20
> > > I've looked into this and the root cause seems to be that the new code
> > > doesn't handle dma_fence_arrays with zero elements.
> > > 
> > > That is rather easy to fix, but a dma_fence_array with zero number of
> > > elements is most likely a bug because under the wrong circumstances it can
> > > create a dma_fence instance which will never signal.
> > > 
> > > I've send out a patch on Frinday ([PATCH] dma-buf: WIP dma_fence_array_first
> > > fix) which avoids the crash and prints a warning if anybody tries to create
> > > a dma_fence_array with zero length.
> > > 
> > > Can you test this?
> > It's in drm-tip now (in the fixup branch) so drm-tip results should have
> > the result soonish:
> > 
> > https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fintel-gfx-ci.01.org%2Ftree%2Fdrm-tip%2Findex.html&amp;data=04%7C01%7Cchristian.koenig%40amd.com%7C0afc74b5df0c4ea384af08da10c672fa%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637840742273792356%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&amp;sdata=SL2CuMgM6lCSOhDTvs%2FaFg6zRlc7F3X%2BNkf6GuGMGXI%3D&amp;reserved=0?
> 
> How do I find something in there? Is there a search function over all the
> test results?

Not really. You can ask on #intel-gfx-ci, or look at the tests that have
blown up without this to know where to look.

You can also download all the logfiles with wget recursive or so from the
build directory (the links in the top row) and search locally.
-Daniel

> 
> Thanks,
> Christian.
> 
> > 
> > Cheers, Daniel
>
diff mbox series

Patch

diff --git a/Documentation/driver-api/dma-buf.rst b/Documentation/driver-api/dma-buf.rst
index 2cd7db82d9fe..7209500f08c8 100644
--- a/Documentation/driver-api/dma-buf.rst
+++ b/Documentation/driver-api/dma-buf.rst
@@ -194,6 +194,12 @@  DMA Fence Chain
 .. kernel-doc:: include/linux/dma-fence-chain.h
    :internal:
 
+DMA Fence unwrap
+~~~~~~~~~~~~~~~~
+
+.. kernel-doc:: include/linux/dma-fence-unwrap.h
+   :internal:
+
 DMA Fence uABI/Sync File
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
index 511805dbeb75..4c9eb53ba3f8 100644
--- a/drivers/dma-buf/Makefile
+++ b/drivers/dma-buf/Makefile
@@ -12,6 +12,7 @@  dmabuf_selftests-y := \
 	selftest.o \
 	st-dma-fence.o \
 	st-dma-fence-chain.o \
+	st-dma-fence-unwrap.o \
 	st-dma-resv.o
 
 obj-$(CONFIG_DMABUF_SELFTESTS)	+= dmabuf_selftests.o
diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
index 97d73aaa31da..851965867d9c 100644
--- a/drivers/dma-buf/selftests.h
+++ b/drivers/dma-buf/selftests.h
@@ -12,4 +12,5 @@ 
 selftest(sanitycheck, __sanitycheck__) /* keep first (igt selfcheck) */
 selftest(dma_fence, dma_fence)
 selftest(dma_fence_chain, dma_fence_chain)
+selftest(dma_fence_unwrap, dma_fence_unwrap)
 selftest(dma_resv, dma_resv)
diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
new file mode 100644
index 000000000000..d821faaebe93
--- /dev/null
+++ b/drivers/dma-buf/st-dma-fence-unwrap.c
@@ -0,0 +1,279 @@ 
+// SPDX-License-Identifier: MIT
+
+/*
+ * Copyright (C) 2022 Advanced Micro Devices, Inc.
+ */
+
+#include <linux/dma-fence-unwrap.h>
+#if 0
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/mm.h>
+#include <linux/sched/signal.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/random.h>
+#endif
+
+#include "selftest.h"
+
+#define CHAIN_SZ (4 << 10)
+
+static struct kmem_cache *slab_fences;
+
+static inline struct mock_fence {
+	struct dma_fence base;
+	spinlock_t lock;
+} *to_mock_fence(struct dma_fence *f) {
+	return container_of(f, struct mock_fence, base);
+}
+
+static const char *mock_name(struct dma_fence *f)
+{
+	return "mock";
+}
+
+static void mock_fence_release(struct dma_fence *f)
+{
+	kmem_cache_free(slab_fences, to_mock_fence(f));
+}
+
+static const struct dma_fence_ops mock_ops = {
+	.get_driver_name = mock_name,
+	.get_timeline_name = mock_name,
+	.release = mock_fence_release,
+};
+
+static struct dma_fence *mock_fence(void)
+{
+	struct mock_fence *f;
+
+	f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
+	if (!f)
+		return NULL;
+
+	spin_lock_init(&f->lock);
+	dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
+
+	return &f->base;
+}
+
+static struct dma_fence *mock_array(unsigned int num_fences, ...)
+{
+	struct dma_fence_array *array;
+	struct dma_fence **fences;
+	va_list valist;
+	int i;
+
+	fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
+	if (!fences)
+		return NULL;
+
+	va_start(valist, num_fences);
+	for (i = 0; i < num_fences; ++i)
+		fences[i] = va_arg(valist, typeof(*fences));
+	va_end(valist);
+
+	array = dma_fence_array_create(num_fences, fences,
+				       dma_fence_context_alloc(1),
+				       1, false);
+	if (!array)
+		goto cleanup;
+	return &array->base;
+
+cleanup:
+	for (i = 0; i < num_fences; ++i)
+		dma_fence_put(fences[i]);
+	kfree(fences);
+	return NULL;
+}
+
+static struct dma_fence *mock_chain(struct dma_fence *prev,
+				    struct dma_fence *fence)
+{
+	struct dma_fence_chain *f;
+
+	f = dma_fence_chain_alloc();
+	if (!f) {
+		dma_fence_put(prev);
+		dma_fence_put(fence);
+		return NULL;
+	}
+
+	dma_fence_chain_init(f, prev, fence, 1);
+	return &f->base;
+}
+
+static int sanitycheck(void *arg)
+{
+	struct dma_fence *f, *chain, *array;
+	int err = 0;
+
+	f = mock_fence();
+	if (!f)
+		return -ENOMEM;
+
+	array = mock_array(1, f);
+	if (!array)
+		return -ENOMEM;
+
+	chain = mock_chain(NULL, array);
+	if (!chain)
+		return -ENOMEM;
+
+	dma_fence_signal(f);
+	dma_fence_put(chain);
+	return err;
+}
+
+static int unwrap_array(void *arg)
+{
+	struct dma_fence *fence, *f1, *f2, *array;
+	struct dma_fence_unwrap iter;
+	int err = 0;
+
+	f1 = mock_fence();
+	if (!f1)
+		return -ENOMEM;
+
+	f2 = mock_fence();
+	if (!f2) {
+		dma_fence_put(f1);
+		return -ENOMEM;
+	}
+
+	array = mock_array(2, f1, f2);
+	if (!array)
+		return -ENOMEM;
+
+	dma_fence_unwrap_for_each(fence, &iter, array) {
+		if (fence == f1) {
+			f1 = NULL;
+		} else if (fence == f2) {
+			f2 = NULL;
+		} else {
+			pr_err("Unexpected fence!\n");
+			err = -EINVAL;
+		}
+	}
+
+	if (f1 || f2) {
+		pr_err("Not all fences seen!\n");
+		err = -EINVAL;
+	}
+
+	dma_fence_signal(f1);
+	dma_fence_signal(f2);
+	dma_fence_put(array);
+	return 0;
+}
+
+static int unwrap_chain(void *arg)
+{
+	struct dma_fence *fence, *f1, *f2, *chain;
+	struct dma_fence_unwrap iter;
+	int err = 0;
+
+	f1 = mock_fence();
+	if (!f1)
+		return -ENOMEM;
+
+	f2 = mock_fence();
+	if (!f2) {
+		dma_fence_put(f1);
+		return -ENOMEM;
+	}
+
+	chain = mock_chain(f1, f2);
+	if (!chain)
+		return -ENOMEM;
+
+	dma_fence_unwrap_for_each(fence, &iter, chain) {
+		if (fence == f1) {
+			f1 = NULL;
+		} else if (fence == f2) {
+			f2 = NULL;
+		} else {
+			pr_err("Unexpected fence!\n");
+			err = -EINVAL;
+		}
+	}
+
+	if (f1 || f2) {
+		pr_err("Not all fences seen!\n");
+		err = -EINVAL;
+	}
+
+	dma_fence_signal(f1);
+	dma_fence_signal(f2);
+	dma_fence_put(chain);
+	return 0;
+}
+
+static int unwrap_chain_array(void *arg)
+{
+	struct dma_fence *fence, *f1, *f2, *array, *chain;
+	struct dma_fence_unwrap iter;
+	int err = 0;
+
+	f1 = mock_fence();
+	if (!f1)
+		return -ENOMEM;
+
+	f2 = mock_fence();
+	if (!f2) {
+		dma_fence_put(f1);
+		return -ENOMEM;
+	}
+
+	array = mock_array(2, f1, f2);
+	if (!array)
+		return -ENOMEM;
+
+	chain = mock_chain(NULL, array);
+	if (!chain)
+		return -ENOMEM;
+
+	dma_fence_unwrap_for_each(fence, &iter, chain) {
+		if (fence == f1) {
+			f1 = NULL;
+		} else if (fence == f2) {
+			f2 = NULL;
+		} else {
+			pr_err("Unexpected fence!\n");
+			err = -EINVAL;
+		}
+	}
+
+	if (f1 || f2) {
+		pr_err("Not all fences seen!\n");
+		err = -EINVAL;
+	}
+
+	dma_fence_signal(f1);
+	dma_fence_signal(f2);
+	dma_fence_put(chain);
+	return 0;
+}
+
+int dma_fence_unwrap(void)
+{
+	static const struct subtest tests[] = {
+		SUBTEST(sanitycheck),
+		SUBTEST(unwrap_array),
+		SUBTEST(unwrap_chain),
+		SUBTEST(unwrap_chain_array),
+	};
+	int ret;
+
+	slab_fences = KMEM_CACHE(mock_fence,
+				 SLAB_TYPESAFE_BY_RCU |
+				 SLAB_HWCACHE_ALIGN);
+	if (!slab_fences)
+		return -ENOMEM;
+
+	ret = subtests(tests, NULL);
+
+	kmem_cache_destroy(slab_fences);
+	return ret;
+}
diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h
new file mode 100644
index 000000000000..54963df00c98
--- /dev/null
+++ b/include/linux/dma-fence-unwrap.h
@@ -0,0 +1,99 @@ 
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * fence-chain: chain fences together in a timeline
+ *
+ * Copyright (C) 2022 Advanced Micro Devices, Inc.
+ * Authors:
+ *	Christian König <christian.koenig@amd.com>
+ */
+
+#ifndef __LINUX_DMA_FENCE_UNWRAP_H
+#define __LINUX_DMA_FENCE_UNWRAP_H
+
+#include <linux/dma-fence-chain.h>
+#include <linux/dma-fence-array.h>
+
+/**
+ * struct dma_fence_unwrap - cursor into the container structure
+ */
+struct dma_fence_unwrap {
+	/**
+	 * @chain: potential dma_fence_chain, but can be other fence as well
+	 */
+	struct dma_fence *chain;
+	/**
+	 * @array: potential dma_fence_array, but can be other fence as well
+	 */
+	struct dma_fence *array;
+	/**
+	 * @index: last returned index if @array is really a dma_fence_array
+	 */
+	unsigned int index;
+};
+
+/**
+ * dma_fence_unwrap_array - helper to unwrap dma_fence_arrays
+ * @cursor: cursor to initialize
+ *
+ * Helper function to unwrap dma_fence_array containers, don't touch directly.
+ * Use dma_fence_unwrap_first/next instead.
+ */
+static inline struct dma_fence *
+dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)
+{
+	cursor->array = dma_fence_chain_contained(cursor->chain);
+	cursor->index = 0;
+	return dma_fence_array_first(cursor->array);
+}
+
+/**
+ * dma_fence_unwrap_first - return the first fence from fence containers
+ * @head: the entrypoint into the containers
+ * @cursor: current position inside the containers
+ *
+ * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
+ * first fence.
+ */
+static inline struct dma_fence *
+dma_fence_unwrap_first(struct dma_fence *head, struct dma_fence_unwrap *cursor)
+{
+	cursor->chain = dma_fence_get(head);
+	return dma_fence_unwrap_array(cursor);
+}
+
+/**
+ * dma_fence_unwrap_next - return the next fence from a fence containers
+ * @cursor: current position inside the containers
+ *
+ * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
+ * the next fence from them.
+ */
+static inline struct dma_fence *
+dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
+{
+	struct dma_fence *tmp;
+
+	++cursor->index;
+	tmp = dma_fence_array_next(cursor->array, cursor->index);
+	if (tmp)
+		return tmp;
+
+	cursor->chain = dma_fence_chain_walk(cursor->chain);
+	return dma_fence_unwrap_array(cursor);
+}
+
+/**
+ * dma_fence_unwrap_for_each - iterate over all fences in containers
+ * @fence: current fence
+ * @cursor: current position inside the containers
+ * @head: starting point for the iterator
+ *
+ * Unwrap dma_fence_chain and dma_fence_array containers and deep dive into all
+ * potential fences in them. If @head is just a normal fence only that one is
+ * returned.
+ */
+#define dma_fence_unwrap_for_each(fence, cursor, head)			\
+	for (fence = dma_fence_unwrap_first(head, cursor); fence;	\
+	     fence = dma_fence_unwrap_next(cursor))
+
+#endif