nvidia-open-gpu-kernel-modules/kernel-open/nvidia-uvm/uvm_push_test.c

/*******************************************************************************
    Copyright (c) 2015-2022 NVIDIA Corporation

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to
    deal in the Software without restriction, including without limitation the
    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
    sell copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

        The above copyright notice and this permission notice shall be
        included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.

*******************************************************************************/

#include <asm/atomic.h>

#include "uvm_global.h"
#include "uvm_channel.h"
#include "uvm_hal.h"
#include "uvm_mem.h"
#include "uvm_push.h"
#include "uvm_test.h"
#include "uvm_test_rng.h"
#include "uvm_thread_context.h"
#include "uvm_va_space.h"
#include "uvm_tracker.h"
#include "uvm_gpu_semaphore.h"
#include "uvm_kvmalloc.h"

#define TEST_PUSH_INTERLEAVING_NUM_PAUSED_PUSHES 2

static NvU32 get_push_end_size(uvm_channel_t *channel)
{
    if (uvm_channel_is_ce(channel))
        return UVM_PUSH_CE_END_SIZE;

    return 0;
}

static NV_STATUS test_push_end_size(uvm_va_space_t *va_space)
{
    NV_STATUS status = NV_OK;
    uvm_gpu_t *gpu;
    NvU32 push_size;
    NvU32 i;

    for_each_va_space_gpu(gpu, va_space) {
        for (i = 0; i < UVM_CHANNEL_TYPE_COUNT; ++i) {
            uvm_push_t push;
            NvU32 push_end_size;
            uvm_channel_type_t type = i;

            status = uvm_push_begin(gpu->channel_manager, type, &push, "type %u\n", (unsigned)type);
            TEST_CHECK_GOTO(status == NV_OK, done);

            push_end_size = get_push_end_size(push.channel);
            push_size = uvm_push_get_size(&push);
            uvm_push_end(&push);
            if (uvm_push_get_size(&push) - push_size != push_end_size) {
                UVM_TEST_PRINT("push_end_size incorrect, %u instead of %u for GPU %s\n",
                               uvm_push_get_size(&push) - push_size,
                               push_end_size,
                               uvm_gpu_name(gpu));
                status = NV_ERR_INVALID_STATE;
                goto done;
            }
        }
    }

done:
    for_each_va_space_gpu(gpu, va_space) {
        uvm_channel_manager_wait(gpu->channel_manager);
    }

    return status;
}

typedef enum {
    TEST_INLINE_ADD,
    TEST_INLINE_GET,
    TEST_INLINE_SINGLE_BUFFER,
    TEST_INLINE_MAX,
} test_inline_type_t;

static NV_STATUS test_push_inline_data_gpu(uvm_gpu_t *gpu)
{
    static const size_t test_sizes[] = { 1, 2, 3, 4, 8, 31, 32, 1023, 1024, 1025, UVM_PUSH_INLINE_DATA_MAX_SIZE };
    NV_STATUS status;
    int i, j;
    int test_inline_type;
    uvm_push_t push;
    uvm_mem_t *mem = NULL;
    char *verif;

    status = uvm_mem_alloc_sysmem_and_map_cpu_kernel(UVM_PUSH_INLINE_DATA_MAX_SIZE, current->mm, &mem);
    TEST_CHECK_GOTO(status == NV_OK, done);

    status = uvm_mem_map_gpu_kernel(mem, gpu);
    TEST_CHECK_GOTO(status == NV_OK, done);

    verif = (char *)uvm_mem_get_cpu_addr_kernel(mem);

    for (test_inline_type = 0; test_inline_type < TEST_INLINE_MAX; ++test_inline_type) {
        for (i = 0; i < ARRAY_SIZE(test_sizes); ++i) {
            size_t test_size = test_sizes[i];
            uvm_push_inline_data_t data;
            size_t inline_data_size = 0;
            uvm_gpu_address_t data_gpu_address;
            char *inline_buf;

            status = uvm_push_begin(gpu->channel_manager,
                                    UVM_CHANNEL_TYPE_GPU_INTERNAL,
                                    &push,
                                    "Inline data size %zu",
                                    test_size);
            TEST_CHECK_GOTO(status == NV_OK, done);

            // Do a noop first to test inline data starting at different offsets
            gpu->parent->host_hal->noop(&push, roundup(min(test_size, (size_t)4096), UVM_METHOD_SIZE));

            switch (test_inline_type) {
                case TEST_INLINE_ADD:
                    uvm_push_inline_data_begin(&push, &data);
                    for (j = 0; j < test_size; ++j) {
                        char value = 1 + i + j;
                        uvm_push_inline_data_add(&data, &value, 1);
                    }
                    inline_data_size = uvm_push_inline_data_size(&data);
                    data_gpu_address = uvm_push_inline_data_end(&data);
                    break;
                case TEST_INLINE_GET:
                    uvm_push_inline_data_begin(&push, &data);
                    inline_buf = (char*)uvm_push_inline_data_get(&data, test_size);
                    inline_data_size = uvm_push_inline_data_size(&data);
                    data_gpu_address = uvm_push_inline_data_end(&data);
                    for (j = 0; j < test_size; ++j)
                        inline_buf[j] = 1 + i + j;
                    break;
                case TEST_INLINE_SINGLE_BUFFER:
                    inline_buf = (char*)uvm_push_get_single_inline_buffer(&push, test_size, &data_gpu_address);
                    inline_data_size = test_size;
                    for (j = 0; j < test_size; ++j)
                        inline_buf[j] = 1 + i + j;
                    break;
            }


            gpu->parent->ce_hal->memcopy(&push,
                                        uvm_mem_gpu_address_virtual_kernel(mem, gpu),
                                        data_gpu_address,
                                        test_size);
            status = uvm_push_end_and_wait(&push);
            TEST_CHECK_GOTO(status == NV_OK, done);

            TEST_CHECK_GOTO(inline_data_size == test_size, done);

            for (j = 0; j < test_size; ++j) {
                char expected = 1 + i + j;
                if (verif[j] != expected) {
                    UVM_TEST_PRINT("size %zu verif[%d] = %d instead of %d\n", test_size, j, verif[j], expected);
                    status = NV_ERR_INVALID_STATE;
                    goto done;
                }
            }
        }
    }
done:
    uvm_mem_free(mem);

    return status;
}

static NV_STATUS test_push_inline_data(uvm_va_space_t *va_space)
{
    uvm_gpu_t *gpu;

    for_each_va_space_gpu(gpu, va_space) {
        TEST_CHECK_RET(test_push_inline_data_gpu(gpu) == NV_OK);
    }

    return NV_OK;
}

// Test that begins UVM_PUSH_MAX_CONCURRENT_PUSHES number of pushes before
// ending any of them on each GPU.
// Notably starting more than a single push is not safe to do outside of a test
// as if multiple threads tried doing so, it could easily deadlock.
static NV_STATUS test_concurrent_pushes(uvm_va_space_t *va_space)
{
    NV_STATUS status = NV_OK;
    uvm_gpu_t *gpu;
    NvU32 i;
    uvm_push_t *pushes;
    uvm_tracker_t tracker = UVM_TRACKER_INIT();

    // As noted above, this test does unsafe things that would be detected by
    // lock tracking, opt-out.
    uvm_thread_context_lock_disable_tracking();

    pushes = uvm_kvmalloc_zero(sizeof(*pushes) * UVM_PUSH_MAX_CONCURRENT_PUSHES);
    if (pushes == NULL) {
        status = NV_ERR_NO_MEMORY;
        goto done;
    }

    for_each_va_space_gpu(gpu, va_space) {
        for (i = 0; i < UVM_PUSH_MAX_CONCURRENT_PUSHES; ++i) {
            uvm_push_t *push = &pushes[i];
            status = uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_CPU_TO_GPU, push, "concurrent push %u", i);
            TEST_CHECK_GOTO(status == NV_OK, done);
        }
        for (i = 0; i < UVM_PUSH_MAX_CONCURRENT_PUSHES; ++i) {
            uvm_push_t *push = &pushes[i];
            uvm_push_end(push);
            TEST_NV_CHECK_GOTO(uvm_tracker_add_push(&tracker, push), done);
        }
        TEST_CHECK_GOTO(tracker.size != 0, done);

        status = uvm_tracker_wait(&tracker);
        TEST_CHECK_GOTO(status == NV_OK, done);
    }

done:
    uvm_thread_context_lock_enable_tracking();

    uvm_tracker_deinit(&tracker);

    uvm_kvfree(pushes);

    return status;
}

static void add_to_counter(void* ptr, int value)
{
    atomic_t *atomic = (atomic_t*) ptr;
    atomic_add(value, atomic);
}

static void add_one_to_counter(void* ptr)
{
    add_to_counter(ptr, 1);
}

static void add_two_to_counter(void* ptr)
{
    add_to_counter(ptr, 2);
}

static NV_STATUS test_push_interleaving_on_gpu(uvm_gpu_t* gpu)
{
    NV_STATUS status;
    uvm_channel_t *channel;
    uvm_push_t push;
    NvU32 i;
    NvU32 *host_va;
    NvU64 gpu_va;
    NvU32 observed, expected;
    unsigned int num_non_paused_pushes;
    uvm_push_t pushes_not_ended[TEST_PUSH_INTERLEAVING_NUM_PAUSED_PUSHES];
    const NvLength size = sizeof(NvU32) * (1 + TEST_PUSH_INTERLEAVING_NUM_PAUSED_PUSHES);
    uvm_rm_mem_t *mem = NULL;
    atomic_t on_complete_counter = ATOMIC_INIT(0);

    // This test issues virtual memcopies/memsets, which in SR-IOV heavy cannot
    // be pushed to a proxy channel. Pushing to a UVM internal CE channel works
    // in all scenarios.
    channel = uvm_channel_any_of_type(gpu->channel_manager, UVM_CHANNEL_POOL_TYPE_CE);
    TEST_CHECK_RET(channel != NULL);

    if (channel->num_gpfifo_entries <= TEST_PUSH_INTERLEAVING_NUM_PAUSED_PUSHES) {
        UVM_TEST_PRINT("Insufficient number of gpfifo entries per channel to run this test. Expected at least %u "
                       "entries, but found %u\n",
                       TEST_PUSH_INTERLEAVING_NUM_PAUSED_PUSHES + 1,
                       channel->num_gpfifo_entries);
        return NV_ERR_INVALID_STATE;
    }
    num_non_paused_pushes = channel->num_gpfifo_entries;

    // The UVM driver only allows push interleaving across separate threads, but
    // it is hard to consistenly replicate the interleaving. Instead, we
    // temporarily disable lock tracking, so we can interleave pushes from a
    // single thread.
    uvm_thread_context_lock_disable_tracking();

    status = uvm_rm_mem_alloc_and_map_cpu(gpu, UVM_RM_MEM_TYPE_SYS, size, 0, &mem);
    TEST_CHECK_GOTO(status == NV_OK, done);
    host_va = (NvU32*)uvm_rm_mem_get_cpu_va(mem);
    gpu_va = uvm_rm_mem_get_gpu_va(mem, gpu, uvm_channel_is_proxy(channel));
    memset(host_va, 0, size);

    // Begin a few pushes on the channel, but do not end them yet.
    // Each pushed method sets a magic number on an independent memory location.
    for (i = 0; i < TEST_PUSH_INTERLEAVING_NUM_PAUSED_PUSHES; ++i) {
        uvm_push_info_t *push_info;

        status = uvm_push_begin_on_channel(channel, pushes_not_ended + i, "Set to 0x%x", 0xDEADBEEF + i);
        TEST_CHECK_GOTO(status == NV_OK, done);
        gpu->parent->ce_hal->memset_v_4(pushes_not_ended + i,
                                        gpu_va + sizeof(NvU32) * (i + 1),
                                        0xDEADBEEF + i,
                                        sizeof(NvU32));

        push_info = uvm_push_info_from_push(pushes_not_ended + i);
        push_info->on_complete = add_two_to_counter;
        push_info->on_complete_data = &on_complete_counter;
    }

    // Push N (N = #channel entries) value increments to the same channel.
    for (i = 0; i < num_non_paused_pushes; ++i) {
        uvm_push_info_t *push_info;

        status = uvm_push_begin_on_channel(channel, &push, "inc to %u", i + 1);
        TEST_CHECK_GOTO(status == NV_OK, done);
        gpu->parent->ce_hal->semaphore_reduction_inc(&push, gpu_va, num_non_paused_pushes);

        push_info = uvm_push_info_from_push(&push);
        push_info->on_complete = add_one_to_counter;
        push_info->on_complete_data = &on_complete_counter;

        uvm_push_end(&push);
    }

    // End the pending pushes
    for (i = 0; i < TEST_PUSH_INTERLEAVING_NUM_PAUSED_PUSHES; ++i)
        uvm_push_end(pushes_not_ended + i);

    // When the channel manager becomes idle, the GPU methods have been
    // completed, and the CPU completion callbacks associated with the push
    // have been invoked.
    status = uvm_channel_manager_wait(channel->pool->manager);
    TEST_CHECK_GOTO(status == NV_OK, done);

    observed = host_va[0];
    expected = num_non_paused_pushes;
    if (observed != expected) {
        UVM_TEST_PRINT("Observed counter %u but expected %u\n", observed, expected);
        status = NV_ERR_INVALID_STATE;
        goto done;
    }

    for (i = 0; i < TEST_PUSH_INTERLEAVING_NUM_PAUSED_PUSHES; ++i) {
        observed = host_va[i + 1];
        expected = 0xDEADBEEF + i;
        if (observed != expected) {
            UVM_TEST_PRINT("Observed magic number 0x%x but expected 0x%x\n", observed, expected);
            status = NV_ERR_INVALID_STATE;
            goto done;
        }
    }

    observed = atomic_read(&on_complete_counter);
    expected = TEST_PUSH_INTERLEAVING_NUM_PAUSED_PUSHES * 2 + num_non_paused_pushes;
    if (observed != expected) {
        UVM_TEST_PRINT("Wrong value of counter incremented by push info callback. Observed %u but expected %u\n",
                       observed,
                       expected);
        status = NV_ERR_INVALID_STATE;
        goto done;
    }

done:
    uvm_rm_mem_free(mem);
    uvm_thread_context_lock_enable_tracking();

    return status;
}

// Using a single thread, interleave pushes and check that the result is
// consistent with a non-interleaved sequence.
// 1) Begin a few pushes in channel X but do not end them. Each pushed (GPU)
//    method sets a individual value in an independent system memory location.
//    Each push is associated with a push info (CPU) callback that atomically
//    adds 2 to a memory location M
// 2) Begin and end many pushes in the same channel X such that all the gpfifo
//    entries are filled. All the pushed methods do the same thing: atomically
//    increment a given system memory location.
//    Each push is associated with a push info callback that atomically
//    increments the memory location M
// 3) End the pending pushes
//
// The final state should be the same as in the non-interleaved sequence
// (1)-(3)-(2)
//
// Starting more than a single push is not safe to do outside of a test as if
// multiple threads tried doing so, it could easily deadlock.
static NV_STATUS test_push_interleaving(uvm_va_space_t *va_space)
{
    NV_STATUS status;
    uvm_gpu_t *gpu;

    BUILD_BUG_ON(TEST_PUSH_INTERLEAVING_NUM_PAUSED_PUSHES >= UVM_PUSH_MAX_CONCURRENT_PUSHES);

    for_each_va_space_gpu(gpu, va_space) {
        status = test_push_interleaving_on_gpu(gpu);
        if (status != NV_OK)
            return status;
    }

    return NV_OK;
}

// Push exactly UVM_MAX_PUSH_SIZE methods while acquiring a semaphore
// This is very tightly coupled with the pushbuffer implementation and method
// sizes, which is not ideal, but allows to test corner cases in the pushbuffer
// management code.
static NV_STATUS test_push_exactly_max_push(uvm_gpu_t *gpu,
                                            uvm_push_t *push,
                                            uvm_channel_type_t channel_type,
                                            uvm_gpu_semaphore_t *sema_to_acquire,
                                            NvU32 value)
{
    NV_STATUS status;
    NvU64 semaphore_gpu_va;
    NvU32 push_end_size;

    status = uvm_push_begin(gpu->channel_manager, channel_type, push, "Test push");
    if (status != NV_OK)
        return status;

    TEST_CHECK_RET(uvm_push_has_space(push, UVM_MAX_PUSH_SIZE));
    TEST_CHECK_RET(!uvm_push_has_space(push, UVM_MAX_PUSH_SIZE + 1));

    semaphore_gpu_va = uvm_gpu_semaphore_get_gpu_va(sema_to_acquire, gpu, uvm_channel_is_proxy(push->channel));
    gpu->parent->host_hal->semaphore_acquire(push, semaphore_gpu_va, value);

    // Push a noop leaving just push_end_size in the pushbuffer.
    push_end_size = get_push_end_size(push->channel);
    gpu->parent->host_hal->noop(push, UVM_MAX_PUSH_SIZE - uvm_push_get_size(push) - push_end_size);

    TEST_CHECK_RET(uvm_push_has_space(push, push_end_size));
    TEST_CHECK_RET(!uvm_push_has_space(push, push_end_size + 1));
    uvm_push_end(push);

    UVM_ASSERT_MSG(uvm_push_get_size(push) == UVM_MAX_PUSH_SIZE, "push_size %u\n", uvm_push_get_size(push));

    return NV_OK;
}

static NvU32 test_count_idle_chunks(uvm_pushbuffer_t *pushbuffer)
{
    NvU32 i;
    NvU32 count = 0;
    for (i = 0; i < UVM_PUSHBUFFER_CHUNKS; ++i)
        count += test_bit(i, pushbuffer->idle_chunks) ? 1 : 0;
    return count;
}

static NvU32 test_count_available_chunks(uvm_pushbuffer_t *pushbuffer)
{
    NvU32 i;
    NvU32 count = 0;
    for (i = 0; i < UVM_PUSHBUFFER_CHUNKS; ++i)
        count += test_bit(i, pushbuffer->available_chunks) ? 1 : 0;
    return count;
}

// Reuse the whole pushbuffer 4 times, one UVM_MAX_PUSH_SIZE at a time
#define EXTRA_MAX_PUSHES_WHILE_FULL (4 * UVM_PUSHBUFFER_SIZE / UVM_MAX_PUSH_SIZE)

// Test doing pushes of exactly UVM_MAX_PUSH_SIZE size and only allowing them to
// complete one by one.
static NV_STATUS test_max_pushes_on_gpu_and_channel_type(uvm_gpu_t *gpu, uvm_channel_type_t channel_type)
{
    NV_STATUS status;

    uvm_tracker_t tracker;
    uvm_gpu_semaphore_t sema;
    NvU32 total_push_size = 0;
    NvU32 push_count = 0;
    NvU32 i;

    uvm_tracker_init(&tracker);

    status = uvm_gpu_semaphore_alloc(gpu->semaphore_pool, &sema);
    TEST_CHECK_GOTO(status == NV_OK, done);

    uvm_gpu_semaphore_set_payload(&sema, 0);

    // Need to wait for all channels to completely idle so that the pushbuffer
    // is in completely idle state when we begin.
    status = uvm_channel_manager_wait(gpu->channel_manager);
    TEST_CHECK_GOTO(status == NV_OK, done);

    while (uvm_pushbuffer_has_space(gpu->channel_manager->pushbuffer)) {
        uvm_push_t push;

        ++push_count;

        status = test_push_exactly_max_push(gpu, &push, channel_type, &sema, push_count);
        TEST_CHECK_GOTO(status == NV_OK, done);

        total_push_size += uvm_push_get_size(&push);
        TEST_NV_CHECK_GOTO(uvm_tracker_add_push(&tracker, &push), done);
    }

    if (total_push_size != UVM_PUSHBUFFER_SIZE) {
        UVM_TEST_PRINT("Unexpected space in the pushbuffer, total push %u\n", total_push_size);
        uvm_pushbuffer_print(gpu->channel_manager->pushbuffer);
        status = NV_ERR_INVALID_STATE;
        goto done;
    }

    TEST_CHECK_GOTO(test_count_available_chunks(gpu->channel_manager->pushbuffer) == 0, done);
    TEST_CHECK_GOTO(test_count_idle_chunks(gpu->channel_manager->pushbuffer) == 0, done);

    for (i = 0; i < EXTRA_MAX_PUSHES_WHILE_FULL; ++i) {
        uvm_push_t push;

        // There should be no space for another push until the sema is
        // incremented. Incrementing the same allows a single push to complete
        // freeing exactly UVM_MAX_PUSH_SIZE space.
        if (uvm_pushbuffer_has_space(gpu->channel_manager->pushbuffer)) {
            UVM_TEST_PRINT("Unexpected space in the pushbuffer for iter %d\n", i);
            uvm_pushbuffer_print(gpu->channel_manager->pushbuffer);
            status = NV_ERR_INVALID_STATE;
            goto done;
        }

        uvm_gpu_semaphore_set_payload(&sema, i + 1);

        ++push_count;

        // Take UVM_MAX_PUSH_SIZE space. This should leave no space left again.
        status = test_push_exactly_max_push(gpu, &push, channel_type, &sema, push_count);
        TEST_CHECK_GOTO(status == NV_OK, done);

        TEST_NV_CHECK_GOTO(uvm_tracker_add_push(&tracker, &push), done);
    }

done:
    uvm_gpu_semaphore_set_payload(&sema, push_count);
    uvm_tracker_wait_deinit(&tracker);

    uvm_gpu_semaphore_free(&sema);

    return status;
}

static NV_STATUS test_max_pushes_on_gpu(uvm_gpu_t *gpu)
{

    TEST_NV_CHECK_RET(test_max_pushes_on_gpu_and_channel_type(gpu, UVM_CHANNEL_TYPE_GPU_INTERNAL));

    return NV_OK;
}

// Test doing UVM_PUSHBUFFER_CHUNKS independent pushes expecting each one to use
// a different chunk in the pushbuffer.
static NV_STATUS test_idle_chunks_on_gpu(uvm_gpu_t *gpu)
{
    NV_STATUS status;

    uvm_gpu_semaphore_t sema;
    uvm_tracker_t tracker = UVM_TRACKER_INIT();
    NvU32 i;

    uvm_tracker_init(&tracker);

    status = uvm_gpu_semaphore_alloc(gpu->semaphore_pool, &sema);
    TEST_CHECK_GOTO(status == NV_OK, done);

    uvm_gpu_semaphore_set_payload(&sema, 0);

    // Need to wait for all channels to completely idle so that the pushbuffer
    // is in completely idle state when we begin.
    status = uvm_channel_manager_wait(gpu->channel_manager);
    TEST_CHECK_GOTO(status == NV_OK, done);

    for (i = 0; i < UVM_PUSHBUFFER_CHUNKS; ++i) {
        NvU64 semaphore_gpu_va;
        uvm_push_t push;

        status = uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_INTERNAL, &push, "Push using chunk %u", i);
        TEST_CHECK_GOTO(status == NV_OK, done);

        semaphore_gpu_va = uvm_gpu_semaphore_get_gpu_va(&sema, gpu, uvm_channel_is_proxy(push.channel));
        gpu->parent->host_hal->semaphore_acquire(&push, semaphore_gpu_va, i + 1);
        uvm_push_end(&push);

        TEST_NV_CHECK_GOTO(uvm_tracker_add_push(&tracker, &push), done);

        if (test_count_idle_chunks(gpu->channel_manager->pushbuffer) != UVM_PUSHBUFFER_CHUNKS - i - 1) {
            UVM_TEST_PRINT("Unexpected count of idle chunks in the pushbuffer %u instead of %u\n",
                           test_count_idle_chunks(gpu->channel_manager->pushbuffer), UVM_PUSHBUFFER_CHUNKS - i - 1);
            uvm_pushbuffer_print(gpu->channel_manager->pushbuffer);
            status = NV_ERR_INVALID_STATE;
            goto done;
        }
    }
    uvm_gpu_semaphore_set_payload(&sema, UVM_PUSHBUFFER_CHUNKS + 1);

    status = uvm_channel_manager_wait(gpu->channel_manager);
    TEST_CHECK_GOTO(status == NV_OK, done);

    if (test_count_idle_chunks(gpu->channel_manager->pushbuffer) != UVM_PUSHBUFFER_CHUNKS) {
        UVM_TEST_PRINT("Unexpected count of idle chunks in the pushbuffer %u\n",
                       test_count_idle_chunks(gpu->channel_manager->pushbuffer));
        uvm_pushbuffer_print(gpu->channel_manager->pushbuffer);
        status = NV_ERR_INVALID_STATE;
        goto done;
    }

done:
    uvm_gpu_semaphore_set_payload(&sema, UVM_PUSHBUFFER_CHUNKS + 1);
    uvm_tracker_wait(&tracker);

    uvm_gpu_semaphore_free(&sema);
    uvm_tracker_deinit(&tracker);

    return status;
}

static NV_STATUS test_pushbuffer(uvm_va_space_t *va_space)
{
    uvm_gpu_t *gpu;

    for_each_va_space_gpu(gpu, va_space) {
        TEST_NV_CHECK_RET(test_max_pushes_on_gpu(gpu));
        TEST_NV_CHECK_RET(test_idle_chunks_on_gpu(gpu));
    }
    return NV_OK;
}

typedef struct
{
    NvU64 *timestmap_in_pushbuffer;
    NvU64 timestamp;
} timestamp_test_t;

static void timestamp_on_complete(void *void_data)
{
    timestamp_test_t *data = (timestamp_test_t *)void_data;

    if (uvm_global_get_status() != NV_OK) {
        // Do nothing if a global error has been set as the callback might be
        // called from teardown where the reference to test data is no longer
        // valid.
        return;
    }

    data->timestamp = *data->timestmap_in_pushbuffer;
}

static NV_STATUS test_timestamp_on_gpu(uvm_gpu_t *gpu)
{
    NV_STATUS status;
    uvm_push_t push;
    timestamp_test_t test_data = {0};
    NvU32 i;
    NvU64 last_stamp = 0;

    for (i = 0; i < 10; ++i) {
        status = uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_INTERNAL, &push, "Releasing a timestamp");
        if (status != NV_OK)
            return status;

        test_data.timestmap_in_pushbuffer = uvm_push_timestamp(&push);
        uvm_push_info_from_push(&push)->on_complete = timestamp_on_complete;
        uvm_push_info_from_push(&push)->on_complete_data = &test_data;
        uvm_push_end(&push);

        // Synchronize the channel manager to make sure the on_complete
        // callbacks have a chance to run.
        status = uvm_channel_manager_wait(gpu->channel_manager);
        TEST_CHECK_RET(status == NV_OK);

        TEST_CHECK_RET(test_data.timestamp != 0);
        TEST_CHECK_RET(test_data.timestamp > last_stamp);
        last_stamp = test_data.timestamp;
    }

    return NV_OK;
}

static NV_STATUS test_timestamp(uvm_va_space_t *va_space)
{
    uvm_gpu_t *gpu;

    for_each_va_space_gpu(gpu, va_space)
        TEST_CHECK_RET(test_timestamp_on_gpu(gpu) == NV_OK);

    return NV_OK;
}

static NV_STATUS sync_memcopy(uvm_channel_type_t type, uvm_mem_t *dst, uvm_mem_t *src)
{
    uvm_push_t push;
    uvm_gpu_address_t dst_va;
    uvm_gpu_address_t src_va;
    uvm_gpu_t *gpu;
    NV_STATUS status;

    UVM_ASSERT(uvm_mem_is_vidmem(src) || uvm_mem_is_vidmem(dst));

    if (type == UVM_CHANNEL_TYPE_CPU_TO_GPU || type == UVM_CHANNEL_TYPE_GPU_TO_CPU) {
        gpu = (type == UVM_CHANNEL_TYPE_CPU_TO_GPU) ? dst->backing_gpu : src->backing_gpu;
        status = uvm_push_begin(gpu->channel_manager, type, &push, uvm_channel_type_to_string(type));
        if (status != NV_OK)
            return status;

        dst_va = uvm_mem_gpu_address_virtual_kernel(dst, gpu);
        src_va = uvm_mem_gpu_address_virtual_kernel(src, gpu);
        gpu->parent->ce_hal->memcopy(&push, dst_va, src_va, src->size);
    }
    else {
        unsigned i;
        const NvU32 chunk_size = src->chunk_size;

        UVM_ASSERT((src->size % chunk_size) == 0);

        gpu = src->backing_gpu;
        status = uvm_push_begin_gpu_to_gpu(gpu->channel_manager,
                                           dst->backing_gpu,
                                           &push,
                                           uvm_channel_type_to_string(type));

        for (i = 0; i < src->size / chunk_size; i++) {
            dst_va = uvm_mem_gpu_address_copy(dst, gpu, i * chunk_size, chunk_size);
            src_va = uvm_mem_gpu_address_copy(src, gpu, i * chunk_size, chunk_size);
            gpu->parent->ce_hal->memcopy(&push, dst_va, src_va, chunk_size);
        }
    }

    return uvm_push_end_and_wait(&push);
}

static bool can_do_peer_copies(uvm_va_space_t *va_space, uvm_gpu_t *gpu_a, uvm_gpu_t *gpu_b)
{
    if (gpu_a == gpu_b || !uvm_processor_mask_test(&va_space->can_copy_from[uvm_id_value(gpu_a->id)], gpu_b->id))
        return false;

    UVM_ASSERT(uvm_processor_mask_test(&va_space->can_copy_from[uvm_id_value(gpu_b->id)], gpu_a->id));

    // TODO: Bug 2028875. Indirect peers are not supported for now.
    if (uvm_gpus_are_indirect_peers(gpu_a, gpu_b))
        return false;

    return true;
}

// Test the GPU to GPU push interface by transferring data between each
// permutation of GPU peers.
static NV_STATUS test_push_gpu_to_gpu(uvm_va_space_t *va_space)
{
    NvU32 i;
    NV_STATUS status;
    uvm_gpu_t *gpu, *gpu_a, *gpu_b;
    uvm_mem_t *mem[UVM_ID_MAX_PROCESSORS] = {NULL};
    NvU32 *host_ptr;
    const size_t size = 1024 * 1024;
    bool waive = true;

    for_each_va_space_gpu(gpu_a, va_space) {
        for_each_va_space_gpu(gpu_b, va_space) {
            if (can_do_peer_copies(va_space, gpu_a, gpu_b)) {
                waive = false;
                break;
            }
        }
    }

    if (waive)
        return NV_OK;

    // Alloc and initialize host buffer
    status = uvm_mem_alloc_sysmem_and_map_cpu_kernel(size, current->mm, &mem[UVM_ID_CPU_VALUE]);
    TEST_CHECK_GOTO(status == NV_OK, done);

    host_ptr = (NvU32 *)uvm_mem_get_cpu_addr_kernel(mem[UVM_ID_CPU_VALUE]);

    for (i = 0; i < size / sizeof(NvU32); ++i)
        host_ptr[i] = i + 1;

    // Allocate vidmem on each GPU, and map the host buffer
    for_each_va_space_gpu(gpu, va_space) {
        status = uvm_mem_alloc_vidmem(size, gpu, &mem[uvm_id_value(gpu->id)]);
        TEST_CHECK_GOTO(status == NV_OK, done);

        status = uvm_mem_map_gpu_kernel(mem[uvm_id_value(gpu->id)], gpu);
        TEST_CHECK_GOTO(status == NV_OK, done);

        status = uvm_mem_map_gpu_kernel(mem[UVM_ID_CPU_VALUE], gpu);
        TEST_CHECK_GOTO(status == NV_OK, done);
    }

    // Copy buffer between each pair of GPU peers, in both directions
    for_each_va_space_gpu(gpu_a, va_space) {
        for_each_va_space_gpu(gpu_b, va_space) {
            if (!can_do_peer_copies(va_space, gpu_a, gpu_b))
                continue;

            // Copy from CPU to the first GPU, and then zero out the host copy
            status = sync_memcopy(UVM_CHANNEL_TYPE_CPU_TO_GPU,
                                  mem[uvm_id_value(gpu_a->id)],
                                  mem[UVM_ID_CPU_VALUE]);
            TEST_CHECK_GOTO(status == NV_OK, done);

            memset(host_ptr, 0, size / sizeof(NvU32));

            // Copy from the first GPU to the second GPU
            status = sync_memcopy(UVM_CHANNEL_TYPE_GPU_TO_GPU,
                                  mem[uvm_id_value(gpu_b->id)],
                                  mem[uvm_id_value(gpu_a->id)]);
            TEST_CHECK_GOTO(status == NV_OK, done);

            // Copy from the second GPU back to the host, and check result
            status = sync_memcopy(UVM_CHANNEL_TYPE_GPU_TO_CPU,
                                  mem[UVM_ID_CPU_VALUE],
                                  mem[uvm_id_value(gpu_b->id)]);
            TEST_CHECK_GOTO(status == NV_OK, done);

            for (i = 0; i < size / sizeof(NvU32); ++i) {
                if (host_ptr[i] != i + 1) {
                    UVM_TEST_PRINT("host_ptr[%u] = %u instead of %u when copying between %s and %s\n",
                                   i,
                                   host_ptr[i],
                                   i + 1,
                                   uvm_gpu_name(gpu_a),
                                   uvm_gpu_name(gpu_b));
                    status = NV_ERR_INVALID_STATE;
                    TEST_CHECK_GOTO(status == NV_OK, done);
                }
            }
        }
    }

 done:
    for_each_va_space_gpu(gpu, va_space)
        uvm_mem_free(mem[uvm_id_value(gpu->id)]);

    uvm_mem_free(mem[UVM_ID_CPU_VALUE]);

    return status;
}

NV_STATUS uvm_test_push_sanity(UVM_TEST_PUSH_SANITY_PARAMS *params, struct file *filp)
{
    NV_STATUS status;
    uvm_va_space_t *va_space = uvm_va_space_get(filp);

    // Take the global lock as some of the tests rely on being the
    // only thread doing pushes and could deadlock otherwise.
    uvm_mutex_lock(&g_uvm_global.global_lock);
    uvm_va_space_down_read_rm(va_space);

    status = test_push_end_size(va_space);
    if (status != NV_OK)
        goto done;

    status = test_push_inline_data(va_space);
    if (status != NV_OK)
        goto done;

    status = test_concurrent_pushes(va_space);
    if (status != NV_OK)
        goto done;

    status = test_push_interleaving(va_space);
    if (status != NV_OK)
        goto done;

    status = test_push_gpu_to_gpu(va_space);
    if (status != NV_OK)
        goto done;

    status = test_pushbuffer(va_space);
    if (status != NV_OK)
        goto done;

    if (!params->skipTimestampTest) {
        status = test_timestamp(va_space);
        if (status != NV_OK)
            goto done;
    }

done:
    uvm_va_space_up_read_rm(va_space);
    uvm_mutex_unlock(&g_uvm_global.global_lock);

    return status;
}