CUDA Cooperative Groups

Introduction

CUDA cooperative groups is a feature that allows developers to create and manage groups of threads that can synchronize and communicate with each other. Cooperative groups provide a more flexible and efficient way to write parallel algorithms on the GPU compared to traditional CUDA programming models.

In this blog post, we will discuss the parallel reduction algorithm and its implementation in CUDA using cooperative groups.

Batched Reduce Sum and Full Reduce Sum Using Cooperative Groups

In this example, we modified the two batched reduce sum kernels implemented in the previous blog post “CUDA Reduction” to use cooperative groups for synchronization and communication between threads. The reduction algorithms remain exactly the same, only the APIs used for synchronizing groups of threads are different. We also implemented a full reduce sum kernel that reduces an array of elements to a single value using cooperative groups using one single kernel launch.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585

#include <cassert>#include <functional>#include <iostream>#include <string>#include <vector>#include <cooperative_groups.h>#include <cuda_runtime.h>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, char const* func, char const* file, int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() check_last(__FILE__, __LINE__)void check_last(char const* file, int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}template <class T>float measure_performance(std::function<T(cudaStream_t)> bound_function,                          cudaStream_t stream, size_t num_repeats = 10,                          size_t num_warmups = 10){    cudaEvent_t start, stop;    float time;    CHECK_CUDA_ERROR(cudaEventCreate(&start));    CHECK_CUDA_ERROR(cudaEventCreate(&stop));    for (size_t i{0}; i < num_warmups; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaEventRecord(start, stream));    for (size_t i{0}; i < num_repeats; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaEventRecord(stop, stream));    CHECK_CUDA_ERROR(cudaEventSynchronize(stop));    CHECK_LAST_CUDA_ERROR();    CHECK_CUDA_ERROR(cudaEventElapsedTime(&time, start, stop));    CHECK_CUDA_ERROR(cudaEventDestroy(start));    CHECK_CUDA_ERROR(cudaEventDestroy(stop));    float const latency{time / num_repeats};    return latency;}std::string std_string_centered(std::string const& s, size_t width,                                char pad = ' '){    size_t const l{s.length()};    // Throw an exception if width is too small.    if (width < l)    {        throw std::runtime_error("Width is too small.");    }    size_t const left_pad{(width - l) / 2};    size_t const right_pad{width - l - left_pad};    std::string const s_centered{std::string(left_pad, pad) + s +                                 std::string(right_pad, pad)};    return s_centered;}template <size_t NUM_THREADS>__device__ float thread_block_reduce_sum(    cooperative_groups::thread_block_tile<NUM_THREADS> group,    float shared_data[NUM_THREADS], float val){    static_assert(NUM_THREADS % 32 == 0,                  "NUM_THREADS must be a multiple of 32");    size_t thread_idx{group.thread_rank()};    shared_data[thread_idx] = val;    group.sync();#pragma unroll    for (size_t offset{group.size() / 2}; offset > 0; offset /= 2)    {        if (thread_idx < offset)        {            shared_data[thread_idx] += shared_data[thread_idx + offset];        }        group.sync();    }    // There will be no shared memory bank conflicts here.    // Because multiple threads in a warp address the same shared memory    // location, resulting in a broadcast.    return shared_data[0];}__device__ float thread_block_reduce_sum(cooperative_groups::thread_block group,                                         float* shared_data, float val){    size_t const thread_idx{group.thread_rank()};    shared_data[thread_idx] = val;    group.sync();    for (size_t stride{group.size() / 2}; stride > 0; stride /= 2)    {        if (thread_idx < stride)        {            shared_data[thread_idx] += shared_data[thread_idx + stride];        }        group.sync();    }    return shared_data[0];}template <size_t NUM_WARPS>__device__ float thread_block_reduce_sum(float shared_data[NUM_WARPS]){    float sum{0.0f};#pragma unroll    for (size_t i{0}; i < NUM_WARPS; ++i)    {        // There will be no shared memory bank conflicts here.        // Because multiple threads in a warp address the same shared memory        // location, resulting in a broadcast.        sum += shared_data[i];    }    return sum;}__device__ float thread_reduce_sum(float const* __restrict__ input_data,                                   size_t start_offset, size_t num_elements,                                   size_t stride){    float sum{0.0f};    for (size_t i{start_offset}; i < num_elements; i += stride)    {        sum += input_data[i];    }    return sum;}__device__ floatwarp_reduce_sum(cooperative_groups::thread_block_tile<32> group, float val){#pragma unroll    for (size_t offset{group.size() / 2}; offset > 0; offset /= 2)    {        // The shfl_down function is a warp shuffle operation that only exists        // for thread block tiles of size 32.        val += group.shfl_down(val, offset);    }    // Only the first thread in the warp will return the correct result.    return val;}template <size_t NUM_THREADS>__device__ floatthread_block_reduce_sum_v1(float const* __restrict__ input_data,                           size_t num_elements){    static_assert(NUM_THREADS % 32 == 0,                  "NUM_THREADS must be a multiple of 32");    __shared__ float shared_data[NUM_THREADS];    size_t const thread_idx{        cooperative_groups::this_thread_block().thread_index().x};    float sum{        thread_reduce_sum(input_data, thread_idx, num_elements, NUM_THREADS)};    shared_data[thread_idx] = sum;    // This somehow does not work.    // static thread block cooperative groups is still not supported.    // cooperative_groups::thread_block_tile<NUM_THREADS> const    // thread_block{cooperative_groups::tiled_partition<NUM_THREADS>(cooperative_groups::this_thread_block())};    // float const block_sum{thread_block_reduce_sum<NUM_THREADS>(thread_block,    // shared_data, sum)}; This works.    float const block_sum{thread_block_reduce_sum(        cooperative_groups::this_thread_block(), shared_data, sum)};    return block_sum;}template <size_t NUM_THREADS, size_t NUM_WARPS = NUM_THREADS / 32>__device__ floatthread_block_reduce_sum_v2(float const* __restrict__ input_data,                           size_t num_elements){    static_assert(NUM_THREADS % 32 == 0,                  "NUM_THREADS must be a multiple of 32");    __shared__ float shared_data[NUM_WARPS];    size_t const thread_idx{        cooperative_groups::this_thread_block().thread_index().x};    float sum{        thread_reduce_sum(input_data, thread_idx, num_elements, NUM_THREADS)};    cooperative_groups::thread_block_tile<32> const warp{        cooperative_groups::tiled_partition<32>(            cooperative_groups::this_thread_block())};    sum = warp_reduce_sum(warp, sum);    if (warp.thread_rank() == 0)    {        shared_data[cooperative_groups::this_thread_block().thread_rank() /                    32] = sum;    }    cooperative_groups::this_thread_block().sync();    float const block_sum{thread_block_reduce_sum<NUM_WARPS>(shared_data)};    return block_sum;}template <size_t NUM_THREADS>__global__ void batched_reduce_sum_v1(float* __restrict__ output_data,                                      float const* __restrict__ input_data,                                      size_t num_elements_per_batch){    static_assert(NUM_THREADS % 32 == 0,                  "NUM_THREADS must be a multiple of 32");    size_t const block_idx{cooperative_groups::this_grid().block_rank()};    size_t const thread_idx{        cooperative_groups::this_thread_block().thread_rank()};    float const block_sum{thread_block_reduce_sum_v1<NUM_THREADS>(        input_data + block_idx * num_elements_per_batch,        num_elements_per_batch)};    if (thread_idx == 0)    {        output_data[block_idx] = block_sum;    }}template <size_t NUM_THREADS>__global__ void batched_reduce_sum_v2(float* __restrict__ output_data,                                      float const* __restrict__ input_data,                                      size_t num_elements_per_batch){    static_assert(NUM_THREADS % 32 == 0,                  "NUM_THREADS must be a multiple of 32");    constexpr size_t NUM_WARPS{NUM_THREADS / 32};    size_t const block_idx{cooperative_groups::this_grid().block_rank()};    size_t const thread_idx{        cooperative_groups::this_thread_block().thread_rank()};    float const block_sum{thread_block_reduce_sum_v2<NUM_THREADS, NUM_WARPS>(        input_data + block_idx * num_elements_per_batch,        num_elements_per_batch)};    if (thread_idx == 0)    {        output_data[block_idx] = block_sum;    }}template <size_t NUM_THREADS, size_t NUM_BLOCK_ELEMENTS>__global__ void full_reduce_sum(float* output,                                float const* __restrict__ input_data,                                size_t num_elements, float* workspace){    static_assert(NUM_THREADS % 32 == 0,                  "NUM_THREADS must be a multiple of 32");    static_assert(NUM_BLOCK_ELEMENTS % NUM_THREADS == 0,                  "NUM_BLOCK_ELEMENTS must be a multiple of NUM_THREADS");    // Workspace size: num_elements.    size_t const num_grid_elements{        NUM_BLOCK_ELEMENTS * cooperative_groups::this_grid().num_blocks()};    float* const workspace_ptr_1{workspace};    float* const workspace_ptr_2{workspace + num_elements / 2};    size_t remaining_elements{num_elements};    // The first iteration of the reduction.    float* workspace_output_data{workspace_ptr_1};    size_t const num_grid_iterations{        (remaining_elements + num_grid_elements - 1) / num_grid_elements};    for (size_t i{0}; i < num_grid_iterations; ++i)    {        size_t const grid_offset{i * num_grid_elements};        size_t const block_offset{grid_offset +                                  cooperative_groups::this_grid().block_rank() *                                      NUM_BLOCK_ELEMENTS};        size_t const num_actual_elements_to_reduce_per_block{            remaining_elements >= block_offset                ? min(NUM_BLOCK_ELEMENTS, remaining_elements - block_offset)                : 0};        float const block_sum{thread_block_reduce_sum_v1<NUM_THREADS>(            input_data + block_offset,            num_actual_elements_to_reduce_per_block)};        if (cooperative_groups::this_thread_block().thread_rank() == 0)        {            workspace_output_data                [i * cooperative_groups::this_grid().num_blocks() +                 cooperative_groups::this_grid().block_rank()] = block_sum;        }    }    cooperative_groups::this_grid().sync();    remaining_elements =        (remaining_elements + NUM_BLOCK_ELEMENTS - 1) / NUM_BLOCK_ELEMENTS;    // The rest iterations of the reduction.    float* workspace_input_data{workspace_output_data};    workspace_output_data = workspace_ptr_2;    while (remaining_elements > 1)    {        size_t const num_grid_iterations{            (remaining_elements + num_grid_elements - 1) / num_grid_elements};        for (size_t i{0}; i < num_grid_iterations; ++i)        {            size_t const grid_offset{i * num_grid_elements};            size_t const block_offset{                grid_offset + cooperative_groups::this_grid().block_rank() *                                  NUM_BLOCK_ELEMENTS};            size_t const num_actual_elements_to_reduce_per_block{                remaining_elements >= block_offset                    ? min(NUM_BLOCK_ELEMENTS, remaining_elements - block_offset)                    : 0};            float const block_sum{thread_block_reduce_sum_v1<NUM_THREADS>(                workspace_input_data + block_offset,                num_actual_elements_to_reduce_per_block)};            if (cooperative_groups::this_thread_block().thread_rank() == 0)            {                workspace_output_data                    [i * cooperative_groups::this_grid().num_blocks() +                     cooperative_groups::this_grid().block_rank()] = block_sum;            }        }        cooperative_groups::this_grid().sync();        remaining_elements =            (remaining_elements + NUM_BLOCK_ELEMENTS - 1) / NUM_BLOCK_ELEMENTS;        // Swap the input and output data.        float* const temp{workspace_input_data};        workspace_input_data = workspace_output_data;        workspace_output_data = temp;    }    // Copy the final result to the output.    workspace_output_data = workspace_input_data;    if (cooperative_groups::this_grid().thread_rank() == 0)    {        *output = workspace_output_data[0];    }}template <size_t NUM_THREADS>void launch_batched_reduce_sum_v1(float* output_data, float const* input_data,                                  size_t batch_size,                                  size_t num_elements_per_batch,                                  cudaStream_t stream){    size_t const num_blocks{batch_size};    batched_reduce_sum_v1<NUM_THREADS><<<num_blocks, NUM_THREADS, 0, stream>>>(        output_data, input_data, num_elements_per_batch);    CHECK_LAST_CUDA_ERROR();}template <size_t NUM_THREADS>void launch_batched_reduce_sum_v2(float* output_data, float const* input_data,                                  size_t batch_size,                                  size_t num_elements_per_batch,                                  cudaStream_t stream){    size_t const num_blocks{batch_size};    batched_reduce_sum_v2<NUM_THREADS><<<num_blocks, NUM_THREADS, 0, stream>>>(        output_data, input_data, num_elements_per_batch);    CHECK_LAST_CUDA_ERROR();}template <size_t NUM_THREADS, size_t NUM_BLOCK_ELEMENTS>void launch_full_reduce_sum(float* output, float const* input_data,                            size_t num_elements, float* workspace,                            cudaStream_t stream){    // https://docs.nvidia.com/cuda/archive/12.4.1/cuda-c-programming-guide/index.html#grid-synchronization    void const* func{reinterpret_cast<void const*>(        full_reduce_sum<NUM_THREADS, NUM_BLOCK_ELEMENTS>)};    int dev{0};    cudaDeviceProp deviceProp;    CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, dev));    dim3 const grid_dim{        static_cast<unsigned int>(deviceProp.multiProcessorCount)};    dim3 const block_dim{NUM_THREADS};    // This will launch a grid that can maximally fill the GPU, on the    // default stream with kernel arguments.    // In practice, it's not always the best.    // void const* func{reinterpret_cast<void const*>(    //     full_reduce_sum<NUM_THREADS, NUM_BLOCK_ELEMENTS>)};    // int dev{0};    // dim3 const block_dim{NUM_THREADS};    // int num_blocks_per_sm{0};    // cudaDeviceProp deviceProp;    // cudaGetDeviceProperties(&deviceProp, dev);    // cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks_per_sm, func,    //                                               NUM_THREADS, 0);    // dim3 const grid_dim{static_cast<unsigned int>(num_blocks_per_sm)};    void* args[]{static_cast<void*>(&output), static_cast<void*>(&input_data),                 static_cast<void*>(&num_elements),                 static_cast<void*>(&workspace)};    CHECK_CUDA_ERROR(cudaLaunchCooperativeKernel(func, grid_dim, block_dim,                                                 args, 0, stream));    CHECK_LAST_CUDA_ERROR();}float profile_full_reduce_sum(    std::function<void(float*, float const*, size_t, float*, cudaStream_t)>        full_reduce_sum_launch_function,    size_t num_elements){    cudaStream_t stream;    CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    constexpr float element_value{1.0f};    std::vector<float> input_data(num_elements, element_value);    float output{0.0f};    float* d_input_data;    float* d_workspace;    float* d_output;    CHECK_CUDA_ERROR(cudaMalloc(&d_input_data, num_elements * sizeof(float)));    CHECK_CUDA_ERROR(cudaMalloc(&d_workspace, num_elements * sizeof(float)));    CHECK_CUDA_ERROR(cudaMalloc(&d_output, sizeof(float)));    CHECK_CUDA_ERROR(cudaMemcpy(d_input_data, input_data.data(),                                num_elements * sizeof(float),                                cudaMemcpyHostToDevice));    full_reduce_sum_launch_function(d_output, d_input_data, num_elements,                                    d_workspace, stream);    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    // Verify the correctness of the kernel.    CHECK_CUDA_ERROR(        cudaMemcpy(&output, d_output, sizeof(float), cudaMemcpyDeviceToHost));    if (output != num_elements * element_value)    {        std::cout << "Expected: " << num_elements * element_value                  << " but got: " << output << std::endl;        throw std::runtime_error("Error: incorrect sum");    }    std::function<void(cudaStream_t)> const bound_function{        std::bind(full_reduce_sum_launch_function, d_output, d_input_data,                  num_elements, d_workspace, std::placeholders::_1)};    float const latency{measure_performance<void>(bound_function, stream)};    std::cout << "Latency: " << latency << " ms" << std::endl;    // Compute effective bandwidth.    size_t num_bytes{num_elements * sizeof(float) + sizeof(float)};    float const bandwidth{(num_bytes * 1e-6f) / latency};    std::cout << "Effective Bandwidth: " << bandwidth << " GB/s" << std::endl;    CHECK_CUDA_ERROR(cudaFree(d_input_data));    CHECK_CUDA_ERROR(cudaFree(d_workspace));    CHECK_CUDA_ERROR(cudaFree(d_output));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));    return latency;}float profile_batched_reduce_sum(    std::function<void(float*, float const*, size_t, size_t, cudaStream_t)>        batched_reduce_sum_launch_function,    size_t batch_size, size_t num_elements_per_batch){    size_t const num_elements{batch_size * num_elements_per_batch};    cudaStream_t stream;    CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    constexpr float element_value{1.0f};    std::vector<float> input_data(num_elements, element_value);    std::vector<float> output_data(batch_size, 0.0f);    float* d_input_data;    float* d_output_data;    CHECK_CUDA_ERROR(cudaMalloc(&d_input_data, num_elements * sizeof(float)));    CHECK_CUDA_ERROR(cudaMalloc(&d_output_data, batch_size * sizeof(float)));    CHECK_CUDA_ERROR(cudaMemcpy(d_input_data, input_data.data(),                                num_elements * sizeof(float),                                cudaMemcpyHostToDevice));    batched_reduce_sum_launch_function(d_output_data, d_input_data, batch_size,                                       num_elements_per_batch, stream);    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    // Verify the correctness of the kernel.    CHECK_CUDA_ERROR(cudaMemcpy(output_data.data(), d_output_data,                                batch_size * sizeof(float),                                cudaMemcpyDeviceToHost));    for (size_t i{0}; i < batch_size; ++i)    {        if (output_data.at(i) != num_elements_per_batch * element_value)        {            std::cout << "Expected: " << num_elements_per_batch * element_value                      << " but got: " << output_data.at(i) << std::endl;            throw std::runtime_error("Error: incorrect sum");        }    }    std::function<void(cudaStream_t)> const bound_function{std::bind(        batched_reduce_sum_launch_function, d_output_data, d_input_data,        batch_size, num_elements_per_batch, std::placeholders::_1)};    float const latency{measure_performance<void>(bound_function, stream)};    std::cout << "Latency: " << latency << " ms" << std::endl;    // Compute effective bandwidth.    size_t num_bytes{num_elements * sizeof(float) + batch_size * sizeof(float)};    float const bandwidth{(num_bytes * 1e-6f) / latency};    std::cout << "Effective Bandwidth: " << bandwidth << " GB/s" << std::endl;    CHECK_CUDA_ERROR(cudaFree(d_input_data));    CHECK_CUDA_ERROR(cudaFree(d_output_data));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));    return latency;}int main(){    size_t const batch_size{2048};    size_t const num_elements_per_batch{1024 * 256};    constexpr size_t string_width{50U};    std::cout << std_string_centered("", string_width, '~') << std::endl;    std::cout << std_string_centered("NVIDIA GPU Device Info", string_width,                                     ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '~') << std::endl;    // Query deive name and peak memory bandwidth.    int device_id{0};    cudaGetDevice(&device_id);    cudaDeviceProp device_prop;    cudaGetDeviceProperties(&device_prop, device_id);    std::cout << "Device Name: " << device_prop.name << std::endl;    float const memory_size{static_cast<float>(device_prop.totalGlobalMem) /                            (1 << 30)};    std::cout << "Memory Size: " << memory_size << " GB" << std::endl;    float const peak_bandwidth{        static_cast<float>(2.0f * device_prop.memoryClockRate *                           (device_prop.memoryBusWidth / 8) / 1.0e6)};    std::cout << "Peak Bandwitdh: " << peak_bandwidth << " GB/s" << std::endl;    std::cout << std_string_centered("", string_width, '~') << std::endl;    std::cout << std_string_centered("Reduce Sum Profiling", string_width, ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '~') << std::endl;    std::cout << std_string_centered("", string_width, '=') << std::endl;    std::cout << "Batch Size: " << batch_size << std::endl;    std::cout << "Number of Elements Per Batch: " << num_elements_per_batch              << std::endl;    std::cout << std_string_centered("", string_width, '=') << std::endl;    constexpr size_t NUM_THREADS_PER_BATCH{256};    static_assert(NUM_THREADS_PER_BATCH % 32 == 0,                  "NUM_THREADS_PER_BATCH must be a multiple of 32");    static_assert(NUM_THREADS_PER_BATCH <= 1024,                  "NUM_THREADS_PER_BATCH must be less than or equal to 1024");    std::cout << "Batched Reduce Sum V1" << std::endl;    float const latency_v1{profile_batched_reduce_sum(        launch_batched_reduce_sum_v1<NUM_THREADS_PER_BATCH>, batch_size,        num_elements_per_batch)};    std::cout << std_string_centered("", string_width, '-') << std::endl;    std::cout << "Batched Reduce Sum V2" << std::endl;    float const latency_v2{profile_batched_reduce_sum(        launch_batched_reduce_sum_v2<NUM_THREADS_PER_BATCH>, batch_size,        num_elements_per_batch)};    std::cout << std_string_centered("", string_width, '-') << std::endl;    std::cout << "Full Reduce Sum" << std::endl;    constexpr size_t NUM_THREADS{256};    constexpr size_t NUM_BLOCK_ELEMENTS{NUM_THREADS * 1024};    float const latency_v3{profile_full_reduce_sum(        launch_full_reduce_sum<NUM_THREADS, NUM_BLOCK_ELEMENTS>,        batch_size * num_elements_per_batch)};    std::cout << std_string_centered("", string_width, '-') << std::endl;}

To build and run the reduce sum example, please run the following commands.

123456789101112131415161718192021222324252627

$ nvcc reduce_sum_cooperative_groups.cu -o reduce_sum_cooperative_groups$ ./reduce_sum_cooperative_groups~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~              NVIDIA GPU Device Info~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Device Name: NVIDIA GeForce RTX 3090Memory Size: 23.6694 GBPeak Bandwitdh: 936.096 GB/s~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~               Reduce Sum Profiling~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~==================================================Batch Size: 2048Number of Elements Per Batch: 262144==================================================Batched Reduce Sum V1Latency: 2.43301 msEffective Bandwidth: 882.649 GB/s--------------------------------------------------Batched Reduce Sum V2Latency: 2.43445 msEffective Bandwidth: 882.126 GB/s--------------------------------------------------Full Reduce SumLatency: 2.47788 msEffective Bandwidth: 866.663 GB/s--------------------------------------------------

The performance of the batched reduce sum kernels using cooperative groups is similar to the performance of the batched reduce sum kernels using traditional CUDA programming models.

Large Array Reduce Sum

There could be three approaches to implement a large array reduce sum kernel.

Without using cooperative groups, we could only synchronize threads within a thread block, which leads to the first approach. But there are additional kernel launch overhead due to multiple kernel launches.

With cooperative groups, we could synchronize threads across thread blocks, which leads to the second approach. The second approach, however, also has drawbacks comparing to the first approach that in the later stage of reduction the number of grids being actually utilized is much smaller because the reduction problem size becomes smaller, which is a waste of computation resources.

References

CUDA Cooperative Groups

https://leimao.github.io/blog/CUDA-Cooperative-Groups/

Author

Lei Mao

Posted on

08-06-2024

Updated on

08-06-2024

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


CUDA Reduction

Introduction

Reduction is a common operation in parallel computing. Usually the reduction operation is used to compute the sum, the maximum, the minimum, the product, of a sequence of elements.

In this blog post, we will discuss the parallel reduction algorithm and its implementation in CUDA.

Batched Reduce Sum

In this example, we implemented two batched reduce sum kernels in CUDA. The batched reduce sum kernel computes the sum for each array of elements in a batch of arrays.

The idea of the reduction algorithm is simple. For each array in the batch, we will assign a thread block consisting of a fixed number of threads to compute the sum of the elements in the array. Each thread will access multiple elements in the array from the global memory and store the partial sum in the register file. After all the threads have computed the partial sum, we have two ways to further reduce the partial sum to the final sum. One way is to use shared memory to store the partial sum and reduce the partial sum in the shared memory. The other way is to use warp-level primitives to reduce the partial sum in the register file in a warp followed by a smaller scale reduction in the shared memory.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360

#include <cassert>#include <functional>#include <iostream>#include <string>#include <vector>#include <cuda_runtime.h>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, char const* func, char const* file, int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() check_last(__FILE__, __LINE__)void check_last(char const* file, int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}template <class T>float measure_performance(std::function<T(cudaStream_t)> bound_function,                          cudaStream_t stream, size_t num_repeats = 10,                          size_t num_warmups = 10){    cudaEvent_t start, stop;    float time;    CHECK_CUDA_ERROR(cudaEventCreate(&start));    CHECK_CUDA_ERROR(cudaEventCreate(&stop));    for (size_t i{0}; i < num_warmups; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaEventRecord(start, stream));    for (size_t i{0}; i < num_repeats; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaEventRecord(stop, stream));    CHECK_CUDA_ERROR(cudaEventSynchronize(stop));    CHECK_LAST_CUDA_ERROR();    CHECK_CUDA_ERROR(cudaEventElapsedTime(&time, start, stop));    CHECK_CUDA_ERROR(cudaEventDestroy(start));    CHECK_CUDA_ERROR(cudaEventDestroy(stop));    float const latency{time / num_repeats};    return latency;}std::string std_string_centered(std::string const& s, size_t width,                                char pad = ' '){    size_t const l{s.length()};    // Throw an exception if width is too small.    if (width < l)    {        throw std::runtime_error("Width is too small.");    }    size_t const left_pad{(width - l) / 2};    size_t const right_pad{width - l - left_pad};    std::string const s_centered{std::string(left_pad, pad) + s +                                 std::string(right_pad, pad)};    return s_centered;}template <size_t NUM_THREADS>__device__ float shared_data_reduce_sum_v1(float shared_data[NUM_THREADS]){    static_assert(NUM_THREADS % 32 == 0,                  "NUM_THREADS must be a multiple of 32");    size_t const thread_idx{threadIdx.x};#pragma unroll    for (size_t stride{NUM_THREADS / 2}; stride > 0; stride /= 2)    {        if (thread_idx < stride)        {            shared_data[thread_idx] += shared_data[thread_idx + stride];        }        __syncthreads();    }    return shared_data[0];}template <size_t NUM_WARPS>__device__ float shared_data_reduce_sum_v2(float shared_data[NUM_WARPS]){    float sum{0.0f};#pragma unroll    for (size_t i{0}; i < NUM_WARPS; ++i)    {        // There will be no shared memory bank conflicts here.        // Because multiple threads in a warp address the same shared memory        // location, resulting in a broadcast.        sum += shared_data[i];    }    return sum;}__device__ float warp_reduce_sum(float val){    constexpr unsigned int FULL_MASK{0xffffffff};#pragma unroll    for (size_t offset{16}; offset > 0; offset /= 2)    {        val += __shfl_down_sync(FULL_MASK, val, offset);    }    // Only the first thread in the warp will return the correct result.    return val;}template <size_t NUM_THREADS>__device__ float block_reduce_sum_v1(float const* __restrict__ input_data,                                     float shared_data[NUM_THREADS],                                     size_t num_elements){    static_assert(NUM_THREADS % 32 == 0,                  "NUM_THREADS must be a multiple of 32");    size_t const num_elements_per_thread{(num_elements + NUM_THREADS - 1) /                                         NUM_THREADS};    size_t const thread_idx{threadIdx.x};    float sum{0.0f};    for (size_t i{0}; i < num_elements_per_thread; ++i)    {        size_t const offset{thread_idx + i * NUM_THREADS};        if (offset < num_elements)        {            sum += input_data[offset];        }    }    shared_data[thread_idx] = sum;    __syncthreads();    float const block_sum{shared_data_reduce_sum_v1<NUM_THREADS>(shared_data)};    return block_sum;}template <size_t NUM_THREADS, size_t NUM_WARPS = NUM_THREADS / 32>__device__ float block_reduce_sum_v2(float const* __restrict__ input_data,                                     float shared_data[NUM_WARPS],                                     size_t num_elements){    size_t const num_elements_per_thread{(num_elements + NUM_THREADS - 1) /                                         NUM_THREADS};    size_t const thread_idx{threadIdx.x};    float sum{0.0f};    for (size_t i{0}; i < num_elements_per_thread; ++i)    {        size_t const offset{thread_idx + i * NUM_THREADS};        if (offset < num_elements)        {            sum += input_data[offset];        }    }    sum = warp_reduce_sum(sum);    if (threadIdx.x % 32 == 0)    {        shared_data[threadIdx.x / 32] = sum;    }    __syncthreads();    float const block_sum{shared_data_reduce_sum_v2<NUM_WARPS>(shared_data)};    return block_sum;}template <size_t NUM_THREADS>__global__ void batched_reduce_sum_v1(float* __restrict__ output_data,                                      float const* __restrict__ input_data,                                      size_t num_elements_per_batch){    static_assert(NUM_THREADS % 32 == 0,                  "NUM_THREADS must be a multiple of 32");    size_t const block_idx{blockIdx.x};    size_t const thread_idx{threadIdx.x};    __shared__ float shared_data[NUM_THREADS];    float const block_sum{block_reduce_sum_v1<NUM_THREADS>(        input_data + block_idx * num_elements_per_batch, shared_data,        num_elements_per_batch)};    if (thread_idx == 0)    {        output_data[block_idx] = block_sum;    }}template <size_t NUM_THREADS>__global__ void batched_reduce_sum_v2(float* __restrict__ output_data,                                      float const* __restrict__ input_data,                                      size_t num_elements_per_batch){    static_assert(NUM_THREADS % 32 == 0,                  "NUM_THREADS must be a multiple of 32");    constexpr size_t NUM_WARPS{NUM_THREADS / 32};    size_t const block_idx{blockIdx.x};    size_t const thread_idx{threadIdx.x};    __shared__ float shared_data[NUM_WARPS];    float const block_sum{block_reduce_sum_v2<NUM_THREADS, NUM_WARPS>(        input_data + block_idx * num_elements_per_batch, shared_data,        num_elements_per_batch)};    if (thread_idx == 0)    {        output_data[block_idx] = block_sum;    }}template <size_t NUM_THREADS>void launch_batched_reduce_sum_v1(float* output_data, float const* input_data,                                  size_t batch_size,                                  size_t num_elements_per_batch,                                  cudaStream_t stream){    size_t const num_blocks{batch_size};    batched_reduce_sum_v1<NUM_THREADS><<<num_blocks, NUM_THREADS, 0, stream>>>(        output_data, input_data, num_elements_per_batch);    CHECK_LAST_CUDA_ERROR();}template <size_t NUM_THREADS>void launch_batched_reduce_sum_v2(float* output_data, float const* input_data,                                  size_t batch_size,                                  size_t num_elements_per_batch,                                  cudaStream_t stream){    size_t const num_blocks{batch_size};    batched_reduce_sum_v2<NUM_THREADS><<<num_blocks, NUM_THREADS, 0, stream>>>(        output_data, input_data, num_elements_per_batch);    CHECK_LAST_CUDA_ERROR();}float profile_batched_reduce_sum(    std::function<void(float*, float const*, size_t, size_t, cudaStream_t)>        batched_reduce_sum_launch_function,    size_t batch_size, size_t num_elements_per_batch){    size_t const num_elements{batch_size * num_elements_per_batch};    cudaStream_t stream;    CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    constexpr float element_value{1.0f};    std::vector<float> input_data(num_elements, element_value);    std::vector<float> output_data(batch_size, 0.0f);    float* d_input_data;    float* d_output_data;    CHECK_CUDA_ERROR(cudaMalloc(&d_input_data, num_elements * sizeof(float)));    CHECK_CUDA_ERROR(cudaMalloc(&d_output_data, batch_size * sizeof(float)));    CHECK_CUDA_ERROR(cudaMemcpy(d_input_data, input_data.data(),                                num_elements * sizeof(float),                                cudaMemcpyHostToDevice));    batched_reduce_sum_launch_function(d_output_data, d_input_data, batch_size,                                       num_elements_per_batch, stream);    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    // Verify the correctness of the kernel.    CHECK_CUDA_ERROR(cudaMemcpy(output_data.data(), d_output_data,                                batch_size * sizeof(float),                                cudaMemcpyDeviceToHost));    for (size_t i{0}; i < batch_size; ++i)    {        if (output_data.at(i) != num_elements_per_batch * element_value)        {            std::cout << "Expected: " << num_elements_per_batch * element_value                      << " but got: " << output_data.at(i) << std::endl;            throw std::runtime_error("Error: incorrect sum");        }    }    std::function<void(cudaStream_t)> const bound_function{std::bind(        batched_reduce_sum_launch_function, d_output_data, d_input_data,        batch_size, num_elements_per_batch, std::placeholders::_1)};    float const latency{measure_performance<void>(bound_function, stream)};    std::cout << "Latency: " << latency << " ms" << std::endl;    // Compute effective bandwidth.    size_t num_bytes{num_elements * sizeof(float) + batch_size * sizeof(float)};    float const bandwidth{(num_bytes * 1e-6f) / latency};    std::cout << "Effective Bandwidth: " << bandwidth << " GB/s" << std::endl;    CHECK_CUDA_ERROR(cudaFree(d_input_data));    CHECK_CUDA_ERROR(cudaFree(d_output_data));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));    return latency;}int main(){    size_t const batch_size{2048};    size_t const num_elements_per_batch{1024 * 256};    constexpr size_t string_width{50U};    std::cout << std_string_centered("", string_width, '~') << std::endl;    std::cout << std_string_centered("NVIDIA GPU Device Info", string_width,                                     ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '~') << std::endl;    // Query deive name and peak memory bandwidth.    int device_id{0};    cudaGetDevice(&device_id);    cudaDeviceProp device_prop;    cudaGetDeviceProperties(&device_prop, device_id);    std::cout << "Device Name: " << device_prop.name << std::endl;    float const memory_size{static_cast<float>(device_prop.totalGlobalMem) /                            (1 << 30)};    std::cout << "Memory Size: " << memory_size << " GB" << std::endl;    float const peak_bandwidth{        static_cast<float>(2.0f * device_prop.memoryClockRate *                           (device_prop.memoryBusWidth / 8) / 1.0e6)};    std::cout << "Peak Bandwitdh: " << peak_bandwidth << " GB/s" << std::endl;    std::cout << std_string_centered("", string_width, '~') << std::endl;    std::cout << std_string_centered("Reduce Sum Profiling", string_width, ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '~') << std::endl;    std::cout << std_string_centered("", string_width, '=') << std::endl;    std::cout << "Batch Size: " << batch_size << std::endl;    std::cout << "Number of Elements Per Batch: " << num_elements_per_batch              << std::endl;    std::cout << std_string_centered("", string_width, '=') << std::endl;    constexpr size_t NUM_THREADS_PER_BATCH{256};    static_assert(NUM_THREADS_PER_BATCH % 32 == 0,                  "NUM_THREADS_PER_BATCH must be a multiple of 32");    static_assert(NUM_THREADS_PER_BATCH <= 1024,                  "NUM_THREADS_PER_BATCH must be less than or equal to 1024");    std::cout << "Batched Reduce Sum V1" << std::endl;    float const latency_v1{profile_batched_reduce_sum(        launch_batched_reduce_sum_v1<NUM_THREADS_PER_BATCH>, batch_size,        num_elements_per_batch)};    std::cout << std_string_centered("", string_width, '-') << std::endl;    std::cout << "Batched Reduce Sum V2" << std::endl;    float const latency_v2{profile_batched_reduce_sum(        launch_batched_reduce_sum_v2<NUM_THREADS_PER_BATCH>, batch_size,        num_elements_per_batch)};    std::cout << std_string_centered("", string_width, '-') << std::endl;}

To build and run the reduce sum example, please run the following commands.

1234567891011121314151617181920212223

$ nvcc reduce_sum.cu -o reduce_sum$ ./reduce_sum~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~              NVIDIA GPU Device Info~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Device Name: NVIDIA GeForce RTX 3090Memory Size: 23.6694 GBPeak Bandwitdh: 936.096 GB/s~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~               Reduce Sum Profiling~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~==================================================Batch Size: 2048Number of Elements Per Batch: 262144==================================================Batched Reduce Sum V1Latency: 2.42976 msEffective Bandwidth: 883.83 GB/s--------------------------------------------------Batched Reduce Sum V2Latency: 2.44303 msEffective Bandwidth: 879.028 GB/s--------------------------------------------------

It turns out that the two batched reduce sum kernels have similar performance. The effective bandwidth is about 94% of the peak bandwidth of the GPU. It should be noted that on my system, the effective bandwidth can vary from run to run at different times of the day, from 750 GB/s to 900 GB/s.

Large Array Reduce Sum

What if we have much larger arrays and much smaller batch sizes? The maximum number of threads in a thread block is 1024. If only one thread block is assigned to compute the sum of the elements in a much larger array and the batch size is very small, the GPU utilization and the effective bandwidth will be very low.

In this case, we will need to split a large array into multiple smaller arrays as if each large array is a batch of arrays. We will assign multiple thread blocks to compute the sum of the elements in each smaller array. Once the sum of the elements in each smaller array is computed, we will further reduce the partial sum to the final sum using the batched reduce sum kernels again.

Concretely, suppose a batch of data is of shape (batch_size, num_elements_per_batch), if num_elements_per_batch is very large and batch_size is very small, we can always reshape the data into a shape of (batch_size * inner_batch_size, inner_num_elements_per_batch) and run batched reduce sum kernel. The resulting reduced sum will be of shape (batch_size * inner_batch_size, 1). We can further reshape the reduced sum into a shape of (batch_size, inner_batch_size) (let’s call it (batch_size, num_elements_per_batch) again) and run batched reduce sum kernel. This process can be repeated until the num_elements_per_batch is not too large.

Of course, instead of running the batched reduce sum kernel and synchronization multiple times, we can also try adding the partial sum of each smaller array to the final sum in the global memory using atomic operations. This, however, may or may not have performance degradations comparing to running the batched reduce sum kernel and synchronization multiple times.

References

CUDA Reduction

https://leimao.github.io/blog/CUDA-Reduction/

Author

Lei Mao

Posted on

07-30-2024

Updated on

07-30-2024

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


CUDA Vectorized Memory Access

Introduction

Reading and writing data from and to the DRAM is one of the fundamental operations in CUDA programming. The effective memory bandwidth of the CUDA device is one of the most critical factors that affect the performance of a CUDA function, especially when the CUDA function is memory-bound.

In this blog post, we will show how to improve the effective memory bandwidth of a CUDA function by using vectorized memory access.

CUDA Vectorized Memory Access

In the following example, we will implement a naive custom device memcpy function and show how to improve its effective memory bandwidth via 8-byte or 16-byte per thread vectorized memory transactions for contiguous data of different data types. The consequence of using 8-byte or 16-byte per thread vectorized memory transactions is that the number of memory transactions required for data copy gets reduced, which improves the effective memory bandwidth in almost all the use cases.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638

#include <chrono>#include <functional>#include <iomanip>#include <iostream>#include <tuple>#include <type_traits>#include <vector>#include <cuda_runtime.h>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, const char* const func, const char* const file,           const int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() check_last(__FILE__, __LINE__)void check_last(const char* const file, const int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}std::string std_string_centered(std::string const& s, size_t width,                                char pad = ' '){    size_t const l{s.length()};    // Throw an exception if width is too small.    if (width < l)    {        throw std::runtime_error("Width is too small.");    }    size_t const left_pad{(width - l) / 2};    size_t const right_pad{width - l - left_pad};    std::string const s_centered{std::string(left_pad, pad) + s +                                 std::string(right_pad, pad)};    return s_centered;}template <class T>float measure_performance(std::function<T(cudaStream_t)> const& bound_function,                          cudaStream_t stream, unsigned int num_repeats = 100,                          unsigned int num_warmups = 100){    cudaEvent_t start, stop;    float time;    CHECK_CUDA_ERROR(cudaEventCreate(&start));    CHECK_CUDA_ERROR(cudaEventCreate(&stop));    for (unsigned int i{0U}; i < num_warmups; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaEventRecord(start, stream));    for (unsigned int i{0U}; i < num_repeats; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaEventRecord(stop, stream));    CHECK_CUDA_ERROR(cudaEventSynchronize(stop));    CHECK_LAST_CUDA_ERROR();    CHECK_CUDA_ERROR(cudaEventElapsedTime(&time, start, stop));    CHECK_CUDA_ERROR(cudaEventDestroy(start));    CHECK_CUDA_ERROR(cudaEventDestroy(stop));    float const latency{time / num_repeats};    return latency;}template <typename T>__global__ void custom_device_memcpy(T* __restrict__ output,                                     T const* __restrict__ input, size_t n){    size_t const idx{blockDim.x * blockIdx.x + threadIdx.x};    size_t const stride{blockDim.x * gridDim.x};    for (size_t i{idx}; i < n; i += stride)    {        output[i] = input[i];    }}template <typename T>void launch_custom_device_memcpy(T* output, T const* input, size_t n,                                 cudaStream_t stream){    dim3 const threads_per_block{1024};    dim3 const blocks_per_grid{static_cast<unsigned int>(std::min(        (n + threads_per_block.x - 1U) / threads_per_block.x,        static_cast<size_t>(std::numeric_limits<unsigned int>::max())))};    custom_device_memcpy<<<blocks_per_grid, threads_per_block, 0, stream>>>(        output, input, n);    CHECK_LAST_CUDA_ERROR();}template <typename T, unsigned int BLOCK_DIM_X>__global__ void custom_device_memcpy_shared_memory(T* __restrict__ output,                                                   T const* __restrict__ input,                                                   size_t n){    // Using shared memory as intermediate buffer.    __shared__ T shared_memory[BLOCK_DIM_X];    size_t const idx{blockDim.x * blockIdx.x + threadIdx.x};    size_t const stride{blockDim.x * gridDim.x};    for (size_t i{idx}; i < n; i += stride)    {        shared_memory[threadIdx.x] = input[i];        // Synchronization is not necessary in this case.        // __syncthreads();        output[i] = shared_memory[threadIdx.x];    }}template <typename T>void launch_custom_device_memcpy_shared_memory(T* output, T const* input,                                               size_t n, cudaStream_t stream){    constexpr dim3 threads_per_block{1024};    dim3 const blocks_per_grid{static_cast<unsigned int>(std::min(        (n + threads_per_block.x - 1U) / threads_per_block.x,        static_cast<size_t>(std::numeric_limits<unsigned int>::max())))};    custom_device_memcpy_shared_memory<T, threads_per_block.x>        <<<blocks_per_grid, threads_per_block, 0, stream>>>(output, input, n);    CHECK_LAST_CUDA_ERROR();}// One thread copies sizeof(R) bytes of data.// One warp copies 32 x sizeof(R) bytes of data via one of few memory// transactions.template <typename T, typename R = uint64_t>__global__ void custom_device_memcpy_optimized(T* __restrict__ output,                                               T const* __restrict__ input,                                               size_t n){    size_t const idx{blockDim.x * blockIdx.x + threadIdx.x};    size_t const stride{blockDim.x * gridDim.x};    for (size_t i{idx}; i * sizeof(R) / sizeof(T) < n; i += stride)    {        if ((i + 1U) * sizeof(R) / sizeof(T) < n)        {            reinterpret_cast<R*>(output)[i] =                reinterpret_cast<R const*>(input)[i];        }        else        {            // Remaining units to copy.            size_t const start_index{i * sizeof(R) / sizeof(T)};            size_t const remaining_units_to_copy{(n - start_index)};            for (size_t j{0}; j < remaining_units_to_copy; ++j)            {                output[start_index + j] = input[start_index + j];            }        }    }}template <typename T, typename R = uint64_t>void launch_custom_device_memcpy_optimized(T* output, T const* input, size_t n,                                           cudaStream_t stream){    dim3 const threads_per_block{1024};    size_t const num_units_to_copy_round_up{(n * sizeof(T) + sizeof(R) - 1U) /                                            sizeof(R)};    dim3 const blocks_per_grid{static_cast<unsigned int>(std::min(        (num_units_to_copy_round_up + threads_per_block.x - 1U) /            threads_per_block.x,        static_cast<size_t>(std::numeric_limits<unsigned int>::max())))};    custom_device_memcpy_optimized<<<blocks_per_grid, threads_per_block, 0,                                     stream>>>(output, input, n);    CHECK_LAST_CUDA_ERROR();}template <typename T>void launch_official_device_memcpy(T* output, T const* input, size_t n,                                   cudaStream_t stream){    CHECK_CUDA_ERROR(cudaMemcpyAsync(output, input, n * sizeof(T),                                     cudaMemcpyDeviceToDevice, stream));}// Initialize the buffer so that the unit of the data is the index of the data.template <typename T, std::enable_if_t<std::is_integral<T>::value, bool> = true>void initialize_buffer(T* buffer, size_t n){    for (size_t i{0}; i < n; ++i)    {        buffer[i] = static_cast<T>(            i % static_cast<size_t>(std::numeric_limits<T>::max()));    }}template <typename T, std::enable_if_t<std::is_integral<T>::value, bool> = true>void verify_buffer(T* buffer, size_t n){    for (size_t i{0}; i < n; ++i)    {        if (buffer[i] != static_cast<T>(i % static_cast<size_t>(                                                std::numeric_limits<T>::max())))        {            std::cerr << "Verification failed at index: " << i << std::endl;            std::exit(EXIT_FAILURE);        }    }}// Measure custom device memcpy performance given the number of units to copy,// the device memcpy function to use, and the number of repeats and warmups.template <typename T>float measure_custom_device_memcpy_performance(    size_t n,    std::function<void(T*, T const*, size_t, cudaStream_t)> const&        device_memcpy_function,    int num_repeats = 100, int num_warmups = 100){    cudaStream_t stream;    CHECK_CUDA_ERROR(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));    std::vector<T> input(n);    std::vector<T> output(n, static_cast<T>(0));    initialize_buffer(input.data(), n);    T* d_input;    T* d_output;    CHECK_CUDA_ERROR(cudaMalloc(&d_input, n * sizeof(T)));    CHECK_CUDA_ERROR(cudaMalloc(&d_output, n * sizeof(T)));    CHECK_CUDA_ERROR(cudaMemcpyAsync(d_input, input.data(), n * sizeof(T),                                     cudaMemcpyHostToDevice, stream));    CHECK_CUDA_ERROR(cudaMemcpyAsync(d_output, output.data(), n * sizeof(T),                                     cudaMemcpyHostToDevice, stream));    // Run device memcpy once to check correcness.    device_memcpy_function(d_output, d_input, n, stream);    CHECK_CUDA_ERROR(cudaMemcpyAsync(output.data(), d_output, n * sizeof(T),                                     cudaMemcpyDeviceToHost, stream));    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    // Verify the correctness of the device memcpy.    verify_buffer(output.data(), n);    size_t const num_bytes{n * sizeof(T)};    float const num_giga_bytes{static_cast<float>(num_bytes) / (1 << 30)};    std::function<void(cudaStream_t)> function{std::bind(        device_memcpy_function, d_output, d_input, n, std::placeholders::_1)};    float const latency{        measure_performance(function, stream, num_repeats, num_warmups)};    std::cout << std::fixed << std::setprecision(3) << "Latency: " << latency              << " ms" << std::endl;    std::cout << "Effective Bandwitdh: "              << 2.f * num_giga_bytes / (latency / 1000) << " GB/s"              << std::endl;    CHECK_CUDA_ERROR(cudaFree(d_input));    CHECK_CUDA_ERROR(cudaFree(d_output));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));    // Query deive name and peak memory bandwidth.    int device_id{0};    cudaGetDevice(&device_id);    cudaDeviceProp device_prop;    cudaGetDeviceProperties(&device_prop, device_id);    float const peak_bandwidth{        static_cast<float>(2.0 * device_prop.memoryClockRate *                           (device_prop.memoryBusWidth / 8) / 1.0e6)};    std::cout << "Percentage of Peak Bandwitdh: "              << 2.f * num_giga_bytes / (latency / 1000) / peak_bandwidth * 100              << "%" << std::endl;    return latency;}int main(){    constexpr unsigned int num_repeats{10U};    constexpr unsigned int num_warmups{10U};    constexpr size_t tensor_size_small{1U * 64U * 64U * 64U};    constexpr size_t tensor_size_medium{1U * 128U * 128U * 128U};    constexpr size_t tensor_size_large{1U * 512U * 512U * 512U};    constexpr size_t string_width{50U};    std::cout << std_string_centered("", string_width, '~') << std::endl;    std::cout << std_string_centered("NVIDIA GPU Device Info", string_width,                                     ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '~') << std::endl;    // Query deive name and peak memory bandwidth.    int device_id{0};    cudaGetDevice(&device_id);    cudaDeviceProp device_prop;    cudaGetDeviceProperties(&device_prop, device_id);    std::cout << "Device Name: " << device_prop.name << std::endl;    float const memory_size{static_cast<float>(device_prop.totalGlobalMem) /                            (1 << 30)};    std::cout << "Memory Size: " << memory_size << " GB" << std::endl;    float const peak_bandwidth{        static_cast<float>(2.0f * device_prop.memoryClockRate *                           (device_prop.memoryBusWidth / 8) / 1.0e6)};    std::cout << "Peak Bandwitdh: " << peak_bandwidth << " GB/s" << std::endl;    std::cout << std::endl;    // Measure CUDA official memcpy performance for different tensor sizes.    std::cout << std_string_centered("", string_width, '*') << std::endl;    std::cout << std_string_centered("CUDA Official Memcpy", string_width, ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '*') << std::endl;    for (size_t tensor_size :         {tensor_size_small, tensor_size_medium, tensor_size_large})    {        std::string const tensor_size_string{std::string("Tensor Size: ") +                                             std::to_string(tensor_size) +                                             std::string(" Units")};        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered(tensor_size_string, string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 1 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int8_t>(            tensor_size, launch_official_device_memcpy<int8_t>, num_repeats,            num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 2 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int16_t>(            tensor_size, launch_official_device_memcpy<int16_t>, num_repeats,            num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 4 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int32_t>(            tensor_size, launch_official_device_memcpy<int32_t>, num_repeats,            num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 8 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int64_t>(            tensor_size, launch_official_device_memcpy<int64_t>, num_repeats,            num_warmups);    }    std::cout << std::endl;    // Measure the latency and bandwidth of custom device memcpy for different    // tensor sizes.    std::cout << std_string_centered("", string_width, '*') << std::endl;    std::cout << std_string_centered("Custom Device Memcpy", string_width, ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '*') << std::endl;    for (size_t tensor_size :         {tensor_size_small, tensor_size_medium, tensor_size_large})    {        std::string const tensor_size_string{std::string("Tensor Size: ") +                                             std::to_string(tensor_size) +                                             std::string(" Units")};        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered(tensor_size_string, string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 1 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int8_t>(            tensor_size, launch_custom_device_memcpy<int8_t>, num_repeats,            num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 2 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int16_t>(            tensor_size, launch_custom_device_memcpy<int16_t>, num_repeats,            num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 4 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int32_t>(            tensor_size, launch_custom_device_memcpy<int32_t>, num_repeats,            num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 8 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int64_t>(            tensor_size, launch_custom_device_memcpy<int64_t>, num_repeats,            num_warmups);    }    std::cout << std::endl;    // Conclusions:    // 1. The more units of data we copy, the higher the bandwidth.    // 2. The larger the unit of the data, the higher the bandwidth.    // Check if shared memory can improve the latency of custom device memcpy.    std::cout << std_string_centered("", string_width, '*') << std::endl;    std::cout << std_string_centered("Custom Device Memcpy with Shared Memory",                                     string_width, ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '*') << std::endl;    for (size_t tensor_size :         {tensor_size_small, tensor_size_medium, tensor_size_large})    {        std::string const tensor_size_string{std::string("Tensor Size: ") +                                             std::to_string(tensor_size) +                                             std::string(" Units")};        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered(tensor_size_string, string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 1 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int8_t>(            tensor_size, launch_custom_device_memcpy_shared_memory<int8_t>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 2 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int16_t>(            tensor_size, launch_custom_device_memcpy_shared_memory<int16_t>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 4 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int32_t>(            tensor_size, launch_custom_device_memcpy_shared_memory<int32_t>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 8 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int64_t>(            tensor_size, launch_custom_device_memcpy_shared_memory<int64_t>,            num_repeats, num_warmups);    }    std::cout << std::endl;    // Conclusions:    // 1. The effect of using shared memory for improving the latency of custom    // device memcpy is not obvious.    // Improve the latency of custom device memcpy when the unit of the data is    // small.    std::cout << std_string_centered("", string_width, '*') << std::endl;    std::cout << std_string_centered(                     "Custom Device Memcpy 4-Byte Copy Per Thread",                     string_width, ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '*') << std::endl;    for (size_t tensor_size :         {tensor_size_small, tensor_size_medium, tensor_size_large})    {        std::string const tensor_size_string{std::string("Tensor Size: ") +                                             std::to_string(tensor_size) +                                             std::string(" Units")};        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered(tensor_size_string, string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 1 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int8_t>(            tensor_size,            launch_custom_device_memcpy_optimized<int8_t, uint32_t>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 2 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int16_t>(            tensor_size,            launch_custom_device_memcpy_optimized<int16_t, uint32_t>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 4 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int32_t>(            tensor_size,            launch_custom_device_memcpy_optimized<int32_t, uint32_t>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 8 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int64_t>(            tensor_size,            launch_custom_device_memcpy_optimized<int64_t, uint32_t>,            num_repeats, num_warmups);    }    std::cout << std::endl;    std::cout << std_string_centered("", string_width, '*') << std::endl;    std::cout << std_string_centered(                     "Custom Device Memcpy 8-Byte Copy Per Thread",                     string_width, ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '*') << std::endl;    for (size_t tensor_size :         {tensor_size_small, tensor_size_medium, tensor_size_large})    {        std::string const tensor_size_string{std::string("Tensor Size: ") +                                             std::to_string(tensor_size) +                                             std::string(" Units")};        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered(tensor_size_string, string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 1 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int8_t>(            tensor_size,            launch_custom_device_memcpy_optimized<int8_t, uint64_t>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 2 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int16_t>(            tensor_size,            launch_custom_device_memcpy_optimized<int16_t, uint64_t>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 4 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int32_t>(            tensor_size,            launch_custom_device_memcpy_optimized<int32_t, uint64_t>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 8 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int64_t>(            tensor_size,            launch_custom_device_memcpy_optimized<int64_t, uint64_t>,            num_repeats, num_warmups);    }    std::cout << std::endl;    std::cout << std_string_centered("", string_width, '*') << std::endl;    std::cout << std_string_centered(                     "Custom Device Memcpy 16-Byte Copy Per Thread",                     string_width, ' ')              << std::endl;    std::cout << std_string_centered("", string_width, '*') << std::endl;    for (size_t tensor_size :         {tensor_size_small, tensor_size_medium, tensor_size_large})    {        std::string const tensor_size_string{std::string("Tensor Size: ") +                                             std::to_string(tensor_size) +                                             std::string(" Units")};        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered(tensor_size_string, string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '=') << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 1 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int8_t>(            tensor_size, launch_custom_device_memcpy_optimized<int8_t, uint4>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 2 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int16_t>(            tensor_size, launch_custom_device_memcpy_optimized<int16_t, uint4>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 4 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int32_t>(            tensor_size, launch_custom_device_memcpy_optimized<int32_t, uint4>,            num_repeats, num_warmups);        std::cout << std_string_centered("", string_width, '-') << std::endl;        std::cout << std_string_centered("Unit Size: 8 Byte", string_width, ' ')                  << std::endl;        std::cout << std_string_centered("", string_width, '-') << std::endl;        measure_custom_device_memcpy_performance<int64_t>(            tensor_size, launch_custom_device_memcpy_optimized<int64_t, uint4>,            num_repeats, num_warmups);    }    std::cout << std::endl;    // Conclusions:    // 1. Copying data in units of 8 bytes or 16 bytes can improve the latency    // of custom device memcpy.}

The CUDA program was compiled and profiled on an NVIDIA RTX 3090 GPU with CUDA 12.0.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518

$ nvcc memcpy.cu -o memcpy -std=c++14$ ./memcpy~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~              NVIDIA GPU Device Info~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Device Name: NVIDIA GeForce RTX 3090Memory Size: 23.6694 GBPeak Bandwitdh: 936.096 GB/s**************************************************               CUDA Official Memcpy**************************************************==================================================            Tensor Size: 262144 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.002 msEffective Bandwitdh: 217.362 GB/sPercentage of Peak Bandwitdh: 23.220%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.002 msEffective Bandwitdh: 414.641 GB/sPercentage of Peak Bandwitdh: 44.295%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 706.425 GB/sPercentage of Peak Bandwitdh: 75.465%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.004 msEffective Bandwitdh: 1030.999 GB/sPercentage of Peak Bandwitdh: 110.138%==================================================            Tensor Size: 2097152 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.004 msEffective Bandwitdh: 1059.638 GB/sPercentage of Peak Bandwitdh: 113.198%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.011 msEffective Bandwitdh: 719.754 GB/sPercentage of Peak Bandwitdh: 76.889%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.023 msEffective Bandwitdh: 675.261 GB/sPercentage of Peak Bandwitdh: 72.136%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.043 msEffective Bandwitdh: 719.330 GB/sPercentage of Peak Bandwitdh: 76.844%==================================================           Tensor Size: 134217728 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.321 msEffective Bandwitdh: 778.091 GB/sPercentage of Peak Bandwitdh: 83.121%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.640 msEffective Bandwitdh: 781.539 GB/sPercentage of Peak Bandwitdh: 83.489%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 1.275 msEffective Bandwitdh: 784.214 GB/sPercentage of Peak Bandwitdh: 83.775%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 2.560 msEffective Bandwitdh: 781.282 GB/sPercentage of Peak Bandwitdh: 83.462%**************************************************               Custom Device Memcpy**************************************************==================================================            Tensor Size: 262144 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 183.399 GB/sPercentage of Peak Bandwitdh: 19.592%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 354.443 GB/sPercentage of Peak Bandwitdh: 37.864%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 681.196 GB/sPercentage of Peak Bandwitdh: 72.770%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 1192.093 GB/sPercentage of Peak Bandwitdh: 127.347%==================================================            Tensor Size: 2097152 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.010 msEffective Bandwitdh: 378.747 GB/sPercentage of Peak Bandwitdh: 40.460%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.018 msEffective Bandwitdh: 445.593 GB/sPercentage of Peak Bandwitdh: 47.601%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.024 msEffective Bandwitdh: 660.732 GB/sPercentage of Peak Bandwitdh: 70.584%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.042 msEffective Bandwitdh: 737.140 GB/sPercentage of Peak Bandwitdh: 78.746%==================================================           Tensor Size: 134217728 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.972 msEffective Bandwitdh: 257.207 GB/sPercentage of Peak Bandwitdh: 27.477%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 1.076 msEffective Bandwitdh: 464.543 GB/sPercentage of Peak Bandwitdh: 49.626%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 1.369 msEffective Bandwitdh: 730.586 GB/sPercentage of Peak Bandwitdh: 78.046%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 2.536 msEffective Bandwitdh: 788.727 GB/sPercentage of Peak Bandwitdh: 84.257%**************************************************     Custom Device Memcpy with Shared Memory**************************************************==================================================            Tensor Size: 262144 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 175.995 GB/sPercentage of Peak Bandwitdh: 18.801%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 328.853 GB/sPercentage of Peak Bandwitdh: 35.130%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 653.481 GB/sPercentage of Peak Bandwitdh: 69.809%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 1128.192 GB/sPercentage of Peak Bandwitdh: 120.521%==================================================            Tensor Size: 2097152 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.011 msEffective Bandwitdh: 353.213 GB/sPercentage of Peak Bandwitdh: 37.733%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.018 msEffective Bandwitdh: 433.488 GB/sPercentage of Peak Bandwitdh: 46.308%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.024 msEffective Bandwitdh: 650.261 GB/sPercentage of Peak Bandwitdh: 69.465%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.042 msEffective Bandwitdh: 737.864 GB/sPercentage of Peak Bandwitdh: 78.824%==================================================           Tensor Size: 134217728 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 1.011 msEffective Bandwitdh: 247.181 GB/sPercentage of Peak Bandwitdh: 26.406%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 1.113 msEffective Bandwitdh: 449.172 GB/sPercentage of Peak Bandwitdh: 47.984%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 1.391 msEffective Bandwitdh: 718.748 GB/sPercentage of Peak Bandwitdh: 76.781%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 2.546 msEffective Bandwitdh: 785.429 GB/sPercentage of Peak Bandwitdh: 83.905%**************************************************   Custom Device Memcpy 4-Byte Copy Per Thread**************************************************==================================================            Tensor Size: 262144 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.002 msEffective Bandwitdh: 238.419 GB/sPercentage of Peak Bandwitdh: 25.469%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.002 msEffective Bandwitdh: 437.842 GB/sPercentage of Peak Bandwitdh: 46.773%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 684.251 GB/sPercentage of Peak Bandwitdh: 73.096%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.004 msEffective Bandwitdh: 1003.868 GB/sPercentage of Peak Bandwitdh: 107.240%==================================================            Tensor Size: 2097152 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.004 msEffective Bandwitdh: 968.812 GB/sPercentage of Peak Bandwitdh: 103.495%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.012 msEffective Bandwitdh: 675.168 GB/sPercentage of Peak Bandwitdh: 72.126%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.024 msEffective Bandwitdh: 660.196 GB/sPercentage of Peak Bandwitdh: 70.527%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.045 msEffective Bandwitdh: 690.443 GB/sPercentage of Peak Bandwitdh: 73.758%==================================================           Tensor Size: 134217728 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.366 msEffective Bandwitdh: 682.529 GB/sPercentage of Peak Bandwitdh: 72.912%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.722 msEffective Bandwitdh: 692.125 GB/sPercentage of Peak Bandwitdh: 73.937%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 1.422 msEffective Bandwitdh: 703.431 GB/sPercentage of Peak Bandwitdh: 75.145%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 2.824 msEffective Bandwitdh: 708.144 GB/sPercentage of Peak Bandwitdh: 75.649%**************************************************   Custom Device Memcpy 8-Byte Copy Per Thread**************************************************==================================================            Tensor Size: 262144 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.002 msEffective Bandwitdh: 238.792 GB/sPercentage of Peak Bandwitdh: 25.509%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.002 msEffective Bandwitdh: 434.723 GB/sPercentage of Peak Bandwitdh: 46.440%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 681.196 GB/sPercentage of Peak Bandwitdh: 72.770%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.004 msEffective Bandwitdh: 1030.999 GB/sPercentage of Peak Bandwitdh: 110.138%==================================================            Tensor Size: 2097152 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.004 msEffective Bandwitdh: 978.128 GB/sPercentage of Peak Bandwitdh: 104.490%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.012 msEffective Bandwitdh: 677.416 GB/sPercentage of Peak Bandwitdh: 72.366%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.022 msEffective Bandwitdh: 696.748 GB/sPercentage of Peak Bandwitdh: 74.431%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.042 msEffective Bandwitdh: 738.924 GB/sPercentage of Peak Bandwitdh: 78.937%==================================================           Tensor Size: 134217728 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.320 msEffective Bandwitdh: 781.750 GB/sPercentage of Peak Bandwitdh: 83.512%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.636 msEffective Bandwitdh: 786.536 GB/sPercentage of Peak Bandwitdh: 84.023%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 1.265 msEffective Bandwitdh: 790.547 GB/sPercentage of Peak Bandwitdh: 84.451%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 2.530 msEffective Bandwitdh: 790.419 GB/sPercentage of Peak Bandwitdh: 84.438%**************************************************   Custom Device Memcpy 16-Byte Copy Per Thread**************************************************==================================================            Tensor Size: 262144 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.002 msEffective Bandwitdh: 216.744 GB/sPercentage of Peak Bandwitdh: 23.154%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.002 msEffective Bandwitdh: 414.641 GB/sPercentage of Peak Bandwitdh: 44.295%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.002 msEffective Bandwitdh: 829.282 GB/sPercentage of Peak Bandwitdh: 88.589%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 1192.093 GB/sPercentage of Peak Bandwitdh: 127.347%==================================================            Tensor Size: 2097152 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.003 msEffective Bandwitdh: 1128.192 GB/sPercentage of Peak Bandwitdh: 120.521%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.010 msEffective Bandwitdh: 755.386 GB/sPercentage of Peak Bandwitdh: 80.695%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 0.023 msEffective Bandwitdh: 687.333 GB/sPercentage of Peak Bandwitdh: 73.425%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 0.043 msEffective Bandwitdh: 728.343 GB/sPercentage of Peak Bandwitdh: 77.806%==================================================           Tensor Size: 134217728 Units==================================================--------------------------------------------------                Unit Size: 1 Byte--------------------------------------------------Latency: 0.321 msEffective Bandwitdh: 779.006 GB/sPercentage of Peak Bandwitdh: 83.219%--------------------------------------------------                Unit Size: 2 Byte--------------------------------------------------Latency: 0.639 msEffective Bandwitdh: 782.639 GB/sPercentage of Peak Bandwitdh: 83.607%--------------------------------------------------                Unit Size: 4 Byte--------------------------------------------------Latency: 1.280 msEffective Bandwitdh: 781.520 GB/sPercentage of Peak Bandwitdh: 83.487%--------------------------------------------------                Unit Size: 8 Byte--------------------------------------------------Latency: 2.552 msEffective Bandwitdh: 783.602 GB/sPercentage of Peak Bandwitdh: 83.710%

Conclusions

We could see from the results that:

Note that even though we could just use the CUDA official memcpy for this use case, it is still good to know how to write and improve a custom device memcpy function, because in more practical CUDA applications, the data to copy may not be contiguous in memory, and we may need to copy data from multiple sources to multiple destinations.

References

CUDA Vectorized Memory Access

https://leimao.github.io/blog/CUDA-Vectorized-Memory-Access/

Author

Lei Mao

Posted on

01-14-2024

Updated on

01-14-2024

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV: