CUDA Coalesced Memory Access

Introduction

In CUDA programming, accessing the GPU global memory from a CUDA kernel is usually a factor that will affect the CUDA kernel performance. To reduce global memory IO, we would like to reduce the number of global memory access by coalescing the global memory access and cache the reusable data in the fast shared memory.

In this blog post, I would like to discuss how to coalesce the GPU global memory read and write and use an example to show the performance improvement brought by coalescing both the global memory read and write.

CUDA Matrix Transpose

Implementations

In the following example, I implemented three CUDA kernels for (out-of-place) matrix transpose.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315

#include <algorithm>#include <cassert>#include <chrono>#include <cstdio>#include <functional>#include <iomanip>#include <iostream>#include <random>#include <vector>#include <cuda_runtime.h>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, char const* func, char const* file, int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() check_last(__FILE__, __LINE__)void check_last(char const* file, int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}template <class T>float measure_performance(std::function<T(cudaStream_t)> bound_function,                          cudaStream_t stream, size_t num_repeats = 100,                          size_t num_warmups = 100){    cudaEvent_t start, stop;    float time;    CHECK_CUDA_ERROR(cudaEventCreate(&start));    CHECK_CUDA_ERROR(cudaEventCreate(&stop));    for (size_t i{0}; i < num_warmups; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaEventRecord(start, stream));    for (size_t i{0}; i < num_repeats; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaEventRecord(stop, stream));    CHECK_CUDA_ERROR(cudaEventSynchronize(stop));    CHECK_LAST_CUDA_ERROR();    CHECK_CUDA_ERROR(cudaEventElapsedTime(&time, start, stop));    CHECK_CUDA_ERROR(cudaEventDestroy(start));    CHECK_CUDA_ERROR(cudaEventDestroy(stop));    float const latency{time / num_repeats};    return latency;}constexpr size_t div_up(size_t a, size_t b) { return (a + b - 1) / b; }template <typename T>__global__ void transpose_read_coalesced(T* output_matrix,                                         T const* input_matrix, size_t M,                                         size_t N){    size_t const j{threadIdx.x + blockIdx.x * blockDim.x};    size_t const i{threadIdx.y + blockIdx.y * blockDim.y};    size_t const from_idx{i * N + j};    if ((i < M) && (j < N))    {        size_t const to_idx{j * M + i};        output_matrix[to_idx] = input_matrix[from_idx];    }}template <typename T>__global__ void transpose_write_coalesced(T* output_matrix,                                          T const* input_matrix, size_t M,                                          size_t N){    size_t const j{threadIdx.x + blockIdx.x * blockDim.x};    size_t const i{threadIdx.y + blockIdx.y * blockDim.y};    size_t const to_idx{i * M + j};    if ((i < N) && (j < M))    {        size_t const from_idx{j * N + i};        output_matrix[to_idx] = input_matrix[from_idx];    }}template <typename T>void launch_transpose_read_coalesced(T* output_matrix, T const* input_matrix,                                     size_t M, size_t N, cudaStream_t stream){    constexpr size_t const warp_size{32};    dim3 const threads_per_block{warp_size, warp_size};    dim3 const blocks_per_grid{static_cast<unsigned int>(div_up(N, warp_size)),                               static_cast<unsigned int>(div_up(M, warp_size))};    transpose_read_coalesced<<<blocks_per_grid, threads_per_block, 0, stream>>>(        output_matrix, input_matrix, M, N);    CHECK_LAST_CUDA_ERROR();}template <typename T>void launch_transpose_write_coalesced(T* output_matrix, T const* input_matrix,                                      size_t M, size_t N, cudaStream_t stream){    constexpr size_t const warp_size{32};    dim3 const threads_per_block{warp_size, warp_size};    dim3 const blocks_per_grid{static_cast<unsigned int>(div_up(M, warp_size)),                               static_cast<unsigned int>(div_up(N, warp_size))};    transpose_write_coalesced<<<blocks_per_grid, threads_per_block, 0,                                stream>>>(output_matrix, input_matrix, M, N);    CHECK_LAST_CUDA_ERROR();}template <typename T, size_t BLOCK_SIZE = 32>__global__ void transpose_read_write_coalesced(T* output_matrix,                                               T const* input_matrix, size_t M,                                               size_t N){    // BLOCK_SIZE + 1 for avoiding the shared memory bank conflicts.    // https://leimao.github.io/blog/CUDA-Shared-Memory-Bank/    // Try setting it to BLOCK_SIZE instead of BLOCK_SIZE + 1 to see the    // performance drop.    __shared__ T buffer[BLOCK_SIZE][BLOCK_SIZE + 1];    size_t const matrix_j{threadIdx.x + blockIdx.x * blockDim.x};    size_t const matrix_i{threadIdx.y + blockIdx.y * blockDim.y};    size_t const matrix_from_idx{matrix_i * N + matrix_j};    // We have two ways to write matrix data to the shared memory.    // 1. Write transposed matrix data from the DRAM to the shared memory and    // write the non-transposed matrix data from the shared memory to DRAM.    // 2. Write non-transposed matrix data from the DRAM to the shared memory    // and write the transposed matrix data from the shared memory to DRAM. Both    // should result in the same performance, even if there are shared memory    // access bank conflicts.    if ((matrix_i < M) && (matrix_j < N))    {        // The first approach.        buffer[threadIdx.x][threadIdx.y] = input_matrix[matrix_from_idx];        // The second approach.        // buffer[threadIdx.y][threadIdx.x] = input_matrix[matrix_from_idx];    }    // Make sure the buffer in a block is filled.    __syncthreads();    size_t const matrix_transposed_j{threadIdx.x + blockIdx.y * blockDim.y};    size_t const matrix_transposed_i{threadIdx.y + blockIdx.x * blockDim.x};    if ((matrix_transposed_i < N) && (matrix_transposed_j < M))    {        size_t const to_idx{matrix_transposed_i * M + matrix_transposed_j};        // The first approach.        output_matrix[to_idx] = buffer[threadIdx.y][threadIdx.x];        // The second approach.        // output_matrix[to_idx] = buffer[threadIdx.x][threadIdx.y];    }}template <typename T>void launch_transpose_read_write_coalesced(T* output_matrix,                                           T const* input_matrix, size_t M,                                           size_t N, cudaStream_t stream){    constexpr size_t const warp_size{32};    dim3 const threads_per_block{warp_size, warp_size};    dim3 const blocks_per_grid{static_cast<unsigned int>(div_up(N, warp_size)),                               static_cast<unsigned int>(div_up(M, warp_size))};    transpose_read_write_coalesced<T, warp_size>        <<<blocks_per_grid, threads_per_block, 0, stream>>>(output_matrix,                                                            input_matrix, M, N);    CHECK_LAST_CUDA_ERROR();}template <typename T>bool is_equal(T const* data_1, T const* data_2, size_t size){    for (size_t i{0}; i < size; ++i)    {        if (data_1[i] != data_2[i])        {            return false;        }    }    return true;}template <typename T>bool verify_transpose_implementation(    std::function<void(T*, T const*, size_t, size_t, cudaStream_t)>        transpose_function,    size_t M, size_t N){    // Fixed random seed for reproducibility    std::mt19937 gen{0};    cudaStream_t stream;    size_t const matrix_size{M * N};    std::vector<T> matrix(matrix_size, 0.0f);    std::vector<T> matrix_transposed(matrix_size, 1.0f);    std::vector<T> matrix_transposed_reference(matrix_size, 2.0f);    std::uniform_real_distribution<T> uniform_dist(-256, 256);    for (size_t i{0}; i < matrix_size; ++i)    {        matrix[i] = uniform_dist(gen);    }    // Create the reference transposed matrix using CPU.    for (size_t i{0}; i < M; ++i)    {        for (size_t j{0}; j < N; ++j)        {            size_t const from_idx{i * N + j};            size_t const to_idx{j * M + i};            matrix_transposed_reference[to_idx] = matrix[from_idx];        }    }    T* d_matrix;    T* d_matrix_transposed;    CHECK_CUDA_ERROR(cudaMalloc(&d_matrix, matrix_size * sizeof(T)));    CHECK_CUDA_ERROR(cudaMalloc(&d_matrix_transposed, matrix_size * sizeof(T)));    CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    CHECK_CUDA_ERROR(cudaMemcpy(d_matrix, matrix.data(),                                matrix_size * sizeof(T),                                cudaMemcpyHostToDevice));    transpose_function(d_matrix_transposed, d_matrix, M, N, stream);    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaMemcpy(matrix_transposed.data(), d_matrix_transposed,                                matrix_size * sizeof(T),                                cudaMemcpyDeviceToHost));    bool const correctness{is_equal(matrix_transposed.data(),                                    matrix_transposed_reference.data(),                                    matrix_size)};    CHECK_CUDA_ERROR(cudaFree(d_matrix));    CHECK_CUDA_ERROR(cudaFree(d_matrix_transposed));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));    return correctness;}template <typename T>void profile_transpose_implementation(    std::function<void(T*, T const*, size_t, size_t, cudaStream_t)>        transpose_function,    size_t M, size_t N){    constexpr int num_repeats{100};    constexpr int num_warmups{10};    cudaStream_t stream;    size_t const matrix_size{M * N};    T* d_matrix;    T* d_matrix_transposed;    CHECK_CUDA_ERROR(cudaMalloc(&d_matrix, matrix_size * sizeof(T)));    CHECK_CUDA_ERROR(cudaMalloc(&d_matrix_transposed, matrix_size * sizeof(T)));    CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    std::function<void(cudaStream_t)> const transpose_function_wrapped{        std::bind(transpose_function, d_matrix_transposed, d_matrix, M, N,                  std::placeholders::_1)};    float const transpose_function_latency{measure_performance(        transpose_function_wrapped, stream, num_repeats, num_warmups)};    std::cout << std::fixed << std::setprecision(3)              << "Latency: " << transpose_function_latency << " ms"              << std::endl;    CHECK_CUDA_ERROR(cudaFree(d_matrix));    CHECK_CUDA_ERROR(cudaFree(d_matrix_transposed));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));}int main(){    // Unit tests.    for (size_t m{1}; m <= 64; ++m)    {        for (size_t n{1}; n <= 64; ++n)        {            assert(verify_transpose_implementation<float>(                &launch_transpose_write_coalesced<float>, m, n));            assert(verify_transpose_implementation<float>(                &launch_transpose_read_coalesced<float>, m, n));            assert(verify_transpose_implementation<float>(                &launch_transpose_read_write_coalesced<float>, m, n));        }    }    // M: Number of rows.    size_t const M{12800};    // N: Number of columns.    size_t const N{12800};    std::cout << M << " x " << N << " Matrix" << std::endl;    std::cout << "Transpose Write Coalesced" << std::endl;    profile_transpose_implementation<float>(        &launch_transpose_write_coalesced<float>, M, N);    std::cout << "Transpose Read Coalesced" << std::endl;    profile_transpose_implementation<float>(        &launch_transpose_read_coalesced<float>, M, N);    std::cout << "Transpose Read and Write Coalesced" << std::endl;    profile_transpose_implementation<float>(        &launch_transpose_read_write_coalesced<float>, M, N);}

Performance

The performances of the three CUDA kernels were measured using a $12800 \times 12800$ matrix. The reason why we used a square matrix for perfiormance measurement is that we want to compare the performance of global memory coalesced read and coalesced write as fair as possible.

Using -Xptxas -O0, we could disable all the NVCC compiler optimizations for the CUDA kernel. We could see that the kernel with global memory coalesced write is much faster than the kernel with global memory coalesced read, at least for this use case. By enabling both global memory coalesced read and write in the kernel, the kernel performance is the best among all the three kernels.

123456789

$ nvcc transpose.cu -o transpose -Xptxas -O0$ ./transpose12800 x 12800 MatrixTranspose Write CoalescedLatency: 5.220 msTranspose Read CoalescedLatency: 7.624 msTranspose Read and Write CoalescedLatency: 4.804 ms

Using -Xptxas -O3, which is the compiler default, we could enable all the NVCC compiler optimizations for the CUDA kernel. In this case, the CUDA kernel performance order of the three kernels remains the same.

123456789

$ nvcc transpose.cu -o transpose -Xptxas -O3$ ./transpose12800 x 12800 MatrixTranspose Write CoalescedLatency: 2.924 msTranspose Read CoalescedLatency: 5.337 msTranspose Read and Write CoalescedLatency: 2.345 ms

All the measurements were performed on a platform that has an Intel i9-9900K CPU and an NVIDIA RTX 3090 GPU.

Conclusions

In the CUDA kernel implementation, we should try to coalesce both the global memory read and write whenever it’s possible.

References

CUDA Coalesced Memory Access

https://leimao.github.io/blog/CUDA-Coalesced-Memory-Access/

Author

Lei Mao

Posted on

03-19-2023

Updated on

03-19-2023

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


CUDA Zero Copy Mapped Memory

Introduction

Unified memory is used on NVIDIA embedding platforms, such as NVIDIA Drive series and NVIDIA Jetson series. Since the same memory is used for both the CPU and the integrated GPU, it is possible to eliminate the CUDA memory copy between host and device that normally happens on a system that uses discrete GPU so that the GPU can directly the access the outputs from CPU and the CPU can also directly access the outputs from GPU. In this way, the system performance could be improved significantly in some use cases.

In this blog post, I would like to discuss the CUDA mapped pinned memory versus CUDA non-mapped pinned memory and compare their performance on memory bound kernels.

CUDA Pinned Mapped Memory

CUDA pinned mapped memory enables GPU threads to directly access host memory. For this purpose, it requires mapped pinned (non-pageable, page-locked) memory. On integrated GPUs (i.e., GPUs with the integrated field of the CUDA device properties structure set to 1), mapped pinned memory is always a performance gain because it avoids superfluous copies as integrated GPU and CPU memory are physically the same. On discrete GPUs, mapped pinned memory is advantageous only in certain cases. Because the data is not cached on the GPU, mapped pinned memory should be read or written only once, and the global loads and stores that read and write the memory should be coalesced. Zero copy can be used in place of streams because kernel-originated data transfers automatically overlap kernel execution without the overhead of setting up and determining the optimal number of streams.

CUDA Pinned Memory Non-Mapped VS Mapped

The following implementation compares the latency of a memory-bound kernel and its memory copy between host and device if necessary.

CUDA mapped memory also uses pinned memory. For CUDA pinned memory, we still need to allocate device memory and transfer the data between the host memory and the device memory, whereas for CUDA mapped memory, the device memory allocation and memory transfer, if there is any, are abstracted.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229

#include <cassert>#include <chrono>#include <functional>#include <iomanip>#include <iostream>#include <stdexcept>#include <thread>#include <tuple>#include <utility>#include <vector>#include <cuda_runtime.h>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, const char* const func, const char* const file,           const int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() checkLast(__FILE__, __LINE__)void checkLast(const char* const file, const int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}template <class T>float measure_performance(std::function<T(cudaStream_t)> bound_function,                          cudaStream_t stream, int num_repeats = 100,                          int num_warmups = 100){    cudaEvent_t start, stop;    float time;    CHECK_CUDA_ERROR(cudaEventCreate(&start));    CHECK_CUDA_ERROR(cudaEventCreate(&stop));    for (int i{0}; i < num_warmups; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaEventRecord(start, stream));    for (int i{0}; i < num_repeats; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaEventRecord(stop, stream));    CHECK_CUDA_ERROR(cudaEventSynchronize(stop));    CHECK_LAST_CUDA_ERROR();    CHECK_CUDA_ERROR(cudaEventElapsedTime(&time, start, stop));    CHECK_CUDA_ERROR(cudaEventDestroy(start));    CHECK_CUDA_ERROR(cudaEventDestroy(stop));    float const latency{time / num_repeats};    return latency;}__global__ void float_addition(float* output, float const* input_1,                               float const* input_2, uint32_t n){    const uint32_t idx{blockDim.x * blockIdx.x + threadIdx.x};    const uint32_t stride{blockDim.x * gridDim.x};    for (uint32_t i{idx}; i < n; i += stride)    {        output[i] = input_1[i] + input_2[i];    }}void launch_float_addition_non_mapped_pinned_memory(    float* h_output, float const* h_input_1, float const* h_input_2,    float* d_output, float* d_input_1, float* d_input_2, uint32_t n,    cudaStream_t stream){    CHECK_CUDA_ERROR(cudaMemcpyAsync(d_input_1, h_input_1, n * sizeof(float),                                     cudaMemcpyHostToDevice, stream));    CHECK_CUDA_ERROR(cudaMemcpyAsync(d_input_2, h_input_2, n * sizeof(float),                                     cudaMemcpyHostToDevice, stream));    dim3 const threads_per_block{1024};    dim3 const blocks_per_grid{32};    float_addition<<<blocks_per_grid, threads_per_block, 0, stream>>>(        d_output, d_input_1, d_input_2, n);    CHECK_LAST_CUDA_ERROR();    CHECK_CUDA_ERROR(cudaMemcpyAsync(h_output, d_output, n * sizeof(float),                                     cudaMemcpyDeviceToHost, stream));}void launch_float_addition_mapped_pinned_memory(float* d_output,                                                float* d_input_1,                                                float* d_input_2, uint32_t n,                                                cudaStream_t stream){    dim3 const threads_per_block{1024};    dim3 const blocks_per_grid{32};    float_addition<<<blocks_per_grid, threads_per_block, 0, stream>>>(        d_output, d_input_1, d_input_2, n);    CHECK_LAST_CUDA_ERROR();}void initialize_host_memory(float* h_buffer, uint32_t n, float value){    for (int i{0}; i < n; ++i)    {        h_buffer[i] = value;    }}bool verify_host_memory(float* h_buffer, uint32_t n, float value){    for (int i{0}; i < n; ++i)    {        if (h_buffer[i] != value)        {            return false;        }    }    return true;}int main(){    constexpr int const num_repeats{10};    constexpr int const num_warmups{10};    constexpr int const n{1000000};    cudaStream_t stream;    CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    float const v_input_1{1.0f};    float const v_input_2{1.0f};    float const v_output{0.0f};    float const v_output_reference{v_input_1 + v_input_2};    cudaDeviceProp prop;    CHECK_CUDA_ERROR(cudaGetDeviceProperties(&prop, 0));    if (!prop.canMapHostMemory)    {        throw std::runtime_error{"Device does not supported mapped memory."};    }    float *h_input_1, *h_input_2, *h_output;    float *d_input_1, *d_input_2, *d_output;    float *a_input_1, *a_input_2, *a_output;    float *m_input_1, *m_input_2, *m_output;    CHECK_CUDA_ERROR(cudaMallocHost(&h_input_1, n * sizeof(float)));    CHECK_CUDA_ERROR(cudaMallocHost(&h_input_2, n * sizeof(float)));    CHECK_CUDA_ERROR(cudaMallocHost(&h_output, n * sizeof(float)));    CHECK_CUDA_ERROR(cudaMalloc(&d_input_1, n * sizeof(float)));    CHECK_CUDA_ERROR(cudaMalloc(&d_input_2, n * sizeof(float)));    CHECK_CUDA_ERROR(cudaMalloc(&d_output, n * sizeof(float)));    CHECK_CUDA_ERROR(        cudaHostAlloc(&a_input_1, n * sizeof(float), cudaHostAllocMapped));    CHECK_CUDA_ERROR(        cudaHostAlloc(&a_input_2, n * sizeof(float), cudaHostAllocMapped));    CHECK_CUDA_ERROR(        cudaHostAlloc(&a_output, n * sizeof(float), cudaHostAllocMapped));    CHECK_CUDA_ERROR(cudaHostGetDevicePointer(&m_input_1, a_input_1, 0));    CHECK_CUDA_ERROR(cudaHostGetDevicePointer(&m_input_2, a_input_2, 0));    CHECK_CUDA_ERROR(cudaHostGetDevicePointer(&m_output, a_output, 0));    // Verify the implementation correctness.    initialize_host_memory(h_input_1, n, v_input_1);    initialize_host_memory(h_input_2, n, v_input_2);    initialize_host_memory(h_output, n, v_output);    launch_float_addition_non_mapped_pinned_memory(        h_output, h_input_1, h_input_2, d_output, d_input_1, d_input_2, n,        stream);    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    assert(verify_host_memory(h_output, n, v_output_reference));    initialize_host_memory(a_input_1, n, v_input_1);    initialize_host_memory(a_input_2, n, v_input_2);    initialize_host_memory(a_output, n, v_output);    launch_float_addition_mapped_pinned_memory(m_output, m_input_1, m_input_2,                                               n, stream);    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    assert(verify_host_memory(a_output, n, v_output_reference));    // Measure latencies.    std::function<void(cudaStream_t)> function_non_mapped_pinned_memory{        std::bind(launch_float_addition_non_mapped_pinned_memory, h_output,                  h_input_1, h_input_2, d_output, d_input_1, d_input_2, n,                  std::placeholders::_1)};    std::function<void(cudaStream_t)> function_mapped_pinned_memory{        std::bind(launch_float_addition_mapped_pinned_memory, m_output,                  m_input_1, m_input_2, n, std::placeholders::_1)};    float const latency_non_mapped_pinned_memory{measure_performance(        function_non_mapped_pinned_memory, stream, num_repeats, num_warmups)};    float const latency_mapped_pinned_memory{measure_performance(        function_mapped_pinned_memory, stream, num_repeats, num_warmups)};    std::cout << std::fixed << std::setprecision(3)              << "CUDA Kernel With Non-Mapped Pinned Memory Latency: "              << latency_non_mapped_pinned_memory << " ms" << std::endl;    std::cout << std::fixed << std::setprecision(3)              << "CUDA Kernel With Mapped Pinned Memory Latency: "              << latency_mapped_pinned_memory << " ms" << std::endl;    CHECK_CUDA_ERROR(cudaFree(d_input_1));    CHECK_CUDA_ERROR(cudaFree(d_input_2));    CHECK_CUDA_ERROR(cudaFree(d_output));    CHECK_CUDA_ERROR(cudaFreeHost(h_input_1));    CHECK_CUDA_ERROR(cudaFreeHost(h_input_2));    CHECK_CUDA_ERROR(cudaFreeHost(h_output));    CHECK_CUDA_ERROR(cudaFreeHost(a_input_1));    CHECK_CUDA_ERROR(cudaFreeHost(a_input_2));    CHECK_CUDA_ERROR(cudaFreeHost(a_output));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));}

Discrete GPU

This is the latency profiling on a desktop that has Intel Core i9-9900K CPU and NVIDIA RTX 3090 GPU.

1234

$ nvcc mapped_memory.cu -o mapped_memory -std=c++14$ ./mapped_memoryCUDA Kernel With Non-Mapped Pinned Memory Latency: 0.964 msCUDA Kernel With Mapped Pinned Memory Latency: 0.631 ms

We could see that for memory-bound kernel, on a platform that uses discrete GPU, separate host memory, and device memory, using mapped pinned memory is almost 30% faster than using non-mapped pinned memory.

Integrated GPU

This is the latency profiling on an NVIDIA Jetson Xavier.

1234

$ nvcc mapped_memory.cu -o mapped_memory -std=c++14$ ./mapped_memoryCUDA Kernel With Non-Mapped Pinned Memory Latency: 2.343 msCUDA Kernel With Mapped Pinned Memory Latency: 0.431 ms

We could see that for memory-bound kernel, on a platform that uses integrated GPU and unified memory, using mapped pinned memory is almost 6x faster than using non-mapped pinned memory. This is because the using mapped memory truly eliminated the memory copy between host and device on unified memory.

Caveats

CUDA zero copy memory disables data cache on GPUs, there might be performance drop for math bound kernels.

References

CUDA Zero Copy Mapped Memory

https://leimao.github.io/blog/CUDA-Zero-Copy-Mapped-Memory/

Author

Lei Mao

Posted on

12-16-2022

Updated on

12-16-2022

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


CUDA Data Alignment

Introduction

In order to get the best performance, similar to the data alignment requirement in C++, CUDA also requires data alignment.

In this blog post, I would like to quickly discuss the data alignment requirement in CUDA.

Coalesced Access to Global Memory

Global memory resides in device memory and device memory is accessed via 32-, 64-, or 128-byte memory transactions. These memory transactions must be naturally aligned: Only the 32-, 64-, or 128-byte segments of device memory that are aligned to their size (i.e., whose first address is a multiple of their size) can be read or written by memory transactions.

When a warp executes an instruction that accesses global memory, it coalesces the memory accesses of the threads within the warp into one or more of these memory transactions depending on the size of the word accessed by each thread and the distribution of the memory addresses across the threads. In general, the more transactions are necessary, the more unused words are transferred in addition to the words accessed by the threads, reducing the instruction throughput accordingly.

For devices of compute capability 6.0 or higher, the requirements can be summarized quite easily: the concurrent accesses of the threads of a warp will coalesce into a number of transactions equal to the number of 32-byte transactions necessary to service all of the threads of the warp.

Any address of a variable residing in global memory or returned by one of the memory allocation routines from the driver or runtime API, such as cudaMalloc or cudaMallocPitch, is always aligned to at least 256 bytes.

Example

For example, if the each of the thread in a warp that consists of 32 threads wants to read a 4-byte data, and if the 4-byte data from all the threads in the warp (128-byte data) are adjacent to each other and 32-byte aligned, i.e., the the address of the first 4-byte data is a multiple of 32, the memory access is coalesced and GPU will make $\frac{4 \times 32}{32} = 4$ 32-byte memory transactions. Maximum memory transaction throughput is achieved because GPU made the fewest transactions possible.

If the 128-byte data is not 32-byte aligned on the memory, say it is 4-byte aligned instead, one additional 32-byte memory transactions will have to be made, and therefore the memory access throughput becomes $\frac{4}{5} = 80\%$ of the maximum theoretical throughput (speed of light).

Furthermore, if the 4-byte data from all the threads are not adjacent to each other and are scattered sparsely on the memory, it is possible that at most 32 32-byte memory transactions will have to be made, and the throughput becomes only $\frac{4}{32} = 12.5\%$ of the maximum theoretical throughput.

Size and Alignment Requirement

Global memory instructions support reading or writing words of size equal to 1, 2, 4, 8, or 16 bytes. Any access (via a variable or a pointer) to data residing in global memory compiles to a single global memory instruction if and only if the size of the data type is 1, 2, 4, 8, or 16 bytes and the data is naturally aligned (i.e., its address is a multiple of that size).

If this size and alignment requirement is not fulfilled, the access compiles to multiple instructions with interleaved access patterns that prevent these instructions from fully coalescing. It is therefore recommended to use types that meet this requirement for data that resides in global memory.

Reading non-naturally aligned 8-byte or 16-byte words produces incorrect results (off by a few words), so special care must be taken to maintain alignment of the starting address of any value or array of values of these types.

Therefore, working with word of size equal to 1, 2, 4, 8, or 16 bytes is sometimes straightforward because as mentioned above the starting memory address returned by the memory allocation CUDA APIs is always aligned to at least 256 bytes, which is already 1, 2, 4, 8, or 16 byte-aligned. So we could safely save the word sequence, such as a numerical array, matrix, or tensor, into the allocated memory without having to worry about the reading of the words of 8-byte or 16-byte size produces incorrect results. To achieve the best memory access throughput, special attention is paid to the kernel implementation so that the coalesced memory access is also naturally aligned.

But, what if the word size is not 1, 2, 4, 8, or 16 bytes? The starting memory address returned by the memory allocation CUDA APIs will not guarantee that it is naturally aligned, and therefore the memory access throughput would be compromised significantly. There are usually two ways to handle this.

12345678910111213

struct __align__(4) int8_3_4_t{    int8_t x;    int8_t y;    int8_t z;};struct __align__(16) float3_16_t{    float x;    float y;    float z;};

Conclusions

Always making the word size equal to 1, 2, 4, 8, or 16 bytes and the data naturally aligned.

Reading words and producing incorrect results rarely happen if the memory allocated is only used for a sequence of words of the same type, because the starting memory address returned by the memory allocation CUDA APIs is always aligned to at least 256 bytes. However, if one single piece of large memory is allocated for multiple sequences of words of different types with or without paddings, special care must be taken to maintain alignment of the starting address of any word or word sequence as it may produce incorrect result (for non-naturally aligned 8-byte or 16-byte words).

References

CUDA Data Alignment

https://leimao.github.io/blog/CUDA-Data-Alignment/

Author

Lei Mao

Posted on

10-18-2022

Updated on

10-18-2022

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


CUDA L2 Persistent Cache

Introduction

Starting with CUDA 11.0, devices of compute capability 8.0 and above have the capability to influence persistence of data in the L2 cache. Because L2 cache is on-chip, it potentially provides higher bandwidth and lower latency accesses to global memory.

In this blog post, I created a CUDA example to demonstrate the how to use the L2 persistent cache to accelerate the data traffic.

CUDA L2 Persistent Cache

In this example, I would have a small constant buffer of certain values that will be used for resetting a large streaming buffer. For example, if the constant buffer is of size 4 and has values of [5, 2, 1, 4] and the large streaming buffer to be reset is of size 100, after resetting the large streaming buffer will have values of [5, 2, 1, 4, 5, 2, 1, 4, ...], namely repeating the values of the constant buffer.

Because the streaming buffer is much larger than the constant buffer, each element from the constant buffer is accessed more often than the streaming buffer. Accessing buffer from global memory is very expensive. If we could cache the frequently accessed constant buffer in L2 cache, the access to the frequently accessed constant buffer could be accelerated.

CUDA Data Resetting

For the data resetting CUDA kernel, I created a baseline which launches the kernel without using persistent L2 cache, a variant which launches the kernel using 3 MB persistent L2 cache but has data thrashing when the constant buffer size exceeds 3 MB, and a optimized variant which launches the kernel using 3 MB persistent L2 cache but the data thrashing was eliminated.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246

#include <algorithm>#include <cassert>#include <cstdlib>#include <functional>#include <iomanip>#include <iostream>#include <vector>#include <cuda_runtime.h>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, char const* const func, char const* const file,           int const line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() checkLast(__FILE__, __LINE__)void checkLast(char const* const file, int const line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}template <class T>float measure_performance(std::function<T(cudaStream_t)> bound_function,                          cudaStream_t stream, int num_repeats = 100,                          int num_warmups = 100){    cudaEvent_t start, stop;    float time;    CHECK_CUDA_ERROR(cudaEventCreate(&start));    CHECK_CUDA_ERROR(cudaEventCreate(&stop));    for (int i{0}; i < num_warmups; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaEventRecord(start, stream));    for (int i{0}; i < num_repeats; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaEventRecord(stop, stream));    CHECK_CUDA_ERROR(cudaEventSynchronize(stop));    CHECK_LAST_CUDA_ERROR();    CHECK_CUDA_ERROR(cudaEventElapsedTime(&time, start, stop));    CHECK_CUDA_ERROR(cudaEventDestroy(start));    CHECK_CUDA_ERROR(cudaEventDestroy(stop));    float const latency{time / num_repeats};    return latency;}__global__ void reset_data(int* data_streaming, int const* lut_persistent,                           size_t data_streaming_size,                           size_t lut_persistent_size){    size_t const idx{blockDim.x * blockIdx.x + threadIdx.x};    size_t const stride{blockDim.x * gridDim.x};    for (size_t i{idx}; i < data_streaming_size; i += stride)    {        data_streaming[i] = lut_persistent[i % lut_persistent_size];    }}/** * @brief Reset the data_streaming using lut_persistent so that the * data_streaming is lut_persistent repeatedly. * * @param data_streaming The data for reseting. * @param lut_persistent The values for resetting data_streaming. * @param data_streaming_size The size for data_streaming. * @param lut_persistent_size The size for lut_persistent. * @param stream The CUDA stream. */void launch_reset_data(int* data_streaming, int const* lut_persistent,                       size_t data_streaming_size, size_t lut_persistent_size,                       cudaStream_t stream){    dim3 const threads_per_block{1024};    dim3 const blocks_per_grid{32};    reset_data<<<blocks_per_grid, threads_per_block, 0, stream>>>(        data_streaming, lut_persistent, data_streaming_size,        lut_persistent_size);    CHECK_LAST_CUDA_ERROR();}bool verify_data(int* data, int n, size_t size){    for (size_t i{0}; i < size; ++i)    {        if (data[i] != i % n)        {            return false;        }    }    return true;}int main(int argc, char* argv[]){    size_t num_megabytes_persistent_data{3};    if (argc == 2)    {        num_megabytes_persistent_data = std::atoi(argv[1]);    }    constexpr int const num_repeats{100};    constexpr int const num_warmups{10};    cudaDeviceProp device_prop{};    int current_device{0};    CHECK_CUDA_ERROR(cudaGetDevice(&current_device));    CHECK_CUDA_ERROR(cudaGetDeviceProperties(&device_prop, current_device));    std::cout << "GPU: " << device_prop.name << std::endl;    std::cout << "L2 Cache Size: " << device_prop.l2CacheSize / 1024 / 1024              << " MB" << std::endl;    std::cout << "Max Persistent L2 Cache Size: "              << device_prop.persistingL2CacheMaxSize / 1024 / 1024 << " MB"              << std::endl;    size_t const num_megabytes_streaming_data{1024};    if (num_megabytes_persistent_data > num_megabytes_streaming_data)    {        std::runtime_error(            "Try setting persistent data size smaller than 1024 MB.");    }    size_t const size_persistent(num_megabytes_persistent_data * 1024 * 1024 /                                 sizeof(int));    size_t const size_streaming(num_megabytes_streaming_data * 1024 * 1024 /                                sizeof(int));    std::cout << "Persistent Data Size: " << num_megabytes_persistent_data              << " MB" << std::endl;    std::cout << "Steaming Data Size: " << num_megabytes_streaming_data << " MB"              << std::endl;    cudaStream_t stream;    std::vector<int> lut_persistent_vec(size_persistent, 0);    for (size_t i{0}; i < lut_persistent_vec.size(); ++i)    {        lut_persistent_vec[i] = i;    }    std::vector<int> data_streaming_vec(size_streaming, 0);    int* d_lut_persistent;    int* d_data_streaming;    int* h_lut_persistent = lut_persistent_vec.data();    int* h_data_streaming = data_streaming_vec.data();    CHECK_CUDA_ERROR(        cudaMalloc(&d_lut_persistent, size_persistent * sizeof(int)));    CHECK_CUDA_ERROR(        cudaMalloc(&d_data_streaming, size_streaming * sizeof(int)));    CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    CHECK_CUDA_ERROR(cudaMemcpy(d_lut_persistent, h_lut_persistent,                                size_persistent * sizeof(int),                                cudaMemcpyHostToDevice));    launch_reset_data(d_data_streaming, d_lut_persistent, size_streaming,                      size_persistent, stream);    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaMemcpy(h_data_streaming, d_data_streaming,                                size_streaming * sizeof(int),                                cudaMemcpyDeviceToHost));    assert(verify_data(h_data_streaming, size_persistent, size_streaming));    std::function<void(cudaStream_t)> const function{        std::bind(launch_reset_data, d_data_streaming, d_lut_persistent,                  size_streaming, size_persistent, std::placeholders::_1)};    float const latency{        measure_performance(function, stream, num_repeats, num_warmups)};    std::cout << std::fixed << std::setprecision(3)              << "Latency Without Using Persistent L2 Cache: " << latency              << " ms" << std::endl;    // Start to use persistent cache.    cudaStream_t stream_persistent_cache;    size_t const num_megabytes_persistent_cache{3};    CHECK_CUDA_ERROR(cudaStreamCreate(&stream_persistent_cache));    CHECK_CUDA_ERROR(        cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize,                           num_megabytes_persistent_cache * 1024 * 1024));    cudaStreamAttrValue stream_attribute_thrashing;    stream_attribute_thrashing.accessPolicyWindow.base_ptr =        reinterpret_cast<void*>(d_lut_persistent);    stream_attribute_thrashing.accessPolicyWindow.num_bytes =        num_megabytes_persistent_data * 1024 * 1024;    stream_attribute_thrashing.accessPolicyWindow.hitRatio = 1.0;    stream_attribute_thrashing.accessPolicyWindow.hitProp =        cudaAccessPropertyPersisting;    stream_attribute_thrashing.accessPolicyWindow.missProp =        cudaAccessPropertyStreaming;    CHECK_CUDA_ERROR(cudaStreamSetAttribute(        stream_persistent_cache, cudaStreamAttributeAccessPolicyWindow,        &stream_attribute_thrashing));    float const latency_persistent_cache_thrashing{measure_performance(        function, stream_persistent_cache, num_repeats, num_warmups)};    std::cout << std::fixed << std::setprecision(3) << "Latency With Using "              << num_megabytes_persistent_cache              << " MB Persistent L2 Cache (Potentially Thrashing): "              << latency_persistent_cache_thrashing << " ms" << std::endl;    cudaStreamAttrValue stream_attribute_non_thrashing{        stream_attribute_thrashing};    stream_attribute_non_thrashing.accessPolicyWindow.hitRatio =        std::min(static_cast<double>(num_megabytes_persistent_cache) /                     num_megabytes_persistent_data,                 1.0);    CHECK_CUDA_ERROR(cudaStreamSetAttribute(        stream_persistent_cache, cudaStreamAttributeAccessPolicyWindow,        &stream_attribute_non_thrashing));    float const latency_persistent_cache_non_thrashing{measure_performance(        function, stream_persistent_cache, num_repeats, num_warmups)};    std::cout << std::fixed << std::setprecision(3) << "Latency With Using "              << num_megabytes_persistent_cache              << " MB Persistent L2 Cache (Non-Thrashing): "              << latency_persistent_cache_non_thrashing << " ms" << std::endl;    CHECK_CUDA_ERROR(cudaFree(d_lut_persistent));    CHECK_CUDA_ERROR(cudaFree(d_data_streaming));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream_persistent_cache));}

To avoid data thrashing, the product of accessPolicyWindow.hitRatio and accessPolicyWindow.num_bytes should be less than or equal to the cudaLimitPersistingL2CacheSize. The accessPolicyWindow.hitRatio parameter can be used to specify the fraction of accesses that receive the accessPolicyWindow.hitProp property, which is usually cudaAccessPropertyPersisting. The accessPolicyWindow.num_bytes parameter can be used to specify the number of bytes that the access policy window covers, which is usually the size of the persistent data.

In practice, we could set the accessPolicyWindow.hitRatio to be the ratio of the persistent L2 cache size to the persistent data size. For example, if the the persistent L2 cache size is 3 MB and the persistent data size is 4 MB, we could set the accessPolicyWindow.hitRatio to be 3 / 4 = 0.75.

Run CUDA Data Resetting

We could build and run the example on an NVIDIA Ampere GPU. In my case, I used an NVIDIA RTX 3090 GPU.

12345678910

$ nvcc l2-persistent.cu -o l2-persistent -std=c++14 --gpu-architecture=compute_80$ ./l2-persistentGPU: NVIDIA GeForce RTX 3090L2 Cache Size: 6 MBMax Persistent L2 Cache Size: 4 MBPersistent Data Size: 3 MBSteaming Data Size: 1024 MBLatency Without Using Persistent L2 Cache: 3.071 msLatency With Using 3 MB Persistent L2 Cache (Potentially Thrashing): 2.436 msLatency With Using 3 MB Persistent L2 Cache (Non-Thrashing): 2.443 ms

We could see that when the persistent data size is 3 MB and the persistent L2 cache is 3 MB, the performance of the application is improved by roughly 20%.

Benchmarking

We could also run some mini benchmarking by varying the persistent data size.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354

$ ./l2-persistent 1GPU: NVIDIA GeForce RTX 3090L2 Cache Size: 6 MBMax Persistent L2 Cache Size: 4 MBPersistent Data Size: 1 MBSteaming Data Size: 1024 MBLatency Without Using Persistent L2 Cache: 1.754 msLatency With Using 3 MB Persistent L2 Cache (Potentially Thrashing): 1.685 msLatency With Using 3 MB Persistent L2 Cache (Non-Thrashing): 1.674 ms$ ./l2-persistent 2GPU: NVIDIA GeForce RTX 3090L2 Cache Size: 6 MBMax Persistent L2 Cache Size: 4 MBPersistent Data Size: 2 MBSteaming Data Size: 1024 MBLatency Without Using Persistent L2 Cache: 2.158 msLatency With Using 3 MB Persistent L2 Cache (Potentially Thrashing): 1.997 msLatency With Using 3 MB Persistent L2 Cache (Non-Thrashing): 2.002 ms$ ./l2-persistent 3GPU: NVIDIA GeForce RTX 3090L2 Cache Size: 6 MBMax Persistent L2 Cache Size: 4 MBPersistent Data Size: 3 MBSteaming Data Size: 1024 MBLatency Without Using Persistent L2 Cache: 3.095 msLatency With Using 3 MB Persistent L2 Cache (Potentially Thrashing): 2.510 msLatency With Using 3 MB Persistent L2 Cache (Non-Thrashing): 2.533 ms$ ./l2-persistent 4GPU: NVIDIA GeForce RTX 3090L2 Cache Size: 6 MBMax Persistent L2 Cache Size: 4 MBPersistent Data Size: 4 MBSteaming Data Size: 1024 MBLatency Without Using Persistent L2 Cache: 3.906 msLatency With Using 3 MB Persistent L2 Cache (Potentially Thrashing): 3.632 msLatency With Using 3 MB Persistent L2 Cache (Non-Thrashing): 3.706 ms$ ./l2-persistent 5GPU: NVIDIA GeForce RTX 3090L2 Cache Size: 6 MBMax Persistent L2 Cache Size: 4 MBPersistent Data Size: 5 MBSteaming Data Size: 1024 MBLatency Without Using Persistent L2 Cache: 4.120 msLatency With Using 3 MB Persistent L2 Cache (Potentially Thrashing): 4.554 msLatency With Using 3 MB Persistent L2 Cache (Non-Thrashing): 3.920 ms$ ./l2-persistent 6GPU: NVIDIA GeForce RTX 3090L2 Cache Size: 6 MBMax Persistent L2 Cache Size: 4 MBPersistent Data Size: 6 MBSteaming Data Size: 1024 MBLatency Without Using Persistent L2 Cache: 4.194 msLatency With Using 3 MB Persistent L2 Cache (Potentially Thrashing): 4.583 msLatency With Using 3 MB Persistent L2 Cache (Non-Thrashing): 4.255 ms

We could see that even when the persistent data size is larger than the persistent L2 cache, the latency of using persistent L2 cache that has no-thrashing usually does not perform worse than the baseline.

FAQ

Persistent Cache VS Shared Memory?

The persistent cache is different from the shared memory. The persistent cache is visible to all the threads in the GPU, while the shared memory is only visible to the threads in the same block.

For small-sized frequently accessed data, we could also use the shared memory to accelerate the data access. However, the shared memory is limited to 48 to 96 KB per block of threads, depending on the GPU, while the persistent cache is limited to a few MB per GPU.

References

CUDA L2 Persistent Cache

https://leimao.github.io/blog/CUDA-L2-Persistent-Cache/

Author

Lei Mao

Posted on

09-12-2022

Updated on

11-12-2023

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


CUDA Shared Memory Bank

Introduction

Memory bank is a key concept for CUDA shared memory. To get the best performance out of a CUDA kernel implementation, the user will have to pay attention to memory bank access and avoid memory bank access conflicts.

In this blog post, I would like to quickly discuss memory bank for CUDA shared memory.

Memory Bank

Memory Bank Properties

To achieve high memory bandwidth for concurrent accesses, shared memory is divided into equally sized memory modules (banks) that can be accessed simultaneously. Therefore, any memory load or store of $n$ addresses that spans $n$ distinct memory banks can be serviced simultaneously, yielding an effective bandwidth that is $n$ times as high as the bandwidth of a single bank.

However, if multiple addresses of a memory request map to the same memory bank, the accesses are serialized. The hardware splits a memory request that has bank conflicts into as many separate conflict-free requests as necessary, decreasing the effective bandwidth by a factor equal to the number of separate memory requests. The one exception here is when multiple threads in a warp address the same shared memory location, resulting in a broadcast. In this case, multiple broadcasts from different banks are coalesced into a single multicast from the requested shared memory locations to the threads.

Memory Bank Mapping

The memory bank properties were described above. However, how memory addresses map to memory banks is architecture-specific.

On devices of compute capability 5.x or newer, each bank has a bandwidth of 32 bits every clock cycle, and successive 32-bit words are assigned to successive banks. The warp size is 32 threads and the number of banks is also 32, so bank conflicts can occur between any threads in the warp.

To elaborate on this, let’s see how memory addresses map to memory banks using examples. The following program illustrated the idea of 1D and 2D memory address to memory banks mapping for devices of compute capability 5.x or newer.

12345678910111213141516171819202122232425262728293031323334353637383940414243444546

#include <iostream>#include <memory>#include <vector>template <typename T>void bank_id_1d_mapping(int bank_size, int num_banks, int N){    for (int i{0}; i < N; ++i)    {        // bank_size: Bank size in bits.        // 8: 8 bits per Byte.        int bank_idx = (i * sizeof(T) * 8 / bank_size) % num_banks;        std::cout << "Array Idx: " << i << " "                  << "Bank Idx: " << bank_idx << std::endl;    }}template <typename T>void bank_id_2d_mapping(int bank_size, int num_banks, int M, int N){    for (int i{0}; i < M; ++i)    {        for (int j{0}; j < N; ++j)        {            int bank_idx =                ((i * N + j) * sizeof(T) * 8 / bank_size) % num_banks;            std::cout << "Matrix Idx: (" << i << ", " << j << ") "                      << "Bank Idx: " << bank_idx << std::endl;        }    }}int main(){    constexpr const int bank_size{32}; // bits    constexpr const int num_banks{32};    const int M{4};    const int N{32};    std::cout << "Bank ID Mapping 1D: N = " << N << std::endl;    bank_id_1d_mapping<float>(bank_size, num_banks, N);    std::cout << "Bank 2D Mapping 1D: M = " << M << " N = " << N << std::endl;    bank_id_2d_mapping<float>(bank_size, num_banks, M, N);}

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164

$ g++ memory_bank.cpp -o memory_bank -std=c++14$ ./memory_bankBank ID Mapping 1D: N = 32Array Idx: 0 Bank Idx: 0Array Idx: 1 Bank Idx: 1Array Idx: 2 Bank Idx: 2Array Idx: 3 Bank Idx: 3Array Idx: 4 Bank Idx: 4Array Idx: 5 Bank Idx: 5Array Idx: 6 Bank Idx: 6Array Idx: 7 Bank Idx: 7Array Idx: 8 Bank Idx: 8Array Idx: 9 Bank Idx: 9Array Idx: 10 Bank Idx: 10Array Idx: 11 Bank Idx: 11Array Idx: 12 Bank Idx: 12Array Idx: 13 Bank Idx: 13Array Idx: 14 Bank Idx: 14Array Idx: 15 Bank Idx: 15Array Idx: 16 Bank Idx: 16Array Idx: 17 Bank Idx: 17Array Idx: 18 Bank Idx: 18Array Idx: 19 Bank Idx: 19Array Idx: 20 Bank Idx: 20Array Idx: 21 Bank Idx: 21Array Idx: 22 Bank Idx: 22Array Idx: 23 Bank Idx: 23Array Idx: 24 Bank Idx: 24Array Idx: 25 Bank Idx: 25Array Idx: 26 Bank Idx: 26Array Idx: 27 Bank Idx: 27Array Idx: 28 Bank Idx: 28Array Idx: 29 Bank Idx: 29Array Idx: 30 Bank Idx: 30Array Idx: 31 Bank Idx: 31Bank 2D Mapping 1D: M = 4 N = 32Matrix Idx: (0, 0) Bank Idx: 0Matrix Idx: (0, 1) Bank Idx: 1Matrix Idx: (0, 2) Bank Idx: 2Matrix Idx: (0, 3) Bank Idx: 3Matrix Idx: (0, 4) Bank Idx: 4Matrix Idx: (0, 5) Bank Idx: 5Matrix Idx: (0, 6) Bank Idx: 6Matrix Idx: (0, 7) Bank Idx: 7Matrix Idx: (0, 8) Bank Idx: 8Matrix Idx: (0, 9) Bank Idx: 9Matrix Idx: (0, 10) Bank Idx: 10Matrix Idx: (0, 11) Bank Idx: 11Matrix Idx: (0, 12) Bank Idx: 12Matrix Idx: (0, 13) Bank Idx: 13Matrix Idx: (0, 14) Bank Idx: 14Matrix Idx: (0, 15) Bank Idx: 15Matrix Idx: (0, 16) Bank Idx: 16Matrix Idx: (0, 17) Bank Idx: 17Matrix Idx: (0, 18) Bank Idx: 18Matrix Idx: (0, 19) Bank Idx: 19Matrix Idx: (0, 20) Bank Idx: 20Matrix Idx: (0, 21) Bank Idx: 21Matrix Idx: (0, 22) Bank Idx: 22Matrix Idx: (0, 23) Bank Idx: 23Matrix Idx: (0, 24) Bank Idx: 24Matrix Idx: (0, 25) Bank Idx: 25Matrix Idx: (0, 26) Bank Idx: 26Matrix Idx: (0, 27) Bank Idx: 27Matrix Idx: (0, 28) Bank Idx: 28Matrix Idx: (0, 29) Bank Idx: 29Matrix Idx: (0, 30) Bank Idx: 30Matrix Idx: (0, 31) Bank Idx: 31Matrix Idx: (1, 0) Bank Idx: 0Matrix Idx: (1, 1) Bank Idx: 1Matrix Idx: (1, 2) Bank Idx: 2Matrix Idx: (1, 3) Bank Idx: 3Matrix Idx: (1, 4) Bank Idx: 4Matrix Idx: (1, 5) Bank Idx: 5Matrix Idx: (1, 6) Bank Idx: 6Matrix Idx: (1, 7) Bank Idx: 7Matrix Idx: (1, 8) Bank Idx: 8Matrix Idx: (1, 9) Bank Idx: 9Matrix Idx: (1, 10) Bank Idx: 10Matrix Idx: (1, 11) Bank Idx: 11Matrix Idx: (1, 12) Bank Idx: 12Matrix Idx: (1, 13) Bank Idx: 13Matrix Idx: (1, 14) Bank Idx: 14Matrix Idx: (1, 15) Bank Idx: 15Matrix Idx: (1, 16) Bank Idx: 16Matrix Idx: (1, 17) Bank Idx: 17Matrix Idx: (1, 18) Bank Idx: 18Matrix Idx: (1, 19) Bank Idx: 19Matrix Idx: (1, 20) Bank Idx: 20Matrix Idx: (1, 21) Bank Idx: 21Matrix Idx: (1, 22) Bank Idx: 22Matrix Idx: (1, 23) Bank Idx: 23Matrix Idx: (1, 24) Bank Idx: 24Matrix Idx: (1, 25) Bank Idx: 25Matrix Idx: (1, 26) Bank Idx: 26Matrix Idx: (1, 27) Bank Idx: 27Matrix Idx: (1, 28) Bank Idx: 28Matrix Idx: (1, 29) Bank Idx: 29Matrix Idx: (1, 30) Bank Idx: 30Matrix Idx: (1, 31) Bank Idx: 31Matrix Idx: (2, 0) Bank Idx: 0Matrix Idx: (2, 1) Bank Idx: 1Matrix Idx: (2, 2) Bank Idx: 2Matrix Idx: (2, 3) Bank Idx: 3Matrix Idx: (2, 4) Bank Idx: 4Matrix Idx: (2, 5) Bank Idx: 5Matrix Idx: (2, 6) Bank Idx: 6Matrix Idx: (2, 7) Bank Idx: 7Matrix Idx: (2, 8) Bank Idx: 8Matrix Idx: (2, 9) Bank Idx: 9Matrix Idx: (2, 10) Bank Idx: 10Matrix Idx: (2, 11) Bank Idx: 11Matrix Idx: (2, 12) Bank Idx: 12Matrix Idx: (2, 13) Bank Idx: 13Matrix Idx: (2, 14) Bank Idx: 14Matrix Idx: (2, 15) Bank Idx: 15Matrix Idx: (2, 16) Bank Idx: 16Matrix Idx: (2, 17) Bank Idx: 17Matrix Idx: (2, 18) Bank Idx: 18Matrix Idx: (2, 19) Bank Idx: 19Matrix Idx: (2, 20) Bank Idx: 20Matrix Idx: (2, 21) Bank Idx: 21Matrix Idx: (2, 22) Bank Idx: 22Matrix Idx: (2, 23) Bank Idx: 23Matrix Idx: (2, 24) Bank Idx: 24Matrix Idx: (2, 25) Bank Idx: 25Matrix Idx: (2, 26) Bank Idx: 26Matrix Idx: (2, 27) Bank Idx: 27Matrix Idx: (2, 28) Bank Idx: 28Matrix Idx: (2, 29) Bank Idx: 29Matrix Idx: (2, 30) Bank Idx: 30Matrix Idx: (2, 31) Bank Idx: 31Matrix Idx: (3, 0) Bank Idx: 0Matrix Idx: (3, 1) Bank Idx: 1Matrix Idx: (3, 2) Bank Idx: 2Matrix Idx: (3, 3) Bank Idx: 3Matrix Idx: (3, 4) Bank Idx: 4Matrix Idx: (3, 5) Bank Idx: 5Matrix Idx: (3, 6) Bank Idx: 6Matrix Idx: (3, 7) Bank Idx: 7Matrix Idx: (3, 8) Bank Idx: 8Matrix Idx: (3, 9) Bank Idx: 9Matrix Idx: (3, 10) Bank Idx: 10Matrix Idx: (3, 11) Bank Idx: 11Matrix Idx: (3, 12) Bank Idx: 12Matrix Idx: (3, 13) Bank Idx: 13Matrix Idx: (3, 14) Bank Idx: 14Matrix Idx: (3, 15) Bank Idx: 15Matrix Idx: (3, 16) Bank Idx: 16Matrix Idx: (3, 17) Bank Idx: 17Matrix Idx: (3, 18) Bank Idx: 18Matrix Idx: (3, 19) Bank Idx: 19Matrix Idx: (3, 20) Bank Idx: 20Matrix Idx: (3, 21) Bank Idx: 21Matrix Idx: (3, 22) Bank Idx: 22Matrix Idx: (3, 23) Bank Idx: 23Matrix Idx: (3, 24) Bank Idx: 24Matrix Idx: (3, 25) Bank Idx: 25Matrix Idx: (3, 26) Bank Idx: 26Matrix Idx: (3, 27) Bank Idx: 27Matrix Idx: (3, 28) Bank Idx: 28Matrix Idx: (3, 29) Bank Idx: 29Matrix Idx: (3, 30) Bank Idx: 30Matrix Idx: (3, 31) Bank Idx: 31

Memory Bank Conflicts

Notice that for 2D matrix, assuming the data type bitwidth is 32 bit, if $N$ is a multiple of 32, the elements in the same column of the matrix belongs to the same memory bank. This is where memory bank conflicts can easily happen in the implementation. If the threads in a warp try to access the values in the same column of the matrix, there will be severe memory bank conflicts. Using some other values for $N$, such as 33, can avoid the elements in the same column of the matrix belongs to the same memory bank. So be careful about the stride of memory bank access.

1234567891011121314151617181920212223242526272829303132333435363738394041424344

#include <iostream>#include <memory>#include <vector>template <typename T>void bank_id_1d_mapping(int bank_size, int num_banks, int N){    for (int i{0}; i < N; ++i)    {        int bank_idx = (i * sizeof(T) * 8 / bank_size) % num_banks;        std::cout << "Array Idx: " << i << " "                  << "Bank Idx: " << bank_idx << std::endl;    }}template <typename T>void bank_id_2d_mapping(int bank_size, int num_banks, int M, int N){    for (int i{0}; i < M; ++i)    {        for (int j{0}; j < N; ++j)        {            int bank_idx =                ((i * N + j) * sizeof(T) * 8 / bank_size) % num_banks;            std::cout << "Matrix Idx: (" << i << ", " << j << ") "                      << "Bank Idx: " << bank_idx << std::endl;        }    }}int main(){    constexpr const int bank_size{32}; // bits    constexpr const int num_banks{32};    const int M{4};    const int N{33};    std::cout << "Bank ID Mapping 1D: N = " << N << std::endl;    bank_id_1d_mapping<float>(bank_size, num_banks, N);    std::cout << "Bank 2D Mapping 1D: M = " << M << " N = " << N << std::endl;    bank_id_2d_mapping<float>(bank_size, num_banks, M, N);}

In practice, the additional column is unused and can be filled with any value. Just to make sure that the algorithm implemented will not produce incorrect results by accidentally using the values that are not supposed to be used in the additional column.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169

$ g++ memory_bank.cpp -o memory_bank -std=c++14$ ./memory_bankBank ID Mapping 1D: N = 33Array Idx: 0 Bank Idx: 0Array Idx: 1 Bank Idx: 1Array Idx: 2 Bank Idx: 2Array Idx: 3 Bank Idx: 3Array Idx: 4 Bank Idx: 4Array Idx: 5 Bank Idx: 5Array Idx: 6 Bank Idx: 6Array Idx: 7 Bank Idx: 7Array Idx: 8 Bank Idx: 8Array Idx: 9 Bank Idx: 9Array Idx: 10 Bank Idx: 10Array Idx: 11 Bank Idx: 11Array Idx: 12 Bank Idx: 12Array Idx: 13 Bank Idx: 13Array Idx: 14 Bank Idx: 14Array Idx: 15 Bank Idx: 15Array Idx: 16 Bank Idx: 16Array Idx: 17 Bank Idx: 17Array Idx: 18 Bank Idx: 18Array Idx: 19 Bank Idx: 19Array Idx: 20 Bank Idx: 20Array Idx: 21 Bank Idx: 21Array Idx: 22 Bank Idx: 22Array Idx: 23 Bank Idx: 23Array Idx: 24 Bank Idx: 24Array Idx: 25 Bank Idx: 25Array Idx: 26 Bank Idx: 26Array Idx: 27 Bank Idx: 27Array Idx: 28 Bank Idx: 28Array Idx: 29 Bank Idx: 29Array Idx: 30 Bank Idx: 30Array Idx: 31 Bank Idx: 31Array Idx: 32 Bank Idx: 0Bank 2D Mapping 1D: M = 4 N = 33Matrix Idx: (0, 0) Bank Idx: 0Matrix Idx: (0, 1) Bank Idx: 1Matrix Idx: (0, 2) Bank Idx: 2Matrix Idx: (0, 3) Bank Idx: 3Matrix Idx: (0, 4) Bank Idx: 4Matrix Idx: (0, 5) Bank Idx: 5Matrix Idx: (0, 6) Bank Idx: 6Matrix Idx: (0, 7) Bank Idx: 7Matrix Idx: (0, 8) Bank Idx: 8Matrix Idx: (0, 9) Bank Idx: 9Matrix Idx: (0, 10) Bank Idx: 10Matrix Idx: (0, 11) Bank Idx: 11Matrix Idx: (0, 12) Bank Idx: 12Matrix Idx: (0, 13) Bank Idx: 13Matrix Idx: (0, 14) Bank Idx: 14Matrix Idx: (0, 15) Bank Idx: 15Matrix Idx: (0, 16) Bank Idx: 16Matrix Idx: (0, 17) Bank Idx: 17Matrix Idx: (0, 18) Bank Idx: 18Matrix Idx: (0, 19) Bank Idx: 19Matrix Idx: (0, 20) Bank Idx: 20Matrix Idx: (0, 21) Bank Idx: 21Matrix Idx: (0, 22) Bank Idx: 22Matrix Idx: (0, 23) Bank Idx: 23Matrix Idx: (0, 24) Bank Idx: 24Matrix Idx: (0, 25) Bank Idx: 25Matrix Idx: (0, 26) Bank Idx: 26Matrix Idx: (0, 27) Bank Idx: 27Matrix Idx: (0, 28) Bank Idx: 28Matrix Idx: (0, 29) Bank Idx: 29Matrix Idx: (0, 30) Bank Idx: 30Matrix Idx: (0, 31) Bank Idx: 31Matrix Idx: (0, 32) Bank Idx: 0Matrix Idx: (1, 0) Bank Idx: 1Matrix Idx: (1, 1) Bank Idx: 2Matrix Idx: (1, 2) Bank Idx: 3Matrix Idx: (1, 3) Bank Idx: 4Matrix Idx: (1, 4) Bank Idx: 5Matrix Idx: (1, 5) Bank Idx: 6Matrix Idx: (1, 6) Bank Idx: 7Matrix Idx: (1, 7) Bank Idx: 8Matrix Idx: (1, 8) Bank Idx: 9Matrix Idx: (1, 9) Bank Idx: 10Matrix Idx: (1, 10) Bank Idx: 11Matrix Idx: (1, 11) Bank Idx: 12Matrix Idx: (1, 12) Bank Idx: 13Matrix Idx: (1, 13) Bank Idx: 14Matrix Idx: (1, 14) Bank Idx: 15Matrix Idx: (1, 15) Bank Idx: 16Matrix Idx: (1, 16) Bank Idx: 17Matrix Idx: (1, 17) Bank Idx: 18Matrix Idx: (1, 18) Bank Idx: 19Matrix Idx: (1, 19) Bank Idx: 20Matrix Idx: (1, 20) Bank Idx: 21Matrix Idx: (1, 21) Bank Idx: 22Matrix Idx: (1, 22) Bank Idx: 23Matrix Idx: (1, 23) Bank Idx: 24Matrix Idx: (1, 24) Bank Idx: 25Matrix Idx: (1, 25) Bank Idx: 26Matrix Idx: (1, 26) Bank Idx: 27Matrix Idx: (1, 27) Bank Idx: 28Matrix Idx: (1, 28) Bank Idx: 29Matrix Idx: (1, 29) Bank Idx: 30Matrix Idx: (1, 30) Bank Idx: 31Matrix Idx: (1, 31) Bank Idx: 0Matrix Idx: (1, 32) Bank Idx: 1Matrix Idx: (2, 0) Bank Idx: 2Matrix Idx: (2, 1) Bank Idx: 3Matrix Idx: (2, 2) Bank Idx: 4Matrix Idx: (2, 3) Bank Idx: 5Matrix Idx: (2, 4) Bank Idx: 6Matrix Idx: (2, 5) Bank Idx: 7Matrix Idx: (2, 6) Bank Idx: 8Matrix Idx: (2, 7) Bank Idx: 9Matrix Idx: (2, 8) Bank Idx: 10Matrix Idx: (2, 9) Bank Idx: 11Matrix Idx: (2, 10) Bank Idx: 12Matrix Idx: (2, 11) Bank Idx: 13Matrix Idx: (2, 12) Bank Idx: 14Matrix Idx: (2, 13) Bank Idx: 15Matrix Idx: (2, 14) Bank Idx: 16Matrix Idx: (2, 15) Bank Idx: 17Matrix Idx: (2, 16) Bank Idx: 18Matrix Idx: (2, 17) Bank Idx: 19Matrix Idx: (2, 18) Bank Idx: 20Matrix Idx: (2, 19) Bank Idx: 21Matrix Idx: (2, 20) Bank Idx: 22Matrix Idx: (2, 21) Bank Idx: 23Matrix Idx: (2, 22) Bank Idx: 24Matrix Idx: (2, 23) Bank Idx: 25Matrix Idx: (2, 24) Bank Idx: 26Matrix Idx: (2, 25) Bank Idx: 27Matrix Idx: (2, 26) Bank Idx: 28Matrix Idx: (2, 27) Bank Idx: 29Matrix Idx: (2, 28) Bank Idx: 30Matrix Idx: (2, 29) Bank Idx: 31Matrix Idx: (2, 30) Bank Idx: 0Matrix Idx: (2, 31) Bank Idx: 1Matrix Idx: (2, 32) Bank Idx: 2Matrix Idx: (3, 0) Bank Idx: 3Matrix Idx: (3, 1) Bank Idx: 4Matrix Idx: (3, 2) Bank Idx: 5Matrix Idx: (3, 3) Bank Idx: 6Matrix Idx: (3, 4) Bank Idx: 7Matrix Idx: (3, 5) Bank Idx: 8Matrix Idx: (3, 6) Bank Idx: 9Matrix Idx: (3, 7) Bank Idx: 10Matrix Idx: (3, 8) Bank Idx: 11Matrix Idx: (3, 9) Bank Idx: 12Matrix Idx: (3, 10) Bank Idx: 13Matrix Idx: (3, 11) Bank Idx: 14Matrix Idx: (3, 12) Bank Idx: 15Matrix Idx: (3, 13) Bank Idx: 16Matrix Idx: (3, 14) Bank Idx: 17Matrix Idx: (3, 15) Bank Idx: 18Matrix Idx: (3, 16) Bank Idx: 19Matrix Idx: (3, 17) Bank Idx: 20Matrix Idx: (3, 18) Bank Idx: 21Matrix Idx: (3, 19) Bank Idx: 22Matrix Idx: (3, 20) Bank Idx: 23Matrix Idx: (3, 21) Bank Idx: 24Matrix Idx: (3, 22) Bank Idx: 25Matrix Idx: (3, 23) Bank Idx: 26Matrix Idx: (3, 24) Bank Idx: 27Matrix Idx: (3, 25) Bank Idx: 28Matrix Idx: (3, 26) Bank Idx: 29Matrix Idx: (3, 27) Bank Idx: 30Matrix Idx: (3, 28) Bank Idx: 31Matrix Idx: (3, 29) Bank Idx: 0Matrix Idx: (3, 30) Bank Idx: 1Matrix Idx: (3, 31) Bank Idx: 2Matrix Idx: (3, 32) Bank Idx: 3

Here is an example of memory conflicts due to inappropriate strides.

References

CUDA Shared Memory Bank

https://leimao.github.io/blog/CUDA-Shared-Memory-Bank/

Author

Lei Mao

Posted on

06-22-2022

Updated on

08-19-2022

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


CUDA Kernel Execution Overlap

Introduction

In my previous blog post “CUDA Stream”, I have discussed about how CUDA streams helps the CUDA program achieves concurrency. At the end of the article, I also mentioned that in addition to memory transfer and kernel execution overlap, execution overlap between different kernels is also allowed. However, many CUDA programmers wondered why they have not encountered kernel execution overlap before.

In this blog post, I would like to discuss the CUDA kernel execution overlap and why we could or could not see them in practice.

CUDA Kernel Execution Overlap

Computation Resources

CUDA kernel executions can overlap if there are sufficient computation resource to parallelize multiple kernel executions.

In the following example, by changing the value of blocks_per_grid from small to large, we could see that the kernel executions from different CUDA streams changes from full-parallelization, to partial-parallelization, and finally to almost no-parallelization. This is because, when the computation resource allocated for one CUDA kernel becomes larger, the computation resource for additional CUDA kernels becomes smaller.

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495

#include <cuda_runtime.h>#include <iostream>#include <vector>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, const char* const func, const char* const file,           const int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() checkLast(__FILE__, __LINE__)void checkLast(const char* const file, const int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}__global__ void float_add_one(float* buffer, uint32_t n){    uint32_t const idx{blockDim.x * blockIdx.x + threadIdx.x};    uint32_t const stride{blockDim.x * gridDim.x};    for (uint32_t i{idx}; i < n; i += stride)    {        buffer[i] += 1.0F;    }}void launch_float_add_one(float* buffer, uint32_t n,                          dim3 const& threads_per_block,                          dim3 const& blocks_per_grid, cudaStream_t stream){    float_add_one<<<blocks_per_grid, threads_per_block, 0, stream>>>(buffer, n);    CHECK_LAST_CUDA_ERROR();}int main(int argc, char** argv){    size_t const buffer_size{1024 * 10240};    size_t const num_streams{5};    dim3 const threads_per_block{1024};    // Try different values for blocks_per_grid    // 1, 2, 4, 8, 16, 32, 1024, 2048    dim3 const blocks_per_grid{32};    std::vector<float*> d_buffers(num_streams);    std::vector<cudaStream_t> streams(num_streams);    for (auto& d_buffer : d_buffers)    {        CHECK_CUDA_ERROR(cudaMalloc(&d_buffer, buffer_size * sizeof(float)));    }    for (auto& stream : streams)    {        CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    }    for (size_t i = 0; i < num_streams; ++i)    {        launch_float_add_one(d_buffers[i], buffer_size, threads_per_block,                             blocks_per_grid, streams[i]);    }    for (auto& stream : streams)    {        CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    }    for (auto& d_buffer : d_buffers)    {        CHECK_CUDA_ERROR(cudaFree(d_buffer));    }    for (auto& stream : streams)    {        CHECK_CUDA_ERROR(cudaStreamDestroy(stream));    }    return 0;}

12

$ nvcc overlap.cu -o overlap$ ./overlap

We observed full-parallelization for blocks_per_grid = 1. However, we could also see that the time spent for finishing all the kernels was long because the GPU was not fully utilized.

When we set blocks_per_grid = 32, only some of the kernel executions were parallelized. However, the GPU was fully utilized and the time spent for finishing all the kernels was much less compared to the blocks_per_grid = 1.

Same as blocks_per_grid = 32, when we set blocks_per_grid = 5120, there was almost no kernel executions parallelized. However, the GPU was still fully utilized and the time spent for finishing all the kernels was much less compared to the blocks_per_grid = 1.

Implicit Synchronization

It is also possible that there is no kernel execution overlap even if there are sufficient computation resources. It could be due to that there are CUDA commands issued by the host thread to the default stream between other CUDA commands from other different streams causing implicit synchronization.

In my opinion, this rarely happens in the single-threaded CUDA programs due to the way CUDA programmers usually writes CUDA programs. However, it will definitely happen for the multi-threaded CUDA programs. To overcome this situation, since CUDA 7, a per-thread default stream compile mode has been created. The user would just have to specify --default-stream per-thread in the NVCC compiler building flags without having to change the existing CUDA program to disable implicit synchronization. To see more details about how to simplify CUDA concurrency using per-thread default stream, please read Mark Harris’s blog post.

As of CUDA 11.4, the default building argument is still legacy. The user would have to manually change it to per-thread in order to use the per-thread default stream. From the CUDA 11.4 NVCC help:

12345678910111213141516

--default-stream {legacy|null|per-thread}       (-default-stream)        Specify the stream that CUDA commands from the compiled program will be sent        to by default.        legacy                The CUDA legacy stream (per context, implicitly synchronizes with                other streams).        per-thread                A normal CUDA stream (per thread, does not implicitly                synchronize with other streams).        'null' is a deprecated alias for 'legacy'.        Allowed values for this option:  'legacy','null','per-thread'.        Default value:  'legacy'.

Conclusions

If there is no implicit synchronization from the default CUDA stream, partial or no CUDA kernel execution parallelization usually indicate high GPU utilization, and full CUDA kernel execution parallelization usually indicate GPU might have not been fully utilized.

If the no CUDA kernel execution overlap was due to the implicit synchronization from the default CUDA stream, we should probably think of disabling it by enabling the per-thread default stream.

References

CUDA Kernel Execution Overlap

https://leimao.github.io/blog/CUDA-Kernel-Execution-Overlap/

Author

Lei Mao

Posted on

06-10-2022

Updated on

06-10-2022

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


Proper CUDA Error Checking

Introduction

Proper CUDA error checking is critical for making the CUDA program development smooth and successful. Missing or incorrectly identifying CUDA errors could cause problems in production or waste lots of time in debugging.

In this blog post, I would like to quickly discuss proper CUDA error checking.

CUDA Error Types

CUDA errors could be separated into synchronous and asynchronous errors, or sticky and non-sticky errors.

Synchronous Error VS Asynchronous Error

CUDA kernel launch is asynchronous, meaning when the host thread reaches the code for kernel launch, say kernel<<<...>>>, the host thread issues an request to execute the kernel on GPU, then the host thread that launches the kernel continues, without waiting for the kernel to complete. The kernel might not begin to execute right away either.

There could be two types of error for CUDA kernel launch, synchronous error and asynchronous error.

Synchronous error happens when the host thread knows the kernel is illegal or invalid. For example, when the thread block size or grid size is too large, a synchronous error is resulted immediately after the kernel launch call, and this error could be captured by CUDA runtime error capturing API calls, such as cudaGetLastError, right after the kernel launch call.

Asynchronous error happens during kernel execution or CUDA runtime asynchronous API execution on GPU. It might take a while to encounter the error and send the error to host thread. For example, For example, it might encounter accessing invalid memory address in the late stage of kernel execution or CUDA runtime asynchronous API cudaMemcpyAsync execution, it will abort the execution and then send the error back to thread. Even if there are CUDA runtime error capturing API calls, such as cudaGetLastError, right after the kernel launch call, at the time when the error reaches host, those CUDA runtime error capturing API calls have been executed and they found no error. It is possible to capture the asynchronous error by explicitly synchronizing using the CUDA kernel launch using CUDA runtime API calls, such as cudaDeviceSynchronize, cudaStreamSynchronize, or cudaEventSynchronize, and checking the returned error from those CUDA kernel launch using CUDA runtime API calls or capturing the error using CUDA runtime error capturing API calls, such as cudaGetLastError. However, explicitly synchronization usually affects performance and therefore is not recommended for using in production unless it is extremely necessary.

Sticky VS Non-Sticky Error

CUDA runtime API returns non-sticky error if there is any, whereas CUDA kernel execution resulted in sticky error if there is any.

A non-sticky error is recoverable, meaning subsequent CUDA runtime API calls could behave normally. Therefore, the CUDA context is not corrupted. For example, when we allocate memory using cudaMalloc, it will return a non-sticky error if the GPU memory is insufficient.

A sticky error is not recoverable, meaning subsequent CUDA runtime API calls will always return the same error. Therefore, the CUDA context is corrupted, unless the application host process is terminated. For example, when the kernel tries to access invalid memory address during kernel execution, it will result in a sticky error which will be captured and returned by all the subsequent CUDA runtime API calls.

CUDA Error Checking Best Practice

In a CUDA program implementation, both development and production code, always check the return value of each CUDA runtime synchronous or asynchronous API call to see if there is any CUDA synchronous error, always run CUDA runtime error capturing API calls, such as cudaGetLastError, after kernel launch calls to see if there is any CUDA synchronous error. Check CUDA asynchronous error in development by synchronization and error checking after kernel launch calls and disable it in production.

Quiz

There is a question on the NVIDIA developer forum. Let’s use it as a quiz. Basically, the user has the following code. All calculations are done on the default stream and one thread. The cudaDeviceSynchronize returns cudaSuccess, but the cudaGetLastError call returns an invalid device function error. How would this happen?

1234567

// do some stuff, launch kernels, etcres = cudaDeviceSynchronize();// check resres = cudaGetLastError();// check res

cudaGetLastError returns the last error that has been produced by any of the runtime calls in the same host thread and resets it to cudaSuccess. cudaDeviceSynchronize is a CUDA runtime API call and it got no error. This means the kernel launch got no asynchronous error. However, there could be errors from CUDA runtime API calls prior to launching the kernel or the kernel launching encountered synchronous error which have not been properly error-checked. The last error that produced by those would not be reset until the cudaGetLastError call, even though before the reset there were cudaSuccess from other CUDA runtime API calls.

For example,

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647

#include <cuda_runtime.h>#include <iostream>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, const char* const func, const char* const file,           const int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        // We don't exit when we encounter CUDA errors in this example.        // std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() checkLast(__FILE__, __LINE__)void checkLast(const char* const file, const int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        // We don't exit when we encounter CUDA errors in this example.        // std::exit(EXIT_FAILURE);    }}int main(){    float* p;    // This will produce error.    CHECK_CUDA_ERROR(cudaMalloc(&p, 1000000000000000 * sizeof(float)));    // This will be successful.    CHECK_CUDA_ERROR(cudaMalloc(&p, 10 * sizeof(float)));    // This will be successful.    CHECK_CUDA_ERROR(cudaFree(p));    // The last error still has not been reset here.    // This will produce the same error as    // cudaMalloc(&p, 1000000000000000 * sizeof(float))    CHECK_LAST_CUDA_ERROR();    // The last error has been reset here.    CHECK_LAST_CUDA_ERROR();}

123456

$ nvcc last_error.cu -o last_error$ ./last_errorCUDA Runtime Error at: last_error.cu:37out of memory cudaMalloc(&p, 1000000000000000 * sizeof(float))CUDA Runtime Error at: last_error.cu:45out of memory

Fundamentally, it was due to that the CUDA program error checking was not following the best practice mentioned previously.

References

Proper CUDA Error Checking

https://leimao.github.io/blog/Proper-CUDA-Error-Checking/

Author

Lei Mao

Posted on

05-25-2022

Updated on

12-15-2023

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


CUDA Compilation Architecture Macro

Introduction

In C++, macros are often used for controlling the code for compilation for difference use cases. Similarly, in CUDA, it is often necessary to compile the same source code file for different GPU architectures.

In this blog post, I would like to quickly discuss how to use NVCC compilation architecture macro to control the compilation for different GPU architectures.

Half Addition Example

According to the CUDA arithmetic instructions, FP16 add arithmetic instruction could only be performed with compute capability >= 5.3.

In this example, with architecture macro, different FP16 add implementation could be switched for different virtual GPU architectures.

No Architecture Macro

Without using architecture macro, we could not control the device side implementation for different virtual GPU architectures.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134

#include <cmath>#include <cstdint>#include <cuda_fp16.h>#include <functional>#include <iomanip>#include <iostream>#include <vector>#define checkCuda(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, const char* const func, const char* const file,           const int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}template <class T>float measure_cuda_performance(std::function<T(void)> bound_function,                               const int num_repeats = 100,                               const int num_warmups = 100){    cudaEvent_t start;    cudaEvent_t stop;    cudaEventCreate(&start);    cudaEventCreate(&stop);    for (int i{0}; i < num_warmups; ++i)    {        bound_function();    }    cudaDeviceSynchronize();    cudaEventRecord(start, 0);    for (int i{0}; i < num_repeats; ++i)    {        bound_function();    }    cudaEventRecord(stop, 0);    cudaEventSynchronize(stop);    cudaError_t err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA half addition kernel failed to execute."                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }    float time_elapsed{0.0f};    cudaEventElapsedTime(&time_elapsed, start, stop);    float latency{time_elapsed / static_cast<float>(num_repeats)};    return latency;}__global__ void half_addition(__half* output, __half const* input_1,                              __half const* input_2, uint32_t const n){    const uint32_t idx{blockDim.x * blockIdx.x + threadIdx.x};    const uint32_t stride{blockDim.x * gridDim.x};    for (uint32_t i{idx}; i < n; i += stride)    {        output[i] = __hadd(input_1[i], input_2[i]);    }}void launch_half_addition(__half* output, __half const* input_1,                          __half const* input_2, uint32_t const n){    dim3 threads_per_block{1024};    dim3 blocks_per_grid{32};    half_addition<<<blocks_per_grid, threads_per_block>>>(output, input_1,                                                          input_2, n);}int main(){    constexpr uint32_t n{100000};    constexpr float a{1.0f}, b{2.0f}, c{3.0f};    std::vector<__half> input_1(n, __float2half(a));    std::vector<__half> input_2(n, __float2half(b));    std::vector<__half> output(n, __float2half(0.0f));    __half *d_input_1, *d_input_2, *d_output;    checkCuda(cudaMalloc(&d_input_1, n * sizeof(__half)));    checkCuda(cudaMalloc(&d_input_2, n * sizeof(__half)));    checkCuda(cudaMalloc(&d_output, n * sizeof(__half)));    checkCuda(cudaMemcpy(d_input_1, input_1.data(), n * sizeof(__half),                         cudaMemcpyHostToDevice));    checkCuda(cudaMemcpy(d_input_2, input_2.data(), n * sizeof(__half),                         cudaMemcpyHostToDevice));    launch_half_addition(d_output, d_input_1, d_input_2, n);    cudaDeviceSynchronize();    cudaError_t err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA half addition kernel failed to execute."                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }    checkCuda(cudaMemcpy(output.data(), d_output, n * sizeof(__half),                         cudaMemcpyDeviceToHost));    for (uint32_t i{0}; i < n; ++i)    {        if (std::abs(__half2float(output.at(i)) - c) > 1e-8)        {            std::cerr << "CUDA half addition kernel implementation has error!."                      << std::endl;            std::cout << "Expect " << c << " at position " << i << " but got "                      << __half2float(output.at(i)) << std::endl;            break;        }    }    std::function<void(void)> bound_function{        std::bind(launch_half_addition, d_output, d_input_1, d_input_2, n)};    float latency{measure_cuda_performance(bound_function, 100, 100)};    std::cout << std::fixed << std::setprecision(5) << "Latency: " << latency              << " ms" << std::endl;}

Although compiling the FP16 addition program against the virtual GPU architecture compute_52 did not produce compilation error, runtime sanity check shows that the the CUDA kernel has issues. Compiling the same program against the virtual GPU architecture compute_53 is fine. This is expected because FP16 add arithmetic instruction __hadd could only be performed with virtual GPU architecture >= 5.3.

12345678

$ nvcc half_addition_no_macro.cu -o half_addition_no_macro --gpu-architecture=compute_52$ ./half_addition_no_macroCUDA half addition kernel implementation has error!.Expect 3 at position 0 but got 1Latency: 0.00313 ms$ nvcc half_addition_no_macro.cu -o half_addition_no_macro --gpu-architecture=compute_53$ ./half_addition_no_macroLatency: 0.00286 ms

With Architecture Macro

For virtual GPU architecture < 5.3, if we care less about the performance, we could still do the FP16 addition by converting FP16 values to FP32, perform the FP32 addition, and convert the FP32 sum back to FP16. __CUDA_ARCH__ is the architecture macro representing the virtual GPU architecture.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137

#include <cmath>#include <cstdint>#include <cuda_fp16.h>#include <functional>#include <iomanip>#include <iostream>#include <vector>#define checkCuda(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, const char* const func, const char* const file,           const int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}template <class T>float measure_cuda_performance(std::function<T(void)> bound_function,                               const int num_repeats = 100,                               const int num_warmups = 100){    cudaEvent_t start;    cudaEvent_t stop;    cudaEventCreate(&start);    cudaEventCreate(&stop);    for (int i{0}; i < num_warmups; ++i)    {        bound_function();    }    cudaDeviceSynchronize();    cudaEventRecord(start, 0);    for (int i{0}; i < num_repeats; ++i)    {        bound_function();    }    cudaEventRecord(stop, 0);    cudaEventSynchronize(stop);    cudaError_t err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA half addition kernel failed to execute."                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }    float time_elapsed{0.0f};    cudaEventElapsedTime(&time_elapsed, start, stop);    float latency{time_elapsed / static_cast<float>(num_repeats)};    return latency;}__global__ void half_addition(__half* output, __half const* input_1,                              __half const* input_2, uint32_t const n){    const uint32_t idx{blockDim.x * blockIdx.x + threadIdx.x};    const uint32_t stride{blockDim.x * gridDim.x};    for (uint32_t i{idx}; i < n; i += stride)    {#if __CUDA_ARCH__ >= 530        output[i] = __hadd(input_1[i], input_2[i]);#else        output[i] =            __float2half(__half2float(input_1[i]) + __half2float(input_2[i]));#endif    }}void launch_half_addition(__half* output, __half const* input_1,                          __half const* input_2, uint32_t const n){    dim3 threads_per_block{1024};    dim3 blocks_per_grid{32};    half_addition<<<blocks_per_grid, threads_per_block>>>(output, input_1,                                                          input_2, n);}int main(){    constexpr uint32_t n{100000};    constexpr float a{1.0f}, b{2.0f}, c{3.0f};    std::vector<__half> input_1(n, __float2half(a));    std::vector<__half> input_2(n, __float2half(b));    std::vector<__half> output(n, __float2half(0.0f));    __half *d_input_1, *d_input_2, *d_output;    checkCuda(cudaMalloc(&d_input_1, n * sizeof(__half)));    checkCuda(cudaMalloc(&d_input_2, n * sizeof(__half)));    checkCuda(cudaMalloc(&d_output, n * sizeof(__half)));    checkCuda(cudaMemcpy(d_input_1, input_1.data(), n * sizeof(__half),                         cudaMemcpyHostToDevice));    checkCuda(cudaMemcpy(d_input_2, input_2.data(), n * sizeof(__half),                         cudaMemcpyHostToDevice));    launch_half_addition(d_output, d_input_1, d_input_2, n);    cudaDeviceSynchronize();    cudaError_t err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA half addition kernel failed to execute."                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }    checkCuda(cudaMemcpy(output.data(), d_output, n * sizeof(__half),                         cudaMemcpyDeviceToHost));    for (uint32_t i{0}; i < n; ++i)    {        if (std::abs(__half2float(output.at(i)) - c) > 1e-8)        {            std::cerr << "CUDA half addition kernel implementation has error!."                      << std::endl;            std::cout << "Expect " << c << " at position " << i << " but got "                      << __half2float(output.at(i)) << std::endl;            break;        }    }    std::function<void(void)> bound_function{        std::bind(launch_half_addition, d_output, d_input_1, d_input_2, n)};    float latency{measure_cuda_performance(bound_function, 100, 100)};    std::cout << std::fixed << std::setprecision(5) << "Latency: " << latency              << " ms" << std::endl;}

In this implementation, when the virtual GPU architecture is compute_52, __float2half(__half2float(input_1[i]) + __half2float(input_2[i])) will be used for compilation; when the virtual GPU architecture is compute_53, __hadd(input_1[i], input_2[i]) will be used for compilation.

123456

$ nvcc half_addition_with_macro.cu -o half_addition_with_macro --gpu-architecture=compute_52$ ./half_addition_with_macroLatency: 0.00305 ms$ nvcc half_addition_with_macro.cu -o half_addition_with_macro --gpu-architecture=compute_53$ ./half_addition_with_macroLatency: 0.00292 ms

Caveats

This macro can be used in the implementation of GPU functions for determining the virtual architecture for which it is currently being compiled. The host code (the non-GPU code) must not depend on it. This means the __CUDA_ARCH__ macro could only live inside the functions decorated with __device__.

In the following example, we could see that the __CUDA_ARCH__ macro is useless inside a host function.

123456789101112

#include <iostream>void test_host_function(){#if __CUDA_ARCH__ >= 530    std::cout << "__CUDA_ARCH__ >= 530" << std::endl;#else    std::cout << "__CUDA_ARCH__ < 530" << std::endl;#endif}int main() { test_host_function(); }

123456

$ nvcc host.cu -o host --gpu-architecture=compute_52$ ./host__CUDA_ARCH__ < 530$ nvcc host.cu -o host --gpu-architecture=compute_53$ ./host__CUDA_ARCH__ < 530

References

CUDA Compilation Architecture Macro

https://leimao.github.io/blog/CUDA-Compilation-Architecture-Macro/

Author

Lei Mao

Posted on

05-01-2022

Updated on

05-01-2022

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV: