CUDA Constant Memory

Introduction

CUDA constant memory is a special memory space on the device. It’s cached and read-only.

There are some caveats when using constant memory. In this post, we will discuss the usages and caveats of constant memory.

Constant Memory

There is a total of 64 KB constant memory on a device. The constant memory space is cached. As a result, a read from constant memory costs one memory read from device memory only on a cache miss; otherwise, it just costs one read from the constant cache. Accesses to different addresses by threads within a warp are serialized, thus the cost scales linearly with the number of unique addresses read by all threads within a warp. As such, the constant cache is best when threads in the same warp accesses only a few distinct locations. If all threads of a warp access the same location, then constant memory can be as fast as a register access.

Constant Memory Usage and Performance

In the following example, we perform additions for an array. One of the constant input arrays is stored on global memory, and the other constant input arrays is stored on global memory or constant memory. We compare the performance of accessing constant memory and global memory under different access patterns.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373

#include <functional>#include <iostream>#include <string>#include <vector>#include <cuda_runtime.h>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, const char* const func, const char* const file,           const int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() checkLast(__FILE__, __LINE__)void checkLast(const char* const file, const int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}template <class T>float measure_performance(std::function<T(cudaStream_t)> bound_function,                          cudaStream_t stream, unsigned int num_repeats = 100,                          unsigned int num_warmups = 100){    cudaEvent_t start, stop;    float time;    CHECK_CUDA_ERROR(cudaEventCreate(&start));    CHECK_CUDA_ERROR(cudaEventCreate(&stop));    for (unsigned int i{0}; i < num_warmups; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaEventRecord(start, stream));    for (unsigned int i{0}; i < num_repeats; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaEventRecord(stop, stream));    CHECK_CUDA_ERROR(cudaEventSynchronize(stop));    CHECK_LAST_CUDA_ERROR();    CHECK_CUDA_ERROR(cudaEventElapsedTime(&time, start, stop));    CHECK_CUDA_ERROR(cudaEventDestroy(start));    CHECK_CUDA_ERROR(cudaEventDestroy(stop));    float const latency{time / num_repeats};    return latency;}// Use all the constant memory.constexpr unsigned int N{64U * 1024U / sizeof(int)};__constant__ int const_values[N];// Magic number for generating the pseudo-random access pattern.constexpr unsigned int magic_number{1357U};enum struct AccessPattern{    OneAccessPerBlock,    OneAccessPerWarp,    OneAccessPerThread,    PseudoRandom};void add_constant_cpu(int* sums, int const* inputs, int const* values,                      unsigned int num_sums, unsigned int num_values,                      unsigned int block_size, AccessPattern access_pattern){    for (unsigned int i{0U}; i < num_sums; ++i)    {        unsigned int const block_id{i / block_size};        unsigned int const thread_id{i % block_size};        unsigned int const warp_id{thread_id / 32U};        unsigned int index{0U};        switch (access_pattern)        {            case AccessPattern::OneAccessPerBlock:                index = block_id % num_values;                break;            case AccessPattern::OneAccessPerWarp:                index = warp_id % num_values;                break;            case AccessPattern::OneAccessPerThread:                index = thread_id % num_values;                break;            case AccessPattern::PseudoRandom:                index = (thread_id * magic_number) % num_values;                break;        }        sums[i] = inputs[i] + values[index];    }}__global__ void add_constant_global_memory(    int* sums, int const* inputs, int const* values, unsigned int num_sums,    unsigned int num_values,    AccessPattern access_pattern = AccessPattern::OneAccessPerBlock){    unsigned int const i{blockIdx.x * blockDim.x + threadIdx.x};    unsigned int const block_id{blockIdx.x};    unsigned int const thread_id{threadIdx.x};    unsigned int const warp_id{threadIdx.x / warpSize};    unsigned int index{0U};    switch (access_pattern)    {        case AccessPattern::OneAccessPerBlock:            index = block_id % num_values;            break;        case AccessPattern::OneAccessPerWarp:            index = warp_id % num_values;            break;        case AccessPattern::OneAccessPerThread:            index = thread_id % num_values;            break;        case AccessPattern::PseudoRandom:            index = (thread_id * magic_number) % num_values;            break;    }    if (i < num_sums)    {        sums[i] = inputs[i] + values[index];    }}void launch_add_constant_global_memory(int* sums, int const* inputs,                                       int const* values, unsigned int num_sums,                                       unsigned int num_values,                                       unsigned int block_size,                                       AccessPattern access_pattern,                                       cudaStream_t stream){    add_constant_global_memory<<<(num_sums + block_size - 1) / block_size,                                 block_size, 0, stream>>>(        sums, inputs, values, num_sums, num_values, access_pattern);    CHECK_LAST_CUDA_ERROR();}__global__ void add_constant_constant_memory(int* sums, int const* inputs,                                             unsigned int num_sums,                                             AccessPattern access_pattern){    unsigned int const i{blockIdx.x * blockDim.x + threadIdx.x};    unsigned int const block_id{blockIdx.x};    unsigned int const thread_id{threadIdx.x};    unsigned int const warp_id{threadIdx.x / warpSize};    unsigned int index{0U};    switch (access_pattern)    {        case AccessPattern::OneAccessPerBlock:            index = block_id % N;            break;        case AccessPattern::OneAccessPerWarp:            index = warp_id % N;            break;        case AccessPattern::OneAccessPerThread:            index = thread_id % N;            break;        case AccessPattern::PseudoRandom:            index = (thread_id * magic_number) % N;            break;    }    if (i < num_sums)    {        sums[i] = inputs[i] + const_values[index];    }}void launch_add_constant_constant_memory(int* sums, int const* inputs,                                         unsigned int num_sums,                                         unsigned int block_size,                                         AccessPattern access_pattern,                                         cudaStream_t stream){    add_constant_constant_memory<<<(num_sums + block_size - 1) / block_size,                                   block_size, 0, stream>>>(        sums, inputs, num_sums, access_pattern);    CHECK_LAST_CUDA_ERROR();}void parse_args(int argc, char** argv, AccessPattern& access_pattern,                unsigned int& block_size, unsigned int& num_sums){    if (argc < 4)    {        std::cerr << "Usage: " << argv[0]                  << " <access pattern> <block size> <number of sums>"                  << std::endl;        std::exit(EXIT_FAILURE);    }    std::string const access_pattern_str{argv[1]};    if (access_pattern_str == "one_access_per_block")    {        access_pattern = AccessPattern::OneAccessPerBlock;    }    else if (access_pattern_str == "one_access_per_warp")    {        access_pattern = AccessPattern::OneAccessPerWarp;    }    else if (access_pattern_str == "one_access_per_thread")    {        access_pattern = AccessPattern::OneAccessPerThread;    }    else if (access_pattern_str == "pseudo_random")    {        access_pattern = AccessPattern::PseudoRandom;    }    else    {        std::cerr << "Invalid access pattern: " << access_pattern_str                  << std::endl;        std::exit(EXIT_FAILURE);    }    block_size = std::stoi(argv[2]);    num_sums = std::stoi(argv[3]);}int main(int argc, char** argv){    constexpr unsigned int num_warmups{100U};    constexpr unsigned int num_repeats{100U};    AccessPattern access_pattern{AccessPattern::OneAccessPerBlock};    unsigned int block_size{1024U};    unsigned int num_sums{12800000U};    // Modify access pattern, block size and number of sums from command line.    parse_args(argc, argv, access_pattern, block_size, num_sums);    cudaStream_t stream;    CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    int h_values[N];    // Initialize values on host memory.    for (unsigned int i{0U}; i < N; ++i)    {        h_values[i] = i;    }    // Initialize values on global memory.    int* d_values;    CHECK_CUDA_ERROR(cudaMallocAsync(&d_values, N * sizeof(int), stream));    CHECK_CUDA_ERROR(cudaMemcpyAsync(d_values, h_values, N * sizeof(int),                                     cudaMemcpyHostToDevice, stream));    // Initialize values on constant memory.    CHECK_CUDA_ERROR(cudaMemcpyToSymbolAsync(const_values, h_values,                                             N * sizeof(int), 0,                                             cudaMemcpyHostToDevice, stream));    std::vector<int> inputs(num_sums, 0);    int* h_inputs{inputs.data()};    int* d_inputs_for_constant;    int* d_inputs_for_global;    CHECK_CUDA_ERROR(cudaMallocAsync(&d_inputs_for_constant,                                     num_sums * sizeof(int), stream));    CHECK_CUDA_ERROR(        cudaMallocAsync(&d_inputs_for_global, num_sums * sizeof(int), stream));    CHECK_CUDA_ERROR(cudaMemcpyAsync(d_inputs_for_constant, h_inputs,                                     num_sums * sizeof(int),                                     cudaMemcpyHostToDevice, stream));    CHECK_CUDA_ERROR(cudaMemcpyAsync(d_inputs_for_global, h_inputs,                                     num_sums * sizeof(int),                                     cudaMemcpyHostToDevice, stream));    std::vector<int> reference_sums(num_sums, 0);    std::vector<int> sums_from_constant(num_sums, 1);    std::vector<int> sums_from_global(num_sums, 2);    int* h_reference_sums{reference_sums.data()};    int* h_sums_from_constant{sums_from_constant.data()};    int* h_sums_from_global{sums_from_global.data()};    int* d_sums_from_constant;    int* d_sums_from_global;    CHECK_CUDA_ERROR(        cudaMallocAsync(&d_sums_from_constant, num_sums * sizeof(int), stream));    CHECK_CUDA_ERROR(        cudaMallocAsync(&d_sums_from_global, num_sums * sizeof(int), stream));    // Synchronize.    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    // Compute reference sums on CPU.    add_constant_cpu(h_reference_sums, h_inputs, h_values, num_sums, N,                     block_size, access_pattern);    // Compute reference sums on GPU using global memory.    launch_add_constant_global_memory(d_sums_from_global, d_inputs_for_global,                                      d_values, num_sums, N, block_size,                                      access_pattern, stream);    // Compute reference sums on GPU using constant memory.    launch_add_constant_constant_memory(d_sums_from_constant,                                        d_inputs_for_constant, num_sums,                                        block_size, access_pattern, stream);    // Copy results from device to host.    CHECK_CUDA_ERROR(cudaMemcpyAsync(h_sums_from_constant, d_sums_from_constant,                                     num_sums * sizeof(int),                                     cudaMemcpyDeviceToHost, stream));    CHECK_CUDA_ERROR(cudaMemcpyAsync(h_sums_from_global, d_sums_from_global,                                     num_sums * sizeof(int),                                     cudaMemcpyDeviceToHost, stream));    // Synchronize.    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    // Verify results.    for (unsigned int i{0U}; i < num_sums; ++i)    {        if (h_reference_sums[i] != h_sums_from_constant[i])        {            std::cerr << "Error at index " << i << " for constant memory."                      << std::endl;            std::exit(EXIT_FAILURE);        }        if (h_reference_sums[i] != h_sums_from_global[i])        {            std::cerr << "Error at index " << i << " for global memory."                      << std::endl;            std::exit(EXIT_FAILURE);        }    }    // Measure performance.    std::function<void(cudaStream_t)> bound_function_constant_memory{        std::bind(launch_add_constant_constant_memory, d_sums_from_constant,                  d_inputs_for_constant, num_sums, block_size, access_pattern,                  std::placeholders::_1)};    std::function<void(cudaStream_t)> bound_function_global_memory{        std::bind(launch_add_constant_global_memory, d_sums_from_global,                  d_inputs_for_global, d_values, num_sums, N, block_size,                  access_pattern, std::placeholders::_1)};    float const latency_constant_memory{measure_performance(        bound_function_constant_memory, stream, num_repeats, num_warmups)};    float const latency_global_memory{measure_performance(        bound_function_global_memory, stream, num_repeats, num_warmups)};    std::cout << "Latency for Add using constant memory: "              << latency_constant_memory << " ms" << std::endl;    std::cout << "Latency for Add using global memory: "              << latency_global_memory << " ms" << std::endl;    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));    CHECK_CUDA_ERROR(cudaFree(d_values));    CHECK_CUDA_ERROR(cudaFree(d_inputs_for_constant));    CHECK_CUDA_ERROR(cudaFree(d_inputs_for_global));    CHECK_CUDA_ERROR(cudaFree(d_sums_from_constant));    CHECK_CUDA_ERROR(cudaFree(d_sums_from_global));    return 0;}

The program was compiled and executed on an NVIDIA RTX 3090 GPU.

1

$ nvcc add_constant.cu -o add_constant

If we have 12800000 adds to perform using 1024 threads per block.

123456789101112

$ ./add_constant one_access_per_block 1024 12800000Latency for Add using constant memory: 0.151798 msLatency for Add using global memory: 0.171404 ms$ ./add_constant one_access_per_warp 1024 12800000Latency for Add using constant memory: 0.164012 msLatency for Add using global memory: 0.189501 ms$ ./add_constant one_access_per_thread 1024 12800000Latency for Add using constant memory: 0.281967 msLatency for Add using global memory: 0.164649 ms$ ./add_constant pseudo_random 1024 12800000Latency for Add using constant memory: 1.2925 msLatency for Add using global memory: 0.159621 ms

If we have 128000 adds to perform using 1024 threads per block.

123456789101112

$ ./add_constant one_access_per_block 1024 128000Latency for Add using constant memory: 0.00289792 msLatency for Add using global memory: 0.00323584 ms$ ./add_constant one_access_per_warp 1024 128000Latency for Add using constant memory: 0.00315392 msLatency for Add using global memory: 0.00359392 ms$ ./add_constant one_access_per_thread 1024 128000Latency for Add using constant memory: 0.00596992 msLatency for Add using global memory: 0.00383264 ms$ ./add_constant pseudo_random 1024 128000Latency for Add using constant memory: 0.0215347 msLatency for Add using global memory: 0.00482304 ms

In both cases, we could see that accessing constant memory is ~10% faster than accessing global memory if the it’s one access per block or one access per warp. If it’s one access per thread, then accessing constant memory is ~70% slower than accessing global memory. If it’s pseudo random access, then accessing constant memory is ~800% slower than accessing global memory.

Conclusions

To use constant memory, it’s important to roughly know the access pattern. If the access pattern is one access per block or one access per warp, which is typically used in broadcast, then constant memory is a good choice. If the access pattern is one access per thread or even pseudo random, then constant memory is a very bad choice.

References

CUDA Constant Memory

https://leimao.github.io/blog/CUDA-Constant-Memory/

Author

Lei Mao

Posted on

12-01-2023

Updated on

12-01-2023

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


CUDA Default Stream

Introduction

CUDA default stream can have different synchronization behaviors in different scenarios. Sometimes, it helps the program to run correctly even if we made some mistakes in assigning the CUDA streams to different kernels.

In this blog post, I would like to introduce the two types of the CUDA default streams, the default legacy stream and the default per-thread stream, and discuss their synchronization behaviors in different scenarios.

Default Stream and Non-Default Blocking Stream

In the following example, I created a non-default blocking stream using cudaStreamCreate. For a series of CUDA kernels that is supposed to be run in sequence on the same non-default blocking CUDA stream, I made a mistake and accidentally used the default stream for one of the kernels.

If the default stream is a default legacy stream, when an action is taken in the legacy stream such as a kernel launch or cudaStreamWaitEvent(), the legacy stream first waits on all blocking streams, the action is queued in the legacy stream, and then all blocking streams wait on the legacy stream. Therefore, even if I made a mistake, the CUDA kernels are still run in sequence and the correctness of the application is not affected.

If the default stream is a default per-thread stream, it is non-blocking and will not synchronize with other CUDA streams. Therefore, my mistake will cause the application to run incorrectly.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101

#include <cassert>#include <iostream>#include <vector>#include <cuda_runtime.h>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, const char* const func, const char* const file,           const int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() checkLast(__FILE__, __LINE__)void checkLast(const char* const file, const int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}__global__ void add_val_in_place(int32_t* data, int32_t val, uint32_t n){    uint32_t const idx{blockDim.x * blockIdx.x + threadIdx.x};    uint32_t const stride{blockDim.x * gridDim.x};    for (uint32_t i{idx}; i < n; i += stride)    {        data[i] += val;    }}void launch_add_val_in_place(int32_t* data, int32_t val, uint32_t n,                             cudaStream_t stream){    dim3 const threads_per_block{1024};    dim3 const blocks_per_grid{32};    add_val_in_place<<<blocks_per_grid, threads_per_block, 0, stream>>>(data,                                                                        val, n);    CHECK_LAST_CUDA_ERROR();}bool check_array_value(int32_t const* data, uint32_t n, int32_t val){    for (uint32_t i{0}; i < n; ++i)    {        if (data[i] != val)        {            return false;        }    }    return true;}int main(){    constexpr uint32_t const n{1000000};    constexpr int32_t const val_1{1};    constexpr int32_t const val_2{2};    constexpr int32_t const val_3{3};    // Create an multi-stream application.    cudaStream_t stream_1{0};    cudaStream_t stream_2{0};    // stream_1 is a non-default blocking stream.    CHECK_CUDA_ERROR(cudaStreamCreate(&stream_1));    std::vector<int32_t> vec(n, 0);    int32_t* d_data{nullptr};    CHECK_CUDA_ERROR(cudaMalloc(&d_data, n * sizeof(int32_t)));    CHECK_CUDA_ERROR(cudaMemcpy(d_data, vec.data(), n * sizeof(int32_t),                                cudaMemcpyHostToDevice));    // Run a sequence of CUDA kernels in order on the same CUDA stream.    launch_add_val_in_place(d_data, val_1, n, stream_1);    // The second kernel launch is supposed to be run on stream_1.    // However, the implementation has a typo such that the kernel launch    // is run on the default stream_2.    launch_add_val_in_place(d_data, val_2, n, stream_2);    launch_add_val_in_place(d_data, val_3, n, stream_1);    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream_1));    CHECK_CUDA_ERROR(cudaMemcpy(vec.data(), d_data, n * sizeof(int32_t),                                cudaMemcpyDeviceToHost));    // Check the correctness of the application.    // Yet the result will still be correct if the default stream_2    // is a legacy default stream.    assert(check_array_value(vec.data(), n, val_1 + val_2 + val_3));    CHECK_CUDA_ERROR(cudaFree(d_data));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream_1));}

We made in the implementation that the three kernels are not run in the same CUDA stream, yet the result is still correct.

12

$ nvcc add.cu -o add -std=c++14$ ./add

This is the same as running the follow command as the default value for --default-stream is legacy.

12

$ nvcc add.cu -o add -std=c++14 --default-stream=legacy$ ./add

Depending on the use cases, this kind of mistake sometimes may affect application performance. It could usually be identified using CUDA profiling software, such as Nsight Systems.

However, if the default stream becomes per-thread, the result is no longer correct, because the kernel launch are no longer issued in sequence.

1234

$ nvcc add.cu -o add -std=c++14 --default-stream=per-thread$ ./addadd: add.cu:98: int main(): Assertion `check_array_value(vec.data(), n, val_1 + val_2 + val_3)' failed.Aborted (core dumped)

Default Stream and Non-Default Non-Blocking Stream

In some applications, the non-default stream can be created using cudaStreamCreateWithFlags and the non-default stream created becomes non-blocking. In this case, the default stream, even if it is the default legacy stream, cannot synchronize with the non-default non-blocking stream. Therefore, Therefore, my mistake will cause the application to run incorrectly, regardless whether the non-default stream is legacy or per-thread.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101

#include <cassert>#include <iostream>#include <vector>#include <cuda_runtime.h>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)void check(cudaError_t err, const char* const func, const char* const file,           const int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() checkLast(__FILE__, __LINE__)void checkLast(const char* const file, const int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}__global__ void add_val_in_place(int32_t* data, int32_t val, uint32_t n){    uint32_t const idx{blockDim.x * blockIdx.x + threadIdx.x};    uint32_t const stride{blockDim.x * gridDim.x};    for (uint32_t i{idx}; i < n; i += stride)    {        data[i] += val;    }}void launch_add_val_in_place(int32_t* data, int32_t val, uint32_t n,                             cudaStream_t stream){    dim3 const threads_per_block{1024};    dim3 const blocks_per_grid{32};    add_val_in_place<<<blocks_per_grid, threads_per_block, 0, stream>>>(data,                                                                        val, n);    CHECK_LAST_CUDA_ERROR();}bool check_array_value(int32_t const* data, uint32_t n, int32_t val){    for (uint32_t i{0}; i < n; ++i)    {        if (data[i] != val)        {            return false;        }    }    return true;}int main(){    constexpr uint32_t const n{1000000};    constexpr int32_t const val_1{1};    constexpr int32_t const val_2{2};    constexpr int32_t const val_3{3};    // Create an multi-stream application.    cudaStream_t stream_1{0};    cudaStream_t stream_2{0};    // stream_1 is a non-default non-blocking stream.    CHECK_CUDA_ERROR(cudaStreamCreateWithFlags(&stream_1, cudaStreamNonBlocking));    std::vector<int32_t> vec(n, 0);    int32_t* d_data{nullptr};    CHECK_CUDA_ERROR(cudaMalloc(&d_data, n * sizeof(int32_t)));    CHECK_CUDA_ERROR(cudaMemcpy(d_data, vec.data(), n * sizeof(int32_t),                                cudaMemcpyHostToDevice));    // Run a sequence of CUDA kernels in order on the same CUDA stream.    launch_add_val_in_place(d_data, val_1, n, stream_1);    // The second kernel launch is supposed to be run on stream_1.    // However, the implementation has a typo so that the kernel launch    // is run on the default stream_2.    launch_add_val_in_place(d_data, val_2, n, stream_2);    launch_add_val_in_place(d_data, val_3, n, stream_1);    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream_1));    CHECK_CUDA_ERROR(cudaMemcpy(vec.data(), d_data, n * sizeof(int32_t),                                cudaMemcpyDeviceToHost));    // Check the correctness of the application.    // Yet the result will still be correct if the default stream_2    // is a legacy default stream.    assert(check_array_value(vec.data(), n, val_1 + val_2 + val_3));    CHECK_CUDA_ERROR(cudaFree(d_data));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream_1));}

1234

$ nvcc add.cu -o add -std=c++14 --default-stream=legacy$ ./addadd: add.cu:98: int main(): Assertion `check_array_value(vec.data(), n, val_1 + val_2 + val_3)' failed.Aborted (core dumped)

1234

$ nvcc add.cu -o add -std=c++14 --default-stream=per-thread$ ./addadd: add.cu:98: int main(): Assertion `check_array_value(vec.data(), n, val_1 + val_2 + val_3)' failed.Aborted (core dumped)

References

CUDA Default Stream

https://leimao.github.io/blog/CUDA-Default-Stream/

Author

Lei Mao

Posted on

11-06-2023

Updated on

11-06-2023

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


CUDA Tensor Layouts for Convolution

Introduction

There are commonly two layouts for the activation tensors involved in the convolution operations in neural networks, NCHW, NHWC, and NC/xHWx.

In general, the performance of convolution using NHWC is much faster than using NCHW. The NC/xHWx layout is an variant of NHWC that is prepared for NVIDIA Tensor Core operations.

In this blog post, I would like to discuss how to perform convolution on GPU and why NHWC and NC/xHWx activation tensor layouts are much more favored than the NCHW activation tensor layout for convolutional neural network inference.

Convolution

The description of convolution in neural networks can be found in the documentation of many deep learning frameworks, such as PyTorch.

Convolution Dimensions

The 2D convolution operation in neural networks consists of an input activation tensor, a filter tensor, an optional bias tensor, and an output activation tensor. We will ignore the bias tensor in this article since it is usually simple to deal with.

The input and output activation tensors and the filter tensor are all 4D tensors that consist of four dimensions. We use N to describe the batch dimension for the input and output tensors. We use C, H, W to describe the number of channels, the spatial height, and the spatial width of the input activation tensor. In order to distinguish the output activation tensor from the input activation tensor, we use K, P, Q to describe the number of channels, the spatial height, and the spatial width of the output activation tensor instead. The filter height and width are described using R and S, respectively.

Implicit GEMM for Convolution

In my previous article “Fast Fourier Transform for Convolution”, I described how to perform convolution using the asymptotically faster fast Fourier transform. But this technique is still not the most common way of performing convolution nowadays on GPU and it is out of the scope of this article.

In my previous article “Convolution and Transposed Convolution as Matrix Multiplication”, I described how to perform convolution using matrix multiplication in which the activation tensors are dense but the filter tensor is sparse.

On GPUs, convolution is usually performed using a method called implicit GEMM. GEMM stands for general matrix multiplication. The difference between the implicit GEMM method that I am about to describe and the method I described in the article “Convolution and Transposed Convolution as Matrix Multiplication” is that all the matrices used in the implicit GEMM method are dense matrices.

The implicit GEMM method for convolution can be described using the following figure. We will focus on the forward propagation (a) only as the gradient updates (b and c) usually do not happen in the neural network inference.

Theoretically, if we transpose, expand and reshape the input activation from a 4D tensor of shape $(N, C, H, W)$ to a 2D tensor of shape $(NPQ, CRS)$, transpose and reshape the weight tensor from 4D $(K, C, S, R)$ to 2D $(CRS, K)$, multiply the two tensors, the 2D output tensor is a tensor of shape $(NPQ, K)$ and can be further transposed to the 4D output activation tensor of shape $(N, K, P, Q)$. For example, suppose $N = 1$, $C = K = 1$, $H = W = 3$, $R = S = 2$, $P = Q = 2$ (the convolution stride is 1 and the padding is “valid”). Because there is only one input channel, the only spatial feature in the input activation tensor is a matrix and its values can be assumed to be

$$\begin{bmatrix}1 & 2 & 3 \\4 & 5 & 6 \\7 & 8 & 9 \\\end{bmatrix}$$

The reconstructed input activation matrix will be of shape $(4, 4)$ and its values are

$$\begin{bmatrix}1 & 2 & 4 & 5 \\2 & 3 & 5 & 6 \\4 & 5 & 7 & 8 \\5 & 6 & 8 & 9 \\\end{bmatrix}$$

The problem of this theoretical formulation is that a new matrices always need to be constructed during inference because the output activation tensor, which will usually be used as the input activation tensor for the next convolution layer, is not of the reconstructed format. Even though the theoretical ration between the number of values in the reconstructed input activation matrix and the number of values in the original input activation tensor is $\frac{PQRS}{HW}$ which sometimes can be 1, constructing such a new matrix is not a no-op and will introduce overhead in computing, not to mention consuming additional memory when this ratio is high. Therefore, in practice, this reconstructed input activation matrix is never constructed in the implicit GEMM method for convolution. The values are read from the input activation tensor of its original layout instead.

NVIDIA Tensor Core

NVIDIA Tensor Core performs small matrix multiplications to accelerate GEMM with extremely high throughput. For example, NVIDIA Tensor Core could perform 16×16×16 GEMM, 16x16 and 16x16 matrix multiplication (and accumulation) for half precision floating point data on a warp basis. Fundamentally, the mathematical motivation of Tensor Core GEMM acceleration has been described in my previous article CUDA Matrix Multiplication, although not explicitly at that time.

$$\mathbf{A} =\begin{bmatrix}\mathbf{A}_{1,1}^{d \times d} & \mathbf{A}_{1,2}^{d \times d} & \cdots & \mathbf{A}_{1,n/d}^{d \times d} \\\mathbf{A}_{2,1}^{d \times d} & \mathbf{A}_{2,2}^{d \times d} & \cdots & \mathbf{A}_{2,n/d}^{d \times d} \\\vdots & \vdots & \ddots & \vdots \\\mathbf{A}_{m/d,1}^{d \times d} & \mathbf{A}_{m/d,2}^{d \times d} & \cdots & \mathbf{A}_{m/d,n/d}^{d \times d} \\\end{bmatrix}$$

$$\mathbf{B} =\begin{bmatrix}\mathbf{B}_{1,1}^{d \times d} & \mathbf{B}_{1,2}^{d \times d} & \cdots & \mathbf{B}_{1,p/d}^{d \times d} \\\mathbf{B}_{2,1}^{d \times d} & \mathbf{B}_{2,2}^{d \times d} & \cdots & \mathbf{B}_{2,p/d}^{d \times d} \\\vdots & \vdots & \ddots & \vdots \\\mathbf{B}_{n/d,1}^{d \times d} & \mathbf{B}_{n/d,2}^{d \times d} & \cdots & \mathbf{B}_{n/d,p/d}^{d \times d} \\\end{bmatrix}$$

$$\mathbf{C} =\begin{bmatrix}\mathbf{C}_{1,1}^{d \times d} & \mathbf{C}_{1,2}^{d \times d} & \cdots & \mathbf{C}_{1,p/d}^{d \times d} \\\mathbf{C}_{2,1}^{d \times d} & \mathbf{C}_{2,2}^{d \times d} & \cdots & \mathbf{C}_{2,p/d}^{d \times d} \\\vdots & \vdots & \ddots & \vdots \\\mathbf{C}_{m/d,1}^{d \times d} & \mathbf{C}_{m/d,2}^{d \times d} & \cdots & \mathbf{C}_{m/d,p/d}^{d \times d} \\\end{bmatrix}$$

$$\mathbf{C}_{i,j}^{d \times d} = \sum_{k=1}^{n/d} \mathbf{A}_{i,k}^{d \times d} \mathbf{B}_{k,j}^{d \times d}$$

Basically, by decomposing the large matrix multiplication into smaller matrix multiplications and accumulation and caching the small matrices, we could make GEMM extremely math bound. Specifically, the small matrices $\mathbf{A}_{i,k}^{d \times d}$ and $\mathbf{B}_{k,j}^{d \times d}$ are cached in the registers of a warp, and each warp computes a $\mathbf{C}_{i,j}^{d \times d}$ using Tensor Core by iterating the small matrice multiplication and accumulation $\frac{n}{d}$ times.

Tensor Layouts

Now that we have some basic idea of how convolution is performed on GPU via implicit GEMM. Let’s check the impact of different activation layouts on the performance of convolution.

NCHW

In the NCHW layout, C is not the fastest dimension. This means, even without assuming the implementation, getting the entire channels from the input activations for implicit GEMM ($CRS$) needs to stride lots of times, which significantly reduced the valid memory throughput on GPU. For example, suppose the input activation tensor has $N = 1$, $C = 256$, and $H = W = 128$, to get an entire channel from the spatial indices $(12, 35)$ for 1x1 convolution, the slicing operation we will perform for the first sample is $X[1, :, 12, 35]$. Under the hood, getting an entire channel of size $C = 256$ needs to stride $C = 256$ times of size $HW = 16384$ and this invalids the coalesced reading of the data from the DRAM on GPU.

Therefore, the NCHW layout is not favored for the implicit GEMM for convolution.

NHWC

In the NHWC layout, C becomes the fastest dimension. Unlike NCHW slicing for the C dimension, the NHWC slicing for the C dimension can be fully coalesced from the DRAM.

Therefore, the NHWC layout is favored over the NCHW layout for the implicit GEMM for convolution.

NC/xHWx

To take the advantage of NVIDIA Tensor Core, the “virtual” reconstructed input activation matrix needs to be divided in a way that is compatible with the Tensor Core GEMM. This requires the “virtual” reconstructed input activation matrix to be padded (with zeros) so that $CSR$ could be divided by the small matrix dimension requirements from Tensor Core operations. The NHWC layout provides no such guarantee therefore applying the NHWC tensor for Tensor Core GEMM requires padding during the runtime and therefore is a little bit cumbersome to use with Tensor Core.

The NC/xHWx layout is always padded to x elements for the fastest (C) dimension, where x is usually the Tensor Core GEMM dimension requirement. Therefore, it is immediately ready to be used with Tensor Core.

One might ask, is there a NHWC variant layout whose C dimension is not divided like the NC/xHWx layout but padded to x elements according to the Tensor Core GEMM dimension requirement. The answer is yes and using that layout can be very performant for the implicit GEMM method for convolution as well. My educative guess for the reason why NC/xHWx is slightly more often seen than the padded NHWC layout is that the indexing and slicing for the NC/xHWx layout might be more natural in the implementation than that for the padded NHWC layout. For example, using the padded NHWC layout, the indexing and slicing of the input activation tensor would be like this.

$$\begin{align}&X[1, 0:4, 0:4, 0:16] \\&X[1, 0:4, 0:4, 16:32] \\&X[1, 0:4, 0:4, 32:48] \\\end{align}$$

Using the NC/16HW16 layout instead, the indexing and slicing of the input activation tensor to get the equivalent matrices would be like this.

$$\begin{align}&X[1, 0, 0:4, 0:4, :] \\&X[1, 1, 0:4, 0:4, :] \\&X[1, 2, 0:4, 0:4, :] \\\end{align}$$

References

CUDA Tensor Layouts for Convolution

https://leimao.github.io/blog/CUDA-Convolution-Tensor-Layouts/

Author

Lei Mao

Posted on

06-04-2023

Updated on

06-04-2023

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


NVIDIA Tensor Core Programming

Introduction

NVIDIA Tensor Cores are dedicated accelerators for general matrix multiplication (GEMM) operations on NVIDIA GPUs since the Volta architecture. Because the artificial intelligence computations are usually dominated by GEMM operations, NVIDIA Tensor Core is critical for accelerating the artificial intelligence applications.

NVIDIA Tensor Core

NVIDIA Tensor Cores are specialized in performing the GEMM operations in mixed precision, i.e., the GEMM input matrices are in lower precision whereas the GEMM output matrix are in high precision. The mixed precision training and inference are the key techniques for accelerating the training and inference of neural networks.

Because NVIDIA Tensor Cores are specifically designed for GEMM, the GEMM throughput using NVIDIA Tensor Core is incredibly much higher than what can be achieved using NVIDIA CUDA Cores which are more suitable for more general parallel programming.

For the NVIDIA Ampere architecture, each SM has 4 Tensor Cores. In particular, NVIDIA A100 GPU has 108 streaming multiprocessors (SMs) which accounts for 432 Tensor Cores in total.

NVIDIA Tensor Cores are fully programmable. The Tensor Core programming API at the warp level has been declared in the mma.h header under the nvcuda::wmma namespace.

NVIDIA Tensor Core Programming

Matrix Multiplication Decomposition

NVIDIA CUDA allows the user to program Tensor Core GEMM operations $D = AB + C$ at the warp level. While each Tensor Core could only perform matrix multiplication of some specific small sizes for different data types, as discussed in my previous article “CUDA Matrix Multiplication”, large GEMM can be divided into multiple small GEMMs and accumulation.

Given a GEMM operation $D = AB + C$, where $D \in \mathbb{R}^{m \times n}$, $A \in \mathbb{R}^{m \times k}$, $B \in \mathbb{R}^{k \times n}$, $C \in \mathbb{R}^{m \times n}$, the matrices could be divided into smaller matrices.

$$A =\begin{bmatrix}A_{1,1}^{d \times d} & A_{1,2}^{d \times d} & \cdots & A_{1,k/d}^{d \times d} \\A_{2,1}^{d \times d} & A_{2,2}^{d \times d} & \cdots & A_{2,k/d}^{d \times d} \\\vdots & \vdots & \ddots & \vdots \\A_{m/d,1}^{d \times d} & A_{m/d,2}^{d \times d} & \cdots & A_{m/d,k/d}^{d \times d} \\\end{bmatrix}$$

$$B =\begin{bmatrix}B_{1,1}^{d \times d} & B_{1,2}^{d \times d} & \cdots & B_{1,n/d}^{d \times d} \\B_{2,1}^{d \times d} & B_{2,2}^{d \times d} & \cdots & B_{2,n/d}^{d \times d} \\\vdots & \vdots & \ddots & \vdots \\B_{k/d,1}^{d \times d} & B_{k/d,2}^{d \times d} & \cdots & B_{k/d,n/d}^{d \times d} \\\end{bmatrix}$$

$$C =\begin{bmatrix}C_{1,1}^{d \times d} & C_{1,2}^{d \times d} & \cdots & C_{1,n/d}^{d \times d} \\C_{2,1}^{d \times d} & C_{2,2}^{d \times d} & \cdots & C_{2,n/d}^{d \times d} \\\vdots & \vdots & \ddots & \vdots \\C_{m/d,1}^{d \times d} & C_{m/d,2}^{d \times d} & \cdots & C_{m/d,n/d}^{d \times d} \\\end{bmatrix}$$

$$D =\begin{bmatrix}D_{1,1}^{d \times d} & D_{1,2}^{d \times d} & \cdots & D_{1,n/d}^{d \times d} \\D_{2,1}^{d \times d} & D_{2,2}^{d \times d} & \cdots & D_{2,n/d}^{d \times d} \\\vdots & \vdots & \ddots & \vdots \\D_{m/d,1}^{d \times d} & D_{m/d,2}^{d \times d} & \cdots & D_{m/d,n/d}^{d \times d} \\\end{bmatrix}$$

Each small matrix in $D$ is computed as multiple small GEMMs and accumulation.

$$D_{i_m,i_n}^{d \times d} = \sum_{i_k=1}^{k/d} A_{i_m,i_k}^{d \times d} B_{i_k,i_n}^{d \times d}$$

In my previous article “CUDA Matrix Multiplication”, I used CUDA Core and CUDA shared memory to perform the above mathematics and each thread block computes one $D_{i_m,i_n}^{d \times d}$. This time instead, I will use Tensor Core to compute exactly the same mathematics where each warp computes one $D_{i_m,i_n}^{d \times d}$. More specifically, each warp computes a $16 \times 16 \times 16$ GEMM resulting in a $16 \times 16$ tile in the $D$ matrix, i.e., $d = 16$.

Matrix Multiplication Implementation Using NVIDIA Tensor Core

In this implementation, we will use Tensor Core to perform GEMM operations using HMMA (half matrix multiplication and accumulation) and IMMA (integer matrix multiplication and accumulation) instructions. In addition, four different types of GEMM which involves transposed matrix multiplications have been implemented and verified.

In this implementation, we will mainly focus on the matrix multiplication part in the GEMM operation by treating the $C = 0$.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599

#include <cassert>#include <chrono>#include <functional>#include <iomanip>#include <iostream>#include <random>#include <utility>#include <vector>#include <cuda_runtime.h>#include <mma.h>#define CHECK_CUDA_ERROR(val) check((val), #val, __FILE__, __LINE__)template <typename T>void check(T err, const char* const func, const char* const file,           int const line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() checkLast(__FILE__, __LINE__)void checkLast(const char* const file, int const line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}template <class T>float measure_performance(std::function<T(cudaStream_t)> bound_function,                          cudaStream_t stream, int num_repeats = 100,                          int num_warmups = 100){    cudaEvent_t start, stop;    float time;    CHECK_CUDA_ERROR(cudaEventCreate(&start));    CHECK_CUDA_ERROR(cudaEventCreate(&stop));    for (int i{0}; i < num_warmups; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaEventRecord(start, stream));    for (int i{0}; i < num_repeats; ++i)    {        bound_function(stream);    }    CHECK_CUDA_ERROR(cudaEventRecord(stop, stream));    CHECK_CUDA_ERROR(cudaEventSynchronize(stop));    CHECK_LAST_CUDA_ERROR();    CHECK_CUDA_ERROR(cudaEventElapsedTime(&time, start, stop));    CHECK_CUDA_ERROR(cudaEventDestroy(start));    CHECK_CUDA_ERROR(cudaEventDestroy(stop));    float const latency{time / num_repeats};    return latency;}// All the data in the matrices are stored in a column-major order,// which is the consistent with most of the cuBLAS GEMM APIs.// For matrix A of shape M x N, the leading dimension is M.// For matrix A that is transposed and is of shape N x M,// the leading dimension is N.// Matrix A: M x K, or K x N (if transposed).// Matrix B: K x M, or M x K (if transposed).// Matrix C: M x N.// WMMA_FRAG_LAYOUT_A: nvcuda::wmma::row_major if A is// transposed, otherwise nvcuda::wmma::col_major.// WMMA_FRAG_LAYOUT_B: nvcuda::wmma::row_major if B is// transposed, otherwise nvcuda::wmma::col_major.template <typename T1, typename T2, int WMMA_M, int WMMA_N, int WMMA_K,          typename WMMA_FRAG_LAYOUT_A, typename WMMA_FRAG_LAYOUT_B>__global__ void wmma_gemm_a_col_major_b_col_major(    T1 const* A, T1 const* B, T2* C, uint32_t m, uint32_t n, uint32_t k,    uint32_t lda, uint32_t ldb, uint32_t ldc, bool is_A_transpose,    bool is_B_transpose, float alpha, float beta){    // Tile using a 2D grid.    // Determine the warp 2D index.    uint32_t const warpM{(blockIdx.x * blockDim.x + threadIdx.x) / warpSize};    uint32_t const warpN{blockIdx.y * blockDim.y + threadIdx.y};    // Declare the fragments.    nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, WMMA_M, WMMA_N, WMMA_K, T1,                           WMMA_FRAG_LAYOUT_A>        a_frag{};    nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, WMMA_M, WMMA_N, WMMA_K, T1,                           WMMA_FRAG_LAYOUT_B>        b_frag{};    nvcuda::wmma::fragment<nvcuda::wmma::accumulator, WMMA_M, WMMA_N, WMMA_K,                           T2>        acc_frag{};    nvcuda::wmma::fragment<nvcuda::wmma::accumulator, WMMA_M, WMMA_N, WMMA_K,                           T2>        c_frag{};    // Make sure the accumulator starts from 0.    nvcuda::wmma::fill_fragment(acc_frag, static_cast<T2>(0));    // Loop over K.    for (uint32_t ki{0}; ki < k; ki += WMMA_K)    {        // Determine the first element of the mma matrices on the linear memory.        // Matrix A mma matrix        uint32_t const matrix_mma_a_row_idx{is_A_transpose ? ki                                                           : warpM * WMMA_M};        uint32_t const matrix_mma_a_col_idx{is_A_transpose ? warpM * WMMA_M                                                           : ki};        // Matrix B mma matrix        uint32_t const matrix_mma_b_row_idx{is_B_transpose ? warpN * WMMA_N                                                           : ki};        uint32_t const matrix_mma_b_col_idx{is_B_transpose ? ki                                                           : warpN * WMMA_N};        // Bounds checking        if (matrix_mma_a_row_idx < (is_A_transpose ? k : m) &&            matrix_mma_a_col_idx < (is_A_transpose ? m : k) &&            matrix_mma_b_row_idx < (is_B_transpose ? n : k) &&            matrix_mma_b_col_idx < (is_B_transpose ? k : n))        {            // Determine the memory address of the first element of the mma            // matrices. Notice that all the matrices are assumed to be            // column-major. Therefore, the indexing is different from the            // row-major indexing that we commonly see.            T1 const* matrix_mma_a_mptr{A + matrix_mma_a_row_idx +                                        matrix_mma_a_col_idx * lda};            T1 const* matrix_mma_b_mptr{B + matrix_mma_b_row_idx +                                        matrix_mma_b_col_idx * ldb};            // Load the mma matrix inputs.            nvcuda::wmma::load_matrix_sync(a_frag, matrix_mma_a_mptr, lda);            nvcuda::wmma::load_matrix_sync(b_frag, matrix_mma_b_mptr, ldb);            // Perform the matrix multiplication            nvcuda::wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);        }    }    // Load in the current value of c, scale it by beta, and add this our result    // scaled by alpha.    uint32_t const matrix_mma_c_row_idx{warpM * WMMA_M};    uint32_t const matrix_mma_c_col_idx{warpN * WMMA_N};    if (matrix_mma_c_row_idx < m && matrix_mma_c_col_idx < n)    {        T2* matrix_mma_c_mptr{C + matrix_mma_c_row_idx +                              matrix_mma_c_col_idx * ldc};        nvcuda::wmma::load_matrix_sync(c_frag, matrix_mma_c_mptr, ldc,                                       nvcuda::wmma::mem_col_major);        // Let the compiler figure out how to do the elementwise operation.        // Such elementwise operation can be scaling, accumulation,        // quantization, etc.        // https://docs.nvidia.com/cuda/archive/12.0.1/cuda-c-programming-guide/#id40        // Be careful when dealing with the integer types.        for (uint32_t i = 0; i < c_frag.num_elements; i++)        {            c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];        }        // Store the output        nvcuda::wmma::store_matrix_sync(matrix_mma_c_mptr, c_frag, ldc,                                        nvcuda::wmma::mem_col_major);    }}template <typename T1, typename T2>void launch_wmma_mm(T1 const* A, T1 const* B, T2* C, uint32_t m, uint32_t n,                    uint32_t k, bool is_A_transpose, bool is_B_transpose,                    cudaStream_t stream){    // Assume there is no padding in our data.    uint32_t const lda{is_A_transpose ? k : m};    uint32_t const ldb{is_B_transpose ? n : k};    uint32_t const ldc{m};    float const alpha{1.0f};    float const beta{0.0f};    constexpr int WMMA_M{16};    constexpr int WMMA_N{16};    constexpr int WMMA_K{16};    constexpr int WARP_SIZE{32};    dim3 gridDim;    dim3 blockDim;    // blockDim.x must be a multple of warpSize    // Block size of 128x4 means we have 16 (4x4) warps,    // each warp computes a 16x16 output tile,    // and a block computes a 64x64 output tile.    // Each block has 4x4 warps, totalling 4x4x32 threads.    int const num_warps_x = 4;    int const num_warps_y = 4;    blockDim.x = num_warps_x * WARP_SIZE;    blockDim.y = num_warps_y;    // Round up.    gridDim.x = (m + (WMMA_M * num_warps_x - 1)) / (WMMA_M * num_warps_x);    gridDim.y = (n + WMMA_N * num_warps_y - 1) / (WMMA_N * num_warps_y);    // C = A * B    if ((!is_A_transpose) && (!is_B_transpose))    {        wmma_gemm_a_col_major_b_col_major<T1, T2, WMMA_M, WMMA_N, WMMA_K,                                          nvcuda::wmma::col_major,                                          nvcuda::wmma::col_major>            <<<gridDim, blockDim, 0, stream>>>(A, B, C, m, n, k, lda, ldb, ldc,                                               is_A_transpose, is_B_transpose,                                               alpha, beta);    }    // C = A^T * B    else if ((is_A_transpose) && (!is_B_transpose))    {        wmma_gemm_a_col_major_b_col_major<T1, T2, WMMA_M, WMMA_N, WMMA_K,                                          nvcuda::wmma::row_major,                                          nvcuda::wmma::col_major>            <<<gridDim, blockDim, 0, stream>>>(A, B, C, m, n, k, lda, ldb, ldc,                                               is_A_transpose, is_B_transpose,                                               alpha, beta);    }    // C = A * B^T    else if ((!is_A_transpose) && (is_B_transpose))    {        wmma_gemm_a_col_major_b_col_major<T1, T2, WMMA_M, WMMA_N, WMMA_K,                                          nvcuda::wmma::col_major,                                          nvcuda::wmma::row_major>            <<<gridDim, blockDim, 0, stream>>>(A, B, C, m, n, k, lda, ldb, ldc,                                               is_A_transpose, is_B_transpose,                                               alpha, beta);    }    // C = A^T * B^T    else    {        wmma_gemm_a_col_major_b_col_major<T1, T2, WMMA_M, WMMA_N, WMMA_K,                                          nvcuda::wmma::row_major,                                          nvcuda::wmma::row_major>            <<<gridDim, blockDim, 0, stream>>>(A, B, C, m, n, k, lda, ldb, ldc,                                               is_A_transpose, is_B_transpose,                                               alpha, beta);    }    CHECK_LAST_CUDA_ERROR();}// A and B are column-major matrices.template <typename T1, typename T2>void mm_a_col_major_b_col_major(T1 const* A, T1 const* B, T2* C, uint32_t m,                                uint32_t n, uint32_t k, uint32_t lda,                                uint32_t ldb, uint32_t ldc, bool is_A_transpose,                                bool is_B_transpose){    for (uint32_t ni{0}; ni < n; ++ni)    {        for (uint32_t mi{0}; mi < m; ++mi)        {            // Compute C[mi, ni]            T2 accum{0};            // C = A * B            if ((!is_A_transpose) && (!is_B_transpose))            {                for (uint32_t ki{0}; ki < k; ++ki)                {                    // A[mi, ki] * B[ki, ni]                    accum += A[ki * lda + mi] * B[ni * ldb + ki];                }            }            // C = A^T * B            else if ((is_A_transpose) && (!is_B_transpose))            {                for (uint32_t ki{0}; ki < k; ++ki)                {                    // A[ki, mi] * B[ki, ni]                    accum += A[mi * lda + ki] * B[ni * ldb + ki];                }            }            // C = A * B^T            else if ((!is_A_transpose) && (is_B_transpose))            {                for (uint32_t ki{0}; ki < k; ++ki)                {                    // A[mi, ki] * B[ni, ki]                    accum += A[ki * lda + mi] * B[ki * ldb + ni];                }            }            // C = A^T * B^T            else            {                for (uint32_t ki{0}; ki < k; ++ki)                {                    // A[ki, mi] * B[ni, ki]                    accum += A[mi * lda + ki] * B[ki * ldb + ni];                }            }            C[ni * ldc + mi] = accum;        }    }}template <typename T1, typename T2>void launch_mm(T1 const* A, T1 const* B, T2* C, uint32_t m, uint32_t n,               uint32_t k, bool is_A_transpose, bool is_B_transpose){    // Assume there is no padding in our data.    uint32_t const lda{is_A_transpose ? k : m};    uint32_t const ldb{is_B_transpose ? n : k};    uint32_t const ldc{m};    mm_a_col_major_b_col_major(A, B, C, m, n, k, lda, ldb, ldc, is_A_transpose,                               is_B_transpose);}void fill_random_float_values(float* arr, size_t n,                              std::default_random_engine& e){    std::uniform_real_distribution<float> uniform_dist(-256, 256);    for (size_t i{0}; i < n; ++i)    {        arr[i] = uniform_dist(e);    }}void fill_random_int8_values(int8_t* arr, size_t n,                             std::default_random_engine& e){    std::uniform_int_distribution<int8_t> uniform_dist(-128, 127);    for (size_t i{0}; i < n; ++i)    {        arr[i] = uniform_dist(e);    }}void fill_random_int32_values(int32_t* arr, size_t n,                              std::default_random_engine& e){    std::uniform_int_distribution<int32_t> uniform_dist(-128, 127);    for (size_t i{0}; i < n; ++i)    {        arr[i] = uniform_dist(e);    }}void float2half(__half* half_arr, float const* float_arr, size_t n){    for (size_t i{0}; i < n; ++i)    {        half_arr[i] = __float2half(float_arr[i]);    }}template <typename T>float get_avg_abs_diff_ratio(T const* arr_1, T const* arr_2, size_t n){    float sum_abs_diff_ratio{0};    for (size_t i{0}; i < n; ++i)    {        sum_abs_diff_ratio += std::abs(static_cast<float>(arr_1[i]) -                                       static_cast<float>(arr_2[i])) /                              std::abs(static_cast<float>(arr_1[i]) +                                       static_cast<float>(arr_2[i]));    }    return sum_abs_diff_ratio / n;}template <typename T>bool array_equal(T const* arr_1, T const* arr_2, size_t n){    for (size_t i{0}; i < n; ++i)    {        if (arr_1[i] != arr_2[i])        {            return false;        }    }    return true;}void print_test_header(bool is_A_transpose, bool is_B_transpose){    // C = A * B    if ((!is_A_transpose) && (!is_B_transpose))    {        std::cout << "C = A * B" << std::endl;    }    // C = A^T * B    else if ((is_A_transpose) && (!is_B_transpose))    {        std::cout << "C = A^T * B" << std::endl;    }    // C = A * B^T    else if ((!is_A_transpose) && (is_B_transpose))    {        std::cout << "C = A * B^T" << std::endl;    }    // C = A^T * B^T    else    {        std::cout << "C = A^T * B^T" << std::endl;    }}int main(){    constexpr int num_repeats{10};    constexpr int num_warmups{10};    uint32_t const matrix_size_m{1024};    uint32_t const matrix_size_n{1024};    uint32_t const matrix_size_k{1024};    std::cout << "Matrix Sizes" << std::endl;    std::cout << "M: " << matrix_size_m << std::endl;    std::cout << "N: " << matrix_size_n << std::endl;    std::cout << "K: " << matrix_size_k << std::endl;    std::default_random_engine random_engine(0);    cudaStream_t stream;    CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    // HMMA    std::cout << "FP16 HMMA" << std::endl;    std::vector<float> matrix_a_float(matrix_size_m * matrix_size_k);    std::vector<float> matrix_b_float(matrix_size_k * matrix_size_n);    std::vector<__half> matrix_a_half(matrix_size_m * matrix_size_k);    std::vector<__half> matrix_b_half(matrix_size_k * matrix_size_n);    std::vector<float> matrix_c_float(matrix_size_m * matrix_size_n);    std::vector<float> matrix_c_float_reference(matrix_size_m * matrix_size_n);    float* h_matrix_a_float{matrix_a_float.data()};    float* h_matrix_b_float{matrix_b_float.data()};    __half* h_matrix_a_half{matrix_a_half.data()};    __half* h_matrix_b_half{matrix_b_half.data()};    float* h_matrix_c_float{matrix_c_float.data()};    float* h_matrix_c_float_reference{matrix_c_float_reference.data()};    fill_random_float_values(h_matrix_a_float, matrix_a_float.size(),                             random_engine);    fill_random_float_values(h_matrix_b_float, matrix_b_float.size(),                             random_engine);    fill_random_float_values(h_matrix_c_float, matrix_c_float.size(),                             random_engine);    fill_random_float_values(h_matrix_c_float_reference,                             matrix_c_float_reference.size(), random_engine);    float2half(h_matrix_a_half, h_matrix_a_float, matrix_a_float.size());    float2half(h_matrix_b_half, h_matrix_b_float, matrix_b_float.size());    half *d_matrix_a_half, *d_matrix_b_half;    float* d_matrix_c_float;    CHECK_CUDA_ERROR(cudaMalloc(&d_matrix_a_half,                                matrix_size_m * matrix_size_k * sizeof(half)));    CHECK_CUDA_ERROR(cudaMalloc(&d_matrix_b_half,                                matrix_size_k * matrix_size_n * sizeof(half)));    CHECK_CUDA_ERROR(cudaMalloc(&d_matrix_c_float,                                matrix_size_m * matrix_size_n * sizeof(float)));    // Copy data from host to device.    CHECK_CUDA_ERROR(cudaMemcpy(d_matrix_a_half, h_matrix_a_half,                                matrix_a_float.size() * sizeof(__half),                                cudaMemcpyHostToDevice));    CHECK_CUDA_ERROR(cudaMemcpy(d_matrix_b_half, h_matrix_b_half,                                matrix_b_float.size() * sizeof(__half),                                cudaMemcpyHostToDevice));    for (bool is_A_transpose : {true, false})    {        for (bool is_B_transpose : {true, false})        {            print_test_header(is_A_transpose, is_B_transpose);            // Compute matrix multiplication reference output using CPU.            launch_mm(h_matrix_a_float, h_matrix_b_float,                      h_matrix_c_float_reference, matrix_size_m, matrix_size_n,                      matrix_size_k, is_A_transpose, is_B_transpose);            // Compute matrix multiplication reference output using CUDA WMMA.            launch_wmma_mm(d_matrix_a_half, d_matrix_b_half, d_matrix_c_float,                           matrix_size_m, matrix_size_n, matrix_size_k,                           is_A_transpose, is_B_transpose, stream);            CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));            CHECK_CUDA_ERROR(cudaMemcpy(h_matrix_c_float, d_matrix_c_float,                                        matrix_c_float.size() * sizeof(float),                                        cudaMemcpyDeviceToHost));            float const avg_abs_diff_ratio{get_avg_abs_diff_ratio(                h_matrix_c_float, h_matrix_c_float_reference,                matrix_c_float.size())};            if (avg_abs_diff_ratio > 0.01)            {                std::cout << "Got high average absolute diff ratio: "                          << avg_abs_diff_ratio << std::endl;            }            // Performance measurement.            std::function<void(cudaStream_t)> const function_hmma{std::bind(                launch_wmma_mm<__half, float>, d_matrix_a_half, d_matrix_b_half,                d_matrix_c_float, matrix_size_m, matrix_size_n, matrix_size_k,                is_A_transpose, is_B_transpose, std::placeholders::_1)};            float const latency_hmma{measure_performance(                function_hmma, stream, num_repeats, num_warmups)};            std::cout << std::fixed << std::setprecision(3)                      << "HMMA Latency: " << latency_hmma << " ms" << std::endl;        }    }    CHECK_CUDA_ERROR(cudaFree(d_matrix_a_half));    CHECK_CUDA_ERROR(cudaFree(d_matrix_b_half));    CHECK_CUDA_ERROR(cudaFree(d_matrix_c_float));    // IMMA    std::cout << "INT8 IMMA" << std::endl;    std::vector<int8_t> matrix_a_int8(matrix_size_m * matrix_size_k);    std::vector<int8_t> matrix_b_int8(matrix_size_k * matrix_size_n);    std::vector<int32_t> matrix_c_int32(matrix_size_m * matrix_size_n);    std::vector<int32_t> matrix_c_int32_reference(matrix_size_m *                                                  matrix_size_n);    int8_t* h_matrix_a_int8{matrix_a_int8.data()};    int8_t* h_matrix_b_int8{matrix_b_int8.data()};    int32_t* h_matrix_c_int32{matrix_c_int32.data()};    int32_t* h_matrix_c_int32_reference{matrix_c_int32_reference.data()};    fill_random_int8_values(h_matrix_a_int8, matrix_a_int8.size(),                            random_engine);    fill_random_int8_values(h_matrix_b_int8, matrix_b_int8.size(),                            random_engine);    fill_random_int32_values(h_matrix_c_int32, matrix_c_int32.size(),                             random_engine);    fill_random_int32_values(h_matrix_c_int32_reference,                             matrix_c_int32_reference.size(), random_engine);    // Profile INT8 IMMA without verifying the correctness.    int8_t *d_matrix_a_int8, *d_matrix_b_int8;    int32_t* d_matrix_c_int32;    CHECK_CUDA_ERROR(cudaMalloc(        &d_matrix_a_int8, matrix_size_m * matrix_size_k * sizeof(int8_t)));    CHECK_CUDA_ERROR(cudaMalloc(        &d_matrix_b_int8, matrix_size_k * matrix_size_n * sizeof(int8_t)));    CHECK_CUDA_ERROR(cudaMalloc(        &d_matrix_c_int32, matrix_size_m * matrix_size_n * sizeof(int32_t)));    CHECK_CUDA_ERROR(cudaMemcpy(d_matrix_a_int8, h_matrix_a_int8,                                matrix_a_int8.size() * sizeof(int8_t),                                cudaMemcpyHostToDevice));    CHECK_CUDA_ERROR(cudaMemcpy(d_matrix_b_int8, h_matrix_b_int8,                                matrix_b_int8.size() * sizeof(int8_t),                                cudaMemcpyHostToDevice));    for (bool is_A_transpose : {true, false})    {        for (bool is_B_transpose : {true, false})        {            print_test_header(is_A_transpose, is_B_transpose);            // Compute matrix multiplication reference output using CPU.            launch_mm(h_matrix_a_int8, h_matrix_b_int8,                      h_matrix_c_int32_reference, matrix_size_m, matrix_size_n,                      matrix_size_k, is_A_transpose, is_B_transpose);            // Compute matrix multiplication reference output using CUDA WMMA.            launch_wmma_mm(d_matrix_a_int8, d_matrix_b_int8, d_matrix_c_int32,                           matrix_size_m, matrix_size_n, matrix_size_k,                           is_A_transpose, is_B_transpose, stream);            CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));            CHECK_CUDA_ERROR(cudaMemcpy(h_matrix_c_int32, d_matrix_c_int32,                                        matrix_c_int32.size() * sizeof(int32_t),                                        cudaMemcpyDeviceToHost));            // Integer matrix multiplications from CPU and CUDA should be            // bitwise identical.            assert(array_equal(h_matrix_c_int32, h_matrix_c_int32_reference,                               matrix_c_int32.size()));            // Performance measurement.            std::function<void(cudaStream_t)> const function_imma{                std::bind(launch_wmma_mm<int8_t, int32_t>, d_matrix_a_int8,                          d_matrix_b_int8, d_matrix_c_int32, matrix_size_m,                          matrix_size_n, matrix_size_k, is_A_transpose,                          is_B_transpose, std::placeholders::_1)};            float const latency_imma{measure_performance(                function_imma, stream, num_repeats, num_warmups)};            std::cout << std::fixed << std::setprecision(3)                      << "IMMA Latency: " << latency_imma << " ms" << std::endl;        }    }    CHECK_CUDA_ERROR(cudaFree(d_matrix_a_int8));    CHECK_CUDA_ERROR(cudaFree(d_matrix_b_int8));    CHECK_CUDA_ERROR(cudaFree(d_matrix_c_int32));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));}

All the transposed matrix multiplication implementations did not actually transpose the matrices. Instead, we used the row-major and column-major trick introduced in my previous article “Row-Major VS Column-Major”.

We also observed that for matrix multiplication for matrices stored in column-major order, $C = A^{\top}B$ is the fastest and $C = A B^{\top}$ is the slowest, for GEMM implementations using HMMA and IMMA instructions on an NVIDIA RTX 3090 GPU.

123456789101112131415161718192021222324

$ nvcc mma.cu -o mma --gpu-architecture=compute_86$ ./mmaMatrix SizesM: 1024N: 1024K: 1024FP16 HMMAC = A^T * B^THMMA Latency: 0.177 msC = A^T * BHMMA Latency: 0.169 msC = A * B^THMMA Latency: 0.189 msC = A * BHMMA Latency: 0.177 msINT8 IMMAC = A^T * B^TIMMA Latency: 0.129 msC = A^T * BIMMA Latency: 0.090 msC = A * B^TIMMA Latency: 0.170 msC = A * BIMMA Latency: 0.129 ms

Conclusions

NVIDIA Tensor Cores are programmable and can be used for accelerating computations that are dominated by GEMM operations.

References

NVIDIA Tensor Core Programming

https://leimao.github.io/blog/NVIDIA-Tensor-Core-Programming/

Author

Lei Mao

Posted on

05-18-2023

Updated on

12-27-2023

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV:


Row-Major VS Column-Major

Introduction

In computing, row-major order and column-major order are two ways for storing multidimensional arrays in linear storage such as random access memory. The following figure demonstrated the row-major order and the column-major for a 2D matrix.

In this blog post, I would like to discuss the difference between the row-major order and the column-major order, and their consequence for matrix multiplication performance.

Row-Major VS Column-Major

Given a matrix $A$ of shape $(M, N)$, if it is stored in row-major order, its leading dimension is $N$, and if it is stored in column-major order, its leading dimension is $M$.

To read $A^{\top}$ from the same piece of the memory in which $A$ was stored in row-major order with a leading dimension of $N$, we could just treat the matrix in the memory as if it were stored in column-major order and the leading dimension is still $N$.

To read $A^{\top}$ from the same piece of the memory in which $A$ was stored in column-major order with a leading dimension of $M$, we could just treat the matrix in the memory as if it were stored in row-major order and the leading dimension is still $M$.

For example, we have a matrix $A$,

$$\begin{align}A &=\begin{bmatrix}1 & 2 & 3 \\4 & 5 & 6 \\\end{bmatrix} \\\end{align}$$

If $A$ is stored in row-major order, the matrix values in the linear memory is $[1, 2, 3, 4, 5, 6]$.If $A$ is stored in column-major order, the matrix values in the linear memory is $[1, 4, 2, 5, 3, 6]$.

The transpose of $A$, $A^{\top}$, is

$$\begin{align}A^{\top} &=\begin{bmatrix}1 & 4 \\2 & 5 \\3 & 6 \\\end{bmatrix} \\\end{align}$$

If $A^{\top}$ is stored in row-major order, the matrix values in the linear memory is $[1, 4, 2, 5, 3, 6]$.If $A^{\top}$ is stored in column-major order, the matrix values in the linear memory is $[1, 2, 3, 4, 5, 6]$.

It’s easy to see that $A$ is stored in row-major order is exactly the same as $A^{\top}$ is stored in column-major order in the memory and $A$ is stored in column-major order is exactly the same as $A^{\top}$ is stored in row-major order in the memory.

For a matrix $A$ stored in row-major order, reading rows of $A$ and reading columns of $A^{\top}$ are fast and cache-friendly, whereas reading columns of $A$ and reading rows of $A^{\top}$ are slow and invalids caching.

For a matrix $A$ stored in column-major order, reading columns of $A$ and reading rows of $A^{\top}$ are fast and cache-friendly, whereas reading rows of $A$ and reading columns of $A^{\top}$ are slow and invalids caching.

Matrix Multiplication

The way of storing matrices in memory affects the performance of matrix multiplication on lots of processors, such as CPU and GPU. Usually, depending on whether the matrices for multiplications needs to be mathematically transposed for matrix multiplication, there are four ways of computing matrix multiplication, $C=AB$, $C=A^{\top}B$, $C=AB^{\top}$, and $C=A^{\top}B^{\top}$. Each of them performs better than others depending on the storage orderings of matrices $A$ and $B$, even though the theoretical MACs of the operations remain the same.

$C=AB$

Suppose a matrix $A$ is of shape $(M, K)$ and a matrix $B$ is of shape $(K, N)$, to compute $C = AB$ where $C$ is an matrix of shape $(M, N)$, each element in $C$ is an accumulated sum of one row of size $K$ in the matrix $A$ and one column of size $K$ in the matrix $B$.

Depending on the storage ordering of the two matrices, there are four scenarios.

When $A$ is stored in row-major order and $B$ is stored in column-major order, reading both rows from $A$ and columns from $B$ are fast for matrix multiplication because of the caching mechanism of modern processors, and faster readings results in better performance, given the same amount of math to compute.

Therefore, the matrix multiplication $C=AB$ is more suitable for the situation where $A$ is stored in row-major order and $B$ is stored in column-major order.

$C=A^{\top}B$

Suppose a matrix $A$ is of shape $(K, M)$ and a matrix $B$ is of shape $(K, N)$, to compute $C = A^{\top}B$ where $C$ is an matrix of shape $(M, N)$, each element in $C$ is an accumulated sum of one column of size $K$ in the matrix $A$ and one column of size $K$ in the matrix $B$.

Depending on the storage ordering of the two matrices, there are four scenarios.

When $A$ is stored in column-major order and $B$ is stored in column-major order, reading both columns from $A$ and columns from $B$ are fast for matrix multiplication because of the caching mechanism of modern processors, and faster readings results in better performance, given the same amount of math to compute.

Therefore, the matrix multiplication $C=A^{\top}B$ is more suitable for the situation where $A$ is stored in column-major order and $B$ is stored in column-major order.

$C=AB^{\top}$

Suppose a matrix $A$ is of shape $(M, K)$ and a matrix $B$ is of shape $(N, K)$, to compute $C = AB^{\top}$ where $C$ is an matrix of shape $(M, N)$, each element in $C$ is an accumulated sum of one row of size $K$ in the matrix $A$ and one row of size $K$ in the matrix $B$.

Depending on the storage ordering of the two matrices, there are four scenarios.

When $A$ is stored in row-major order and $B$ is stored in row-major order, reading both rows from $A$ and rows from $B$ are fast for matrix multiplication because of the caching mechanism of modern processors, and faster readings results in better performance, given the same amount of math to compute.

Therefore, the matrix multiplication $C=AB^{\top}$ is more suitable for the situation where $A$ is stored in row-major order and $B$ is stored in row-major order.

$C=A^{\top}B^{\top}$

Suppose a matrix $A$ is of shape $(K, M)$ and a matrix $B$ is of shape $(N, K)$, to compute $C = A^{\top}B^{\top}$ where $C$ is an matrix of shape $(M, N)$, each element in $C$ is an accumulated sum of one column of size $K$ in the matrix $A$ and one row of size $K$ in the matrix $B$.

Depending on the storage ordering of the two matrices, there are four scenarios.

When $A$ is stored in column-major order and $B$ is stored in row-major order, reading both columns from $A$ and rows from $B$ are fast for matrix multiplication because of the caching mechanism of modern processors, and faster readings results in better performance, given the same amount of math to compute.

Therefore, the matrix multiplication $C=A^{\top}B^{\top}$ is more suitable for the situation where $A$ is stored in column-major order and $B$ is stored in row-major order.

Matrix Multiplication Preference

The matrix multiplication preference for different combinations of the storage orders of matrices beings multiplied can be summarized as follows.

Because usually the all matrices in one software framework would use the same storage order, this means only $C = A^{\top}B$ and $C = AB^{\top}$ are preferred for those scenarios.

Optimizations can reduce the performance gap between the optimal matrix multiplication option and the other options, sometimes even to almost zero, depending on the implementation and the processor.

In addition, sometimes it’s not a good idea to physically transpose a matrix in memory just in order to use the most performant matrix multiplication option among the four, because the overhead of transposing a matrix in memory might be much larger than the difference between the four matrix multiplication options especially when they are all well optimized.

Matrix Multiplication Benchmarks

Additionally, we could verify our analysis using C++ single-threaded naive implementations for matrix multiplication.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177

#include <cassert>#include <chrono>#include <cstdint>#include <functional>#include <iomanip>#include <iostream>#include <tuple>#include <utility>#include <vector>template <class T>float measure_performance(std::function<T(void)> bound_function,                          int num_repeats = 100, int num_warmups = 100){    for (int i{0}; i < num_warmups; ++i)    {        bound_function();    }    std::chrono::steady_clock::time_point time_start{        std::chrono::steady_clock::now()};    for (int i{0}; i < num_repeats; ++i)    {        bound_function();    }    std::chrono::steady_clock::time_point time_end{        std::chrono::steady_clock::now()};    auto time_elapsed{std::chrono::duration_cast<std::chrono::milliseconds>(                          time_end - time_start)                          .count()};    float latency{time_elapsed / static_cast<float>(num_repeats)};    return latency;}// A and B are column-major matrices.template <typename T>void mm_a_col_major_b_col_major(T const* A, T const* B, T* C, uint32_t m,                                uint32_t n, uint32_t k, uint32_t lda,                                uint32_t ldb, uint32_t ldc, bool is_A_transpose,                                bool is_B_transpose){    for (uint32_t ni{0}; ni < n; ++ni)    {        for (uint32_t mi{0}; mi < m; ++mi)        {            // Compute C[mi, ni]            T accum{0};            // A * B            if ((!is_A_transpose) && (!is_B_transpose))            {                for (uint32_t ki{0}; ki < k; ++ki)                {                    // A[mi, ki] * B[ki, ni]                    accum += A[ki * lda + mi] * B[ni * ldb + ki];                }            }            // A^T * B            else if ((is_A_transpose) && (!is_B_transpose))            {                for (uint32_t ki{0}; ki < k; ++ki)                {                    // A[ki, mi] * B[ki, ni]                    accum += A[mi * lda + ki] * B[ni * ldb + ki];                }            }            // A * B^T            else if ((!is_A_transpose) && (is_B_transpose))            {                for (uint32_t ki{0}; ki < k; ++ki)                {                    // A[mi, ki] * B[ni, ki]                    accum += A[ki * lda + mi] * B[ki * ldb + ni];                }            }            // A^T * B^T            else            {                for (uint32_t ki{0}; ki < k; ++ki)                {                    // A[ki, mi] * B[ni, ki]                    accum += A[mi * lda + ki] * B[ki * ldb + ni];                }            }            C[ni * ldc + mi] = accum;        }    }}void print_latency(float latency){    std::cout << std::fixed << std::setprecision(3) << "Latency: " << latency              << " ms" << std::endl;}int main(){    constexpr uint32_t num_repeats{10};    constexpr uint32_t num_warmups{10};    constexpr uint32_t M{256};    constexpr uint32_t K{256};    constexpr uint32_t N{256};    std::vector<float> matrix_a(M * K);    std::vector<float> matrix_b(K * N);    std::vector<float> matrix_c(M * N);    float const* A{matrix_a.data()};    float const* B{matrix_b.data()};    float* C{matrix_c.data()};    uint32_t const matrix_a_col_major_ld{M};    uint32_t const matrix_a_row_major_ld{K};    uint32_t const matrix_a_transpose_col_major_ld{matrix_a_row_major_ld};    uint32_t const matrix_a_transpose_row_major_ld{matrix_a_col_major_ld};    uint32_t const matrix_b_col_major_ld{K};    uint32_t const matrix_b_row_major_ld{N};    uint32_t const matrix_b_transpose_col_major_ld{matrix_b_row_major_ld};    uint32_t const matrix_b_transpose_row_major_ld{matrix_b_col_major_ld};    uint32_t const matrix_c_col_major_ld{M};    uint32_t const matrix_c_row_major_ld{N};    uint32_t const matrix_c_transpose_col_major_ld{matrix_c_row_major_ld};    uint32_t const matrix_c_transpose_row_major_ld{matrix_c_col_major_ld};    std::function<void(void)> const mm_a_col_major_b_col_major_a_b{        std::bind(mm_a_col_major_b_col_major<float>, A, B, C, M, N, K,                  matrix_a_col_major_ld, matrix_b_col_major_ld,                  matrix_c_col_major_ld, false, false)};    std::function<void(void)> const mm_a_col_major_b_col_major_a_transpose_b{        std::bind(mm_a_col_major_b_col_major<float>, A, B, C, M, N, K,                  matrix_a_transpose_col_major_ld, matrix_b_col_major_ld,                  matrix_c_col_major_ld, true, false)};    std::function<void(void)> const        mm_a_col_major_b_col_major_a_transpose_b_transpose{std::bind(            mm_a_col_major_b_col_major<float>, A, B, C, M, N, K,            matrix_a_transpose_col_major_ld, matrix_b_transpose_col_major_ld,            matrix_c_col_major_ld, true, true)};    std::function<void(void)> const mm_a_col_major_b_col_major_a_b_transpose{        std::bind(mm_a_col_major_b_col_major<float>, A, B, C, M, N, K,                  matrix_a_col_major_ld, matrix_b_transpose_col_major_ld,                  matrix_c_col_major_ld, false, true)};    std::cout << "C = A * B" << std::endl;    float const latency_a_b = measure_performance(        mm_a_col_major_b_col_major_a_b, num_repeats, num_warmups);    print_latency(latency_a_b);    std::cout << "C = A^T * B" << std::endl;    float const latency_a_transpose_b = measure_performance(        mm_a_col_major_b_col_major_a_transpose_b, num_repeats, num_warmups);    print_latency(latency_a_transpose_b);    std::cout << "C = A * B^T" << std::endl;    float const latency_a_b_transpose = measure_performance(        mm_a_col_major_b_col_major_a_b_transpose, num_repeats, num_warmups);    print_latency(latency_a_b_transpose);    std::cout << "C = A^T * B^T" << std::endl;    float const latency_a_transpose_b_transpose =        measure_performance(mm_a_col_major_b_col_major_a_transpose_b_transpose,                            num_repeats, num_warmups);    print_latency(latency_a_transpose_b_transpose);    assert(latency_a_transpose_b ==           std::min({latency_a_b, latency_a_transpose_b, latency_a_b_transpose,                     latency_a_transpose_b_transpose}));    assert(latency_a_b_transpose ==           std::max({latency_a_b, latency_a_transpose_b, latency_a_b_transpose,                     latency_a_transpose_b_transpose}));}

We could see that given matrix $A$ and matrix $B$ are stored in column-major order, as expected, the performance of $C = A^{\top}B$ is the best and the performance of $C = AB^{\top}$ is the worst.

12345678910

$ g++ naive_mm.cpp -o naive_mm$ ./naive_mmC = A * BLatency: 45.400 msC = A^T * BLatency: 32.500 msC = A * B^TLatency: 57.800 msC = A^T * B^TLatency: 48.300 ms

Using multi-threaded optimized matrix multiplication implementations, such as the GEMM functions from the cuBLAS library, can eliminate the difference between the four options.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188

#include <cassert>#include <chrono>#include <cstdint>#include <functional>#include <iomanip>#include <iostream>#include <tuple>#include <utility>#include <vector>#include <cublas_v2.h>#include <cuda_runtime.h>#define CHECK_CUBLAS_ERROR(val) checkCuBlas((val), #val, __FILE__, __LINE__)template <typename T>void checkCuBlas(T err, const char* const func, const char* const file,                 const int line){    if (err != CUBLAS_STATUS_SUCCESS)    {        std::cerr << "cuBlas Runtime Error at: " << file << ":" << line                  << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_CUDA_ERROR(val) checkCuda((val), #val, __FILE__, __LINE__)template <typename T>void checkCuda(T err, const char* const func, const char* const file,               const int line){    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << " " << func << std::endl;        std::exit(EXIT_FAILURE);    }}#define CHECK_LAST_CUDA_ERROR() checkCudaLast(__FILE__, __LINE__)void checkCudaLast(const char* const file, const int line){    cudaError_t const err{cudaGetLastError()};    if (err != cudaSuccess)    {        std::cerr << "CUDA Runtime Error at: " << file << ":" << line                  << std::endl;        std::cerr << cudaGetErrorString(err) << std::endl;        std::exit(EXIT_FAILURE);    }}float measure_cublas_performance(    std::function<cublasStatus_t(void)> bound_cublas_function,    cudaStream_t stream, int num_repeats = 100, int num_warmups = 100){    cudaEvent_t start, stop;    float time;    CHECK_CUDA_ERROR(cudaEventCreate(&start));    CHECK_CUDA_ERROR(cudaEventCreate(&stop));    for (int i{0}; i < num_warmups; ++i)    {        CHECK_CUBLAS_ERROR(bound_cublas_function());    }    CHECK_CUDA_ERROR(cudaStreamSynchronize(stream));    CHECK_CUDA_ERROR(cudaEventRecord(start, stream));    for (int i{0}; i < num_repeats; ++i)    {        CHECK_CUBLAS_ERROR(bound_cublas_function());    }    CHECK_CUDA_ERROR(cudaEventRecord(stop, stream));    CHECK_CUDA_ERROR(cudaEventSynchronize(stop));    CHECK_LAST_CUDA_ERROR();    CHECK_CUDA_ERROR(cudaEventElapsedTime(&time, start, stop));    CHECK_CUDA_ERROR(cudaEventDestroy(start));    CHECK_CUDA_ERROR(cudaEventDestroy(stop));    float const latency{time / num_repeats};    return latency;}void print_latency(float latency){    std::cout << std::fixed << std::setprecision(3) << "Latency: " << latency              << " ms" << std::endl;}int main(){    constexpr uint32_t num_repeats{100};    constexpr uint32_t num_warmups{100};    constexpr uint32_t M{256};    constexpr uint32_t K{256};    constexpr uint32_t N{256};    float* A{nullptr};    float* B{nullptr};    float* C{nullptr};    CHECK_CUDA_ERROR(cudaMalloc(&A, M * K * sizeof(float)));    CHECK_CUDA_ERROR(cudaMalloc(&B, K * N * sizeof(float)));    CHECK_CUDA_ERROR(cudaMalloc(&C, M * N * sizeof(float)));    uint32_t const matrix_a_col_major_ld{M};    uint32_t const matrix_a_row_major_ld{K};    uint32_t const matrix_a_transpose_col_major_ld{matrix_a_row_major_ld};    uint32_t const matrix_a_transpose_row_major_ld{matrix_a_col_major_ld};    uint32_t const matrix_b_col_major_ld{K};    uint32_t const matrix_b_row_major_ld{N};    uint32_t const matrix_b_transpose_col_major_ld{matrix_b_row_major_ld};    uint32_t const matrix_b_transpose_row_major_ld{matrix_b_col_major_ld};    uint32_t const matrix_c_col_major_ld{M};    uint32_t const matrix_c_row_major_ld{N};    uint32_t const matrix_c_transpose_col_major_ld{matrix_c_row_major_ld};    uint32_t const matrix_c_transpose_row_major_ld{matrix_c_col_major_ld};    cublasHandle_t cublas_handle;    cudaStream_t stream;    CHECK_CUDA_ERROR(cudaStreamCreate(&stream));    CHECK_CUBLAS_ERROR(cublasCreate(&cublas_handle));    CHECK_CUBLAS_ERROR(cublasSetStream(cublas_handle, stream));    float const alpha{1.0};    float const beta{0.0};    // cublasSgemm assumes column-major matrices.    std::function<cublasStatus_t(void)> const mm_a_col_major_b_col_major_a_b{        std::bind(cublasSgemm, cublas_handle, CUBLAS_OP_N, CUBLAS_OP_N, M, N, K,                  &alpha, A, matrix_a_col_major_ld, B, matrix_b_col_major_ld,                  &beta, C, matrix_c_col_major_ld)};    std::function<cublasStatus_t(void)> const        mm_a_col_major_b_col_major_a_transpose_b{            std::bind(cublasSgemm, cublas_handle, CUBLAS_OP_T, CUBLAS_OP_N, M,                      N, K, &alpha, A, matrix_a_transpose_col_major_ld, B,                      matrix_b_col_major_ld, &beta, C, matrix_c_col_major_ld)};    std::function<cublasStatus_t(void)> const        mm_a_col_major_b_col_major_a_transpose_b_transpose{std::bind(            cublasSgemm, cublas_handle, CUBLAS_OP_T, CUBLAS_OP_T, M, N, K,            &alpha, A, matrix_a_transpose_col_major_ld, B,            matrix_b_transpose_col_major_ld, &beta, C, matrix_c_col_major_ld)};    std::function<cublasStatus_t(void)> const        mm_a_col_major_b_col_major_a_b_transpose{std::bind(            cublasSgemm, cublas_handle, CUBLAS_OP_N, CUBLAS_OP_T, M, N, K,            &alpha, A, matrix_a_col_major_ld, B,            matrix_b_transpose_col_major_ld, &beta, C, matrix_c_col_major_ld)};    std::cout << "C = A * B" << std::endl;    float const latency_a_b = measure_cublas_performance(        mm_a_col_major_b_col_major_a_b, stream, num_repeats, num_warmups);    print_latency(latency_a_b);    std::cout << "C = A^T * B" << std::endl;    float const latency_a_transpose_b =        measure_cublas_performance(mm_a_col_major_b_col_major_a_transpose_b,                                   stream, num_repeats, num_warmups);    print_latency(latency_a_transpose_b);    std::cout << "C = A * B^T" << std::endl;    float const latency_a_b_transpose =        measure_cublas_performance(mm_a_col_major_b_col_major_a_b_transpose,                                   stream, num_repeats, num_warmups);    print_latency(latency_a_b_transpose);    std::cout << "C = A^T * B^T" << std::endl;    float const latency_a_transpose_b_transpose = measure_cublas_performance(        mm_a_col_major_b_col_major_a_transpose_b_transpose, stream, num_repeats,        num_warmups);    print_latency(latency_a_transpose_b_transpose);    CHECK_CUDA_ERROR(cudaFree(A));    CHECK_CUDA_ERROR(cudaFree(B));    CHECK_CUDA_ERROR(cudaFree(C));    CHECK_CUBLAS_ERROR(cublasDestroy(cublas_handle));    CHECK_CUDA_ERROR(cudaStreamDestroy(stream));}

With the highly optimized implementations, there is almost no difference between the four options.

12345678910

$ nvcc cublas_mm.cu -o cublas_mm -lcublas$ ./cublas_mmC = A * BLatency: 0.008 msC = A^T * BLatency: 0.010 msC = A * B^TLatency: 0.009 msC = A^T * B^TLatency: 0.008 ms

References

Row-Major VS Column-Major

https://leimao.github.io/blog/Row-Major-VS-Column-Major/

Author

Lei Mao

Posted on

05-12-2023

Updated on

05-12-2023

Licensed under

Like this article? Support the author with

Comments

Lei Mao

Artificial Intelligence
Machine Learning
Computer Science

Santa Clara, California

Posts

1008

Categories

7

Tags

662

Advertisement

Catalogue

© 2017-2025 Lei Mao  Powered by Hexo & IcarusSite UV:  Site PV: