common/nn_data_tools.cpp
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Read all labels from given mnist labels file and all corressponding images from mnist images file
Read all labels from given mnist labels file and all corressponding images from mnist images file
Read all labels from given mnist labels file and all corressponding images from mnist images file
void nn_data_load_images_and_labels_from_mnist_files( nn::data< float, 3 >* &images, nn::data< char, 1 >* &labels, std::string &mnist_images, std::string &mnist_labels)
{
    char int_read[sizeof(uint32_t)];
    const int mnist_labels_magic_number = 2049;
    const int mnist_images_magic_number = 2051;
    auto block_to_int = [](char *memblock)
    {
        uint32_t int_value = ((((((uint8_t)memblock[0] << 8) + (uint8_t)memblock[1]) << 8 )+ (uint8_t)memblock[2]) << 8)  + (uint8_t)memblock[3];
        return int_value;
    };
    std::ifstream lifs;
    lifs.open(mnist_labels, std::ifstream::binary );
    if(lifs.good() == false) {
        std::string err_msg("Error reading: " + mnist_labels);
        throw std::runtime_error(err_msg.c_str());
    }
    lifs.read(int_read,sizeof(uint32_t));
    uint32_t magic_number = block_to_int(int_read);
    if( magic_number != mnist_labels_magic_number ) {
        std::string err_msg("Error reading MNIST Labels eg. Wrong Magic number read: ");
        err_msg += std::to_string(magic_number );
        throw std::runtime_error(err_msg);
    }
    lifs.read(int_read,sizeof(uint32_t));
    uint32_t num_labels = block_to_int(int_read);
    labels = new nn::data<char, 1>(num_labels);
    char* pbuf = (char*)labels->buffer;
    for(unsigned int l=0 ; l< num_labels; ++l)
    {
        lifs.read(pbuf+l,sizeof(char));
    }
    lifs.close();
    std::ifstream ifs;
    ifs.open(mnist_images, std::ifstream::binary );
    if(ifs.good() == false) {
        std::string err_msg("Error reading: " + mnist_images);
        throw std::runtime_error(err_msg.c_str());
    }
    ifs.read(int_read,sizeof(uint32_t));
    magic_number = block_to_int(int_read);
    if( magic_number != mnist_images_magic_number ) {
        std::string err_msg("Error reading MNIST Images eg. Wrong Magic number read: ");
        err_msg += std::to_string(magic_number);
        throw std::runtime_error(err_msg);
    }
    ifs.read(int_read,sizeof(uint32_t));
    uint32_t num_images = block_to_int(int_read);
    ifs.read(int_read,sizeof(uint32_t));
    uint32_t images_width = block_to_int(int_read);
    ifs.read(int_read,sizeof(uint32_t));
    uint32_t images_height = block_to_int(int_read);
    images = new nn::data<float, 3>(images_width, images_height, num_images);
    std::unique_ptr<char[]> single_image(new char[images_height * images_width]);
    for(unsigned int img=0; img < num_images; ++img)
    {
        ifs.read(single_image.get(),images_width*images_height*sizeof(char));
        float* pdata_image = (float*)images->buffer + img*images_width*images_height;   
        for(unsigned int pix=0; pix < images_width*images_height; ++pix )
        {
             pdata_image[pix]= *((unsigned char*)single_image.get() + pix)*1.0f/255.0f;
        }
    }
    ifs.close();
}
void nn_data_load_images_and_labels_from_mnist_files ( nn :: data < float , 3 > * & images , nn :: data < char , 1 > * & labels , std :: string & mnist_images , std :: string & mnist_labels ) { char int_read [ sizeof ( uint32_t ) ] ; const int mnist_labels_magic_number = 2049 ; const int mnist_images_magic_number = 2051 ; auto block_to_int = [ ] ( char * memblock ) { uint32_t int_value = ( ( ( ( ( ( uint8_t ) memblock [ 0 ] << 8 ) + ( uint8_t ) memblock [ 1 ] ) << 8 ) + ( uint8_t ) memblock [ 2 ] ) << 8 ) + ( uint8_t ) memblock [ 3 ] ; return int_value ; } ; std :: ifstream lifs ; lifs . open ( mnist_labels , std :: ifstream :: binary ) ; if ( lifs . good ( ) == false ) { std :: string err_msg ( " " + mnist_labels ) ; throw std :: runtime_error ( err_msg . c_str ( ) ) ; } lifs . read ( int_read , sizeof ( uint32_t ) ) ; uint32_t magic_number = block_to_int ( int_read ) ; if ( magic_number != mnist_labels_magic_number ) { std :: string err_msg ( " " ) ; err_msg += std :: to_string ( magic_number ) ; throw std :: runtime_error ( err_msg ) ; } lifs . read ( int_read , sizeof ( uint32_t ) ) ; uint32_t num_labels = block_to_int ( int_read ) ; labels = new nn :: data < char , 1 > ( num_labels ) ; char * pbuf = ( char * ) labels -> buffer ; for ( unsigned int l = 0 ; l < num_labels ; ++ l ) { lifs . read ( pbuf + l , sizeof ( char ) ) ; } lifs . close ( ) ; std :: ifstream ifs ; ifs . open ( mnist_images , std :: ifstream :: binary ) ; if ( ifs . good ( ) == false ) { std :: string err_msg ( " " + mnist_images ) ; throw std :: runtime_error ( err_msg . c_str ( ) ) ; } ifs . read ( int_read , sizeof ( uint32_t ) ) ; magic_number = block_to_int ( int_read ) ; if ( magic_number != mnist_images_magic_number ) { std :: string err_msg ( " " ) ; err_msg += std :: to_string ( magic_number ) ; throw std :: runtime_error ( err_msg ) ; } ifs . read ( int_read , sizeof ( uint32_t ) ) ; uint32_t num_images = block_to_int ( int_read ) ; ifs . read ( int_read , sizeof ( uint32_t ) ) ; uint32_t images_width = block_to_int ( int_read ) ; ifs . read ( int_read , sizeof ( uint32_t ) ) ; uint32_t images_height = block_to_int ( int_read ) ; images = new nn :: data < float , 3 > ( images_width , images_height , num_images ) ; std :: unique_ptr < char [ ] > single_image ( new char [ images_height * images_width ] ) ; for ( unsigned int img = 0 ; img < num_images ; ++ img ) { ifs . read ( single_image . get ( ) , images_width * images_height * sizeof ( char ) ) ; float * pdata_image = ( float * ) images -> buffer + img * images_width * images_height ; for ( unsigned int pix = 0 ; pix < images_width * images_height ; ++ pix ) { pdata_image [ pix ] = * ( ( unsigned char * ) single_image . get ( ) + pix ) * 1.0f / 255.0f ; } } ifs . close ( ) ; }
Read all labels from given mnist labels file and all corressponding images from mnist images file
Read all labels from given mnist labels file and all corressponding images from mnist images file
// Convert 4 chars in big endian order into 4bytes long integer //1. Read MNIST labels // 1.1 Get Magic number // 1.2 Get Number of labels // 1.3 Get labels //2. Read MNIST images // 2.1 Get Magic number and number of images and width and height of each image // Get Number of images // Get images' width // Get images' height // 2.2 Get all images // Read all images (all at one) , scale its values from: <0-255> to <0.0 - 1.0> // as caffe training was processing data that way //?? is it safe