View source for Converting the original MNIST files into numpy

Questions to [mailto:davrot@uni-bremen.de David Rotermund]

'''I will use Linux. You will need a replacement for gzip under Windows.'''

== Download the files ==
We need to download the MNIST database files

* t10k-images.idx3-ubyte.gz
* t10k-labels.idx1-ubyte.gz
* train-images.idx3-ubyte.gz
* train-labels.idx1-ubyte.gz

A source for that is for example https://www.kaggle.com/datasets/hojjatk/mnist-dataset?resource=download

== Unpack the gz files ==
In a terminal:<syntaxhighlight lang="shell">gzip -d *.gz</syntaxhighlight>

== Convert the data into numpy files ==

=== [https://numpy.org/doc/stable/reference/generated/numpy.dtype.newbyteorder.html numpy.dtype.newbyteorder] ===
<syntaxhighlight lang="python">type.newbyteorder(new_order='S', /)</syntaxhighlight><blockquote>Return a new dtype with a different byte order.

Changes are also made in all fields and sub-arrays of the data type.

'''new_order''' : string, optional

Byte order to force; a value from the byte order specifications below. The default value (‘S’) results in swapping the current byte order. new_order codes can be any of:

‘S’ - swap dtype from current to opposite endian

{‘&#x3C;’, ‘little’} - little endian

{‘&#x3E;’, ‘big’} - big endian

{‘=’, ‘native’} - native order

{‘|’, ‘I’} - ignore (no change to byte order)</blockquote>

=== Label file structure ===
<blockquote>[offset] [type] [value] [description]

0000 32 bit integer 0x00000801(2049) magic number (MSB first)

0004 32 bit integer 60000 number of items

0008 unsigned byte ?? label

0009 unsigned byte ?? label

……..

xxxx unsigned byte ?? label</blockquote>The labels values are 0 to 9.

=== Pattern file structure ===
<blockquote>[offset] [type] [value] [description]

0000 32 bit integer 0x00000803(2051) magic number

0004 32 bit integer 60000 number of images

0008 32 bit integer 28 number of rows

0012 32 bit integer 28 number of columns

0016 unsigned byte ?? pixel

0017 unsigned byte ?? pixel

……..

xxxx unsigned byte ?? pixel</blockquote>Pixels are organized row-wise.

Pixel values are 0 to 255. 0 means background (white),

255 means foreground (black).

== Converting the dataset to numpy ==
My source code for that task: convert.py<syntaxhighlight lang="python">import numpy as np

# [offset] [type]          [value]          [description]
# 0000     32 bit integer  0x00000801(2049) magic number (MSB first)
# 0004     32 bit integer  60000            number of items
# 0008     unsigned byte   ??               label
# 0009     unsigned byte   ??               label
# ........
# xxxx     unsigned byte   ??               label
# The labels values are 0 to 9.


class ReadLabel:
    """Class for reading the labels from an MNIST label file"""

    def __init__(self, filename: str) -> None:
        self.filename: str = filename
        self.data = self.read_from_file(filename)

    def read_from_file(self, filename: str) -> np.ndarray:

        int_32bit_data = np.dtype(np.uint32)
        int_32bit_data = int_32bit_data.newbyteorder(">")

        with open(filename, "rb") as file:

            magic_flag: np.uint32 = np.frombuffer(file.read(4), int_32bit_data)[0]

            if magic_flag != 2049:
                data: np.ndarray = np.zeros(0)
                number_of_elements: int = 0
            else:
                number_of_elements = np.frombuffer(file.read(4), int_32bit_data)[0]

            if number_of_elements < 1:
                data = np.zeros(0)
            else:
                data = np.frombuffer(file.read(number_of_elements), dtype=np.uint8)

        return data


# [offset] [type]          [value]          [description]
# 0000     32 bit integer  0x00000803(2051) magic number
# 0004     32 bit integer  60000            number of images
# 0008     32 bit integer  28               number of rows
# 0012     32 bit integer  28               number of columns
# 0016     unsigned byte   ??               pixel
# 0017     unsigned byte   ??               pixel
# ........
# xxxx     unsigned byte   ??               pixel
# Pixels are organized row-wise.
# Pixel values are 0 to 255. 0 means background (white),
# 255 means foreground (black).


class ReadPicture:
    """Class for reading the images from an MNIST image file"""

    def __init__(self, filename: str) -> None:
        self.filename: str = filename
        self.Data = self.read_from_file(filename)

    def read_from_file(self, filename: str) -> np.ndarray:

        int_32bit_data = np.dtype(np.uint32)
        int_32bit_data = int_32bit_data.newbyteorder(">")

        with open(filename, "rb") as file:

            magic_flag = np.frombuffer(file.read(4), int_32bit_data)[0]

            if magic_flag != 2051:
                data = np.zeros(0)
                number_of_elements: int = 0
            else:
                number_of_elements = np.frombuffer(file.read(4), int_32bit_data)[0]

            if number_of_elements < 1:
                data = np.zeros(0)
                number_of_rows: int = 0
            else:
                number_of_rows = np.frombuffer(file.read(4), int_32bit_data)[0]

            if number_of_rows != 28:
                data = np.zeros(0)
                number_of_columns: int = 0
            else:
                number_of_columns = np.frombuffer(file.read(4), int_32bit_data)[0]

            if number_of_columns != 28:
                data = np.zeros(0)
            else:
                data = np.frombuffer(
                    file.read(number_of_elements * number_of_rows * number_of_columns),
                    dtype=np.uint8,
                )
                data = data.reshape(
                    number_of_elements, number_of_columns, number_of_rows
                )

        return data


def proprocess_dataset(testdata_mode: bool) -> None:

    if testdata_mode is True:
        filename_out_pattern: str = "test_pattern_storage.npy"
        filename_out_label: str = "test_label_storage.npy"
        filename_in_image: str = "t10k-images.idx3-ubyte"
        filename_in_label: str = "t10k-labels.idx1-ubyte"
    else:
        filename_out_pattern = "train_pattern_storage.npy"
        filename_out_label = "train_label_storage.npy"
        filename_in_image = "train-images.idx3-ubyte"
        filename_in_label = "train-labels.idx1-ubyte"

    pictures = ReadPicture(filename_in_image)
    labels = ReadLabel(filename_in_label)

    # Down to 0 ... 1.0
    max_value = np.max(pictures.Data.astype(np.float32))
    pattern_storage = np.float32(pictures.Data.astype(np.float32) / max_value).astype(
        np.float32
    )

    label_storage = np.uint64(labels.data)

    np.save(filename_out_pattern, pattern_storage)
    np.save(filename_out_label, label_storage)


proprocess_dataset(testdata_mode=True)
proprocess_dataset(testdata_mode=False)</syntaxhighlight>Now we have the files:

* test_label_storage.npy
* test_pattern_storage.npy
* train_label_storage.npy
* train_pattern_storage.npy