Converting the original MNIST files into numpy - Revision history

Davrot: Created page with "Questions to [mailto:davrot@uni-bremen.de David Rotermund] '''I will use Linux. You will need a replacement for gzip under Windows.''' == Download the files == We need to download the MNIST database files * t10k-images.idx3-ubyte.gz * t10k-labels.idx1-ubyte.gz * train-images.idx3-ubyte.gz * train-labels.idx1-ubyte.gz A source for that is for example https://www.kaggle.com/datasets/hojjatk/mnist-dataset?resource=download == Unpack the gz files == In a terminal:
2025-10-21T09:47:28Z

Created page with "Questions to [mailto:davrot@uni-bremen.de David Rotermund] '''I will use Linux. You will need a replacement for gzip under Windows.''' == Download the files == We need to download the MNIST database files * t10k-images.idx3-ubyte.gz * t10k-labels.idx1-ubyte.gz * train-images.idx3-ubyte.gz * train-labels.idx1-ubyte.gz A source for that is for example https://www.kaggle.com/datasets/hojjatk/mnist-dataset?resource=download == Unpack the gz files == In a terminal:<synta..."

New page
Questions to [mailto:davrot@uni-bremen.de David Rotermund]

'''I will use Linux. You will need a replacement for gzip under Windows.'''

== Download the files ==
We need to download the MNIST database files

* t10k-images.idx3-ubyte.gz
* t10k-labels.idx1-ubyte.gz
* train-images.idx3-ubyte.gz
* train-labels.idx1-ubyte.gz

A source for that is for example https://www.kaggle.com/datasets/hojjatk/mnist-dataset?resource=download

== Unpack the gz files ==
In a terminal:<syntaxhighlight lang="shell">gzip -d .gz</syntaxhighlight>

== Convert the data into numpy files ==

=== [https://numpy.org/doc/stable/reference/generated/numpy.dtype.newbyteorder.html numpy.dtype.newbyteorder] ===
<syntaxhighlight lang="python">type.newbyteorder(new_order='S', /)</syntaxhighlight><blockquote>Return a new dtype with a different byte order.

Changes are also made in all fields and sub-arrays of the data type.

'''new_order''' : string, optional

Byte order to force; a value from the byte order specifications below. The default value (‘S’) results in swapping the current byte order. new_order codes can be any of:

‘S’ - swap dtype from current to opposite endian

{‘<’, ‘little’} - little endian

{‘>’, ‘big’} - big endian

{‘=’, ‘native’} - native order

{‘|’, ‘I’} - ignore (no change to byte order)</blockquote>

=== Label file structure ===
<blockquote>[offset] [type] [value] [description]

0000 32 bit integer 0x00000801(2049) magic number (MSB first)

0004 32 bit integer 60000 number of items

0008 unsigned byte ?? label

0009 unsigned byte ?? label

……..

xxxx unsigned byte ?? label</blockquote>The labels values are 0 to 9.

=== Pattern file structure ===
<blockquote>[offset] [type] [value] [description]

0000 32 bit integer 0x00000803(2051) magic number

0004 32 bit integer 60000 number of images

0008 32 bit integer 28 number of rows

0012 32 bit integer 28 number of columns

0016 unsigned byte ?? pixel

0017 unsigned byte ?? pixel

……..

xxxx unsigned byte ?? pixel</blockquote>Pixels are organized row-wise.

Pixel values are 0 to 255. 0 means background (white),

255 means foreground (black).

== Converting the dataset to numpy ==
My source code for that task: convert.py<syntaxhighlight lang="python">import numpy as np

# [offset] [type] [value] [description]
# 0000 32 bit integer 0x00000801(2049) magic number (MSB first)
# 0004 32 bit integer 60000 number of items
# 0008 unsigned byte ?? label
# 0009 unsigned byte ?? label
# ........
# xxxx unsigned byte ?? label
# The labels values are 0 to 9.

class ReadLabel:
"""Class for reading the labels from an MNIST label file"""

def init(self, filename: str) -> None:
self.filename: str = filename
self.data = self.read_from_file(filename)

def read_from_file(self, filename: str) -> np.ndarray:

int_32bit_data = np.dtype(np.uint32)
int_32bit_data = int_32bit_data.newbyteorder(">")

with open(filename, "rb") as file:

magic_flag: np.uint32 = np.frombuffer(file.read(4), int_32bit_data)[0]

if magic_flag != 2049:
data: np.ndarray = np.zeros(0)
number_of_elements: int = 0
else:
number_of_elements = np.frombuffer(file.read(4), int_32bit_data)[0]

if number_of_elements < 1:
data = np.zeros(0)
else:
data = np.frombuffer(file.read(number_of_elements), dtype=np.uint8)

return data

# [offset] [type] [value] [description]
# 0000 32 bit integer 0x00000803(2051) magic number
# 0004 32 bit integer 60000 number of images
# 0008 32 bit integer 28 number of rows
# 0012 32 bit integer 28 number of columns
# 0016 unsigned byte ?? pixel
# 0017 unsigned byte ?? pixel
# ........
# xxxx unsigned byte ?? pixel
# Pixels are organized row-wise.
# Pixel values are 0 to 255. 0 means background (white),
# 255 means foreground (black).

class ReadPicture:
"""Class for reading the images from an MNIST image file"""

def init(self, filename: str) -> None:
self.filename: str = filename
self.Data = self.read_from_file(filename)

def read_from_file(self, filename: str) -> np.ndarray:

int_32bit_data = np.dtype(np.uint32)
int_32bit_data = int_32bit_data.newbyteorder(">")

with open(filename, "rb") as file:

magic_flag = np.frombuffer(file.read(4), int_32bit_data)[0]

if magic_flag != 2051:
data = np.zeros(0)
number_of_elements: int = 0
else:
number_of_elements = np.frombuffer(file.read(4), int_32bit_data)[0]

if number_of_elements < 1:
data = np.zeros(0)
number_of_rows: int = 0
else:
number_of_rows = np.frombuffer(file.read(4), int_32bit_data)[0]

if number_of_rows != 28:
data = np.zeros(0)
number_of_columns: int = 0
else:
number_of_columns = np.frombuffer(file.read(4), int_32bit_data)[0]

if number_of_columns != 28:
data = np.zeros(0)
else:
data = np.frombuffer(
file.read(number_of_elements number_of_rows * number_of_columns),
dtype=np.uint8,
)
data = data.reshape(
number_of_elements, number_of_columns, number_of_rows
)

return data

def proprocess_dataset(testdata_mode: bool) -> None:

if testdata_mode is True:
filename_out_pattern: str = "test_pattern_storage.npy"
filename_out_label: str = "test_label_storage.npy"
filename_in_image: str = "t10k-images.idx3-ubyte"
filename_in_label: str = "t10k-labels.idx1-ubyte"
else:
filename_out_pattern = "train_pattern_storage.npy"
filename_out_label = "train_label_storage.npy"
filename_in_image = "train-images.idx3-ubyte"
filename_in_label = "train-labels.idx1-ubyte"

pictures = ReadPicture(filename_in_image)
labels = ReadLabel(filename_in_label)

# Down to 0 ... 1.0
max_value = np.max(pictures.Data.astype(np.float32))
pattern_storage = np.float32(pictures.Data.astype(np.float32) / max_value).astype(
np.float32
)

label_storage = np.uint64(labels.data)

np.save(filename_out_pattern, pattern_storage)
np.save(filename_out_label, label_storage)

proprocess_dataset(testdata_mode=True)
proprocess_dataset(testdata_mode=False)</syntaxhighlight>Now we have the files:

* test_label_storage.npy
* test_pattern_storage.npy
* train_label_storage.npy
* train_pattern_storage.npy