from PythonInterface import Python
= Python.import_module("pathlib") # Python standard library
let pathlib = Python.import_module("gzip") # Python standard library
let gzip = Python.import_module("pickle") # Python standard library
let pickle = Python.import_module("numpy") let np
π₯ Dataset struct
= pathlib.Path('./lost+found/data/mnist.pkl.gz')
path_gz = gzip.open(path_gz, 'rb')
f = pickle._Unpickler(f)
u = 'latin1'
u.encoding = u.load()
data
= data[0]
data_train = data[1]
data_valid
= data_train[0]
x_train = data_train[1]
y_train = np.expand_dims(y_train, 1)
y_train
= data_valid[0]
x_valid = data_valid[1]
y_valid = np.expand_dims(y_valid, 1)
y_valid f.close()
from DType import DType
from Memory import memset_zero
from Object import object, Attr
from Pointer import DTypePointer, Pointer
from Random import rand
from Range import range
from TargetInfo import dtype_sizeof
type: DType]:
struct Matrix[type]
var data: DTypePointer[
var rows: Int
var cols: Int
__init__(inout self, rows: Int, cols: Int):
fn self.data = DTypePointer[type].alloc(rows * cols)
self.data, rows*cols)
rand(self.rows = rows
self.cols = cols
self, other: Self):
fn __copyinit__(inout self.data = other.data
self.rows = other.rows
self.cols = other.cols
__del__(owned self):
fn self.data.free()
self):
fn zero(inout self.data, self.rows * self.cols)
memset_zero(
@always_inline
__getitem__(self, y: Int, x: Int) -> SIMD[type, 1]:
fn return self.load[1](y, x)
@always_inline
self, y: Int, x: Int) -> SIMD[type, nelts]:
fn load[nelts:Int](return self.data.simd_load[nelts](y * self.cols + x)
@always_inline
__setitem__(self, y: Int, x: Int, val: SIMD[type, 1]):
fn return self.store[1](y, x, val)
@always_inline
self, y: Int, x: Int, val: SIMD[type, nelts]):
fn store[nelts:Int](self.data.simd_store[nelts](y * self.cols + x, val)
@value
type: DType, n_feats: Int]:
struct Dataset[
var np_x: PythonObject
var np_y: PythonObjecttype]
var x: Matrix[type]
var y: Matrix[len: Int
var
__init__(inout self, np_x:PythonObject, np_y:PythonObject) raises:
fn self.np_x = np_x
self.np_y = np_y
self.x = Matrix[type](1,n_feats)
self.x.zero()
self.y = Matrix[type](1,1)
self.y.zero()
self.len = np_x.shape[0].__index__()
__len__(self) -> Int:
fn return self.len
__getitem__(self, i: Int) raises -> Tuple[Matrix[type], Matrix[type]]:
fn self.y[0,0] = self.np_y[i][0].to_float64().cast[type]()
for j in range(n_feats):
self.x[i,j] = self.np_x[i][j].to_float64().cast[type]()
return Tuple(self.x, self.y)
= Dataset[DType.float32, 28*28](x_train, y_train)
var ds print(ds.__len__())
= ds[5]
var ds_item print(ds_item.__len__())
50000
2
Letβs take a stab at creating a Dataloader.
@value
type: DType, n_feats: Int]:
struct Dataloader[len: Int
var
var bs: Inttype, n_feats]
var ds: Dataset[
var current: Inttype]
var xb: Matrix[type]
var yb: Matrix[
__init__(inout self, ds: Dataset[type, n_feats], bs: Int):
fn self.ds = ds
self.bs = bs
self.len = ds.__len__()//bs
self.current = 0
self.xb = Matrix[type](bs,n_feats)
self.xb.zero()
self.yb = Matrix[type](bs,1)
self.yb.zero()
__len__(self) -> Int:
fn return self.len
__iter__(self) -> Self:
fn return self
__next__(inout self) raises -> Matrix[type]:
fn self.len = self.len - 1
for i in range(self.current*self.bs, (self.current+1)*self.bs):
for j in range(n_feats):
self.xb[i,j] = self.ds[i].get[0, Matrix]()[0,j]
self.current = self.current + 1
return self.xb
error: Expression [7]:49:57: invalid call to 'get': result cannot bind generic !mlirtype to memory-only type 'Matrix'
self.xb[i,j] = self.ds[i].get[0, Matrix]()[0,j]
~~~~~~~~~~~~~~~~~~~~~~~~~^~
/.modular/Kernels/mojo/Builtin/Tuple.mojo:58:5: function declared here
fn get[i: Int, T: AnyType](self) -> T:
^
expression failed to parse (no further compiler diagnostics)