Skip to content

Optimizer API

Both optimizers update parameter arrays in place and expose zero_grad() for clearing accumulated gradients.

SGD

SGD 

SGD(parameters: list[Variable], lr: float = 0.01, momentum: float = 0.0, weight_decay: float = 0.0)

Stochastic Gradient Descent (with optional momentum).

Parameters

parameters : list of Variable lr : learning rate momentum : momentum factor (0 = plain SGD) weight_decay : L2 regularisation coefficient

Source code in MiniTorch/optim/sgd.py 
20
21
22
23
24
25
26
27
28
29
30
31
def __init__(
    self,
    parameters: list[Variable],
    lr: float = 0.01,
    momentum: float = 0.0,
    weight_decay: float = 0.0,
) -> None:
    self.parameters = parameters
    self.lr = lr
    self.momentum = momentum
    self.weight_decay = weight_decay
    self._velocity: list[np.ndarray] = [np.zeros_like(p.data) for p in parameters]

step 

step() -> None
Source code in MiniTorch/optim/sgd.py 
33
34
35
36
37
38
39
40
41
42
def step(self) -> None:
    for v, p in zip(self._velocity, self.parameters):
        if p.grad is None:
            continue
        g: np.ndarray = p.grad.data  # type: ignore[assignment]
        if self.weight_decay != 0:
            g = g + self.weight_decay * p.data  # type: ignore[operator]
        v *= self.momentum
        v += g
        p.data -= self.lr * v  # type: ignore[operator]

zero_grad 

zero_grad() -> None
Source code in MiniTorch/optim/sgd.py 
44
45
46
def zero_grad(self) -> None:
    for p in self.parameters:
        p.clear_grad()

Adam

Adam 

Adam(parameters: list[Variable], lr: float = 0.001, beta1: float = 0.9, beta2: float = 0.999, eps: float = 1e-08, weight_decay: float = 0.0)

Adam optimiser (Kingma & Ba, 2015).

Parameters

parameters : list of Variable lr : step size (α) beta1 : first-moment decay (default 0.9) beta2 : second-moment decay (default 0.999) eps : numerical stability term (default 1e-8) weight_decay : L2 regularisation coefficient

Source code in MiniTorch/optim/adam.py 
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
def __init__(
    self,
    parameters: list[Variable],
    lr: float = 1e-3,
    beta1: float = 0.9,
    beta2: float = 0.999,
    eps: float = 1e-8,
    weight_decay: float = 0.0,
) -> None:
    self.parameters = parameters
    self.lr = lr
    self.beta1 = beta1
    self.beta2 = beta2
    self.eps = eps
    self.weight_decay = weight_decay
    self.t: int = 0
    self._m: list[np.ndarray] = [np.zeros_like(p.data) for p in parameters]
    self._v: list[np.ndarray] = [np.zeros_like(p.data) for p in parameters]
    self._scratch: list[np.ndarray] = [
        np.empty_like(p.data) for p in parameters
    ]

step 

step() -> None
Source code in MiniTorch/optim/adam.py 
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
def step(self) -> None:
    self.t += 1
    lr_t = (
        self.lr
        * math.sqrt(1 - self.beta2 ** self.t)
        / (1 - self.beta1 ** self.t)
    )
    for i, p in enumerate(self.parameters):
        if p.grad is None:
            continue
        g: np.ndarray = p.grad.data  # type: ignore[assignment]
        if self.weight_decay != 0:
            g = g + self.weight_decay * p.data  # type: ignore[operator]
        first_moment = self._m[i]
        second_moment = self._v[i]
        scratch = self._scratch[i]

        first_moment *= self.beta1
        first_moment += (1 - self.beta1) * g
        second_moment *= self.beta2
        np.multiply(g, g, out=scratch)
        second_moment += (1 - self.beta2) * scratch

        np.sqrt(second_moment, out=scratch)
        scratch += self.eps
        np.divide(first_moment, scratch, out=scratch)
        scratch *= lr_t
        p.data -= scratch  # type: ignore[operator]

zero_grad 

zero_grad() -> None
Source code in MiniTorch/optim/adam.py 
75
76
77
def zero_grad(self) -> None:
    for p in self.parameters:
        p.clear_grad()