Create optimizers

Sure, here are the implementations of some common optimizers in Python:

1. Stochastic Gradient Descent (SGD)

class SGD:
    def __init__(self, learning_rate):
        self.learning_rate = learning_rate

    def update(self, weights, gradients):
        for i in range(len(weights)):
            weights[i] -= self.learning_rate * gradients[i]

2. Momentum

class Momentum:
    def __init__(self, learning_rate, momentum):
        self.learning_rate = learning_rate
        self.momentum = momentum
        self.velocity = None

    def update(self, weights, gradients):
        if self.velocity is None:
            self.velocity = [np.zeros_like(w) for w in weights]

        for i in range(len(weights)):
            self.velocity[i] = self.momentum * self.velocity[i] - self.learning_rate * gradients[i]
            weights[i] += self.velocity[i]

3. Adagrad

class Adagrad:
    def __init__(self, learning_rate, eps=1e-8):
        self.learning_rate = learning_rate
        self.eps = eps
        self.cache = None

    def update(self, weights, gradients):
        if self.cache is None:
            self.cache = [np.zeros_like(w) for w in weights]

        for i in range(len(weights)):
            self.cache[i] += gradients[i] ** 2
            weights[i] -= self.learning_rate * gradients[i] / (np.sqrt(self.cache[i]) + self.eps)

4. Adadelta

class Adadelta:
    def __init__(self, rho=0.95, eps=1e-6):
        self.rho = rho
        self.eps = eps
        self.cache = None
        self.delta_cache = None

    def update(self, weights, gradients):
        if self.cache is None:
            self.cache = [np.zeros_like(w) for w in weights]
            self.delta_cache = [np.zeros_like(w) for w in weights]

        for i in range(len(weights)):
            self.cache[i] = self.rho * self.cache[i] + (1 - self.rho) * gradients[i] ** 2
            delta = np.sqrt((self.delta_cache[i] + self.eps) / (self.cache[i] + self.eps)) * gradients[i]
            weights[i] -= delta
            self.delta_cache[i] = self.rho * self.delta_cache[i] + (1 - self.rho) * delta ** 2

5. Adam

class Adam:
    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, eps=1e-8):
        self.learning_rate = learning_rate
        self.beta1 = beta1
        self.beta2 = beta2
        self.eps = eps
        self.m = None
        self.v = None
        self.t = 0

    def update(self, weights, gradients):
        if self.m is None:
            self.m = [np.zeros_like(w) for w in weights]
            self.v = [np.zeros_like(w) for w in weights]

        self.t += 1
        alpha = self.learning_rate * np.sqrt(1 - self.beta2 ** self.t) / (1 - self.beta1 ** self.t)

        for i in range(len(weights)):
            self.m[i] = self.beta1 * self.m[i] + (1 - self.beta1) * gradients[i]
            self.v[i] = self.beta2 * self.v[i] + (1 - self.beta2) * gradients[i] ** 2
            weights[i] -= alpha * self.m[i] / (np.sqrt(self.v[i]) + self.eps)

Note that these are just basic implementations and there are many variations and improvements to these optimizers.