my-grad.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "\n",
    "class V:\n",
    "    def __init__(self, value, op = None , parents = None, label = None):\n",
    "        self.value = value\n",
    "        self.grad = 0\n",
    "        self.parents = parents\n",
    "        self.label = label\n",
    "        self.op = op\n",
    "        self._backward = lambda: None\n",
    "\n",
    "    def __add__(self, other):\n",
    "        other = V._cast_to_v(other)\n",
    "        new_value = self.value + other.value\n",
    "        result = V(new_value, op='+', parents=(self,other))\n",
    "\n",
    "        def backward():\n",
    "            self.grad += result.grad * 1\n",
    "            other.grad += result.grad * 1\n",
    "                            \n",
    "        result._backward = backward\n",
    "\n",
    "        return result\n",
    "\n",
    "    def __mul__(self, other):\n",
    "        other = V._cast_to_v(other)\n",
    "        new_value = self.value * other.value\n",
    "        result = V(new_value, op='*', parents=(self,other))\n",
    "\n",
    "        def backward():\n",
    "            self.grad += result.grad * other.value\n",
    "            other.grad += result.grad * self.value\n",
    "\n",
    "        result._backward = backward\n",
    "        \n",
    "        return result  \n",
    "    \n",
    "    def __pow__(self, other):\n",
    "        assert isinstance(other, (float, int)), \"only float and int supported in pow\"\n",
    "\n",
    "        new_value = self.value ** other\n",
    "        result = V(new_value, op='**', parents=(self,))\n",
    "\n",
    "        def backward():\n",
    "            self.grad += other * (self.value ** (other - 1)) * result.grad \n",
    "\n",
    "        result._backward = backward\n",
    "\n",
    "        return result\n",
    "\n",
    "    def __radd__(self, other):\n",
    "        return self.__add__(other)\n",
    "\n",
    "    def __sub__(self, other):\n",
    "        other = V._cast_to_v(other)\n",
    "        return self.__add__(other.__mul__(-1))\n",
    "    \n",
    "    def __rsub__(self, other):\n",
    "        return self.__sub__(other)\n",
    "    \n",
    "    def __rmul__(self,other):\n",
    "        return self.__mul__(other)\n",
    "    \n",
    "    def tanh(self):\n",
    "        new_value = tanh(self.value)\n",
    "        result = V(new_value, op='tanh', parents=(self,))\n",
    "\n",
    "        def backward():\n",
    "            self.grad += result.grad * (1 - new_value ** 2)\n",
    "\n",
    "        result._backward = backward\n",
    "\n",
    "        return result\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "       return f\"{self.label}: value={self.value} grad={self.grad} op={self.op}\"\n",
    "\n",
    "    def backward(self, isFirst = True):\n",
    "        if isFirst:\n",
    "            self.grad = 1.0   \n",
    "            self._backward()\n",
    "        \n",
    "        if self.parents == None:\n",
    "            return\n",
    "\n",
    "        for parent in self.parents:\n",
    "            parent._backward()    \n",
    "\n",
    "        for parent in self.parents:\n",
    "            parent.backward(False)\n",
    " \n",
    "    @staticmethod\n",
    "    def _cast_to_v(other):\n",
    "        return other if isinstance(other, V) else V(other)\n",
    "    \n",
    "def tanh(value):\n",
    "    e2x = math.exp(2 * value) \n",
    "    top = e2x - 1\n",
    "    bottom = e2x + 1\n",
    "    new_value = top / bottom\n",
    "    return new_value\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09966799462495583"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "assert V(1).value == 1 \n",
    "v3 = V(1) + V(2)\n",
    "assert v3.value == 3\n",
    "v4 = V(2) * V(8)\n",
    "assert v4.value == 16\n",
    "tanh(0.1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "f = a * x\n",
    "\n",
    "df/dx = [(a * (x+h)) - a * x] / h\n",
    "df/dx = [ax + ah - ax] / h\n",
    "df/dx = [ah]/h = a\n",
    "\n",
    "\n",
    "g = b + y\n",
    "dg/dy =  [b + y + h - (b +y)] /h\n",
    "dg/dy =  [b + y + h - b -y] /h\n",
    "dg/dy =  1\n",
    "\n",
    "L = f + g\n",
    "dL/df = 1\n",
    "dL/dg = 1\n",
    "\n",
    "g = b + y\n",
    "dL/db = 1 * dL/df(1)\n",
    "dL/dy = 1 * dL/df(1)\n",
    "\n",
    "f = a * x\n",
    "dL/da = x * dL/dg(1)  \n",
    "dL/dx = a * dL/dg(1) \n",
    "\n",
    "for a = 3 x = 4 b = 6 y = 7\n",
    "L = f + g\n",
    "dL/df = 1\n",
    "dL/dg = 1\n",
    "\n",
    "g = b + y\n",
    "dL/db = 1 * dL/df(1)  = 1\n",
    "dL/dy = 1 * dL/df(1)  = 1\n",
    "\n",
    "f = a * x\n",
    "dL/da = x * dL/dg(1) = 4\n",
    "dL/dx = a * dL/dg(1) = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = V(3, label='a')\n",
    "x = V(4, label='x')\n",
    "b = V(-4.5, label='b')\n",
    "y = V(-7, label='y')\n",
    "\n",
    "\n",
    "f = a * x; f.label = 'f'\n",
    "g = b + y; g.label = 'g'\n",
    "L = f + g; L.label = 'L'\n",
    "tan_L  = L.tanh(); tan_L.label = 'tanh(L)'\n",
    "\n",
    "tan_L.backward()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Any\n",
    "\n",
    "\n",
    "class Neuron():\n",
    "    def __init__(self, weights_count):\n",
    "        self.weights = [V(random.uniform(-1,1)) for _ in range(weights_count)]\n",
    "        self.bias = V(random.uniform(-1,1))\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        assert len(inputs) == len(self.weights), \"Inputs should have same length as weights len\"\n",
    "        result = sum([i * w for (i,w) in zip(inputs,self.weights)]) + self.bias\n",
    "        return result.tanh()\n",
    "\n",
    "    def params(self):\n",
    "        return [self.bias] + self.weights \n",
    "\n",
    "class Layer():       \n",
    "    def __init__(self, neurons_count, weights_count_per_neuron):\n",
    "        self.neurons = [Neuron(weights_count_per_neuron) for _ in range(neurons_count)]\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        return [neuron(inputs) for neuron in self.neurons]\n",
    "\n",
    "    def params(self):\n",
    "        return [params for neuron in self.neurons for params in neuron.params()]\n",
    "\n",
    "class Net():\n",
    "    def __init__(self, layer_sizes):\n",
    "        self.layers = [Layer(layer_sizes[index+1],layer_sizes[index]) for index in  range(len(layer_sizes) - 1)]\n",
    "    \n",
    "    def __call__(self, inputs):\n",
    "        for layer in self.layers:\n",
    "            inputs = layer(inputs)\n",
    "        return unwrap_if_single_element(inputs) \n",
    "    \n",
    "    def params(self):\n",
    "        return [params for layer in self.layers for params in layer.params()]\n",
    "\n",
    "    def reset_grad(self):\n",
    "        for param in self.params():\n",
    "            param.grad = 0.0\n",
    "        \n",
    "\n",
    "def unwrap_if_single_element(array):\n",
    "    return array[0] if len(array) == 1 else array\n",
    "\n",
    "def loss(actual_outputs, expected_outputs):\n",
    "    return sum([(actual - expected) ** 2 for actual, expected in zip(actual_outputs, expected_outputs)])\n",
    "\n",
    "def forward(net, inputs):\n",
    "    actual_outputs = []\n",
    "    for single_input in inputs:\n",
    "        actual_outputs.append(net(single_input))\n",
    "    return actual_outputs\n",
    "\n",
    "def step(net, inputs, expected_outputs, step_size = 0.002):\n",
    "    net.reset_grad()\n",
    "    actual_outputs = forward(net, inputs)\n",
    "    cost = loss(actual_outputs, expected_outputs)\n",
    "    cost.backward()\n",
    "\n",
    "    for param in net.params():\n",
    "        param.value -= step_size * param.grad\n",
    "\n",
    "    return cost\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "expected output: 1 actual: None: value=0.9305031489849274 grad=0 op=tanh\n",
      "expected output: 0 actual: None: value=0.005431402594292498 grad=0 op=tanh\n",
      "expected output: 1 actual: None: value=0.933125569444634 grad=0 op=tanh\n"
     ]
    }
   ],
   "source": [
    "net = Net([3,4,1])\n",
    "\n",
    "inputs = [\n",
    "    [1,2,3],\n",
    "    [5,4,1],\n",
    "    [3,2,5]\n",
    "]\n",
    "\n",
    "expected_outputs = [1,0,1]\n",
    "\n",
    "for i in range(1000):\n",
    "    cost = step(net, inputs, expected_outputs)\n",
    "\n",
    "for (input,expected_output) in zip(inputs,expected_outputs):\n",
    "    print(f\"expected output: {expected_output} actual: {net(input)}\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}