A Single Neuron (from Scratch)

Code

forward and accuracy — write them yourself

neuron and softmax work. For the two remaining functions there are no holes this time — only the skeleton.

Both are simple, and both put together what you already have.

`forward`

forward(x, W, b):
    scores ← neuron(x, W, b)        # [n, 784] @ [784, 10] + [10] → [n, 10]
    return softmax(scores)          # [n, 10], every row adds up to 1

One line — and that is exactly the whole "model". Pictures go in, probabilities come out.

`accuracy`

For every picture the model gives ten probabilities. Its answer is the class with the highest probability. Accuracy is the share of answers that match the label.

accuracy(probs, y):
    answer ← the index of each row's largest probability      # [n]
    correct ← the number of places where answer == y
    return correct / n   (as a Python float)

Two things worth thinking about:

the index of the largest value (argmax), not the value itself — a class is a number, not a probability;
comparing with y gives a True/False tensor. Its mean is the share of correct ones, but a bool is not a number: convert it to floating point for mean to work. And the result must be a plain Python number — .item().

Your job

neuron.py (right panel) has grown: two new skeletons at the bottom, forward and accuracy, with raise NotImplementedError. Above, your neuron, sigmoid, softmax.

The panel shows the skeleton: the HOLE places are empty in it, in your copy they are filled in.

test_neuron.py grew too. If you run it before writing anything (with the holes filled in):

$ python -m unittest
EEEEE........
ERROR: test_all_right_and_all_wrong (...)
NotImplementedError
ERROR: test_half_right (...)
NotImplementedError
ERROR: test_returns_a_plain_float (...)
NotImplementedError
ERROR: test_forward_gives_probabilities_per_image (...)
NotImplementedError
ERROR: test_forward_picks_the_class_with_the_big_weight (...)
NotImplementedError
----------------------------------------------------------------------
Ran 13 tests in 0.010s

FAILED (errors=5)

Eight dots are the earlier tests, still passing.

When it does not pass

False is not true in the forward test — you returned neuron(x, W, b) without the softmax. The shapes are right, but the rows do not add up to 1.
tensor(1.) is not an instance of <class 'float'> — you forgot .item(). A tensor holding one number looks like a number, but is not one.
The size of tensor a (3) must match the size of tensor b (4) at non-singleton dimension 0 — argmax(dim=0) instead of dim=1. You got "which picture is best for this class" (3 numbers, one per class), not "which class for this picture" (4 numbers).
0.0 != 0.5 — you compare the largest probability (e.g. 0.8) with y, not its index. A probability never equals a label.

neuron.py MODIFY 18 lines changed

Edit this file in ./

"""neuron.py — a neuron built by hand from raw tensor math. No nn.Module."""

import torch


def neuron(x: torch.Tensor, w: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
    """One neuron per column of w: a weighted sum of the inputs plus a bias."""
    # HOLE 1: the weighted sum of the inputs, plus the bias.
    # (This placeholder returns zeros of the right shape, so everything runs.)
    return torch.zeros(x.shape[:-1] + w.shape[1:])


def sigmoid(z: torch.Tensor) -> torch.Tensor:
    """Squash any number into (0, 1): 'how sure is this neuron?'"""
    return 1 / (1 + torch.exp(-z))


def softmax(z: torch.Tensor) -> torch.Tensor:
    """Turn a row of scores into probabilities that sum to 1."""
    # HOLE 2: subtract each row's largest score BEFORE exp(), so that big
    # scores cannot overflow. (Dividing by the sum below stays as it is.)
    e = torch.exp(z)
    return e / e.sum(dim=1, keepdim=True)


def forward(x: torch.Tensor, W: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
    """Scores for every image, turned into probabilities.

    x is [n, 784], W is [784, 10], b is [10]; the result is [n, 10] and every
    row sums to 1. Use neuron() and softmax() from above.
    """
    raise NotImplementedError


def accuracy(probs: torch.Tensor, y: torch.Tensor) -> float:
    """The fraction of images whose most probable class is the true label.

    probs is [n, 10], y is [n] of class numbers. Return a plain Python float
    between 0 and 1.
    """
    raise NotImplementedError

test_neuron.py MODIFY 39 lines changed

Edit this file in ./

"""test_neuron.py — the verdict for neuron.py.

Run it from this folder:   python -m unittest
"""

import unittest

import torch

import neuron as nr


class TestNeuron(unittest.TestCase):
    def test_one_neuron_one_image(self):
        # 1*0.5 + 2*(-1) + 3*2 + 0.25 = 4.75
        x = torch.tensor([1.0, 2.0, 3.0])
        w = torch.tensor([0.5, -1.0, 2.0])
        b = torch.tensor(0.25)
        self.assertAlmostEqual(nr.neuron(x, w, b).item(), 4.75, places=5)

    def test_bias_is_added(self):
        x = torch.zeros(3)  # all-zero input: only the bias is left
        w = torch.ones(3)
        b = torch.tensor(0.7)
        self.assertAlmostEqual(nr.neuron(x, w, b).item(), 0.7, places=5)

    def test_many_neurons_many_images(self):
        # two images, three inputs each; two neurons side by side = [3, 2]
        x = torch.tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 1.0]])
        W = torch.tensor([[1.0, 10.0], [2.0, 20.0], [3.0, 30.0]])
        b = torch.tensor([0.5, -0.5])
        out = nr.neuron(x, W, b)
        self.assertEqual(tuple(out.shape), (2, 2))
        self.assertEqual(out.tolist(), [[1.5, 9.5], [5.5, 49.5]])


class TestSigmoid(unittest.TestCase):
    def test_sigmoid_squashes_into_zero_one(self):
        z = torch.tensor([-100.0, 0.0, 100.0])
        out = nr.sigmoid(z)
        self.assertAlmostEqual(out[1].item(), 0.5, places=6)
        self.assertTrue(bool((out >= 0).all() and (out <= 1).all()))


class TestSoftmax(unittest.TestCase):
    def test_rows_sum_to_one(self):
        z = torch.tensor([[1.0, 2.0, 3.0], [0.0, 0.0, 0.0]])
        sums = nr.softmax(z).sum(dim=1)
        self.assertTrue(torch.allclose(sums, torch.ones(2)))

    def test_known_values(self):
        out = nr.softmax(torch.tensor([[1.0, 2.0, 3.0]]))
        want = torch.tensor([[0.0900, 0.2447, 0.6652]])
        self.assertTrue(torch.allclose(out, want, atol=1e-4), f"got {out.tolist()}")

    def test_big_scores_do_not_overflow(self):
        out = nr.softmax(torch.tensor([[1000.0, 1.0, 2.0]]))
        self.assertFalse(bool(torch.isnan(out).any()), f"got {out.tolist()} - overflow?")
        self.assertAlmostEqual(out[0, 0].item(), 1.0, places=5)

    def test_every_row_is_stabilised_on_its_own(self):
        # row 1 is huge, row 2 is hugely negative: one shared maximum fails row 2
        z = torch.tensor([[1000.0, 1.0, 2.0], [-1000.0, -1001.0, -1002.0]])
        out = nr.softmax(z)
        self.assertFalse(bool(torch.isnan(out).any()), f"got {out.tolist()}")
        want = torch.tensor([0.6652, 0.2447, 0.0900])
        self.assertTrue(torch.allclose(out[1], want, atol=1e-4), f"row 2 is {out[1].tolist()}")


class TestForward(unittest.TestCase):
    def test_forward_gives_probabilities_per_image(self):
        torch.manual_seed(0)
        x = torch.rand(6, 784)
        W = torch.randn(784, 10) * 0.01
        b = torch.zeros(10)
        probs = nr.forward(x, W, b)
        self.assertEqual(tuple(probs.shape), (6, 10))
        self.assertTrue(torch.allclose(probs.sum(dim=1), torch.ones(6)))

    def test_forward_picks_the_class_with_the_big_weight(self):
        # class 1 listens to input 0, class 0 listens to input 1
        W = torch.tensor([[0.0, 5.0], [5.0, 0.0]])
        b = torch.zeros(2)
        x = torch.tensor([[1.0, 0.0], [0.0, 1.0]])
        probs = nr.forward(x, W, b)
        self.assertEqual(probs.argmax(dim=1).tolist(), [1, 0])


class TestAccuracy(unittest.TestCase):
    def setUp(self):
        # four images, three classes; the model's answers are 0, 1, 2, 0
        self.probs = torch.tensor(
            [[0.7, 0.2, 0.1], [0.1, 0.8, 0.1], [0.2, 0.2, 0.6], [0.5, 0.3, 0.2]]
        )

    def test_half_right(self):
        y = torch.tensor([0, 1, 0, 1])  # right, right, wrong, wrong
        self.assertEqual(nr.accuracy(self.probs, y), 0.5)

    def test_all_right_and_all_wrong(self):
        self.assertEqual(nr.accuracy(self.probs, torch.tensor([0, 1, 2, 0])), 1.0)
        self.assertEqual(nr.accuracy(self.probs, torch.tensor([2, 2, 1, 2])), 0.0)

    def test_returns_a_plain_float(self):
        acc = nr.accuracy(self.probs, torch.tensor([0, 1, 2, 0]))
        self.assertIsInstance(acc, float)


if __name__ == "__main__":
    unittest.main()