Ready. Set. Go!

build.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -xe
+
+clang -Wall -Wextra -o twice twice.c -lm
+clang -Wall -Wextra -o gates gates.c -lm
+clang -Wall -Wextra -o xor xor.c -lm

gates.c

@@ -0,0 +1,104 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <math.h>
+
+float sigmoidf(float x)
+{
+    return 1.f / (1.f + expf(-x));
+}
+
+typedef float sample[3];
+
+// NAND-gate
+sample or_train[] = {
+    {0, 0, 0},
+    {1, 0, 1},
+    {0, 1, 1},
+    {1, 1, 1},
+};
+
+sample and_train[] = {
+    {0, 0, 0},
+    {1, 0, 0},
+    {0, 1, 0},
+    {1, 1, 1},
+};
+
+sample nand_train[] = {
+    {0, 0, 1},
+    {1, 0, 1},
+    {0, 1, 1},
+    {1, 1, 0},
+};
+
+sample xor_train[] = {
+    {0, 0, 0},
+    {1, 0, 1},
+    {0, 1, 1},
+    {1, 1, 0},
+};
+
+sample *train = xor_train;
+size_t train_count = 4;
+
+float cost(float w1, float w2, float b)
+{
+    float result = 0.0f;
+    for (size_t i = 0; i < train_count; ++i) {
+        float x1 = train[i][0];
+        float x2 = train[i][1];
+        float y = sigmoidf(x1*w1 + x2*w2 + b);
+        float d = y - train[i][2];
+        result += d*d;
+    }
+    result /= train_count;
+    return result;
+}
+
+float rand_float(void)
+{
+    return (float) rand()/ (float) RAND_MAX;
+}
+
+int main2(void)
+{
+    // (x|y) & ~(x&y)
+    for (size_t x = 0; x < 2; ++x) {
+        for (size_t y = 0; y < 2; ++y) {
+            printf("%zu ^ %zu = %zu\n", x, y, (x|y) & (~(x&y)));
+        }
+    }
+    return 0;
+}
+
+int main(void)
+{
+    srand(time(0));
+    float w1 = rand_float();
+    float w2 = rand_float();
+    float b  = rand_float();
+
+    float eps = 1e-1;
+    float rate = 1e-1;
+
+    for (size_t i = 0; i < 500*1000; ++i) {
+        float c = cost(w1, w2, b);
+        printf("w1 = %f, w2 = %f, b = %f, c = %f\n", w1, w2, b, c);
+        float dw1 = (cost(w1 + eps, w2, b) - c)/eps;
+        float dw2 = (cost(w1, w2 + eps, b) - c)/eps;
+        float db  = (cost(w1, w2, b + eps) - c)/eps;
+        w1 -= rate*dw1;
+        w2 -= rate*dw2;
+        b  -= rate*db;
+    }
+    printf("w1 = %f, w2 = %f, b = %f, c = %f\n", w1, w2, b, cost(w1, w2, b));
+
+    for (size_t i = 0; i < 2; ++i) {
+        for (size_t j = 0; j < 2; ++j) {
+            printf("%zu | %zu = %f\n", i, j, sigmoidf(i*w1 + j*w2 + b));
+        }
+    }
+
+    return 0;
+}

twice.c

@@ -0,0 +1,59 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+float train[][2] = {
+    {0, 0},
+    {1, 2},
+    {2, 4},
+    {3, 6},
+    {4, 8},
+};
+#define train_count (sizeof(train)/sizeof(train[0]))
+
+float rand_float(void)
+{
+    return (float) rand()/ (float) RAND_MAX;
+}
+
+// x1, x2, x3, ..., b
+// w1, w2, w3, ...
+// y = x1*w1 + x2*w2 + x3*w3 + ... + b
+
+float cost(float w, float b)
+{
+    float result = 0.0f;
+    for (size_t i = 0; i < train_count; ++i) {
+        float x = train[i][0];
+        float y = x*w + b;
+        float d = y - train[i][1];
+        result += d*d;
+    }
+    result /= train_count;
+    return result;
+}
+
+int main()
+{
+    srand(time(0));
+    float w = rand_float()*10.0f;
+    float b = rand_float()*5.0f;
+
+    float eps = 1e-3;
+    float rate = 1e-3;
+
+    printf("%f\n", cost(w, b));
+    for (size_t i = 0; i < 500; ++i) {
+        float c = cost(w, b);
+        float dw = (cost(w + eps, b) - c)/eps;
+        float db = (cost(w, b + eps) - c)/eps;
+        w -= rate*dw;
+        b -= rate*db;
+        printf("cost = %f, w = %f, b = %f\n", cost(w, b), w, b);
+    }
+
+    printf("------------------------------\n");
+    printf("w = %f, b = %f\n", w, b);
+
+    return 0;
+}

xor.c

@@ -0,0 +1,228 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <time.h>
+
+typedef struct {
+    float or_w1;
+    float or_w2;
+    float or_b;
+    float nand_w1;
+    float nand_w2;
+    float nand_b;
+    float and_w1;
+    float and_w2;
+    float and_b;
+} Xor;
+
+float sigmoidf(float x)
+{
+    return 1.f / (1.f + expf(-x));
+}
+
+float forward(Xor m, float x1, float x2)
+{
+    float a = sigmoidf(m.or_w1*x1 + m.or_w2*x2 + m.or_b);
+    float b = sigmoidf(m.nand_w1*x1 + m.nand_w2*x2 + m.nand_b);
+    return sigmoidf(a*m.and_w1 + b*m.and_w2 + m.and_b);
+}
+
+typedef float sample[3];
+sample xor_train[] = {
+    {0, 0, 0},
+    {1, 0, 1},
+    {0, 1, 1},
+    {1, 1, 0},
+};
+
+// NAND-gate
+sample or_train[] = {
+    {0, 0, 0},
+    {1, 0, 1},
+    {0, 1, 1},
+    {1, 1, 1},
+};
+
+sample and_train[] = {
+    {0, 0, 0},
+    {1, 0, 0},
+    {0, 1, 0},
+    {1, 1, 1},
+};
+
+sample nand_train[] = {
+    {0, 0, 1},
+    {1, 0, 1},
+    {0, 1, 1},
+    {1, 1, 0},
+};
+
+sample nor_train[] = {
+    {0, 0, 1},
+    {1, 0, 0},
+    {0, 1, 0},
+    {1, 1, 0},
+};
+
+sample *train = xor_train;
+size_t train_count = 4;
+
+float cost(Xor m)
+{
+    float result = 0.0f;
+    for (size_t i = 0; i < train_count; ++i) {
+        float x1 = train[i][0];
+        float x2 = train[i][1];
+        float y = forward(m, x1, x2);
+        float d = y - train[i][2];
+        result += d*d;
+    }
+    result /= train_count;
+    return result;
+}
+
+float rand_float(void)
+{
+    return (float) rand()/ (float) RAND_MAX;
+}
+
+Xor rand_xor(void)
+{
+    Xor m;
+    m.or_w1 = rand_float();
+    m.or_w2 = rand_float();
+    m.or_b = rand_float();
+    m.nand_w1 = rand_float();
+    m.nand_w2 = rand_float();
+    m.nand_b = rand_float();
+    m.and_w1 = rand_float();
+    m.and_w2 = rand_float();
+    m.and_b = rand_float();
+    return m;
+}
+
+void print_xor(Xor m)
+{
+    printf("or_w1 = %f\n", m.or_w1);
+    printf("or_w2 = %f\n", m.or_w2);
+    printf("or_b = %f\n", m.or_b);
+    printf("nand_w1 = %f\n", m.nand_w1);
+    printf("nand_w2 = %f\n", m.nand_w2);
+    printf("nand_b = %f\n", m.nand_b);
+    printf("and_w1 = %f\n", m.and_w1);
+    printf("and_w2 = %f\n", m.and_w2);
+    printf("and_b = %f\n", m.and_b);
+}
+
+Xor learn(Xor m, Xor g, float rate)
+{
+    m.or_w1 -= rate*g.or_w1;
+    m.or_w2 -= rate*g.or_w2;
+    m.or_b -= rate*g.or_b;
+    m.nand_w1 -= rate*g.nand_w1;
+    m.nand_w2 -= rate*g.nand_w2;
+    m.nand_b -= rate*g.nand_b;
+    m.and_w1 -= rate*g.and_w1;
+    m.and_w2 -= rate*g.and_w2;
+    m.and_b -= rate*g.and_b;
+    return m;
+}
+
+Xor finite_diff(Xor m, float eps)
+{
+    Xor g;
+    float c = cost(m);
+    float saved;
+
+    saved = m.or_w1;
+    m.or_w1 += eps;
+    g.or_w1 = (cost(m) - c)/eps;
+    m.or_w1 = saved;
+
+    saved = m.or_w2;
+    m.or_w2 += eps;
+    g.or_w2 = (cost(m) - c)/eps;
+    m.or_w2 = saved;
+
+    saved = m.or_b;
+    m.or_b += eps;
+    g.or_b = (cost(m) - c)/eps;
+    m.or_b = saved;
+
+    saved = m.nand_w1;
+    m.nand_w1 += eps;
+    g.nand_w1 = (cost(m) - c)/eps;
+    m.nand_w1 = saved;
+
+    saved = m.nand_w2;
+    m.nand_w2 += eps;
+    g.nand_w2 = (cost(m) - c)/eps;
+    m.nand_w2 = saved;
+
+    saved = m.nand_b;
+    m.nand_b += eps;
+    g.nand_b = (cost(m) - c)/eps;
+    m.nand_b = saved;
+
+    saved = m.and_w1;
+    m.and_w1 += eps;
+    g.and_w1 = (cost(m) - c)/eps;
+    m.and_w1 = saved;
+
+    saved = m.and_w2;
+    m.and_w2 += eps;
+    g.and_w2 = (cost(m) - c)/eps;
+    m.and_w2 = saved;
+
+    saved = m.and_b;
+    m.and_b += eps;
+    g.and_b = (cost(m) - c)/eps;
+    m.and_b = saved;
+
+    return g;
+}
+
+int main(void)
+{
+    srand(time(0));
+    Xor m = rand_xor();
+
+    float eps = 1e-1;
+    float rate = 1e-1;
+
+    for (size_t i = 0; i < 100*1000; ++i) {
+        Xor g = finite_diff(m, eps);
+        m = learn(m, g, rate);
+        // printf("cost = %f\n", cost(m));
+    }
+    printf("cost = %f\n", cost(m));
+
+    printf("------------------------------\n");
+    for (size_t i = 0; i < 2; ++i) {
+        for (size_t j = 0; j < 2; ++j) {
+            printf("%zu ^ %zu = %f\n", i, j, forward(m, i, j));
+        }
+    }
+    printf("------------------------------\n");
+    printf("\"OR\" neuron:\n");
+    for (size_t i = 0; i < 2; ++i) {
+        for (size_t j = 0; j < 2; ++j) {
+            printf("%zu | %zu = %f\n", i, j, sigmoidf(m.or_w1*i + m.or_w2*j + m.or_b));
+        }
+    }
+    printf("------------------------------\n");
+    printf("\"NAND\" neuron:\n");
+    for (size_t i = 0; i < 2; ++i) {
+        for (size_t j = 0; j < 2; ++j) {
+            printf("~(%zu & %zu) = %f\n", i, j, sigmoidf(m.nand_w1*i + m.nand_w2*j + m.nand_b));
+        }
+    }
+    printf("------------------------------\n");
+    printf("\"AND\" neuron:\n");
+    for (size_t i = 0; i < 2; ++i) {
+        for (size_t j = 0; j < 2; ++j) {
+            printf("%zu & %zu = %f\n", i, j, sigmoidf(m.and_w1*i + m.and_w2*j + m.and_b));
+        }
+    }
+    return 0;
+}