|
|
@@ -0,0 +1,1474 @@
|
|
|
+#include <stdio.h>
|
|
|
+#include "squirrel.h"
|
|
|
+#include <string.h>
|
|
|
+#include <inttypes.h>
|
|
|
+#include <math.h>
|
|
|
+#include <stdlib.h>
|
|
|
+#include <sys/time.h>
|
|
|
+//#include <pthread.h>
|
|
|
+
|
|
|
+SQ_OPT_STRING_STRLEN();
|
|
|
+
|
|
|
+extern "C" {
|
|
|
+#include "nn.h"
|
|
|
+
|
|
|
+void *ann_malloc(size_t sz)
|
|
|
+{
|
|
|
+ return sq_malloc(sz);
|
|
|
+}
|
|
|
+void ann_free(void *p)
|
|
|
+{
|
|
|
+ sq_free(p, 0);
|
|
|
+}
|
|
|
+}
|
|
|
+
|
|
|
+#define NR_FLAG_NONE 0
|
|
|
+#define NR_FLAG_TRAINING (1<<0) /* NN is training in a thread. */
|
|
|
+#define NR_FLAG_REGRESSOR (1<<1) /* NN will be used for regression. */
|
|
|
+#define NR_FLAG_CLASSIFIER (1<<2) /* NN will be used for classification.*/
|
|
|
+#define NR_FLAG_NORMALIZE (1<<3) /* Perform input/output normalization.*/
|
|
|
+#define NR_FLAG_AUTO_STOP (1<<4) /* Auto stop on training. */
|
|
|
+#define NR_FLAG_OF_DETECTED (1<<5) /* Auto stopped on overfitting. */
|
|
|
+#define NR_FLAG_BACKTRACK (1<<6) /* Auto stop with backtracking. */
|
|
|
+
|
|
|
+/* Flags to persist when saving the NN. */
|
|
|
+#define NR_FLAG_TO_PERSIST (NR_FLAG_REGRESSOR| \
|
|
|
+ NR_FLAG_CLASSIFIER| \
|
|
|
+ NR_FLAG_NORMALIZE| \
|
|
|
+ NR_FLAG_OF_DETECTED)
|
|
|
+
|
|
|
+/* Flags to transfer after training. */
|
|
|
+#define NR_FLAG_TO_TRANSFER (NR_FLAG_OF_DETECTED)
|
|
|
+
|
|
|
+#define NR_MAX_LAYERS 32
|
|
|
+#define NR_RDB_ENC_VER 2
|
|
|
+
|
|
|
+typedef struct {
|
|
|
+ uint32_t len, maxlen;
|
|
|
+ float *inputs, *outputs;
|
|
|
+} NRDataset;
|
|
|
+
|
|
|
+typedef struct {
|
|
|
+ uint64_t id; /* Neural network unique ID. */
|
|
|
+ uint64_t training_total_steps; /* How many steps of trainig the network
|
|
|
+ received. A step is a single input/output
|
|
|
+ pattern presented to the net (counting
|
|
|
+ the same pattern multiple times) */
|
|
|
+ uint64_t training_total_ms; /* Total milliseconds time of training. */
|
|
|
+ uint64_t training_max_cycles; /* Max cycles of a single training. */
|
|
|
+ uint64_t training_max_ms; /* Max time of a single training. */
|
|
|
+ uint32_t flags; /* NR_FLAG_... */
|
|
|
+ uint32_t epochs; /* Number of training epochs so far. */
|
|
|
+ AnnRprop *nn; /* Neural network structure. */
|
|
|
+ NRDataset dataset; /* Training dataset. */
|
|
|
+ NRDataset test; /* Testing dataset. */
|
|
|
+ float dataset_error; /* Average error in the training dataset. */
|
|
|
+ float test_error; /* Average error in the test dataset. */
|
|
|
+ float test_class_error; /* Percentage of wrong classifications in test
|
|
|
+ dataset. Only applicable to nets flagged with
|
|
|
+ NR_FLAG_CLASSIFIER. */
|
|
|
+ /* For normalized (NR_FLAG_NORMALIZE) networks. */
|
|
|
+ float *inorm; /* Inputs normalization factors. */
|
|
|
+ float *onorm; /* Outputs normalization factors. */
|
|
|
+} NRTypeObject;
|
|
|
+#if 0
|
|
|
+typedef struct {
|
|
|
+ //RedisModuleString *key; /* Key name of the NN we are training.
|
|
|
+ // Set to NULL for unused slots. */
|
|
|
+ int db_id; /* DB ID where the key is. */
|
|
|
+ pthread_t tid; /* Thread ID of the trainer. */
|
|
|
+ int in_progress; /* 0 if training terminated. */
|
|
|
+ NRTypeObject *nr; /* A copy of the NN we are training. */
|
|
|
+ float dataset_error; /* Dataset error in the last cycle. */
|
|
|
+ float test_error; /* Test error in the last cycle. */
|
|
|
+ float class_error; /* Percentage of wrong classifications. */
|
|
|
+ int curcycle; /* Current cycle. */
|
|
|
+} NRPendingTraining;
|
|
|
+#endif
|
|
|
+/* We take an array with NNs currently training in other threads.
|
|
|
+ * Every time an NN command is called, we try to see if there are
|
|
|
+ * finished trainings, in order to udpate weights of the original
|
|
|
+ * NN stored into the key (we work on a copy on the other thread).*/
|
|
|
+#define NR_PENDING_TRAINING_MAX_LEN 32
|
|
|
+
|
|
|
+#if 0
|
|
|
+#define REDISMODULE_ERR -1
|
|
|
+#define REDISMODULE_OK 0
|
|
|
+
|
|
|
+#define RedisModuleCtx void
|
|
|
+
|
|
|
+#define RedisModule_Log(ctx, log_level, msg)
|
|
|
+
|
|
|
+#define UNUSED(V) ((void) V)
|
|
|
+typedef SQString RedisModuleString;
|
|
|
+
|
|
|
+static uint64_t NRNextId = 1; /* Next neural network unique ID. */
|
|
|
+
|
|
|
+#define RedisModule_Alloc(x) sq_malloc(x)
|
|
|
+
|
|
|
+static void *RedisModule_Calloc(size_t nelm, size_t sz)
|
|
|
+{
|
|
|
+ size_t malloc_size = nelm * sz;
|
|
|
+ void *ptr = sq_malloc(malloc_size);
|
|
|
+ if(ptr) memset(ptr, 0, malloc_size);
|
|
|
+ return ptr;
|
|
|
+}
|
|
|
+
|
|
|
+static void *RedisModule_Realloc(void *oldPtr, size_t sz)
|
|
|
+{
|
|
|
+ void *ptr = sq_realloc(oldPtr, 0, sz);
|
|
|
+ return ptr;
|
|
|
+}
|
|
|
+#define RedisModule_Free(x) sq_free(x, 0)
|
|
|
+
|
|
|
+static pthread_mutex_t NRPendingTrainingMutex = PTHREAD_MUTEX_INITIALIZER;
|
|
|
+/* All the followings must be accessed after acquiring the mutex. */
|
|
|
+static NRPendingTraining NRTrainings[NR_PENDING_TRAINING_MAX_LEN];
|
|
|
+static int NRPendingTrainingCount = 0; /* Number of pending trainings. */
|
|
|
+
|
|
|
+/* ========================== Low level object API ========================== */
|
|
|
+
|
|
|
+long long NRMilliseconds(void) {
|
|
|
+ struct timeval tv;
|
|
|
+ long long ust;
|
|
|
+
|
|
|
+ gettimeofday(&tv, NULL);
|
|
|
+ ust = ((long long)tv.tv_sec)*1000000;
|
|
|
+ ust += tv.tv_usec;
|
|
|
+ return ust/1000;
|
|
|
+}
|
|
|
+
|
|
|
+/* Create a network with the specified parameters. Note that the layers
|
|
|
+ * must be specified from the output layer[0] to the input
|
|
|
+ * layer[N]. Each element in the integer array 'layer' specify how many
|
|
|
+ * units there are in the corresponding layer. */
|
|
|
+static NRTypeObject *createNRTypeObject(int flags, int *layers, int numlayers, int dset_len, int test_len) {
|
|
|
+ NRTypeObject *o;
|
|
|
+ o = (NRTypeObject*)RedisModule_Calloc(1,sizeof(*o));
|
|
|
+ o->id = NRNextId++;
|
|
|
+ o->flags = flags;
|
|
|
+ o->nn = AnnCreateNet(numlayers,layers);
|
|
|
+ o->dataset.maxlen = dset_len;
|
|
|
+ o->test.maxlen = test_len;
|
|
|
+ int ilen = ANN_INPUT_UNITS(o->nn);
|
|
|
+ int olen = ANN_OUTPUT_UNITS(o->nn);
|
|
|
+ o->inorm = (float*)RedisModule_Calloc(1,sizeof(float)*ilen);
|
|
|
+ o->onorm = (float*)RedisModule_Calloc(1,sizeof(float)*olen);
|
|
|
+ for (int j = 0; j < ilen; j++) o->inorm[j] = 1;
|
|
|
+ for (int j = 0; j < olen; j++) o->onorm[j] = 1;
|
|
|
+ return o;
|
|
|
+}
|
|
|
+
|
|
|
+/* Insert data (observations needed to train and test the NN) into the
|
|
|
+ * NN object. While the learning and testing datasets are yet not full
|
|
|
+ * the observed pattern is inserted evenly in one or the other side in
|
|
|
+ * order to make sure the two datasets are populated evenly. When both
|
|
|
+ * are already full, a random elmenet from one or the other (doing
|
|
|
+ * a random weighted choice depending on the length) is substituted with
|
|
|
+ * the new item. */
|
|
|
+#define NR_INSERT_NO_TARGET 0 /* Auto select where to insert. */
|
|
|
+#define NR_INSERT_TRAIN 1 /* Insert in training dataset. */
|
|
|
+#define NR_INSERT_TEST 2 /* Insert in testing dataset. */
|
|
|
+static void NRTypeInsertData(NRTypeObject *o, float *inputs, float *outputs,
|
|
|
+ int target_ds) {
|
|
|
+ NRDataset *target = NULL;
|
|
|
+
|
|
|
+ /* Check if there is no dataset at all. This may be a valid setup
|
|
|
+ * with online learning, sample by sample. */
|
|
|
+ if (o->dataset.maxlen == 0 && o->test.maxlen == 0) return;
|
|
|
+
|
|
|
+ /* If the user specified a target, select it. */
|
|
|
+ if (target_ds == NR_INSERT_TRAIN) target = &o->dataset;
|
|
|
+ else if (target_ds == NR_INSERT_TEST) target = &o->test;
|
|
|
+
|
|
|
+ /* If no target is specified, but there is only one possible
|
|
|
+ * target, select it ASAP. */
|
|
|
+ if (o->dataset.maxlen == 0) {
|
|
|
+ target = &o->test;
|
|
|
+ } else if (o->test.maxlen == 0) {
|
|
|
+ target = &o->dataset;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Otherwise choose as the target to populate the one with less data
|
|
|
+ * relatively to its size. */
|
|
|
+ if (target == NULL) {
|
|
|
+ /* If one of the two datasets are still not full, pick
|
|
|
+ * based on fill percentage. Otherwise pick a random
|
|
|
+ * target relatively to their size. */
|
|
|
+ if (o->dataset.len != o->dataset.maxlen ||
|
|
|
+ o->test.len != o->dataset.len)
|
|
|
+ {
|
|
|
+ float fill_a = (float)o->dataset.len / o->dataset.maxlen;
|
|
|
+ float fill_b = (float)o->test.len / o->test.maxlen;
|
|
|
+ target = (fill_a <= fill_b) ? &o->dataset : &o->test;
|
|
|
+ } else {
|
|
|
+ double r = rand()/RAND_MAX;
|
|
|
+ double sumlen = o->dataset.maxlen + o->test.maxlen;
|
|
|
+ if (r < (double)o->dataset.maxlen/sumlen) {
|
|
|
+ target = &o->dataset;
|
|
|
+ } else {
|
|
|
+ target = &o->test;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Append if there is room or substitute with a random entry. */
|
|
|
+ size_t idx;
|
|
|
+ int j, numin = ANN_INPUT_UNITS(o->nn),
|
|
|
+ numout = ANN_OUTPUT_UNITS(o->nn);
|
|
|
+
|
|
|
+ if (target->maxlen == target->len) {
|
|
|
+ idx = rand() % target->maxlen;
|
|
|
+ } else {
|
|
|
+ idx = target->len;
|
|
|
+ target->len++;
|
|
|
+ target->inputs = (float*)RedisModule_Realloc(target->inputs,
|
|
|
+ sizeof(float)*numin*target->len);
|
|
|
+ target->outputs = (float*)RedisModule_Realloc(target->outputs,
|
|
|
+ sizeof(float)*numout*target->len);
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Finally store the values at position. */
|
|
|
+ for (j = 0; j < numin; j++)
|
|
|
+ target->inputs[idx*numin+j] = inputs[j];
|
|
|
+ for (j = 0; j < numout; j++)
|
|
|
+ target->outputs[idx*numout+j] = outputs[j];
|
|
|
+}
|
|
|
+
|
|
|
+/* Free the specified dataset. */
|
|
|
+void NRDatasetFree(NRDataset *dset) {
|
|
|
+ RedisModule_Free(dset->inputs);
|
|
|
+ RedisModule_Free(dset->outputs);
|
|
|
+}
|
|
|
+
|
|
|
+/* Free a whole NN object. */
|
|
|
+void NRTypeReleaseObject(NRTypeObject *o) {
|
|
|
+ AnnFree(o->nn);
|
|
|
+ NRDatasetFree(&o->dataset);
|
|
|
+ NRDatasetFree(&o->test);
|
|
|
+ RedisModule_Free(o->inorm);
|
|
|
+ RedisModule_Free(o->onorm);
|
|
|
+ RedisModule_Free(o);
|
|
|
+}
|
|
|
+
|
|
|
+/* ================================ Training =============================== */
|
|
|
+
|
|
|
+/* Clone a neural network object, including the training and test dataset.
|
|
|
+ * We use cloning in order to train in a different thread, and later
|
|
|
+ * copy the weights back into the original NN.
|
|
|
+ *
|
|
|
+ * Note when 'newid' is 0, the copied object NN unique ID is the same as the
|
|
|
+ * original as normally this is what we want, in order to later match the
|
|
|
+ * trained network with the object stored at the specified key
|
|
|
+ * in the pending traning structure.
|
|
|
+ *
|
|
|
+ * However if the copy is performed with other goals, 'newid' should
|
|
|
+ * be set to non-zero in order to create a net with a different ID. */
|
|
|
+NRTypeObject *NRClone(NRTypeObject *o, int newid) {
|
|
|
+ NRTypeObject *copy;
|
|
|
+ copy = (NRTypeObject*)RedisModule_Calloc(1,sizeof(*o));
|
|
|
+ *copy = *o;
|
|
|
+ if (newid) copy->id = NRNextId++;
|
|
|
+ copy->nn = AnnClone(o->nn);
|
|
|
+ copy->dataset = o->dataset;
|
|
|
+ copy->test = o->test;
|
|
|
+
|
|
|
+ int ilen = ANN_INPUT_UNITS(o->nn);
|
|
|
+ int olen = ANN_OUTPUT_UNITS(o->nn);
|
|
|
+ copy->dataset.inputs = (float*)RedisModule_Alloc(sizeof(float)*ilen*o->dataset.len);
|
|
|
+ copy->dataset.outputs = (float*)RedisModule_Alloc(sizeof(float)*olen*o->dataset.len);
|
|
|
+ copy->test.inputs = (float*)RedisModule_Alloc(sizeof(float)*ilen*o->test.len);
|
|
|
+ copy->test.outputs = (float*)RedisModule_Alloc(sizeof(float)*olen*o->test.len);
|
|
|
+ memcpy(copy->dataset.inputs,o->dataset.inputs,sizeof(float)*ilen*o->dataset.len);
|
|
|
+ memcpy(copy->dataset.outputs,o->dataset.outputs,sizeof(float)*olen*o->dataset.len);
|
|
|
+ memcpy(copy->test.inputs,o->test.inputs,sizeof(float)*ilen*o->test.len);
|
|
|
+ memcpy(copy->test.outputs,o->test.outputs,sizeof(float)*olen*o->test.len);
|
|
|
+
|
|
|
+ copy->inorm = (float*)RedisModule_Alloc(sizeof(float)*ilen);
|
|
|
+ copy->onorm = (float*)RedisModule_Alloc(sizeof(float)*olen);
|
|
|
+ memcpy(copy->inorm,o->inorm,sizeof(float)*ilen);
|
|
|
+ memcpy(copy->onorm,o->onorm,sizeof(float)*olen);
|
|
|
+ return copy;
|
|
|
+}
|
|
|
+
|
|
|
+/* Transfer the weights from the source to the destination NN.
|
|
|
+ * This is used after the learning process finished in a different
|
|
|
+ * thread in order to transfer the learning back to the orignal
|
|
|
+ * NN. */
|
|
|
+static void NRTransferWeights(RedisModuleCtx *ctx, NRTypeObject *dst, NRTypeObject *src) {
|
|
|
+ if (dst->id != src->id) {
|
|
|
+ RedisModule_Log(ctx,"warning",
|
|
|
+ "NSTransferWeight(): source and destination neural network IDs "
|
|
|
+ "don't match. This is unexpected, probably a bug inside the "
|
|
|
+ "module. Weights not transferred back to the origina NN.");
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* It would be faster to memcpy just the weight array for each layer,
|
|
|
+ * however this way we access the NN in a more abstract way, and should
|
|
|
+ * be fast enough in most cases. We can always optimized it later. */
|
|
|
+ AnnFree(dst->nn);
|
|
|
+ dst->nn = AnnClone(src->nn);
|
|
|
+ dst->training_total_steps = src->training_total_steps;
|
|
|
+ dst->training_total_ms = src->training_total_ms;
|
|
|
+ dst->dataset_error = src->dataset_error;
|
|
|
+ dst->test_error = src->test_error;
|
|
|
+ dst->test_class_error = src->test_class_error;
|
|
|
+ dst->flags |= src->flags & NR_FLAG_TO_TRANSFER;
|
|
|
+
|
|
|
+ int ilen = ANN_INPUT_UNITS(src->nn);
|
|
|
+ int olen = ANN_OUTPUT_UNITS(src->nn);
|
|
|
+ memcpy(dst->inorm,src->inorm,sizeof(float)*ilen);
|
|
|
+ memcpy(dst->onorm,src->onorm,sizeof(float)*olen);
|
|
|
+}
|
|
|
+
|
|
|
+/* Threaded training entry point.
|
|
|
+ *
|
|
|
+ * To get some clue about overfitting algorithm behavior:
|
|
|
+ * #define NR_TRAINING_DEBUG 1
|
|
|
+ */
|
|
|
+void *NRTrainingThreadMain(void *arg) {
|
|
|
+ NRPendingTraining *pt = (NRPendingTraining*)arg;
|
|
|
+ NRTypeObject *nr = pt->nr;
|
|
|
+ int training_iterations = 1;
|
|
|
+ float train_error = 0;
|
|
|
+ float test_error = 0;
|
|
|
+ float class_error = 0;
|
|
|
+ float past_train_error = 1.0/0.0;
|
|
|
+ float past_test_error = 1.0/0.0;
|
|
|
+ int auto_stop = nr->flags & NR_FLAG_AUTO_STOP;
|
|
|
+ int backtrack = nr->flags & NR_FLAG_BACKTRACK;
|
|
|
+
|
|
|
+ uint64_t cycles = 0;
|
|
|
+ long long start = NRMilliseconds();
|
|
|
+ long long cycle_time;
|
|
|
+ int overfitting_count = 0;
|
|
|
+ int overfitting_limit = 5;
|
|
|
+ float best_test_error = 1.0/0.0;
|
|
|
+
|
|
|
+ nr->flags &= ~NR_FLAG_TO_TRANSFER;
|
|
|
+
|
|
|
+ /* If the network is auto normalized, we need to trasnform the inputs
|
|
|
+ * in a way that's acceptable for the NN. We just find the maximum
|
|
|
+ * absolute value, and divide for it, to get a -1,1 range. There
|
|
|
+ * are more advanced transformations that are usually performed that
|
|
|
+ * could be implemented in the future.
|
|
|
+ *
|
|
|
+ * Note that we compute the normalization vectors for all the inputs
|
|
|
+ * and outputs, however if the network is a classifier, flagged with
|
|
|
+ * (NR_FLAG_CLASSIFIER), no output normalization will be done since
|
|
|
+ * the data is already in 0/1 format. */
|
|
|
+ if ((nr->flags & NR_FLAG_NORMALIZE) && nr->dataset.len) {
|
|
|
+ int ilen = ANN_INPUT_UNITS(nr->nn);
|
|
|
+ int olen = ANN_OUTPUT_UNITS(nr->nn);
|
|
|
+ float *imax = nr->inorm;
|
|
|
+ float *omax = nr->onorm;
|
|
|
+ float *inputs = nr->dataset.inputs;
|
|
|
+ float *outputs = nr->dataset.outputs;
|
|
|
+ for (int i = 0; i < ilen; i++) imax[i] = 1;
|
|
|
+ for (int i = 0; i < olen; i++) omax[i] = 1;
|
|
|
+
|
|
|
+ /* Compute the max values vectors. */
|
|
|
+ for (uint32_t j = 0; j < nr->dataset.len; j++) {
|
|
|
+ for (int i = 0; i < ilen; i++)
|
|
|
+ if (fabs(inputs[i]) > imax[i]) imax[i] = fabs(inputs[i]);
|
|
|
+ for (int i = 0; i < olen; i++)
|
|
|
+ if (fabs(outputs[i]) > omax[i]) omax[i] = fabs(outputs[i]);
|
|
|
+ inputs += ilen;
|
|
|
+ outputs += olen;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Likely we are not seeing what will really be the true input/output
|
|
|
+ * maximum value, so we multiply the maximum values found by a constant.
|
|
|
+ * However if the max is exactly "1" we assume it's a classification
|
|
|
+ * input and don't alter it. */
|
|
|
+ for (int i = 0; i < ilen; i++) if (imax[i] != 1) imax[i] *= 1.2;
|
|
|
+ for (int i = 0; i < olen; i++) if (omax[i] != 1) omax[i] *= 1.2;
|
|
|
+
|
|
|
+ /* We can normalize the dataset directly: after the training it will
|
|
|
+ * be discarded anyway. */
|
|
|
+ inputs = nr->dataset.inputs;
|
|
|
+ outputs = nr->dataset.outputs;
|
|
|
+ for (uint32_t j = 0; j < nr->dataset.len; j++) {
|
|
|
+ for (int i = 0; i < ilen; i++) inputs[i] /= nr->inorm[i];
|
|
|
+ if (!(nr->flags & NR_FLAG_CLASSIFIER))
|
|
|
+ for (int i = 0; i < olen; i++) outputs[i] /= nr->onorm[i];
|
|
|
+ inputs += ilen;
|
|
|
+ outputs += olen;
|
|
|
+ }
|
|
|
+
|
|
|
+ inputs = nr->test.inputs;
|
|
|
+ outputs = nr->test.outputs;
|
|
|
+ for (uint32_t j = 0; j < nr->test.len; j++) {
|
|
|
+ for (int i = 0; i < ilen; i++) inputs[i] /= nr->inorm[i];
|
|
|
+ if (!(nr->flags & NR_FLAG_CLASSIFIER))
|
|
|
+ for (int i = 0; i < olen; i++) outputs[i] /= nr->onorm[i];
|
|
|
+ inputs += ilen;
|
|
|
+ outputs += olen;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ AnnRprop *saved = NULL; /* Saved to recover on overfitting. */
|
|
|
+ float saved_error; /* The test error of the saved NN. */
|
|
|
+ float saved_train_error; /* The training dataset error of the saved NN */
|
|
|
+ float saved_class_error; /* The % of classification errors of saved NN */
|
|
|
+
|
|
|
+ while(1) {
|
|
|
+ long long cycle_start = NRMilliseconds();
|
|
|
+
|
|
|
+ train_error = AnnTrain(nr->nn,
|
|
|
+ nr->dataset.inputs,
|
|
|
+ nr->dataset.outputs,
|
|
|
+ 0,
|
|
|
+ training_iterations,
|
|
|
+ nr->dataset.len,
|
|
|
+ ANN_ALGO_BPROP);
|
|
|
+ cycle_time = NRMilliseconds() - cycle_start;
|
|
|
+ nr->training_total_steps += nr->dataset.len*training_iterations;
|
|
|
+
|
|
|
+ /* Evaluate the error in the case of auto training, stop it
|
|
|
+ * once we see that the error in the traning set is decreasing
|
|
|
+ * while the one in the test set is not. */
|
|
|
+ if (auto_stop) {
|
|
|
+ AnnTestError(nr->nn,
|
|
|
+ nr->test.inputs,
|
|
|
+ nr->test.outputs,
|
|
|
+ nr->test.len, &test_error, &class_error);
|
|
|
+
|
|
|
+ if (train_error < past_train_error &&
|
|
|
+ test_error > past_test_error)
|
|
|
+ {
|
|
|
+ overfitting_count++;
|
|
|
+ #ifdef NR_TRAINING_DEBUG
|
|
|
+ printf("+YCLE %lld: [%d] %f VS %f\n", (long long)cycles,
|
|
|
+ overfitting_count, train_error, test_error);
|
|
|
+ #endif
|
|
|
+ if (overfitting_count == overfitting_limit) {
|
|
|
+ nr->flags |= NR_FLAG_OF_DETECTED;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ } else if (overfitting_count > 0) {
|
|
|
+ #ifdef NR_TRAINING_DEBUG
|
|
|
+ printf("-YCLE %lld: [%d] %f VS %f\n", (long long)cycles,
|
|
|
+ overfitting_count, train_error, test_error);
|
|
|
+ #endif
|
|
|
+ overfitting_count--;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Save all the networks with a score better than the currently
|
|
|
+ * saved network. This can be a bit costly, but is safe: one
|
|
|
+ * cycle of training more and overfitting can ruin it all. */
|
|
|
+ if (backtrack && (saved == NULL || test_error < saved_error)) {
|
|
|
+ #ifdef NR_TRAINING_DEBUG
|
|
|
+ printf("SAVED! %f < %f\n", test_error, saved_error);
|
|
|
+ #endif
|
|
|
+ saved_error = test_error;
|
|
|
+ saved_train_error = train_error;
|
|
|
+ saved_class_error = class_error;
|
|
|
+ if (saved) AnnFree(saved);
|
|
|
+ saved = AnnClone(nr->nn);
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Best network found? Reset the overfitting hints counter. */
|
|
|
+ if (test_error < best_test_error) {
|
|
|
+ overfitting_count = 0;
|
|
|
+ best_test_error = test_error;
|
|
|
+ #ifdef NR_TRAINING_DEBUG
|
|
|
+ printf("BEST! %lld: <%d> %f VS %f\n", (long long)cycles,
|
|
|
+ overfitting_limit,train_error, test_error);
|
|
|
+ #endif
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Also stop if the loss is zero in both datasets. */
|
|
|
+ if (train_error < 0.000000000000001 &&
|
|
|
+ test_error < 0.000000000000001) break;
|
|
|
+ }
|
|
|
+
|
|
|
+ cycles++;
|
|
|
+ long long total_time = NRMilliseconds()-start;
|
|
|
+
|
|
|
+ /* Cycles and milliseconds stop conditions. */
|
|
|
+ if (nr->training_max_cycles && cycles == nr->training_max_cycles)
|
|
|
+ break;
|
|
|
+ if (nr->training_max_ms && total_time > (long long)nr->training_max_ms)
|
|
|
+ break;
|
|
|
+
|
|
|
+ /* If this is a long training, to do just a single training iteration
|
|
|
+ * for each cycle is not optimal: tune the number of iterations to
|
|
|
+ * at least take 100 milliseconds. */
|
|
|
+ if (total_time > 10000 && cycle_time < 100) training_iterations++;
|
|
|
+
|
|
|
+ past_train_error = train_error;
|
|
|
+ past_test_error = test_error;
|
|
|
+
|
|
|
+ /* Update stats for NR.THREADS to show progresses. */
|
|
|
+ pthread_mutex_lock(&NRPendingTrainingMutex);
|
|
|
+ pt->dataset_error = train_error;
|
|
|
+ pt->test_error = test_error;
|
|
|
+ if (nr->flags & NR_FLAG_CLASSIFIER) pt->class_error = class_error;
|
|
|
+ pt->curcycle = cycles;
|
|
|
+ pthread_mutex_unlock(&NRPendingTrainingMutex);
|
|
|
+ }
|
|
|
+
|
|
|
+ /* If auto stop is disabled, we still need to compute the test error
|
|
|
+ * in order to return this information to the main thread. */
|
|
|
+ if (!auto_stop) {
|
|
|
+ AnnTestError(nr->nn,
|
|
|
+ nr->test.inputs,
|
|
|
+ nr->test.outputs,
|
|
|
+ nr->test.len, &test_error, &class_error);
|
|
|
+ }
|
|
|
+
|
|
|
+ /* If both autostop and backtracking are enabled, we may have
|
|
|
+ * a better network saved! */
|
|
|
+ if (auto_stop && backtrack) {
|
|
|
+ if (saved && saved_error < test_error) {
|
|
|
+ #ifdef NR_TRAINING_DEBUG
|
|
|
+ printf("BACKTRACK: Saved network used!\n");
|
|
|
+ #endif
|
|
|
+ AnnFree(nr->nn);
|
|
|
+ nr->nn = saved;
|
|
|
+ test_error = saved_error;
|
|
|
+ train_error = saved_train_error;
|
|
|
+ class_error = saved_class_error;
|
|
|
+ } else if (saved) {
|
|
|
+ AnnFree(saved);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (nr->flags & NR_FLAG_CLASSIFIER) nr->test_class_error = class_error;
|
|
|
+ nr->dataset_error = train_error;
|
|
|
+ nr->test_error = test_error;
|
|
|
+ nr->training_total_ms += NRMilliseconds()-start;
|
|
|
+
|
|
|
+ /* Signal that the training process has finished, it's up to the main
|
|
|
+ * thread to cleanup this training slot, copying the weights to the
|
|
|
+ * original neural network and reclaiming memory for the copy we
|
|
|
+ * used to work. */
|
|
|
+ pthread_mutex_lock(&NRPendingTrainingMutex);
|
|
|
+ pt->in_progress = 0;
|
|
|
+ pthread_mutex_unlock(&NRPendingTrainingMutex);
|
|
|
+ return NULL;
|
|
|
+}
|
|
|
+
|
|
|
+/* Start a background training in another thread. Return REDISMODULE_ERR if
|
|
|
+ * there is no free slot for training, as we already reached the maximum of
|
|
|
+ * networks we can train in parallel.
|
|
|
+ *
|
|
|
+ * The 'flags' argument specifies the additional NN flags to pass to the
|
|
|
+ * training ruotine:
|
|
|
+ *
|
|
|
+ * NR_FLAG_AUTO_STOP -- Automatically stop training on overtraining.
|
|
|
+ * NR_FLAG_BACKTRACK -- Save current NN state when overfitting is likely.
|
|
|
+ */
|
|
|
+int NRStartTraining(RedisModuleCtx *ctx, RedisModuleString *key, int dbid, NRTypeObject *nr) {
|
|
|
+ pthread_mutex_lock(&NRPendingTrainingMutex);
|
|
|
+ if (NRPendingTrainingCount == NR_PENDING_TRAINING_MAX_LEN) {
|
|
|
+ pthread_mutex_unlock(&NRPendingTrainingMutex);
|
|
|
+ return REDISMODULE_ERR;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Setup our trainig data. */
|
|
|
+ NRPendingTraining *pt = &NRTrainings[NRPendingTrainingCount];
|
|
|
+ //pt->key = RedisModule_CreateStringFromString(ctx,key);
|
|
|
+ //RedisModule_RetainString(ctx,pt->key);
|
|
|
+ pt->db_id = dbid;
|
|
|
+ pt->in_progress = 1;
|
|
|
+ pt->nr = NRClone(nr,0);
|
|
|
+ pt->dataset_error = 0;
|
|
|
+ pt->test_error = 0;
|
|
|
+ pt->class_error = 0;
|
|
|
+ pt->curcycle = 0;
|
|
|
+ if (pthread_create(&pt->tid,NULL,NRTrainingThreadMain,pt) != 0) {
|
|
|
+ RedisModule_Log(ctx,"warning","Unable to create a new pthread in NRStartTraining()");
|
|
|
+ //RedisModule_FreeString(ctx,pt->key);
|
|
|
+ pt->key = NULL;
|
|
|
+ NRTypeReleaseObject(pt->nr);
|
|
|
+ pthread_mutex_unlock(&NRPendingTrainingMutex);
|
|
|
+ return REDISMODULE_ERR;
|
|
|
+ }
|
|
|
+ NRPendingTrainingCount++;
|
|
|
+ nr->flags |= NR_FLAG_TRAINING;
|
|
|
+ nr->flags &= ~NR_FLAG_TO_TRANSFER;
|
|
|
+ pthread_mutex_unlock(&NRPendingTrainingMutex);
|
|
|
+ return REDISMODULE_OK;
|
|
|
+}
|
|
|
+
|
|
|
+/* Check if there are threads that terminated the NN training, and
|
|
|
+ * collect the info they computed (that is the new NN). */
|
|
|
+int NRCollectThreads(RedisModuleCtx *ctx) {
|
|
|
+ int collected = 0;
|
|
|
+ pthread_mutex_lock(&NRPendingTrainingMutex);
|
|
|
+ for (int j = 0; j < NRPendingTrainingCount; j++) {
|
|
|
+ NRPendingTraining *pt = &NRTrainings[j];
|
|
|
+ if (pt->in_progress == 0) {
|
|
|
+ /* Training terminated. Let's see if the key
|
|
|
+ * is still there and NN ID matches. */
|
|
|
+ int orig_id = RedisModule_GetSelectedDb(ctx);
|
|
|
+ if (orig_id != pt->db_id) RedisModule_SelectDb(ctx,pt->db_id);
|
|
|
+ RedisModuleKey *key = RedisModule_OpenKey(ctx,pt->key,
|
|
|
+ REDISMODULE_READ|REDISMODULE_WRITE);
|
|
|
+ if (RedisModule_ModuleTypeGetType(key) == NRType) {
|
|
|
+ NRTypeObject *nr = RedisModule_ModuleTypeGetValue(key);
|
|
|
+ if (nr->id == pt->nr->id) {
|
|
|
+ NRTransferWeights(ctx,nr,pt->nr);
|
|
|
+ nr->flags &= ~NR_FLAG_TRAINING;
|
|
|
+ }
|
|
|
+ RedisModule_FreeString(ctx,pt->key);
|
|
|
+ pt->key = NULL;
|
|
|
+ NRTypeReleaseObject(pt->nr);
|
|
|
+ NRPendingTrainingCount--;
|
|
|
+ memcpy(&NRTrainings[j],&NRTrainings[j+1],
|
|
|
+ (NRPendingTrainingCount-j)*sizeof(NRTrainings[0]));
|
|
|
+ }
|
|
|
+ if (orig_id != pt->db_id) RedisModule_SelectDb(ctx,orig_id);
|
|
|
+ collected++;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ pthread_mutex_unlock(&NRPendingTrainingMutex);
|
|
|
+ return collected;
|
|
|
+}
|
|
|
+#endif // 0
|
|
|
+
|
|
|
+#define RedisModule_Free(x) sq_free(x, 0)
|
|
|
+
|
|
|
+static void *RedisModule_Calloc(size_t nelm, size_t sz)
|
|
|
+{
|
|
|
+ size_t malloc_size = nelm * sz;
|
|
|
+ void *ptr = sq_malloc(malloc_size);
|
|
|
+ if(ptr) memset(ptr, 0, malloc_size);
|
|
|
+ return ptr;
|
|
|
+}
|
|
|
+
|
|
|
+static void *RedisModule_Realloc(void *oldPtr, size_t sz)
|
|
|
+{
|
|
|
+ void *ptr = sq_realloc(oldPtr, 0, sz);
|
|
|
+ return ptr;
|
|
|
+}
|
|
|
+
|
|
|
+static uint64_t NRNextId = 1; /* Next neural network unique ID. */
|
|
|
+
|
|
|
+long long NRMilliseconds(void) {
|
|
|
+ struct timeval tv;
|
|
|
+ long long ust;
|
|
|
+
|
|
|
+ gettimeofday(&tv, NULL);
|
|
|
+ ust = ((long long)tv.tv_sec)*1000000;
|
|
|
+ ust += tv.tv_usec;
|
|
|
+ return ust/1000;
|
|
|
+}
|
|
|
+
|
|
|
+/* Create a network with the specified parameters. Note that the layers
|
|
|
+ * must be specified from the output layer[0] to the input
|
|
|
+ * layer[N]. Each element in the integer array 'layer' specify how many
|
|
|
+ * units there are in the corresponding layer. */
|
|
|
+static NRTypeObject *createNRTypeObject(int flags, int *layers, int numlayers, int dset_len, int test_len) {
|
|
|
+ NRTypeObject *o;
|
|
|
+ o = (NRTypeObject*)RedisModule_Calloc(1,sizeof(*o));
|
|
|
+ o->id = NRNextId++;
|
|
|
+ o->flags = flags;
|
|
|
+ o->nn = AnnCreateNet(numlayers,layers);
|
|
|
+ o->dataset.maxlen = dset_len;
|
|
|
+ o->test.maxlen = test_len;
|
|
|
+ int ilen = ANN_INPUT_UNITS(o->nn);
|
|
|
+ int olen = ANN_OUTPUT_UNITS(o->nn);
|
|
|
+ o->inorm = (float*)RedisModule_Calloc(1,sizeof(float)*ilen);
|
|
|
+ o->onorm = (float*)RedisModule_Calloc(1,sizeof(float)*olen);
|
|
|
+ for (int j = 0; j < ilen; j++) o->inorm[j] = 1;
|
|
|
+ for (int j = 0; j < olen; j++) o->onorm[j] = 1;
|
|
|
+ return o;
|
|
|
+}
|
|
|
+
|
|
|
+/* Insert data (observations needed to train and test the NN) into the
|
|
|
+ * NN object. While the learning and testing datasets are yet not full
|
|
|
+ * the observed pattern is inserted evenly in one or the other side in
|
|
|
+ * order to make sure the two datasets are populated evenly. When both
|
|
|
+ * are already full, a random elmenet from one or the other (doing
|
|
|
+ * a random weighted choice depending on the length) is substituted with
|
|
|
+ * the new item. */
|
|
|
+#define NR_INSERT_NO_TARGET 0 /* Auto select where to insert. */
|
|
|
+#define NR_INSERT_TRAIN 1 /* Insert in training dataset. */
|
|
|
+#define NR_INSERT_TEST 2 /* Insert in testing dataset. */
|
|
|
+static void NRTypeInsertData(NRTypeObject *o, float *inputs, float *outputs,
|
|
|
+ int target_ds) {
|
|
|
+ NRDataset *target = NULL;
|
|
|
+
|
|
|
+ /* Check if there is no dataset at all. This may be a valid setup
|
|
|
+ * with online learning, sample by sample. */
|
|
|
+ if (o->dataset.maxlen == 0 && o->test.maxlen == 0) return;
|
|
|
+
|
|
|
+ /* If the user specified a target, select it. */
|
|
|
+ if (target_ds == NR_INSERT_TRAIN) target = &o->dataset;
|
|
|
+ else if (target_ds == NR_INSERT_TEST) target = &o->test;
|
|
|
+
|
|
|
+ /* If no target is specified, but there is only one possible
|
|
|
+ * target, select it ASAP. */
|
|
|
+ if (o->dataset.maxlen == 0) {
|
|
|
+ target = &o->test;
|
|
|
+ } else if (o->test.maxlen == 0) {
|
|
|
+ target = &o->dataset;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Otherwise choose as the target to populate the one with less data
|
|
|
+ * relatively to its size. */
|
|
|
+ if (target == NULL) {
|
|
|
+ /* If one of the two datasets are still not full, pick
|
|
|
+ * based on fill percentage. Otherwise pick a random
|
|
|
+ * target relatively to their size. */
|
|
|
+ if (o->dataset.len != o->dataset.maxlen ||
|
|
|
+ o->test.len != o->dataset.len)
|
|
|
+ {
|
|
|
+ float fill_a = (float)o->dataset.len / o->dataset.maxlen;
|
|
|
+ float fill_b = (float)o->test.len / o->test.maxlen;
|
|
|
+ target = (fill_a <= fill_b) ? &o->dataset : &o->test;
|
|
|
+ } else {
|
|
|
+ double r = rand()/RAND_MAX;
|
|
|
+ double sumlen = o->dataset.maxlen + o->test.maxlen;
|
|
|
+ if (r < (double)o->dataset.maxlen/sumlen) {
|
|
|
+ target = &o->dataset;
|
|
|
+ } else {
|
|
|
+ target = &o->test;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Append if there is room or substitute with a random entry. */
|
|
|
+ size_t idx;
|
|
|
+ int j, numin = ANN_INPUT_UNITS(o->nn),
|
|
|
+ numout = ANN_OUTPUT_UNITS(o->nn);
|
|
|
+
|
|
|
+ if (target->maxlen == target->len) {
|
|
|
+ idx = rand() % target->maxlen;
|
|
|
+ } else {
|
|
|
+ idx = target->len;
|
|
|
+ target->len++;
|
|
|
+ target->inputs = (float*)RedisModule_Realloc(target->inputs,
|
|
|
+ sizeof(float)*numin*target->len);
|
|
|
+ target->outputs = (float*)RedisModule_Realloc(target->outputs,
|
|
|
+ sizeof(float)*numout*target->len);
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Finally store the values at position. */
|
|
|
+ for (j = 0; j < numin; j++)
|
|
|
+ target->inputs[idx*numin+j] = inputs[j];
|
|
|
+ for (j = 0; j < numout; j++)
|
|
|
+ target->outputs[idx*numout+j] = outputs[j];
|
|
|
+}
|
|
|
+
|
|
|
+/* Free the specified dataset. */
|
|
|
+void NRDatasetFree(NRDataset *dset) {
|
|
|
+ RedisModule_Free(dset->inputs);
|
|
|
+ RedisModule_Free(dset->outputs);
|
|
|
+}
|
|
|
+
|
|
|
+/* Free a whole NN object. */
|
|
|
+void NRTypeReleaseObject(NRTypeObject *o) {
|
|
|
+ AnnFree(o->nn);
|
|
|
+ NRDatasetFree(&o->dataset);
|
|
|
+ NRDatasetFree(&o->test);
|
|
|
+ RedisModule_Free(o->inorm);
|
|
|
+ RedisModule_Free(o->onorm);
|
|
|
+ RedisModule_Free(o);
|
|
|
+}
|
|
|
+
|
|
|
+static const SQChar sq_nn_TAG[] = _SC("AnnRprop");
|
|
|
+
|
|
|
+static SQRESULT sq_nn_release_hook(SQUserPointer p, SQInteger size, void */*ep*/) {
|
|
|
+ NRTypeObject *self = (NRTypeObject *)p;
|
|
|
+ if(self) NRTypeReleaseObject(self);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+** Creates a new AnnRprop.
|
|
|
+*/
|
|
|
+static SQRESULT sq_nn_constructor (HSQUIRRELVM v) {
|
|
|
+ SQ_FUNC_VARS(v);
|
|
|
+ SQ_GET_INTEGER(v, 2, flags);
|
|
|
+ SQ_GET_INTEGER(v, 3, ninputs);
|
|
|
+ const SQInteger nhidden_pos = 4;
|
|
|
+ SQ_GET_INTEGER(v, 5, noutputs);
|
|
|
+ SQ_OPT_INTEGER(v, 6, ndata, 0);
|
|
|
+ SQ_OPT_INTEGER(v, 7, ntest, 0);
|
|
|
+
|
|
|
+ if(!(
|
|
|
+ ((flags & NR_FLAG_CLASSIFIER) && !(flags & NR_FLAG_REGRESSOR))
|
|
|
+ || (!(flags & NR_FLAG_CLASSIFIER) && (flags & NR_FLAG_REGRESSOR))
|
|
|
+ )
|
|
|
+ )
|
|
|
+ return sq_throwerror(v, _SC("invalid neural network type. Must be "
|
|
|
+ "CLASSIFIER or REGRESSOR"));
|
|
|
+
|
|
|
+ int layers[NR_MAX_LAYERS], num_layers=0;
|
|
|
+ layers[num_layers++] = noutputs;
|
|
|
+
|
|
|
+ /* Our NN library takes the definition of layers in the opposite
|
|
|
+ * order, swap the layers array. */
|
|
|
+ SQInteger asize = sq_getsize(v, nhidden_pos);
|
|
|
+ for(int i=asize-1; i >= 0; --i)
|
|
|
+ {
|
|
|
+ sq_pushinteger(v, i);
|
|
|
+ sq_get(v, nhidden_pos);
|
|
|
+ SQInteger nhidden;
|
|
|
+ SQRESULT rc = sq_getinteger(v, -1, &nhidden);
|
|
|
+ if(rc != SQ_OK) return sq_throwerror(v, _SC("only integers expected on hidden layers array"));
|
|
|
+ layers[num_layers++] = nhidden;
|
|
|
+ sq_poptop(v);
|
|
|
+ }
|
|
|
+
|
|
|
+ layers[num_layers++] = ninputs;
|
|
|
+ //for(int i=0; i < num_layers; ++i) printf("layers %d : %d\n", i, layers[i]);
|
|
|
+
|
|
|
+ NRTypeObject *self = createNRTypeObject(flags, layers, num_layers, ndata, ntest);
|
|
|
+
|
|
|
+ if(self){
|
|
|
+ self->flags = flags;
|
|
|
+ sq_setinstanceup(v, 1, self);
|
|
|
+ sq_setreleasehook(v, 1, sq_nn_release_hook);
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+ delete self;
|
|
|
+ return sq_throwerror(v, _SC("failed to create AnnRprop"));
|
|
|
+}
|
|
|
+
|
|
|
+#define SQ_GET_NN_INSTANCE(v, at) SQ_GET_INSTANCE_VAR(v, at, NRTypeObject, self, sq_nn_TAG)
|
|
|
+
|
|
|
+static SQRESULT sq_nn_observe(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+ SQ_OPT_INTEGER(v, 4, target, NR_INSERT_NO_TARGET);
|
|
|
+
|
|
|
+ SQInteger ilen = ANN_INPUT_UNITS(self->nn);
|
|
|
+ SQInteger olen = ANN_OUTPUT_UNITS(self->nn);
|
|
|
+ SQInteger oargs = (self->flags & NR_FLAG_CLASSIFIER) ? 1 : olen;
|
|
|
+
|
|
|
+ const SQInteger inputs_pos = 2;
|
|
|
+ const SQInteger outputs_pos = 3;
|
|
|
+
|
|
|
+ SQInteger asize_inputs = sq_getsize(v, inputs_pos);
|
|
|
+ SQInteger asize_outputs = sq_getsize(v, outputs_pos);
|
|
|
+
|
|
|
+ if((ilen != asize_inputs) || (oargs != asize_outputs))
|
|
|
+ return sq_throwerror(v, _SC( "number of arguments does not "
|
|
|
+ "match the number of " _PRINT_INT_FMT " inputs and " _PRINT_INT_FMT " outputs in the neural network"),
|
|
|
+ ilen, oargs);
|
|
|
+
|
|
|
+ const SQInteger inputs_alloc_size = sizeof(float)*ilen;
|
|
|
+ const SQInteger outputs_alloc_size = sizeof(float)*olen;
|
|
|
+
|
|
|
+ float *inputs = (float*)sq_malloc(inputs_alloc_size);
|
|
|
+ for(SQInteger i=0; i < ilen; ++i)
|
|
|
+ {
|
|
|
+ sq_pushinteger(v, i);
|
|
|
+ sq_get(v, inputs_pos);
|
|
|
+ SQFloat fnum;
|
|
|
+ SQRESULT rc = sq_getfloat(v, -1, &fnum);
|
|
|
+ if(rc != SQ_OK)
|
|
|
+ {
|
|
|
+ sq_free(inputs, inputs_alloc_size);
|
|
|
+ return sq_throwerror(v, _SC("only numbers expected on input array"));
|
|
|
+ }
|
|
|
+ inputs[i] = fnum;
|
|
|
+ sq_poptop(v);
|
|
|
+ }
|
|
|
+
|
|
|
+ float *outputs = (float*)sq_malloc(outputs_alloc_size);
|
|
|
+ for(SQInteger i=0; i < oargs; ++i)
|
|
|
+ {
|
|
|
+ sq_pushinteger(v, i);
|
|
|
+ sq_get(v, outputs_pos);
|
|
|
+ SQFloat fnum;
|
|
|
+ SQRESULT rc = sq_getfloat(v, -1, &fnum);
|
|
|
+ if(rc != SQ_OK)
|
|
|
+ {
|
|
|
+ sq_free(inputs, inputs_alloc_size);
|
|
|
+ sq_free(outputs, outputs_alloc_size);
|
|
|
+ return sq_throwerror(v, _SC("only numbers expected on output array"));
|
|
|
+ }
|
|
|
+ if (self->flags & NR_FLAG_CLASSIFIER) {
|
|
|
+ int classid = fnum;
|
|
|
+ if (classid != fnum || fnum >= olen || fnum < 0) {
|
|
|
+ sq_free(inputs, inputs_alloc_size);
|
|
|
+ sq_free(outputs, outputs_alloc_size);
|
|
|
+ return sq_throwerror(v, _SC("classifier network output must be an integer "
|
|
|
+ "in the range from 0 to outputs-1."));
|
|
|
+ }
|
|
|
+ memset(outputs,0, outputs_alloc_size);
|
|
|
+ outputs[classid] = 1;
|
|
|
+ } else {
|
|
|
+ outputs[i] = fnum;
|
|
|
+ }
|
|
|
+
|
|
|
+ sq_poptop(v);
|
|
|
+ }
|
|
|
+
|
|
|
+ NRTypeInsertData(self,inputs,outputs,target);
|
|
|
+ sq_free(inputs, inputs_alloc_size);
|
|
|
+ sq_free(outputs, outputs_alloc_size);
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static SQRESULT sq_nn_train(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+ SQ_OPT_INTEGER(v, 2, opt_max_cycles, 0);
|
|
|
+ SQ_OPT_INTEGER(v, 3, opt_max_ms, 10000);
|
|
|
+ SQ_OPT_INTEGER(v, 4, opt_flags, 0);
|
|
|
+
|
|
|
+ NRTypeObject *nr = self;
|
|
|
+
|
|
|
+ nr->training_max_cycles = opt_max_cycles;
|
|
|
+ nr->training_max_ms = opt_max_ms;
|
|
|
+ if(opt_flags & NR_FLAG_AUTO_STOP) nr->flags |= NR_FLAG_AUTO_STOP;
|
|
|
+ if(opt_flags & NR_FLAG_BACKTRACK) nr->flags |= NR_FLAG_BACKTRACK;
|
|
|
+
|
|
|
+ /* Overfitting detection compares error rate in testing/training data,
|
|
|
+ * so does not work without entries in the testing dataset. */
|
|
|
+ if (nr->flags & NR_FLAG_AUTO_STOP && nr->test.len == 0) {
|
|
|
+ return sq_throwerror(v, _SC("Can't start training with AUTOSTOP option: "
|
|
|
+ "overfitting detection requires a non zero length testing dataset"));
|
|
|
+ }
|
|
|
+
|
|
|
+ int training_iterations = 1;
|
|
|
+ float train_error = 0;
|
|
|
+ float test_error = 0;
|
|
|
+ float class_error = 0;
|
|
|
+ float past_train_error = 1.0/0.0;
|
|
|
+ float past_test_error = 1.0/0.0;
|
|
|
+ int auto_stop = nr->flags & NR_FLAG_AUTO_STOP;
|
|
|
+ int backtrack = nr->flags & NR_FLAG_BACKTRACK;
|
|
|
+
|
|
|
+ uint64_t cycles = 0;
|
|
|
+ long long start = NRMilliseconds();
|
|
|
+ long long cycle_time;
|
|
|
+ int overfitting_count = 0;
|
|
|
+ int overfitting_limit = 5;
|
|
|
+ float best_test_error = 1.0/0.0;
|
|
|
+
|
|
|
+ nr->flags &= ~NR_FLAG_TO_TRANSFER;
|
|
|
+
|
|
|
+ /* If the network is auto normalized, we need to trasnform the inputs
|
|
|
+ * in a way that's acceptable for the NN. We just find the maximum
|
|
|
+ * absolute value, and divide for it, to get a -1,1 range. There
|
|
|
+ * are more advanced transformations that are usually performed that
|
|
|
+ * could be implemented in the future.
|
|
|
+ *
|
|
|
+ * Note that we compute the normalization vectors for all the inputs
|
|
|
+ * and outputs, however if the network is a classifier, flagged with
|
|
|
+ * (NR_FLAG_CLASSIFIER), no output normalization will be done since
|
|
|
+ * the data is already in 0/1 format. */
|
|
|
+ if ((nr->flags & NR_FLAG_NORMALIZE) && nr->dataset.len) {
|
|
|
+ int ilen = ANN_INPUT_UNITS(nr->nn);
|
|
|
+ int olen = ANN_OUTPUT_UNITS(nr->nn);
|
|
|
+ float *imax = nr->inorm;
|
|
|
+ float *omax = nr->onorm;
|
|
|
+ float *inputs = nr->dataset.inputs;
|
|
|
+ float *outputs = nr->dataset.outputs;
|
|
|
+ for (int i = 0; i < ilen; i++) imax[i] = 1;
|
|
|
+ for (int i = 0; i < olen; i++) omax[i] = 1;
|
|
|
+
|
|
|
+ /* Compute the max values vectors. */
|
|
|
+ for (uint32_t j = 0; j < nr->dataset.len; j++) {
|
|
|
+ for (int i = 0; i < ilen; i++)
|
|
|
+ if (fabs(inputs[i]) > imax[i]) imax[i] = fabs(inputs[i]);
|
|
|
+ for (int i = 0; i < olen; i++)
|
|
|
+ if (fabs(outputs[i]) > omax[i]) omax[i] = fabs(outputs[i]);
|
|
|
+ inputs += ilen;
|
|
|
+ outputs += olen;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Likely we are not seeing what will really be the true input/output
|
|
|
+ * maximum value, so we multiply the maximum values found by a constant.
|
|
|
+ * However if the max is exactly "1" we assume it's a classification
|
|
|
+ * input and don't alter it. */
|
|
|
+ for (int i = 0; i < ilen; i++) if (imax[i] != 1) imax[i] *= 1.2;
|
|
|
+ for (int i = 0; i < olen; i++) if (omax[i] != 1) omax[i] *= 1.2;
|
|
|
+
|
|
|
+ /* We can normalize the dataset directly: after the training it will
|
|
|
+ * be discarded anyway. */
|
|
|
+ inputs = nr->dataset.inputs;
|
|
|
+ outputs = nr->dataset.outputs;
|
|
|
+ for (uint32_t j = 0; j < nr->dataset.len; j++) {
|
|
|
+ for (int i = 0; i < ilen; i++) inputs[i] /= nr->inorm[i];
|
|
|
+ if (!(nr->flags & NR_FLAG_CLASSIFIER))
|
|
|
+ for (int i = 0; i < olen; i++) outputs[i] /= nr->onorm[i];
|
|
|
+ inputs += ilen;
|
|
|
+ outputs += olen;
|
|
|
+ }
|
|
|
+
|
|
|
+ inputs = nr->test.inputs;
|
|
|
+ outputs = nr->test.outputs;
|
|
|
+ for (uint32_t j = 0; j < nr->test.len; j++) {
|
|
|
+ for (int i = 0; i < ilen; i++) inputs[i] /= nr->inorm[i];
|
|
|
+ if (!(nr->flags & NR_FLAG_CLASSIFIER))
|
|
|
+ for (int i = 0; i < olen; i++) outputs[i] /= nr->onorm[i];
|
|
|
+ inputs += ilen;
|
|
|
+ outputs += olen;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ AnnRprop *saved = NULL; /* Saved to recover on overfitting. */
|
|
|
+ float saved_error; /* The test error of the saved NN. */
|
|
|
+ float saved_train_error; /* The training dataset error of the saved NN */
|
|
|
+ float saved_class_error; /* The % of classification errors of saved NN */
|
|
|
+
|
|
|
+ while(1) {
|
|
|
+ long long cycle_start = NRMilliseconds();
|
|
|
+
|
|
|
+ train_error = AnnTrain(nr->nn,
|
|
|
+ nr->dataset.inputs,
|
|
|
+ nr->dataset.outputs,
|
|
|
+ 0,
|
|
|
+ training_iterations,
|
|
|
+ nr->dataset.len,
|
|
|
+ ANN_ALGO_BPROP);
|
|
|
+ cycle_time = NRMilliseconds() - cycle_start;
|
|
|
+ nr->training_total_steps += nr->dataset.len*training_iterations;
|
|
|
+
|
|
|
+ /* Evaluate the error in the case of auto training, stop it
|
|
|
+ * once we see that the error in the traning set is decreasing
|
|
|
+ * while the one in the test set is not. */
|
|
|
+ if (auto_stop) {
|
|
|
+ AnnTestError(nr->nn,
|
|
|
+ nr->test.inputs,
|
|
|
+ nr->test.outputs,
|
|
|
+ nr->test.len, &test_error, &class_error);
|
|
|
+
|
|
|
+ if (train_error < past_train_error &&
|
|
|
+ test_error > past_test_error)
|
|
|
+ {
|
|
|
+ overfitting_count++;
|
|
|
+ #ifdef NR_TRAINING_DEBUG
|
|
|
+ printf("+YCLE %lld: [%d] %f VS %f\n", (long long)cycles,
|
|
|
+ overfitting_count, train_error, test_error);
|
|
|
+ #endif
|
|
|
+ if (overfitting_count == overfitting_limit) {
|
|
|
+ nr->flags |= NR_FLAG_OF_DETECTED;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ } else if (overfitting_count > 0) {
|
|
|
+ #ifdef NR_TRAINING_DEBUG
|
|
|
+ printf("-YCLE %lld: [%d] %f VS %f\n", (long long)cycles,
|
|
|
+ overfitting_count, train_error, test_error);
|
|
|
+ #endif
|
|
|
+ overfitting_count--;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Save all the networks with a score better than the currently
|
|
|
+ * saved network. This can be a bit costly, but is safe: one
|
|
|
+ * cycle of training more and overfitting can ruin it all. */
|
|
|
+ if (backtrack && (saved == NULL || test_error < saved_error)) {
|
|
|
+ #ifdef NR_TRAINING_DEBUG
|
|
|
+ printf("SAVED! %f < %f\n", test_error, saved_error);
|
|
|
+ #endif
|
|
|
+ saved_error = test_error;
|
|
|
+ saved_train_error = train_error;
|
|
|
+ saved_class_error = class_error;
|
|
|
+ if (saved) AnnFree(saved);
|
|
|
+ saved = AnnClone(nr->nn);
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Best network found? Reset the overfitting hints counter. */
|
|
|
+ if (test_error < best_test_error) {
|
|
|
+ overfitting_count = 0;
|
|
|
+ best_test_error = test_error;
|
|
|
+ #ifdef NR_TRAINING_DEBUG
|
|
|
+ printf("BEST! %lld: <%d> %f VS %f\n", (long long)cycles,
|
|
|
+ overfitting_limit,train_error, test_error);
|
|
|
+ #endif
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Also stop if the loss is zero in both datasets. */
|
|
|
+ if (train_error < 0.000000000000001 &&
|
|
|
+ test_error < 0.000000000000001) break;
|
|
|
+ }
|
|
|
+
|
|
|
+ cycles++;
|
|
|
+ long long total_time = NRMilliseconds()-start;
|
|
|
+
|
|
|
+ /* Cycles and milliseconds stop conditions. */
|
|
|
+ if (nr->training_max_cycles && cycles == nr->training_max_cycles)
|
|
|
+ break;
|
|
|
+ if (nr->training_max_ms && total_time > (long long)nr->training_max_ms)
|
|
|
+ break;
|
|
|
+
|
|
|
+ /* If this is a long training, to do just a single training iteration
|
|
|
+ * for each cycle is not optimal: tune the number of iterations to
|
|
|
+ * at least take 100 milliseconds. */
|
|
|
+ if (total_time > 10000 && cycle_time < 100) training_iterations++;
|
|
|
+
|
|
|
+ past_train_error = train_error;
|
|
|
+ past_test_error = test_error;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* If auto stop is disabled, we still need to compute the test error
|
|
|
+ * in order to return this information to the main thread. */
|
|
|
+ if (!auto_stop) {
|
|
|
+ AnnTestError(nr->nn,
|
|
|
+ nr->test.inputs,
|
|
|
+ nr->test.outputs,
|
|
|
+ nr->test.len, &test_error, &class_error);
|
|
|
+ }
|
|
|
+
|
|
|
+ /* If both autostop and backtracking are enabled, we may have
|
|
|
+ * a better network saved! */
|
|
|
+ if (auto_stop && backtrack) {
|
|
|
+ if (saved && saved_error < test_error) {
|
|
|
+ #ifdef NR_TRAINING_DEBUG
|
|
|
+ printf("BACKTRACK: Saved network used!\n");
|
|
|
+ #endif
|
|
|
+ AnnFree(nr->nn);
|
|
|
+ nr->nn = saved;
|
|
|
+ test_error = saved_error;
|
|
|
+ train_error = saved_train_error;
|
|
|
+ class_error = saved_class_error;
|
|
|
+ } else if (saved) {
|
|
|
+ AnnFree(saved);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (nr->flags & NR_FLAG_CLASSIFIER) nr->test_class_error = class_error;
|
|
|
+ nr->dataset_error = train_error;
|
|
|
+ nr->test_error = test_error;
|
|
|
+ nr->training_total_ms += NRMilliseconds()-start;
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static SQRESULT sq_nn_run(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS_NO_TOP(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+
|
|
|
+ SQInteger asize_inputs = sq_getsize(v, 2);
|
|
|
+ SQInteger ilen = ANN_INPUT_UNITS(self->nn);
|
|
|
+ if(ilen != asize_inputs)
|
|
|
+ return sq_throwerror(v, _SC("wrong number of inputs " _PRINT_INT_FMT " for expected " _PRINT_INT_FMT), asize_inputs, ilen);
|
|
|
+
|
|
|
+ for(SQInteger i=0; i < ilen; ++i)
|
|
|
+ {
|
|
|
+ sq_pushinteger(v, i);
|
|
|
+ sq_get(v, 2);
|
|
|
+ SQFloat fnum;
|
|
|
+ SQRESULT rc = sq_getfloat(v, -1, &fnum);
|
|
|
+ if(rc != SQ_OK)
|
|
|
+ {
|
|
|
+ return sq_throwerror(v, _SC("only numbers expected on input array"));
|
|
|
+ }
|
|
|
+ if (self->flags & NR_FLAG_NORMALIZE) fnum /= self->inorm[i];
|
|
|
+ ANN_INPUT_NODE(self->nn,i) = fnum;
|
|
|
+ sq_poptop(v);
|
|
|
+ }
|
|
|
+ AnnSimulate(self->nn);
|
|
|
+
|
|
|
+ /* Output the raw net output or the class ID if the network
|
|
|
+ * is a classifier and the command invoked was NR.CLASS. */
|
|
|
+ int olen = ANN_OUTPUT_UNITS(self->nn);
|
|
|
+ sq_newarray(v, olen);
|
|
|
+ for(int j = 0; j < olen; j++) {
|
|
|
+ float output = ANN_OUTPUT_NODE(self->nn,j);
|
|
|
+ if (!(self->flags & NR_FLAG_CLASSIFIER) &&
|
|
|
+ (self->flags & NR_FLAG_NORMALIZE))
|
|
|
+ {
|
|
|
+ output *= self->onorm[j];
|
|
|
+ }
|
|
|
+ sq_pushfloat(v, output);
|
|
|
+ sq_arrayset(v, -2, j);
|
|
|
+ }
|
|
|
+
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+static SQRESULT sq_nn_classify(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS_NO_TOP(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+
|
|
|
+ if (!(self->flags & NR_FLAG_CLASSIFIER))
|
|
|
+ return sq_throwerror(v, _SC("you can't call classify with a regressor network."));
|
|
|
+
|
|
|
+ SQInteger asize_inputs = sq_getsize(v, 2);
|
|
|
+ SQInteger ilen = ANN_INPUT_UNITS(self->nn);
|
|
|
+ if(ilen != asize_inputs)
|
|
|
+ return sq_throwerror(v, _SC("wrong number of inputs %d for expected %d"), (int)asize_inputs, (int)ilen);
|
|
|
+
|
|
|
+ for(SQInteger i=0; i < ilen; ++i)
|
|
|
+ {
|
|
|
+ sq_pushinteger(v, i);
|
|
|
+ sq_get(v, 2);
|
|
|
+ SQFloat fnum;
|
|
|
+ SQRESULT rc = sq_getfloat(v, -1, &fnum);
|
|
|
+ if(rc != SQ_OK)
|
|
|
+ {
|
|
|
+ return sq_throwerror(v, _SC("only numbers expected on input array"));
|
|
|
+ }
|
|
|
+ if (self->flags & NR_FLAG_NORMALIZE) fnum /= self->inorm[i];
|
|
|
+ ANN_INPUT_NODE(self->nn,i) = fnum;
|
|
|
+ sq_poptop(v);
|
|
|
+ }
|
|
|
+ AnnSimulate(self->nn);
|
|
|
+
|
|
|
+ /* Output the raw net output or the class ID if the network
|
|
|
+ * is a classifier and the command invoked was NR.CLASS. */
|
|
|
+ int olen = ANN_OUTPUT_UNITS(self->nn);
|
|
|
+ float fmax = ANN_OUTPUT_NODE(self->nn,0);
|
|
|
+ int max_class = 0;
|
|
|
+ for(int j = 1; j < olen; j++) {
|
|
|
+ float output = ANN_OUTPUT_NODE(self->nn,j);
|
|
|
+ if (output > fmax) {
|
|
|
+ fmax = output;
|
|
|
+ max_class = j;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ sq_pushinteger(v, max_class);
|
|
|
+
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+#define ADD_T_TABLE_STR(sk, sv) \
|
|
|
+ sq_pushstring(v, sk, -1); \
|
|
|
+ sq_pushstring(v, sv, -1); \
|
|
|
+ sq_rawset(v, -3);
|
|
|
+
|
|
|
+#define ADD_T_TABLE_INT(sk, sv) \
|
|
|
+ sq_pushstring(v, sk, -1); \
|
|
|
+ sq_pushinteger(v, sv); \
|
|
|
+ sq_rawset(v, -3);
|
|
|
+
|
|
|
+#define ADD_T_TABLE_FLOAT(sk, sv) \
|
|
|
+ sq_pushstring(v, sk, -1); \
|
|
|
+ sq_pushfloat(v, sv); \
|
|
|
+ sq_rawset(v, -3);
|
|
|
+
|
|
|
+static SQRESULT sq_nn_info(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS_NO_TOP(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+
|
|
|
+ sq_newtable(v);
|
|
|
+
|
|
|
+ ADD_T_TABLE_INT("id", self->id);
|
|
|
+ ADD_T_TABLE_STR("type", (self->flags & NR_FLAG_CLASSIFIER) ? "classifier" : "regressor");
|
|
|
+ ADD_T_TABLE_INT("auto-normalization", !!(self->flags & NR_FLAG_NORMALIZE));
|
|
|
+ ADD_T_TABLE_INT("training", !!(self->flags & NR_FLAG_TRAINING));
|
|
|
+
|
|
|
+ sq_pushliteral(v, _SC("layout"));
|
|
|
+ sq_newarray(v, ANN_LAYERS(self->nn));
|
|
|
+ for (int ai=0, i = ANN_LAYERS(self->nn)-1; i >= 0; i--, ++ai) {
|
|
|
+ int units = ANN_UNITS(self->nn,i);
|
|
|
+ if (i != 0) units--; /* Don't count the bias unit. */
|
|
|
+ sq_pushinteger(v, units);
|
|
|
+ sq_arrayset(v, -2, ai);
|
|
|
+ }
|
|
|
+ sq_rawset(v, -3);
|
|
|
+
|
|
|
+ ADD_T_TABLE_INT("training-dataset-maxlen", self->dataset.maxlen);
|
|
|
+ ADD_T_TABLE_INT("training-dataset-len", self->dataset.len);
|
|
|
+ ADD_T_TABLE_INT("test-dataset-maxlen", self->test.maxlen);
|
|
|
+ ADD_T_TABLE_INT("test-dataset-len", self->test.len);
|
|
|
+ ADD_T_TABLE_INT("training-total-steps", self->training_total_steps);
|
|
|
+ ADD_T_TABLE_INT("training-total-cycles", self->dataset.len ?
|
|
|
+ (self->training_total_steps / self->dataset.len) : 0);
|
|
|
+
|
|
|
+ float tms = (float)self->training_total_ms/1000;
|
|
|
+ ADD_T_TABLE_FLOAT("training-total-seconds", tms);
|
|
|
+ ADD_T_TABLE_FLOAT("dataset-error", self->dataset_error);
|
|
|
+ ADD_T_TABLE_FLOAT("test-error", self->test_error);
|
|
|
+
|
|
|
+ if (self->flags & NR_FLAG_CLASSIFIER) {
|
|
|
+ ADD_T_TABLE_FLOAT("classification-errors-perc", self->test_class_error);
|
|
|
+ }
|
|
|
+
|
|
|
+ ADD_T_TABLE_STR("overfitting-detected", (self->flags & NR_FLAG_OF_DETECTED) ? "yes" : "no");
|
|
|
+
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+static SQRESULT sq_nn_clone(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS_NO_TOP(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+ AnnRprop *clone = AnnClone(self->nn);
|
|
|
+ if(clone)
|
|
|
+ {
|
|
|
+ sq_pushstring(v, sq_nn_TAG, -1);
|
|
|
+ if(sq_getonregistrytable(v) == SQ_ERROR) return SQ_ERROR;
|
|
|
+ sq_createinstance(v, -1);
|
|
|
+ sq_setinstanceup(v, -1, clone);
|
|
|
+ sq_setreleasehook(v, -1, sq_nn_release_hook);
|
|
|
+ }
|
|
|
+ else sq_pushnull(v);
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+#define SQ_NN_GET_SET_FLOAT(func_name) \
|
|
|
+static SQRESULT sq_nn_##func_name(HSQUIRRELVM v)\
|
|
|
+{\
|
|
|
+ SQ_FUNC_VARS(v);\
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);\
|
|
|
+ if(_top_ == 1)\
|
|
|
+ {\
|
|
|
+ sq_pushfloat(v, self->nn->func_name);\
|
|
|
+ return 1;\
|
|
|
+ }\
|
|
|
+ SQ_GET_FLOAT(v, 2, func_name);\
|
|
|
+ self->nn->func_name = func_name;\
|
|
|
+ return 0;\
|
|
|
+}
|
|
|
+
|
|
|
+SQ_NN_GET_SET_FLOAT(learn_rate);
|
|
|
+SQ_NN_GET_SET_FLOAT(rprop_nminus);
|
|
|
+SQ_NN_GET_SET_FLOAT(rprop_nplus);
|
|
|
+SQ_NN_GET_SET_FLOAT(rprop_maxupdate);
|
|
|
+SQ_NN_GET_SET_FLOAT(rprop_minupdate);
|
|
|
+
|
|
|
+static SQRESULT sq_nn_flags(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+ if(_top_ == 1)
|
|
|
+ {
|
|
|
+ sq_pushinteger(v, self->nn->flags);
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+ SQ_GET_INTEGER(v, 2, flags);
|
|
|
+ self->nn->flags = flags;
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static SQRESULT sq_nn_weights(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS_NO_TOP(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+ sq_pushfloat(v, AnnCountWeights(self->nn));
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+static SQRESULT sq_nn_weight(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+ SQ_GET_INTEGER(v, 2, layer);
|
|
|
+ SQ_GET_INTEGER(v, 3, i);
|
|
|
+ SQ_GET_INTEGER(v, 4, j);
|
|
|
+
|
|
|
+ if(layer < 0 && layer >= self->nn->layers) return sq_throwerror(v, _SC("layer out of range"));
|
|
|
+ //if(i < 0 && i >= self->layer[layer]) return sq_throwerror(v, _("layer out of range"));
|
|
|
+
|
|
|
+ float *weight = &ANN_WEIGHT(self->nn, layer, i, j);
|
|
|
+ if(_top_ == 4)
|
|
|
+ {
|
|
|
+ sq_pushfloat(v, *weight);
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+ SQ_GET_FLOAT(v, 5, new_weight);
|
|
|
+ *weight = new_weight;
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static SQRESULT sq_nn_Ann2Tcl(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS_NO_TOP(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+ Ann2Tcl(self->nn);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static SQRESULT sq_nn_Ann2Js(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS_NO_TOP(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+ Ann2Js(self->nn);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static SQRESULT sq_nn_AnnPrint(HSQUIRRELVM v)
|
|
|
+{
|
|
|
+ SQ_FUNC_VARS_NO_TOP(v);
|
|
|
+ SQ_GET_NN_INSTANCE(v, 1);
|
|
|
+ AnnPrint(self->nn);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+#define _DECL_FUNC(name,nparams,tycheck) {_SC(#name),sq_nn_##name,nparams,tycheck}
|
|
|
+static SQRegFunction sq_nn_methods[] =
|
|
|
+{
|
|
|
+ _DECL_FUNC(constructor, -5,_SC("xiiaiii")),
|
|
|
+ _DECL_FUNC(clone, 1,_SC("x")),
|
|
|
+ _DECL_FUNC(Ann2Tcl, 1,_SC("x")),
|
|
|
+ _DECL_FUNC(Ann2Js, 1,_SC("x")),
|
|
|
+ _DECL_FUNC(AnnPrint, 1,_SC("x")),
|
|
|
+ _DECL_FUNC(flags, -1,_SC("xi")),
|
|
|
+ _DECL_FUNC(learn_rate, -1,_SC("xf")),
|
|
|
+ _DECL_FUNC(rprop_nminus, -1,_SC("xf")),
|
|
|
+ _DECL_FUNC(rprop_nplus, -1,_SC("xf")),
|
|
|
+ _DECL_FUNC(rprop_maxupdate, -1,_SC("xf")),
|
|
|
+ _DECL_FUNC(rprop_minupdate, -1,_SC("xf")),
|
|
|
+ _DECL_FUNC(weights, 1,_SC("x")),
|
|
|
+ _DECL_FUNC(weight, -4,_SC("xiiif")),
|
|
|
+ _DECL_FUNC(observe, -3,_SC("xaai")),
|
|
|
+ _DECL_FUNC(train, -1,_SC("xiii")),
|
|
|
+ _DECL_FUNC(run, 2,_SC("xa")),
|
|
|
+ _DECL_FUNC(classify, 2,_SC("xa")),
|
|
|
+ _DECL_FUNC(info, 1,_SC("x")),
|
|
|
+ {0,0}
|
|
|
+};
|
|
|
+#undef _DECL_FUNC
|
|
|
+
|
|
|
+typedef struct {
|
|
|
+ const SQChar *Str;
|
|
|
+ SQInteger Val;
|
|
|
+} KeyIntType, * KeyIntPtrType;
|
|
|
+
|
|
|
+static KeyIntType sqpcre2_constants[] = {
|
|
|
+ #define MK_CONST(c) {_SC(#c), NR_##c}
|
|
|
+ #define MK_CONST_FLAG(c) {_SC(#c), NR_FLAG_##c}
|
|
|
+
|
|
|
+ MK_CONST_FLAG(NONE),
|
|
|
+ MK_CONST_FLAG(TRAINING),
|
|
|
+ MK_CONST_FLAG(REGRESSOR),
|
|
|
+ MK_CONST_FLAG(CLASSIFIER),
|
|
|
+ MK_CONST_FLAG(NORMALIZE),
|
|
|
+ MK_CONST_FLAG(AUTO_STOP),
|
|
|
+ MK_CONST_FLAG(OF_DETECTED),
|
|
|
+ MK_CONST_FLAG(BACKTRACK),
|
|
|
+ MK_CONST_FLAG(TO_PERSIST),
|
|
|
+ MK_CONST_FLAG(TO_TRANSFER),
|
|
|
+ MK_CONST(MAX_LAYERS),
|
|
|
+ MK_CONST(RDB_ENC_VER),
|
|
|
+ MK_CONST(INSERT_TRAIN),
|
|
|
+ MK_CONST(INSERT_TEST),
|
|
|
+ {0,0}
|
|
|
+};
|
|
|
+
|
|
|
+#ifdef __cplusplus
|
|
|
+extern "C" {
|
|
|
+#endif
|
|
|
+/* This defines a function that opens up your library. */
|
|
|
+SQRESULT sqext_register_nn (HSQUIRRELVM v) {
|
|
|
+ sq_pushstring(v,sq_nn_TAG,-1);
|
|
|
+ sq_newclass(v,SQFalse);
|
|
|
+ sq_settypetag(v,-1,(void*)sq_nn_TAG);
|
|
|
+ sq_insert_reg_funcs(v, sq_nn_methods);
|
|
|
+
|
|
|
+ //add constants
|
|
|
+ KeyIntPtrType KeyIntPtr;
|
|
|
+ for (KeyIntPtr = sqpcre2_constants; KeyIntPtr->Str; KeyIntPtr++) {
|
|
|
+ sq_pushstring(v, KeyIntPtr->Str, -1); //first the key
|
|
|
+ sq_pushinteger(v, KeyIntPtr->Val); //then the value
|
|
|
+ sq_newslot(v, -3, SQFalse); //store then
|
|
|
+ }
|
|
|
+
|
|
|
+ sq_newslot(v,-3,SQTrue);
|
|
|
+
|
|
|
+ return SQ_OK;
|
|
|
+}
|
|
|
+
|
|
|
+#ifdef __cplusplus
|
|
|
+}
|
|
|
+#endif
|
|
|
+
|
|
|
+
|
mingodad