chore: import upstream snapshot with attribution

LICENSE

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2014 Andrej Karpathy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

README.wehub.md

@@ -0,0 +1,7 @@
+# WeHub 来源说明
+
+- 原始项目：`karpathy/convnetjs`
+- 原始仓库：https://github.com/karpathy/convnetjs
+- 导入方式：上游默认分支的最新快照
+- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
+- 本文件仅用于记录来源，不代表 WeHub 是原项目作者

Readme.md

@@ -0,0 +1,118 @@
+
+# ConvNetJS
+
+ConvNetJS is a Javascript implementation of Neural networks, together with nice browser-based demos. It currently supports:
+
+- Common **Neural Network modules** (fully connected layers, non-linearities)
+- Classification (SVM/Softmax) and Regression (L2) **cost functions**
+- Ability to specify and train **Convolutional Networks** that process images
+- An experimental **Reinforcement Learning** module, based on Deep Q Learning
+
+For much more information, see the main page at [convnetjs.com](http://convnetjs.com)
+
+**Note**: I am not actively maintaining ConvNetJS anymore because I simply don't have time. I think the npm repo might not work at this point.
+
+## Online Demos
+- [Convolutional Neural Network on MNIST digits](http://cs.stanford.edu/~karpathy/convnetjs/demo/mnist.html)
+- [Convolutional Neural Network on CIFAR-10](http://cs.stanford.edu/~karpathy/convnetjs/demo/cifar10.html)
+- [Toy 2D data](http://cs.stanford.edu/~karpathy/convnetjs/demo/classify2d.html)
+- [Toy 1D regression](http://cs.stanford.edu/~karpathy/convnetjs/demo/regression.html)
+- [Training an Autoencoder on MNIST digits](http://cs.stanford.edu/~karpathy/convnetjs/demo/autoencoder.html)
+- [Deep Q Learning Reinforcement Learning demo](http://cs.stanford.edu/people/karpathy/convnetjs/demo/rldemo.html)
+- [Image Regression ("Painting")](http://cs.stanford.edu/~karpathy/convnetjs/demo/image_regression.html)
+- [Comparison of SGD/Adagrad/Adadelta on MNIST](http://cs.stanford.edu/people/karpathy/convnetjs/demo/trainers.html)
+
+## Example Code
+
+Here's a minimum example of defining a **2-layer neural network** and training
+it on a single data point:
+
+```javascript
+// species a 2-layer neural network with one hidden layer of 20 neurons
+var layer_defs = [];
+// input layer declares size of input. here: 2-D data
+// ConvNetJS works on 3-Dimensional volumes (sx, sy, depth), but if you're not dealing with images
+// then the first two dimensions (sx, sy) will always be kept at size 1
+layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:2});
+// declare 20 neurons, followed by ReLU (rectified linear unit non-linearity)
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'}); 
+// declare the linear classifier on top of the previous hidden layer
+layer_defs.push({type:'softmax', num_classes:10});
+
+var net = new convnetjs.Net();
+net.makeLayers(layer_defs);
+
+// forward a random data point through the network
+var x = new convnetjs.Vol([0.3, -0.5]);
+var prob = net.forward(x); 
+
+// prob is a Vol. Vols have a field .w that stores the raw data, and .dw that stores gradients
+console.log('probability that x is class 0: ' + prob.w[0]); // prints 0.50101
+
+var trainer = new convnetjs.SGDTrainer(net, {learning_rate:0.01, l2_decay:0.001});
+trainer.train(x, 0); // train the network, specifying that x is class zero
+
+var prob2 = net.forward(x);
+console.log('probability that x is class 0: ' + prob2.w[0]);
+// now prints 0.50374, slightly higher than previous 0.50101: the networks
+// weights have been adjusted by the Trainer to give a higher probability to
+// the class we trained the network with (zero)
+```
+
+and here is a small **Convolutional Neural Network** if you wish to predict on images:
+
+```javascript
+var layer_defs = [];
+layer_defs.push({type:'input', out_sx:32, out_sy:32, out_depth:3}); // declare size of input
+// output Vol is of size 32x32x3 here
+layer_defs.push({type:'conv', sx:5, filters:16, stride:1, pad:2, activation:'relu'});
+// the layer will perform convolution with 16 kernels, each of size 5x5.
+// the input will be padded with 2 pixels on all sides to make the output Vol of the same size
+// output Vol will thus be 32x32x16 at this point
+layer_defs.push({type:'pool', sx:2, stride:2});
+// output Vol is of size 16x16x16 here
+layer_defs.push({type:'conv', sx:5, filters:20, stride:1, pad:2, activation:'relu'});
+// output Vol is of size 16x16x20 here
+layer_defs.push({type:'pool', sx:2, stride:2});
+// output Vol is of size 8x8x20 here
+layer_defs.push({type:'conv', sx:5, filters:20, stride:1, pad:2, activation:'relu'});
+// output Vol is of size 8x8x20 here
+layer_defs.push({type:'pool', sx:2, stride:2});
+// output Vol is of size 4x4x20 here
+layer_defs.push({type:'softmax', num_classes:10});
+// output Vol is of size 1x1x10 here
+
+net = new convnetjs.Net();
+net.makeLayers(layer_defs);
+
+// helpful utility for converting images into Vols is included
+var x = convnetjs.img_to_vol(document.getElementById('some_image'))
+var output_probabilities_vol = net.forward(x)
+```
+
+## Getting Started
+A [Getting Started](http://cs.stanford.edu/people/karpathy/convnetjs/started.html) tutorial is available on main page.
+
+The full [Documentation](http://cs.stanford.edu/people/karpathy/convnetjs/docs.html) can also be found there.
+
+See the **releases** page for this project to get the minified, compiled library, and a direct link to is also available below for convenience (but please host your own copy)
+
+- [convnet.js](http://cs.stanford.edu/people/karpathy/convnetjs/build/convnet.js)
+- [convnet-min.js](http://cs.stanford.edu/people/karpathy/convnetjs/build/convnet-min.js)
+
+## Compiling the library from src/ to build/
+If you would like to add features to the library, you will have to change the code in `src/` and then compile the library into the `build/` directory. The compilation script simply concatenates files in `src/` and then minifies the result.
+
+The compilation is done using an ant task: it compiles `build/convnet.js` by concatenating the source files in `src/` and then minifies the result into `build/convnet-min.js`. Make sure you have **ant** installed (on Ubuntu you can simply *sudo apt-get install* it), then cd into `compile/` directory and run:
+
+    $ ant -lib yuicompressor-2.4.8.jar -f build.xml
+
+The output files will be in `build/`
+## Use in Node
+The library is also available on *node.js*:
+
+1. Install it: `$ npm install convnetjs`
+2. Use it: `var convnetjs = require("convnetjs");`
+
+## License
+MIT

bower.json

@@ -0,0 +1,31 @@
+{
+  "name": "convnetjs",
+  "version": "0.0.0",
+  "authors": [
+    "Andrej Karpathy <andrej.karpathy@gmail.com>"
+  ],
+  "description": "Deep Learning in Javascript. Train Convolutional Neural Networks (or ordinary ones) in your browser.",
+  "main": "build/convnet.js",
+  "moduleType": [
+    "amd",
+    "es6",
+    "globals",
+    "node",
+    "yui"
+  ],
+  "keywords": [
+    "machine",
+    "learning",
+    "AI",
+    "convnet"
+  ],
+  "license": "MIT",
+  "homepage": "http://cs.stanford.edu/people/karpathy/convnetjs/",
+  "ignore": [
+    "**/.*",
+    "node_modules",
+    "bower_components",
+    "test",
+    "tests"
+  ]
+}

build/deepqlearn.js

@@ -0,0 +1,292 @@
+var deepqlearn = deepqlearn || { REVISION: 'ALPHA' };
+
+(function(global) {
+  "use strict";
+  
+  // An agent is in state0 and does action0
+  // environment then assigns reward0 and provides new state, state1
+  // Experience nodes store all this information, which is used in the
+  // Q-learning update step
+  var Experience = function(state0, action0, reward0, state1) {
+    this.state0 = state0;
+    this.action0 = action0;
+    this.reward0 = reward0;
+    this.state1 = state1;
+  }
+
+  // A Brain object does all the magic.
+  // over time it receives some inputs and some rewards
+  // and its job is to set the outputs to maximize the expected reward
+  var Brain = function(num_states, num_actions, opt) {
+    var opt = opt || {};
+    // in number of time steps, of temporal memory
+    // the ACTUAL input to the net will be (x,a) temporal_window times, and followed by current x
+    // so to have no information from previous time step going into value function, set to 0.
+    this.temporal_window = typeof opt.temporal_window !== 'undefined' ? opt.temporal_window : 1; 
+    // size of experience replay memory
+    this.experience_size = typeof opt.experience_size !== 'undefined' ? opt.experience_size : 30000;
+    // number of examples in experience replay memory before we begin learning
+    this.start_learn_threshold = typeof opt.start_learn_threshold !== 'undefined'? opt.start_learn_threshold : Math.floor(Math.min(this.experience_size*0.1, 1000)); 
+    // gamma is a crucial parameter that controls how much plan-ahead the agent does. In [0,1]
+    this.gamma = typeof opt.gamma !== 'undefined' ? opt.gamma : 0.8;
+    
+    // number of steps we will learn for
+    this.learning_steps_total = typeof opt.learning_steps_total !== 'undefined' ? opt.learning_steps_total : 100000;
+    // how many steps of the above to perform only random actions (in the beginning)?
+    this.learning_steps_burnin = typeof opt.learning_steps_burnin !== 'undefined' ? opt.learning_steps_burnin : 3000;
+    // what epsilon value do we bottom out on? 0.0 => purely deterministic policy at end
+    this.epsilon_min = typeof opt.epsilon_min !== 'undefined' ? opt.epsilon_min : 0.05;
+    // what epsilon to use at test time? (i.e. when learning is disabled)
+    this.epsilon_test_time = typeof opt.epsilon_test_time !== 'undefined' ? opt.epsilon_test_time : 0.01;
+    
+    // advanced feature. Sometimes a random action should be biased towards some values
+    // for example in flappy bird, we may want to choose to not flap more often
+    if(typeof opt.random_action_distribution !== 'undefined') {
+      // this better sum to 1 by the way, and be of length this.num_actions
+      this.random_action_distribution = opt.random_action_distribution;
+      if(this.random_action_distribution.length !== num_actions) {
+        console.log('TROUBLE. random_action_distribution should be same length as num_actions.');
+      }
+      var a = this.random_action_distribution;
+      var s = 0.0; for(var k=0;k<a.length;k++) { s+= a[k]; }
+      if(Math.abs(s-1.0)>0.0001) { console.log('TROUBLE. random_action_distribution should sum to 1!'); }
+    } else {
+      this.random_action_distribution = [];
+    }
+    
+    // states that go into neural net to predict optimal action look as
+    // x0,a0,x1,a1,x2,a2,...xt
+    // this variable controls the size of that temporal window. Actions are
+    // encoded as 1-of-k hot vectors
+    this.net_inputs = num_states * this.temporal_window + num_actions * this.temporal_window + num_states;
+    this.num_states = num_states;
+    this.num_actions = num_actions;
+    this.window_size = Math.max(this.temporal_window, 2); // must be at least 2, but if we want more context even more
+    this.state_window = new Array(this.window_size);
+    this.action_window = new Array(this.window_size);
+    this.reward_window = new Array(this.window_size);
+    this.net_window = new Array(this.window_size);
+    
+    // create [state -> value of all possible actions] modeling net for the value function
+    var layer_defs = [];
+    if(typeof opt.layer_defs !== 'undefined') {
+      // this is an advanced usage feature, because size of the input to the network, and number of
+      // actions must check out. This is not very pretty Object Oriented programming but I can't see
+      // a way out of it :(
+      layer_defs = opt.layer_defs;
+      if(layer_defs.length < 2) { console.log('TROUBLE! must have at least 2 layers'); }
+      if(layer_defs[0].type !== 'input') { console.log('TROUBLE! first layer must be input layer!'); }
+      if(layer_defs[layer_defs.length-1].type !== 'regression') { console.log('TROUBLE! last layer must be input regression!'); }
+      if(layer_defs[0].out_depth * layer_defs[0].out_sx * layer_defs[0].out_sy !== this.net_inputs) {
+        console.log('TROUBLE! Number of inputs must be num_states * temporal_window + num_actions * temporal_window + num_states!');
+      }
+      if(layer_defs[layer_defs.length-1].num_neurons !== this.num_actions) {
+        console.log('TROUBLE! Number of regression neurons should be num_actions!');
+      }
+    } else {
+      // create a very simple neural net by default
+      layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:this.net_inputs});
+      if(typeof opt.hidden_layer_sizes !== 'undefined') {
+        // allow user to specify this via the option, for convenience
+        var hl = opt.hidden_layer_sizes;
+        for(var k=0;k<hl.length;k++) {
+          layer_defs.push({type:'fc', num_neurons:hl[k], activation:'relu'}); // relu by default
+        }
+      }
+      layer_defs.push({type:'regression', num_neurons:num_actions}); // value function output
+    }
+    this.value_net = new convnetjs.Net();
+    this.value_net.makeLayers(layer_defs);
+    
+    // and finally we need a Temporal Difference Learning trainer!
+    var tdtrainer_options = {learning_rate:0.01, momentum:0.0, batch_size:64, l2_decay:0.01};
+    if(typeof opt.tdtrainer_options !== 'undefined') {
+      tdtrainer_options = opt.tdtrainer_options; // allow user to overwrite this
+    }
+    this.tdtrainer = new convnetjs.SGDTrainer(this.value_net, tdtrainer_options);
+    
+    // experience replay
+    this.experience = [];
+    
+    // various housekeeping variables
+    this.age = 0; // incremented every backward()
+    this.forward_passes = 0; // incremented every forward()
+    this.epsilon = 1.0; // controls exploration exploitation tradeoff. Should be annealed over time
+    this.latest_reward = 0;
+    this.last_input_array = [];
+    this.average_reward_window = new cnnutil.Window(1000, 10);
+    this.average_loss_window = new cnnutil.Window(1000, 10);
+    this.learning = true;
+  }
+  Brain.prototype = {
+    random_action: function() {
+      // a bit of a helper function. It returns a random action
+      // we are abstracting this away because in future we may want to 
+      // do more sophisticated things. For example some actions could be more
+      // or less likely at "rest"/default state.
+      if(this.random_action_distribution.length === 0) {
+        return convnetjs.randi(0, this.num_actions);
+      } else {
+        // okay, lets do some fancier sampling:
+        var p = convnetjs.randf(0, 1.0);
+        var cumprob = 0.0;
+        for(var k=0;k<this.num_actions;k++) {
+          cumprob += this.random_action_distribution[k];
+          if(p < cumprob) { return k; }
+        }
+      }
+    },
+    policy: function(s) {
+      // compute the value of doing any action in this state
+      // and return the argmax action and its value
+      var svol = new convnetjs.Vol(1, 1, this.net_inputs);
+      svol.w = s;
+      var action_values = this.value_net.forward(svol);
+      var maxk = 0; 
+      var maxval = action_values.w[0];
+      for(var k=1;k<this.num_actions;k++) {
+        if(action_values.w[k] > maxval) { maxk = k; maxval = action_values.w[k]; }
+      }
+      return {action:maxk, value:maxval};
+    },
+    getNetInput: function(xt) {
+      // return s = (x,a,x,a,x,a,xt) state vector. 
+      // It's a concatenation of last window_size (x,a) pairs and current state x
+      var w = [];
+      w = w.concat(xt); // start with current state
+      // and now go backwards and append states and actions from history temporal_window times
+      var n = this.window_size; 
+      for(var k=0;k<this.temporal_window;k++) {
+        // state
+        w = w.concat(this.state_window[n-1-k]);
+        // action, encoded as 1-of-k indicator vector. We scale it up a bit because
+        // we dont want weight regularization to undervalue this information, as it only exists once
+        var action1ofk = new Array(this.num_actions);
+        for(var q=0;q<this.num_actions;q++) action1ofk[q] = 0.0;
+        action1ofk[this.action_window[n-1-k]] = 1.0*this.num_states;
+        w = w.concat(action1ofk);
+      }
+      return w;
+    },
+    forward: function(input_array) {
+      // compute forward (behavior) pass given the input neuron signals from body
+      this.forward_passes += 1;
+      this.last_input_array = input_array; // back this up
+      
+      // create network input
+      var action;
+      if(this.forward_passes > this.temporal_window) {
+        // we have enough to actually do something reasonable
+        var net_input = this.getNetInput(input_array);
+        if(this.learning) {
+          // compute epsilon for the epsilon-greedy policy
+          this.epsilon = Math.min(1.0, Math.max(this.epsilon_min, 1.0-(this.age - this.learning_steps_burnin)/(this.learning_steps_total - this.learning_steps_burnin))); 
+        } else {
+          this.epsilon = this.epsilon_test_time; // use test-time value
+        }
+        var rf = convnetjs.randf(0,1);
+        if(rf < this.epsilon) {
+          // choose a random action with epsilon probability
+          action = this.random_action();
+        } else {
+          // otherwise use our policy to make decision
+          var maxact = this.policy(net_input);
+          action = maxact.action;
+       }
+      } else {
+        // pathological case that happens first few iterations 
+        // before we accumulate window_size inputs
+        var net_input = [];
+        action = this.random_action();
+      }
+      
+      // remember the state and action we took for backward pass
+      this.net_window.shift();
+      this.net_window.push(net_input);
+      this.state_window.shift(); 
+      this.state_window.push(input_array);
+      this.action_window.shift(); 
+      this.action_window.push(action);
+      
+      return action;
+    },
+    backward: function(reward) {
+      this.latest_reward = reward;
+      this.average_reward_window.add(reward);
+      this.reward_window.shift();
+      this.reward_window.push(reward);
+      
+      if(!this.learning) { return; } 
+      
+      // various book-keeping
+      this.age += 1;
+      
+      // it is time t+1 and we have to store (s_t, a_t, r_t, s_{t+1}) as new experience
+      // (given that an appropriate number of state measurements already exist, of course)
+      if(this.forward_passes > this.temporal_window + 1) {
+        var e = new Experience();
+        var n = this.window_size;
+        e.state0 = this.net_window[n-2];
+        e.action0 = this.action_window[n-2];
+        e.reward0 = this.reward_window[n-2];
+        e.state1 = this.net_window[n-1];
+        if(this.experience.length < this.experience_size) {
+          this.experience.push(e);
+        } else {
+          // replace. finite memory!
+          var ri = convnetjs.randi(0, this.experience_size);
+          this.experience[ri] = e;
+        }
+      }
+      
+      // learn based on experience, once we have some samples to go on
+      // this is where the magic happens...
+      if(this.experience.length > this.start_learn_threshold) {
+        var avcost = 0.0;
+        for(var k=0;k < this.tdtrainer.batch_size;k++) {
+          var re = convnetjs.randi(0, this.experience.length);
+          var e = this.experience[re];
+          var x = new convnetjs.Vol(1, 1, this.net_inputs);
+          x.w = e.state0;
+          var maxact = this.policy(e.state1);
+          var r = e.reward0 + this.gamma * maxact.value;
+          var ystruct = {dim: e.action0, val: r};
+          var loss = this.tdtrainer.train(x, ystruct);
+          avcost += loss.loss;
+        }
+        avcost = avcost/this.tdtrainer.batch_size;
+        this.average_loss_window.add(avcost);
+      }
+    },
+    visSelf: function(elt) {
+      elt.innerHTML = ''; // erase elt first
+      
+      // elt is a DOM element that this function fills with brain-related information
+      var brainvis = document.createElement('div');
+      
+      // basic information
+      var desc = document.createElement('div');
+      var t = '';
+      t += 'experience replay size: ' + this.experience.length + '<br>';
+      t += 'exploration epsilon: ' + this.epsilon + '<br>';
+      t += 'age: ' + this.age + '<br>';
+      t += 'average Q-learning loss: ' + this.average_loss_window.get_average() + '<br />';
+      t += 'smooth-ish reward: ' + this.average_reward_window.get_average() + '<br />';
+      desc.innerHTML = t;
+      brainvis.appendChild(desc);
+      
+      elt.appendChild(brainvis);
+    }
+  }
+  
+  global.Brain = Brain;
+})(deepqlearn);
+
+(function(lib) {
+  "use strict";
+  if (typeof module === "undefined" || typeof module.exports === "undefined") {
+    window.deepqlearn = lib; // in ordinary browser attach library to window
+  } else {
+    module.exports = lib; // in nodejs
+  }
+})(deepqlearn);

build/util.js

@@ -0,0 +1,64 @@
+
+// contains various utility functions 
+var cnnutil = (function(exports){
+
+  // a window stores _size_ number of values
+  // and returns averages. Useful for keeping running
+  // track of validation or training accuracy during SGD
+  var Window = function(size, minsize) {
+    this.v = [];
+    this.size = typeof(size)==='undefined' ? 100 : size;
+    this.minsize = typeof(minsize)==='undefined' ? 20 : minsize;
+    this.sum = 0;
+  }
+  Window.prototype = {
+    add: function(x) {
+      this.v.push(x);
+      this.sum += x;
+      if(this.v.length>this.size) {
+        var xold = this.v.shift();
+        this.sum -= xold;
+      }
+    },
+    get_average: function() {
+      if(this.v.length < this.minsize) return -1;
+      else return this.sum/this.v.length;
+    },
+    reset: function(x) {
+      this.v = [];
+      this.sum = 0;
+    }
+  }
+
+  // returns min, max and indeces of an array
+  var maxmin = function(w) {
+    if(w.length === 0) { return {}; } // ... ;s
+
+    var maxv = w[0];
+    var minv = w[0];
+    var maxi = 0;
+    var mini = 0;
+    for(var i=1;i<w.length;i++) {
+      if(w[i] > maxv) { maxv = w[i]; maxi = i; } 
+      if(w[i] < minv) { minv = w[i]; mini = i; } 
+    }
+    return {maxi: maxi, maxv: maxv, mini: mini, minv: minv, dv:maxv-minv};
+  }
+
+  // returns string representation of float
+  // but truncated to length of d digits
+  var f2t = function(x, d) {
+    if(typeof(d)==='undefined') { var d = 5; }
+    var dd = 1.0 * Math.pow(10, d);
+    return '' + Math.floor(x*dd)/dd;
+  }
+
+  exports = exports || {};
+  exports.Window = Window;
+  exports.maxmin = maxmin;
+  exports.f2t = f2t;
+  return exports;
+
+})(typeof module != 'undefined' && module.exports);  // add exports to module.exports if in node.js
+
+

build/vis.js

@@ -0,0 +1,195 @@
+
+// contains various utility functions 
+var cnnvis = (function(exports){
+
+  // can be used to graph loss, or accuract over time
+  var Graph = function(options) {
+    var options = options || {};
+    this.step_horizon = options.step_horizon || 1000;
+    
+    this.pts = [];
+    
+    this.maxy = -9999;
+    this.miny = 9999;
+  }
+
+  Graph.prototype = {
+    // canv is the canvas we wish to update with this new datapoint
+    add: function(step, y) {
+      var time = new Date().getTime(); // in ms
+      if(y>this.maxy*0.99) this.maxy = y*1.05;
+      if(y<this.miny*1.01) this.miny = y*0.95;
+
+      this.pts.push({step: step, time: time, y: y});
+      if(step > this.step_horizon) this.step_horizon *= 2;
+    },
+    // elt is a canvas we wish to draw into
+    drawSelf: function(canv) {
+      
+      var pad = 25;
+      var H = canv.height;
+      var W = canv.width;
+      var ctx = canv.getContext('2d');
+
+      ctx.clearRect(0, 0, W, H);
+      ctx.font="10px Georgia";
+
+      var f2t = function(x) {
+        var dd = 1.0 * Math.pow(10, 2);
+        return '' + Math.floor(x*dd)/dd;
+      }
+
+      // draw guidelines and values
+      ctx.strokeStyle = "#999";
+      ctx.beginPath();
+      var ng = 10;
+      for(var i=0;i<=ng;i++) {
+        var xpos = i/ng*(W-2*pad)+pad;
+        ctx.moveTo(xpos, pad);
+        ctx.lineTo(xpos, H-pad);
+        ctx.fillText(f2t(i/ng*this.step_horizon/1000)+'k',xpos,H-pad+14);
+      }
+      for(var i=0;i<=ng;i++) {
+        var ypos = i/ng*(H-2*pad)+pad;
+        ctx.moveTo(pad, ypos);
+        ctx.lineTo(W-pad, ypos);
+        ctx.fillText(f2t((ng-i)/ng*(this.maxy-this.miny) + this.miny), 0, ypos);
+      }
+      ctx.stroke();
+
+      var N = this.pts.length;
+      if(N<2) return;
+
+      // draw the actual curve
+      var t = function(x, y, s) {
+        var tx = x / s.step_horizon * (W-pad*2) + pad;
+        var ty = H - ((y-s.miny) / (s.maxy-s.miny) * (H-pad*2) + pad);
+        return {tx:tx, ty:ty}
+      }
+
+      ctx.strokeStyle = "red";
+      ctx.beginPath()
+      for(var i=0;i<N;i++) {
+        // draw line from i-1 to i
+        var p = this.pts[i];
+        var pt = t(p.step, p.y, this);
+        if(i===0) ctx.moveTo(pt.tx, pt.ty);
+        else ctx.lineTo(pt.tx, pt.ty);
+      }
+      ctx.stroke();
+    }
+  }
+
+  // same as graph but draws multiple lines. For now I'm lazy and duplicating
+  // the code, but in future I will merge these two more nicely.
+  var MultiGraph = function(legend, options) {
+    var options = options || {};
+    this.step_horizon = options.step_horizon || 1000;
+
+    if(typeof options.maxy !== 'undefined') this.maxy_forced = options.maxy;
+    if(typeof options.miny !== 'undefined') this.miny_forced = options.miny;
+
+    this.pts = [];
+    
+    this.maxy = -9999;
+    this.miny = 9999;
+    this.numlines = 0;
+
+    this.numlines = legend.length;
+    this.legend = legend;
+    this.styles = ["red", "blue", "green", "black", "magenta", "cyan", "purple", "aqua", "olive", "lime", "navy"];
+    // 17 basic colors: aqua, black, blue, fuchsia, gray, green, lime, maroon, navy, olive, orange, purple, red, silver, teal, white, and yellow
+  }
+
+  MultiGraph.prototype = {
+    // canv is the canvas we wish to update with this new datapoint
+    add: function(step, yl) {
+      var time = new Date().getTime(); // in ms
+      var n = yl.length;
+      for(var k=0;k<n;k++) {
+        var y = yl[k];
+        if(y>this.maxy*0.99) this.maxy = y*1.05;
+        if(y<this.miny*1.01) this.miny = y*0.95;
+      }
+
+      if(typeof this.maxy_forced !== 'undefined') this.maxy = this.maxy_forced;
+      if(typeof this.miny_forced !== 'undefined') this.miny = this.miny_forced;
+
+      this.pts.push({step: step, time: time, yl: yl});
+      if(step > this.step_horizon) this.step_horizon *= 2;
+    },
+    // elt is a canvas we wish to draw into
+    drawSelf: function(canv) {
+      
+      var pad = 25;
+      var H = canv.height;
+      var W = canv.width;
+      var ctx = canv.getContext('2d');
+
+      ctx.clearRect(0, 0, W, H);
+      ctx.font="10px Georgia";
+
+      var f2t = function(x) {
+        var dd = 1.0 * Math.pow(10, 2);
+        return '' + Math.floor(x*dd)/dd;
+      }
+
+      // draw guidelines and values
+      ctx.strokeStyle = "#999";
+      ctx.beginPath();
+      var ng = 10;
+      for(var i=0;i<=ng;i++) {
+        var xpos = i/ng*(W-2*pad)+pad;
+        ctx.moveTo(xpos, pad);
+        ctx.lineTo(xpos, H-pad);
+        ctx.fillText(f2t(i/ng*this.step_horizon/1000)+'k',xpos,H-pad+14);
+      }
+      for(var i=0;i<=ng;i++) {
+        var ypos = i/ng*(H-2*pad)+pad;
+        ctx.moveTo(pad, ypos);
+        ctx.lineTo(W-pad, ypos);
+        ctx.fillText(f2t((ng-i)/ng*(this.maxy-this.miny) + this.miny), 0, ypos);
+      }
+      ctx.stroke();
+
+      var N = this.pts.length;
+      if(N<2) return;
+
+      // draw legend
+      for(var k=0;k<this.numlines;k++) {
+        ctx.fillStyle = this.styles[k % this.styles.length];
+        ctx.fillText(this.legend[k], W-pad-100, pad+20+k*16);
+      }
+      ctx.fillStyle = "black";
+
+      // draw the actual curve
+      var t = function(x, y, s) {
+        var tx = x / s.step_horizon * (W-pad*2) + pad;
+        var ty = H - ((y-s.miny) / (s.maxy-s.miny) * (H-pad*2) + pad);
+        return {tx:tx, ty:ty}
+      }
+      for(var k=0;k<this.numlines;k++) {
+
+        ctx.strokeStyle = this.styles[k % this.styles.length];
+        ctx.beginPath()
+        for(var i=0;i<N;i++) {
+          // draw line from i-1 to i
+          var p = this.pts[i];
+          var pt = t(p.step, p.yl[k], this);
+          if(i===0) ctx.moveTo(pt.tx, pt.ty);
+          else ctx.lineTo(pt.tx, pt.ty);
+        }
+        ctx.stroke();
+      }
+
+    }
+  }
+
+  exports = exports || {};
+  exports.Graph = Graph;
+  exports.MultiGraph = MultiGraph;
+  return exports;
+
+})(typeof module != 'undefined' && module.exports);  // add exports to module.exports if in node.js
+
+

compile/build.xml

@@ -0,0 +1,34 @@
+<project name="Build" default="compress">
+  <target name="concatenate">
+    <concat destfile="../build/convnet.js" encoding="UTF-8" outputencoding="UTF-8" fixlastline="true">
+      <filelist id="filelist" dir="../src">
+        <file name="convnet_init.js"/>
+        <file name="convnet_util.js" />
+        <file name="convnet_vol.js" />
+        <file name="convnet_vol_util.js" />
+        <file name="convnet_layers_dotproducts.js" />
+        <file name="convnet_layers_pool.js" />
+        <file name="convnet_layers_input.js" />
+        <file name="convnet_layers_loss.js" />
+        <file name="convnet_layers_nonlinearities.js" />
+        <file name="convnet_layers_dropout.js" />
+        <file name="convnet_layers_normalization.js" />
+        <file name="convnet_net.js" />
+        <file name="convnet_trainers.js" />
+        <file name="convnet_magicnet.js" />
+        <file name="convnet_export.js" />
+      </filelist>
+    </concat>
+  </target>
+  <target name="compress" depends="concatenate" description="Minify convnet.js to convnet-min.js">
+    <apply executable="java" parallel="false">
+      <filelist dir="../build" files="convnet.js" />
+      <arg line="-jar" />
+      <arg path="yuicompressor-2.4.8.jar" />
+      <srcfile />
+      <arg line="-o" />
+      <mapper type="glob" from="*.js" to="../build/*-min.js" />
+      <targetfile />
+    </apply>
+  </target>
+</project>

demo/autoencoder.html

@@ -0,0 +1,85 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+  <title>ConvNetJS MNIST demo</title>
+  <meta name="description" content="">
+  <meta name="author" content="">
+  <link rel="stylesheet" href="css/style.css">
+</head>
+
+<script src="js/jquery-1.8.3.min.js"></script>
+<script src="../build/vis.js"></script>
+<script src="../build/util.js"></script>
+<script src="../build/convnet.js"></script>
+
+
+<script src="mnist/mnist_labels.js"></script>
+<script src="js/autoencoder.js"></script>
+
+<body>
+  <div id="wrap">
+  <h2 style="text-align: center;"><a href="http://cs.stanford.edu/people/karpathy/convnetjs/">ConvNetJS</a> Denoising Autoencoder demo</h2>
+  <h1>Description</h1>
+  <p>
+    All the other demos are examples of Supervised Learning, so in this demo I wanted to show an example of Unsupervised Learning. We are going to train an autoencoder on MNIST digits.
+  </p>
+  <p>
+    An autoencoder is a regression task where the network is asked to predict its input (in other words, model the identity function). Sounds simple enough, except the network has a tight bottleneck of a few neurons in the middle (in the default example only two!), forcing it to create effective representations that compress the input into a low-dimensional code that can be used by the decoder to reproduce the original input.
+  </p>
+  <p>Report questions/bugs/suggestions to <a href="https://twitter.com/karpathy">@karpathy</a>.
+  </p>
+  <h1>Training Stats</h1>
+  <div class="divsec" style="270px;">
+    <div class="secpart">
+      Current image: <img id="input_image" src=""></img><input id="buttontp" type="submit" value="pause" onclick="toggle_pause();"/>
+      <div id="trainstats"></div>
+
+      <div id="controls">
+        Learning rate: <input name="lri" type="text" maxlength="20" id="lr_input"/>
+        <input id="buttonlr" type="submit" value="change" onclick="change_lr();"/>
+        <br />
+      </div>
+
+      <input id="buttondj" type="submit" value="save network snapshot as JSON" onclick="dump_json();"/><br />
+      <input id="buttonlfj" type="submit" value="init network from JSON snapshot" onclick="load_from_json();"/><br />
+      <textarea id="dumpjson"></textarea>
+    </div>
+    <div class="secpart">
+      
+      <div>
+        Loss:<br />
+        <canvas id="lossgraph">
+        </canvas>
+        <br />
+        <input id="buttoncg" type="submit" value="clear graph" onclick="clear_graph();"/>
+      </div>
+
+    </div>
+    <div>
+      <textarea id="newnet" style="width:100%; height:100px;"></textarea><br />
+      <input id="buttonnn" type="submit" value="reload" onclick="change_net();" style="width:200px; height: 40px;"/>
+    </div>
+
+    <div style="margin-top: 1px solid #000; margin-top:20px;">
+      <div id="layer_ixes"></div>
+      <input id="buttoncycle" type="submit" value="cycle through visualized neurons at selected layer (if more than 2)" onclick="cycle();" style="height: 40px;"/>
+      <div id="cyclestatus"></div>
+      <div style="text-align:center;"><canvas id="embedding" width="500" height="500"></canvas></div>
+    </div>
+
+    <div style="clear:both;"></div>
+  </div>
+
+  <div class="divsec">
+  <h1>Network Visualization</h1>
+    <div id="visnet"></div>
+  </div>
+
+  </div>  
+</body>
+</html>
+
+
+

demo/automatic.html

@@ -0,0 +1,183 @@
+<!doctype html>
+<html lang="en">
+ <head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+  <title>ConvNetJS Automatic</title>
+  <meta name="description" content="">
+  <meta name="author" content="">
+  <link href='http://fonts.googleapis.com/css?family=Lato:300,400,900' rel='stylesheet' type='text/css'>
+
+  <script src="../build/convnet.js"></script>
+  <script src="../build/util.js"></script>
+  <script src="../build/vis.js"></script>
+  <script src="js/jquery-1.8.3.min.js"></script>
+  <script src="js/jquery.csv-0.71.min.js"></script>
+  
+  <!-- JS syntax highlighting -->
+  <script type="text/javascript" src="../syntaxhighlighter_3.0.83/scripts/shCore.js"></script>
+  <script type="text/javascript" src="../syntaxhighlighter_3.0.83/scripts/shBrushJScript.js"></script>
+  <link type="text/css" rel="stylesheet" href="../syntaxhighlighter_3.0.83/styles/shCoreDefault.css"/>
+  <script type="text/javascript">SyntaxHighlighter.all();</script>
+
+  <link rel="stylesheet" href="css/automatic.css">
+  
+  <script src="js/automatic.js"></script>
+ </head>
+ <body onload="start();">
+  <div id="wrap">
+    <h2>ConvNetJS Automatic Prediction Demo</h2>
+    <h1>Introduction</h1>
+    <p>
+    This demo illustrates the usage of ConvNetJS' <b>MagicNet</b> class, which performs fully automatic prediction given your arbitrary data. Internally, the MagicNet tries out many different types of networks, performs cross-validations of network hyper-parameters across folds of your data, and creates a final classifier by model averaging the best architectures. The API for MagicNet looks as follows:
+    </p>
+
+<pre class="brush: js; toolbar: false;">
+var opts = {}; // options struct
+opts.train_ratio = 0.7;
+opts.num_folds = 10; // number of folds to eval per candidate
+opts.num_candidates = 10; // number of candidates to eval in parallel
+opts.num_epochs = 50; // epochs to make through data per fold
+// below, train_data is a list of input Vols and train_labels is a 
+// list of integer correct labels (in 0...K).
+var magicNet = new convnetjs.MagicNet(train_data, train_labels, opts);
+magicNet.onFinishBatch(finishedBatch); // example of setting callback for events
+
+// start training magicNet. Every step() call all candidates train on one example
+setInterval(function(){ magicNet.step() }, 0);
+
+// once at least one batch of candidates is evaluated on all folds we can do prediction!
+function finishedBatch() {
+  // prediction example. xout is Vol of scores
+  // there is also predict_soft(), which returns the full score volume for all labels
+  var predicted_label = magicNet.predict(some_test_vol);
+}
+</pre>
+
+    <h1>Your data</h1>
+    <p>
+      Currently made input data assumptions:
+    </p>
+    <ul>
+      <li>Provide data as CSV (comma-separated) values. Leave out any header rows.</li>
+      <li>Every row is a data point.</li>
+      <li>No missing values.</li>
+      <li>Last column is the class (only classification is currently supported).</li>
+    </ul>
+    <p>
+      The text area is pre-filled with a Car Quality Evaluation dataset to show you example input, but there are a few buttons that load some example datasets (more details on these: <a href="https://archive.ics.uci.edu/ml/datasets/Iris">Iris data</a>, <a href="https://archive.ics.uci.edu/ml/datasets/Car+Evaluation">Car Eval data</a>, <a href="https://archive.ics.uci.edu/ml/datasets/Yeast">Yeast Data</a>). A nice place to find more datasets are <a href="https://archive.ics.uci.edu/ml/datasets.html?format=mat&task=cla&att=&area=&numAtt=&numIns=&type=&sort=dateDown&view=table">UCI Repository</a> or <a href="http://mldata.org/repository/data/by_views/">mldata.org</a>.
+    </p>
+
+    <div style="text-align: center;">
+      <button class="clouds-flat-button" onclick="loadDB('data/iris.data.txt')" style="height: 40px; width: 260px; margin-bottom: 5px;">Fill Iris data</button>
+      <button class="clouds-flat-button" onclick="loadDB('data/car.data.txt')" style="height: 40px; width: 260px; margin-bottom: 5px;">Fill Car Evaluation data</button>
+      <button class="clouds-flat-button" onclick="loadDB('data/yeast.data.txt')" style="height: 40px; width: 260px; margin-bottom: 5px;">Fill Yeast data</button>
+    </div>
+
+    <textarea id="data-ta" style="width:100%; height:200px;">
+    </textarea>
+
+    <button class="clouds-flat-button" onclick="importTrainData()" style="width: 200px; height: 40px; margin-top: 5px;">Import Data</button>
+    (and send <input type="text" id="testsplit" maxlength="3" value="20" style="width:40px;">% of imported data randomly into Test Set below)
+
+    <div id='prepromsg'></div>
+    <div id='datamsg'></div>
+    
+    <h1>Cross-Validation</h1>
+    
+    <div>
+    <input type="text" id="labelix" maxlength="4" value="-1" style="width:40px;">
+    Index of column to classify as target. (e.g. 0 = first column, -1 = last column)
+    </div>
+
+    <div>
+    <input type="text" id="trainp" name="train_split" maxlength="3" value="70" style="width:20px;">
+    Percent of data to use for training (rest will be validation)
+    </div>
+
+    <div>
+    <input type="text" id="foldsnum" name="folds_number" maxlength="3" value="3" style="width:40px;"> Number of data folds to evaluate per candidate
+    </div>
+
+    <div>
+    <input type="text" id="candsnum" name="candidates_number" maxlength="4" value="50" style="width:40px;"> Number of candidates in a batch, to evaluate in parallel
+    </div>
+
+    <div>
+    <input type="text" id="epochsnum" name="epochs_number" maxlength="4" value="40" style="width:40px;"> Number of epochs to make over each fold
+    </div>
+
+    <div>
+    Number of Neurons in each layer: Min <input type="text" id="nnmin" maxlength="4" value="5" style="width:40px;"> Max <input type="text" id="nnmax" maxlength="4" value="30" style="width:40px;"> 
+    </div>
+
+    <!--
+    <div class="sopts">
+    Batch size sampling:<br>
+    Min: <input type="text" id="bsmin" maxlength="4" value="10" style="width:40px;"> 
+    Max: <input type="text" id="bsmax" maxlength="4" value="300" style="width:40px;"> 
+    </div>
+
+    <div class="sopts">
+    L2 weight decay (values will be raised to power of 10):<br>
+    Min: <input type="text" id="l2min" maxlength="4" value="-4" style="width:40px;"> 
+    Max: <input type="text" id="l2max" maxlength="4" value="2" style="width:40px;"> 
+    </div>
+
+    <div class="sopts">
+    Learning rate for SGD/Adagrad (values will be raised to power of 10):<br>
+    Min: <input type="text" id="lrmin" maxlength="4" value="-4" style="width:40px;"> 
+    Max: <input type="text" id="lrmax" maxlength="4" value="0" style="width:40px;"> 
+    </div>
+
+    <div class="sopts">
+    Momentum for SGD (must be between 0 and 1):<br>
+    Min: <input type="text" id="mommin" maxlength="4" value="0.9" style="width:40px;"> 
+    Max: <input type="text" id="mommax" maxlength="4" value="0.9" style="width:40px;"> 
+    </div>
+    -->
+    
+    <button class="clouds-flat-button" onclick="startCV()" style="width: 200px; height: 40px; margin-top: 5px;">Start/Restart</button>
+    <hr>
+
+    <p>Below: graph of the <b>validation accuracy</b> for current batch of candidate models as a function of the number of training points they have seen during training. Good networks will rise up as high as possible and stay there. The best performer is printed in detail below the graph. The graph is less wiggly if there is more data.</p>
+    <div id="foldreport"></div>
+    <div id="candsreport"></div>
+
+    <canvas id="valgraph" width="800" height="400"></canvas>
+    <div id="bestmodel"></div>
+    <div id="bestmodeloverall"></div>
+    
+    <h1>Evaluate on Test Set</h1>
+    <p>
+      Paste a test set in box below to evaluate the final test accuracy, which is based on a model-averaged ensemble of the best discovered network from the training data above. The CSV pasted below should be in the same format as the one used for training data above. The text field is pre-filled with the training data.
+    </p>
+    
+    <div>
+    <input type="text" id="ensemblenum" maxlength="3" value="10" style="width:20px;"> Number of best models to average in the ensemble network
+    </div>
+
+    <textarea id="data-te" style="width:100%; height:200px;">
+    </textarea>
+    <button class="clouds-flat-button" onclick="testEval()" style="width: 200px; height: 40px; margin-top: 5px;">Evaluate Test Accuracy</button>
+    <div id='datamsgtest'></div>
+    <br>
+    <div id="testresult"></div>
+
+    <h1>Export Best Network</h1>
+    <button class="clouds-flat-button" onclick="exportMagicNet()" style="width: 200px; height: 40px; margin-top: 5px;">Export</button><br><br>
+    <textarea id="taexport" style="width:100%; height:200px;">
+    </textarea>
+
+    Above you can export a trained MagicNet in JSON format. The exported MagicNet is simply a thin wrapper around a list of the best networks that were discovered during cross-validation and it can be loaded and used again as follows:
+<pre class="brush: js; toolbar: false;">
+var magicNet = new convnetjs.MagicNet();
+magicNet.fromJSON(json);
+magicNet.predict(some_vol); // ready to use!
+</pre>
+    
+    <br><br><br><br><br><br><br><br>
+    
+  </div>
+ </body>
+</html>

demo/cifar10.html

@@ -0,0 +1,145 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+  <title>ConvNetJS CIFAR-10 demo</title>
+  <meta name="description" content="">
+  <meta name="author" content="">
+  <link rel="stylesheet" href="css/style.css">
+
+<script src="js/jquery-1.8.3.min.js"></script>
+<script src="../build/vis.js"></script>
+<script src="../build/util.js"></script>
+<script src="../build/convnet.js"></script>
+
+<script src="js/image-helpers.js"></script>
+<script src="js/pica.js"></script>
+
+<script src="cifar10/cifar10_labels.js"></script>
+<script type="text/javascript">
+  // ------------------------
+  // BEGIN CIFAR-10 SPECIFIC STUFF
+  // ------------------------
+  var classes_txt = ['airplane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'];
+  var dataset_name = "cifar10";
+  var num_batches = 51; // 20 training batches, 1 test
+  var test_batch = 50;
+  var num_samples_per_batch = 1000;
+  var image_dimension = 32;
+  var image_channels = 3;
+  var use_validation_data = true;
+  var random_flip = true;
+  var random_position = true;
+
+  var layer_defs, net, trainer;
+  var t = "layer_defs = [];\n\
+  layer_defs.push({type:'input', out_sx:32, out_sy:32, out_depth:3});\n\
+  layer_defs.push({type:'conv', sx:5, filters:16, stride:1, pad:2, activation:'relu'});\n\
+  layer_defs.push({type:'pool', sx:2, stride:2});\n\
+  layer_defs.push({type:'conv', sx:5, filters:20, stride:1, pad:2, activation:'relu'});\n\
+  layer_defs.push({type:'pool', sx:2, stride:2});\n\
+  layer_defs.push({type:'conv', sx:5, filters:20, stride:1, pad:2, activation:'relu'});\n\
+  layer_defs.push({type:'pool', sx:2, stride:2});\n\
+  layer_defs.push({type:'softmax', num_classes:10});\n\
+  \n\
+  net = new convnetjs.Net();\n\
+  net.makeLayers(layer_defs);\n\
+  \n\
+  trainer = new convnetjs.SGDTrainer(net, {method:'adadelta', batch_size:4, l2_decay:0.0001});\n\
+  ";
+  // ------------------------
+  // END CIFAR-10 SPECIFIC STUFF
+  // ------------------------
+</script>
+<script src="js/images-demo.js"></script>
+
+</head>
+<body>
+  <div id="wrap">
+  <h2 style="text-align: center;"><a href="http://cs.stanford.edu/people/karpathy/convnetjs/">ConvNetJS</a> CIFAR-10 demo</h2>
+  <h1>Description</h1>
+  <p>
+    This demo trains a Convolutional Neural Network on the <a href="http://www.cs.toronto.edu/~kriz/cifar.html">CIFAR-10 dataset</a> in your browser, with nothing but Javascript. The state of the art on this dataset is about 90% accuracy and human performance is at about 94% (not perfect as the dataset can be a bit ambiguous). I used <a href="cifar10_parse.zip">this python script</a> to parse the <a href="http://www.cs.toronto.edu/~kriz/cifar.html">original files</a> (python version) into batches of images that can be easily loaded into page DOM with img tags.
+  </p>
+  <p>This dataset is more difficult and it takes longer to train a network. Data augmentation includes random flipping and random image shifts by up to 2px horizontally and verically.</p>
+  <p>
+    By default, in this demo we're using Adadelta which is one of per-parameter adaptive step size methods, so we don't have to worry about changing learning rates or momentum over time. However, I still included the text fields for changing these if you'd like to play around with SGD+Momentum trainer.
+  </p>
+  <p>Report questions/bugs/suggestions to <a href="https://twitter.com/karpathy">@karpathy</a>.</p>
+  <h1>Training Stats</h1>
+  <div class="divsec" style="270px;">
+    <div class="secpart">
+      <input id="buttontp" type="submit" value="pause" onclick="toggle_pause();"/>
+      <div id="trainstats"></div>
+
+      <div id="controls">
+        Learning rate: <input name="lri" type="text" maxlength="20" id="lr_input"/>
+        <input id="buttonlr" type="submit" value="change" onclick="change_lr();"/>
+        <br />
+
+        Momentum: <input name="momi" type="text" maxlength="20" id="momentum_input"/>
+        <input id="buttonmom" type="submit" value="change" onclick="change_momentum();"/>
+        <br />
+
+        Batch size: <input name="bsi" type="text" maxlength="20" id="batch_size_input"/>
+        <input id="buttonbs" type="submit" value="change" onclick="change_batch_size();"/>
+        <br />
+
+        Weight decay: <input name="wdi" type="text" maxlength="20" id="decay_input"/>
+        <input id="buttonwd" type="submit" value="change" onclick="change_decay();"/>
+      </div>
+
+      <input id="buttondj" type="submit" value="save network snapshot as JSON" onclick="dump_json();"/><br />
+      <input id="buttonlfj" type="submit" value="init network from JSON snapshot" onclick="load_from_json();"/><br />
+      <textarea id="dumpjson"></textarea>
+      <br>
+
+      <input id="buttonpre" type="submit" value="load a pretrained network (achieves ~80% accuracy)" onclick="load_pretrained();" style="height: 30px; width: 400px;"/><br />
+    </div>
+    <div class="secpart">
+      <div>
+        Loss:<br />
+        <canvas id="lossgraph">
+        </canvas>
+        <br />
+        <input id="buttoncg" type="submit" value="clear graph" onclick="clear_graph();"/>
+      </div>
+    </div>
+    <div class="secpart"> 
+      <div id="upload_box">
+        Test an image from your computer:
+        <div id="img_div">
+            <img id="preview_img"/>
+        </div>
+        <input name="image" type="file" accept="image/*" onchange="loadFile(event)">
+
+        <input type="submit" value="Test Image" onclick="testImage(document.getElementById('preview_img'))">
+      </div>
+    </div>
+    <div style="clear:both;"></div>
+  </div>
+
+  <h1>Instantiate a Network and Trainer</h1>
+  <div>
+    <textarea id="newnet" style="width:100%; height:200px;"></textarea><br />
+    <input id="buttonnn" type="submit" value="change network" onclick="change_net();" style="width:200px;height:30px;"/>
+  </div>
+
+  <div class="divsec">
+  <h1>Network Visualization</h1>
+    <div id="visnet"></div>
+  </div>
+  
+  <div class="divsec">
+  <h1>Example predictions on Test set</h1>
+    <div id="testset_acc"></div>
+    <div id="testset_vis"></div>
+  </div>
+
+  </div>  
+</body>
+</html>
+
+
+

demo/classify2d.html

@@ -0,0 +1,86 @@
+<html>
+<head>
+<title>ConvNetJS demo: Classify toy 2D data</title>
+<link href='http://fonts.googleapis.com/css?family=Cabin' rel='stylesheet' type='text/css'>
+
+<script src="js/jquery-1.8.3.min.js"></script>
+<script src="js/npgmain.js"></script>
+<script src="../build/convnet.js"></script>
+<script src="../build/util.js"></script>
+<script src="js/classify2d.js"></script>
+
+<style type="text/css">
+body {
+  font-family: Arial, "Helvetica Neue", Helvetica, sans-serif;
+  color: #333;
+  padding: 20px;
+}
+canvas { 
+    border: 1px solid #555;
+}
+#wrap {
+  width: 900px;
+  margin-right: auto;
+  margin-left: auto;
+  margin-bottom: 200px;
+}
+.pane {
+  width: 420px;
+  display: inline-block;
+  vertical-align: top;
+}
+input[type="submit"]
+{
+  margin: 3px;
+}
+
+</style>
+
+</head>
+
+<body>
+
+<div id="wrap">
+<h1><a href="http://cs.stanford.edu/people/karpathy/convnetjs">ConvnetJS</a> demo: toy 2d classification with 2-layer neural network</h1>
+
+<p>The simulation below shows a toy binary problem with a few data points of class 0 (red) and 1 (green). The network is set up as:</p>
+
+<textarea id="layerdef" style="width:100%; height:200px;">
+</textarea>
+<br>
+<input id="buttontp" type="submit" value="change network" onclick="reload();" style="width: 300px; height: 50px;"/>
+
+<p>Feel free to change this, the text area above gets eval()'d when you hit the button and the network gets reloaded. Every 10th of a second, all points are fed to the network multiple times through the trainer class to train the network. The resulting predictions of the network are then "painted" under the data points to show you the generalization.</p>
+
+<p>On the right we visualize the transformed representation of all grid points in the original space and the data, for a given layer and only for 2 neurons at a time. The number in the bracket shows the total number of neurons at that level of representation. If the number is more than 2, you will only see the two visualized but you can cycle through all of them with the cycle button.</p>
+
+<div class="panes">
+<div class="pane">
+<canvas id="NPGcanvas" width="400" height="400">Browser not supported for Canvas. Get a real browser.</canvas>
+<input id="buttondata1" type="submit" value="simple data" onclick="original_data();" style="width: 100px; height: 30px;"/>
+<input id="buttondata2" type="submit" value="circle data" onclick="circle_data();" style="width: 100px; height: 30px;"/>
+<input id="buttondata3" type="submit" value="spiral data" onclick="spiral_data();" style="width: 100px; height: 30px;"/><br>
+<input id="buttondata4" type="submit" value="random data" onclick="random_data();" style="width: 100px; height: 30px;"/><br>
+<p>
+Controls:<br>
+<b>CLICK</b>: Add red data point<br>
+<b>SHIFT+CLICK</b>: Add green data point<br>
+<b>CTRL+CLICK</b>: Remove closest data point<br>
+</p>
+</div>
+
+<div class="pane">
+  <canvas id="viscanvas" width="400" height="400">Browser not supported for Canvas. Get a real browser.</canvas>
+  <div id="cyclestatus"></div>
+<div id="layer_ixes"></div>
+<input id="buttoncycle" type="submit" value="cycle through visualized neurons at selected layer (if more than 2)" onclick="cycle();" style="height: 40px;"/>
+</div>
+
+</div>
+
+<p>Go <a href="http://cs.stanford.edu/people/karpathy/convnetjs/">back to ConvNetJS</a></p>
+
+</div>
+
+</body>
+</html>

demo/css/automatic.css

@@ -0,0 +1,82 @@
+#wrap {
+  width:800px;
+  margin-left: auto;
+  margin-right: auto;
+}
+h2 {
+  text-align: center;
+  font-size: 34px;
+  font-weight: 300;
+  margin-bottom: 50px;
+}
+h1 {
+  font-size: 26px;
+  font-weight: 400;
+  border-bottom: 1px #999 solid;
+}
+#datamsg{
+  background-color: white;
+  margin-top: 2px;
+  padding: 10px;
+}
+#prepromsg{
+  margin-top: 10px;
+  padding: 10px;
+}
+.msg{
+  padding: 2px;
+}
+body {
+  font-family: 'Lato', sans-serif;
+  color: #333;
+  font-size: 20px;
+  font-weight: 300;
+}
+input[type=text] {
+  border: 1px solid #999;
+  padding: 3px;
+  font-size: 18px;
+  color: #333;
+}
+.clouds-flat-button {
+  position: relative;
+  vertical-align: top;
+  width: 100%;
+  height: 80px;
+  padding: 0;
+  color:#454545;
+  text-align: center;
+  font-size: 16px;
+  background: #ecf0f1;
+  border: 0;
+  border-bottom: 2px solid #dadedf;
+  cursor: pointer;
+  -webkit-box-shadow: inset 0 -2px #dadedf;
+  box-shadow: inset 0 -2px #dadedf;
+}
+.clouds-flat-button:active {
+  top: 1px;
+  outline: none;
+  -webkit-box-shadow: none;
+  box-shadow: none;
+}
+#bestmodel {
+  font-size: 16px;
+  background-color: #FAFAFA;
+  padding: 10px;
+}
+#bestmodeloverall {
+  font-size: 16px;
+  background-color: #FAFAFA;
+  padding: 10px;
+  margin-top: 10px;
+}
+.syntaxhighlighter {
+  font-size: 14px !important;
+  overflow-y: hidden !important; 
+}
+.sopts {
+  box-shadow: 0px 0px 2px 0px #555;
+  margin-bottom: 5px;
+  padding: 10px;
+}

demo/css/style.css

@@ -0,0 +1,102 @@
+.layer {
+  border: 1px solid #999;
+  margin-bottom: 5px;
+  text-align: left;
+  padding: 10px;
+}
+.layer_act {
+  width: 500px;
+  float: right;
+}
+.ltconv {
+  background-color: #FDD;
+}
+.ltrelu {
+  background-color: #FDF;
+}
+.ltpool {
+  background-color: #DDF;
+}
+.ltsoftmax {
+  background-color: #FFD;
+}
+.ltfc {
+  background-color: #DFF;
+}
+.ltlrn {
+  background-color: #DFD; 
+}
+.ltdropout {
+  background-color: #AAA;  
+}
+.ltitle {
+  color: #333;
+  font-size: 18px;
+}
+.actmap {
+  margin: 1px;
+}
+#trainstats {
+  text-align: left;
+}
+.clear {
+  clear: both;
+}
+#wrap {
+  width: 800px;
+  margin-left: auto;
+  margin-right: auto;
+}
+h1 {
+  font-size: 16px;
+  color: #333;
+  background-color: #DDD;
+  border-bottom: 1px #999 solid;
+  text-align: center;
+}
+h2 {
+  text-align: center;
+}
+.secpart {
+  width: 400px;
+  float: left;
+}
+#lossgraph {
+  /*border: 1px solid #F0F;*/
+  width: 100%;
+}
+.testdiv canvas {
+  float: left;
+  width: 64px;
+}
+.testdiv {
+  background-color: rgba(255,255,255,0.65);
+  height: auto;
+  width: auto;
+  display: inline-block;
+  font-size: 12px;
+  box-shadow: 0px 0px 2px 2px #EEE;
+  margin: 5px;
+  padding: 5px;
+  color: black;
+}
+.probsdiv {
+  float: left;
+  width: 100px;
+  margin-left: 1px;
+}
+.pp {
+  margin: 1px;
+  padding: 1px;
+}
+#testset_acc {
+  /* margin-bottom: 200px; */
+}
+#testset_vis {
+  margin-bottom: 200px;
+}
+body {
+  font-family: Arial, "Helvetica Neue", Helvetica, sans-serif;
+  /* color: #333; */
+  /* padding: 20px; */
+}

demo/image_regression.html

@@ -0,0 +1,111 @@
+<html>
+<head>
+<title>ConvNetJS demo: Image Painting</title>
+<link href='http://fonts.googleapis.com/css?family=Cabin' rel='stylesheet' type='text/css'>
+<link rel="stylesheet" href="css/style.css">
+
+<link rel="stylesheet" href="css/jquery-ui.min.css">
+<script src="js/jquery-1.8.3.min.js"></script>
+<script src="js/jquery-ui.min.js"></script>
+
+<script src="../build/convnet.js"></script>
+<script src="js/image_regression.js"></script>
+
+<style type="text/css">
+canvas { 
+    border: 1px solid #555;
+    margin-top: 10px;
+}
+#wrap {
+  width: 800px;
+  margin-right: auto;
+  margin-left: auto;
+  margin-bottom: 200px;
+}
+#gallery img {
+  width: 195;
+  height: 195px;
+  display: inline-block;
+}
+</style>
+
+</head>
+
+<body>
+
+<div id="wrap">
+<h1><a href="http://cs.stanford.edu/people/karpathy/convnetjs/">ConvnetJS</a> demo: Image "Painting"</h1>
+
+<p>This demo that treats the pixels of an image as a learning problem: it takes the (x,y) position on a grid and learns to predict the color at that point using regression to (r,g,b). It's a bit like compression, since the image information is encoded in the weights of the network, but almost certainly not of practical kind :)</p>
+<p>
+  Note that the entire ConvNetJS definition is shown in textbox below and it gets eval()'d to create the network, so feel free to fiddle with the parameters and hit "reload". I found that, empirically and interestingly, deeper networks tend to work much better on this task given a fixed parameter budget.
+</p>
+
+<p>Report questions/bugs/suggestions to <a href="https://twitter.com/karpathy">@karpathy</a>.</p>
+
+<textarea id="layerdef" style="width:100%; height:250px;"></textarea>
+<br /><br  />
+<input id="buttontp" type="submit" value="reload network" onclick="reload();" style="width: 200px; height: 30px;"/>
+
+<div style="float: right;">
+Choose your own image: 
+<input id="f" type="file" />
+</div>
+
+<br>
+
+<div style="background-color: #EEE; padding: 10px; margin-top: 10px;">
+<div style="float: left; margin-left: 200px;">
+  Original Image<br>
+  <canvas id="canv_original"></canvas>
+</div>
+<div style="float: left; margin-left: 10px;">
+  Neural Network output<br>
+  <canvas id="canv_net"></canvas>
+</div>
+
+<div style="clear:both;"></div>
+</div>
+
+<br>
+<div id="report"></div>
+<br>
+
+<div id="lr">Learning rate:</div>
+<div id="slider"></div>
+<div>The learning rate should probably be decreased over time (slide left) to let the network better overfit the training data. It's nice to not have to worry about overfitting.</div>
+
+<br><br>
+<div>
+  You can upload your own image above (click Choose File), or you can click on any of the images below to load them.
+</div>
+<div id="gallery">
+  <img src="imgs/cat.jpg" class="ci" />
+  <img src="imgs/esher.png"  class="ci" />
+  <img src="imgs/pencils.png" class="ci" />
+  <img src="imgs/usa.png"  class="ci" />
+  <img src="imgs/fractal.jpg"  class="ci" />
+  <img src="imgs/dora.jpg" class="ci" />
+  <img src="imgs/rubiks.jpg" class="ci" />
+  <img src="imgs/earth.jpg" class="ci" />
+  <img src="imgs/fox.png" class="ci" />
+  <img src="imgs/twitter.png" class="ci" />
+  <img src="imgs/reddit.jpg" class="ci" />
+  <img src="imgs/chip.jpg" class="ci" />
+  <img src="imgs/jitendra.jpg" class="ci" />
+  <img src="imgs/tesla.jpg" class="ci" />
+  <img src="imgs/rainforest.jpg" class="ci" />
+  <img src="imgs/chess.png" class="ci" />
+  <img src="imgs/gradient.png" class="ci" />
+  <img src="imgs/battery.jpg" class="ci" />
+  <img src="imgs/starry.jpg" class="ci" />
+</div>
+
+<br><br>
+<p>Go <a href="http://cs.stanford.edu/people/karpathy/convnetjs/">back to ConvNetJS</a></p>
+
+</div>
+
+</body>
+</html>
+

demo/js/autoencoder.js

@@ -0,0 +1,495 @@
+// globals
+var layer_defs, net, trainer;
+var t = "\
+layer_defs = [];\n\
+layer_defs.push({type:'input', out_sx:28, out_sy:28, out_depth:1});\n\
+layer_defs.push({type:'fc', num_neurons:50, activation:'tanh'});\n\
+layer_defs.push({type:'fc', num_neurons:50, activation:'tanh'});\n\
+layer_defs.push({type:'fc', num_neurons:2});\n\
+layer_defs.push({type:'fc', num_neurons:50, activation:'tanh'});\n\
+layer_defs.push({type:'fc', num_neurons:50, activation:'tanh'});\n\
+layer_defs.push({type:'regression', num_neurons:28*28});\n\
+\n\
+net = new convnetjs.Net();\n\
+net.makeLayers(layer_defs);\n\
+\n\
+trainer = new convnetjs.SGDTrainer(net, {learning_rate:1, method:'adadelta', batch_size:50, l2_decay:0.001, l1_decay:0.001});\n\
+";
+
+// ------------------------
+// BEGIN MNIST SPECIFIC STUFF
+// ------------------------
+var sample_training_instance = function() {
+
+  // find an unloaded batch
+  var bi = Math.floor(Math.random()*loaded_train_batches.length);
+  var b = loaded_train_batches[bi];
+  var k = Math.floor(Math.random()*3000); // sample within the batch
+  var n = b*3000+k;
+
+  // load more batches over time
+  if(step_num%5000===0 && step_num>0) {
+    for(var i=0;i<num_batches;i++) {
+      if(!loaded[i]) {
+        // load it
+        load_data_batch(i);
+        break; // okay for now
+      }
+    }
+  }
+
+  // fetch the appropriate row of the training image and reshape into a Vol
+  var p = img_data[b].data;
+  var x = new convnetjs.Vol(28,28,1,0.0);
+  var W = 28*28;
+  for(var i=0;i<W;i++) {
+    var ix = ((W * k) + i) * 4;
+    x.w[i] = p[ix]/255.0;
+  }
+
+  return {x:x, label:labels[n]};
+}
+
+var num_batches = 21; // 20 training batches, 1 test
+var data_img_elts = new Array(num_batches);
+var img_data = new Array(num_batches);
+var loaded = new Array(num_batches);
+var loaded_train_batches = [];
+
+// int main
+$(window).load(function() {
+
+  $("#newnet").val(t);
+  change_net();
+
+  for(var k=0;k<loaded.length;k++) { loaded[k] = false; }
+
+  load_data_batch(0); // async load train set batch 0 (6 total train batches)
+  load_data_batch(20); // async load test set (batch 6)
+  start_fun();
+});
+
+var start_fun = function() {
+  if(loaded[0] && loaded[20]) { 
+    console.log('starting!'); 
+    setInterval(load_and_step, 0); // lets go!
+  }
+  else { setTimeout(start_fun, 200); } // keep checking
+}
+
+var load_data_batch = function(batch_num) {
+  // Load the dataset with JS in background
+  data_img_elts[batch_num] = new Image();
+  var data_img_elt = data_img_elts[batch_num];
+  data_img_elt.onload = function() { 
+    var data_canvas = document.createElement('canvas');
+    data_canvas.width = data_img_elt.width;
+    data_canvas.height = data_img_elt.height;
+    var data_ctx = data_canvas.getContext("2d");
+    data_ctx.drawImage(data_img_elt, 0, 0); // copy it over... bit wasteful :(
+    img_data[batch_num] = data_ctx.getImageData(0, 0, data_canvas.width, data_canvas.height);
+    loaded[batch_num] = true;
+    if(batch_num < 20) { loaded_train_batches.push(batch_num); }
+    console.log('finished loading data batch ' + batch_num);
+  };
+  data_img_elt.src = "mnist/mnist_batch_" + batch_num + ".png";
+}
+
+// ------------------------
+// END MNIST SPECIFIC STUFF
+// ------------------------
+
+var maxmin = cnnutil.maxmin;
+var f2t = cnnutil.f2t;
+
+var render_act = function(A) {
+  var w = A.w;
+  var mm = maxmin(w);
+  var s = 1;
+
+  var canv = document.createElement('canvas');
+  canv.className = 'rendera';
+  var W = A.sx * s;
+  var H = A.sy * s;
+  canv.width = W;
+  canv.height = H;
+  var ctx = canv.getContext('2d');
+  var g = ctx.createImageData(W, H);
+  var d = 0;
+  for(var x=0;x<A.sx;x++) {
+    for(var y=0;y<A.sy;y++) {
+      var dval = Math.floor((A.get(x,y,d)-mm.minv)/mm.dv*255);  
+      for(var dx=0;dx<s;dx++) {
+        for(var dy=0;dy<s;dy++) {
+          var pp = ((W * (y*s+dy)) + (dx + x*s)) * 4;
+          for(var i=0;i<3;i++) { g.data[pp + i] = dval; } // rgb
+          g.data[pp+3] = 255; // alpha channel
+        }
+      }
+    }
+  }
+  ctx.putImageData(g, 0, 0);
+  return canv;
+}
+
+// elt is the element to add all the canvas activation drawings into
+// A is the Vol() to use
+// scale is a multiplier to make the visualizations larger. Make higher for larger pictures
+// if grads is true then gradients are used instead
+var draw_activations = function(elt, A, scale, grads) {
+
+  var s = scale || 2; // scale
+  var draw_grads = false;
+  if(typeof(grads) !== 'undefined') draw_grads = grads;
+
+  // get max and min activation to scale the maps automatically
+  var w = draw_grads ? A.dw : A.w;
+  var mm = maxmin(w);
+
+  // create the canvas elements, draw and add to DOM
+  for(var d=0;d<A.depth;d++) {
+
+    var canv = document.createElement('canvas');
+    canv.className = 'actmap';
+    var W = A.sx * s;
+    var H = A.sy * s;
+    canv.width = W;
+    canv.height = H;
+    var ctx = canv.getContext('2d');
+    var g = ctx.createImageData(W, H);
+
+    for(var x=0;x<A.sx;x++) {
+      for(var y=0;y<A.sy;y++) {
+        if(draw_grads) {
+          var dval = Math.floor((A.get_grad(x,y,d)-mm.minv)/mm.dv*255);
+        } else {
+          var dval = Math.floor((A.get(x,y,d)-mm.minv)/mm.dv*255);  
+        }
+        for(var dx=0;dx<s;dx++) {
+          for(var dy=0;dy<s;dy++) {
+            var pp = ((W * (y*s+dy)) + (dx + x*s)) * 4;
+            for(var i=0;i<3;i++) { g.data[pp + i] = dval; } // rgb
+            g.data[pp+3] = 255; // alpha channel
+          }
+        }
+      }
+    }
+    ctx.putImageData(g, 0, 0);
+    elt.appendChild(canv);
+  }  
+}
+
+var visualize_activations = function(net, elt) {
+
+  // clear the element
+  elt.innerHTML = "";
+
+  // show activations in each layer
+  var N = net.layers.length;
+  for(var i=0;i<N;i++) {
+    var L = net.layers[i];
+
+    var layer_div = document.createElement('div');
+
+    // visualize activations
+    var activations_div = document.createElement('div');
+    activations_div.appendChild(document.createTextNode('Activations:'));
+    activations_div.appendChild(document.createElement('br'));
+    activations_div.className = 'layer_act';
+    var scale = 2;
+    if(L.layer_type==='fc') scale = 10; // for softmax
+    if(L.layer_type==='regression') {
+      var Vvis = L.out_act.clone();
+      Vvis.sx = 28; 
+      Vvis.sy = 28;
+      Vvis.depth = 1;
+      draw_activations(activations_div, Vvis, scale);
+    } else {
+      draw_activations(activations_div, L.out_act, scale);
+
+      if(i===0) {
+        // also append the regression layer right nex tto input
+        // so that it's easy to compare
+        activations_div.appendChild(document.createElement('br'));
+        activations_div.appendChild(document.createTextNode('Predicted reconstruction:'));
+        activations_div.appendChild(document.createElement('br'));
+        var Vvis = net.layers[net.layers.length-1].out_act.clone();
+        Vvis.sx = 28; 
+        Vvis.sy = 28;
+        Vvis.depth = 1;
+        draw_activations(activations_div, Vvis, scale);
+      }
+    }
+
+    if(L.layer_type === 'fc' && i===1) {
+      var filters_div = document.createElement('div');
+      filters_div.appendChild(document.createTextNode('Weights:'));
+      filters_div.appendChild(document.createElement('br'));
+      for(var j=0;j<L.filters.length;j++) {
+        var Lshow = L.filters[j].clone();
+        Lshow.sx = 28; 
+        Lshow.sy = 28; 
+        Lshow.depth = 1;
+        draw_activations(filters_div, Lshow, 2);
+      }
+      activations_div.appendChild(filters_div);
+    }
+
+    // visualize filters if they are of reasonable size
+    if(L.layer_type === 'conv') {
+      var filters_div = document.createElement('div');
+      if(L.filters[0].sx>3) {
+        // actual weights
+        filters_div.appendChild(document.createTextNode('Weights:'));
+        filters_div.appendChild(document.createElement('br'));
+        for(var j=0;j<L.filters.length;j++) {
+          draw_activations(filters_div, L.filters[j], 2);
+        }
+        // gradients
+        filters_div.appendChild(document.createElement('br'));
+        filters_div.appendChild(document.createTextNode('Gradients:'));
+        filters_div.appendChild(document.createElement('br'));
+        for(var j=0;j<L.filters.length;j++) {
+          draw_activations(filters_div, L.filters[j], 2, true);
+        }
+      } else {
+        filters_div.appendChild(document.createTextNode('Weights hidden, too small'));
+      }
+      activations_div.appendChild(filters_div);
+    }
+    layer_div.appendChild(activations_div);
+
+    // print some stats on left of the layer
+    layer_div.className = 'layer ' + 'lt' + L.layer_type;
+    var title_div = document.createElement('div');
+    title_div.className = 'ltitle'
+    var t = L.layer_type + ' (' + L.out_sx + 'x' + L.out_sy + 'x' + L.out_depth + ')';
+    title_div.appendChild(document.createTextNode(t));
+    layer_div.appendChild(title_div);
+
+    if(L.layer_type==='conv') {
+      var t = 'filter size ' + L.filters[0].sx + 'x' + L.filters[0].sy + 'x' + L.filters[0].depth + ', stride ' + L.stride;
+      layer_div.appendChild(document.createTextNode(t));
+      layer_div.appendChild(document.createElement('br'));
+    }
+    if(L.layer_type==='pool') {
+      var t = 'pooling size ' + L.sx + 'x' + L.sy + ', stride ' + L.stride;
+      layer_div.appendChild(document.createTextNode(t));
+      layer_div.appendChild(document.createElement('br'));
+    }
+
+    // find min, max activations and display them
+    var mma = maxmin(L.out_act.w);
+    var t = 'max activation: ' + f2t(mma.maxv) + ', min: ' + f2t(mma.minv);
+    layer_div.appendChild(document.createTextNode(t));
+    layer_div.appendChild(document.createElement('br'));
+
+    // number of parameters
+    if(L.layer_type==='conv') {
+      var tot_params = L.sx*L.sy*L.in_depth*L.filters.length + L.filters.length;
+      var t = 'parameters: ' + L.filters.length + 'x' + L.sx + 'x' + L.sy + 'x' + L.in_depth + '+' + L.filters.length + ' = ' + tot_params;
+      layer_div.appendChild(document.createTextNode(t));
+      layer_div.appendChild(document.createElement('br'));
+    }
+    if(L.layer_type==='fc') {
+      var tot_params = L.num_inputs*L.filters.length + L.filters.length;
+      var t = 'parameters: ' + L.filters.length + 'x' + L.num_inputs + '+' + L.filters.length + ' = ' + tot_params;
+      layer_div.appendChild(document.createTextNode(t));
+      layer_div.appendChild(document.createElement('br'));
+    }
+
+    // css madness needed here...
+    var clear = document.createElement('div');
+    clear.className = 'clear';
+    layer_div.appendChild(clear);
+
+    elt.appendChild(layer_div);
+  }
+}
+
+// loads a training image and trains on it with the network
+var paused = false;
+var embed_samples = [];
+var embed_imgs = [];
+var load_and_step = function() {
+  if(paused) return; 
+
+  if(embed_samples.length === 0) { // happens once
+    for(var k=0;k<200;k++) {
+      var s = sample_training_instance();
+      embed_samples.push(s);
+      // render x and save it too
+      var I = render_act(s.x);
+      embed_imgs.push(I);
+    }
+  }
+  var sample = sample_training_instance();
+  step(sample); // process this image
+}
+
+var lix = 5;
+var d0 = 0;
+var d1 = 1;
+function cycle() {
+  var selected_layer = net.layers[lix];
+  d0 += 1;
+  d1 += 1;
+  if(d1 >= selected_layer.out_depth) d1 = 0; // and wrap
+  if(d0 >= selected_layer.out_depth) d0 = 0; // and wrap
+  $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer #' + lix + ' (' + net.layers[lix].layer_type + ')');
+}
+function updateLix(newlix) {
+  $("#button"+lix).css('background-color', ''); // erase highlight
+  lix = newlix;
+  d0 = 0;
+  d1 = 1; // reset these
+  $("#button"+lix).css('background-color', '#FFA');
+
+  $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer with index ' + lix + ' (' + net.layers[lix].layer_type + ')');
+}
+
+
+var lossGraph = new cnnvis.Graph();
+var xLossWindow = new cnnutil.Window(100);
+var w2LossWindow = new cnnutil.Window(100);
+var w1LossWindow = new cnnutil.Window(100);
+var step_num = 0;
+var colors = ["red", "blue", "green", "orange", "magenta", "cyan", "purple", "silver", "olive", "lime", "yellow"];
+var step = function(sample) {
+
+  // train on it with network
+  var stats = trainer.train(sample.x, sample.x.w);
+
+  // keep track of stats such as the average training error and loss  
+  xLossWindow.add(stats.cost_loss);
+  w1LossWindow.add(stats.l1_decay_loss);
+  w2LossWindow.add(stats.l2_decay_loss);
+
+  // visualize training status
+  var train_elt = document.getElementById("trainstats");
+  train_elt.innerHTML = '';
+  var t = 'Forward time per example: ' + stats.fwd_time + 'ms';
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'Backprop time per example: ' + stats.bwd_time + 'ms';
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'Regression loss: ' + f2t(xLossWindow.get_average());
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'L2 Weight decay loss: ' + f2t(w2LossWindow.get_average());
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'L1 Weight decay loss: ' + f2t(w1LossWindow.get_average());
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'Examples seen: ' + step_num;
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+
+  // visualize activations
+  if(step_num % 100 === 0) {
+    var vis_elt = document.getElementById("visnet");
+    visualize_activations(net, vis_elt);
+  }
+
+  // visualize embedding
+  if(step_num % 100 === 0) {
+
+    var embcanvas = document.getElementById('embedding');
+    var ctx = embcanvas.getContext("2d");
+    var EW = embcanvas.width;
+    var EH = embcanvas.height;
+
+    // propagate a few training examples through the network and grab codes
+    var xcodes = [];
+    var ycodes = [];
+    var ns = embed_samples.length; // number of samples
+    for(var k=0;k<ns;k++) {
+      var sample = embed_samples[k];
+      net.forward(sample.x);
+      var xcode = net.layers[lix].out_act.w[d0];
+      var ycode = net.layers[lix].out_act.w[d1];
+      xcodes.push(xcode);
+      ycodes.push(ycode);
+    }
+    var mmx = cnnutil.maxmin(xcodes);
+    var mmy = cnnutil.maxmin(ycodes);
+
+    // draw every example into the canvas
+    ctx.clearRect(0,0,EW,EH);
+    for(var k=0;k<ns;k++) {
+      var xpos = (EW-28*2)*(xcodes[k]-mmx.minv)/mmx.dv+28;
+      var ypos = (EH-28*2)*(ycodes[k]-mmy.minv)/mmy.dv+28;
+      // draw border according to class
+      ctx.fillStyle = colors[embed_samples[k].label];
+      ctx.fillRect(xpos-2,ypos-2,32,32);
+
+      ctx.drawImage(embed_imgs[k], xpos, ypos );
+    }
+
+  }
+
+  // log progress to graph, (full loss)
+  if(step_num % 200 === 0) {
+    var xa = xLossWindow.get_average();
+    var xw1 = w1LossWindow.get_average();
+    var xw2 = w2LossWindow.get_average();
+    if(xa >= 0 && xw1 >= 0 && xw2 >= 0) { // if they are -1 it means not enough data was accumulated yet for estimates
+      lossGraph.add(step_num, xa + xw1 + xw2);
+      lossGraph.drawSelf(document.getElementById("lossgraph"));
+    }
+  }
+
+  step_num++;
+}
+
+// user settings 
+var change_lr = function() {
+  trainer.learning_rate = parseFloat(document.getElementById("lr_input").value);
+  update_net_param_display();
+}
+var update_net_param_display = function() {
+  document.getElementById('lr_input').value = trainer.learning_rate;
+}
+var toggle_pause = function() {
+  paused = !paused;
+  var btn = document.getElementById('buttontp');
+  if(paused) { btn.value = 'resume' }
+  else { btn.value = 'pause'; }
+}
+var dump_json = function() {
+  document.getElementById("dumpjson").value = JSON.stringify(net.toJSON());
+}
+var clear_graph = function() {
+  lossGraph = new cnnvis.Graph(); // reinit graph too 
+}
+var reset_all = function() {
+  update_net_param_display();
+
+  // reinit windows that keep track of val/train accuracies
+  lossGraph = new cnnvis.Graph(); // reinit graph too
+  step_num = 0;
+
+  // enter buttons for layers
+  var t = '';
+  for(var i=1;i<net.layers.length-1;i++) { // ignore input and regression layers (first and last)
+    var butid = "button" + i;
+    t += "<input id=\""+butid+"\" value=\"" + net.layers[i].layer_type + "(" + net.layers[i].out_depth + ")" +"\" type=\"submit\" onclick=\"updateLix("+i+")\" style=\"width:80px; height: 30px; margin:5px;\";>";
+  }
+  $("#layer_ixes").html(t);
+  $("#button"+lix).css('background-color', '#FFA');
+  $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer with index ' + lix + ' (' + net.layers[lix].layer_type + ')');
+
+}
+var load_from_json = function() {
+  var jsonString = document.getElementById("dumpjson").value;
+  var json = JSON.parse(jsonString);
+  net = new convnetjs.Net();
+  net.fromJSON(json);
+  reset_all();
+}
+var change_net = function() {
+  eval($("#newnet").val());
+  reset_all();
+}

demo/js/automatic.js

@@ -0,0 +1,355 @@
+ 
+    // utility functions
+    Array.prototype.contains = function(v) {
+    for(var i = 0; i < this.length; i++) {
+      if(this[i] === v) return true;
+    }
+    return false;
+    };
+    Array.prototype.unique = function() {
+      var arr = [];
+      for(var i = 0; i < this.length; i++) {
+        if(!arr.contains(this[i])) {
+          arr.push(this[i]);
+        }
+      }
+      return arr; 
+    }
+    
+    function FAIL(outdivid, msg) {
+      $(outdivid).prepend("<div class=\"msg\" style=\"background-color:#FCC;\">"+msg+"</div>")
+    }
+    function SUCC(outdivid, msg) {
+      $(outdivid).prepend("<div class=\"msg\" style=\"background-color:#CFC;\">"+msg+"</div>")
+    }
+
+    // looks at a column i of data and guesses what's in it
+    // returns results of analysis: is column numeric? How many unique entries and what are they?
+    function guessColumn(data, c) {
+      var numeric = true;
+      var vs = [];
+      for(var i=0,n=data.length;i<n;i++) {
+        var v = data[i][c];
+        vs.push(v);
+        if(isNaN(v)) numeric = false;
+      }
+      var u = vs.unique();
+      if(!numeric) {
+        // if we have a non-numeric we will map it through uniques to an index
+        return {numeric:numeric, num:u.length, uniques:u};
+      } else {
+        return {numeric:numeric, num:u.length};
+      }
+    }
+    
+    // returns arr (csv parse)
+    // and colstats, which contains statistics about the columns of the input
+    // parsing results will be appended to a div with id outdivid
+    function importData(arr, outdivid) {
+      $(outdivid).empty(); // flush messages
+
+      // find number of datapoints
+      N = arr.length;
+      var t = [];
+      SUCC(outdivid, "found " + N + " data points");
+      if(N === 0) { FAIL(outdivid, 'no data points found?'); return; }
+      
+      // find dimensionality and enforce consistency
+      D = arr[0].length;
+      for(var i=0;i<N;i++) {
+        var d = arr[i].length;
+        if(d !== D) { FAIL(outdivid, 'data dimension not constant: line ' + i + ' has ' + d + ' entries.'); return; }
+      }
+      SUCC(outdivid, "data dimensionality is " + (D-1));
+      
+      // go through columns of data and figure out what they are
+      var colstats = [];
+      for(var i=0;i<D;i++) {
+        var res = guessColumn(arr, i);
+        colstats.push(res);
+        if(D > 20 && i>3 && i < D-3) {
+          if(i==4) {
+            SUCC(outdivid, "..."); // suppress output for too many columns
+          }
+        } else {
+          SUCC(outdivid, "column " + i + " looks " + (res.numeric ? "numeric" : "NOT numeric") + " and has " + res.num + " unique elements");
+        }
+      }
+
+      return {arr: arr, colstats: colstats};
+   }
+    
+  // process input mess into vols and labels
+  function makeDataset(arr, colstats) {
+
+    var labelix = parseInt($("#labelix").val());
+    if(labelix < 0) labelix = D + labelix; // -1 should turn to D-1
+
+    var data = [];
+    var labels = [];
+    for(var i=0;i<N;i++) {
+      var arri = arr[i];
+      
+      // create the input datapoint Vol()
+      var p = arri.slice(0, D-1);
+      var xarr = [];
+      for(var j=0;j<D;j++) {
+        if(j===labelix) continue; // skip!
+
+        if(colstats[j].numeric) {
+          xarr.push(parseFloat(arri[j]));
+        } else {
+          var u = colstats[j].uniques;
+          var ix = u.indexOf(arri[j]); // turn into 1ofk encoding
+          for(var q=0;q<u.length;q++) {
+            if(q === ix) { xarr.push(1.0); }
+            else { xarr.push(0.0); }
+          }
+        }
+      }
+      var x = new convnetjs.Vol(xarr);
+      
+      // process the label (last column)
+      if(colstats[labelix].numeric) {
+        var L = parseFloat(arri[labelix]); // regression
+      } else {
+        var L = colstats[labelix].uniques.indexOf(arri[labelix]); // classification
+        if(L==-1) {
+          console.log('whoa label not found! CRITICAL ERROR, very fishy.');
+        }
+      }
+      data.push(x);
+      labels.push(L);
+    }
+    
+    var dataset = {};
+    dataset.data = data;
+    dataset.labels = labels;
+    return dataset;
+  }
+
+  // optionally provide a magic net
+  function testEval(optional_net) {
+    if (typeof optional_net !== 'undefined') {
+      var net = optional_net;
+    } else {
+      var net = magicNet;
+    }
+
+    // set options for magic net
+    net.ensemble_size = parseInt($("#ensemblenum").val())
+
+    // read in the data in the text field
+    var test_dataset = importTestData();
+    // use magic net to predict
+    var n = test_dataset.data.length;
+    var acc = 0.0;
+    for(var i=0;i<n;i++) {
+      var yhat = net.predict(test_dataset.data[i]);
+      if(yhat === -1) {
+        $("#testresult").html("The MagicNet is not yet ready! It must process at least one batch of candidates across all folds first. Wait a bit.");
+        $("#testresult").css('background-color', '#FCC');
+        return;
+      }
+      var l = test_dataset.labels[i];
+      acc += (yhat === l ? 1 : 0); // 0-1 loss
+      console.log('test example ' + i + ': predicting ' + yhat + ', ground truth is ' + l);
+    }
+    acc /= n;
+
+    // report accuracy
+    $("#testresult").html("Test set accuracy: " + acc);
+    $("#testresult").css('background-color', '#CFC');
+  }
+
+  function reinitGraph() {
+    var legend = [];
+    for(var i=0;i<magicNet.candidates.length;i++) {
+      legend.push('model ' + i);
+    }
+    valGraph = new cnnvis.MultiGraph(legend, {miny: 0, maxy: 1});
+  }
+
+  var folds_evaluated = 0;
+  function finishedFold() {
+    folds_evaluated++;
+    $("#foldreport").html("So far evaluated a total of " + folds_evaluated + "/" + magicNet.num_folds + " folds in current batch");
+    reinitGraph();
+  }
+  var batches_evaluated = 0;
+  function finishedBatch() {
+    batches_evaluated++;
+    $("#candsreport").html("So far evaluated a total of " + batches_evaluated + " batches of candidates");
+  }
+
+  var magicNet = null;
+  function startCV() { // takes in train_dataset global
+    var opts = {}
+    opts.train_ratio = parseInt($("#trainp").val())/100.0;
+    opts.num_folds = parseInt($("#foldsnum").val());
+    opts.num_candidates = parseInt($("#candsnum").val());
+    opts.num_epochs = parseInt($("#epochsnum").val());
+    opts.neurons_min = parseInt($("#nnmin").val());
+    opts.neurons_max = parseInt($("#nnmin").val());
+    magicNet = new convnetjs.MagicNet(train_dataset.data, train_dataset.labels, opts);
+    magicNet.onFinishFold(finishedFold);
+    magicNet.onFinishBatch(finishedBatch);
+
+    folds_evaluated = 0;
+    batches_evaluated = 0;
+    $("#candsreport").html("So far evaluated a total of " + batches_evaluated + " batches of candidates");
+    $("#foldreport").html("So far evaluated a total of " + folds_evaluated + "/" + magicNet.num_folds + " folds in current batch");
+    reinitGraph();
+
+    var legend = [];
+    for(var i=0;i<magicNet.candidates.length;i++) {
+      legend.push('model ' + i);
+    }
+    valGraph = new cnnvis.MultiGraph(legend, {miny: 0, maxy: 1});
+    setInterval(step, 0);
+  }
+      
+    var fold;
+    var cands = [];
+    var dostep = false;
+    var valGraph;
+    var iter = 0;
+    function step() {
+      iter++;
+      
+      magicNet.step();
+      if(iter % 300 == 0) {
+
+        var vals = magicNet.evalValErrors();
+        valGraph.add(magicNet.iter, vals);
+        valGraph.drawSelf(document.getElementById("valgraph"));
+    
+        // print out the best models so far
+        var cands = magicNet.candidates; // naughty: get pointer to internal data
+        var scores = [];
+        for(var k=0;k<cands.length;k++) {
+          var c = cands[k];
+          var s = c.acc.length === 0 ? 0 : c.accv / c.acc.length;
+          scores.push(s);
+        }
+        var mm = convnetjs.maxmin(scores);
+        var cm = cands[mm.maxi];
+        var t = '';
+        if(c.acc.length > 0) {
+          t += 'Results based on ' + c.acc.length + ' folds:';
+          t += 'best model in current batch (validation accuracy ' + mm.maxv + '):<br>';
+          t += '<b>Net layer definitions:</b><br>';
+          t += JSON.stringify(cm.layer_defs);
+          t += '<br><b>Trainer definition:</b><br>';
+          t += JSON.stringify(cm.trainer_def);
+          t += '<br>';
+        }
+        $('#bestmodel').html(t);
+
+        // also print out the best model so far
+        var t = '';
+        if(magicNet.evaluated_candidates.length > 0) {
+          var cm = magicNet.evaluated_candidates[0];
+          t += 'validation accuracy of best model so far, overall: ' + cm.accv / cm.acc.length + '<br>';
+          t += '<b>Net layer definitions:</b><br>';
+          t += JSON.stringify(cm.layer_defs);
+          t += '<br><b>Trainer definition:</b><br>';
+          t += JSON.stringify(cm.trainer_def);
+          t += '<br>';
+        }
+        $('#bestmodeloverall').html(t);
+      }
+    }
+    
+    // TODO: MOVE TO CONVNETJS UTILS
+    var randperm = function(n) {
+      var i = n,
+          j = 0,
+          temp;
+      var array = [];
+      for(var q=0;q<n;q++)array[q]=q;
+      while (i--) {
+          j = Math.floor(Math.random() * (i+1));
+          temp = array[i];
+          array[i] = array[j];
+          array[j] = temp;
+      }
+      return array;
+    }
+
+    var train_dataset, train_import_data; // globals
+    function importTrainData() {
+      var csv_txt = $('#data-ta').val();
+      var arr = $.csv.toArrays(csv_txt);
+      var arr_train = arr;
+      var arr_test = [];
+
+      var test_ratio = Math.floor($("#testsplit").val());
+      if(test_ratio !== 0) {
+        // send some lines to test set
+        var test_lines_num = Math.floor(arr.length * test_ratio / 100.0);
+        var rp = randperm(arr.length);
+        arr_train = [];
+        for(var i=0;i<arr.length;i++) {
+          if(i<test_lines_num) {
+            arr_test.push(arr[rp[i]]);
+          } else {
+            arr_train.push(arr[rp[i]]);
+          }
+        }
+        // enter test lines to test box
+        var t = "";
+        for(var i=0;i<arr_test.length;i++) {
+          t+= arr_test[i].join(",")+"\n";
+        }
+        $("#data-te").val(t);
+        $("#datamsgtest").empty();
+      }
+
+      $("#prepromsg").empty(); // flush
+      SUCC("#prepromsg", "Sent " + arr_test.length + " data to test, keeping " + arr_train.length + " for train.");
+      train_import_data = importData(arr_train,'#datamsg');
+      train_dataset = makeDataset(train_import_data.arr, train_import_data.colstats);
+      return train_dataset;
+    }
+
+    function importTestData() {
+      var csv_txt = $('#data-te').val();
+      var arr = $.csv.toArrays(csv_txt);
+      var import_data = importData(arr,'#datamsgtest');
+      // note important that we use colstats of train data!
+      test_dataset = makeDataset(import_data.arr, train_import_data.colstats);
+      return test_dataset;
+    }
+
+    function loadDB(url) {
+      // load a dataset from a url with ajax
+      $.ajax({
+        url: url,
+        dataType: "text",
+        success: function(txt) {
+          $("#data-ta").val(txt);
+        }
+      });
+    }
+
+    function start() {
+      loadDB('data/car.data.txt');
+    }
+
+    function exportMagicNet() {
+      $("#taexport").val(JSON.stringify(magicNet.toJSON()));
+
+      /*
+      // for debugging
+      var j = JSON.parse($("#taexport").val());
+      var m = new convnetjs.MagicNet();
+      m.fromJSON(j);
+      testEval(m);
+      */
+    }
+
+    function changeNNRange() {
+      magicNet.neurons_min = parseInt($("#nnmin").val());
+      magicNet.neurons_max = parseInt($("#nnmax").val());
+    }

demo/js/classify2d.js

@@ -0,0 +1,321 @@
+
+var data, labels, N;
+var ss = 50.0; // scale for drawing
+
+// create neural net
+var layer_defs, net, trainer;
+var t = "\n\
+layer_defs = [];\n\
+layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:2});\n\
+layer_defs.push({type:'fc', num_neurons:6, activation: 'tanh'});\n\
+layer_defs.push({type:'fc', num_neurons:2, activation: 'tanh'});\n\
+layer_defs.push({type:'softmax', num_classes:2});\n\
+\n\
+net = new convnetjs.Net();\n\
+net.makeLayers(layer_defs);\n\
+\n\
+trainer = new convnetjs.SGDTrainer(net, {learning_rate:0.01, momentum:0.1, batch_size:10, l2_decay:0.001});\n\
+";
+
+function reload() {
+  eval($("#layerdef").val());
+
+  // enter buttons for layers
+  var t = '';
+  for(var i=1;i<net.layers.length-1;i++) { // ignore input and regression layers (first and last)
+    var butid = "button" + i;
+    t += "<input id=\""+butid+"\" value=\"" + net.layers[i].layer_type + "(" + net.layers[i].out_depth + ")" +"\" type=\"submit\" onclick=\"updateLix("+i+")\" style=\"width:80px; height: 30px; margin:5px;\";>";
+  }
+  $("#layer_ixes").html(t);
+  $("#button"+lix).css('background-color', '#FFA');
+  $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer with index ' + lix + ' (' + net.layers[lix].layer_type + ')');
+}
+function updateLix(newlix) {
+  $("#button"+lix).css('background-color', ''); // erase highlight
+  lix = newlix;
+  d0 = 0;
+  d1 = 1; // reset these
+  $("#button"+lix).css('background-color', '#FFA');
+
+  $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer with index ' + lix + ' (' + net.layers[lix].layer_type + ')');
+}
+ 
+
+function myinit() { }
+
+function random_data(){
+  data = [];
+  labels = [];
+  for(var k=0;k<40;k++) {
+    data.push([convnetjs.randf(-3,3), convnetjs.randf(-3,3)]); labels.push(convnetjs.randf(0,1) > 0.5 ? 1 : 0);
+  }
+  N = labels.length;
+}
+
+function original_data(){
+  
+  data = [];
+  labels = [];
+  data.push([-0.4326  ,  1.1909 ]); labels.push(1);
+  data.push([3.0, 4.0]); labels.push(1);
+  data.push([0.1253 , -0.0376   ]); labels.push(1);
+  data.push([0.2877 ,   0.3273  ]); labels.push(1);
+  data.push([-1.1465 ,   0.1746 ]); labels.push(1);
+  data.push([1.8133 ,   1.0139  ]); labels.push(0);
+  data.push([2.7258 ,   1.0668  ]); labels.push(0);
+  data.push([1.4117 ,   0.5593  ]); labels.push(0);
+  data.push([4.1832 ,   0.3044  ]); labels.push(0);
+  data.push([1.8636 ,   0.1677  ]); labels.push(0);
+  data.push([0.5 ,   3.2  ]); labels.push(1);
+  data.push([0.8 ,   3.2  ]); labels.push(1);
+  data.push([1.0 ,   -2.2  ]); labels.push(1);
+  N = labels.length;
+}
+
+function circle_data() {
+  data = [];
+  labels = [];
+  for(var i=0;i<50;i++) {
+    var r = convnetjs.randf(0.0, 2.0);
+    var t = convnetjs.randf(0.0, 2*Math.PI);
+    data.push([r*Math.sin(t), r*Math.cos(t)]);
+    labels.push(1);
+  }
+  for(var i=0;i<50;i++) {
+    var r = convnetjs.randf(3.0, 5.0);
+    //var t = convnetjs.randf(0.0, 2*Math.PI);
+    var t = 2*Math.PI*i/50.0
+    data.push([r*Math.sin(t), r*Math.cos(t)]);
+    labels.push(0);
+  }
+  N = data.length;
+}
+
+function spiral_data() {
+  data = [];
+  labels = [];
+  var n = 100;
+  for(var i=0;i<n;i++) {
+    var r = i/n*5 + convnetjs.randf(-0.1, 0.1);
+    var t = 1.25*i/n*2*Math.PI + convnetjs.randf(-0.1, 0.1);
+    data.push([r*Math.sin(t), r*Math.cos(t)]);
+    labels.push(1);
+  }
+  for(var i=0;i<n;i++) {
+    var r = i/n*5 + convnetjs.randf(-0.1, 0.1);
+    var t = 1.25*i/n*2*Math.PI + Math.PI + convnetjs.randf(-0.1, 0.1);
+    data.push([r*Math.sin(t), r*Math.cos(t)]);
+    labels.push(0);
+  }
+  N = data.length;
+}
+ 
+function update(){
+  // forward prop the data
+
+  var start = new Date().getTime();
+
+  var x = new convnetjs.Vol(1,1,2);
+  //x.w = data[ix];
+  var avloss = 0.0;
+  for(var iters=0;iters<20;iters++) {
+    for(var ix=0;ix<N;ix++) {
+      x.w = data[ix];
+      var stats = trainer.train(x, labels[ix]);
+      avloss += stats.loss;
+    }
+  }
+  avloss /= N*iters;
+
+  var end = new Date().getTime();
+  var time = end - start;
+      
+  //console.log('loss = ' + avloss + ', 100 cycles through data in ' + time + 'ms');
+}
+
+function cycle() {
+  var selected_layer = net.layers[lix];
+  d0 += 1;
+  d1 += 1;
+  if(d1 >= selected_layer.out_depth) d1 = 0; // and wrap
+  if(d0 >= selected_layer.out_depth) d0 = 0; // and wrap
+  $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer #' + lix + ' (' + net.layers[lix].layer_type + ')');
+}
+
+var lix = 4; // layer id to track first 2 neurons of
+var d0 = 0; // first dimension to show visualized
+var d1 = 1; // second dimension to show visualized
+function draw(){
+    
+    ctx.clearRect(0,0,WIDTH,HEIGHT);
+    
+    var netx = new convnetjs.Vol(1,1,2);
+    // draw decisions in the grid
+    var density= 5.0;
+    var gridstep = 2;
+    var gridx = [];
+    var gridy = [];
+    var gridl = []; 
+    for(var x=0.0, cx=0; x<=WIDTH; x+= density, cx++) {
+      for(var y=0.0, cy=0; y<=HEIGHT; y+= density, cy++) {
+        //var dec= svm.marginOne([(x-WIDTH/2)/ss, (y-HEIGHT/2)/ss]);
+        netx.w[0] = (x-WIDTH/2)/ss;
+        netx.w[1] = (y-HEIGHT/2)/ss;
+        var a = net.forward(netx, false);
+        
+        if(a.w[0] > a.w[1]) ctx.fillStyle = 'rgb(250, 150, 150)';
+        else ctx.fillStyle = 'rgb(150, 250, 150)';
+
+        //ctx.fillStyle = 'rgb(150,' + Math.floor(a.w[0]*105)+150 + ',150)';
+        //ctx.fillStyle = 'rgb(' + Math.floor(a.w[0]*255) + ',' + Math.floor(a.w[1]*255) + ', 0)';
+        ctx.fillRect(x-density/2-1, y-density/2-1, density+2, density+2);
+
+        if(cx%gridstep === 0 && cy%gridstep===0) {
+          // record the transformation information
+          var xt = net.layers[lix].out_act.w[d0]; // in screen coords
+          var yt = net.layers[lix].out_act.w[d1]; // in screen coords
+          gridx.push(xt);
+          gridy.push(yt);
+          gridl.push(a.w[0] > a.w[1]); // remember final label as well
+        }
+      }
+    }
+
+    // draw axes
+    ctx.beginPath();
+    ctx.strokeStyle = 'rgb(50,50,50)';
+    ctx.lineWidth = 1;
+    ctx.moveTo(0, HEIGHT/2);
+    ctx.lineTo(WIDTH, HEIGHT/2);
+    ctx.moveTo(WIDTH/2, 0);
+    ctx.lineTo(WIDTH/2, HEIGHT);
+    ctx.stroke();
+
+    // draw representation transformation axes for two neurons at some layer
+    var mmx = cnnutil.maxmin(gridx);
+    var mmy = cnnutil.maxmin(gridy);
+    visctx.clearRect(0,0,visWIDTH,visHEIGHT);
+    visctx.strokeStyle = 'rgb(0, 0, 0)';
+    var n = Math.floor(Math.sqrt(gridx.length)); // size of grid. Should be fine?
+    var ng = gridx.length;
+    var c = 0; // counter
+    visctx.beginPath() 
+    for(var x=0;x<n;x++) {
+      for(var y=0;y<n;y++) {
+
+        // down
+        var ix1 = x*n+y;
+        var ix2 = x*n+y+1;
+        if(ix1 >= 0 && ix2 >= 0 && ix1 < ng && ix2 < ng && y<n-1) { // check oob
+          var xraw = gridx[ix1];
+          xraw1 = visWIDTH*(gridx[ix1] - mmx.minv)/mmx.dv;
+          yraw1 = visHEIGHT*(gridy[ix1] - mmy.minv)/mmy.dv;
+          xraw2 = visWIDTH*(gridx[ix2] - mmx.minv)/mmx.dv;
+          yraw2 = visHEIGHT*(gridy[ix2] - mmy.minv)/mmy.dv;
+          visctx.moveTo(xraw1, yraw1);
+          visctx.lineTo(xraw2, yraw2);
+        }
+
+        // and draw its color
+        if(gridl[ix1]) visctx.fillStyle = 'rgb(250, 150, 150)';
+        else visctx.fillStyle = 'rgb(150, 250, 150)';
+        var sz = density * gridstep;
+        visctx.fillRect(xraw1-sz/2-1, yraw1-sz/2-1, sz+2, sz+2);
+
+        // right
+        var ix1 = (x+1)*n+y;
+        var ix2 = x*n+y;
+        if(ix1 >= 0 && ix2 >= 0 && ix1 < ng && ix2 < ng && x <n-1) { // check oob
+          var xraw = gridx[ix1];
+          xraw1 = visWIDTH*(gridx[ix1] - mmx.minv)/mmx.dv;
+          yraw1 = visHEIGHT*(gridy[ix1] - mmy.minv)/mmy.dv;
+          xraw2 = visWIDTH*(gridx[ix2] - mmx.minv)/mmx.dv;
+          yraw2 = visHEIGHT*(gridy[ix2] - mmy.minv)/mmy.dv;
+          visctx.moveTo(xraw1, yraw1);
+          visctx.lineTo(xraw2, yraw2);
+        }
+ 
+      }
+    }
+    visctx.stroke();
+
+    // draw datapoints.
+    ctx.strokeStyle = 'rgb(0,0,0)';
+    ctx.lineWidth = 1;
+    for(var i=0;i<N;i++) {
+      
+      if(labels[i]==1) ctx.fillStyle = 'rgb(100,200,100)';
+      else ctx.fillStyle = 'rgb(200,100,100)';
+      
+      drawCircle(data[i][0]*ss+WIDTH/2, data[i][1]*ss+HEIGHT/2, 5.0);
+
+      // also draw transformed data points while we're at it
+      netx.w[0] = data[i][0];
+      netx.w[1] = data[i][1]
+      var a = net.forward(netx, false);
+      var xt = visWIDTH * (net.layers[lix].out_act.w[d0] - mmx.minv) / mmx.dv; // in screen coords
+      var yt = visHEIGHT * (net.layers[lix].out_act.w[d1] - mmy.minv) / mmy.dv; // in screen coords
+      if(labels[i]==1) visctx.fillStyle = 'rgb(100,200,100)';
+      else visctx.fillStyle = 'rgb(200,100,100)';
+      visctx.beginPath();
+      visctx.arc(xt, yt, 5.0, 0, Math.PI*2, true); 
+      visctx.closePath();
+      visctx.stroke();
+      visctx.fill();
+    }
+}
+
+function mouseClick(x, y, shiftPressed, ctrlPressed){
+  
+  // x and y transformed to data space coordinates
+  var xt = (x-WIDTH/2)/ss;
+  var yt = (y-HEIGHT/2)/ss;
+
+  if(ctrlPressed) {
+    // remove closest data point
+    var mink = -1;
+    var mind = 99999;
+    for(var k=0, n=data.length;k<n;k++) {
+      var dx = data[k][0] - xt;
+      var dy = data[k][1] - yt;
+      var d = dx*dx+dy*dy;
+      if(d < mind || k==0) {
+        mind = d;
+        mink = k;
+      }
+    }
+    if(mink>=0) {
+      console.log('splicing ' + mink);
+      data.splice(mink, 1);
+      labels.splice(mink, 1);
+      N -= 1;
+    }
+
+  } else {
+    // add datapoint at location of click
+    data.push([xt, yt]);
+    labels.push(shiftPressed ? 1 : 0);
+    N += 1;
+  }
+
+}
+
+function keyDown(key){
+}
+
+function keyUp(key) {
+}
+
+$(function() {
+    // note, globals
+    viscanvas = document.getElementById('viscanvas');
+    visctx = viscanvas.getContext('2d');
+    visWIDTH = viscanvas.width;
+    visHEIGHT = viscanvas.height;
+
+    circle_data();
+    $("#layerdef").val(t);
+    reload();
+    NPGinit(20);
+
+});

demo/js/image-helpers.js

@@ -0,0 +1,54 @@
+
+var loadFile = function(event) {
+var reader = new FileReader();
+reader.onload = function(){
+  var preview = document.getElementById('preview_img');
+  preview.src = centerCrop(reader.result);
+  preview.src = resize(preview.src);
+};
+reader.readAsDataURL(event.target.files[0]);
+};
+
+function centerCrop(src){
+    var image = new Image();
+    image.src = src;
+
+    var max_width = Math.min(image.width, image.height);
+    var max_height = Math.min(image.width, image.height);
+
+    var canvas = document.createElement('canvas');
+    var ctx = canvas.getContext("2d");
+
+    ctx.clearRect(0, 0, canvas.width, canvas.height);
+    canvas.width = max_width;
+    canvas.height = max_height;
+    ctx.drawImage(image, (max_width - image.width)/2, (max_height - image.height)/2, image.width, image.height);
+    return canvas.toDataURL("image/png");
+}
+
+function resize(src){
+    var image = new Image();
+    image.src = src;
+
+    var canvas = document.createElement('canvas');
+    canvas.width = image.width;
+    canvas.height = image.height;
+    var ctx = canvas.getContext("2d");
+    ctx.clearRect(0, 0, canvas.width, canvas.height);
+    ctx.drawImage(image, 0, 0, image.width, image.height);
+
+    var dst = document.createElement('canvas');
+    dst.width = image_dimension;
+    dst.height = image_dimension;
+
+    window.pica.WW = false;
+    window.pica.resizeCanvas(canvas, dst, {
+    quality: 2,
+    unsharpAmount: 500,
+    unsharpThreshold: 100,
+    transferable: false
+  }, function (err) {  });
+    window.pica.WW = true;
+    return dst.toDataURL("image/png");
+}
+  

demo/js/image_regression.js

@@ -0,0 +1,173 @@
+
+var data, labels;
+var layer_defs, net, trainer;
+
+// create neural net
+var t = "layer_defs = [];\n\
+layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:2}); // 2 inputs: x, y \n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
+layer_defs.push({type:'regression', num_neurons:3}); // 3 outputs: r,g,b \n\
+\n\
+net = new convnetjs.Net();\n\
+net.makeLayers(layer_defs);\n\
+\n\
+trainer = new convnetjs.SGDTrainer(net, {learning_rate:0.01, momentum:0.9, batch_size:5, l2_decay:0.0});\n\
+";
+
+var batches_per_iteration = 100;
+var mod_skip_draw = 100;
+var smooth_loss = -1;
+
+function update(){
+  // forward prop the data
+  var W = nn_canvas.width;
+  var H = nn_canvas.height;
+
+  var p = oridata.data;
+
+  var v = new convnetjs.Vol(1,1,2);
+  var loss = 0;
+  var lossi = 0;
+  var N = batches_per_iteration;
+  for(var iters=0;iters<trainer.batch_size;iters++) {
+    for(var i=0;i<N;i++) {
+      // sample a coordinate
+      var x = convnetjs.randi(0, W);
+      var y = convnetjs.randi(0, H);
+      var ix = ((W*y)+x)*4;
+      var r = [p[ix]/255.0, p[ix+1]/255.0, p[ix+2]/255.0]; // r g b
+      v.w[0] = (x-W/2)/W;
+      v.w[1] = (y-H/2)/H;
+      var stats = trainer.train(v, r);
+      loss += stats.loss;
+      lossi += 1;
+    }
+  }
+  loss /= lossi;
+
+  if(counter === 0) smooth_loss = loss;
+  else smooth_loss = 0.99*smooth_loss + 0.01*loss;
+
+  var t = '';
+  t += 'loss: ' + smooth_loss;
+  t += '<br>'
+  t += 'iteration: ' + counter;
+  $("#report").html(t);
+}
+
+function draw() {
+  if(counter % mod_skip_draw !== 0) return;
+
+  // iterate over all pixels in the target array, evaluate them
+  // and draw
+  var W = nn_canvas.width;
+  var H = nn_canvas.height;
+
+  var g = nn_ctx.getImageData(0, 0, W, H);
+  var v = new convnetjs.Vol(1, 1, 2);
+  for(var x=0;x<W;x++) {
+    v.w[0] = (x-W/2)/W;
+    for(var y=0;y<H;y++) {
+      v.w[1] = (y-H/2)/H;
+
+      var ix = ((W*y)+x)*4;
+      var r = net.forward(v);
+      g.data[ix+0] = Math.floor(255*r.w[0]);
+      g.data[ix+1] = Math.floor(255*r.w[1]);
+      g.data[ix+2] = Math.floor(255*r.w[2]);
+      g.data[ix+3] = 255; // alpha...
+    }
+  }
+  nn_ctx.putImageData(g, 0, 0);
+}
+
+function tick() {
+  update();
+  draw();
+  counter += 1;
+}
+
+function reload() {
+  counter = 0;
+  eval($("#layerdef").val());
+  //$("#slider").slider("value", Math.log(trainer.learning_rate) / Math.LN10);
+  //$("#lr").html('Learning rate: ' + trainer.learning_rate);
+}
+function refreshSwatch() {
+  var lr = $("#slider").slider("value");
+  trainer.learning_rate = Math.pow(10, lr);
+  $("#lr").html('Learning rate: ' + trainer.learning_rate);
+}
+
+var ori_canvas, nn_canvas, ori_ctx, nn_ctx, oridata;
+var sz = 200; // size of our drawing area
+var counter = 0;
+$(function() {
+    // dynamically load lena image into original image canvas
+    var image = new Image();
+    //image.src = "lena.png";
+    image.onload = function() {
+
+      ori_canvas = document.getElementById('canv_original');
+      nn_canvas = document.getElementById('canv_net');
+      ori_canvas.width = sz;
+      ori_canvas.height = sz;
+      nn_canvas.width = sz;
+      nn_canvas.height = sz;
+
+      ori_ctx = ori_canvas.getContext("2d");
+      nn_ctx = nn_canvas.getContext("2d");
+      ori_ctx.drawImage(image, 0, 0, sz, sz);
+      oridata = ori_ctx.getImageData(0, 0, sz, sz); // grab the data pointer. Our dataset.
+
+      // start the regression!
+      setInterval(tick, 1);
+    }
+    image.src = "imgs/cat.jpg";
+
+    // init put text into textarea
+    $("#layerdef").val(t);
+
+    // load the net
+    reload();
+
+    // set up slider for learning rate
+    $("#slider").slider({
+      orientation: "horizontal",
+      min: -4,
+      max: -1,
+      step: 0.05,
+      value: Math.log(trainer.learning_rate) / Math.LN10,
+      slide: refreshSwatch,
+      change: refreshSwatch
+    });
+    $("#lr").html('Learning rate: ' + trainer.learning_rate);
+
+    $("#f").on('change', function(ev) {
+      var f = ev.target.files[0];
+      var fr = new FileReader();
+      fr.onload = function(ev2) {
+        var image = new Image();
+        image.onload = function(){
+          ori_ctx.drawImage(image, 0, 0, sz, sz);
+          oridata = ori_ctx.getImageData(0, 0, sz, sz);
+          reload();
+        }
+        image.src = ev2.target.result;
+      };
+      fr.readAsDataURL(f);
+    });
+
+    $('.ci').click(function(){
+      var src = $(this).attr('src');
+      ori_ctx.drawImage(this, 0, 0, sz, sz);
+      oridata = ori_ctx.getImageData(0, 0, sz, sz);
+      reload();
+    });
+});

demo/js/images-demo.js

@@ -0,0 +1,651 @@
+var sample_training_instance = function() {
+  // find an unloaded batch
+  var bi = Math.floor(Math.random()*loaded_train_batches.length);
+  var b = loaded_train_batches[bi];
+  var k = Math.floor(Math.random()*num_samples_per_batch); // sample within the batch
+  var n = b*num_samples_per_batch+k;
+
+  // load more batches over time
+  if(step_num%(2 * num_samples_per_batch)===0 && step_num>0) {
+    for(var i=0;i<num_batches;i++) {
+      if(!loaded[i]) {
+        // load it
+        load_data_batch(i);
+        break; // okay for now
+      }
+    }
+  }
+
+  // fetch the appropriate row of the training image and reshape into a Vol
+  var p = img_data[b].data;
+  var x = new convnetjs.Vol(image_dimension,image_dimension,image_channels,0.0);
+  var W = image_dimension*image_dimension;
+  var j=0;
+  for(var dc=0;dc<image_channels;dc++) {
+    var i=0;
+    for(var xc=0;xc<image_dimension;xc++) {
+      for(var yc=0;yc<image_dimension;yc++) {
+        var ix = ((W * k) + i) * 4 + dc;
+        x.set(yc,xc,dc,p[ix]/255.0-0.5);
+        i++;
+      }
+    }
+  }
+
+  if(random_position){
+    var dx = Math.floor(Math.random()*5-2);
+    var dy = Math.floor(Math.random()*5-2);
+    x = convnetjs.augment(x, image_dimension, dx, dy, false); //maybe change position
+  }
+
+  if(random_flip){
+    x = convnetjs.augment(x, image_dimension, 0, 0, Math.random()<0.5); //maybe flip horizontally
+  }
+
+  var isval = use_validation_data && n%10===0 ? true : false;
+  return {x:x, label:labels[n], isval:isval};
+}
+
+// sample a random testing instance
+var sample_test_instance = function() {
+
+  var b = test_batch;
+  var k = Math.floor(Math.random()*num_samples_per_batch);
+  var n = b*num_samples_per_batch+k;
+
+  var p = img_data[b].data;
+  var x = new convnetjs.Vol(image_dimension,image_dimension,image_channels,0.0);
+  var W = image_dimension*image_dimension;
+  var j=0;
+  for(var dc=0;dc<image_channels;dc++) {
+    var i=0;
+    for(var xc=0;xc<image_dimension;xc++) {
+      for(var yc=0;yc<image_dimension;yc++) {
+        var ix = ((W * k) + i) * 4 + dc;
+        x.set(yc,xc,dc,p[ix]/255.0-0.5);
+        i++;
+      }
+    }
+  }
+
+  // distort position and maybe flip
+  var xs = [];
+  
+  if (random_flip || random_position){
+    for(var k=0;k<6;k++) {
+      var test_variation = x;
+      if(random_position){
+        var dx = Math.floor(Math.random()*5-2);
+        var dy = Math.floor(Math.random()*5-2);
+        test_variation = convnetjs.augment(test_variation, image_dimension, dx, dy, false);
+      }
+      
+      if(random_flip){
+        test_variation = convnetjs.augment(test_variation, image_dimension, 0, 0, Math.random()<0.5); 
+      }
+
+      xs.push(test_variation);
+    }
+  }else{
+    xs.push(x, image_dimension, 0, 0, false); // push an un-augmented copy
+  }
+  
+  // return multiple augmentations, and we will average the network over them
+  // to increase performance
+  return {x:xs, label:labels[n]};
+}
+
+
+var data_img_elts = new Array(num_batches);
+var img_data = new Array(num_batches);
+var loaded = new Array(num_batches);
+var loaded_train_batches = [];
+
+// int main
+$(window).load(function() {
+
+  $("#newnet").val(t);
+  eval($("#newnet").val());
+
+  update_net_param_display();
+
+  for(var k=0;k<loaded.length;k++) { loaded[k] = false; }
+
+  load_data_batch(0); // async load train set batch 0
+  load_data_batch(test_batch); // async load test set
+  start_fun();
+});
+
+var start_fun = function() {
+  if(loaded[0] && loaded[test_batch]) { 
+    console.log('starting!'); 
+    setInterval(load_and_step, 0); // lets go!
+  }
+  else { setTimeout(start_fun, 200); } // keep checking
+}
+
+var load_data_batch = function(batch_num) {
+  // Load the dataset with JS in background
+  data_img_elts[batch_num] = new Image();
+  var data_img_elt = data_img_elts[batch_num];
+  data_img_elt.onload = function() { 
+    var data_canvas = document.createElement('canvas');
+    data_canvas.width = data_img_elt.width;
+    data_canvas.height = data_img_elt.height;
+    var data_ctx = data_canvas.getContext("2d");
+    data_ctx.drawImage(data_img_elt, 0, 0); // copy it over... bit wasteful :(
+    img_data[batch_num] = data_ctx.getImageData(0, 0, data_canvas.width, data_canvas.height);
+    loaded[batch_num] = true;
+    if(batch_num < test_batch) { loaded_train_batches.push(batch_num); }
+    console.log('finished loading data batch ' + batch_num);
+  };
+  data_img_elt.src = dataset_name + "/" + dataset_name + "_batch_" + batch_num + ".png";
+}
+
+var maxmin = cnnutil.maxmin;
+var f2t = cnnutil.f2t;
+
+// elt is the element to add all the canvas activation drawings into
+// A is the Vol() to use
+// scale is a multiplier to make the visualizations larger. Make higher for larger pictures
+// if grads is true then gradients are used instead
+var draw_activations = function(elt, A, scale, grads) {
+
+  var s = scale || 2; // scale
+  var draw_grads = false;
+  if(typeof(grads) !== 'undefined') draw_grads = grads;
+
+  // get max and min activation to scale the maps automatically
+  var w = draw_grads ? A.dw : A.w;
+  var mm = maxmin(w);
+
+  // create the canvas elements, draw and add to DOM
+  for(var d=0;d<A.depth;d++) {
+
+    var canv = document.createElement('canvas');
+    canv.className = 'actmap';
+    var W = A.sx * s;
+    var H = A.sy * s;
+    canv.width = W;
+    canv.height = H;
+    var ctx = canv.getContext('2d');
+    var g = ctx.createImageData(W, H);
+
+    for(var x=0;x<A.sx;x++) {
+      for(var y=0;y<A.sy;y++) {
+        if(draw_grads) {
+          var dval = Math.floor((A.get_grad(x,y,d)-mm.minv)/mm.dv*255);
+        } else {
+          var dval = Math.floor((A.get(x,y,d)-mm.minv)/mm.dv*255);  
+        }
+        for(var dx=0;dx<s;dx++) {
+          for(var dy=0;dy<s;dy++) {
+            var pp = ((W * (y*s+dy)) + (dx + x*s)) * 4;
+            for(var i=0;i<3;i++) { g.data[pp + i] = dval; } // rgb
+            g.data[pp+3] = 255; // alpha channel
+          }
+        }
+      }
+    }
+    ctx.putImageData(g, 0, 0);
+    elt.appendChild(canv);
+  }  
+}
+
+var draw_activations_COLOR = function(elt, A, scale, grads) {
+
+  var s = scale || 2; // scale
+  var draw_grads = false;
+  if(typeof(grads) !== 'undefined') draw_grads = grads;
+
+  // get max and min activation to scale the maps automatically
+  var w = draw_grads ? A.dw : A.w;
+  var mm = maxmin(w);
+
+  var canv = document.createElement('canvas');
+  canv.className = 'actmap';
+  var W = A.sx * s;
+  var H = A.sy * s;
+  canv.width = W;
+  canv.height = H;
+  var ctx = canv.getContext('2d');
+  var g = ctx.createImageData(W, H);
+  for(var d=0;d<3;d++) {
+    for(var x=0;x<A.sx;x++) {
+      for(var y=0;y<A.sy;y++) {
+        if(draw_grads) {
+          var dval = Math.floor((A.get_grad(x,y,d)-mm.minv)/mm.dv*255);
+        } else {
+          var dval = Math.floor((A.get(x,y,d)-mm.minv)/mm.dv*255);  
+        }
+        for(var dx=0;dx<s;dx++) {
+          for(var dy=0;dy<s;dy++) {
+            var pp = ((W * (y*s+dy)) + (dx + x*s)) * 4;
+            g.data[pp + d] = dval;
+            if(d===0) g.data[pp+3] = 255; // alpha channel
+          }
+        }
+      }
+    }
+  }
+  ctx.putImageData(g, 0, 0);
+  elt.appendChild(canv);
+}
+
+var visualize_activations = function(net, elt) {
+
+  // clear the element
+  elt.innerHTML = "";
+
+  // show activations in each layer
+  var N = net.layers.length;
+  for(var i=0;i<N;i++) {
+    var L = net.layers[i];
+
+    var layer_div = document.createElement('div');
+
+    // visualize activations
+    var activations_div = document.createElement('div');
+    activations_div.appendChild(document.createTextNode('Activations:'));
+    activations_div.appendChild(document.createElement('br'));
+    activations_div.className = 'layer_act';
+    var scale = 2;
+    if(L.layer_type==='softmax' || L.layer_type==='fc') scale = 10; // for softmax
+    
+    // HACK to draw in color in input layer
+    if(i===0) {
+      draw_activations_COLOR(activations_div, L.out_act, scale);
+      draw_activations_COLOR(activations_div, L.out_act, scale, true);
+
+      /*
+      // visualize positive and negative components of the gradient separately
+      var dd = L.out_act.clone();
+      var ni = L.out_act.w.length;
+      for(var q=0;q<ni;q++) { var dwq = L.out_act.dw[q]; dd.w[q] = dwq > 0 ? dwq : 0.0; }
+      draw_activations_COLOR(activations_div, dd, scale);
+      for(var q=0;q<ni;q++) { var dwq = L.out_act.dw[q]; dd.w[q] = dwq < 0 ? -dwq : 0.0; }
+      draw_activations_COLOR(activations_div, dd, scale);
+      */
+
+      /*
+      // visualize what the network would like the image to look like more
+      var dd = L.out_act.clone();
+      var ni = L.out_act.w.length;
+      for(var q=0;q<ni;q++) { var dwq = L.out_act.dw[q]; dd.w[q] -= 20*dwq; }
+      draw_activations_COLOR(activations_div, dd, scale);
+      */
+
+      /*
+      // visualize gradient magnitude
+      var dd = L.out_act.clone();
+      var ni = L.out_act.w.length;
+      for(var q=0;q<ni;q++) { var dwq = L.out_act.dw[q]; dd.w[q] = dwq*dwq; }
+      draw_activations_COLOR(activations_div, dd, scale);
+      */
+
+    } else {
+      draw_activations(activations_div, L.out_act, scale);
+    } 
+
+    // visualize data gradients
+    if(L.layer_type !== 'softmax' && L.layer_type !== 'input' ) {
+      var grad_div = document.createElement('div');
+      grad_div.appendChild(document.createTextNode('Activation Gradients:'));
+      grad_div.appendChild(document.createElement('br'));
+      grad_div.className = 'layer_grad';
+      var scale = 2;
+      if(L.layer_type==='softmax' || L.layer_type==='fc') scale = 10; // for softmax
+      draw_activations(grad_div, L.out_act, scale, true);
+      activations_div.appendChild(grad_div);
+    }
+
+    // visualize filters if they are of reasonable size
+    if(L.layer_type === 'conv') {
+      var filters_div = document.createElement('div');
+      if(L.filters[0].sx>3) {
+        // actual weights
+        filters_div.appendChild(document.createTextNode('Weights:'));
+        filters_div.appendChild(document.createElement('br'));
+        for(var j=0;j<L.filters.length;j++) {
+          // HACK to draw in color for first layer conv filters
+          if(i===1) {
+            draw_activations_COLOR(filters_div, L.filters[j], 2);
+          } else {
+            filters_div.appendChild(document.createTextNode('('));
+            draw_activations(filters_div, L.filters[j], 2);
+            filters_div.appendChild(document.createTextNode(')'));
+          }
+        }
+        // gradients
+        filters_div.appendChild(document.createElement('br'));
+        filters_div.appendChild(document.createTextNode('Weight Gradients:'));
+        filters_div.appendChild(document.createElement('br'));
+        for(var j=0;j<L.filters.length;j++) {
+          if(i===1) { draw_activations_COLOR(filters_div, L.filters[j], 2, true); }
+          else {
+            filters_div.appendChild(document.createTextNode('('));
+            draw_activations(filters_div, L.filters[j], 2, true);
+            filters_div.appendChild(document.createTextNode(')'));
+          }
+        }
+      } else {
+        filters_div.appendChild(document.createTextNode('Weights hidden, too small'));
+      }
+      activations_div.appendChild(filters_div);
+    }
+    layer_div.appendChild(activations_div);
+
+    // print some stats on left of the layer
+    layer_div.className = 'layer ' + 'lt' + L.layer_type;
+    var title_div = document.createElement('div');
+    title_div.className = 'ltitle'
+    var t = L.layer_type + ' (' + L.out_sx + 'x' + L.out_sy + 'x' + L.out_depth + ')';
+    title_div.appendChild(document.createTextNode(t));
+    layer_div.appendChild(title_div);
+
+    if(L.layer_type==='conv') {
+      var t = 'filter size ' + L.filters[0].sx + 'x' + L.filters[0].sy + 'x' + L.filters[0].depth + ', stride ' + L.stride;
+      layer_div.appendChild(document.createTextNode(t));
+      layer_div.appendChild(document.createElement('br'));
+    }
+    if(L.layer_type==='pool') {
+      var t = 'pooling size ' + L.sx + 'x' + L.sy + ', stride ' + L.stride;
+      layer_div.appendChild(document.createTextNode(t));
+      layer_div.appendChild(document.createElement('br'));
+    }
+
+    // find min, max activations and display them
+    var mma = maxmin(L.out_act.w);
+    var t = 'max activation: ' + f2t(mma.maxv) + ', min: ' + f2t(mma.minv);
+    layer_div.appendChild(document.createTextNode(t));
+    layer_div.appendChild(document.createElement('br'));
+
+    var mma = maxmin(L.out_act.dw);
+    var t = 'max gradient: ' + f2t(mma.maxv) + ', min: ' + f2t(mma.minv);
+    layer_div.appendChild(document.createTextNode(t));
+    layer_div.appendChild(document.createElement('br'));
+
+    // number of parameters
+    if(L.layer_type==='conv' || L.layer_type==='local') {
+      var tot_params = L.sx*L.sy*L.in_depth*L.filters.length + L.filters.length;
+      var t = 'parameters: ' + L.filters.length + 'x' + L.sx + 'x' + L.sy + 'x' + L.in_depth + '+' + L.filters.length + ' = ' + tot_params;
+      layer_div.appendChild(document.createTextNode(t));
+      layer_div.appendChild(document.createElement('br'));
+    }
+    if(L.layer_type==='fc') {
+      var tot_params = L.num_inputs*L.filters.length + L.filters.length;
+      var t = 'parameters: ' + L.filters.length + 'x' + L.num_inputs + '+' + L.filters.length + ' = ' + tot_params;
+      layer_div.appendChild(document.createTextNode(t));
+      layer_div.appendChild(document.createElement('br'));
+    }
+
+    // css madness needed here...
+    var clear = document.createElement('div');
+    clear.className = 'clear';
+    layer_div.appendChild(clear);
+
+    elt.appendChild(layer_div);
+  }
+}
+
+// loads a training image and trains on it with the network
+var paused = false;
+var load_and_step = function() {
+  if(paused) return; 
+
+  var sample = sample_training_instance();
+  step(sample); // process this image
+  
+  //setTimeout(load_and_step, 0); // schedule the next iteration
+}
+
+// evaluate current network on test set
+var test_predict = function() {
+  var num_classes = net.layers[net.layers.length-1].out_depth;
+
+  document.getElementById('testset_acc').innerHTML = '';
+  var num_total = 0;
+  var num_correct = 0;
+
+  // grab a random test image
+  for(num=0;num<4;num++) {
+    var sample = sample_test_instance();
+    var y = sample.label;  // ground truth label
+
+    // forward prop it through the network
+    var aavg = new convnetjs.Vol(1,1,num_classes,0.0);
+    // ensures we always have a list, regardless if above returns single item or list
+    var xs = [].concat(sample.x);
+    var n = xs.length;
+    for(var i=0;i<n;i++) {
+      var a = net.forward(xs[i]);
+      aavg.addFrom(a);
+    }
+    var preds = [];
+    for(var k=0;k<aavg.w.length;k++) { preds.push({k:k,p:aavg.w[k]}); }
+    preds.sort(function(a,b){return a.p<b.p ? 1:-1;});
+    
+    var correct = preds[0].k===y;
+    if(correct) num_correct++;
+    num_total++;
+
+    var div = document.createElement('div');
+    div.className = 'testdiv';
+
+    // draw the image into a canvas
+    draw_activations_COLOR(div, xs[0], 2); // draw Vol into canv
+
+    // add predictions
+    var probsdiv = document.createElement('div');
+    
+    var t = '';
+    for(var k=0;k<3;k++) {
+      var col = preds[k].k===y ? 'rgb(85,187,85)' : 'rgb(187,85,85)';
+      t += '<div class=\"pp\" style=\"width:' + Math.floor(preds[k].p/n*100) + 'px; background-color:' + col + ';\">' + classes_txt[preds[k].k] + '</div>'
+    }
+    probsdiv.innerHTML = t;
+    probsdiv.className = 'probsdiv';
+    div.appendChild(probsdiv);
+
+    // add it into DOM
+    $(div).prependTo($("#testset_vis")).hide().fadeIn('slow').slideDown('slow');
+    if($(".probsdiv").length>200) {
+      $("#testset_vis > .probsdiv").last().remove(); // pop to keep upper bound of shown items
+    }
+  }
+  testAccWindow.add(num_correct/num_total);
+  $("#testset_acc").text('test accuracy based on last 200 test images: ' + testAccWindow.get_average());  
+}
+var testImage = function(img) {
+  var x = convnetjs.img_to_vol(img);
+  var out_p = net.forward(x);
+
+
+  var vis_elt = document.getElementById("visnet");
+  visualize_activations(net, vis_elt);
+
+  var preds =[]
+  for(var k=0;k<out_p.w.length;k++) { preds.push({k:k,p:out_p.w[k]}); }
+  preds.sort(function(a,b){return a.p<b.p ? 1:-1;});
+
+  // add predictions
+  var div = document.createElement('div');
+  div.className = 'testdiv';
+
+  // draw the image into a canvas
+  draw_activations_COLOR(div, x, 2);
+
+  var probsdiv = document.createElement('div');
+
+
+  var t = '';
+  for(var k=0;k<3;k++) {
+    var col = k===0 ? 'rgb(85,187,85)' : 'rgb(187,85,85)';
+    t += '<div class=\"pp\" style=\"width:' + Math.floor(preds[k].p/1*100) + 'px; background-color:' + col + ';\">' + classes_txt[preds[k].k] + '</div>'
+  }
+  
+  probsdiv.innerHTML = t;
+  probsdiv.className = 'probsdiv';
+  div.appendChild(probsdiv);
+
+  // add it into DOM
+  $(div).prependTo($("#testset_vis")).hide().fadeIn('slow').slideDown('slow');
+  if($(".probsdiv").length>200) {
+    $("#testset_vis > .probsdiv").last().remove(); // pop to keep upper bound of shown items
+  }
+}
+
+var lossGraph = new cnnvis.Graph();
+var xLossWindow = new cnnutil.Window(100);
+var wLossWindow = new cnnutil.Window(100);
+var trainAccWindow = new cnnutil.Window(100);
+var valAccWindow = new cnnutil.Window(100);
+var testAccWindow = new cnnutil.Window(50, 1);
+var step_num = 0;
+var step = function(sample) {
+
+  var x = sample.x;
+  var y = sample.label;
+
+  if(sample.isval) {
+    // use x to build our estimate of validation error
+    net.forward(x);
+    var yhat = net.getPrediction();
+    var val_acc = yhat === y ? 1.0 : 0.0;
+    valAccWindow.add(val_acc);
+    return; // get out
+  }
+
+  // train on it with network
+  var stats = trainer.train(x, y);
+  var lossx = stats.cost_loss;
+  var lossw = stats.l2_decay_loss;
+
+  // keep track of stats such as the average training error and loss
+  var yhat = net.getPrediction();
+  var train_acc = yhat === y ? 1.0 : 0.0;
+  xLossWindow.add(lossx);
+  wLossWindow.add(lossw);
+  trainAccWindow.add(train_acc);
+
+  // visualize training status
+  var train_elt = document.getElementById("trainstats");
+  train_elt.innerHTML = '';
+  var t = 'Forward time per example: ' + stats.fwd_time + 'ms';
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'Backprop time per example: ' + stats.bwd_time + 'ms';
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'Classification loss: ' + f2t(xLossWindow.get_average());
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'L2 Weight decay loss: ' + f2t(wLossWindow.get_average());
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'Training accuracy: ' + f2t(trainAccWindow.get_average());
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'Validation accuracy: ' + f2t(valAccWindow.get_average());
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+  var t = 'Examples seen: ' + step_num;
+  train_elt.appendChild(document.createTextNode(t));
+  train_elt.appendChild(document.createElement('br'));
+
+  // visualize activations
+  if(step_num % 100 === 0) {
+    var vis_elt = document.getElementById("visnet");
+    visualize_activations(net, vis_elt);
+  }
+
+  // log progress to graph, (full loss)
+  if(step_num % 200 === 0) {
+    var xa = xLossWindow.get_average();
+    var xw = wLossWindow.get_average();
+    if(xa >= 0 && xw >= 0) { // if they are -1 it means not enough data was accumulated yet for estimates
+      lossGraph.add(step_num, xa + xw);
+      lossGraph.drawSelf(document.getElementById("lossgraph"));
+    }
+  }
+
+  // run prediction on test set
+  if((step_num % 100 === 0 && step_num > 0) || step_num===100) {
+    test_predict();
+  }
+  step_num++;
+}
+
+// user settings 
+var change_lr = function() {
+  trainer.learning_rate = parseFloat(document.getElementById("lr_input").value);
+  update_net_param_display();
+}
+var change_momentum = function() {
+  trainer.momentum = parseFloat(document.getElementById("momentum_input").value);
+  update_net_param_display();
+}
+var change_batch_size = function() {
+  trainer.batch_size = parseFloat(document.getElementById("batch_size_input").value);
+  update_net_param_display();
+}
+var change_decay = function() {
+  trainer.l2_decay = parseFloat(document.getElementById("decay_input").value);
+  update_net_param_display();
+}
+var update_net_param_display = function() {
+  document.getElementById('lr_input').value = trainer.learning_rate;
+  document.getElementById('momentum_input').value = trainer.momentum;
+  document.getElementById('batch_size_input').value = trainer.batch_size;
+  document.getElementById('decay_input').value = trainer.l2_decay;
+}
+var toggle_pause = function() {
+  paused = !paused;
+  var btn = document.getElementById('buttontp');
+  if(paused) { btn.value = 'resume' }
+  else { btn.value = 'pause'; }
+}
+var dump_json = function() {
+  document.getElementById("dumpjson").value = JSON.stringify(this.net.toJSON());
+}
+var clear_graph = function() {
+  lossGraph = new cnnvis.Graph(); // reinit graph too 
+}
+var reset_all = function() {
+  // reinit trainer
+  trainer = new convnetjs.SGDTrainer(net, {learning_rate:trainer.learning_rate, momentum:trainer.momentum, batch_size:trainer.batch_size, l2_decay:trainer.l2_decay});
+  update_net_param_display();
+
+  // reinit windows that keep track of val/train accuracies
+  xLossWindow.reset();
+  wLossWindow.reset();
+  trainAccWindow.reset();
+  valAccWindow.reset();
+  testAccWindow.reset();
+  lossGraph = new cnnvis.Graph(); // reinit graph too
+  step_num = 0;
+}
+var load_from_json = function() {
+  var jsonString = document.getElementById("dumpjson").value;
+  var json = JSON.parse(jsonString);
+  net = new convnetjs.Net();
+  net.fromJSON(json);
+  reset_all();
+}
+
+var load_pretrained = function() {
+  $.getJSON(dataset_name + "_snapshot.json", function(json){
+    net = new convnetjs.Net();
+    net.fromJSON(json);
+    trainer.learning_rate = 0.0001;
+    trainer.momentum = 0.9;
+    trainer.batch_size = 2;
+    trainer.l2_decay = 0.00001;
+    reset_all();
+  });
+}
+
+var change_net = function() {
+  eval($("#newnet").val());
+  reset_all();
+}

demo/js/npgmain.js

@@ -0,0 +1,134 @@
+//Simple game engine
+//Author: Andrej Karpathy
+//License: BSD
+//This function does all the boring canvas stuff. To use it, just create functions:
+//update()          gets called every frame
+//draw()            gets called every frame
+//myinit()          gets called once in beginning
+//mouseClick(x, y)  gets called on mouse click
+//keyUp(keycode)    gets called when key is released
+//keyDown(keycode)  gets called when key is pushed
+
+var canvas;
+var ctx;
+var WIDTH;
+var HEIGHT; 
+var FPS;
+
+function drawBubble(x, y, w, h, radius)
+{
+  var r = x + w;
+  var b = y + h;
+  ctx.beginPath();
+  ctx.strokeStyle="black";
+  ctx.lineWidth="2";
+  ctx.moveTo(x+radius, y);
+  ctx.lineTo(x+radius/2, y-10);
+  ctx.lineTo(x+radius * 2, y);
+  ctx.lineTo(r-radius, y);
+  ctx.quadraticCurveTo(r, y, r, y+radius);
+  ctx.lineTo(r, y+h-radius);
+  ctx.quadraticCurveTo(r, b, r-radius, b);
+  ctx.lineTo(x+radius, b);
+  ctx.quadraticCurveTo(x, b, x, b-radius);
+  ctx.lineTo(x, y+radius);
+  ctx.quadraticCurveTo(x, y, x+radius, y);
+  ctx.stroke();
+}
+
+function drawRect(x, y, w, h){
+  ctx.beginPath();
+  ctx.rect(x,y,w,h);
+  ctx.closePath();
+  ctx.fill();
+  ctx.stroke();
+}
+
+function drawCircle(x, y, r){
+  ctx.beginPath();
+  ctx.arc(x, y, r, 0, Math.PI*2, true); 
+  ctx.closePath();
+  ctx.stroke();
+  ctx.fill();
+}
+
+//uniform distribution integer
+function randi(s, e) {
+  return Math.floor(Math.random()*(e-s) + s);
+}
+
+//uniform distribution
+function randf(s, e) {
+  return Math.random()*(e-s) + s;
+}
+
+//normal distribution random number
+function randn(mean, variance) {
+  var V1, V2, S;
+  do {
+    var U1 = Math.random();
+    var U2 = Math.random();
+    V1 = 2 * U1 - 1;
+    V2 = 2 * U2 - 1;
+    S = V1 * V1 + V2 * V2;
+  } while (S > 1);
+  X = Math.sqrt(-2 * Math.log(S) / S) * V1;
+  X = mean + Math.sqrt(variance) * X;
+  return X;
+}
+
+function eventClick(e) {
+    
+  //get position of cursor relative to top left of canvas
+  var x;
+  var y;
+  if (e.pageX || e.pageY) { 
+    x = e.pageX;
+    y = e.pageY;
+  } else { 
+    x = e.clientX + document.body.scrollLeft + document.documentElement.scrollLeft; 
+    y = e.clientY + document.body.scrollTop + document.documentElement.scrollTop; 
+  } 
+  x -= canvas.offsetLeft;
+  y -= canvas.offsetTop;
+  
+  //call user-defined callback
+  mouseClick(x, y, e.shiftKey, e.ctrlKey);
+}
+
+//event codes can be found here:
+//http://www.aspdotnetfaq.com/Faq/What-is-the-list-of-KeyCodes-for-JavaScript-KeyDown-KeyPress-and-KeyUp-events.aspx
+function eventKeyUp(e) {
+  var keycode = ('which' in e) ? e.which : e.keyCode;
+  keyUp(keycode);
+}
+
+function eventKeyDown(e) {
+  var keycode = ('which' in e) ? e.which : e.keyCode;
+  keyDown(keycode);
+}
+
+function NPGinit(FPS){
+  //takes frames per secont to run at
+  
+  canvas = document.getElementById('NPGcanvas');
+  ctx = canvas.getContext('2d');
+  WIDTH = canvas.width;
+  HEIGHT = canvas.height;
+  canvas.addEventListener('click', eventClick, false);
+  
+  //canvas element cannot get focus by default. Requires to either set 
+  //tabindex to 1 so that it's focusable, or we need to attach listeners
+  //to the document. Here we do the latter
+  document.addEventListener('keyup', eventKeyUp, true);
+  document.addEventListener('keydown', eventKeyDown, true);
+  
+  setInterval(NPGtick, 1000/FPS);
+  
+  myinit();
+}
+
+function NPGtick() {
+    update();
+    draw();
+}

demo/js/pica.js

@@ -0,0 +1,709 @@
+/* pica 1.0.7 nodeca/pica */!function(e){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=e();else if("function"==typeof define&&define.amd)define([],e);else{var f;"undefined"!=typeof window?f=window:"undefined"!=typeof global?f=global:"undefined"!=typeof self&&(f=self),f.pica=e()}}(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({"./":[function(require,module,exports){
+'use strict';
+
+/*global window:true*/
+/*eslint space-infix-ops:0*/
+
+// Feature detect
+var WORKER = (typeof window !== 'undefined') && ('Worker' in window);
+if (WORKER) {
+  // IE don't allow to create webworkers from string. We should check it.
+  // https://connect.microsoft.com/IE/feedback/details/801810/web-workers-from-blob-urls-in-ie-10-and-11
+  try {
+    var wkr = require('webworkify')(function () {});
+    wkr.terminate();
+  } catch (__) {
+    WORKER = false;
+  }
+}
+
+var resize       = require('./lib/resize');
+var resizeWorker = require('./lib/resize_worker');
+
+
+////////////////////////////////////////////////////////////////////////////////
+// Helpers
+function _class(obj) { return Object.prototype.toString.call(obj); }
+function isFunction(obj) { return _class(obj) === '[object Function]'; }
+
+
+////////////////////////////////////////////////////////////////////////////////
+// API methods
+
+
+// RGBA buffer async resize
+//
+function resizeBuffer(options, callback) {
+  var wr;
+
+  var _opts = {
+    src:      options.src,
+    dest:     null,
+    width:    options.width|0,
+    height:   options.height|0,
+    toWidth:  options.toWidth|0,
+    toHeight: options.toHeight|0,
+    quality:  options.quality,
+    alpha:    options.alpha,
+    unsharpAmount:    options.unsharpAmount,
+    unsharpThreshold: options.unsharpThreshold
+  };
+
+  if (WORKER && exports.WW) {
+    wr = require('webworkify')(resizeWorker);
+
+    wr.onmessage = function(ev) {
+      var i, l,
+          dest = options.dest,
+          output = ev.data.output;
+
+      // If we got output buffer by reference, we should copy data,
+      // because WW returns independent instance
+      if (dest) {
+        // IE ImageData can return old-style CanvasPixelArray
+        // without .set() method. Copy manually for such case.
+        if (dest.set) {
+          dest.set(output);
+        } else {
+          for (i = 0, l = output.length; i < l; i++) {
+            dest[i] = output[i];
+          }
+        }
+      }
+      callback(ev.data.err, output);
+      wr.terminate();
+    };
+
+    if (options.transferable) {
+      wr.postMessage(_opts, [ options.src.buffer ]);
+    } else {
+      wr.postMessage(_opts);
+    }
+    // Expose worker when available, to allow early termination.
+    return wr;
+  }
+
+  // Fallback to sync call, if WebWorkers not available
+  _opts.dest = options.dest;
+  resize(_opts, callback);
+  return undefined;
+}
+
+
+// Canvas async resize
+//
+function resizeCanvas(from, to, options, callback) {
+  var w = from.width,
+      h = from.height,
+      w2 = to.width,
+      h2 = to.height;
+
+  if (isFunction(options)) {
+    callback = options;
+    options = {};
+  }
+
+  if (!isNaN(options)) {
+    options = { quality: options, alpha: false };
+  }
+
+  var ctxTo = to.getContext('2d');
+  var imageDataTo = ctxTo.getImageData(0, 0, w2, h2);
+
+  var _opts = {
+    src:      from.getContext('2d').getImageData(0, 0, w, h).data,
+    dest:     imageDataTo.data,
+    width:    from.width,
+    height:   from.height,
+    toWidth:  to.width,
+    toHeight: to.height,
+    quality:  options.quality,
+    alpha:    options.alpha,
+    unsharpAmount:    options.unsharpAmount,
+    unsharpThreshold: options.unsharpThreshold,
+    transferable: true
+  };
+
+  return resizeBuffer(_opts, function (err/*, output*/) {
+    if (err) {
+      callback(err);
+      return;
+    }
+
+    ctxTo.putImageData(imageDataTo, 0, 0);
+    callback();
+  });
+}
+
+
+exports.resizeBuffer = resizeBuffer;
+exports.resizeCanvas = resizeCanvas;
+exports.WW = WORKER;
+
+},{"./lib/resize":4,"./lib/resize_worker":5,"webworkify":6}],1:[function(require,module,exports){
+// Blur filter
+//
+
+'use strict';
+
+
+var _blurKernel = new Uint8Array([
+  1, 2, 1,
+  2, 4, 2,
+  1, 2, 1
+]);
+
+var _bkWidth = Math.floor(Math.sqrt(_blurKernel.length));
+var _bkHalf = Math.floor(_bkWidth / 2);
+var _bkWsum = 0;
+for (var wc=0; wc < _blurKernel.length; wc++) { _bkWsum += _blurKernel[wc]; }
+
+
+function blurPoint(gs, x, y, srcW, srcH) {
+  var bx, by, sx, sy, w, wsum, br;
+  var bPtr = 0;
+  var blurKernel = _blurKernel;
+  var bkHalf = _bkHalf;
+
+  wsum = 0; // weight sum to normalize result
+  br   = 0;
+
+  if (x >= bkHalf && y >= bkHalf && x + bkHalf < srcW && y + bkHalf < srcH) {
+    for (by = 0; by < 3; by++) {
+      for (bx = 0; bx < 3; bx++) {
+        sx = x + bx - bkHalf;
+        sy = y + by - bkHalf;
+
+        br += gs[sx + sy * srcW] * blurKernel[bPtr++];
+      }
+    }
+    return (br - (br % _bkWsum)) / _bkWsum;
+  }
+
+  for (by = 0; by < 3; by++) {
+    for (bx = 0; bx < 3; bx++) {
+      sx = x + bx - bkHalf;
+      sy = y + by - bkHalf;
+
+      if (sx >= 0 && sx < srcW && sy >= 0 && sy < srcH) {
+        w = blurKernel[bPtr];
+        wsum += w;
+        br += gs[sx + sy * srcW] * w;
+      }
+      bPtr++;
+    }
+  }
+  return ((br - (br % wsum)) / wsum)|0;
+}
+
+function blur(src, srcW, srcH/*, radius*/) {
+  var x, y,
+      output = new Uint16Array(src.length);
+
+  for (x = 0; x < srcW; x++) {
+    for (y = 0; y < srcH; y++) {
+      output[y * srcW + x] = blurPoint(src, x, y, srcW, srcH);
+    }
+  }
+
+  return output;
+}
+
+module.exports = blur;
+
+},{}],2:[function(require,module,exports){
+// High speed resize with tuneable speed/quality ratio
+
+'use strict';
+
+
+var unsharp = require('./unsharp');
+
+
+// Precision of fixed FP values
+var FIXED_FRAC_BITS = 14;
+var FIXED_FRAC_VAL  = 1 << FIXED_FRAC_BITS;
+
+
+//
+// Presets for quality 0..3. Filter functions + window size
+//
+var FILTER_INFO = [
+  { // Nearest neibor (Box)
+    win: 0.5,
+    filter: function (x) {
+      return (x >= -0.5 && x < 0.5) ? 1.0 : 0.0;
+    }
+  },
+  { // Hamming
+    win: 1.0,
+    filter: function (x) {
+      if (x <= -1.0 || x >= 1.0) { return 0.0; }
+      if (x > -1.19209290E-07 && x < 1.19209290E-07) { return 1.0; }
+      var xpi = x * Math.PI;
+      return ((Math.sin(xpi) / xpi) *  (0.54 + 0.46 * Math.cos(xpi / 1.0)));
+    }
+  },
+  { // Lanczos, win = 2
+    win: 2.0,
+    filter: function (x) {
+      if (x <= -2.0 || x >= 2.0) { return 0.0; }
+      if (x > -1.19209290E-07 && x < 1.19209290E-07) { return 1.0; }
+      var xpi = x * Math.PI;
+      return (Math.sin(xpi) / xpi) * Math.sin(xpi / 2.0) / (xpi / 2.0);
+    }
+  },
+  { // Lanczos, win = 3
+    win: 3.0,
+    filter: function (x) {
+      if (x <= -3.0 || x >= 3.0) { return 0.0; }
+      if (x > -1.19209290E-07 && x < 1.19209290E-07) { return 1.0; }
+      var xpi = x * Math.PI;
+      return (Math.sin(xpi) / xpi) * Math.sin(xpi / 3.0) / (xpi / 3.0);
+    }
+  }
+];
+
+function clampTo8(i) { return i < 0 ? 0 : (i > 255 ? 255 : i); }
+
+function toFixedPoint(num) { return Math.floor(num * FIXED_FRAC_VAL); }
+
+
+// Calculate convolution filters for each destination point,
+// and pack data to Int16Array:
+//
+// [ shift, length, data..., shift2, length2, data..., ... ]
+//
+// - shift - offset in src image
+// - length - filter length (in src points)
+// - data - filter values sequence
+//
+function createFilters(quality, srcSize, destSize) {
+
+  var filterFunction = FILTER_INFO[quality].filter;
+
+  var scale         = destSize / srcSize;
+  var scaleInverted = 1.0 / scale;
+  var scaleClamped  = Math.min(1.0, scale); // For upscale
+
+  // Filter window (averaging interval), scaled to src image
+  var srcWindow = FILTER_INFO[quality].win / scaleClamped;
+
+  var destPixel, srcPixel, srcFirst, srcLast, filterElementSize,
+      floatFilter, fxpFilter, total, fixedTotal, pxl, idx, floatVal, fixedVal;
+  var leftNotEmpty, rightNotEmpty, filterShift, filterSize;
+
+  var maxFilterElementSize = Math.floor((srcWindow + 1) * 2 );
+  var packedFilter    = new Int16Array((maxFilterElementSize + 2) * destSize);
+  var packedFilterPtr = 0;
+
+  // For each destination pixel calculate source range and built filter values
+  for (destPixel = 0; destPixel < destSize; destPixel++) {
+
+    // Scaling should be done relative to central pixel point
+    srcPixel = (destPixel + 0.5) * scaleInverted;
+
+    srcFirst = Math.max(0, Math.floor(srcPixel - srcWindow));
+    srcLast  = Math.min(srcSize - 1, Math.ceil(srcPixel + srcWindow));
+
+    filterElementSize = srcLast - srcFirst + 1;
+    floatFilter = new Float32Array(filterElementSize);
+    fxpFilter = new Int16Array(filterElementSize);
+
+    total = 0.0;
+
+    // Fill filter values for calculated range
+    for (pxl = srcFirst, idx = 0; pxl <= srcLast; pxl++, idx++) {
+      floatVal = filterFunction(((pxl + 0.5) - srcPixel) * scaleClamped);
+      total += floatVal;
+      floatFilter[idx] = floatVal;
+    }
+
+    // Normalize filter, convert to fixed point and accumulate conversion error
+    fixedTotal = 0;
+
+    for (idx = 0; idx < floatFilter.length; idx++) {
+      fixedVal = toFixedPoint(floatFilter[idx] / total);
+      fixedTotal += fixedVal;
+      fxpFilter[idx] = fixedVal;
+    }
+
+    // Compensate normalization error, to minimize brightness drift
+    fxpFilter[destSize >> 1] += toFixedPoint(1.0) - fixedTotal;
+
+    //
+    // Now pack filter to useable form
+    //
+    // 1. Trim heading and tailing zero values, and compensate shitf/length
+    // 2. Put all to single array in this format:
+    //
+    //    [ pos shift, data length, value1, value2, value3, ... ]
+    //
+
+    leftNotEmpty = 0;
+    while (leftNotEmpty < fxpFilter.length && fxpFilter[leftNotEmpty] === 0) {
+      leftNotEmpty++;
+    }
+
+    if (leftNotEmpty < fxpFilter.length) {
+      rightNotEmpty = fxpFilter.length - 1;
+      while (rightNotEmpty > 0 && fxpFilter[rightNotEmpty] === 0) {
+        rightNotEmpty--;
+      }
+
+      filterShift = srcFirst + leftNotEmpty;
+      filterSize = rightNotEmpty - leftNotEmpty + 1;
+
+      packedFilter[packedFilterPtr++] = filterShift; // shift
+      packedFilter[packedFilterPtr++] = filterSize; // size
+
+      packedFilter.set(fxpFilter.subarray(leftNotEmpty, rightNotEmpty + 1), packedFilterPtr);
+      packedFilterPtr += filterSize;
+    } else {
+      // zero data, write header only
+      packedFilter[packedFilterPtr++] = 0; // shift
+      packedFilter[packedFilterPtr++] = 0; // size
+    }
+  }
+  return packedFilter;
+}
+
+// Convolve image in horizontal directions and transpose output. In theory,
+// transpose allow:
+//
+// - use the same convolver for both passes (this fails due different
+//   types of input array and temporary buffer)
+// - making vertical pass by horisonltal lines inprove CPU cache use.
+//
+// But in real life this doesn't work :)
+//
+function convolveHorizontally(src, dest, srcW, srcH, destW, filters) {
+
+  var r, g, b, a;
+  var filterPtr, filterShift, filterSize;
+  var srcPtr, srcY, destX, filterVal;
+  var srcOffset = 0, destOffset = 0;
+
+  // For each row
+  for (srcY = 0; srcY < srcH; srcY++) {
+    filterPtr  = 0;
+
+    // Apply precomputed filters to each destination row point
+    for (destX = 0; destX < destW; destX++) {
+      // Get the filter that determines the current output pixel.
+      filterShift = filters[filterPtr++];
+      filterSize  = filters[filterPtr++];
+
+      srcPtr = (srcOffset + (filterShift * 4))|0;
+
+      r = g = b = a = 0;
+
+      // Apply the filter to the row to get the destination pixel r, g, b, a
+      for (; filterSize > 0; filterSize--) {
+        filterVal = filters[filterPtr++];
+
+        // Use reverse order to workaround deopts in old v8 (node v.10)
+        // Big thanks to @mraleph (Vyacheslav Egorov) for the tip.
+        a = (a + filterVal * src[srcPtr + 3])|0;
+        b = (b + filterVal * src[srcPtr + 2])|0;
+        g = (g + filterVal * src[srcPtr + 1])|0;
+        r = (r + filterVal * src[srcPtr])|0;
+        srcPtr = (srcPtr + 4)|0;
+      }
+
+      // Bring this value back in range. All of the filter scaling factors
+      // are in fixed point with FIXED_FRAC_BITS bits of fractional part.
+      dest[destOffset + 3] = clampTo8(a >> 14/*FIXED_FRAC_BITS*/);
+      dest[destOffset + 2] = clampTo8(b >> 14/*FIXED_FRAC_BITS*/);
+      dest[destOffset + 1] = clampTo8(g >> 14/*FIXED_FRAC_BITS*/);
+      dest[destOffset]     = clampTo8(r >> 14/*FIXED_FRAC_BITS*/);
+      destOffset = (destOffset + srcH * 4)|0;
+    }
+
+    destOffset = ((srcY + 1) * 4)|0;
+    srcOffset  = ((srcY + 1) * srcW * 4)|0;
+  }
+}
+
+// Technically, convolvers are the same. But input array and temporary
+// buffer can be of different type (especially, in old browsers). So,
+// keep code in separate functions to avoid deoptimizations & speed loss.
+
+function convolveVertically(src, dest, srcW, srcH, destW, filters) {
+
+  var r, g, b, a;
+  var filterPtr, filterShift, filterSize;
+  var srcPtr, srcY, destX, filterVal;
+  var srcOffset = 0, destOffset = 0;
+
+  // For each row
+  for (srcY = 0; srcY < srcH; srcY++) {
+    filterPtr  = 0;
+
+    // Apply precomputed filters to each destination row point
+    for (destX = 0; destX < destW; destX++) {
+      // Get the filter that determines the current output pixel.
+      filterShift = filters[filterPtr++];
+      filterSize  = filters[filterPtr++];
+
+      srcPtr = (srcOffset + (filterShift * 4))|0;
+
+      r = g = b = a = 0;
+
+      // Apply the filter to the row to get the destination pixel r, g, b, a
+      for (; filterSize > 0; filterSize--) {
+        filterVal = filters[filterPtr++];
+
+        // Use reverse order to workaround deopts in old v8 (node v.10)
+        // Big thanks to @mraleph (Vyacheslav Egorov) for the tip.
+        a = (a + filterVal * src[srcPtr + 3])|0;
+        b = (b + filterVal * src[srcPtr + 2])|0;
+        g = (g + filterVal * src[srcPtr + 1])|0;
+        r = (r + filterVal * src[srcPtr])|0;
+        srcPtr = (srcPtr + 4)|0;
+      }
+
+      // Bring this value back in range. All of the filter scaling factors
+      // are in fixed point with FIXED_FRAC_BITS bits of fractional part.
+      dest[destOffset + 3] = clampTo8(a >> 14/*FIXED_FRAC_BITS*/);
+      dest[destOffset + 2] = clampTo8(b >> 14/*FIXED_FRAC_BITS*/);
+      dest[destOffset + 1] = clampTo8(g >> 14/*FIXED_FRAC_BITS*/);
+      dest[destOffset]     = clampTo8(r >> 14/*FIXED_FRAC_BITS*/);
+      destOffset = (destOffset + srcH * 4)|0;
+    }
+
+    destOffset = ((srcY + 1) * 4)|0;
+    srcOffset  = ((srcY + 1) * srcW * 4)|0;
+  }
+}
+
+
+function resetAlpha(dst, width, height) {
+  var ptr = 3, len = (width * height * 4)|0;
+  while (ptr < len) { dst[ptr] = 0xFF; ptr = (ptr + 4)|0; }
+}
+
+
+function resize(options) {
+  var src   = options.src;
+  var srcW  = options.width;
+  var srcH  = options.height;
+  var destW = options.toWidth;
+  var destH = options.toHeight;
+  var dest  = options.dest || new Uint8Array(destW * destH * 4);
+  var quality = options.quality === undefined ? 3 : options.quality;
+  var alpha = options.alpha || false;
+  var unsharpAmount = options.unsharpAmount === undefined ? 0 : (options.unsharpAmount|0);
+  var unsharpThreshold = options.unsharpThreshold === undefined ? 0 : (options.unsharpThreshold|0);
+
+  if (srcW < 1 || srcH < 1 || destW < 1 || destH < 1) { return []; }
+
+  var filtersX = createFilters(quality, srcW, destW),
+      filtersY = createFilters(quality, srcH, destH);
+
+  var tmp  = new Uint8Array(destW * srcH * 4);
+
+  // To use single function we need src & tmp of the same type.
+  // But src can be CanvasPixelArray, and tmp - Uint8Array. So, keep
+  // vertical and horizontal passes separately to avoid deoptimization.
+
+  convolveHorizontally(src, tmp, srcW, srcH, destW, filtersX);
+  convolveVertically(tmp, dest, srcH, destW, destH, filtersY);
+
+  // That's faster than doing checks in convolver.
+  // !!! Note, canvas data is not premultipled. We don't need other
+  // alpha corrections.
+
+  if (!alpha) {
+    resetAlpha(dest, destW, destH);
+  }
+
+  if (unsharpAmount) {
+    unsharp(dest, destW, destH, unsharpAmount, 1.0, unsharpThreshold);
+  }
+
+  return dest;
+}
+
+
+module.exports = resize;
+
+},{"./unsharp":3}],3:[function(require,module,exports){
+// Unsharp mask filter
+//
+// http://stackoverflow.com/a/23322820/1031804
+// USM(O) = O + (2 * (Amount / 100) * (O - GB))
+// GB - gaussial blur.
+//
+// brightness = 0.299*R + 0.587*G + 0.114*B
+// http://stackoverflow.com/a/596243/1031804
+//
+// To simplify math, normalize brighness mutipliers to 2^16:
+//
+// brightness = (19595*R + 38470*G + 7471*B) / 65536
+
+'use strict';
+
+
+var blur = require('./blur');
+
+
+function clampTo8(i) { return i < 0 ? 0 : (i > 255 ? 255 : i); }
+
+// Convert image to greyscale, 16bits FP result (8.8)
+//
+function greyscale(src, srcW, srcH) {
+  var size = srcW * srcH;
+  var result = new Uint16Array(size); // We don't use sign, but that helps to JIT
+  var i, srcPtr;
+
+  for (i = 0, srcPtr = 0; i < size; i++) {
+    result[i] = (src[srcPtr + 2] * 7471       // blue
+               + src[srcPtr + 1] * 38470      // green
+               + src[srcPtr] * 19595) >>> 8;  // red
+    srcPtr = (srcPtr + 4)|0;
+  }
+
+  return result;
+}
+
+
+// Apply unsharp mask to src
+//
+// NOTE: radius is ignored to simplify gaussian blur calculation
+// on practice we need radius 0.3..2.0. Use 1.0 now.
+//
+function unsharp(src, srcW, srcH, amount, radius, threshold) {
+  var x, y, c, diff = 0, corr, srcPtr;
+
+  // Normalized delta multiplier. Expect that:
+  var AMOUNT_NORM = Math.floor(amount * 256 / 50);
+
+  // Convert to grayscale:
+  //
+  // - prevent color drift
+  // - speedup blur calc
+  //
+  var gs = greyscale(src, srcW, srcH);
+  var blured = blur(gs, srcW, srcH, 1);
+  var fpThreshold = threshold << 8;
+  var gsPtr = 0;
+
+  for (y = 0; y < srcH; y++) {
+    for (x = 0; x < srcW; x++) {
+
+      // calculate brightness blur, difference & update source buffer
+
+      diff = gs[gsPtr] - blured[gsPtr];
+
+      // Update source image if thresold exceeded
+      if (Math.abs(diff) > fpThreshold) {
+        // Calculate correction multiplier
+        corr = 65536 + ((diff * AMOUNT_NORM) >> 8);
+        srcPtr = gsPtr * 4;
+
+        c = src[srcPtr];
+        src[srcPtr++] = clampTo8((c * corr) >> 16);
+        c = src[srcPtr];
+        src[srcPtr++] = clampTo8((c * corr) >> 16);
+        c = src[srcPtr];
+        src[srcPtr] = clampTo8((c * corr) >> 16);
+      }
+
+      gsPtr++;
+
+    } // end row
+  } // end column
+}
+
+
+module.exports = unsharp;
+
+},{"./blur":1}],4:[function(require,module,exports){
+// Proxy to simplify split between webworker/plain calls
+'use strict';
+
+var resize = require('./pure/resize');
+
+module.exports = function (options, callback) {
+  var output = resize(options);
+
+  callback(null, output);
+};
+
+},{"./pure/resize":2}],5:[function(require,module,exports){
+// Web Worker wrapper for image resize function
+
+'use strict';
+
+module.exports = function(self) {
+  var resize = require('./resize');
+
+  self.onmessage = function (ev) {
+    resize(ev.data, function(err, output) {
+      if (err) {
+        self.postMessage({ err: err });
+        return;
+      }
+
+      self.postMessage({ output: output }, [ output.buffer ]);
+    });
+  };
+};
+
+},{"./resize":4}],6:[function(require,module,exports){
+var bundleFn = arguments[3];
+var sources = arguments[4];
+var cache = arguments[5];
+
+var stringify = JSON.stringify;
+
+module.exports = function (fn) {
+    var keys = [];
+    var wkey;
+    var cacheKeys = Object.keys(cache);
+    
+    for (var i = 0, l = cacheKeys.length; i < l; i++) {
+        var key = cacheKeys[i];
+        if (cache[key].exports === fn) {
+            wkey = key;
+            break;
+        }
+    }
+    
+    if (!wkey) {
+        wkey = Math.floor(Math.pow(16, 8) * Math.random()).toString(16);
+        var wcache = {};
+        for (var i = 0, l = cacheKeys.length; i < l; i++) {
+            var key = cacheKeys[i];
+            wcache[key] = key;
+        }
+        sources[wkey] = [
+            Function(['require','module','exports'], '(' + fn + ')(self)'),
+            wcache
+        ];
+    }
+    var skey = Math.floor(Math.pow(16, 8) * Math.random()).toString(16);
+    
+    var scache = {}; scache[wkey] = wkey;
+    sources[skey] = [
+        Function(['require'],'require(' + stringify(wkey) + ')(self)'),
+        scache
+    ];
+    
+    var src = '(' + bundleFn + ')({'
+        + Object.keys(sources).map(function (key) {
+            return stringify(key) + ':['
+                + sources[key][0]
+                + ',' + stringify(sources[key][1]) + ']'
+            ;
+        }).join(',')
+        + '},{},[' + stringify(skey) + '])'
+    ;
+    return new Worker(window.URL.createObjectURL(
+        new Blob([src], { type: 'text/javascript' })
+    ));
+};
+
+},{}]},{},[])("./")
+});

demo/js/regression.js

@@ -0,0 +1,159 @@
+
+var N, data, labels;
+var ss = 30.0; // scale for drawing
+
+var layer_defs, net, trainer;
+
+// create neural net
+var t = "layer_defs = [];\n\
+layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:1});\n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'sigmoid'});\n\
+layer_defs.push({type:'regression', num_neurons:1});\n\
+\n\
+net = new convnetjs.Net();\n\
+net.makeLayers(layer_defs);\n\
+\n\
+trainer = new convnetjs.SGDTrainer(net, {learning_rate:0.01, momentum:0.0, batch_size:1, l2_decay:0.001});\n\
+";
+
+var lix=2; // layer id of layer we'd like to draw outputs of
+function reload() {
+  eval($("#layerdef").val());
+
+  // refresh buttons
+  var t = '';
+  for(var i=1;i<net.layers.length-1;i++) { // ignore input and regression layers (first and last)
+    var butid = "button" + i;
+    t += "<input id=\""+butid+"\" value=\"" + net.layers[i].layer_type +"\" type=\"submit\" onclick=\"updateLix("+i+")\" style=\"width:80px; height: 30px; margin:5px;\";>";
+  }
+  $("#layer_ixes").html(t);
+  $("#button"+lix).css('background-color', '#FFA');
+}
+function updateLix(newlix) {
+  $("#button"+lix).css('background-color', ''); // erase highlight
+  lix = newlix;
+  $("#button"+lix).css('background-color', '#FFA');
+}
+ 
+function regen_data() {
+  N = parseInt($("#num_data").val());
+  data = [];
+  labels = [];
+  for(var i=0;i<N;i++) {
+    var x = Math.random()*10-5;
+    var y = x*Math.sin(x);
+    data.push([x]);
+    labels.push([y]);
+  }
+}
+
+function myinit(){
+  regen_data();
+  $("#layerdef").val(t);
+  reload();
+}
+ 
+function update(){
+  // forward prop the data
+  
+  var netx = new convnetjs.Vol(1,1,1);
+  avloss = 0.0;
+
+  for(var iters=0;iters<50;iters++) {
+    for(var ix=0;ix<N;ix++) {
+      netx.w = data[ix];
+      var stats = trainer.train(netx, labels[ix]);
+      avloss += stats.loss;
+    }
+  }
+  avloss /= N*iters;
+
+}
+
+function draw(){
+    
+    ctx.clearRect(0,0,WIDTH,HEIGHT);
+    ctx.fillStyle = "black";
+
+    var netx = new convnetjs.Vol(1,1,1);
+
+    // draw decisions in the grid
+    var density= 5.0;
+    var draw_neuron_outputs = $("#layer_outs").is(':checked');
+    
+    // draw final decision
+    var neurons = [];
+    ctx.beginPath();
+    for(var x=0.0; x<=WIDTH; x+= density) {
+
+      netx.w[0] = (x-WIDTH/2)/ss;
+      var a = net.forward(netx);
+      var y = a.w[0];
+
+      if(draw_neuron_outputs) {
+        neurons.push(net.layers[lix].out_act.w); // back these up
+      }
+
+      if(x===0) ctx.moveTo(x, -y*ss+HEIGHT/2);
+      else ctx.lineTo(x, -y*ss+HEIGHT/2);
+    }
+    ctx.stroke();
+
+    // draw individual neurons on first layer
+    if(draw_neuron_outputs) {
+      var NL = neurons.length;
+      ctx.strokeStyle = 'rgb(250,50,50)';
+      for(var k=0;k<NL;k++) {
+        ctx.beginPath();
+        var n = 0;
+        for(var x=0.0; x<=WIDTH; x+= density) {
+          if(x===0) ctx.moveTo(x, -neurons[n][k]*ss+HEIGHT/2);
+          else ctx.lineTo(x, -neurons[n][k]*ss+HEIGHT/2);
+          n++;
+        }
+        ctx.stroke();
+      }
+    }
+  
+    // draw axes
+    ctx.beginPath();
+    ctx.strokeStyle = 'rgb(50,50,50)';
+    ctx.lineWidth = 1;
+    ctx.moveTo(0, HEIGHT/2);
+    ctx.lineTo(WIDTH, HEIGHT/2);
+    ctx.moveTo(WIDTH/2, 0);
+    ctx.lineTo(WIDTH/2, HEIGHT);
+    ctx.stroke();
+
+    // draw datapoints. Draw support vectors larger
+    ctx.strokeStyle = 'rgb(0,0,0)';
+    ctx.lineWidth = 1;
+    for(var i=0;i<N;i++) {
+      drawCircle(data[i]*ss+WIDTH/2, -labels[i]*ss+HEIGHT/2, 5.0);
+    }
+
+    ctx.fillStyle = "blue";
+    ctx.font = "bold 16px Arial";
+    ctx.fillText("average loss: " + avloss, 20, 20);
+}
+
+function mouseClick(x, y, shiftPressed){
+  
+  // add datapoint at location of click
+  data.push([(x-WIDTH/2)/ss]);
+  labels.push([-(y-HEIGHT/2)/ss]);
+  N += 1;
+
+}
+
+function keyDown(key){
+}
+
+function keyUp(key) {
+
+}
+
+$(function() {
+    
+});

demo/js/rldemo.js

@@ -0,0 +1,578 @@
+  var canvas, ctx;
+    
+    // A 2D vector utility
+    var Vec = function(x, y) {
+      this.x = x;
+      this.y = y;
+    }
+    Vec.prototype = {
+      
+      // utilities
+      dist_from: function(v) { return Math.sqrt(Math.pow(this.x-v.x,2) + Math.pow(this.y-v.y,2)); },
+      length: function() { return Math.sqrt(Math.pow(this.x,2) + Math.pow(this.y,2)); },
+      
+      // new vector returning operations
+      add: function(v) { return new Vec(this.x + v.x, this.y + v.y); },
+      sub: function(v) { return new Vec(this.x - v.x, this.y - v.y); },
+      rotate: function(a) {  // CLOCKWISE
+        return new Vec(this.x * Math.cos(a) + this.y * Math.sin(a),
+                       -this.x * Math.sin(a) + this.y * Math.cos(a));
+      },
+      
+      // in place operations
+      scale: function(s) { this.x *= s; this.y *= s; },
+      normalize: function() { var d = this.length(); this.scale(1.0/d); }
+    }
+    
+    // line intersection helper function: does line segment (p1,p2) intersect segment (p3,p4) ?
+    var line_intersect = function(p1,p2,p3,p4) {
+      var denom = (p4.y-p3.y)*(p2.x-p1.x)-(p4.x-p3.x)*(p2.y-p1.y);
+      if(denom===0.0) { return false; } // parallel lines
+      var ua = ((p4.x-p3.x)*(p1.y-p3.y)-(p4.y-p3.y)*(p1.x-p3.x))/denom;
+      var ub = ((p2.x-p1.x)*(p1.y-p3.y)-(p2.y-p1.y)*(p1.x-p3.x))/denom;
+      if(ua>0.0&&ua<1.0&&ub>0.0&&ub<1.0) {
+        var up = new Vec(p1.x+ua*(p2.x-p1.x), p1.y+ua*(p2.y-p1.y));
+        return {ua:ua, ub:ub, up:up}; // up is intersection point
+      }
+      return false;
+    }
+    
+    var line_point_intersect = function(p1,p2,p0,rad) {
+      var v = new Vec(p2.y-p1.y,-(p2.x-p1.x)); // perpendicular vector
+      var d = Math.abs((p2.x-p1.x)*(p1.y-p0.y)-(p1.x-p0.x)*(p2.y-p1.y));
+      d = d / v.length();
+      if(d > rad) { return false; }
+      
+      v.normalize();
+      v.scale(d);
+      var up = p0.add(v);
+      if(Math.abs(p2.x-p1.x)>Math.abs(p2.y-p1.y)) {
+        var ua = (up.x - p1.x) / (p2.x - p1.x);
+      } else {
+        var ua = (up.y - p1.y) / (p2.y - p1.y);
+      }
+      if(ua>0.0&&ua<1.0) {
+        return {ua:ua, up:up};
+      }
+      return false;
+    }
+    
+    // Wall is made up of two points
+    var Wall = function(p1, p2) {
+      this.p1 = p1;
+      this.p2 = p2;
+    }
+    
+    // World object contains many agents and walls and food and stuff
+    var util_add_box = function(lst, x, y, w, h) {
+      lst.push(new Wall(new Vec(x,y), new Vec(x+w,y)));
+      lst.push(new Wall(new Vec(x+w,y), new Vec(x+w,y+h)));
+      lst.push(new Wall(new Vec(x+w,y+h), new Vec(x,y+h)));
+      lst.push(new Wall(new Vec(x,y+h), new Vec(x,y)));
+    }
+    
+    // item is circle thing on the floor that agent can interact with (see or eat, etc)
+    var Item = function(x, y, type) {
+      this.p = new Vec(x, y); // position
+      this.type = type;
+      this.rad = 10; // default radius
+      this.age = 0;
+      this.cleanup_ = false;
+    }
+    
+    var World = function() {
+      this.agents = [];
+      this.W = canvas.width;
+      this.H = canvas.height;
+      
+      this.clock = 0;
+      
+      // set up walls in the world
+      this.walls = []; 
+      var pad = 10;
+      util_add_box(this.walls, pad, pad, this.W-pad*2, this.H-pad*2);
+      util_add_box(this.walls, 100, 100, 200, 300); // inner walls
+      this.walls.pop();
+      util_add_box(this.walls, 400, 100, 200, 300);
+      this.walls.pop();
+      
+      // set up food and poison
+      this.items = []
+      for(var k=0;k<30;k++) {
+        var x = convnetjs.randf(20, this.W-20);
+        var y = convnetjs.randf(20, this.H-20);
+        var t = convnetjs.randi(1, 3); // food or poison (1 and 2)
+        var it = new Item(x, y, t);
+        this.items.push(it);
+      }
+    }
+    
+    World.prototype = {      
+      // helper function to get closest colliding walls/items
+      stuff_collide_: function(p1, p2, check_walls, check_items) {
+        var minres = false;
+        
+        // collide with walls
+        if(check_walls) {
+          for(var i=0,n=this.walls.length;i<n;i++) {
+            var wall = this.walls[i];
+            var res = line_intersect(p1, p2, wall.p1, wall.p2);
+            if(res) {
+              res.type = 0; // 0 is wall
+              if(!minres) { minres=res; }
+              else {
+                // check if its closer
+                if(res.ua < minres.ua) {
+                  // if yes replace it
+                  minres = res;
+                }
+              }
+            }
+          }
+        }
+        
+        // collide with items
+        if(check_items) {
+          for(var i=0,n=this.items.length;i<n;i++) {
+            var it = this.items[i];
+            var res = line_point_intersect(p1, p2, it.p, it.rad);
+            if(res) {
+              res.type = it.type; // store type of item
+              if(!minres) { minres=res; }
+              else { if(res.ua < minres.ua) { minres = res; }
+              }
+            }
+          }
+        }
+        
+        return minres;
+      },
+      tick: function() {
+        // tick the environment
+        this.clock++;
+        
+        // fix input to all agents based on environment
+        // process eyes
+        this.collpoints = [];
+        for(var i=0,n=this.agents.length;i<n;i++) {
+          var a = this.agents[i];
+          for(var ei=0,ne=a.eyes.length;ei<ne;ei++) {
+            var e = a.eyes[ei];
+            // we have a line from p to p->eyep
+            var eyep = new Vec(a.p.x + e.max_range * Math.sin(a.angle + e.angle),
+                               a.p.y + e.max_range * Math.cos(a.angle + e.angle));
+            var res = this.stuff_collide_(a.p, eyep, true, true);
+            if(res) {
+              // eye collided with wall
+              e.sensed_proximity = res.up.dist_from(a.p);
+              e.sensed_type = res.type;
+            } else {
+              e.sensed_proximity = e.max_range;
+              e.sensed_type = -1;
+            }
+          }
+        }
+        
+        // let the agents behave in the world based on their input
+        for(var i=0,n=this.agents.length;i<n;i++) {
+          this.agents[i].forward();
+        }
+        
+        // apply outputs of agents on evironment
+        for(var i=0,n=this.agents.length;i<n;i++) {
+          var a = this.agents[i];
+          a.op = a.p; // back up old position
+          a.oangle = a.angle; // and angle
+          
+          // steer the agent according to outputs of wheel velocities
+          var v = new Vec(0, a.rad / 2.0);
+          v = v.rotate(a.angle + Math.PI/2);
+          var w1p = a.p.add(v); // positions of wheel 1 and 2
+          var w2p = a.p.sub(v);
+          var vv = a.p.sub(w2p);
+          vv = vv.rotate(-a.rot1);
+          var vv2 = a.p.sub(w1p);
+          vv2 = vv2.rotate(a.rot2);
+          var np = w2p.add(vv);
+          np.scale(0.5);
+          var np2 = w1p.add(vv2);
+          np2.scale(0.5);
+          a.p = np.add(np2);
+          
+          a.angle -= a.rot1;
+          if(a.angle<0)a.angle+=2*Math.PI;
+          a.angle += a.rot2;
+          if(a.angle>2*Math.PI)a.angle-=2*Math.PI;
+          
+          // agent is trying to move from p to op. Check walls
+          var res = this.stuff_collide_(a.op, a.p, true, false);
+          if(res) {
+            // wall collision! reset position
+            a.p = a.op;
+          }
+          
+          // handle boundary conditions
+          if(a.p.x<0)a.p.x=0;
+          if(a.p.x>this.W)a.p.x=this.W;
+          if(a.p.y<0)a.p.y=0;
+          if(a.p.y>this.H)a.p.y=this.H;
+        }
+        
+        // tick all items
+        var update_items = false;
+        for(var i=0,n=this.items.length;i<n;i++) {
+          var it = this.items[i];
+          it.age += 1;
+          
+          // see if some agent gets lunch
+          for(var j=0,m=this.agents.length;j<m;j++) {
+            var a = this.agents[j];
+            var d = a.p.dist_from(it.p);
+            if(d < it.rad + a.rad) {
+              
+              // wait lets just make sure that this isn't through a wall
+              var rescheck = this.stuff_collide_(a.p, it.p, true, false);
+              if(!rescheck) { 
+                // ding! nom nom nom
+                if(it.type === 1) a.digestion_signal += 5.0; // mmm delicious apple
+                if(it.type === 2) a.digestion_signal += -6.0; // ewww poison
+                it.cleanup_ = true;
+                update_items = true;
+                break; // break out of loop, item was consumed
+              }
+            }
+          }
+          
+          if(it.age > 5000 && this.clock % 100 === 0 && convnetjs.randf(0,1)<0.1) {
+            it.cleanup_ = true; // replace this one, has been around too long
+            update_items = true;
+          }
+        }
+        if(update_items) {
+          var nt = [];
+          for(var i=0,n=this.items.length;i<n;i++) {
+            var it = this.items[i];
+            if(!it.cleanup_) nt.push(it);
+          }
+          this.items = nt; // swap
+        }
+        if(this.items.length < 30 && this.clock % 10 === 0 && convnetjs.randf(0,1)<0.25) {
+          var newitx = convnetjs.randf(20, this.W-20);
+          var newity = convnetjs.randf(20, this.H-20);
+          var newitt = convnetjs.randi(1, 3); // food or poison (1 and 2)
+          var newit = new Item(newitx, newity, newitt);
+          this.items.push(newit);
+        }
+        
+        // agents are given the opportunity to learn based on feedback of their action on environment
+        for(var i=0,n=this.agents.length;i<n;i++) {
+          this.agents[i].backward();
+        }
+      }
+    }
+    
+    // Eye sensor has a maximum range and senses walls
+    var Eye = function(angle) {
+      this.angle = angle; // angle relative to agent its on
+      this.max_range = 85;
+      this.sensed_proximity = 85; // what the eye is seeing. will be set in world.tick()
+      this.sensed_type = -1; // what does the eye see?
+    }
+    
+    // A single agent
+    var Agent = function() {
+    
+      // positional information
+      this.p = new Vec(50, 50);
+      this.op = this.p; // old position
+      this.angle = 0; // direction facing
+      
+      this.actions = [];
+      this.actions.push([1,1]);
+      this.actions.push([0.8,1]);
+      this.actions.push([1,0.8]);
+      this.actions.push([0.5,0]);
+      this.actions.push([0,0.5]);
+      
+      // properties
+      this.rad = 10;
+      this.eyes = [];
+      for(var k=0;k<9;k++) { this.eyes.push(new Eye((k-3)*0.25)); }
+      
+      // braaain
+      //this.brain = new deepqlearn.Brain(this.eyes.length * 3, this.actions.length);
+      var spec = document.getElementById('qspec').value;
+      eval(spec);
+      this.brain = brain;
+      
+      this.reward_bonus = 0.0;
+      this.digestion_signal = 0.0;
+      
+      // outputs on world
+      this.rot1 = 0.0; // rotation speed of 1st wheel
+      this.rot2 = 0.0; // rotation speed of 2nd wheel
+      
+      this.prevactionix = -1;
+    }
+    Agent.prototype = {
+      forward: function() {
+        // in forward pass the agent simply behaves in the environment
+        // create input to brain
+        var num_eyes = this.eyes.length;
+        var input_array = new Array(num_eyes * 3);
+        for(var i=0;i<num_eyes;i++) {
+          var e = this.eyes[i];
+          input_array[i*3] = 1.0;
+          input_array[i*3+1] = 1.0;
+          input_array[i*3+2] = 1.0;
+          if(e.sensed_type !== -1) {
+            // sensed_type is 0 for wall, 1 for food and 2 for poison.
+            // lets do a 1-of-k encoding into the input array
+            input_array[i*3 + e.sensed_type] = e.sensed_proximity/e.max_range; // normalize to [0,1]
+          }
+        }
+        
+        // get action from brain
+        var actionix = this.brain.forward(input_array);
+        var action = this.actions[actionix];
+        this.actionix = actionix; //back this up
+        
+        // demultiplex into behavior variables
+        this.rot1 = action[0]*1;
+        this.rot2 = action[1]*1;
+        
+        //this.rot1 = 0;
+        //this.rot2 = 0;
+      },
+      backward: function() {
+        // in backward pass agent learns.
+        // compute reward 
+        var proximity_reward = 0.0;
+        var num_eyes = this.eyes.length;
+        for(var i=0;i<num_eyes;i++) {
+          var e = this.eyes[i];
+          // agents dont like to see walls, especially up close
+          proximity_reward += e.sensed_type === 0 ? e.sensed_proximity/e.max_range : 1.0;
+        }
+        proximity_reward = proximity_reward/num_eyes;
+        proximity_reward = Math.min(1.0, proximity_reward * 2);
+        
+        // agents like to go straight forward
+        var forward_reward = 0.0;
+        if(this.actionix === 0 && proximity_reward > 0.75) forward_reward = 0.1 * proximity_reward;
+        
+        // agents like to eat good things
+        var digestion_reward = this.digestion_signal;
+        this.digestion_signal = 0.0;
+        
+        var reward = proximity_reward + forward_reward + digestion_reward;
+        
+        // pass to brain for learning
+        this.brain.backward(reward);
+      }
+    }
+    
+    function draw_net() {
+      if(simspeed <=1) {
+        // we will always draw at these speeds
+      } else {
+        if(w.clock % 50 !== 0) return;  // do this sparingly
+      }
+      
+      var canvas = document.getElementById("net_canvas");
+      var ctx = canvas.getContext("2d");
+      var W = canvas.width;
+      var H = canvas.height;
+      ctx.clearRect(0, 0, canvas.width, canvas.height);
+      var L = w.agents[0].brain.value_net.layers;
+      var dx = (W - 50)/L.length;
+      var x = 10;
+      var y = 40;
+      ctx.font="12px Verdana";
+      ctx.fillStyle = "rgb(0,0,0)";
+      ctx.fillText("Value Function Approximating Neural Network:", 10, 14);
+      for(var k=0;k<L.length;k++) {
+        if(typeof(L[k].out_act)==='undefined') continue; // maybe not yet ready
+        var kw = L[k].out_act.w;
+        var n = kw.length;
+        var dy = (H-50)/n;
+        ctx.fillStyle = "rgb(0,0,0)";
+        ctx.fillText(L[k].layer_type + "(" + n + ")", x, 35);
+        for(var q=0;q<n;q++) {
+          var v = Math.floor(kw[q]*100);
+          if(v >= 0) ctx.fillStyle = "rgb(0,0," + v + ")";
+          if(v < 0) ctx.fillStyle = "rgb(" + (-v) + ",0,0)";
+          ctx.fillRect(x,y,10,10);
+          y += 12;
+          if(y>H-25) { y = 40; x += 12};
+        }
+        x += 50;
+        y = 40;
+      }
+    }
+    
+    var reward_graph = new cnnvis.Graph();
+    function draw_stats() {
+      var canvas = document.getElementById("vis_canvas");
+      var ctx = canvas.getContext("2d");
+      var W = canvas.width;
+      var H = canvas.height;
+      ctx.clearRect(0, 0, canvas.width, canvas.height);
+      var a = w.agents[0];
+      var b = a.brain;
+      var netin = b.last_input_array;
+      ctx.strokeStyle = "rgb(0,0,0)";
+      //ctx.font="12px Verdana";
+      //ctx.fillText("Current state:",10,10);
+      ctx.lineWidth = 10;
+      ctx.beginPath();
+      for(var k=0,n=netin.length;k<n;k++) {
+        ctx.moveTo(10+k*12, 120);
+        ctx.lineTo(10+k*12, 120 - netin[k] * 100);
+      }
+      ctx.stroke();
+      
+      if(w.clock % 200 === 0) {
+        reward_graph.add(w.clock/200, b.average_reward_window.get_average());
+        var gcanvas = document.getElementById("graph_canvas");
+        reward_graph.drawSelf(gcanvas);
+      }
+    }
+    
+    // Draw everything
+    function draw() {  
+      ctx.clearRect(0, 0, canvas.width, canvas.height);
+      ctx.lineWidth = 1;
+      var agents = w.agents;
+      
+      // draw walls in environment
+      ctx.strokeStyle = "rgb(0,0,0)";
+      ctx.beginPath();
+      for(var i=0,n=w.walls.length;i<n;i++) {
+        var q = w.walls[i];
+        ctx.moveTo(q.p1.x, q.p1.y);
+        ctx.lineTo(q.p2.x, q.p2.y);
+      }
+      ctx.stroke();
+  
+      // draw agents
+      // color agent based on reward it is experiencing at the moment
+      var r = Math.floor(agents[0].brain.latest_reward * 200);
+      if(r>255)r=255;if(r<0)r=0;
+      ctx.fillStyle = "rgb(" + r + ", 150, 150)";
+      ctx.strokeStyle = "rgb(0,0,0)";
+      for(var i=0,n=agents.length;i<n;i++) {
+        var a = agents[i];
+        
+        // draw agents body
+        ctx.beginPath();
+        ctx.arc(a.op.x, a.op.y, a.rad, 0, Math.PI*2, true); 
+        ctx.fill();
+        ctx.stroke();
+        
+        // draw agents sight
+        for(var ei=0,ne=a.eyes.length;ei<ne;ei++) {
+          var e = a.eyes[ei];
+          var sr = e.sensed_proximity;
+          if(e.sensed_type === -1 || e.sensed_type === 0) { 
+            ctx.strokeStyle = "rgb(0,0,0)"; // wall or nothing
+          }
+          if(e.sensed_type === 1) { ctx.strokeStyle = "rgb(255,150,150)"; } // apples
+          if(e.sensed_type === 2) { ctx.strokeStyle = "rgb(150,255,150)"; } // poison
+          ctx.beginPath();
+          ctx.moveTo(a.op.x, a.op.y);
+          ctx.lineTo(a.op.x + sr * Math.sin(a.oangle + e.angle),
+                     a.op.y + sr * Math.cos(a.oangle + e.angle));
+          ctx.stroke();
+        }
+      }
+      
+      // draw items
+      ctx.strokeStyle = "rgb(0,0,0)";
+      for(var i=0,n=w.items.length;i<n;i++) {
+        var it = w.items[i];
+        if(it.type === 1) ctx.fillStyle = "rgb(255, 150, 150)";
+        if(it.type === 2) ctx.fillStyle = "rgb(150, 255, 150)";
+        ctx.beginPath();
+        ctx.arc(it.p.x, it.p.y, it.rad, 0, Math.PI*2, true); 
+        ctx.fill();
+        ctx.stroke();
+      }
+      
+      w.agents[0].brain.visSelf(document.getElementById('brain_info_div'));
+    }
+    
+    // Tick the world
+    function tick() {
+      w.tick();
+      if(!skipdraw || w.clock % 50 === 0) {
+        draw();
+        draw_stats();
+        draw_net();
+      }
+    }
+    
+    var simspeed = 2;
+    function goveryfast() {
+      window.clearInterval(current_interval_id);
+      current_interval_id = setInterval(tick, 0);
+      skipdraw = true;
+      simspeed = 3;
+    }
+    function gofast() {
+      window.clearInterval(current_interval_id);
+      current_interval_id = setInterval(tick, 0);
+      skipdraw = false;
+      simspeed = 2;
+    }
+    function gonormal() {
+      window.clearInterval(current_interval_id);
+      current_interval_id = setInterval(tick, 30);
+      skipdraw = false;
+      simspeed = 1;
+    }
+    function goslow() {
+      window.clearInterval(current_interval_id);
+      current_interval_id = setInterval(tick, 200);
+      skipdraw = false;
+      simspeed = 0;
+    }
+    
+    function savenet() {
+      var j = w.agents[0].brain.value_net.toJSON();
+      var t = JSON.stringify(j);
+      document.getElementById('tt').value = t;
+    }
+    
+    function loadnet() {
+      var t = document.getElementById('tt').value;
+      var j = JSON.parse(t);
+      w.agents[0].brain.value_net.fromJSON(j);
+      stoplearn(); // also stop learning
+      gonormal();
+    }
+    
+    function startlearn() {
+      w.agents[0].brain.learning = true;
+    }
+    function stoplearn() {
+      w.agents[0].brain.learning = false;
+    }
+    
+    function reload() {
+      w.agents = [new Agent()]; // this should simply work. I think... ;\
+      reward_graph = new cnnvis.Graph(); // reinit
+    }
+    
+    var w; // global world object
+    var current_interval_id;
+    var skipdraw = false;
+    function start() {
+      canvas = document.getElementById("canvas");
+      ctx = canvas.getContext("2d");
+      
+      w = new World();
+      w.agents = [new Agent()];
+      
+      gofast();
+    }

demo/js/trainers.js

@@ -0,0 +1,208 @@
+
+var t = "\n\
+// lets use an example fully-connected 2-layer ReLU net\n\
+var layer_defs = [];\n\
+layer_defs.push({type:'input', out_sx:24, out_sy:24, out_depth:1});\n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
+layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
+layer_defs.push({type:'softmax', num_classes:10});\n\
+\n\
+// below fill out the trainer specs you wish to evaluate, and give them names for legend\n\
+var LR = 0.01; // learning rate\n\
+var BS = 8; // batch size\n\
+var L2 = 0.001; // L2 weight decay\n\
+nets = [];\n\
+trainer_defs = [];\n\
+trainer_defs.push({learning_rate:LR, method: 'sgd', momentum: 0.0, batch_size:BS, l2_decay:L2});\n\
+trainer_defs.push({learning_rate:LR, method: 'sgd', momentum: 0.9, batch_size:BS, l2_decay:L2});\n\
+trainer_defs.push({learning_rate:LR, method: 'adam', eps: 1e-8, beta1: 0.9, beta2: 0.99, batch_size:BS, l2_decay:L2});\n\
+trainer_defs.push({learning_rate:LR, method: 'adagrad', eps: 1e-6, batch_size:BS, l2_decay:L2});\n\
+trainer_defs.push({learning_rate:LR, method: 'windowgrad', eps: 1e-6, ro: 0.95, batch_size:BS, l2_decay:L2});\n\
+trainer_defs.push({learning_rate:1.0, method: 'adadelta', eps: 1e-6, ro:0.95, batch_size:BS, l2_decay:L2});\n\
+trainer_defs.push({learning_rate:LR, method: 'nesterov', momentum: 0.9, batch_size:BS, l2_decay:L2});\n\
+\n\
+// names for all trainers above\n\
+legend = ['sgd', 'sgd+momentum', 'adam', 'adagrad', 'windowgrad', 'adadelta', 'nesterov'];\n\
+"
+
+// ------------------------
+// BEGIN MNIST SPECIFIC STUFF
+// ------------------------
+classes_txt = ['0','1','2','3','4','5','6','7','8','9'];
+
+var use_validation_data = false;
+var sample_training_instance = function() {
+
+  // find an unloaded batch
+  var bi = Math.floor(Math.random()*loaded_train_batches.length);
+  var b = loaded_train_batches[bi];
+  var k = Math.floor(Math.random()*3000); // sample within the batch
+  var n = b*3000+k;
+
+  // load more batches over time
+  if(step_num%5000===0 && step_num>0) {
+    for(var i=0;i<num_batches;i++) {
+      if(!loaded[i]) {
+        // load it
+        load_data_batch(i);
+        break; // okay for now
+      }
+    }
+  }
+
+  // fetch the appropriate row of the training image and reshape into a Vol
+  var p = img_data[b].data;
+  var x = new convnetjs.Vol(28,28,1,0.0);
+  var W = 28*28;
+  for(var i=0;i<W;i++) {
+    var ix = ((W * k) + i) * 4;
+    x.w[i] = p[ix]/255.0;
+  }
+  x = convnetjs.augment(x, 24);
+
+  var isval = use_validation_data && n%10===0 ? true : false;
+  return {x:x, label:labels[n], isval:isval};
+}
+
+var sample_test_instance = function() {
+  var b = 20;
+  var k = Math.floor(Math.random()*3000);
+  var n = b*3000+k;
+
+  var p = img_data[b].data;
+  var x = new convnetjs.Vol(28,28,1,0.0);
+  var W = 28*28;
+  for(var i=0;i<W;i++) {
+    var ix = ((W * k) + i) * 4;
+    x.w[i] = p[ix]/255.0;
+  }
+  x = convnetjs.augment(x, 24);
+  return {x:x, label:labels[n]};
+}
+
+var num_batches = 21; // 20 training batches, 1 test
+var data_img_elts = new Array(num_batches);
+var img_data = new Array(num_batches);
+var loaded = new Array(num_batches);
+var loaded_train_batches = [];
+var step_num = 0;
+
+// int main
+var lossWindows = [];
+var trainAccWindows = [];
+var testAccWindows = [];
+var lossGraph, trainGraph, testGraph;
+$(window).load(function() {
+
+  $("#layerdef").val(t);
+
+  for(var k=0;k<loaded.length;k++) { loaded[k] = false; }
+  load_data_batch(0); // async load train set batch 0 (6 total train batches)
+  load_data_batch(20); // async load test set (batch 6)
+  start_fun();
+
+  reload();
+});
+
+var reload = function() {
+  
+  eval($("#layerdef").val()); // fills in trainer_spects[] array, and layer_defs
+
+  var N = trainer_defs.length;
+  nets = [];
+  trainers = [];
+  for(var i=0;i<N;i++) {
+    var net = new convnetjs.Net();
+    net.makeLayers(layer_defs);
+    var trainer = new convnetjs.Trainer(net, trainer_defs[i]);
+    nets.push(net); 
+    trainers.push(trainer);
+  }
+
+  step_num = 0;
+
+  lossWindows = [];
+  trainAccWindows = [];
+  testAccWindows = [];
+  for(var i=0;i<N;i++) {
+    lossWindows.push(new cnnutil.Window(800));
+    trainAccWindows.push(new cnnutil.Window(800));
+    testAccWindows.push(new cnnutil.Window(800));
+  }
+  lossGraph = new cnnvis.MultiGraph(legend);
+  trainGraph = new cnnvis.MultiGraph(legend);
+  testGraph = new cnnvis.MultiGraph(legend);
+}
+
+var start_fun = function() {
+  if(loaded[0] && loaded[20]) { 
+    console.log('starting!'); 
+    setInterval(load_and_step, 0); // lets go!
+  }
+  else { setTimeout(start_fun, 200); } // keep checking
+}
+
+var load_data_batch = function(batch_num) {
+  // Load the dataset with JS in background
+  data_img_elts[batch_num] = new Image();
+  var data_img_elt = data_img_elts[batch_num];
+  data_img_elt.onload = function() { 
+    var data_canvas = document.createElement('canvas');
+    data_canvas.width = data_img_elt.width;
+    data_canvas.height = data_img_elt.height;
+    var data_ctx = data_canvas.getContext("2d");
+    data_ctx.drawImage(data_img_elt, 0, 0); // copy it over... bit wasteful :(
+    img_data[batch_num] = data_ctx.getImageData(0, 0, data_canvas.width, data_canvas.height);
+    loaded[batch_num] = true;
+    if(batch_num < 20) { loaded_train_batches.push(batch_num); }
+    console.log('finished loading data batch ' + batch_num);
+  };
+  data_img_elt.src = "mnist/mnist_batch_" + batch_num + ".png";
+}
+
+// ------------------------
+// END MNIST SPECIFIC STUFF
+// ------------------------
+
+// main iterator function
+var load_and_step = function() {
+  step_num++;
+  var sample = sample_training_instance();
+  var test_sample = sample_test_instance();
+
+  // train on all networks
+  var N = nets.length;
+  var losses = [];
+  var trainacc = [];
+  testacc = [];
+  for(var i=0;i<N;i++) {
+
+    // train on training example
+    var stats = trainers[i].train(sample.x, sample.label);
+    var yhat = nets[i].getPrediction();
+    trainAccWindows[i].add(yhat === sample.label ? 1.0 : 0.0);
+    lossWindows[i].add(stats.loss);
+
+    // evaluate a test example
+    nets[i].forward(test_sample.x);
+    var yhat_test = nets[i].getPrediction();
+    testAccWindows[i].add(yhat_test === test_sample.label ? 1.0 : 0.0);
+
+    // every 100 iterations also draw
+    if(step_num % 100 === 0) {
+      losses.push(lossWindows[i].get_average());
+      trainacc.push(trainAccWindows[i].get_average());
+      testacc.push(testAccWindows[i].get_average());
+    }
+  }
+  if(step_num % 100 === 0) {
+    lossGraph.add(step_num, losses);
+    lossGraph.drawSelf(document.getElementById("lossgraph"));
+
+    trainGraph.add(step_num, trainacc);
+    trainGraph.drawSelf(document.getElementById("trainaccgraph"));
+
+    testGraph.add(step_num, testacc);
+    testGraph.drawSelf(document.getElementById("testaccgraph"));
+  }
+}

demo/mnist.html

@@ -0,0 +1,142 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+  <title>ConvNetJS MNIST demo</title>
+  <meta name="description" content="">
+  <meta name="author" content="">
+  <link rel="stylesheet" href="css/style.css">
+
+<script src="js/jquery-1.8.3.min.js"></script>
+<script src="../build/vis.js"></script>
+<script src="../build/util.js"></script>
+<script src="../build/convnet.js"></script>
+
+<script src="js/image-helpers.js"></script>
+<script src="js/pica.js"></script>
+
+<script src="mnist/mnist_labels.js"></script>
+<script type="text/javascript">
+  // ------------------------
+  // BEGIN MNIST SPECIFIC STUFF
+  // ------------------------
+  classes_txt = ['0','1','2','3','4','5','6','7','8','9'];
+  var dataset_name = "mnist";
+  var num_batches = 21; // 20 training batches, 1 test
+  var test_batch = 20;
+  var num_samples_per_batch = 3000;
+  var image_dimension = 28;
+  var image_channels = 1;
+  var use_validation_data = true;
+  var random_flip = false;
+  var random_position = false;
+
+  var layer_defs, net, trainer;
+  var t = "layer_defs = [];\n\
+  layer_defs.push({type:'input', out_sx:24, out_sy:24, out_depth:1});\n\
+  layer_defs.push({type:'conv', sx:5, filters:8, stride:1, pad:2, activation:'relu'});\n\
+  layer_defs.push({type:'pool', sx:2, stride:2});\n\
+  layer_defs.push({type:'conv', sx:5, filters:16, stride:1, pad:2, activation:'relu'});\n\
+  layer_defs.push({type:'pool', sx:3, stride:3});\n\
+  layer_defs.push({type:'softmax', num_classes:10});\n\
+  \n\
+  net = new convnetjs.Net();\n\
+  net.makeLayers(layer_defs);\n\
+  \n\
+  trainer = new convnetjs.SGDTrainer(net, {method:'adadelta', batch_size:20, l2_decay:0.001});\n\
+  ";
+  // ------------------------
+  // END MNIST SPECIFIC STUFF
+  // ------------------------
+</script>
+<script src="js/images-demo.js"></script>
+
+</head>
+<body>
+  <div id="wrap">
+  <h2 style="text-align: center;"><a href="http://cs.stanford.edu/people/karpathy/convnetjs/">ConvNetJS</a> MNIST demo</h2>
+  <h1>Description</h1>
+  <p>
+    This demo trains a Convolutional Neural Network on the <a href="http://yann.lecun.com/exdb/mnist/">MNIST digits dataset</a> in your browser, with nothing but Javascript. The dataset is fairly easy and one should expect to get somewhere around 99% accuracy within few minutes. I used <a href="mnist_parse.zip">this python script</a> to parse the <a href="http://deeplearning.net/tutorial/gettingstarted.html">original files</a> into batches of images that can be easily loaded into page DOM with img tags.
+  </p>
+  <p>
+    This network takes a 28x28 MNIST image and crops a random 24x24 window before training on it (this technique is called data augmentation and improves generalization). Similarly to do prediction, 4 random crops are sampled and the probabilities across all crops are averaged to produce final predictions. The network runs at about 5ms for both forward and backward pass on my reasonably decent Ubuntu+Chrome machine.
+  </p>
+  <p>
+    By default, in this demo we're using Adadelta which is one of per-parameter adaptive step size methods, so we don't have to worry about changing learning rates or momentum over time. However, I still included the text fields for changing these if you'd like to play around with SGD+Momentum trainer.
+  </p>
+  <p>Report questions/bugs/suggestions to <a href="https://twitter.com/karpathy">@karpathy</a>.</p>
+  <h1>Training Stats</h1>
+  <div class="divsec" style="270px;">
+    <div class="secpart">
+      <input id="buttontp" type="submit" value="pause" onclick="toggle_pause();" style="width: 100px; height:30px; background-color: #FCC;"/>
+      <div id="trainstats"></div>
+
+      <div id="controls">
+        Learning rate: <input name="lri" type="text" maxlength="20" id="lr_input"/>
+        <input id="buttonlr" type="submit" value="change" onclick="change_lr();"/>
+        <br />
+
+        Momentum: <input name="momi" type="text" maxlength="20" id="momentum_input"/>
+        <input id="buttonmom" type="submit" value="change" onclick="change_momentum();"/>
+        <br />
+
+        Batch size: <input name="bsi" type="text" maxlength="20" id="batch_size_input"/>
+        <input id="buttonbs" type="submit" value="change" onclick="change_batch_size();"/>
+        <br />
+
+        Weight decay: <input name="wdi" type="text" maxlength="20" id="decay_input"/>
+        <input id="buttonwd" type="submit" value="change" onclick="change_decay();"/>
+      </div>
+
+      <input id="buttondj" type="submit" value="save network snapshot as JSON" onclick="dump_json();"/><br />
+      <input id="buttonlfj" type="submit" value="init network from JSON snapshot" onclick="load_from_json();"/><br />
+      <textarea id="dumpjson"></textarea>
+    </div>
+    <div class="secpart">
+      <div>
+        Loss:<br />
+        <canvas id="lossgraph">
+        </canvas>
+        <br />
+        <input id="buttoncg" type="submit" value="clear graph" onclick="clear_graph();"/>
+      </div>
+    </div>
+    <div class="secpart"> 
+      <div id="upload_box">
+        Test an image from your computer:
+        <div id="img_div">
+            <img id="preview_img"/>
+        </div>
+        <input name="image" type="file" accept="image/*" onchange="loadFile(event)">
+
+        <input type="submit" value="Test Image" onclick="testImage(document.getElementById('preview_img'))">
+      </div>
+    </div>
+    <div style="clear:both;"></div>
+  </div>
+
+  <h1>Instantiate a Network and Trainer</h1>
+  <div>
+    <textarea id="newnet" style="width:100%; height:200px;"></textarea><br />
+    <input id="buttonnn" type="submit" value="change network" onclick="change_net();" style="width:200px;height:30px;"/>
+  </div>
+
+  <div class="divsec">
+  <h1>Network Visualization</h1>
+    <div id="visnet"></div>
+  </div>
+  
+  <div class="divsec">
+  <h1>Example predictions on Test set</h1>
+    <div id="testset_acc"></div>
+	<div id="testset_vis"></div>
+  </div>
+
+  </div>  
+</body>
+</html>
+
+
+

demo/regression.html

@@ -0,0 +1,74 @@
+<html>
+<head>
+<title>ConvNetJS demo: Classify toy 2D data</title>
+<link href='http://fonts.googleapis.com/css?family=Cabin' rel='stylesheet' type='text/css'>
+<link rel="stylesheet" href="css/style.css">
+
+<script>
+      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+      })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+      ga('create', 'UA-3698471-21', 'stanford.edu');
+      ga('send', 'pageview');
+
+</script>
+
+<script src="js/jquery-1.8.3.min.js"></script>
+<script src="js/npgmain.js"></script>
+<script src="../build/convnet.js"></script>
+<script src="js/regression.js"></script>
+
+<style type="text/css">
+canvas { 
+    border: 1px solid #555;
+    margin-top: 10px;
+}
+#wrap {
+  width: 800px;
+  margin-right: auto;
+  margin-left: auto;
+  margin-bottom: 200px;
+}
+</style>
+
+</head>
+
+<body onLoad="NPGinit(10);">
+
+<div id="wrap">
+<h1><a href="http://cs.stanford.edu/people/karpathy/convnetjs">ConvnetJS</a> demo: toy 1d regression</h1>
+
+<p>The simulation below is a 1-dimensional regression where a neural network is trained to regress to y coordinates for every given point x through an L2 loss. That is, the minimized cost function computes the squared difference between the predicted y-coordinate and the "correct" y coordinate. Every 10th of a second, all points are fed to the network multiple times through the trainer class to train the network.</p>
+
+<p>The simulation below will eval() whatever you have in the text area and reload. Feel free to explore and use ConvNetJS to instantiate your own network!</p>
+
+<p>Report questions/bugs/suggestions to <a href="https://twitter.com/karpathy">@karpathy</a>.</p>
+
+<textarea id="layerdef" style="width:100%; height:250px;">
+</textarea>
+<br /><br  />
+<input id="buttontp" type="submit" value="change network" onclick="reload();" style="width: 300px; height: 50px;"/>
+
+<div style="float: right;">
+Number of points to generate:  <input type="text" name="num_points" id="num_data" value="20">
+<input type="submit" value="regenerate data" style="height:50px;" onclick="regen_data();" />
+</div>
+
+<br />
+<p style="color: red">Add data points by clicking!</p>
+<br />
+<input type="checkbox" name="draw_layer_outputs" id="layer_outs">
+Also draw outputs of a layer (click layer button below) in red.
+<br /><div id="layer_ixes"></div>
+
+<canvas id="NPGcanvas" width="800" height="500">Browser not supported for Canvas. Get a real browser.</canvas>
+
+<p>Go <a href="http://cs.stanford.edu/people/karpathy/convnetjs/">back to ConvNetJS</a></p>
+
+</div>
+
+</body>
+</html>
+

demo/speedtest.html

@@ -0,0 +1,55 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+  <title>Simple speed test</title>
+  <meta name="description" content="">
+  <meta name="author" content="">
+
+<script src="../build/convnet.js"></script>
+
+<script>
+
+function logEvent(str) {
+  console.log(str);
+  var d = document.createElement('div');
+  d.innerHTML = str;
+  document.getElementById('result').appendChild(d);
+}
+
+n = 0;
+dtall = 0;
+function runExample() {
+
+  var t0 = +new Date();
+  layer.forward(x);
+  //layer.backward();
+  var t1 = +new Date();
+  var diff = t1 - t0;
+  dtall += diff;
+  n++;
+
+  logEvent('ran example ' + n + ' in ' + diff + 'ms, estimated average = ' + (dtall / n).toFixed(3) + 'ms');
+}
+
+function start() {
+
+  // Conv Layer definition used in convnet benchmarks
+  opt = { in_sx:128, in_sy:128, in_depth:3, sx:11, filters:96, stride: 1, pad: 0};
+  layer = new convnetjs.ConvLayer(opt);
+  x = new convnetjs.Vol(128, 128, 3);
+
+  run1i = setInterval(runExample, 5); // start
+}
+</script>
+</head>
+<body>
+  <button onclick="start()">Convolution Test Start</button>
+  <button onclick="window.clearInterval(run1i);">Stop</button>
+  <div id="result"></div>
+</body>
+</html>
+
+
+

demo/trainers.html

@@ -0,0 +1,47 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+  <title>ConvNetJS Trainer Comparison on MNIST</title>
+  <meta name="description" content="">
+  <meta name="author" content="">
+  <link rel="stylesheet" href="css/style.css">
+
+<script src="js/jquery-1.8.3.min.js"></script>
+<script src="../build/vis.js"></script>
+<script src="../build/util.js"></script>
+<script src="../build/convnet.js"></script>
+<script src="mnist/mnist_labels.js"></script>
+<script src="js/trainers.js"></script>
+
+</head>
+<body>
+  <div id="wrap">
+  <h2 style="text-align: center;"><a href="http://cs.stanford.edu/people/karpathy/convnetjs/">ConvNetJS</a> Trainer demo on MNIST</h2>
+  <h1>Description</h1>
+  <p>
+    This demo lets you evaluate multiple trainers against each other on MNIST. By default I've set up a little benchmark that puts SGD/SGD with momentum/Adam/Adagrad/Adadelta/Nesterov against each other. For reference math and explanations on these refer to Matthew Zeiler's <a href="http://www.matthewzeiler.com/pubs/googleTR2012/googleTR2012.pdf">Adadelta paper</a> (Windowgrad is Idea #1 in the paper). In my own experience, Adagrad/Adadelta are "safer" because they don't depend so strongly on setting of learning rates (with Adadelta being slightly better), but well-tuned SGD+Momentum almost always converges faster and at better final values.
+  </p>
+  <p>Report questions/bugs/suggestions to <a href="https://twitter.com/karpathy">@karpathy</a>.</p>
+
+  <textarea id="layerdef" style="width:100%; height:400px;">
+  </textarea>
+  <br /><br  />
+  <input id="buttontp" type="submit" value="re-run" onclick="reload();" style="width: 300px; height: 50px;"/>
+
+  <h1>Loss vs. Number of examples seen</h1>
+  <canvas id="lossgraph" width="800" height="400"></canvas>
+
+  <h1>Testing Accuracy vs. Number of examples seen</h1>
+  <canvas id="testaccgraph" width="800" height="400"></canvas>
+
+  <h1>Training Accuracy vs. Number of examples seen</h1>
+  <canvas id="trainaccgraph" width="800" height="400"></canvas>
+
+  </div>
+</body>
+</html>
+
+
+

src/convnet_export.js

@@ -0,0 +1,8 @@
+(function(lib) {
+  "use strict";
+  if (typeof module === "undefined" || typeof module.exports === "undefined") {
+    window.convnetjs = lib; // in ordinary browser attach library to window
+  } else {
+    module.exports = lib; // in nodejs
+  }
+})(convnetjs);

src/convnet_init.js

@@ -0,0 +1 @@
+var convnetjs = convnetjs || { REVISION: 'ALPHA' };

src/convnet_layers_dotproducts.js

@@ -0,0 +1,275 @@
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+
+  // This file contains all layers that do dot products with input,
+  // but usually in a different connectivity pattern and weight sharing
+  // schemes: 
+  // - FullyConn is fully connected dot products 
+  // - ConvLayer does convolutions (so weight sharing spatially)
+  // putting them together in one file because they are very similar
+  var ConvLayer = function(opt) {
+    var opt = opt || {};
+
+    // required
+    this.out_depth = opt.filters;
+    this.sx = opt.sx; // filter size. Should be odd if possible, it's cleaner.
+    this.in_depth = opt.in_depth;
+    this.in_sx = opt.in_sx;
+    this.in_sy = opt.in_sy;
+    
+    // optional
+    this.sy = typeof opt.sy !== 'undefined' ? opt.sy : this.sx;
+    this.stride = typeof opt.stride !== 'undefined' ? opt.stride : 1; // stride at which we apply filters to input volume
+    this.pad = typeof opt.pad !== 'undefined' ? opt.pad : 0; // amount of 0 padding to add around borders of input volume
+    this.l1_decay_mul = typeof opt.l1_decay_mul !== 'undefined' ? opt.l1_decay_mul : 0.0;
+    this.l2_decay_mul = typeof opt.l2_decay_mul !== 'undefined' ? opt.l2_decay_mul : 1.0;
+
+    // computed
+    // note we are doing floor, so if the strided convolution of the filter doesnt fit into the input
+    // volume exactly, the output volume will be trimmed and not contain the (incomplete) computed
+    // final application.
+    this.out_sx = Math.floor((this.in_sx + this.pad * 2 - this.sx) / this.stride + 1);
+    this.out_sy = Math.floor((this.in_sy + this.pad * 2 - this.sy) / this.stride + 1);
+    this.layer_type = 'conv';
+
+    // initializations
+    var bias = typeof opt.bias_pref !== 'undefined' ? opt.bias_pref : 0.0;
+    this.filters = [];
+    for(var i=0;i<this.out_depth;i++) { this.filters.push(new Vol(this.sx, this.sy, this.in_depth)); }
+    this.biases = new Vol(1, 1, this.out_depth, bias);
+  }
+  ConvLayer.prototype = {
+    forward: function(V, is_training) {
+      // optimized code by @mdda that achieves 2x speedup over previous version
+
+      this.in_act = V;
+      var A = new Vol(this.out_sx |0, this.out_sy |0, this.out_depth |0, 0.0);
+      
+      var V_sx = V.sx |0;
+      var V_sy = V.sy |0;
+      var xy_stride = this.stride |0;
+
+      for(var d=0;d<this.out_depth;d++) {
+        var f = this.filters[d];
+        var x = -this.pad |0;
+        var y = -this.pad |0;
+        for(var ay=0; ay<this.out_sy; y+=xy_stride,ay++) {  // xy_stride
+          x = -this.pad |0;
+          for(var ax=0; ax<this.out_sx; x+=xy_stride,ax++) {  // xy_stride
+
+            // convolve centered at this particular location
+            var a = 0.0;
+            for(var fy=0;fy<f.sy;fy++) {
+              var oy = y+fy; // coordinates in the original input array coordinates
+              for(var fx=0;fx<f.sx;fx++) {
+                var ox = x+fx;
+                if(oy>=0 && oy<V_sy && ox>=0 && ox<V_sx) {
+                  for(var fd=0;fd<f.depth;fd++) {
+                    // avoid function call overhead (x2) for efficiency, compromise modularity :(
+                    a += f.w[((f.sx * fy)+fx)*f.depth+fd] * V.w[((V_sx * oy)+ox)*V.depth+fd];
+                  }
+                }
+              }
+            }
+            a += this.biases.w[d];
+            A.set(ax, ay, d, a);
+          }
+        }
+      }
+      this.out_act = A;
+      return this.out_act;
+    },
+    backward: function() {
+
+      var V = this.in_act;
+      V.dw = global.zeros(V.w.length); // zero out gradient wrt bottom data, we're about to fill it
+
+      var V_sx = V.sx |0;
+      var V_sy = V.sy |0;
+      var xy_stride = this.stride |0;
+
+      for(var d=0;d<this.out_depth;d++) {
+        var f = this.filters[d];
+        var x = -this.pad |0;
+        var y = -this.pad |0;
+        for(var ay=0; ay<this.out_sy; y+=xy_stride,ay++) {  // xy_stride
+          x = -this.pad |0;
+          for(var ax=0; ax<this.out_sx; x+=xy_stride,ax++) {  // xy_stride
+
+            // convolve centered at this particular location
+            var chain_grad = this.out_act.get_grad(ax,ay,d); // gradient from above, from chain rule
+            for(var fy=0;fy<f.sy;fy++) {
+              var oy = y+fy; // coordinates in the original input array coordinates
+              for(var fx=0;fx<f.sx;fx++) {
+                var ox = x+fx;
+                if(oy>=0 && oy<V_sy && ox>=0 && ox<V_sx) {
+                  for(var fd=0;fd<f.depth;fd++) {
+                    // avoid function call overhead (x2) for efficiency, compromise modularity :(
+                    var ix1 = ((V_sx * oy)+ox)*V.depth+fd;
+                    var ix2 = ((f.sx * fy)+fx)*f.depth+fd;
+                    f.dw[ix2] += V.w[ix1]*chain_grad;
+                    V.dw[ix1] += f.w[ix2]*chain_grad;
+                  }
+                }
+              }
+            }
+            this.biases.dw[d] += chain_grad;
+          }
+        }
+      }
+    },
+    getParamsAndGrads: function() {
+      var response = [];
+      for(var i=0;i<this.out_depth;i++) {
+        response.push({params: this.filters[i].w, grads: this.filters[i].dw, l2_decay_mul: this.l2_decay_mul, l1_decay_mul: this.l1_decay_mul});
+      }
+      response.push({params: this.biases.w, grads: this.biases.dw, l1_decay_mul: 0.0, l2_decay_mul: 0.0});
+      return response;
+    },
+    toJSON: function() {
+      var json = {};
+      json.sx = this.sx; // filter size in x, y dims
+      json.sy = this.sy;
+      json.stride = this.stride;
+      json.in_depth = this.in_depth;
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.l1_decay_mul = this.l1_decay_mul;
+      json.l2_decay_mul = this.l2_decay_mul;
+      json.pad = this.pad;
+      json.filters = [];
+      for(var i=0;i<this.filters.length;i++) {
+        json.filters.push(this.filters[i].toJSON());
+      }
+      json.biases = this.biases.toJSON();
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.sx = json.sx; // filter size in x, y dims
+      this.sy = json.sy;
+      this.stride = json.stride;
+      this.in_depth = json.in_depth; // depth of input volume
+      this.filters = [];
+      this.l1_decay_mul = typeof json.l1_decay_mul !== 'undefined' ? json.l1_decay_mul : 1.0;
+      this.l2_decay_mul = typeof json.l2_decay_mul !== 'undefined' ? json.l2_decay_mul : 1.0;
+      this.pad = typeof json.pad !== 'undefined' ? json.pad : 0;
+      for(var i=0;i<json.filters.length;i++) {
+        var v = new Vol(0,0,0,0);
+        v.fromJSON(json.filters[i]);
+        this.filters.push(v);
+      }
+      this.biases = new Vol(0,0,0,0);
+      this.biases.fromJSON(json.biases);
+    }
+  }
+
+  var FullyConnLayer = function(opt) {
+    var opt = opt || {};
+
+    // required
+    // ok fine we will allow 'filters' as the word as well
+    this.out_depth = typeof opt.num_neurons !== 'undefined' ? opt.num_neurons : opt.filters;
+
+    // optional 
+    this.l1_decay_mul = typeof opt.l1_decay_mul !== 'undefined' ? opt.l1_decay_mul : 0.0;
+    this.l2_decay_mul = typeof opt.l2_decay_mul !== 'undefined' ? opt.l2_decay_mul : 1.0;
+
+    // computed
+    this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
+    this.out_sx = 1;
+    this.out_sy = 1;
+    this.layer_type = 'fc';
+
+    // initializations
+    var bias = typeof opt.bias_pref !== 'undefined' ? opt.bias_pref : 0.0;
+    this.filters = [];
+    for(var i=0;i<this.out_depth ;i++) { this.filters.push(new Vol(1, 1, this.num_inputs)); }
+    this.biases = new Vol(1, 1, this.out_depth, bias);
+  }
+
+  FullyConnLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      var A = new Vol(1, 1, this.out_depth, 0.0);
+      var Vw = V.w;
+      for(var i=0;i<this.out_depth;i++) {
+        var a = 0.0;
+        var wi = this.filters[i].w;
+        for(var d=0;d<this.num_inputs;d++) {
+          a += Vw[d] * wi[d]; // for efficiency use Vols directly for now
+        }
+        a += this.biases.w[i];
+        A.w[i] = a;
+      }
+      this.out_act = A;
+      return this.out_act;
+    },
+    backward: function() {
+      var V = this.in_act;
+      V.dw = global.zeros(V.w.length); // zero out the gradient in input Vol
+      
+      // compute gradient wrt weights and data
+      for(var i=0;i<this.out_depth;i++) {
+        var tfi = this.filters[i];
+        var chain_grad = this.out_act.dw[i];
+        for(var d=0;d<this.num_inputs;d++) {
+          V.dw[d] += tfi.w[d]*chain_grad; // grad wrt input data
+          tfi.dw[d] += V.w[d]*chain_grad; // grad wrt params
+        }
+        this.biases.dw[i] += chain_grad;
+      }
+    },
+    getParamsAndGrads: function() {
+      var response = [];
+      for(var i=0;i<this.out_depth;i++) {
+        response.push({params: this.filters[i].w, grads: this.filters[i].dw, l1_decay_mul: this.l1_decay_mul, l2_decay_mul: this.l2_decay_mul});
+      }
+      response.push({params: this.biases.w, grads: this.biases.dw, l1_decay_mul: 0.0, l2_decay_mul: 0.0});
+      return response;
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.num_inputs = this.num_inputs;
+      json.l1_decay_mul = this.l1_decay_mul;
+      json.l2_decay_mul = this.l2_decay_mul;
+      json.filters = [];
+      for(var i=0;i<this.filters.length;i++) {
+        json.filters.push(this.filters[i].toJSON());
+      }
+      json.biases = this.biases.toJSON();
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.num_inputs = json.num_inputs;
+      this.l1_decay_mul = typeof json.l1_decay_mul !== 'undefined' ? json.l1_decay_mul : 1.0;
+      this.l2_decay_mul = typeof json.l2_decay_mul !== 'undefined' ? json.l2_decay_mul : 1.0;
+      this.filters = [];
+      for(var i=0;i<json.filters.length;i++) {
+        var v = new Vol(0,0,0,0);
+        v.fromJSON(json.filters[i]);
+        this.filters.push(v);
+      }
+      this.biases = new Vol(0,0,0,0);
+      this.biases.fromJSON(json.biases);
+    }
+  }
+
+  global.ConvLayer = ConvLayer;
+  global.FullyConnLayer = FullyConnLayer;
+  
+})(convnetjs);

src/convnet_layers_dropout.js

@@ -0,0 +1,75 @@
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+
+  // An inefficient dropout layer
+  // Note this is not most efficient implementation since the layer before
+  // computed all these activations and now we're just going to drop them :(
+  // same goes for backward pass. Also, if we wanted to be efficient at test time
+  // we could equivalently be clever and upscale during train and copy pointers during test
+  // todo: make more efficient.
+  var DropoutLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = opt.in_depth;
+    this.layer_type = 'dropout';
+    this.drop_prob = typeof opt.drop_prob !== 'undefined' ? opt.drop_prob : 0.5;
+    this.dropped = global.zeros(this.out_sx*this.out_sy*this.out_depth);
+  }
+  DropoutLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      if(typeof(is_training)==='undefined') { is_training = false; } // default is prediction mode
+      var V2 = V.clone();
+      var N = V.w.length;
+      if(is_training) {
+        // do dropout
+        for(var i=0;i<N;i++) {
+          if(Math.random()<this.drop_prob) { V2.w[i]=0; this.dropped[i] = true; } // drop!
+          else {this.dropped[i] = false;}
+        }
+      } else {
+        // scale the activations during prediction
+        for(var i=0;i<N;i++) { V2.w[i]*=this.drop_prob; }
+      }
+      this.out_act = V2;
+      return this.out_act; // dummy identity function for now
+    },
+    backward: function() {
+      var V = this.in_act; // we need to set dw of this
+      var chain_grad = this.out_act;
+      var N = V.w.length;
+      V.dw = global.zeros(N); // zero out gradient wrt data
+      for(var i=0;i<N;i++) {
+        if(!(this.dropped[i])) { 
+          V.dw[i] = chain_grad.dw[i]; // copy over the gradient
+        }
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.drop_prob = this.drop_prob;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+      this.drop_prob = json.drop_prob;
+    }
+  }
+  
+
+  global.DropoutLayer = DropoutLayer;
+})(convnetjs);

src/convnet_layers_input.js

@@ -0,0 +1,47 @@
+
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  var getopt = global.getopt;
+
+  var InputLayer = function(opt) {
+    var opt = opt || {};
+
+    // required: depth
+    this.out_depth = getopt(opt, ['out_depth', 'depth'], 0);
+
+    // optional: default these dimensions to 1
+    this.out_sx = getopt(opt, ['out_sx', 'sx', 'width'], 1);
+    this.out_sy = getopt(opt, ['out_sy', 'sy', 'height'], 1);
+    
+    // computed
+    this.layer_type = 'input';
+  }
+  InputLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      this.out_act = V;
+      return this.out_act; // simply identity function for now
+    },
+    backward: function() { },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+    }
+  }
+
+  global.InputLayer = InputLayer;
+})(convnetjs);

src/convnet_layers_loss.js

@@ -0,0 +1,234 @@
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  
+  // Layers that implement a loss. Currently these are the layers that 
+  // can initiate a backward() pass. In future we probably want a more 
+  // flexible system that can accomodate multiple losses to do multi-task
+  // learning, and stuff like that. But for now, one of the layers in this
+  // file must be the final layer in a Net.
+
+  // This is a classifier, with N discrete classes from 0 to N-1
+  // it gets a stream of N incoming numbers and computes the softmax
+  // function (exponentiate and normalize to sum to 1 as probabilities should)
+  var SoftmaxLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
+    this.out_depth = this.num_inputs;
+    this.out_sx = 1;
+    this.out_sy = 1;
+    this.layer_type = 'softmax';
+  }
+
+  SoftmaxLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+
+      var A = new Vol(1, 1, this.out_depth, 0.0);
+
+      // compute max activation
+      var as = V.w;
+      var amax = V.w[0];
+      for(var i=1;i<this.out_depth;i++) {
+        if(as[i] > amax) amax = as[i];
+      }
+
+      // compute exponentials (carefully to not blow up)
+      var es = global.zeros(this.out_depth);
+      var esum = 0.0;
+      for(var i=0;i<this.out_depth;i++) {
+        var e = Math.exp(as[i] - amax);
+        esum += e;
+        es[i] = e;
+      }
+
+      // normalize and output to sum to one
+      for(var i=0;i<this.out_depth;i++) {
+        es[i] /= esum;
+        A.w[i] = es[i];
+      }
+
+      this.es = es; // save these for backprop
+      this.out_act = A;
+      return this.out_act;
+    },
+    backward: function(y) {
+
+      // compute and accumulate gradient wrt weights and bias of this layer
+      var x = this.in_act;
+      x.dw = global.zeros(x.w.length); // zero out the gradient of input Vol
+
+      for(var i=0;i<this.out_depth;i++) {
+        var indicator = i === y ? 1.0 : 0.0;
+        var mul = -(indicator - this.es[i]);
+        x.dw[i] = mul;
+      }
+
+      // loss is the class negative log likelihood
+      return -Math.log(this.es[y]);
+    },
+    getParamsAndGrads: function() { 
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.num_inputs = this.num_inputs;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.num_inputs = json.num_inputs;
+    }
+  }
+
+  // implements an L2 regression cost layer,
+  // so penalizes \sum_i(||x_i - y_i||^2), where x is its input
+  // and y is the user-provided array of "correct" values.
+  var RegressionLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
+    this.out_depth = this.num_inputs;
+    this.out_sx = 1;
+    this.out_sy = 1;
+    this.layer_type = 'regression';
+  }
+
+  RegressionLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      this.out_act = V;
+      return V; // identity function
+    },
+    // y is a list here of size num_inputs
+    // or it can be a number if only one value is regressed
+    // or it can be a struct {dim: i, val: x} where we only want to 
+    // regress on dimension i and asking it to have value x
+    backward: function(y) { 
+
+      // compute and accumulate gradient wrt weights and bias of this layer
+      var x = this.in_act;
+      x.dw = global.zeros(x.w.length); // zero out the gradient of input Vol
+      var loss = 0.0;
+      if(y instanceof Array || y instanceof Float64Array) {
+        for(var i=0;i<this.out_depth;i++) {
+          var dy = x.w[i] - y[i];
+          x.dw[i] = dy;
+          loss += 0.5*dy*dy;
+        }
+      } else if(typeof y === 'number') {
+        // lets hope that only one number is being regressed
+        var dy = x.w[0] - y;
+        x.dw[0] = dy;
+        loss += 0.5*dy*dy;
+      } else {
+        // assume it is a struct with entries .dim and .val
+        // and we pass gradient only along dimension dim to be equal to val
+        var i = y.dim;
+        var yi = y.val;
+        var dy = x.w[i] - yi;
+        x.dw[i] = dy;
+        loss += 0.5*dy*dy;
+      }
+      return loss;
+    },
+    getParamsAndGrads: function() { 
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.num_inputs = this.num_inputs;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.num_inputs = json.num_inputs;
+    }
+  }
+
+  var SVMLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
+    this.out_depth = this.num_inputs;
+    this.out_sx = 1;
+    this.out_sy = 1;
+    this.layer_type = 'svm';
+  }
+
+  SVMLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      this.out_act = V; // nothing to do, output raw scores
+      return V;
+    },
+    backward: function(y) {
+
+      // compute and accumulate gradient wrt weights and bias of this layer
+      var x = this.in_act;
+      x.dw = global.zeros(x.w.length); // zero out the gradient of input Vol
+
+      // we're using structured loss here, which means that the score
+      // of the ground truth should be higher than the score of any other 
+      // class, by a margin
+      var yscore = x.w[y]; // score of ground truth
+      var margin = 1.0;
+      var loss = 0.0;
+      for(var i=0;i<this.out_depth;i++) {
+        if(y === i) { continue; }
+        var ydiff = -yscore + x.w[i] + margin;
+        if(ydiff > 0) {
+          // violating dimension, apply loss
+          x.dw[i] += 1;
+          x.dw[y] -= 1;
+          loss += ydiff;
+        }
+      }
+
+      return loss;
+    },
+    getParamsAndGrads: function() { 
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.num_inputs = this.num_inputs;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.num_inputs = json.num_inputs;
+    }
+  }
+  
+  global.RegressionLayer = RegressionLayer;
+  global.SoftmaxLayer = SoftmaxLayer;
+  global.SVMLayer = SVMLayer;
+
+})(convnetjs);
+

src/convnet_layers_nonlinearities.js

@@ -0,0 +1,291 @@
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  
+  // Implements ReLU nonlinearity elementwise
+  // x -> max(0, x)
+  // the output is in [0, inf)
+  var ReluLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = opt.in_depth;
+    this.layer_type = 'relu';
+  }
+  ReluLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      var V2 = V.clone();
+      var N = V.w.length;
+      var V2w = V2.w;
+      for(var i=0;i<N;i++) { 
+        if(V2w[i] < 0) V2w[i] = 0; // threshold at 0
+      }
+      this.out_act = V2;
+      return this.out_act;
+    },
+    backward: function() {
+      var V = this.in_act; // we need to set dw of this
+      var V2 = this.out_act;
+      var N = V.w.length;
+      V.dw = global.zeros(N); // zero out gradient wrt data
+      for(var i=0;i<N;i++) {
+        if(V2.w[i] <= 0) V.dw[i] = 0; // threshold
+        else V.dw[i] = V2.dw[i];
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+    }
+  }
+
+  // Implements Sigmoid nnonlinearity elementwise
+  // x -> 1/(1+e^(-x))
+  // so the output is between 0 and 1.
+  var SigmoidLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = opt.in_depth;
+    this.layer_type = 'sigmoid';
+  }
+  SigmoidLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      var V2 = V.cloneAndZero();
+      var N = V.w.length;
+      var V2w = V2.w;
+      var Vw = V.w;
+      for(var i=0;i<N;i++) { 
+        V2w[i] = 1.0/(1.0+Math.exp(-Vw[i]));
+      }
+      this.out_act = V2;
+      return this.out_act;
+    },
+    backward: function() {
+      var V = this.in_act; // we need to set dw of this
+      var V2 = this.out_act;
+      var N = V.w.length;
+      V.dw = global.zeros(N); // zero out gradient wrt data
+      for(var i=0;i<N;i++) {
+        var v2wi = V2.w[i];
+        V.dw[i] =  v2wi * (1.0 - v2wi) * V2.dw[i];
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+    }
+  }
+
+  // Implements Maxout nnonlinearity that computes
+  // x -> max(x)
+  // where x is a vector of size group_size. Ideally of course,
+  // the input size should be exactly divisible by group_size
+  var MaxoutLayer = function(opt) {
+    var opt = opt || {};
+
+    // required
+    this.group_size = typeof opt.group_size !== 'undefined' ? opt.group_size : 2;
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = Math.floor(opt.in_depth / this.group_size);
+    this.layer_type = 'maxout';
+
+    this.switches = global.zeros(this.out_sx*this.out_sy*this.out_depth); // useful for backprop
+  }
+  MaxoutLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      var N = this.out_depth; 
+      var V2 = new Vol(this.out_sx, this.out_sy, this.out_depth, 0.0);
+
+      // optimization branch. If we're operating on 1D arrays we dont have
+      // to worry about keeping track of x,y,d coordinates inside
+      // input volumes. In convnets we do :(
+      if(this.out_sx === 1 && this.out_sy === 1) {
+        for(var i=0;i<N;i++) {
+          var ix = i * this.group_size; // base index offset
+          var a = V.w[ix];
+          var ai = 0;
+          for(var j=1;j<this.group_size;j++) {
+            var a2 = V.w[ix+j];
+            if(a2 > a) {
+              a = a2;
+              ai = j;
+            }
+          }
+          V2.w[i] = a;
+          this.switches[i] = ix + ai;
+        }
+      } else {
+        var n=0; // counter for switches
+        for(var x=0;x<V.sx;x++) {
+          for(var y=0;y<V.sy;y++) {
+            for(var i=0;i<N;i++) {
+              var ix = i * this.group_size;
+              var a = V.get(x, y, ix);
+              var ai = 0;
+              for(var j=1;j<this.group_size;j++) {
+                var a2 = V.get(x, y, ix+j);
+                if(a2 > a) {
+                  a = a2;
+                  ai = j;
+                }
+              }
+              V2.set(x,y,i,a);
+              this.switches[n] = ix + ai;
+              n++;
+            }
+          }
+        }
+
+      }
+      this.out_act = V2;
+      return this.out_act;
+    },
+    backward: function() {
+      var V = this.in_act; // we need to set dw of this
+      var V2 = this.out_act;
+      var N = this.out_depth;
+      V.dw = global.zeros(V.w.length); // zero out gradient wrt data
+
+      // pass the gradient through the appropriate switch
+      if(this.out_sx === 1 && this.out_sy === 1) {
+        for(var i=0;i<N;i++) {
+          var chain_grad = V2.dw[i];
+          V.dw[this.switches[i]] = chain_grad;
+        }
+      } else {
+        // bleh okay, lets do this the hard way
+        var n=0; // counter for switches
+        for(var x=0;x<V2.sx;x++) {
+          for(var y=0;y<V2.sy;y++) {
+            for(var i=0;i<N;i++) {
+              var chain_grad = V2.get_grad(x,y,i);
+              V.set_grad(x,y,this.switches[n],chain_grad);
+              n++;
+            }
+          }
+        }
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.group_size = this.group_size;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+      this.group_size = json.group_size;
+      this.switches = global.zeros(this.group_size);
+    }
+  }
+
+  // a helper function, since tanh is not yet part of ECMAScript. Will be in v6.
+  function tanh(x) {
+    var y = Math.exp(2 * x);
+    return (y - 1) / (y + 1);
+  }
+  // Implements Tanh nnonlinearity elementwise
+  // x -> tanh(x) 
+  // so the output is between -1 and 1.
+  var TanhLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = opt.in_depth;
+    this.layer_type = 'tanh';
+  }
+  TanhLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      var V2 = V.cloneAndZero();
+      var N = V.w.length;
+      for(var i=0;i<N;i++) { 
+        V2.w[i] = tanh(V.w[i]);
+      }
+      this.out_act = V2;
+      return this.out_act;
+    },
+    backward: function() {
+      var V = this.in_act; // we need to set dw of this
+      var V2 = this.out_act;
+      var N = V.w.length;
+      V.dw = global.zeros(N); // zero out gradient wrt data
+      for(var i=0;i<N;i++) {
+        var v2wi = V2.w[i];
+        V.dw[i] = (1.0 - v2wi * v2wi) * V2.dw[i];
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+    }
+  }
+  
+  global.TanhLayer = TanhLayer;
+  global.MaxoutLayer = MaxoutLayer;
+  global.ReluLayer = ReluLayer;
+  global.SigmoidLayer = SigmoidLayer;
+
+})(convnetjs);
+

src/convnet_layers_normalization.js

@@ -0,0 +1,114 @@
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  
+  // a bit experimental layer for now. I think it works but I'm not 100%
+  // the gradient check is a bit funky. I'll look into this a bit later.
+  // Local Response Normalization in window, along depths of volumes
+  var LocalResponseNormalizationLayer = function(opt) {
+    var opt = opt || {};
+
+    // required
+    this.k = opt.k;
+    this.n = opt.n;
+    this.alpha = opt.alpha;
+    this.beta = opt.beta;
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = opt.in_depth;
+    this.layer_type = 'lrn';
+
+    // checks
+    if(this.n%2 === 0) { console.log('WARNING n should be odd for LRN layer'); }
+  }
+  LocalResponseNormalizationLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+
+      var A = V.cloneAndZero();
+      this.S_cache_ = V.cloneAndZero();
+      var n2 = Math.floor(this.n/2);
+      for(var x=0;x<V.sx;x++) {
+        for(var y=0;y<V.sy;y++) {
+          for(var i=0;i<V.depth;i++) {
+
+            var ai = V.get(x,y,i);
+
+            // normalize in a window of size n
+            var den = 0.0;
+            for(var j=Math.max(0,i-n2);j<=Math.min(i+n2,V.depth-1);j++) {
+              var aa = V.get(x,y,j);
+              den += aa*aa;
+            }
+            den *= this.alpha / this.n;
+            den += this.k;
+            this.S_cache_.set(x,y,i,den); // will be useful for backprop
+            den = Math.pow(den, this.beta);
+            A.set(x,y,i,ai/den);
+          }
+        }
+      }
+
+      this.out_act = A;
+      return this.out_act; // dummy identity function for now
+    },
+    backward: function() { 
+      // evaluate gradient wrt data
+      var V = this.in_act; // we need to set dw of this
+      V.dw = global.zeros(V.w.length); // zero out gradient wrt data
+      var A = this.out_act; // computed in forward pass 
+
+      var n2 = Math.floor(this.n/2);
+      for(var x=0;x<V.sx;x++) {
+        for(var y=0;y<V.sy;y++) {
+          for(var i=0;i<V.depth;i++) {
+
+            var chain_grad = this.out_act.get_grad(x,y,i);
+            var S = this.S_cache_.get(x,y,i);
+            var SB = Math.pow(S, this.beta);
+            var SB2 = SB*SB;
+
+            // normalize in a window of size n
+            for(var j=Math.max(0,i-n2);j<=Math.min(i+n2,V.depth-1);j++) {              
+              var aj = V.get(x,y,j); 
+              var g = -aj*this.beta*Math.pow(S,this.beta-1)*this.alpha/this.n*2*aj;
+              if(j===i) g+= SB;
+              g /= SB2;
+              g *= chain_grad;
+              V.add_grad(x,y,j,g);
+            }
+
+          }
+        }
+      }
+    },
+    getParamsAndGrads: function() { return []; },
+    toJSON: function() {
+      var json = {};
+      json.k = this.k;
+      json.n = this.n;
+      json.alpha = this.alpha; // normalize by size
+      json.beta = this.beta;
+      json.out_sx = this.out_sx; 
+      json.out_sy = this.out_sy;
+      json.out_depth = this.out_depth;
+      json.layer_type = this.layer_type;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.k = json.k;
+      this.n = json.n;
+      this.alpha = json.alpha; // normalize by size
+      this.beta = json.beta;
+      this.out_sx = json.out_sx; 
+      this.out_sy = json.out_sy;
+      this.out_depth = json.out_depth;
+      this.layer_type = json.layer_type;
+    }
+  }
+  
+
+  global.LocalResponseNormalizationLayer = LocalResponseNormalizationLayer;
+})(convnetjs);

src/convnet_layers_pool.js

@@ -0,0 +1,126 @@
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  
+  var PoolLayer = function(opt) {
+
+    var opt = opt || {};
+
+    // required
+    this.sx = opt.sx; // filter size
+    this.in_depth = opt.in_depth;
+    this.in_sx = opt.in_sx;
+    this.in_sy = opt.in_sy;
+
+    // optional
+    this.sy = typeof opt.sy !== 'undefined' ? opt.sy : this.sx;
+    this.stride = typeof opt.stride !== 'undefined' ? opt.stride : 2;
+    this.pad = typeof opt.pad !== 'undefined' ? opt.pad : 0; // amount of 0 padding to add around borders of input volume
+
+    // computed
+    this.out_depth = this.in_depth;
+    this.out_sx = Math.floor((this.in_sx + this.pad * 2 - this.sx) / this.stride + 1);
+    this.out_sy = Math.floor((this.in_sy + this.pad * 2 - this.sy) / this.stride + 1);
+    this.layer_type = 'pool';
+    // store switches for x,y coordinates for where the max comes from, for each output neuron
+    this.switchx = global.zeros(this.out_sx*this.out_sy*this.out_depth);
+    this.switchy = global.zeros(this.out_sx*this.out_sy*this.out_depth);
+  }
+
+  PoolLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+
+      var A = new Vol(this.out_sx, this.out_sy, this.out_depth, 0.0);
+      
+      var n=0; // a counter for switches
+      for(var d=0;d<this.out_depth;d++) {
+        var x = -this.pad;
+        var y = -this.pad;
+        for(var ax=0; ax<this.out_sx; x+=this.stride,ax++) {
+          y = -this.pad;
+          for(var ay=0; ay<this.out_sy; y+=this.stride,ay++) {
+
+            // convolve centered at this particular location
+            var a = -99999; // hopefully small enough ;\
+            var winx=-1,winy=-1;
+            for(var fx=0;fx<this.sx;fx++) {
+              for(var fy=0;fy<this.sy;fy++) {
+                var oy = y+fy;
+                var ox = x+fx;
+                if(oy>=0 && oy<V.sy && ox>=0 && ox<V.sx) {
+                  var v = V.get(ox, oy, d);
+                  // perform max pooling and store pointers to where
+                  // the max came from. This will speed up backprop 
+                  // and can help make nice visualizations in future
+                  if(v > a) { a = v; winx=ox; winy=oy;}
+                }
+              }
+            }
+            this.switchx[n] = winx;
+            this.switchy[n] = winy;
+            n++;
+            A.set(ax, ay, d, a);
+          }
+        }
+      }
+      this.out_act = A;
+      return this.out_act;
+    },
+    backward: function() { 
+      // pooling layers have no parameters, so simply compute 
+      // gradient wrt data here
+      var V = this.in_act;
+      V.dw = global.zeros(V.w.length); // zero out gradient wrt data
+      var A = this.out_act; // computed in forward pass 
+
+      var n = 0;
+      for(var d=0;d<this.out_depth;d++) {
+        var x = -this.pad;
+        var y = -this.pad;
+        for(var ax=0; ax<this.out_sx; x+=this.stride,ax++) {
+          y = -this.pad;
+          for(var ay=0; ay<this.out_sy; y+=this.stride,ay++) {
+
+            var chain_grad = this.out_act.get_grad(ax,ay,d);
+            V.add_grad(this.switchx[n], this.switchy[n], d, chain_grad);
+            n++;
+
+          }
+        }
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.sx = this.sx;
+      json.sy = this.sy;
+      json.stride = this.stride;
+      json.in_depth = this.in_depth;
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.pad = this.pad;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.sx = json.sx;
+      this.sy = json.sy;
+      this.stride = json.stride;
+      this.in_depth = json.in_depth;
+      this.pad = typeof json.pad !== 'undefined' ? json.pad : 0; // backwards compatibility
+      this.switchx = global.zeros(this.out_sx*this.out_sy*this.out_depth); // need to re-init these appropriately
+      this.switchy = global.zeros(this.out_sx*this.out_sy*this.out_depth);
+    }
+  }
+
+  global.PoolLayer = PoolLayer;
+
+})(convnetjs);

src/convnet_magicnet.js

@@ -0,0 +1,321 @@
+(function(global) {
+  "use strict";
+
+  // used utilities, make explicit local references
+  var randf = global.randf;
+  var randi = global.randi;
+  var Net = global.Net;
+  var Trainer = global.Trainer;
+  var maxmin = global.maxmin;
+  var randperm = global.randperm;
+  var weightedSample = global.weightedSample;
+  var getopt = global.getopt;
+  var arrUnique = global.arrUnique;
+
+  /*
+  A MagicNet takes data: a list of convnetjs.Vol(), and labels
+  which for now are assumed to be class indeces 0..K. MagicNet then:
+  - creates data folds for cross-validation
+  - samples candidate networks
+  - evaluates candidate networks on all data folds
+  - produces predictions by model-averaging the best networks
+  */
+  var MagicNet = function(data, labels, opt) {
+    var opt = opt || {};
+    if(typeof data === 'undefined') { data = []; }
+    if(typeof labels === 'undefined') { labels = []; }
+
+    // required inputs
+    this.data = data; // store these pointers to data
+    this.labels = labels;
+
+    // optional inputs
+    this.train_ratio = getopt(opt, 'train_ratio', 0.7);
+    this.num_folds = getopt(opt, 'num_folds', 10);
+    this.num_candidates = getopt(opt, 'num_candidates', 50); // we evaluate several in parallel
+    // how many epochs of data to train every network? for every fold?
+    // higher values mean higher accuracy in final results, but more expensive
+    this.num_epochs = getopt(opt, 'num_epochs', 50); 
+    // number of best models to average during prediction. Usually higher = better
+    this.ensemble_size = getopt(opt, 'ensemble_size', 10);
+
+    // candidate parameters
+    this.batch_size_min = getopt(opt, 'batch_size_min', 10);
+    this.batch_size_max = getopt(opt, 'batch_size_max', 300);
+    this.l2_decay_min = getopt(opt, 'l2_decay_min', -4);
+    this.l2_decay_max = getopt(opt, 'l2_decay_max', 2);
+    this.learning_rate_min = getopt(opt, 'learning_rate_min', -4);
+    this.learning_rate_max = getopt(opt, 'learning_rate_max', 0);
+    this.momentum_min = getopt(opt, 'momentum_min', 0.9);
+    this.momentum_max = getopt(opt, 'momentum_max', 0.9);
+    this.neurons_min = getopt(opt, 'neurons_min', 5);
+    this.neurons_max = getopt(opt, 'neurons_max', 30);
+
+    // computed
+    this.folds = []; // data fold indices, gets filled by sampleFolds()
+    this.candidates = []; // candidate networks that are being currently evaluated
+    this.evaluated_candidates = []; // history of all candidates that were fully evaluated on all folds
+    this.unique_labels = arrUnique(labels);
+    this.iter = 0; // iteration counter, goes from 0 -> num_epochs * num_training_data
+    this.foldix = 0; // index of active fold
+
+    // callbacks
+    this.finish_fold_callback = null;
+    this.finish_batch_callback = null;
+
+    // initializations
+    if(this.data.length > 0) {
+      this.sampleFolds();
+      this.sampleCandidates();
+    }
+  };
+
+  MagicNet.prototype = {
+
+    // sets this.folds to a sampling of this.num_folds folds
+    sampleFolds: function() {
+      var N = this.data.length;
+      var num_train = Math.floor(this.train_ratio * N);
+      this.folds = []; // flush folds, if any
+      for(var i=0;i<this.num_folds;i++) {
+        var p = randperm(N);
+        this.folds.push({train_ix: p.slice(0, num_train), test_ix: p.slice(num_train, N)});
+      }
+    },
+
+    // returns a random candidate network
+    sampleCandidate: function() {
+      var input_depth = this.data[0].w.length;
+      var num_classes = this.unique_labels.length;
+
+      // sample network topology and hyperparameters
+      var layer_defs = [];
+      layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth: input_depth});
+      var nl = weightedSample([0,1,2,3], [0.2, 0.3, 0.3, 0.2]); // prefer nets with 1,2 hidden layers
+      for(var q=0;q<nl;q++) {
+        var ni = randi(this.neurons_min, this.neurons_max);
+        var act = ['tanh','maxout','relu'][randi(0,3)];
+        if(randf(0,1)<0.5) {
+          var dp = Math.random();
+          layer_defs.push({type:'fc', num_neurons: ni, activation: act, drop_prob: dp});
+        } else {
+          layer_defs.push({type:'fc', num_neurons: ni, activation: act});
+        }
+      }
+      layer_defs.push({type:'softmax', num_classes: num_classes});
+      var net = new Net();
+      net.makeLayers(layer_defs);
+
+      // sample training hyperparameters
+      var bs = randi(this.batch_size_min, this.batch_size_max); // batch size
+      var l2 = Math.pow(10, randf(this.l2_decay_min, this.l2_decay_max)); // l2 weight decay
+      var lr = Math.pow(10, randf(this.learning_rate_min, this.learning_rate_max)); // learning rate
+      var mom = randf(this.momentum_min, this.momentum_max); // momentum. Lets just use 0.9, works okay usually ;p
+      var tp = randf(0,1); // trainer type
+      var trainer_def;
+      if(tp<0.33) {
+        trainer_def = {method:'adadelta', batch_size:bs, l2_decay:l2};
+      } else if(tp<0.66) {
+        trainer_def = {method:'adagrad', learning_rate: lr, batch_size:bs, l2_decay:l2};
+      } else {
+        trainer_def = {method:'sgd', learning_rate: lr, momentum: mom, batch_size:bs, l2_decay:l2};
+      }
+      
+      var trainer = new Trainer(net, trainer_def);
+
+      var cand = {};
+      cand.acc = [];
+      cand.accv = 0; // this will maintained as sum(acc) for convenience
+      cand.layer_defs = layer_defs;
+      cand.trainer_def = trainer_def;
+      cand.net = net;
+      cand.trainer = trainer;
+      return cand;
+    },
+
+    // sets this.candidates with this.num_candidates candidate nets
+    sampleCandidates: function() {
+      this.candidates = []; // flush, if any
+      for(var i=0;i<this.num_candidates;i++) {
+        var cand = this.sampleCandidate();
+        this.candidates.push(cand);
+      }
+    },
+
+    step: function() {
+      
+      // run an example through current candidate
+      this.iter++;
+
+      // step all candidates on a random data point
+      var fold = this.folds[this.foldix]; // active fold
+      var dataix = fold.train_ix[randi(0, fold.train_ix.length)];
+      for(var k=0;k<this.candidates.length;k++) {
+        var x = this.data[dataix];
+        var l = this.labels[dataix];
+        this.candidates[k].trainer.train(x, l);
+      }
+
+      // process consequences: sample new folds, or candidates
+      var lastiter = this.num_epochs * fold.train_ix.length;
+      if(this.iter >= lastiter) {
+        // finished evaluation of this fold. Get final validation
+        // accuracies, record them, and go on to next fold.
+        var val_acc = this.evalValErrors();
+        for(var k=0;k<this.candidates.length;k++) {
+          var c = this.candidates[k];
+          c.acc.push(val_acc[k]);
+          c.accv += val_acc[k];
+        }
+        this.iter = 0; // reset step number
+        this.foldix++; // increment fold
+
+        if(this.finish_fold_callback !== null) {
+          this.finish_fold_callback();
+        }
+
+        if(this.foldix >= this.folds.length) {
+          // we finished all folds as well! Record these candidates
+          // and sample new ones to evaluate.
+          for(var k=0;k<this.candidates.length;k++) {
+            this.evaluated_candidates.push(this.candidates[k]);
+          }
+          // sort evaluated candidates according to accuracy achieved
+          this.evaluated_candidates.sort(function(a, b) { 
+            return (a.accv / a.acc.length) 
+                 > (b.accv / b.acc.length) 
+                 ? -1 : 1;
+          });
+          // and clip only to the top few ones (lets place limit at 3*ensemble_size)
+          // otherwise there are concerns with keeping these all in memory 
+          // if MagicNet is being evaluated for a very long time
+          if(this.evaluated_candidates.length > 3 * this.ensemble_size) {
+            this.evaluated_candidates = this.evaluated_candidates.slice(0, 3 * this.ensemble_size);
+          }
+          if(this.finish_batch_callback !== null) {
+            this.finish_batch_callback();
+          }
+          this.sampleCandidates(); // begin with new candidates
+          this.foldix = 0; // reset this
+        } else {
+          // we will go on to another fold. reset all candidates nets
+          for(var k=0;k<this.candidates.length;k++) {
+            var c = this.candidates[k];
+            var net = new Net();
+            net.makeLayers(c.layer_defs);
+            var trainer = new Trainer(net, c.trainer_def);
+            c.net = net;
+            c.trainer = trainer;
+          }
+        }
+      }
+    },
+
+    evalValErrors: function() {
+      // evaluate candidates on validation data and return performance of current networks
+      // as simple list
+      var vals = [];
+      var fold = this.folds[this.foldix]; // active fold
+      for(var k=0;k<this.candidates.length;k++) {
+        var net = this.candidates[k].net;
+        var v = 0.0;
+        for(var q=0;q<fold.test_ix.length;q++) {
+          var x = this.data[fold.test_ix[q]];
+          var l = this.labels[fold.test_ix[q]];
+          net.forward(x);
+          var yhat = net.getPrediction();
+          v += (yhat === l ? 1.0 : 0.0); // 0 1 loss
+        }
+        v /= fold.test_ix.length; // normalize
+        vals.push(v);
+      }
+      return vals;
+    },
+
+    // returns prediction scores for given test data point, as Vol
+    // uses an averaged prediction from the best ensemble_size models
+    // x is a Vol.
+    predict_soft: function(data) {
+      // forward prop the best networks
+      // and accumulate probabilities at last layer into a an output Vol
+
+      var eval_candidates = [];
+      var nv = 0;
+      if(this.evaluated_candidates.length === 0) {
+        // not sure what to do here, first batch of nets hasnt evaluated yet
+        // lets just predict with current candidates.
+        nv = this.candidates.length;
+        eval_candidates = this.candidates;
+      } else {
+        // forward prop the best networks from evaluated_candidates
+        nv = Math.min(this.ensemble_size, this.evaluated_candidates.length);
+        eval_candidates = this.evaluated_candidates
+      }
+
+      // forward nets of all candidates and average the predictions
+      var xout, n;
+      for(var j=0;j<nv;j++) {
+        var net = eval_candidates[j].net;
+        var x = net.forward(data);
+        if(j===0) { 
+          xout = x; 
+          n = x.w.length; 
+        } else {
+          // add it on
+          for(var d=0;d<n;d++) {
+            xout.w[d] += x.w[d];
+          }
+        }
+      }
+      // produce average
+      for(var d=0;d<n;d++) {
+        xout.w[d] /= nv;
+      }
+      return xout;
+    },
+
+    predict: function(data) {
+      var xout = this.predict_soft(data);
+      if(xout.w.length !== 0) {
+        var stats = maxmin(xout.w);
+        var predicted_label = stats.maxi; 
+      } else {
+        var predicted_label = -1; // error out
+      }
+      return predicted_label;
+
+    },
+
+    toJSON: function() {
+      // dump the top ensemble_size networks as a list
+      var nv = Math.min(this.ensemble_size, this.evaluated_candidates.length);
+      var json = {};
+      json.nets = [];
+      for(var i=0;i<nv;i++) {
+        json.nets.push(this.evaluated_candidates[i].net.toJSON());
+      }
+      return json;
+    },
+
+    fromJSON: function(json) {
+      this.ensemble_size = json.nets.length;
+      this.evaluated_candidates = [];
+      for(var i=0;i<this.ensemble_size;i++) {
+        var net = new Net();
+        net.fromJSON(json.nets[i]);
+        var dummy_candidate = {};
+        dummy_candidate.net = net;
+        this.evaluated_candidates.push(dummy_candidate);
+      }
+    },
+
+    // callback functions
+    // called when a fold is finished, while evaluating a batch
+    onFinishFold: function(f) { this.finish_fold_callback = f; },
+    // called when a batch of candidates has finished evaluating
+    onFinishBatch: function(f) { this.finish_batch_callback = f; }
+    
+  };
+
+  global.MagicNet = MagicNet;
+})(convnetjs);

src/convnet_net.js

@@ -0,0 +1,188 @@
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  var assert = global.assert;
+
+  // Net manages a set of layers
+  // For now constraints: Simple linear order of layers, first layer input last layer a cost layer
+  var Net = function(options) {
+    this.layers = [];
+  }
+
+  Net.prototype = {
+    
+    // takes a list of layer definitions and creates the network layer objects
+    makeLayers: function(defs) {
+
+      // few checks
+      assert(defs.length >= 2, 'Error! At least one input layer and one loss layer are required.');
+      assert(defs[0].type === 'input', 'Error! First layer must be the input layer, to declare size of inputs');
+
+      // desugar layer_defs for adding activation, dropout layers etc
+      var desugar = function() {
+        var new_defs = [];
+        for(var i=0;i<defs.length;i++) {
+          var def = defs[i];
+          
+          if(def.type==='softmax' || def.type==='svm') {
+            // add an fc layer here, there is no reason the user should
+            // have to worry about this and we almost always want to
+            new_defs.push({type:'fc', num_neurons: def.num_classes});
+          }
+
+          if(def.type==='regression') {
+            // add an fc layer here, there is no reason the user should
+            // have to worry about this and we almost always want to
+            new_defs.push({type:'fc', num_neurons: def.num_neurons});
+          }
+
+          if((def.type==='fc' || def.type==='conv') 
+              && typeof(def.bias_pref) === 'undefined'){
+            def.bias_pref = 0.0;
+            if(typeof def.activation !== 'undefined' && def.activation === 'relu') {
+              def.bias_pref = 0.1; // relus like a bit of positive bias to get gradients early
+              // otherwise it's technically possible that a relu unit will never turn on (by chance)
+              // and will never get any gradient and never contribute any computation. Dead relu.
+            }
+          }
+
+          new_defs.push(def);
+
+          if(typeof def.activation !== 'undefined') {
+            if(def.activation==='relu') { new_defs.push({type:'relu'}); }
+            else if (def.activation==='sigmoid') { new_defs.push({type:'sigmoid'}); }
+            else if (def.activation==='tanh') { new_defs.push({type:'tanh'}); }
+            else if (def.activation==='maxout') {
+              // create maxout activation, and pass along group size, if provided
+              var gs = def.group_size !== 'undefined' ? def.group_size : 2;
+              new_defs.push({type:'maxout', group_size:gs});
+            }
+            else { console.log('ERROR unsupported activation ' + def.activation); }
+          }
+          if(typeof def.drop_prob !== 'undefined' && def.type !== 'dropout') {
+            new_defs.push({type:'dropout', drop_prob: def.drop_prob});
+          }
+
+        }
+        return new_defs;
+      }
+      defs = desugar(defs);
+
+      // create the layers
+      this.layers = [];
+      for(var i=0;i<defs.length;i++) {
+        var def = defs[i];
+        if(i>0) {
+          var prev = this.layers[i-1];
+          def.in_sx = prev.out_sx;
+          def.in_sy = prev.out_sy;
+          def.in_depth = prev.out_depth;
+        }
+
+        switch(def.type) {
+          case 'fc': this.layers.push(new global.FullyConnLayer(def)); break;
+          case 'lrn': this.layers.push(new global.LocalResponseNormalizationLayer(def)); break;
+          case 'dropout': this.layers.push(new global.DropoutLayer(def)); break;
+          case 'input': this.layers.push(new global.InputLayer(def)); break;
+          case 'softmax': this.layers.push(new global.SoftmaxLayer(def)); break;
+          case 'regression': this.layers.push(new global.RegressionLayer(def)); break;
+          case 'conv': this.layers.push(new global.ConvLayer(def)); break;
+          case 'pool': this.layers.push(new global.PoolLayer(def)); break;
+          case 'relu': this.layers.push(new global.ReluLayer(def)); break;
+          case 'sigmoid': this.layers.push(new global.SigmoidLayer(def)); break;
+          case 'tanh': this.layers.push(new global.TanhLayer(def)); break;
+          case 'maxout': this.layers.push(new global.MaxoutLayer(def)); break;
+          case 'svm': this.layers.push(new global.SVMLayer(def)); break;
+          default: console.log('ERROR: UNRECOGNIZED LAYER TYPE: ' + def.type);
+        }
+      }
+    },
+
+    // forward prop the network. 
+    // The trainer class passes is_training = true, but when this function is
+    // called from outside (not from the trainer), it defaults to prediction mode
+    forward: function(V, is_training) {
+      if(typeof(is_training) === 'undefined') is_training = false;
+      var act = this.layers[0].forward(V, is_training);
+      for(var i=1;i<this.layers.length;i++) {
+        act = this.layers[i].forward(act, is_training);
+      }
+      return act;
+    },
+
+    getCostLoss: function(V, y) {
+      this.forward(V, false);
+      var N = this.layers.length;
+      var loss = this.layers[N-1].backward(y);
+      return loss;
+    },
+    
+    // backprop: compute gradients wrt all parameters
+    backward: function(y) {
+      var N = this.layers.length;
+      var loss = this.layers[N-1].backward(y); // last layer assumed to be loss layer
+      for(var i=N-2;i>=0;i--) { // first layer assumed input
+        this.layers[i].backward();
+      }
+      return loss;
+    },
+    getParamsAndGrads: function() {
+      // accumulate parameters and gradients for the entire network
+      var response = [];
+      for(var i=0;i<this.layers.length;i++) {
+        var layer_reponse = this.layers[i].getParamsAndGrads();
+        for(var j=0;j<layer_reponse.length;j++) {
+          response.push(layer_reponse[j]);
+        }
+      }
+      return response;
+    },
+    getPrediction: function() {
+      // this is a convenience function for returning the argmax
+      // prediction, assuming the last layer of the net is a softmax
+      var S = this.layers[this.layers.length-1];
+      assert(S.layer_type === 'softmax', 'getPrediction function assumes softmax as last layer of the net!');
+
+      var p = S.out_act.w;
+      var maxv = p[0];
+      var maxi = 0;
+      for(var i=1;i<p.length;i++) {
+        if(p[i] > maxv) { maxv = p[i]; maxi = i;}
+      }
+      return maxi; // return index of the class with highest class probability
+    },
+    toJSON: function() {
+      var json = {};
+      json.layers = [];
+      for(var i=0;i<this.layers.length;i++) {
+        json.layers.push(this.layers[i].toJSON());
+      }
+      return json;
+    },
+    fromJSON: function(json) {
+      this.layers = [];
+      for(var i=0;i<json.layers.length;i++) {
+        var Lj = json.layers[i]
+        var t = Lj.layer_type;
+        var L;
+        if(t==='input') { L = new global.InputLayer(); }
+        if(t==='relu') { L = new global.ReluLayer(); }
+        if(t==='sigmoid') { L = new global.SigmoidLayer(); }
+        if(t==='tanh') { L = new global.TanhLayer(); }
+        if(t==='dropout') { L = new global.DropoutLayer(); }
+        if(t==='conv') { L = new global.ConvLayer(); }
+        if(t==='pool') { L = new global.PoolLayer(); }
+        if(t==='lrn') { L = new global.LocalResponseNormalizationLayer(); }
+        if(t==='softmax') { L = new global.SoftmaxLayer(); }
+        if(t==='regression') { L = new global.RegressionLayer(); }
+        if(t==='fc') { L = new global.FullyConnLayer(); }
+        if(t==='maxout') { L = new global.MaxoutLayer(); }
+        if(t==='svm') { L = new global.SVMLayer(); }
+        L.fromJSON(Lj);
+        this.layers.push(L);
+      }
+    }
+  }
+  
+  global.Net = Net;
+})(convnetjs);

src/convnet_trainers.js

@@ -0,0 +1,156 @@
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+
+  var Trainer = function(net, options) {
+
+    this.net = net;
+
+    var options = options || {};
+    this.learning_rate = typeof options.learning_rate !== 'undefined' ? options.learning_rate : 0.01;
+    this.l1_decay = typeof options.l1_decay !== 'undefined' ? options.l1_decay : 0.0;
+    this.l2_decay = typeof options.l2_decay !== 'undefined' ? options.l2_decay : 0.0;
+    this.batch_size = typeof options.batch_size !== 'undefined' ? options.batch_size : 1;
+    this.method = typeof options.method !== 'undefined' ? options.method : 'sgd'; // sgd/adam/adagrad/adadelta/windowgrad/netsterov
+
+    this.momentum = typeof options.momentum !== 'undefined' ? options.momentum : 0.9;
+    this.ro = typeof options.ro !== 'undefined' ? options.ro : 0.95; // used in adadelta
+    this.eps = typeof options.eps !== 'undefined' ? options.eps : 1e-8; // used in adam or adadelta
+    this.beta1 = typeof options.beta1 !== 'undefined' ? options.beta1 : 0.9; // used in adam
+    this.beta2 = typeof options.beta2 !== 'undefined' ? options.beta2 : 0.999; // used in adam
+
+    this.k = 0; // iteration counter
+    this.gsum = []; // last iteration gradients (used for momentum calculations)
+    this.xsum = []; // used in adam or adadelta
+
+    // check if regression is expected 
+    if(this.net.layers[this.net.layers.length - 1].layer_type === "regression")
+      this.regression = true;
+    else
+      this.regression = false;
+  }
+
+  Trainer.prototype = {
+    train: function(x, y) {
+
+      var start = new Date().getTime();
+      this.net.forward(x, true); // also set the flag that lets the net know we're just training
+      var end = new Date().getTime();
+      var fwd_time = end - start;
+
+      var start = new Date().getTime();
+      var cost_loss = this.net.backward(y);
+      var l2_decay_loss = 0.0;
+      var l1_decay_loss = 0.0;
+      var end = new Date().getTime();
+      var bwd_time = end - start;
+
+      if(this.regression && y.constructor !== Array)
+        console.log("Warning: a regression net requires an array as training output vector.");
+      
+      this.k++;
+      if(this.k % this.batch_size === 0) {
+
+        var pglist = this.net.getParamsAndGrads();
+
+        // initialize lists for accumulators. Will only be done once on first iteration
+        if(this.gsum.length === 0 && (this.method !== 'sgd' || this.momentum > 0.0)) {
+          // only vanilla sgd doesnt need either lists
+          // momentum needs gsum
+          // adagrad needs gsum
+          // adam and adadelta needs gsum and xsum
+          for(var i=0;i<pglist.length;i++) {
+            this.gsum.push(global.zeros(pglist[i].params.length));
+            if(this.method === 'adam' || this.method === 'adadelta') {
+              this.xsum.push(global.zeros(pglist[i].params.length));
+            } else {
+              this.xsum.push([]); // conserve memory
+            }
+          }
+        }
+
+        // perform an update for all sets of weights
+        for(var i=0;i<pglist.length;i++) {
+          var pg = pglist[i]; // param, gradient, other options in future (custom learning rate etc)
+          var p = pg.params;
+          var g = pg.grads;
+
+          // learning rate for some parameters.
+          var l2_decay_mul = typeof pg.l2_decay_mul !== 'undefined' ? pg.l2_decay_mul : 1.0;
+          var l1_decay_mul = typeof pg.l1_decay_mul !== 'undefined' ? pg.l1_decay_mul : 1.0;
+          var l2_decay = this.l2_decay * l2_decay_mul;
+          var l1_decay = this.l1_decay * l1_decay_mul;
+
+          var plen = p.length;
+          for(var j=0;j<plen;j++) {
+            l2_decay_loss += l2_decay*p[j]*p[j]/2; // accumulate weight decay loss
+            l1_decay_loss += l1_decay*Math.abs(p[j]);
+            var l1grad = l1_decay * (p[j] > 0 ? 1 : -1);
+            var l2grad = l2_decay * (p[j]);
+
+            var gij = (l2grad + l1grad + g[j]) / this.batch_size; // raw batch gradient
+
+            var gsumi = this.gsum[i];
+            var xsumi = this.xsum[i];
+            if(this.method === 'adam') {
+              // adam update
+              gsumi[j] = gsumi[j] * this.beta1 + (1- this.beta1) * gij; // update biased first moment estimate
+              xsumi[j] = xsumi[j] * this.beta2 + (1-this.beta2) * gij * gij; // update biased second moment estimate
+              var biasCorr1 = gsumi[j] * (1 - Math.pow(this.beta1, this.k)); // correct bias first moment estimate
+              var biasCorr2 = xsumi[j] * (1 - Math.pow(this.beta2, this.k)); // correct bias second moment estimate
+              var dx =  - this.learning_rate * biasCorr1 / (Math.sqrt(biasCorr2) + this.eps);
+              p[j] += dx;
+            } else if(this.method === 'adagrad') {
+              // adagrad update
+              gsumi[j] = gsumi[j] + gij * gij;
+              var dx = - this.learning_rate / Math.sqrt(gsumi[j] + this.eps) * gij;
+              p[j] += dx;
+            } else if(this.method === 'windowgrad') {
+              // this is adagrad but with a moving window weighted average
+              // so the gradient is not accumulated over the entire history of the run. 
+              // it's also referred to as Idea #1 in Zeiler paper on Adadelta. Seems reasonable to me!
+              gsumi[j] = this.ro * gsumi[j] + (1-this.ro) * gij * gij;
+              var dx = - this.learning_rate / Math.sqrt(gsumi[j] + this.eps) * gij; // eps added for better conditioning
+              p[j] += dx;
+            } else if(this.method === 'adadelta') {
+              gsumi[j] = this.ro * gsumi[j] + (1-this.ro) * gij * gij;
+              var dx = - Math.sqrt((xsumi[j] + this.eps)/(gsumi[j] + this.eps)) * gij;
+              xsumi[j] = this.ro * xsumi[j] + (1-this.ro) * dx * dx; // yes, xsum lags behind gsum by 1.
+              p[j] += dx;
+            } else if(this.method === 'nesterov') {
+            	var dx = gsumi[j];
+            	gsumi[j] = gsumi[j] * this.momentum + this.learning_rate * gij;
+                dx = this.momentum * dx - (1.0 + this.momentum) * gsumi[j];
+                p[j] += dx;
+            } else {
+              // assume SGD
+              if(this.momentum > 0.0) {
+                // momentum update
+                var dx = this.momentum * gsumi[j] - this.learning_rate * gij; // step
+                gsumi[j] = dx; // back this up for next iteration of momentum
+                p[j] += dx; // apply corrected gradient
+              } else {
+                // vanilla sgd
+                p[j] +=  - this.learning_rate * gij;
+              }
+            }
+            g[j] = 0.0; // zero out gradient so that we can begin accumulating anew
+          }
+        }
+      }
+
+      // appending softmax_loss for backwards compatibility, but from now on we will always use cost_loss
+      // in future, TODO: have to completely redo the way loss is done around the network as currently 
+      // loss is a bit of a hack. Ideally, user should specify arbitrary number of loss functions on any layer
+      // and it should all be computed correctly and automatically. 
+      return {fwd_time: fwd_time, bwd_time: bwd_time, 
+              l2_decay_loss: l2_decay_loss, l1_decay_loss: l1_decay_loss,
+              cost_loss: cost_loss, softmax_loss: cost_loss, 
+              loss: cost_loss + l1_decay_loss + l2_decay_loss}
+    }
+  }
+  
+  global.Trainer = Trainer;
+  global.SGDTrainer = Trainer; // backwards compatibility
+})(convnetjs);
+

src/convnet_util.js

@@ -0,0 +1,137 @@
+(function(global) {
+  "use strict";
+
+  // Random number utilities
+  var return_v = false;
+  var v_val = 0.0;
+  var gaussRandom = function() {
+    if(return_v) { 
+      return_v = false;
+      return v_val; 
+    }
+    var u = 2*Math.random()-1;
+    var v = 2*Math.random()-1;
+    var r = u*u + v*v;
+    if(r == 0 || r > 1) return gaussRandom();
+    var c = Math.sqrt(-2*Math.log(r)/r);
+    v_val = v*c; // cache this
+    return_v = true;
+    return u*c;
+  }
+  var randf = function(a, b) { return Math.random()*(b-a)+a; }
+  var randi = function(a, b) { return Math.floor(Math.random()*(b-a)+a); }
+  var randn = function(mu, std){ return mu+gaussRandom()*std; }
+
+  // Array utilities
+  var zeros = function(n) {
+    if(typeof(n)==='undefined' || isNaN(n)) { return []; }
+    if(typeof ArrayBuffer === 'undefined') {
+      // lacking browser support
+      var arr = new Array(n);
+      for(var i=0;i<n;i++) { arr[i]= 0; }
+      return arr;
+    } else {
+      return new Float64Array(n);
+    }
+  }
+
+  var arrContains = function(arr, elt) {
+    for(var i=0,n=arr.length;i<n;i++) {
+      if(arr[i]===elt) return true;
+    }
+    return false;
+  }
+
+  var arrUnique = function(arr) {
+    var b = [];
+    for(var i=0,n=arr.length;i<n;i++) {
+      if(!arrContains(b, arr[i])) {
+        b.push(arr[i]);
+      }
+    }
+    return b;
+  }
+
+  // return max and min of a given non-empty array.
+  var maxmin = function(w) {
+    if(w.length === 0) { return {}; } // ... ;s
+    var maxv = w[0];
+    var minv = w[0];
+    var maxi = 0;
+    var mini = 0;
+    var n = w.length;
+    for(var i=1;i<n;i++) {
+      if(w[i] > maxv) { maxv = w[i]; maxi = i; } 
+      if(w[i] < minv) { minv = w[i]; mini = i; } 
+    }
+    return {maxi: maxi, maxv: maxv, mini: mini, minv: minv, dv:maxv-minv};
+  }
+
+  // create random permutation of numbers, in range [0...n-1]
+  var randperm = function(n) {
+    var i = n,
+        j = 0,
+        temp;
+    var array = [];
+    for(var q=0;q<n;q++)array[q]=q;
+    while (i--) {
+        j = Math.floor(Math.random() * (i+1));
+        temp = array[i];
+        array[i] = array[j];
+        array[j] = temp;
+    }
+    return array;
+  }
+
+  // sample from list lst according to probabilities in list probs
+  // the two lists are of same size, and probs adds up to 1
+  var weightedSample = function(lst, probs) {
+    var p = randf(0, 1.0);
+    var cumprob = 0.0;
+    for(var k=0,n=lst.length;k<n;k++) {
+      cumprob += probs[k];
+      if(p < cumprob) { return lst[k]; }
+    }
+  }
+
+  // syntactic sugar function for getting default parameter values
+  var getopt = function(opt, field_name, default_value) {
+    if(typeof field_name === 'string') {
+      // case of single string
+      return (typeof opt[field_name] !== 'undefined') ? opt[field_name] : default_value;
+    } else {
+      // assume we are given a list of string instead
+      var ret = default_value;
+      for(var i=0;i<field_name.length;i++) {
+        var f = field_name[i];
+        if (typeof opt[f] !== 'undefined') {
+          ret = opt[f]; // overwrite return value
+        }
+      }
+      return ret;
+    }
+  }
+
+  function assert(condition, message) {
+    if (!condition) {
+      message = message || "Assertion failed";
+      if (typeof Error !== "undefined") {
+        throw new Error(message);
+      }
+      throw message; // Fallback
+    }
+  }
+
+  global.randf = randf;
+  global.randi = randi;
+  global.randn = randn;
+  global.zeros = zeros;
+  global.maxmin = maxmin;
+  global.randperm = randperm;
+  global.weightedSample = weightedSample;
+  global.arrUnique = arrUnique;
+  global.arrContains = arrContains;
+  global.getopt = getopt;
+  global.assert = assert;
+  
+})(convnetjs);

src/convnet_vol.js

@@ -0,0 +1,111 @@
+(function(global) {
+  "use strict";
+
+  // Vol is the basic building block of all data in a net.
+  // it is essentially just a 3D volume of numbers, with a
+  // width (sx), height (sy), and depth (depth).
+  // it is used to hold data for all filters, all volumes,
+  // all weights, and also stores all gradients w.r.t. 
+  // the data. c is optionally a value to initialize the volume
+  // with. If c is missing, fills the Vol with random numbers.
+  var Vol = function(sx, sy, depth, c) {
+    // this is how you check if a variable is an array. Oh, Javascript :)
+    if(Object.prototype.toString.call(sx) === '[object Array]') {
+      // we were given a list in sx, assume 1D volume and fill it up
+      this.sx = 1;
+      this.sy = 1;
+      this.depth = sx.length;
+      // we have to do the following copy because we want to use
+      // fast typed arrays, not an ordinary javascript array
+      this.w = global.zeros(this.depth);
+      this.dw = global.zeros(this.depth);
+      for(var i=0;i<this.depth;i++) {
+        this.w[i] = sx[i];
+      }
+    } else {
+      // we were given dimensions of the vol
+      this.sx = sx;
+      this.sy = sy;
+      this.depth = depth;
+      var n = sx*sy*depth;
+      this.w = global.zeros(n);
+      this.dw = global.zeros(n);
+      if(typeof c === 'undefined') {
+        // weight normalization is done to equalize the output
+        // variance of every neuron, otherwise neurons with a lot
+        // of incoming connections have outputs of larger variance
+        var scale = Math.sqrt(1.0/(sx*sy*depth));
+        for(var i=0;i<n;i++) { 
+          this.w[i] = global.randn(0.0, scale);
+        }
+      } else {
+        for(var i=0;i<n;i++) { 
+          this.w[i] = c;
+        }
+      }
+    }
+  }
+
+  Vol.prototype = {
+    get: function(x, y, d) { 
+      var ix=((this.sx * y)+x)*this.depth+d;
+      return this.w[ix];
+    },
+    set: function(x, y, d, v) { 
+      var ix=((this.sx * y)+x)*this.depth+d;
+      this.w[ix] = v; 
+    },
+    add: function(x, y, d, v) { 
+      var ix=((this.sx * y)+x)*this.depth+d;
+      this.w[ix] += v; 
+    },
+    get_grad: function(x, y, d) { 
+      var ix = ((this.sx * y)+x)*this.depth+d;
+      return this.dw[ix]; 
+    },
+    set_grad: function(x, y, d, v) { 
+      var ix = ((this.sx * y)+x)*this.depth+d;
+      this.dw[ix] = v; 
+    },
+    add_grad: function(x, y, d, v) { 
+      var ix = ((this.sx * y)+x)*this.depth+d;
+      this.dw[ix] += v; 
+    },
+    cloneAndZero: function() { return new Vol(this.sx, this.sy, this.depth, 0.0)},
+    clone: function() {
+      var V = new Vol(this.sx, this.sy, this.depth, 0.0);
+      var n = this.w.length;
+      for(var i=0;i<n;i++) { V.w[i] = this.w[i]; }
+      return V;
+    },
+    addFrom: function(V) { for(var k=0;k<this.w.length;k++) { this.w[k] += V.w[k]; }},
+    addFromScaled: function(V, a) { for(var k=0;k<this.w.length;k++) { this.w[k] += a*V.w[k]; }},
+    setConst: function(a) { for(var k=0;k<this.w.length;k++) { this.w[k] = a; }},
+
+    toJSON: function() {
+      // todo: we may want to only save d most significant digits to save space
+      var json = {}
+      json.sx = this.sx; 
+      json.sy = this.sy;
+      json.depth = this.depth;
+      json.w = this.w;
+      return json;
+      // we wont back up gradients to save space
+    },
+    fromJSON: function(json) {
+      this.sx = json.sx;
+      this.sy = json.sy;
+      this.depth = json.depth;
+
+      var n = this.sx*this.sy*this.depth;
+      this.w = global.zeros(n);
+      this.dw = global.zeros(n);
+      // copy over the elements.
+      for(var i=0;i<n;i++) {
+        this.w[i] = json.w[i];
+      }
+    }
+  }
+
+  global.Vol = Vol;
+})(convnetjs);

src/convnet_vol_util.js

@@ -0,0 +1,108 @@
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+
+  // Volume utilities
+  // intended for use with data augmentation
+  // crop is the size of output
+  // dx,dy are offset wrt incoming volume, of the shift
+  // fliplr is boolean on whether we also want to flip left<->right
+  var augment = function(V, crop, dx, dy, fliplr) {
+    // note assumes square outputs of size crop x crop
+    if(typeof(fliplr)==='undefined') var fliplr = false;
+    if(typeof(dx)==='undefined') var dx = global.randi(0, V.sx - crop);
+    if(typeof(dy)==='undefined') var dy = global.randi(0, V.sy - crop);
+    
+    // randomly sample a crop in the input volume
+    var W;
+    if(crop !== V.sx || dx!==0 || dy!==0) {
+      W = new Vol(crop, crop, V.depth, 0.0);
+      for(var x=0;x<crop;x++) {
+        for(var y=0;y<crop;y++) {
+          if(x+dx<0 || x+dx>=V.sx || y+dy<0 || y+dy>=V.sy) continue; // oob
+          for(var d=0;d<V.depth;d++) {
+           W.set(x,y,d,V.get(x+dx,y+dy,d)); // copy data over
+          }
+        }
+      }
+    } else {
+      W = V;
+    }
+
+    if(fliplr) {
+      // flip volume horziontally
+      var W2 = W.cloneAndZero();
+      for(var x=0;x<W.sx;x++) {
+        for(var y=0;y<W.sy;y++) {
+          for(var d=0;d<W.depth;d++) {
+           W2.set(x,y,d,W.get(W.sx - x - 1,y,d)); // copy data over
+          }
+        }
+      }
+      W = W2; //swap
+    }
+    return W;
+  }
+
+  // img is a DOM element that contains a loaded image
+  // returns a Vol of size (W, H, 4). 4 is for RGBA
+  var img_to_vol = function(img, convert_grayscale) {
+
+    if(typeof(convert_grayscale)==='undefined') var convert_grayscale = false;
+
+    var canvas = document.createElement('canvas');
+    canvas.width = img.width;
+    canvas.height = img.height;
+    var ctx = canvas.getContext("2d");
+
+    // due to a Firefox bug
+    try {
+      ctx.drawImage(img, 0, 0);
+    } catch (e) {
+      if (e.name === "NS_ERROR_NOT_AVAILABLE") {
+        // sometimes happens, lets just abort
+        return false;
+      } else {
+        throw e;
+      }
+    }
+
+    try {
+      var img_data = ctx.getImageData(0, 0, canvas.width, canvas.height);
+    } catch (e) {
+      if(e.name === 'IndexSizeError') {
+        return false; // not sure what causes this sometimes but okay abort
+      } else {
+        throw e;
+      }
+    }
+
+    // prepare the input: get pixels and normalize them
+    var p = img_data.data;
+    var W = img.width;
+    var H = img.height;
+    var pv = []
+    for(var i=0;i<p.length;i++) {
+      pv.push(p[i]/255.0-0.5); // normalize image pixels to [-0.5, 0.5]
+    }
+    var x = new Vol(W, H, 4, 0.0); //input volume (image)
+    x.w = pv;
+
+    if(convert_grayscale) {
+      // flatten into depth=1 array
+      var x1 = new Vol(W, H, 1, 0.0);
+      for(var i=0;i<W;i++) {
+        for(var j=0;j<H;j++) {
+          x1.set(i,j,0,x.get(i,j,0));
+        }
+      }
+      x = x1;
+    }
+
+    return x;
+  }
+  
+  global.augment = augment;
+  global.img_to_vol = img_to_vol;
+
+})(convnetjs);

test/jasmine/MIT.LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008-2011 Pivotal Labs
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

test/jasmine/SpecRunner.html

@@ -0,0 +1,24 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+  <title>Jasmine Spec Runner v2.0.0</title>
+
+  <link rel="shortcut icon" type="image/png" href="lib/jasmine-2.0.0/jasmine_favicon.png">
+  <link rel="stylesheet" type="text/css" href="lib/jasmine-2.0.0/jasmine.css">
+
+  <script type="text/javascript" src="lib/jasmine-2.0.0/jasmine.js"></script>
+  <script type="text/javascript" src="lib/jasmine-2.0.0/jasmine-html.js"></script>
+  <script type="text/javascript" src="lib/jasmine-2.0.0/boot.js"></script>
+
+  <!-- include source files here... -->
+  <script type="text/javascript" src="../../build/convnet.js"></script>
+
+  <!-- include spec files here... -->
+  <script type="text/javascript" src="spec/NeuralNetSpec.js"></script>
+
+</head>
+
+<body>
+</body>
+</html>

test/jasmine/lib/jasmine-2.0.0/boot.js

@@ -0,0 +1,181 @@
+/**
+ Starting with version 2.0, this file "boots" Jasmine, performing all of the necessary initialization before executing the loaded environment and all of a project's specs. This file should be loaded after `jasmine.js`, but before any project source files or spec files are loaded. Thus this file can also be used to customize Jasmine for a project.
+
+ If a project is using Jasmine via the standalone distribution, this file can be customized directly. If a project is using Jasmine via the [Ruby gem][jasmine-gem], this file can be copied into the support directory via `jasmine copy_boot_js`. Other environments (e.g., Python) will have different mechanisms.
+
+ The location of `boot.js` can be specified and/or overridden in `jasmine.yml`.
+
+ [jasmine-gem]: http://github.com/pivotal/jasmine-gem
+ */
+
+(function() {
+
+  /**
+   * ## Require & Instantiate
+   *
+   * Require Jasmine's core files. Specifically, this requires and attaches all of Jasmine's code to the `jasmine` reference.
+   */
+  window.jasmine = jasmineRequire.core(jasmineRequire);
+
+  /**
+   * Since this is being run in a browser and the results should populate to an HTML page, require the HTML-specific Jasmine code, injecting the same reference.
+   */
+  jasmineRequire.html(jasmine);
+
+  /**
+   * Create the Jasmine environment. This is used to run all specs in a project.
+   */
+  var env = jasmine.getEnv();
+
+  /**
+   * ## The Global Interface
+   *
+   * Build up the functions that will be exposed as the Jasmine public interface. A project can customize, rename or alias any of these functions as desired, provided the implementation remains unchanged.
+   */
+  var jasmineInterface = {
+    describe: function(description, specDefinitions) {
+      return env.describe(description, specDefinitions);
+    },
+
+    xdescribe: function(description, specDefinitions) {
+      return env.xdescribe(description, specDefinitions);
+    },
+
+    it: function(desc, func) {
+      return env.it(desc, func);
+    },
+
+    xit: function(desc, func) {
+      return env.xit(desc, func);
+    },
+
+    beforeEach: function(beforeEachFunction) {
+      return env.beforeEach(beforeEachFunction);
+    },
+
+    afterEach: function(afterEachFunction) {
+      return env.afterEach(afterEachFunction);
+    },
+
+    expect: function(actual) {
+      return env.expect(actual);
+    },
+
+    pending: function() {
+      return env.pending();
+    },
+
+    spyOn: function(obj, methodName) {
+      return env.spyOn(obj, methodName);
+    },
+
+    jsApiReporter: new jasmine.JsApiReporter({
+      timer: new jasmine.Timer()
+    })
+  };
+
+  /**
+   * Add all of the Jasmine global/public interface to the proper global, so a project can use the public interface directly. For example, calling `describe` in specs instead of `jasmine.getEnv().describe`.
+   */
+  if (typeof window == "undefined" && typeof exports == "object") {
+    extend(exports, jasmineInterface);
+  } else {
+    extend(window, jasmineInterface);
+  }
+
+  /**
+   * Expose the interface for adding custom equality testers.
+   */
+  jasmine.addCustomEqualityTester = function(tester) {
+    env.addCustomEqualityTester(tester);
+  };
+
+  /**
+   * Expose the interface for adding custom expectation matchers
+   */
+  jasmine.addMatchers = function(matchers) {
+    return env.addMatchers(matchers);
+  };
+
+  /**
+   * Expose the mock interface for the JavaScript timeout functions
+   */
+  jasmine.clock = function() {
+    return env.clock;
+  };
+
+  /**
+   * ## Runner Parameters
+   *
+   * More browser specific code - wrap the query string in an object and to allow for getting/setting parameters from the runner user interface.
+   */
+
+  var queryString = new jasmine.QueryString({
+    getWindowLocation: function() { return window.location; }
+  });
+
+  var catchingExceptions = queryString.getParam("catch");
+  env.catchExceptions(typeof catchingExceptions === "undefined" ? true : catchingExceptions);
+
+  /**
+   * ## Reporters
+   * The `HtmlReporter` builds all of the HTML UI for the runner page. This reporter paints the dots, stars, and x's for specs, as well as all spec names and all failures (if any).
+   */
+  var htmlReporter = new jasmine.HtmlReporter({
+    env: env,
+    onRaiseExceptionsClick: function() { queryString.setParam("catch", !env.catchingExceptions()); },
+    getContainer: function() { return document.body; },
+    createElement: function() { return document.createElement.apply(document, arguments); },
+    createTextNode: function() { return document.createTextNode.apply(document, arguments); },
+    timer: new jasmine.Timer()
+  });
+
+  /**
+   * The `jsApiReporter` also receives spec results, and is used by any environment that needs to extract the results  from JavaScript.
+   */
+  env.addReporter(jasmineInterface.jsApiReporter);
+  env.addReporter(htmlReporter);
+
+  /**
+   * Filter which specs will be run by matching the start of the full name against the `spec` query param.
+   */
+  var specFilter = new jasmine.HtmlSpecFilter({
+    filterString: function() { return queryString.getParam("spec"); }
+  });
+
+  env.specFilter = function(spec) {
+    return specFilter.matches(spec.getFullName());
+  };
+
+  /**
+   * Setting up timing functions to be able to be overridden. Certain browsers (Safari, IE 8, phantomjs) require this hack.
+   */
+  window.setTimeout = window.setTimeout;
+  window.setInterval = window.setInterval;
+  window.clearTimeout = window.clearTimeout;
+  window.clearInterval = window.clearInterval;
+
+  /**
+   * ## Execution
+   *
+   * Replace the browser window's `onload`, ensure it's called, and then run all of the loaded specs. This includes initializing the `HtmlReporter` instance and then executing the loaded Jasmine environment. All of this will happen after all of the specs are loaded.
+   */
+  var currentWindowOnload = window.onload;
+
+  window.onload = function() {
+    if (currentWindowOnload) {
+      currentWindowOnload();
+    }
+    htmlReporter.initialize();
+    env.execute();
+  };
+
+  /**
+   * Helper function for readability above.
+   */
+  function extend(destination, source) {
+    for (var property in source) destination[property] = source[property];
+    return destination;
+  }
+
+}());

test/jasmine/lib/jasmine-2.0.0/console.js

@@ -0,0 +1,160 @@
+/*
+Copyright (c) 2008-2013 Pivotal Labs
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+function getJasmineRequireObj() {
+  if (typeof module !== "undefined" && module.exports) {
+    return exports;
+  } else {
+    window.jasmineRequire = window.jasmineRequire || {};
+    return window.jasmineRequire;
+  }
+}
+
+getJasmineRequireObj().console = function(jRequire, j$) {
+  j$.ConsoleReporter = jRequire.ConsoleReporter();
+};
+
+getJasmineRequireObj().ConsoleReporter = function() {
+
+  var noopTimer = {
+    start: function(){},
+    elapsed: function(){ return 0; }
+  };
+
+  function ConsoleReporter(options) {
+    var print = options.print,
+      showColors = options.showColors || false,
+      onComplete = options.onComplete || function() {},
+      timer = options.timer || noopTimer,
+      specCount,
+      failureCount,
+      failedSpecs = [],
+      pendingCount,
+      ansi = {
+        green: '\x1B[32m',
+        red: '\x1B[31m',
+        yellow: '\x1B[33m',
+        none: '\x1B[0m'
+      };
+
+    this.jasmineStarted = function() {
+      specCount = 0;
+      failureCount = 0;
+      pendingCount = 0;
+      print("Started");
+      printNewline();
+      timer.start();
+    };
+
+    this.jasmineDone = function() {
+      printNewline();
+      for (var i = 0; i < failedSpecs.length; i++) {
+        specFailureDetails(failedSpecs[i]);
+      }
+
+      printNewline();
+      var specCounts = specCount + " " + plural("spec", specCount) + ", " +
+        failureCount + " " + plural("failure", failureCount);
+
+      if (pendingCount) {
+        specCounts += ", " + pendingCount + " pending " + plural("spec", pendingCount);
+      }
+
+      print(specCounts);
+
+      printNewline();
+      var seconds = timer.elapsed() / 1000;
+      print("Finished in " + seconds + " " + plural("second", seconds));
+
+      printNewline();
+
+      onComplete(failureCount === 0);
+    };
+
+    this.specDone = function(result) {
+      specCount++;
+
+      if (result.status == "pending") {
+        pendingCount++;
+        print(colored("yellow", "*"));
+        return;
+      }
+
+      if (result.status == "passed") {
+        print(colored("green", '.'));
+        return;
+      }
+
+      if (result.status == "failed") {
+        failureCount++;
+        failedSpecs.push(result);
+        print(colored("red", 'F'));
+      }
+    };
+
+    return this;
+
+    function printNewline() {
+      print("\n");
+    }
+
+    function colored(color, str) {
+      return showColors ? (ansi[color] + str + ansi.none) : str;
+    }
+
+    function plural(str, count) {
+      return count == 1 ? str : str + "s";
+    }
+
+    function repeat(thing, times) {
+      var arr = [];
+      for (var i = 0; i < times; i++) {
+        arr.push(thing);
+      }
+      return arr;
+    }
+
+    function indent(str, spaces) {
+      var lines = (str || '').split("\n");
+      var newArr = [];
+      for (var i = 0; i < lines.length; i++) {
+        newArr.push(repeat(" ", spaces).join("") + lines[i]);
+      }
+      return newArr.join("\n");
+    }
+
+    function specFailureDetails(result) {
+      printNewline();
+      print(result.fullName);
+
+      for (var i = 0; i < result.failedExpectations.length; i++) {
+        var failedExpectation = result.failedExpectations[i];
+        printNewline();
+        print(indent(failedExpectation.stack, 2));
+      }
+
+      printNewline();
+    }
+  }
+
+  return ConsoleReporter;
+};

test/jasmine/lib/jasmine-2.0.0/jasmine-html.js

@@ -0,0 +1,359 @@
+/*
+Copyright (c) 2008-2013 Pivotal Labs
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+jasmineRequire.html = function(j$) {
+  j$.ResultsNode = jasmineRequire.ResultsNode();
+  j$.HtmlReporter = jasmineRequire.HtmlReporter(j$);
+  j$.QueryString = jasmineRequire.QueryString();
+  j$.HtmlSpecFilter = jasmineRequire.HtmlSpecFilter();
+};
+
+jasmineRequire.HtmlReporter = function(j$) {
+
+  var noopTimer = {
+    start: function() {},
+    elapsed: function() { return 0; }
+  };
+
+  function HtmlReporter(options) {
+    var env = options.env || {},
+      getContainer = options.getContainer,
+      createElement = options.createElement,
+      createTextNode = options.createTextNode,
+      onRaiseExceptionsClick = options.onRaiseExceptionsClick || function() {},
+      timer = options.timer || noopTimer,
+      results = [],
+      specsExecuted = 0,
+      failureCount = 0,
+      pendingSpecCount = 0,
+      htmlReporterMain,
+      symbols;
+
+    this.initialize = function() {
+      htmlReporterMain = createDom("div", {className: "html-reporter"},
+        createDom("div", {className: "banner"},
+          createDom("span", {className: "title"}, "Jasmine"),
+          createDom("span", {className: "version"}, j$.version)
+        ),
+        createDom("ul", {className: "symbol-summary"}),
+        createDom("div", {className: "alert"}),
+        createDom("div", {className: "results"},
+          createDom("div", {className: "failures"})
+        )
+      );
+      getContainer().appendChild(htmlReporterMain);
+
+      symbols = find(".symbol-summary");
+    };
+
+    var totalSpecsDefined;
+    this.jasmineStarted = function(options) {
+      totalSpecsDefined = options.totalSpecsDefined || 0;
+      timer.start();
+    };
+
+    var summary = createDom("div", {className: "summary"});
+
+    var topResults = new j$.ResultsNode({}, "", null),
+      currentParent = topResults;
+
+    this.suiteStarted = function(result) {
+      currentParent.addChild(result, "suite");
+      currentParent = currentParent.last();
+    };
+
+    this.suiteDone = function(result) {
+      if (currentParent == topResults) {
+        return;
+      }
+
+      currentParent = currentParent.parent;
+    };
+
+    this.specStarted = function(result) {
+      currentParent.addChild(result, "spec");
+    };
+
+    var failures = [];
+    this.specDone = function(result) {
+      if (result.status != "disabled") {
+        specsExecuted++;
+      }
+
+      symbols.appendChild(createDom("li", {
+          className: result.status,
+          id: "spec_" + result.id,
+          title: result.fullName
+        }
+      ));
+
+      if (result.status == "failed") {
+        failureCount++;
+
+        var failure =
+          createDom("div", {className: "spec-detail failed"},
+            createDom("div", {className: "description"},
+              createDom("a", {title: result.fullName, href: specHref(result)}, result.fullName)
+            ),
+            createDom("div", {className: "messages"})
+          );
+        var messages = failure.childNodes[1];
+
+        for (var i = 0; i < result.failedExpectations.length; i++) {
+          var expectation = result.failedExpectations[i];
+          messages.appendChild(createDom("div", {className: "result-message"}, expectation.message));
+          messages.appendChild(createDom("div", {className: "stack-trace"}, expectation.stack));
+        }
+
+        failures.push(failure);
+      }
+
+      if (result.status == "pending") {
+        pendingSpecCount++;
+      }
+    };
+
+    this.jasmineDone = function() {
+      var banner = find(".banner");
+      banner.appendChild(createDom("span", {className: "duration"}, "finished in " + timer.elapsed() / 1000 + "s"));
+
+      var alert = find(".alert");
+
+      alert.appendChild(createDom("span", { className: "exceptions" },
+        createDom("label", { className: "label", 'for': "raise-exceptions" }, "raise exceptions"),
+        createDom("input", {
+          className: "raise",
+          id: "raise-exceptions",
+          type: "checkbox"
+        })
+      ));
+      var checkbox = find("input");
+
+      checkbox.checked = !env.catchingExceptions();
+      checkbox.onclick = onRaiseExceptionsClick;
+
+      if (specsExecuted < totalSpecsDefined) {
+        var skippedMessage = "Ran " + specsExecuted + " of " + totalSpecsDefined + " specs - run all";
+        alert.appendChild(
+          createDom("span", {className: "bar skipped"},
+            createDom("a", {href: "?", title: "Run all specs"}, skippedMessage)
+          )
+        );
+      }
+      var statusBarMessage = "" + pluralize("spec", specsExecuted) + ", " + pluralize("failure", failureCount);
+      if (pendingSpecCount) { statusBarMessage += ", " + pluralize("pending spec", pendingSpecCount); }
+
+      var statusBarClassName = "bar " + ((failureCount > 0) ? "failed" : "passed");
+      alert.appendChild(createDom("span", {className: statusBarClassName}, statusBarMessage));
+
+      var results = find(".results");
+      results.appendChild(summary);
+
+      summaryList(topResults, summary);
+
+      function summaryList(resultsTree, domParent) {
+        var specListNode;
+        for (var i = 0; i < resultsTree.children.length; i++) {
+          var resultNode = resultsTree.children[i];
+          if (resultNode.type == "suite") {
+            var suiteListNode = createDom("ul", {className: "suite", id: "suite-" + resultNode.result.id},
+              createDom("li", {className: "suite-detail"},
+                createDom("a", {href: specHref(resultNode.result)}, resultNode.result.description)
+              )
+            );
+
+            summaryList(resultNode, suiteListNode);
+            domParent.appendChild(suiteListNode);
+          }
+          if (resultNode.type == "spec") {
+            if (domParent.getAttribute("class") != "specs") {
+              specListNode = createDom("ul", {className: "specs"});
+              domParent.appendChild(specListNode);
+            }
+            specListNode.appendChild(
+              createDom("li", {
+                  className: resultNode.result.status,
+                  id: "spec-" + resultNode.result.id
+                },
+                createDom("a", {href: specHref(resultNode.result)}, resultNode.result.description)
+              )
+            );
+          }
+        }
+      }
+
+      if (failures.length) {
+        alert.appendChild(
+          createDom('span', {className: "menu bar spec-list"},
+            createDom("span", {}, "Spec List | "),
+            createDom('a', {className: "failures-menu", href: "#"}, "Failures")));
+        alert.appendChild(
+          createDom('span', {className: "menu bar failure-list"},
+            createDom('a', {className: "spec-list-menu", href: "#"}, "Spec List"),
+            createDom("span", {}, " | Failures ")));
+
+        find(".failures-menu").onclick = function() {
+          setMenuModeTo('failure-list');
+        };
+        find(".spec-list-menu").onclick = function() {
+          setMenuModeTo('spec-list');
+        };
+
+        setMenuModeTo('failure-list');
+
+        var failureNode = find(".failures");
+        for (var i = 0; i < failures.length; i++) {
+          failureNode.appendChild(failures[i]);
+        }
+      }
+    };
+
+    return this;
+
+    function find(selector) {
+      return getContainer().querySelector(selector);
+    }
+
+    function createDom(type, attrs, childrenVarArgs) {
+      var el = createElement(type);
+
+      for (var i = 2; i < arguments.length; i++) {
+        var child = arguments[i];
+
+        if (typeof child === 'string') {
+          el.appendChild(createTextNode(child));
+        } else {
+          if (child) {
+            el.appendChild(child);
+          }
+        }
+      }
+
+      for (var attr in attrs) {
+        if (attr == "className") {
+          el[attr] = attrs[attr];
+        } else {
+          el.setAttribute(attr, attrs[attr]);
+        }
+      }
+
+      return el;
+    }
+
+    function pluralize(singular, count) {
+      var word = (count == 1 ? singular : singular + "s");
+
+      return "" + count + " " + word;
+    }
+
+    function specHref(result) {
+      return "?spec=" + encodeURIComponent(result.fullName);
+    }
+
+    function setMenuModeTo(mode) {
+      htmlReporterMain.setAttribute("class", "html-reporter " + mode);
+    }
+  }
+
+  return HtmlReporter;
+};
+
+jasmineRequire.HtmlSpecFilter = function() {
+  function HtmlSpecFilter(options) {
+    var filterString = options && options.filterString() && options.filterString().replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
+    var filterPattern = new RegExp(filterString);
+
+    this.matches = function(specName) {
+      return filterPattern.test(specName);
+    };
+  }
+
+  return HtmlSpecFilter;
+};
+
+jasmineRequire.ResultsNode = function() {
+  function ResultsNode(result, type, parent) {
+    this.result = result;
+    this.type = type;
+    this.parent = parent;
+
+    this.children = [];
+
+    this.addChild = function(result, type) {
+      this.children.push(new ResultsNode(result, type, this));
+    };
+
+    this.last = function() {
+      return this.children[this.children.length - 1];
+    };
+  }
+
+  return ResultsNode;
+};
+
+jasmineRequire.QueryString = function() {
+  function QueryString(options) {
+
+    this.setParam = function(key, value) {
+      var paramMap = queryStringToParamMap();
+      paramMap[key] = value;
+      options.getWindowLocation().search = toQueryString(paramMap);
+    };
+
+    this.getParam = function(key) {
+      return queryStringToParamMap()[key];
+    };
+
+    return this;
+
+    function toQueryString(paramMap) {
+      var qStrPairs = [];
+      for (var prop in paramMap) {
+        qStrPairs.push(encodeURIComponent(prop) + "=" + encodeURIComponent(paramMap[prop]));
+      }
+      return "?" + qStrPairs.join('&');
+    }
+
+    function queryStringToParamMap() {
+      var paramStr = options.getWindowLocation().search.substring(1),
+        params = [],
+        paramMap = {};
+
+      if (paramStr.length > 0) {
+        params = paramStr.split('&');
+        for (var i = 0; i < params.length; i++) {
+          var p = params[i].split('=');
+          var value = decodeURIComponent(p[1]);
+          if (value === "true" || value === "false") {
+            value = JSON.parse(value);
+          }
+          paramMap[decodeURIComponent(p[0])] = value;
+        }
+      }
+
+      return paramMap;
+    }
+
+  }
+
+  return QueryString;
+};

test/jasmine/lib/jasmine-2.0.0/jasmine.css

@@ -0,0 +1,55 @@
+body { background-color: #eeeeee; padding: 0; margin: 5px; overflow-y: scroll; }
+
+.html-reporter { font-size: 11px; font-family: Monaco, "Lucida Console", monospace; line-height: 14px; color: #333333; }
+.html-reporter a { text-decoration: none; }
+.html-reporter a:hover { text-decoration: underline; }
+.html-reporter p, .html-reporter h1, .html-reporter h2, .html-reporter h3, .html-reporter h4, .html-reporter h5, .html-reporter h6 { margin: 0; line-height: 14px; }
+.html-reporter .banner, .html-reporter .symbol-summary, .html-reporter .summary, .html-reporter .result-message, .html-reporter .spec .description, .html-reporter .spec-detail .description, .html-reporter .alert .bar, .html-reporter .stack-trace { padding-left: 9px; padding-right: 9px; }
+.html-reporter .banner .version { margin-left: 14px; }
+.html-reporter #jasmine_content { position: fixed; right: 100%; }
+.html-reporter .version { color: #aaaaaa; }
+.html-reporter .banner { margin-top: 14px; }
+.html-reporter .duration { color: #aaaaaa; float: right; }
+.html-reporter .symbol-summary { overflow: hidden; *zoom: 1; margin: 14px 0; }
+.html-reporter .symbol-summary li { display: inline-block; height: 8px; width: 14px; font-size: 16px; }
+.html-reporter .symbol-summary li.passed { font-size: 14px; }
+.html-reporter .symbol-summary li.passed:before { color: #5e7d00; content: "\02022"; }
+.html-reporter .symbol-summary li.failed { line-height: 9px; }
+.html-reporter .symbol-summary li.failed:before { color: #b03911; content: "x"; font-weight: bold; margin-left: -1px; }
+.html-reporter .symbol-summary li.disabled { font-size: 14px; }
+.html-reporter .symbol-summary li.disabled:before { color: #bababa; content: "\02022"; }
+.html-reporter .symbol-summary li.pending { line-height: 17px; }
+.html-reporter .symbol-summary li.pending:before { color: #ba9d37; content: "*"; }
+.html-reporter .exceptions { color: #fff; float: right; margin-top: 5px; margin-right: 5px; }
+.html-reporter .bar { line-height: 28px; font-size: 14px; display: block; color: #eee; }
+.html-reporter .bar.failed { background-color: #b03911; }
+.html-reporter .bar.passed { background-color: #a6b779; }
+.html-reporter .bar.skipped { background-color: #bababa; }
+.html-reporter .bar.menu { background-color: #fff; color: #aaaaaa; }
+.html-reporter .bar.menu a { color: #333333; }
+.html-reporter .bar a { color: white; }
+.html-reporter.spec-list .bar.menu.failure-list, .html-reporter.spec-list .results .failures { display: none; }
+.html-reporter.failure-list .bar.menu.spec-list, .html-reporter.failure-list .summary { display: none; }
+.html-reporter .running-alert { background-color: #666666; }
+.html-reporter .results { margin-top: 14px; }
+.html-reporter.showDetails .summaryMenuItem { font-weight: normal; text-decoration: inherit; }
+.html-reporter.showDetails .summaryMenuItem:hover { text-decoration: underline; }
+.html-reporter.showDetails .detailsMenuItem { font-weight: bold; text-decoration: underline; }
+.html-reporter.showDetails .summary { display: none; }
+.html-reporter.showDetails #details { display: block; }
+.html-reporter .summaryMenuItem { font-weight: bold; text-decoration: underline; }
+.html-reporter .summary { margin-top: 14px; }
+.html-reporter .summary ul { list-style-type: none; margin-left: 14px; padding-top: 0; padding-left: 0; }
+.html-reporter .summary ul.suite { margin-top: 7px; margin-bottom: 7px; }
+.html-reporter .summary li.passed a { color: #5e7d00; }
+.html-reporter .summary li.failed a { color: #b03911; }
+.html-reporter .summary li.pending a { color: #ba9d37; }
+.html-reporter .description + .suite { margin-top: 0; }
+.html-reporter .suite { margin-top: 14px; }
+.html-reporter .suite a { color: #333333; }
+.html-reporter .failures .spec-detail { margin-bottom: 28px; }
+.html-reporter .failures .spec-detail .description { background-color: #b03911; }
+.html-reporter .failures .spec-detail .description a { color: white; }
+.html-reporter .result-message { padding-top: 14px; color: #333333; white-space: pre; }
+.html-reporter .result-message span.result { display: block; }
+.html-reporter .stack-trace { margin: 5px 0 0 0; max-height: 224px; overflow: auto; line-height: 18px; color: #666666; border: 1px solid #ddd; background: white; white-space: pre; }

test/jasmine/spec/NeuralNetSpec.js

@@ -0,0 +1,88 @@
+describe("Simple Fully-Connected Neural Net Classifier", function() {
+  var net;
+  var trainer;
+
+  beforeEach(function() {
+    net = new convnetjs.Net();
+
+    var layer_defs = [];
+    layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:2});
+    layer_defs.push({type:'fc', num_neurons:5, activation:'tanh'});
+    layer_defs.push({type:'fc', num_neurons:5, activation:'tanh'});
+    layer_defs.push({type:'softmax', num_classes:3});
+    net.makeLayers(layer_defs);
+
+    trainer = new convnetjs.SGDTrainer(net, 
+          {learning_rate:0.0001, momentum:0.0, batch_size:1, l2_decay:0.0});
+  });
+
+  it("should be possible to initialize", function() {
+    
+    // tanh are their own layers. Softmax gets its own fully connected layer.
+    // this should all get desugared just fine.
+    expect(net.layers.length).toEqual(7); 
+    
+  });
+
+  it("should forward prop volumes to probabilities", function() {
+
+    var x = new convnetjs.Vol([0.2, -0.3]);
+    var probability_volume = net.forward(x);
+
+    expect(probability_volume.w.length).toEqual(3); // 3 classes output
+    var w = probability_volume.w;
+    for(var i=0;i<3;i++) {
+      expect(w[i]).toBeGreaterThan(0);
+      expect(w[i]).toBeLessThan(1.0);
+    }
+    expect(w[0]+w[1]+w[2]).toBeCloseTo(1.0);
+
+  });
+
+  it("should increase probabilities for ground truth class when trained", function() {
+
+    // lets test 100 random point and label settings
+    // note that this should work since l2 and l1 regularization are off
+    // an issue is that if step size is too high, this could technically fail...
+    for(var k=0;k<100;k++) {
+      var x = new convnetjs.Vol([Math.random() * 2 - 1, Math.random() * 2 - 1]);
+      var pv = net.forward(x);
+      var gti = Math.floor(Math.random() * 3);
+      trainer.train(x, gti);
+      var pv2 = net.forward(x);
+      expect(pv2.w[gti]).toBeGreaterThan(pv.w[gti]);
+    }
+
+  });
+
+  it("should compute correct gradient at data", function() {
+
+    // here we only test the gradient at data, but if this is
+    // right then that's comforting, because it is a function 
+    // of all gradients above, for all layers.
+
+    var x = new convnetjs.Vol([Math.random() * 2 - 1, Math.random() * 2 - 1]);
+    var gti = Math.floor(Math.random() * 3); // ground truth index
+    trainer.train(x, gti); // computes gradients at all layers, and at x
+
+    var delta = 0.000001;
+
+    for(var i=0;i<x.w.length;i++) {
+
+      var grad_analytic = x.dw[i];
+
+      var xold = x.w[i];
+      x.w[i] += delta;
+      var c0 = net.getCostLoss(x, gti);
+      x.w[i] -= 2*delta;
+      var c1 = net.getCostLoss(x, gti);
+      x.w[i] = xold; // reset
+
+      var grad_numeric = (c0 - c1)/(2 * delta);
+      var rel_error = Math.abs(grad_analytic - grad_numeric)/Math.abs(grad_analytic + grad_numeric);
+      console.log(i + ': numeric: ' + grad_numeric + ', analytic: ' + grad_analytic + ' => rel error ' + rel_error);
+      expect(rel_error).toBeLessThan(1e-2);
+
+    }
+  });
+});