adding msgpack example files

examples/js/libs/msgpack-js.js

@@ -0,0 +1,600 @@
+( // Module boilerplate to support browser globals and browserify and AMD.
+  typeof define === "function" ? function (m) { define("msgpack-js", m); } :
+  typeof exports === "object" ? function (m) { module.exports = m(); } :
+  function(m){ this.msgpack = m(); }
+)(function () {
+"use strict";
+
+var exports = {};
+
+exports.inspect = inspect;
+function inspect(buffer) {
+  if (buffer === undefined) return "undefined";
+  var view;
+  var type;
+  if (buffer instanceof ArrayBuffer) {
+    type = "ArrayBuffer";
+    view = new DataView(buffer);
+  }
+  else if (buffer instanceof DataView) {
+    type = "DataView";
+    view = buffer;
+  }
+  if (!view) return JSON.stringify(buffer);
+  var bytes = [];
+  for (var i = 0; i < buffer.byteLength; i++) {
+    if (i > 20) {
+      bytes.push("...");
+      break;
+    }
+    var byte = view.getUint8(i).toString(16);
+    if (byte.length === 1) byte = "0" + byte;
+    bytes.push(byte);
+  }
+  return "<" + type + " " + bytes.join(" ") + ">";
+}
+
+// Encode string as utf8 into dataview at offset
+exports.utf8Write = utf8Write;
+function utf8Write(view, offset, string) {
+  var byteLength = view.byteLength;
+  for(var i = 0, l = string.length; i < l; i++) {
+    var codePoint = string.charCodeAt(i);
+
+    // One byte of UTF-8
+    if (codePoint < 0x80) {
+      view.setUint8(offset++, codePoint >>> 0 & 0x7f | 0x00);
+      continue;
+    }
+
+    // Two bytes of UTF-8
+    if (codePoint < 0x800) {
+      view.setUint8(offset++, codePoint >>> 6 & 0x1f | 0xc0);
+      view.setUint8(offset++, codePoint >>> 0 & 0x3f | 0x80);
+      continue;
+    }
+
+    // Three bytes of UTF-8.  
+    if (codePoint < 0x10000) {
+      view.setUint8(offset++, codePoint >>> 12 & 0x0f | 0xe0);
+      view.setUint8(offset++, codePoint >>> 6  & 0x3f | 0x80);
+      view.setUint8(offset++, codePoint >>> 0  & 0x3f | 0x80);
+      continue;
+    }
+
+    // Four bytes of UTF-8
+    if (codePoint < 0x110000) {
+      view.setUint8(offset++, codePoint >>> 18 & 0x07 | 0xf0);
+      view.setUint8(offset++, codePoint >>> 12 & 0x3f | 0x80);
+      view.setUint8(offset++, codePoint >>> 6  & 0x3f | 0x80);
+      view.setUint8(offset++, codePoint >>> 0  & 0x3f | 0x80);
+      continue;
+    }
+    throw new Error("bad codepoint " + codePoint);
+  }
+}
+
+exports.utf8Read = utf8Read;
+function utf8Read(view, offset, length) {
+  var string = "";
+  for (var i = offset, end = offset + length; i < end; i++) {
+    var byte = view.getUint8(i);
+    // One byte character
+    if ((byte & 0x80) === 0x00) {
+      string += String.fromCharCode(byte);
+      continue;
+    }
+    // Two byte character
+    if ((byte & 0xe0) === 0xc0) {
+      string += String.fromCharCode(
+        ((byte & 0x0f) << 6) | 
+        (view.getUint8(++i) & 0x3f)
+      );
+      continue;
+    }
+    // Three byte character
+    if ((byte & 0xf0) === 0xe0) {
+      string += String.fromCharCode(
+        ((byte & 0x0f) << 12) |
+        ((view.getUint8(++i) & 0x3f) << 6) |
+        ((view.getUint8(++i) & 0x3f) << 0)
+      );
+      continue;
+    }
+    // Four byte character
+    if ((byte & 0xf8) === 0xf0) {
+      string += String.fromCharCode(
+        ((byte & 0x07) << 18) |
+        ((view.getUint8(++i) & 0x3f) << 12) |
+        ((view.getUint8(++i) & 0x3f) << 6) |
+        ((view.getUint8(++i) & 0x3f) << 0)
+      );
+      continue;
+    }
+    throw new Error("Invalid byte " + byte.toString(16));
+  }
+  return string;
+}
+
+exports.utf8ByteCount = utf8ByteCount;
+function utf8ByteCount(string) {
+  var count = 0;
+  for(var i = 0, l = string.length; i < l; i++) {
+    var codePoint = string.charCodeAt(i);
+    if (codePoint < 0x80) {
+      count += 1;
+      continue;
+    }
+    if (codePoint < 0x800) {
+      count += 2;
+      continue;
+    }
+    if (codePoint < 0x10000) {
+      count += 3;
+      continue;
+    }
+    if (codePoint < 0x110000) {
+      count += 4;
+      continue;
+    }
+    throw new Error("bad codepoint " + codePoint);
+  }
+  return count;
+}
+
+exports.encode = function (value) {
+  var buffer = new ArrayBuffer(sizeof(value));
+  var view = new DataView(buffer);
+  encode(value, view, 0);
+  return buffer;
+}
+
+exports.decode = decode;
+
+// http://wiki.msgpack.org/display/MSGPACK/Format+specification
+// I've extended the protocol to have two new types that were previously reserved.
+//   buffer 16  11011000  0xd8
+//   buffer 32  11011001  0xd9
+// These work just like raw16 and raw32 except they are node buffers instead of strings.
+//
+// Also I've added a type for `undefined`
+//   undefined  11000100  0xc4
+
+function Decoder(view, offset) {
+  this.offset = offset || 0;
+  this.view = view;
+}
+Decoder.prototype.map = function (length) {
+  var value = {};
+  for (var i = 0; i < length; i++) {
+    var key = this.parse();
+    value[key] = this.parse();
+  }
+  return value;
+};
+Decoder.prototype.buf = function (length) {
+  var value = new ArrayBuffer(length);
+  (new Uint8Array(value)).set(new Uint8Array(this.view.buffer, this.offset, length), 0);
+  this.offset += length;
+  return value;
+};
+Decoder.prototype.raw = function (length) {
+  var value = utf8Read(this.view, this.offset, length);
+  this.offset += length;
+  return value;
+};
+Decoder.prototype.array = function (length) {
+  var value = new Array(length);
+  for (var i = 0; i < length; i++) {
+    value[i] = this.parse();
+  }
+  return value;
+};
+Decoder.prototype.parse = function () {
+  var type = this.view.getUint8(this.offset);
+  var value, length;
+  // FixRaw
+  if ((type & 0xe0) === 0xa0) {
+    length = type & 0x1f;
+    this.offset++;
+    return this.raw(length);
+  }
+  // FixMap
+  if ((type & 0xf0) === 0x80) {
+    length = type & 0x0f;
+    this.offset++;
+    return this.map(length);
+  }
+  // FixArray
+  if ((type & 0xf0) === 0x90) {
+    length = type & 0x0f;
+    this.offset++;
+    return this.array(length);
+  }
+  // Positive FixNum
+  if ((type & 0x80) === 0x00) {
+    this.offset++;
+    return type;
+  }
+  // Negative Fixnum
+  if ((type & 0xe0) === 0xe0) {
+    value = this.view.getInt8(this.offset);
+    this.offset++;
+    return value;
+  }
+  switch (type) {
+  // raw 16
+  case 0xda:
+    length = this.view.getUint16(this.offset + 1);
+    this.offset += 3;
+    return this.raw(length);
+  // raw 32
+  case 0xdb:
+    length = this.view.getUint32(this.offset + 1);
+    this.offset += 5;
+    return this.raw(length);
+  // nil
+  case 0xc0:
+    this.offset++;
+    return null;
+  // false
+  case 0xc2:
+    this.offset++;
+    return false;
+  // true
+  case 0xc3:
+    this.offset++;
+    return true;
+  // undefined
+  case 0xc4:
+    this.offset++;
+    return undefined;
+  // uint8
+  case 0xcc:
+    value = this.view.getUint8(this.offset + 1);
+    this.offset += 2;
+    return value;
+  // uint 16
+  case 0xcd:
+    value = this.view.getUint16(this.offset + 1);
+    this.offset += 3;
+    return value;
+  // uint 32
+  case 0xce:
+    value = this.view.getUint32(this.offset + 1);
+    this.offset += 5;
+    return value;
+  // int 8
+  case 0xd0:
+    value = this.view.getInt8(this.offset + 1);
+    this.offset += 2;
+    return value;
+  // int 16
+  case 0xd1:
+    value = this.view.getInt16(this.offset + 1);
+    this.offset += 3;
+    return value;
+  // int 32
+  case 0xd2:
+    value = this.view.getInt32(this.offset + 1);
+    this.offset += 5;
+    return value;
+  // map 16
+  case 0xde:
+    length = this.view.getUint16(this.offset + 1);
+    this.offset += 3;
+    return this.map(length);
+  // map 32
+  case 0xdf:
+    length = this.view.getUint32(this.offset + 1);
+    this.offset += 5;
+    return this.map(length);
+  // array 16
+  case 0xdc:
+    length = this.view.getUint16(this.offset + 1);
+    this.offset += 3;
+    return this.array(length);
+  // array 32
+  case 0xdd:
+    length = this.view.getUint32(this.offset + 1);
+    this.offset += 5;
+    return this.array(length);
+  // buffer 16
+  case 0xd8:
+    length = this.view.getUint16(this.offset + 1);
+    this.offset += 3;
+    return this.buf(length);
+  // buffer 32
+  case 0xd9:
+    length = this.view.getUint32(this.offset + 1);
+    this.offset += 5;
+    return this.buf(length);
+  // float
+  case 0xca:
+    value = this.view.getFloat32(this.offset + 1);
+    this.offset += 5;
+    return value;
+  // double
+  case 0xcb:
+    value = this.view.getFloat64(this.offset + 1);
+    this.offset += 9;
+    return value;
+  }
+  throw new Error("Unknown type 0x" + type.toString(16));
+};
+function decode(buffer) {
+  var view = new DataView(buffer);
+  var decoder = new Decoder(view);
+  var value = decoder.parse();
+  if (decoder.offset !== buffer.byteLength) throw new Error((buffer.byteLength - decoder.offset) + " trailing bytes");
+  return value;
+}
+
+function encode(value, view, offset) {
+  var type = typeof value;
+
+  // Strings Bytes
+  if (type === "string") {
+    var length = utf8ByteCount(value);
+    // fix raw
+    if (length < 0x20) {
+      view.setUint8(offset, length | 0xa0);
+      utf8Write(view, offset + 1, value);
+      return 1 + length;
+    }
+    // raw 16
+    if (length < 0x10000) {
+      view.setUint8(offset, 0xda);
+      view.setUint16(offset + 1, length);
+      utf8Write(view, offset + 3, value);
+      return 3 + length;
+    }
+    // raw 32
+    if (length < 0x100000000) {
+      view.setUint8(offset, 0xdb);
+      view.setUint32(offset + 1, length);
+      utf8Write(view, offset + 5, value);
+      return 5 + length;
+    }
+  }
+
+  if (value instanceof ArrayBuffer) {
+    var length = value.byteLength;
+    // buffer 16
+    if (length < 0x10000) {
+      view.setUint8(offset, 0xd8);
+      view.setUint16(offset + 1, length);
+      (new Uint8Array(view.buffer)).set(new Uint8Array(value), offset + 3);
+      return 3 + length;
+    }
+    // buffer 32
+    if (length < 0x100000000) {
+      view.setUint8(offset, 0xd9);
+      view.setUint32(offset + 1, length);
+      (new Uint8Array(view.buffer)).set(new Uint8Array(value), offset + 5);
+      return 5 + length;
+    }
+  }
+  
+  if (type === "number") {
+    // Floating Point
+    if ((value << 0) !== value) {
+      view.setUint8(offset, 0xcb);
+      view.setFloat64(offset + 1, value);
+      return 9;
+    }
+
+    // Integers
+    if (value >=0) {
+      // positive fixnum
+      if (value < 0x80) {
+        view.setUint8(offset, value);
+        return 1;
+      }
+      // uint 8
+      if (value < 0x100) {
+        view.setUint8(offset, 0xcc);
+        view.setUint8(offset + 1, value);
+        return 2;
+      }
+      // uint 16
+      if (value < 0x10000) {
+        view.setUint8(offset, 0xcd);
+        view.setUint16(offset + 1, value);
+        return 3;
+      }
+      // uint 32
+      if (value < 0x100000000) {
+        view.setUint8(offset, 0xce);
+        view.setUint32(offset + 1, value);
+        return 5;
+      }
+      throw new Error("Number too big 0x" + value.toString(16));
+    }
+    // negative fixnum
+    if (value >= -0x20) {
+      view.setInt8(offset, value);
+      return 1;
+    }
+    // int 8
+    if (value >= -0x80) {
+      view.setUint8(offset, 0xd0);
+      view.setInt8(offset + 1, value);
+      return 2;
+    }
+    // int 16
+    if (value >= -0x8000) {
+      view.setUint8(offset, 0xd1);
+      view.setInt16(offset + 1, value);
+      return 3;
+    }
+    // int 32
+    if (value >= -0x80000000) {
+      view.setUint8(offset, 0xd2);
+      view.setInt32(offset + 1, value);
+      return 5;
+    }
+    throw new Error("Number too small -0x" + (-value).toString(16).substr(1));
+  }
+  
+  // undefined
+  if (type === "undefined") {
+    view.setUint8(offset, 0xc4);
+    return 1;
+  }
+  
+  // null
+  if (value === null) {
+    view.setUint8(offset, 0xc0);
+    return 1;
+  }
+
+  // Boolean
+  if (type === "boolean") {
+    view.setUint8(offset, value ? 0xc3 : 0xc2);
+    return 1;
+  }
+  
+  // Container Types
+  if (type === "object") {
+    var length, size = 0;
+    var isArray = Array.isArray(value);
+
+    if (isArray) {
+      length = value.length;
+    }
+    else {
+      var keys = Object.keys(value);
+      length = keys.length;
+    }
+
+    var size;
+    if (length < 0x10) {
+      view.setUint8(offset, length | (isArray ? 0x90 : 0x80));
+      size = 1;
+    }
+    else if (length < 0x10000) {
+      view.setUint8(offset, isArray ? 0xdc : 0xde);
+      view.setUint16(offset + 1, length);
+      size = 3;
+    }
+    else if (length < 0x100000000) {
+      view.setUint8(offset, isArray ? 0xdd : 0xdf);
+      view.setUint32(offset + 1, length);
+      size = 5;
+    }
+
+    if (isArray) {
+      for (var i = 0; i < length; i++) {
+        size += encode(value[i], view, offset + size);
+      }
+    }
+    else {
+      for (var i = 0; i < length; i++) {
+        var key = keys[i];
+        size += encode(key, view, offset + size);
+        size += encode(value[key], view, offset + size);
+      }
+    }
+    
+    return size;
+  }
+  throw new Error("Unknown type " + type);
+}
+
+function sizeof(value) {
+  var type = typeof value;
+
+  // Raw Bytes
+  if (type === "string") {
+    var length = utf8ByteCount(value);
+    if (length < 0x20) {
+      return 1 + length;
+    }
+    if (length < 0x10000) {
+      return 3 + length;
+    }
+    if (length < 0x100000000) {
+      return 5 + length;
+    }
+  }
+  
+  if (value instanceof ArrayBuffer) {
+    var length = value.byteLength;
+    if (length < 0x10000) {
+      return 3 + length;
+    }
+    if (length < 0x100000000) {
+      return 5 + length;
+    }
+  }
+  
+  if (type === "number") {
+    // Floating Point
+    // double
+    if (value << 0 !== value) return 9;
+
+    // Integers
+    if (value >=0) {
+      // positive fixnum
+      if (value < 0x80) return 1;
+      // uint 8
+      if (value < 0x100) return 2;
+      // uint 16
+      if (value < 0x10000) return 3;
+      // uint 32
+      if (value < 0x100000000) return 5;
+      // uint 64
+      if (value < 0x10000000000000000) return 9;
+      throw new Error("Number too big 0x" + value.toString(16));
+    }
+    // negative fixnum
+    if (value >= -0x20) return 1;
+    // int 8
+    if (value >= -0x80) return 2;
+    // int 16
+    if (value >= -0x8000) return 3;
+    // int 32
+    if (value >= -0x80000000) return 5;
+    // int 64
+    if (value >= -0x8000000000000000) return 9;
+    throw new Error("Number too small -0x" + value.toString(16).substr(1));
+  }
+  
+  // Boolean, null, undefined
+  if (type === "boolean" || type === "undefined" || value === null) return 1;
+  
+  // Container Types
+  if (type === "object") {
+    var length, size = 0;
+    if (Array.isArray(value)) {
+      length = value.length;
+      for (var i = 0; i < length; i++) {
+        size += sizeof(value[i]);
+      }
+    }
+    else {
+      var keys = Object.keys(value);
+      length = keys.length;
+      for (var i = 0; i < length; i++) {
+        var key = keys[i];
+        size += sizeof(key) + sizeof(value[key]);
+      }
+    }
+    if (length < 0x10) {
+      return 1 + size;
+    }
+    if (length < 0x10000) {
+      return 3 + size;
+    }
+    if (length < 0x100000000) {
+      return 5 + size;
+    }
+    throw new Error("Array or object too long 0x" + length.toString(16));
+  }
+  throw new Error("Unknown type " + type);
+}
+
+return exports;
+
+});

examples/js/libs/require.js

@@ -0,0 +1,36 @@
+/*
+ RequireJS 2.1.11 Copyright (c) 2010-2014, The Dojo Foundation All Rights Reserved.
+ Available via the MIT or new BSD license.
+ see: http://github.com/jrburke/requirejs for details
+*/
+var requirejs,require,define;
+(function(ca){function G(b){return"[object Function]"===M.call(b)}function H(b){return"[object Array]"===M.call(b)}function v(b,c){if(b){var d;for(d=0;d<b.length&&(!b[d]||!c(b[d],d,b));d+=1);}}function U(b,c){if(b){var d;for(d=b.length-1;-1<d&&(!b[d]||!c(b[d],d,b));d-=1);}}function s(b,c){return ga.call(b,c)}function j(b,c){return s(b,c)&&b[c]}function B(b,c){for(var d in b)if(s(b,d)&&c(b[d],d))break}function V(b,c,d,g){c&&B(c,function(c,h){if(d||!s(b,h))g&&"object"===typeof c&&c&&!H(c)&&!G(c)&&!(c instanceof
+RegExp)?(b[h]||(b[h]={}),V(b[h],c,d,g)):b[h]=c});return b}function t(b,c){return function(){return c.apply(b,arguments)}}function da(b){throw b;}function ea(b){if(!b)return b;var c=ca;v(b.split("."),function(b){c=c[b]});return c}function C(b,c,d,g){c=Error(c+"\nhttp://requirejs.org/docs/errors.html#"+b);c.requireType=b;c.requireModules=g;d&&(c.originalError=d);return c}function ha(b){function c(a,e,b){var f,n,c,d,g,h,i,I=e&&e.split("/");n=I;var m=l.map,k=m&&m["*"];if(a&&"."===a.charAt(0))if(e){n=
+I.slice(0,I.length-1);a=a.split("/");e=a.length-1;l.nodeIdCompat&&R.test(a[e])&&(a[e]=a[e].replace(R,""));n=a=n.concat(a);d=n.length;for(e=0;e<d;e++)if(c=n[e],"."===c)n.splice(e,1),e-=1;else if(".."===c)if(1===e&&(".."===n[2]||".."===n[0]))break;else 0<e&&(n.splice(e-1,2),e-=2);a=a.join("/")}else 0===a.indexOf("./")&&(a=a.substring(2));if(b&&m&&(I||k)){n=a.split("/");e=n.length;a:for(;0<e;e-=1){d=n.slice(0,e).join("/");if(I)for(c=I.length;0<c;c-=1)if(b=j(m,I.slice(0,c).join("/")))if(b=j(b,d)){f=b;
+g=e;break a}!h&&(k&&j(k,d))&&(h=j(k,d),i=e)}!f&&h&&(f=h,g=i);f&&(n.splice(0,g,f),a=n.join("/"))}return(f=j(l.pkgs,a))?f:a}function d(a){z&&v(document.getElementsByTagName("script"),function(e){if(e.getAttribute("data-requiremodule")===a&&e.getAttribute("data-requirecontext")===i.contextName)return e.parentNode.removeChild(e),!0})}function g(a){var e=j(l.paths,a);if(e&&H(e)&&1<e.length)return e.shift(),i.require.undef(a),i.require([a]),!0}function u(a){var e,b=a?a.indexOf("!"):-1;-1<b&&(e=a.substring(0,
+b),a=a.substring(b+1,a.length));return[e,a]}function m(a,e,b,f){var n,d,g=null,h=e?e.name:null,l=a,m=!0,k="";a||(m=!1,a="_@r"+(M+=1));a=u(a);g=a[0];a=a[1];g&&(g=c(g,h,f),d=j(p,g));a&&(g?k=d&&d.normalize?d.normalize(a,function(a){return c(a,h,f)}):c(a,h,f):(k=c(a,h,f),a=u(k),g=a[0],k=a[1],b=!0,n=i.nameToUrl(k)));b=g&&!d&&!b?"_unnormalized"+(Q+=1):"";return{prefix:g,name:k,parentMap:e,unnormalized:!!b,url:n,originalName:l,isDefine:m,id:(g?g+"!"+k:k)+b}}function q(a){var e=a.id,b=j(k,e);b||(b=k[e]=new i.Module(a));
+return b}function r(a,e,b){var f=a.id,n=j(k,f);if(s(p,f)&&(!n||n.defineEmitComplete))"defined"===e&&b(p[f]);else if(n=q(a),n.error&&"error"===e)b(n.error);else n.on(e,b)}function w(a,e){var b=a.requireModules,f=!1;if(e)e(a);else if(v(b,function(e){if(e=j(k,e))e.error=a,e.events.error&&(f=!0,e.emit("error",a))}),!f)h.onError(a)}function x(){S.length&&(ia.apply(A,[A.length,0].concat(S)),S=[])}function y(a){delete k[a];delete W[a]}function F(a,e,b){var f=a.map.id;a.error?a.emit("error",a.error):(e[f]=
+!0,v(a.depMaps,function(f,c){var d=f.id,g=j(k,d);g&&(!a.depMatched[c]&&!b[d])&&(j(e,d)?(a.defineDep(c,p[d]),a.check()):F(g,e,b))}),b[f]=!0)}function D(){var a,e,b=(a=1E3*l.waitSeconds)&&i.startTime+a<(new Date).getTime(),f=[],c=[],h=!1,k=!0;if(!X){X=!0;B(W,function(a){var i=a.map,m=i.id;if(a.enabled&&(i.isDefine||c.push(a),!a.error))if(!a.inited&&b)g(m)?h=e=!0:(f.push(m),d(m));else if(!a.inited&&(a.fetched&&i.isDefine)&&(h=!0,!i.prefix))return k=!1});if(b&&f.length)return a=C("timeout","Load timeout for modules: "+
+f,null,f),a.contextName=i.contextName,w(a);k&&v(c,function(a){F(a,{},{})});if((!b||e)&&h)if((z||fa)&&!Y)Y=setTimeout(function(){Y=0;D()},50);X=!1}}function E(a){s(p,a[0])||q(m(a[0],null,!0)).init(a[1],a[2])}function K(a){var a=a.currentTarget||a.srcElement,e=i.onScriptLoad;a.detachEvent&&!Z?a.detachEvent("onreadystatechange",e):a.removeEventListener("load",e,!1);e=i.onScriptError;(!a.detachEvent||Z)&&a.removeEventListener("error",e,!1);return{node:a,id:a&&a.getAttribute("data-requiremodule")}}function L(){var a;
+for(x();A.length;){a=A.shift();if(null===a[0])return w(C("mismatch","Mismatched anonymous define() module: "+a[a.length-1]));E(a)}}var X,$,i,N,Y,l={waitSeconds:7,baseUrl:"./",paths:{},bundles:{},pkgs:{},shim:{},config:{}},k={},W={},aa={},A=[],p={},T={},ba={},M=1,Q=1;N={require:function(a){return a.require?a.require:a.require=i.makeRequire(a.map)},exports:function(a){a.usingExports=!0;if(a.map.isDefine)return a.exports?p[a.map.id]=a.exports:a.exports=p[a.map.id]={}},module:function(a){return a.module?
+a.module:a.module={id:a.map.id,uri:a.map.url,config:function(){return j(l.config,a.map.id)||{}},exports:a.exports||(a.exports={})}}};$=function(a){this.events=j(aa,a.id)||{};this.map=a;this.shim=j(l.shim,a.id);this.depExports=[];this.depMaps=[];this.depMatched=[];this.pluginMaps={};this.depCount=0};$.prototype={init:function(a,e,b,f){f=f||{};if(!this.inited){this.factory=e;if(b)this.on("error",b);else this.events.error&&(b=t(this,function(a){this.emit("error",a)}));this.depMaps=a&&a.slice(0);this.errback=
+b;this.inited=!0;this.ignore=f.ignore;f.enabled||this.enabled?this.enable():this.check()}},defineDep:function(a,e){this.depMatched[a]||(this.depMatched[a]=!0,this.depCount-=1,this.depExports[a]=e)},fetch:function(){if(!this.fetched){this.fetched=!0;i.startTime=(new Date).getTime();var a=this.map;if(this.shim)i.makeRequire(this.map,{enableBuildCallback:!0})(this.shim.deps||[],t(this,function(){return a.prefix?this.callPlugin():this.load()}));else return a.prefix?this.callPlugin():this.load()}},load:function(){var a=
+this.map.url;T[a]||(T[a]=!0,i.load(this.map.id,a))},check:function(){if(this.enabled&&!this.enabling){var a,e,b=this.map.id;e=this.depExports;var f=this.exports,c=this.factory;if(this.inited)if(this.error)this.emit("error",this.error);else{if(!this.defining){this.defining=!0;if(1>this.depCount&&!this.defined){if(G(c)){if(this.events.error&&this.map.isDefine||h.onError!==da)try{f=i.execCb(b,c,e,f)}catch(d){a=d}else f=i.execCb(b,c,e,f);this.map.isDefine&&void 0===f&&((e=this.module)?f=e.exports:this.usingExports&&
+(f=this.exports));if(a)return a.requireMap=this.map,a.requireModules=this.map.isDefine?[this.map.id]:null,a.requireType=this.map.isDefine?"define":"require",w(this.error=a)}else f=c;this.exports=f;if(this.map.isDefine&&!this.ignore&&(p[b]=f,h.onResourceLoad))h.onResourceLoad(i,this.map,this.depMaps);y(b);this.defined=!0}this.defining=!1;this.defined&&!this.defineEmitted&&(this.defineEmitted=!0,this.emit("defined",this.exports),this.defineEmitComplete=!0)}}else this.fetch()}},callPlugin:function(){var a=
+this.map,b=a.id,d=m(a.prefix);this.depMaps.push(d);r(d,"defined",t(this,function(f){var d,g;g=j(ba,this.map.id);var J=this.map.name,u=this.map.parentMap?this.map.parentMap.name:null,p=i.makeRequire(a.parentMap,{enableBuildCallback:!0});if(this.map.unnormalized){if(f.normalize&&(J=f.normalize(J,function(a){return c(a,u,!0)})||""),f=m(a.prefix+"!"+J,this.map.parentMap),r(f,"defined",t(this,function(a){this.init([],function(){return a},null,{enabled:!0,ignore:!0})})),g=j(k,f.id)){this.depMaps.push(f);
+if(this.events.error)g.on("error",t(this,function(a){this.emit("error",a)}));g.enable()}}else g?(this.map.url=i.nameToUrl(g),this.load()):(d=t(this,function(a){this.init([],function(){return a},null,{enabled:!0})}),d.error=t(this,function(a){this.inited=!0;this.error=a;a.requireModules=[b];B(k,function(a){0===a.map.id.indexOf(b+"_unnormalized")&&y(a.map.id)});w(a)}),d.fromText=t(this,function(f,c){var g=a.name,J=m(g),k=O;c&&(f=c);k&&(O=!1);q(J);s(l.config,b)&&(l.config[g]=l.config[b]);try{h.exec(f)}catch(j){return w(C("fromtexteval",
+"fromText eval for "+b+" failed: "+j,j,[b]))}k&&(O=!0);this.depMaps.push(J);i.completeLoad(g);p([g],d)}),f.load(a.name,p,d,l))}));i.enable(d,this);this.pluginMaps[d.id]=d},enable:function(){W[this.map.id]=this;this.enabling=this.enabled=!0;v(this.depMaps,t(this,function(a,b){var c,f;if("string"===typeof a){a=m(a,this.map.isDefine?this.map:this.map.parentMap,!1,!this.skipMap);this.depMaps[b]=a;if(c=j(N,a.id)){this.depExports[b]=c(this);return}this.depCount+=1;r(a,"defined",t(this,function(a){this.defineDep(b,
+a);this.check()}));this.errback&&r(a,"error",t(this,this.errback))}c=a.id;f=k[c];!s(N,c)&&(f&&!f.enabled)&&i.enable(a,this)}));B(this.pluginMaps,t(this,function(a){var b=j(k,a.id);b&&!b.enabled&&i.enable(a,this)}));this.enabling=!1;this.check()},on:function(a,b){var c=this.events[a];c||(c=this.events[a]=[]);c.push(b)},emit:function(a,b){v(this.events[a],function(a){a(b)});"error"===a&&delete this.events[a]}};i={config:l,contextName:b,registry:k,defined:p,urlFetched:T,defQueue:A,Module:$,makeModuleMap:m,
+nextTick:h.nextTick,onError:w,configure:function(a){a.baseUrl&&"/"!==a.baseUrl.charAt(a.baseUrl.length-1)&&(a.baseUrl+="/");var b=l.shim,c={paths:!0,bundles:!0,config:!0,map:!0};B(a,function(a,b){c[b]?(l[b]||(l[b]={}),V(l[b],a,!0,!0)):l[b]=a});a.bundles&&B(a.bundles,function(a,b){v(a,function(a){a!==b&&(ba[a]=b)})});a.shim&&(B(a.shim,function(a,c){H(a)&&(a={deps:a});if((a.exports||a.init)&&!a.exportsFn)a.exportsFn=i.makeShimExports(a);b[c]=a}),l.shim=b);a.packages&&v(a.packages,function(a){var b,
+a="string"===typeof a?{name:a}:a;b=a.name;a.location&&(l.paths[b]=a.location);l.pkgs[b]=a.name+"/"+(a.main||"main").replace(ja,"").replace(R,"")});B(k,function(a,b){!a.inited&&!a.map.unnormalized&&(a.map=m(b))});if(a.deps||a.callback)i.require(a.deps||[],a.callback)},makeShimExports:function(a){return function(){var b;a.init&&(b=a.init.apply(ca,arguments));return b||a.exports&&ea(a.exports)}},makeRequire:function(a,e){function g(f,c,d){var j,l;e.enableBuildCallback&&(c&&G(c))&&(c.__requireJsBuild=
+!0);if("string"===typeof f){if(G(c))return w(C("requireargs","Invalid require call"),d);if(a&&s(N,f))return N[f](k[a.id]);if(h.get)return h.get(i,f,a,g);j=m(f,a,!1,!0);j=j.id;return!s(p,j)?w(C("notloaded",'Module name "'+j+'" has not been loaded yet for context: '+b+(a?"":". Use require([])"))):p[j]}L();i.nextTick(function(){L();l=q(m(null,a));l.skipMap=e.skipMap;l.init(f,c,d,{enabled:!0});D()});return g}e=e||{};V(g,{isBrowser:z,toUrl:function(b){var e,d=b.lastIndexOf("."),g=b.split("/")[0];if(-1!==
+d&&(!("."===g||".."===g)||1<d))e=b.substring(d,b.length),b=b.substring(0,d);return i.nameToUrl(c(b,a&&a.id,!0),e,!0)},defined:function(b){return s(p,m(b,a,!1,!0).id)},specified:function(b){b=m(b,a,!1,!0).id;return s(p,b)||s(k,b)}});a||(g.undef=function(b){x();var c=m(b,a,!0),e=j(k,b);d(b);delete p[b];delete T[c.url];delete aa[b];U(A,function(a,c){a[0]===b&&A.splice(c,1)});e&&(e.events.defined&&(aa[b]=e.events),y(b))});return g},enable:function(a){j(k,a.id)&&q(a).enable()},completeLoad:function(a){var b,
+c,f=j(l.shim,a)||{},d=f.exports;for(x();A.length;){c=A.shift();if(null===c[0]){c[0]=a;if(b)break;b=!0}else c[0]===a&&(b=!0);E(c)}c=j(k,a);if(!b&&!s(p,a)&&c&&!c.inited){if(l.enforceDefine&&(!d||!ea(d)))return g(a)?void 0:w(C("nodefine","No define call for "+a,null,[a]));E([a,f.deps||[],f.exportsFn])}D()},nameToUrl:function(a,b,c){var f,d,g;(f=j(l.pkgs,a))&&(a=f);if(f=j(ba,a))return i.nameToUrl(f,b,c);if(h.jsExtRegExp.test(a))f=a+(b||"");else{f=l.paths;a=a.split("/");for(d=a.length;0<d;d-=1)if(g=a.slice(0,
+d).join("/"),g=j(f,g)){H(g)&&(g=g[0]);a.splice(0,d,g);break}f=a.join("/");f+=b||(/^data\:|\?/.test(f)||c?"":".js");f=("/"===f.charAt(0)||f.match(/^[\w\+\.\-]+:/)?"":l.baseUrl)+f}return l.urlArgs?f+((-1===f.indexOf("?")?"?":"&")+l.urlArgs):f},load:function(a,b){h.load(i,a,b)},execCb:function(a,b,c,d){return b.apply(d,c)},onScriptLoad:function(a){if("load"===a.type||ka.test((a.currentTarget||a.srcElement).readyState))P=null,a=K(a),i.completeLoad(a.id)},onScriptError:function(a){var b=K(a);if(!g(b.id))return w(C("scripterror",
+"Script error for: "+b.id,a,[b.id]))}};i.require=i.makeRequire();return i}var h,x,y,D,K,E,P,L,q,Q,la=/(\/\*([\s\S]*?)\*\/|([^:]|^)\/\/(.*)$)/mg,ma=/[^.]\s*require\s*\(\s*["']([^'"\s]+)["']\s*\)/g,R=/\.js$/,ja=/^\.\//;x=Object.prototype;var M=x.toString,ga=x.hasOwnProperty,ia=Array.prototype.splice,z=!!("undefined"!==typeof window&&"undefined"!==typeof navigator&&window.document),fa=!z&&"undefined"!==typeof importScripts,ka=z&&"PLAYSTATION 3"===navigator.platform?/^complete$/:/^(complete|loaded)$/,
+Z="undefined"!==typeof opera&&"[object Opera]"===opera.toString(),F={},r={},S=[],O=!1;if("undefined"===typeof define){if("undefined"!==typeof requirejs){if(G(requirejs))return;r=requirejs;requirejs=void 0}"undefined"!==typeof require&&!G(require)&&(r=require,require=void 0);h=requirejs=function(b,c,d,g){var u,m="_";!H(b)&&"string"!==typeof b&&(u=b,H(c)?(b=c,c=d,d=g):b=[]);u&&u.context&&(m=u.context);(g=j(F,m))||(g=F[m]=h.s.newContext(m));u&&g.configure(u);return g.require(b,c,d)};h.config=function(b){return h(b)};
+h.nextTick="undefined"!==typeof setTimeout?function(b){setTimeout(b,4)}:function(b){b()};require||(require=h);h.version="2.1.11";h.jsExtRegExp=/^\/|:|\?|\.js$/;h.isBrowser=z;x=h.s={contexts:F,newContext:ha};h({});v(["toUrl","undef","defined","specified"],function(b){h[b]=function(){var c=F._;return c.require[b].apply(c,arguments)}});if(z&&(y=x.head=document.getElementsByTagName("head")[0],D=document.getElementsByTagName("base")[0]))y=x.head=D.parentNode;h.onError=da;h.createNode=function(b){var c=
+b.xhtml?document.createElementNS("http://www.w3.org/1999/xhtml","html:script"):document.createElement("script");c.type=b.scriptType||"text/javascript";c.charset="utf-8";c.async=!0;return c};h.load=function(b,c,d){var g=b&&b.config||{};if(z)return g=h.createNode(g,c,d),g.setAttribute("data-requirecontext",b.contextName),g.setAttribute("data-requiremodule",c),g.attachEvent&&!(g.attachEvent.toString&&0>g.attachEvent.toString().indexOf("[native code"))&&!Z?(O=!0,g.attachEvent("onreadystatechange",b.onScriptLoad)):
+(g.addEventListener("load",b.onScriptLoad,!1),g.addEventListener("error",b.onScriptError,!1)),g.src=d,L=g,D?y.insertBefore(g,D):y.appendChild(g),L=null,g;if(fa)try{importScripts(d),b.completeLoad(c)}catch(j){b.onError(C("importscripts","importScripts failed for "+c+" at "+d,j,[c]))}};z&&!r.skipDataMain&&U(document.getElementsByTagName("script"),function(b){y||(y=b.parentNode);if(K=b.getAttribute("data-main"))return q=K,r.baseUrl||(E=q.split("/"),q=E.pop(),Q=E.length?E.join("/")+"/":"./",r.baseUrl=
+Q),q=q.replace(R,""),h.jsExtRegExp.test(q)&&(q=K),r.deps=r.deps?r.deps.concat(q):[q],!0});define=function(b,c,d){var g,h;"string"!==typeof b&&(d=c,c=b,b=null);H(c)||(d=c,c=null);!c&&G(d)&&(c=[],d.length&&(d.toString().replace(la,"").replace(ma,function(b,d){c.push(d)}),c=(1===d.length?["require"]:["require","exports","module"]).concat(c)));if(O){if(!(g=L))P&&"interactive"===P.readyState||U(document.getElementsByTagName("script"),function(b){if("interactive"===b.readyState)return P=b}),g=P;g&&(b||
+(b=g.getAttribute("data-requiremodule")),h=F[g.getAttribute("data-requirecontext")])}(h?h.defQueue:S).push([b,c,d])};define.amd={jQuery:!0};h.exec=function(b){return eval(b)};h(r)}})(this);

examples/webgl_loader_msgpack.html

@@ -0,0 +1,159 @@
+<!doctype html>
+<!--
+Requires msgpack-js.js
+
+https://github.com/creationix/msgpack-js-browser
+(ported from https://github.com/creationix/msgpack-js)
+-->
+<html lang='en'>
+  <head>
+    <title>three.js webgl - msgpack loader</title>
+    <meta charset='utf-8'>
+    <meta name='viewport' content='width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
+    <style>
+      #viewport {
+        position: absolute;
+        width: 100%;
+        height: 100%;
+        background: #1b1c1e;
+        background-image: linear-gradient(#7d8fa3, #1b1c1e);
+      }
+    
+      #info {
+        color: #fff;
+        position: absolute;
+        top: 10px;
+        width: 100%;
+        text-align: center;
+        z-index: 100;
+        display:block;
+      }
+
+      a { color: orange }
+    </style>
+    <script src='../build/three.min.js'></script>
+    <script src='js/controls/OrbitControls.js'></script>
+    <script src='js/Detector.js'></script>
+    <script src='js/libs/stats.min.js'></script>
+    <script src='js/libs/require.js'></script>
+  </head>
+  <body>
+    <div id='info'>
+      <p>Robo Pigeon, from <a href='http://www.tearsofsteel.org' target='_blank'>Tears of Steel</a>, is licensed under 
+      <a href='http://creativecommons.org/licenses/by/3.0/' target='_blank'>Creative Commons Attribution 3.0</a>.
+      </p>
+    </div>
+    <div id='viewport'></div>
+    <script>
+      
+      if ( ! Detector.webgl ) Detector.addGetWebGLMessage();
+
+      requirejs.config( { baseUrl: 'js/libs' } );
+
+      var container, scene, camera, renderer;
+ 
+      function render() {
+        
+        renderer.render( scene, camera );
+      
+      }
+
+      function threePointLight() {
+
+          var directionalLight = new THREE.DirectionalLight( 0xb8b8b8 );
+          directionalLight.position.set(1, 1, 1).normalize();
+          directionalLight.intensity = 1.0;
+          scene.add( directionalLight );
+     
+          directionalLight = new THREE.DirectionalLight( 0xb8b8b8 );
+          directionalLight.position.set(-1, 0.6, 0.5).normalize();
+          directionalLight.intensity = 0.5;
+          scene.add(directionalLight);
+
+          directionalLight = new THREE.DirectionalLight();
+          directionalLight.position.set(-0.3, 0.6, -0.8).normalize( 0xb8b8b8 );
+          directionalLight.intensity = 0.45;
+          scene.add(directionalLight);
+
+      }
+
+      function setupScene( result ) {
+
+          scene = result;
+          scene.add( new THREE.GridHelper( 10, 2.5 ) );
+
+          threePointLight();
+
+          render();
+
+      }
+
+      function loadMSGPack() {
+
+          require(['msgpack-js'], function ( msgpack ) {
+              
+              var xhr = new XMLHttpRequest();
+              xhr.open('GET', 'scenes/robo_pigeon.pack', true);
+              xhr.responseType = 'arraybuffer';
+
+              xhr.onload = function( e ) {
+
+                  var decoded = msgpack.decode( this.response );
+
+                  var loader = new THREE.ObjectLoader();
+                  
+                  setupScene( loader.parse( decoded ) );
+
+              };
+
+              xhr.send();
+        
+          } );
+
+      }
+
+      function onWindowResize() {
+
+            camera.aspect = container.offsetWidth / container.offsetHeight;
+            camera.updateProjectionMatrix();
+
+            renderer.setSize( container.offsetWidth, container.offsetHeight );
+
+            render();
+
+	  }
+
+      function init() {
+
+          scene = new THREE.Scene();
+
+          scene.add( new THREE.GridHelper( 10, 2.5 ) );
+          container = document.getElementById('viewport');
+
+          renderer = new THREE.WebGLRenderer( { antialias: true, alpha: true  } );
+          renderer.setSize( container.offsetWidth, container.offsetHeight );
+          renderer.setClearColor( 0x000000, 0 );
+          container.appendChild( renderer.domElement );
+
+          var aspect = container.offsetWidth / container.offsetHeight;
+          camera = new THREE.PerspectiveCamera( 60, aspect, 0.01, 50 );
+          orbit = new THREE.OrbitControls( camera, container );
+          orbit.addEventListener( 'change', render );
+          camera.position.z = 5;
+          camera.position.x = 5;
+          camera.position.y = 5;
+          var target = new THREE.Vector3( 0, 1, 0 );
+          camera.lookAt( target );
+          orbit.target = target;
+          camera.updateProjectionMatrix();
+
+          window.addEventListener( 'resize', onWindowResize, false );
+
+          loadMSGPack();
+
+      }
+
+      init();
+    </script>
+  </body>
+</html>

utils/converters/msgpack/json2msgpack.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+__doc__ = '''
+Convert a json file to msgpack.
+
+If fed only an input file the converted will write out a .pack file
+of the same base name in the same directory
+$ json2msgpack.py -i foo.json
+foo.json > foo.pack
+
+Specify an output file path
+$ json2msgpack.py -i foo.json -o /bar/tmp/bar.pack
+foo.json > /bar/tmp/bar.pack
+
+Dependencies:
+https://github.com/msgpack/msgpack-python
+'''
+
+import os
+import sys
+import json
+import argparse
+
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
+
+import msgpack
+
+EXT = '.pack'
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-i', '--infile', required=True,
+        help='Input json file to convert to msgpack')
+    parser.add_argument('-o', '--outfile',
+        help=('Optional output. If not specified the .pack file '\
+            'will write to the same director as the input file.'))
+    args = parser.parse_args()
+    convert(args.infile, args.outfile)
+
+def convert(infile, outfile):
+    if not outfile:
+        ext = infile.split('.')[-1]
+        outfile = '%s%s' % (infile[:-len(ext)-1], EXT)
+
+    print('%s > %s' % (infile, outfile))
+
+    print('reading in JSON')
+    with open(infile) as op:
+        data = json.load(op)
+
+    print('writing to msgpack')
+    with open(outfile, 'wb') as op:
+        msgpack.dump(data, op)
+
+if __name__ == '__main__':
+    main()

utils/converters/msgpack/msgpack/__init__.py

@@ -0,0 +1,54 @@
+# coding: utf-8
+from msgpack._version import version
+from msgpack.exceptions import *
+
+from collections import namedtuple
+
+
+class ExtType(namedtuple('ExtType', 'code data')):
+    """ExtType represents ext type in msgpack."""
+    def __new__(cls, code, data):
+        if not isinstance(code, int):
+            raise TypeError("code must be int")
+        if not isinstance(data, bytes):
+            raise TypeError("data must be bytes")
+        if not 0 <= code <= 127:
+            raise ValueError("code must be 0~127")
+        return super(ExtType, cls).__new__(cls, code, data)
+
+
+import os
+if os.environ.get('MSGPACK_PUREPYTHON'):
+    from msgpack.fallback import Packer, unpack, unpackb, Unpacker
+else:
+    try:
+        from msgpack._packer import Packer
+        from msgpack._unpacker import unpack, unpackb, Unpacker
+    except ImportError:
+        from msgpack.fallback import Packer, unpack, unpackb, Unpacker
+
+
+def pack(o, stream, **kwargs):
+    """
+    Pack object `o` and write it to `stream`
+
+    See :class:`Packer` for options.
+    """
+    packer = Packer(**kwargs)
+    stream.write(packer.pack(o))
+
+
+def packb(o, **kwargs):
+    """
+    Pack object `o` and return packed bytes
+
+    See :class:`Packer` for options.
+    """
+    return Packer(**kwargs).pack(o)
+
+# alias for compatibility to simplejson/marshal/pickle.
+load = unpack
+loads = unpackb
+
+dump = pack
+dumps = packb

utils/converters/msgpack/msgpack/_packer.pyx

@@ -0,0 +1,295 @@
+# coding: utf-8
+#cython: embedsignature=True
+
+from cpython cimport *
+from libc.stdlib cimport *
+from libc.string cimport *
+from libc.limits cimport *
+from libc.stdint cimport int8_t
+
+from msgpack.exceptions import PackValueError
+from msgpack import ExtType
+
+
+cdef extern from "pack.h":
+    struct msgpack_packer:
+        char* buf
+        size_t length
+        size_t buf_size
+        bint use_bin_type
+
+    int msgpack_pack_int(msgpack_packer* pk, int d)
+    int msgpack_pack_nil(msgpack_packer* pk)
+    int msgpack_pack_true(msgpack_packer* pk)
+    int msgpack_pack_false(msgpack_packer* pk)
+    int msgpack_pack_long(msgpack_packer* pk, long d)
+    int msgpack_pack_long_long(msgpack_packer* pk, long long d)
+    int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d)
+    int msgpack_pack_float(msgpack_packer* pk, float d)
+    int msgpack_pack_double(msgpack_packer* pk, double d)
+    int msgpack_pack_array(msgpack_packer* pk, size_t l)
+    int msgpack_pack_map(msgpack_packer* pk, size_t l)
+    int msgpack_pack_raw(msgpack_packer* pk, size_t l)
+    int msgpack_pack_bin(msgpack_packer* pk, size_t l)
+    int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
+    int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l)
+
+cdef int DEFAULT_RECURSE_LIMIT=511
+
+
+cdef class Packer(object):
+    """
+    MessagePack Packer
+
+    usage::
+
+        packer = Packer()
+        astream.write(packer.pack(a))
+        astream.write(packer.pack(b))
+
+    Packer's constructor has some keyword arguments:
+
+    :param callable default:
+        Convert user type to builtin type that Packer supports.
+        See also simplejson's document.
+    :param str encoding:
+        Convert unicode to bytes with this encoding. (default: 'utf-8')
+    :param str unicode_errors:
+        Error handler for encoding unicode. (default: 'strict')
+    :param bool use_single_float:
+        Use single precision float type for float. (default: False)
+    :param bool autoreset:
+        Reset buffer after each pack and return it's content as `bytes`. (default: True).
+        If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+    :param bool use_bin_type:
+        Use bin type introduced in msgpack spec 2.0 for bytes.
+        It also enable str8 type for unicode.
+    """
+    cdef msgpack_packer pk
+    cdef object _default
+    cdef object _bencoding
+    cdef object _berrors
+    cdef char *encoding
+    cdef char *unicode_errors
+    cdef bool use_float
+    cdef bint autoreset
+
+    def __cinit__(self):
+        cdef int buf_size = 1024*1024
+        self.pk.buf = <char*> malloc(buf_size);
+        if self.pk.buf == NULL:
+            raise MemoryError("Unable to allocate internal buffer.")
+        self.pk.buf_size = buf_size
+        self.pk.length = 0
+
+    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
+                 use_single_float=False, bint autoreset=1, bint use_bin_type=0):
+        """
+        """
+        self.use_float = use_single_float
+        self.autoreset = autoreset
+        self.pk.use_bin_type = use_bin_type
+        if default is not None:
+            if not PyCallable_Check(default):
+                raise TypeError("default must be a callable.")
+        self._default = default
+        if encoding is None:
+            self.encoding = NULL
+            self.unicode_errors = NULL
+        else:
+            if isinstance(encoding, unicode):
+                self._bencoding = encoding.encode('ascii')
+            else:
+                self._bencoding = encoding
+            self.encoding = PyBytes_AsString(self._bencoding)
+            if isinstance(unicode_errors, unicode):
+                self._berrors = unicode_errors.encode('ascii')
+            else:
+                self._berrors = unicode_errors
+            self.unicode_errors = PyBytes_AsString(self._berrors)
+
+    def __dealloc__(self):
+        free(self.pk.buf);
+
+    cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
+        cdef long long llval
+        cdef unsigned long long ullval
+        cdef long longval
+        cdef float fval
+        cdef double dval
+        cdef char* rawval
+        cdef int ret
+        cdef dict d
+        cdef size_t L
+        cdef int default_used = 0
+
+        if nest_limit < 0:
+            raise PackValueError("recursion limit exceeded.")
+
+        while True:
+            if o is None:
+                ret = msgpack_pack_nil(&self.pk)
+            elif isinstance(o, bool):
+                if o:
+                    ret = msgpack_pack_true(&self.pk)
+                else:
+                    ret = msgpack_pack_false(&self.pk)
+            elif PyLong_Check(o):
+                # PyInt_Check(long) is True for Python 3.
+                # Sow we should test long before int.
+                if o > 0:
+                    ullval = o
+                    ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
+                else:
+                    llval = o
+                    ret = msgpack_pack_long_long(&self.pk, llval)
+            elif PyInt_Check(o):
+                longval = o
+                ret = msgpack_pack_long(&self.pk, longval)
+            elif PyFloat_Check(o):
+                if self.use_float:
+                   fval = o
+                   ret = msgpack_pack_float(&self.pk, fval)
+                else:
+                   dval = o
+                   ret = msgpack_pack_double(&self.pk, dval)
+            elif PyBytes_Check(o):
+                L = len(o)
+                if L > (2**32)-1:
+                    raise ValueError("bytes is too large")
+                rawval = o
+                ret = msgpack_pack_bin(&self.pk, L)
+                if ret == 0:
+                    ret = msgpack_pack_raw_body(&self.pk, rawval, L)
+            elif PyUnicode_Check(o):
+                if not self.encoding:
+                    raise TypeError("Can't encode unicode string: no encoding is specified")
+                o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
+                L = len(o)
+                if L > (2**32)-1:
+                    raise ValueError("dict is too large")
+                rawval = o
+                ret = msgpack_pack_raw(&self.pk, len(o))
+                if ret == 0:
+                    ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
+            elif PyDict_CheckExact(o):
+                d = <dict>o
+                L = len(d)
+                if L > (2**32)-1:
+                    raise ValueError("dict is too large")
+                ret = msgpack_pack_map(&self.pk, L)
+                if ret == 0:
+                    for k, v in d.iteritems():
+                        ret = self._pack(k, nest_limit-1)
+                        if ret != 0: break
+                        ret = self._pack(v, nest_limit-1)
+                        if ret != 0: break
+            elif PyDict_Check(o):
+                L = len(o)
+                if L > (2**32)-1:
+                    raise ValueError("dict is too large")
+                ret = msgpack_pack_map(&self.pk, L)
+                if ret == 0:
+                    for k, v in o.items():
+                        ret = self._pack(k, nest_limit-1)
+                        if ret != 0: break
+                        ret = self._pack(v, nest_limit-1)
+                        if ret != 0: break
+            elif isinstance(o, ExtType):
+                # This should be before Tuple because ExtType is namedtuple.
+                longval = o.code
+                rawval = o.data
+                L = len(o.data)
+                if L > (2**32)-1:
+                    raise ValueError("EXT data is too large")
+                ret = msgpack_pack_ext(&self.pk, longval, L)
+                ret = msgpack_pack_raw_body(&self.pk, rawval, L)
+            elif PyTuple_Check(o) or PyList_Check(o):
+                L = len(o)
+                if L > (2**32)-1:
+                    raise ValueError("list is too large")
+                ret = msgpack_pack_array(&self.pk, L)
+                if ret == 0:
+                    for v in o:
+                        ret = self._pack(v, nest_limit-1)
+                        if ret != 0: break
+            elif not default_used and self._default:
+                o = self._default(o)
+                default_used = 1
+                continue
+            else:
+                raise TypeError("can't serialize %r" % (o,))
+            return ret
+
+    cpdef pack(self, object obj):
+        cdef int ret
+        ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen.
+            raise TypeError
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
+
+    def pack_ext_type(self, typecode, data):
+        msgpack_pack_ext(&self.pk, typecode, len(data))
+        msgpack_pack_raw_body(&self.pk, data, len(data))
+
+    def pack_array_header(self, size_t size):
+        if size > (2**32-1):
+            raise ValueError
+        cdef int ret = msgpack_pack_array(&self.pk, size)
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen
+            raise TypeError
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
+
+    def pack_map_header(self, size_t size):
+        if size > (2**32-1):
+            raise ValueError
+        cdef int ret = msgpack_pack_map(&self.pk, size)
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen
+            raise TypeError
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
+
+    def pack_map_pairs(self, object pairs):
+        """
+        Pack *pairs* as msgpack map type.
+
+        *pairs* should sequence of pair.
+        (`len(pairs)` and `for k, v in pairs:` should be supported.)
+        """
+        cdef int ret = msgpack_pack_map(&self.pk, len(pairs))
+        if ret == 0:
+            for k, v in pairs:
+                ret = self._pack(k)
+                if ret != 0: break
+                ret = self._pack(v)
+                if ret != 0: break
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen
+            raise TypeError
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
+
+    def reset(self):
+        """Clear internal buffer."""
+        self.pk.length = 0
+
+    def bytes(self):
+        """Return buffer content."""
+        return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)

utils/converters/msgpack/msgpack/_unpacker.pyx

@@ -0,0 +1,426 @@
+# coding: utf-8
+#cython: embedsignature=True
+
+from cpython cimport *
+cdef extern from "Python.h":
+    ctypedef struct PyObject
+    cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1
+
+from libc.stdlib cimport *
+from libc.string cimport *
+from libc.limits cimport *
+
+from msgpack.exceptions import (
+        BufferFull,
+        OutOfData,
+        UnpackValueError,
+        ExtraData,
+        )
+from msgpack import ExtType
+
+
+cdef extern from "unpack.h":
+    ctypedef struct msgpack_user:
+        bint use_list
+        PyObject* object_hook
+        bint has_pairs_hook # call object_hook with k-v pairs
+        PyObject* list_hook
+        PyObject* ext_hook
+        char *encoding
+        char *unicode_errors
+
+    ctypedef struct unpack_context:
+        msgpack_user user
+        PyObject* obj
+        size_t count
+
+    ctypedef int (*execute_fn)(unpack_context* ctx, const char* data,
+                               size_t len, size_t* off) except? -1
+    execute_fn unpack_construct
+    execute_fn unpack_skip
+    execute_fn read_array_header
+    execute_fn read_map_header
+    void unpack_init(unpack_context* ctx)
+    object unpack_data(unpack_context* ctx)
+
+cdef inline init_ctx(unpack_context *ctx,
+                     object object_hook, object object_pairs_hook,
+                     object list_hook, object ext_hook,
+                     bint use_list, char* encoding, char* unicode_errors):
+    unpack_init(ctx)
+    ctx.user.use_list = use_list
+    ctx.user.object_hook = ctx.user.list_hook = <PyObject*>NULL
+
+    if object_hook is not None and object_pairs_hook is not None:
+        raise TypeError("object_pairs_hook and object_hook are mutually exclusive.")
+
+    if object_hook is not None:
+        if not PyCallable_Check(object_hook):
+            raise TypeError("object_hook must be a callable.")
+        ctx.user.object_hook = <PyObject*>object_hook
+
+    if object_pairs_hook is None:
+        ctx.user.has_pairs_hook = False
+    else:
+        if not PyCallable_Check(object_pairs_hook):
+            raise TypeError("object_pairs_hook must be a callable.")
+        ctx.user.object_hook = <PyObject*>object_pairs_hook
+        ctx.user.has_pairs_hook = True
+
+    if list_hook is not None:
+        if not PyCallable_Check(list_hook):
+            raise TypeError("list_hook must be a callable.")
+        ctx.user.list_hook = <PyObject*>list_hook
+
+    if ext_hook is not None:
+        if not PyCallable_Check(ext_hook):
+            raise TypeError("ext_hook must be a callable.")
+        ctx.user.ext_hook = <PyObject*>ext_hook
+
+    ctx.user.encoding = encoding
+    ctx.user.unicode_errors = unicode_errors
+
+def default_read_extended_type(typecode, data):
+    raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode)
+
+def unpackb(object packed, object object_hook=None, object list_hook=None,
+            bint use_list=1, encoding=None, unicode_errors="strict",
+            object_pairs_hook=None, ext_hook=ExtType):
+    """
+    Unpack packed_bytes to object. Returns an unpacked object.
+
+    Raises `ValueError` when `packed` contains extra bytes.
+
+    See :class:`Unpacker` for options.
+    """
+    cdef unpack_context ctx
+    cdef size_t off = 0
+    cdef int ret
+
+    cdef char* buf
+    cdef Py_ssize_t buf_len
+    cdef char* cenc = NULL
+    cdef char* cerr = NULL
+
+    PyObject_AsReadBuffer(packed, <const void**>&buf, &buf_len)
+
+    if encoding is not None:
+        if isinstance(encoding, unicode):
+            encoding = encoding.encode('ascii')
+        cenc = PyBytes_AsString(encoding)
+
+    if unicode_errors is not None:
+        if isinstance(unicode_errors, unicode):
+            unicode_errors = unicode_errors.encode('ascii')
+        cerr = PyBytes_AsString(unicode_errors)
+
+    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
+             use_list, cenc, cerr)
+    ret = unpack_construct(&ctx, buf, buf_len, &off)
+    if ret == 1:
+        obj = unpack_data(&ctx)
+        if off < buf_len:
+            raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
+        return obj
+    else:
+        raise UnpackValueError("Unpack failed: error = %d" % (ret,))
+
+
+def unpack(object stream, object object_hook=None, object list_hook=None,
+           bint use_list=1, encoding=None, unicode_errors="strict",
+           object_pairs_hook=None,
+           ):
+    """
+    Unpack an object from `stream`.
+
+    Raises `ValueError` when `stream` has extra bytes.
+
+    See :class:`Unpacker` for options.
+    """
+    return unpackb(stream.read(), use_list=use_list,
+                   object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
+                   encoding=encoding, unicode_errors=unicode_errors,
+                   )
+
+
+cdef class Unpacker(object):
+    """
+    Streaming unpacker.
+
+    arguments:
+
+    :param file_like:
+        File-like object having `.read(n)` method.
+        If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
+
+    :param int read_size:
+        Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`)
+
+    :param bool use_list:
+        If true, unpack msgpack array to Python list.
+        Otherwise, unpack to Python tuple. (default: True)
+
+    :param callable object_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a dict argument after unpacking msgpack map.
+        (See also simplejson)
+
+    :param callable object_pairs_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
+        (See also simplejson)
+
+    :param str encoding:
+        Encoding used for decoding msgpack raw.
+        If it is None (default), msgpack raw is deserialized to Python bytes.
+
+    :param str unicode_errors:
+        Used for decoding msgpack raw with *encoding*.
+        (default: `'strict'`)
+
+    :param int max_buffer_size:
+        Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
+        Raises `BufferFull` exception when it is insufficient.
+        You shoud set this parameter when unpacking data from untrasted source.
+
+    example of streaming deserialize from file-like object::
+
+        unpacker = Unpacker(file_like)
+        for o in unpacker:
+            process(o)
+
+    example of streaming deserialize from socket::
+
+        unpacker = Unpacker()
+        while True:
+            buf = sock.recv(1024**2)
+            if not buf:
+                break
+            unpacker.feed(buf)
+            for o in unpacker:
+                process(o)
+    """
+    cdef unpack_context ctx
+    cdef char* buf
+    cdef size_t buf_size, buf_head, buf_tail
+    cdef object file_like
+    cdef object file_like_read
+    cdef Py_ssize_t read_size
+    # To maintain refcnt.
+    cdef object object_hook, object_pairs_hook, list_hook, ext_hook
+    cdef object encoding, unicode_errors
+    cdef size_t max_buffer_size
+
+    def __cinit__(self):
+        self.buf = NULL
+
+    def __dealloc__(self):
+        free(self.buf)
+        self.buf = NULL
+
+    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
+                 object object_hook=None, object object_pairs_hook=None, object list_hook=None,
+                 str encoding=None, str unicode_errors='strict', int max_buffer_size=0,
+                 object ext_hook=ExtType):
+        cdef char *cenc=NULL,
+        cdef char *cerr=NULL
+
+        self.object_hook = object_hook
+        self.object_pairs_hook = object_pairs_hook
+        self.list_hook = list_hook
+        self.ext_hook = ext_hook
+
+        self.file_like = file_like
+        if file_like:
+            self.file_like_read = file_like.read
+            if not PyCallable_Check(self.file_like_read):
+                raise TypeError("`file_like.read` must be a callable.")
+        if not max_buffer_size:
+            max_buffer_size = INT_MAX
+        if read_size > max_buffer_size:
+            raise ValueError("read_size should be less or equal to max_buffer_size")
+        if not read_size:
+            read_size = min(max_buffer_size, 1024**2)
+        self.max_buffer_size = max_buffer_size
+        self.read_size = read_size
+        self.buf = <char*>malloc(read_size)
+        if self.buf == NULL:
+            raise MemoryError("Unable to allocate internal buffer.")
+        self.buf_size = read_size
+        self.buf_head = 0
+        self.buf_tail = 0
+
+        if encoding is not None:
+            if isinstance(encoding, unicode):
+                self.encoding = encoding.encode('ascii')
+            else:
+                self.encoding = encoding
+            cenc = PyBytes_AsString(self.encoding)
+
+        if unicode_errors is not None:
+            if isinstance(unicode_errors, unicode):
+                self.unicode_errors = unicode_errors.encode('ascii')
+            else:
+                self.unicode_errors = unicode_errors
+            cerr = PyBytes_AsString(self.unicode_errors)
+
+        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
+                 ext_hook, use_list, cenc, cerr)
+
+    def feed(self, object next_bytes):
+        """Append `next_bytes` to internal buffer."""
+        cdef Py_buffer pybuff
+        if self.file_like is not None:
+            raise AssertionError(
+                    "unpacker.feed() is not be able to use with `file_like`.")
+        PyObject_GetBuffer(next_bytes, &pybuff, PyBUF_SIMPLE)
+        try:
+            self.append_buffer(<char*>pybuff.buf, pybuff.len)
+        finally:
+            PyBuffer_Release(&pybuff)
+
+    cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
+        cdef:
+            char* buf = self.buf
+            char* new_buf
+            size_t head = self.buf_head
+            size_t tail = self.buf_tail
+            size_t buf_size = self.buf_size
+            size_t new_size
+
+        if tail + _buf_len > buf_size:
+            if ((tail - head) + _buf_len) <= buf_size:
+                # move to front.
+                memmove(buf, buf + head, tail - head)
+                tail -= head
+                head = 0
+            else:
+                # expand buffer.
+                new_size = (tail-head) + _buf_len
+                if new_size > self.max_buffer_size:
+                    raise BufferFull
+                new_size = min(new_size*2, self.max_buffer_size)
+                new_buf = <char*>malloc(new_size)
+                if new_buf == NULL:
+                    # self.buf still holds old buffer and will be freed during
+                    # obj destruction
+                    raise MemoryError("Unable to enlarge internal buffer.")
+                memcpy(new_buf, buf + head, tail - head)
+                free(buf)
+
+                buf = new_buf
+                buf_size = new_size
+                tail -= head
+                head = 0
+
+        memcpy(buf + tail, <char*>(_buf), _buf_len)
+        self.buf = buf
+        self.buf_head = head
+        self.buf_size = buf_size
+        self.buf_tail = tail + _buf_len
+
+    cdef read_from_file(self):
+        next_bytes = self.file_like_read(
+                min(self.read_size,
+                    self.max_buffer_size - (self.buf_tail - self.buf_head)
+                    ))
+        if next_bytes:
+            self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes))
+        else:
+            self.file_like = None
+
+    cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0):
+        cdef int ret
+        cdef object obj
+        cdef size_t prev_head
+
+        if self.buf_head >= self.buf_tail and self.file_like is not None:
+            self.read_from_file()
+
+        while 1:
+            prev_head = self.buf_head
+            if prev_head >= self.buf_tail:
+                if iter:
+                    raise StopIteration("No more data to unpack.")
+                else:
+                    raise OutOfData("No more data to unpack.")
+
+            ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+            if write_bytes is not None:
+                write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
+
+            if ret == 1:
+                obj = unpack_data(&self.ctx)
+                unpack_init(&self.ctx)
+                return obj
+            elif ret == 0:
+                if self.file_like is not None:
+                    self.read_from_file()
+                    continue
+                if iter:
+                    raise StopIteration("No more data to unpack.")
+                else:
+                    raise OutOfData("No more data to unpack.")
+            else:
+                raise ValueError("Unpack failed: error = %d" % (ret,))
+
+    def read_bytes(self, Py_ssize_t nbytes):
+        """read a specified number of raw bytes from the stream"""
+        cdef size_t nread
+        nread = min(self.buf_tail - self.buf_head, nbytes)
+        ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread)
+        self.buf_head += nread
+        if len(ret) < nbytes and self.file_like is not None:
+            ret += self.file_like.read(nbytes - len(ret))
+        return ret
+
+    def unpack(self, object write_bytes=None):
+        """
+        unpack one object
+
+        If write_bytes is not None, it will be called with parts of the raw
+        message as it is unpacked.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(unpack_construct, write_bytes)
+
+    def skip(self, object write_bytes=None):
+        """
+        read and ignore one object, returning None
+
+        If write_bytes is not None, it will be called with parts of the raw
+        message as it is unpacked.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(unpack_skip, write_bytes)
+
+    def read_array_header(self, object write_bytes=None):
+        """assuming the next object is an array, return its size n, such that
+        the next n unpack() calls will iterate over its contents.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(read_array_header, write_bytes)
+
+    def read_map_header(self, object write_bytes=None):
+        """assuming the next object is a map, return its size n, such that the
+        next n * 2 unpack() calls will iterate over its key-value pairs.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(read_map_header, write_bytes)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return self._unpack(unpack_construct, None, 1)
+
+    # for debug.
+    #def _buf(self):
+    #    return PyString_FromStringAndSize(self.buf, self.buf_tail)
+
+    #def _off(self):
+    #    return self.buf_head

utils/converters/msgpack/msgpack/_version.py

@@ -0,0 +1 @@
+version = (0, 4, 2)

utils/converters/msgpack/msgpack/exceptions.py

@@ -0,0 +1,29 @@
+class UnpackException(Exception):
+    pass
+
+
+class BufferFull(UnpackException):
+    pass
+
+
+class OutOfData(UnpackException):
+    pass
+
+
+class UnpackValueError(UnpackException, ValueError):
+    pass
+
+
+class ExtraData(ValueError):
+    def __init__(self, unpacked, extra):
+        self.unpacked = unpacked
+        self.extra = extra
+
+    def __str__(self):
+        return "unpack(b) received extra data."
+
+class PackException(Exception):
+    pass
+
+class PackValueError(PackException, ValueError):
+    pass

utils/converters/msgpack/msgpack/fallback.py

@@ -0,0 +1,714 @@
+"""Fallback pure Python implementation of msgpack"""
+
+import sys
+import array
+import struct
+
+if sys.version_info[0] == 3:
+    PY3 = True
+    int_types = int
+    Unicode = str
+    xrange = range
+    def dict_iteritems(d):
+        return d.items()
+else:
+    PY3 = False
+    int_types = (int, long)
+    Unicode = unicode
+    def dict_iteritems(d):
+        return d.iteritems()
+
+
+if hasattr(sys, 'pypy_version_info'):
+    # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
+    # StringBuilder is fastest.
+    from __pypy__ import newlist_hint
+    from __pypy__.builders import StringBuilder
+    USING_STRINGBUILDER = True
+    class StringIO(object):
+        def __init__(self, s=b''):
+            if s:
+                self.builder = StringBuilder(len(s))
+                self.builder.append(s)
+            else:
+                self.builder = StringBuilder()
+        def write(self, s):
+            self.builder.append(s)
+        def getvalue(self):
+            return self.builder.build()
+else:
+    USING_STRINGBUILDER = False
+    from io import BytesIO as StringIO
+    newlist_hint = lambda size: []
+
+from msgpack.exceptions import (
+    BufferFull,
+    OutOfData,
+    UnpackValueError,
+    PackValueError,
+    ExtraData)
+
+from msgpack import ExtType
+
+
+EX_SKIP                 = 0
+EX_CONSTRUCT            = 1
+EX_READ_ARRAY_HEADER    = 2
+EX_READ_MAP_HEADER      = 3
+
+TYPE_IMMEDIATE          = 0
+TYPE_ARRAY              = 1
+TYPE_MAP                = 2
+TYPE_RAW                = 3
+TYPE_BIN                = 4
+TYPE_EXT                = 5
+
+DEFAULT_RECURSE_LIMIT = 511
+
+
+def unpack(stream, **kwargs):
+    """
+    Unpack an object from `stream`.
+
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    unpacker = Unpacker(stream, **kwargs)
+    ret = unpacker._fb_unpack()
+    if unpacker._fb_got_extradata():
+        raise ExtraData(ret, unpacker._fb_get_extradata())
+    return ret
+
+
+def unpackb(packed, **kwargs):
+    """
+    Unpack an object from `packed`.
+
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    unpacker = Unpacker(None, **kwargs)
+    unpacker.feed(packed)
+    try:
+        ret = unpacker._fb_unpack()
+    except OutOfData:
+        raise UnpackValueError("Data is not enough.")
+    if unpacker._fb_got_extradata():
+        raise ExtraData(ret, unpacker._fb_get_extradata())
+    return ret
+
+
+class Unpacker(object):
+    """
+    Streaming unpacker.
+
+    `file_like` is a file-like object having a `.read(n)` method.
+    When `Unpacker` is initialized with a `file_like`, `.feed()` is not
+    usable.
+
+    `read_size` is used for `file_like.read(read_size)`.
+
+    If `use_list` is True (default), msgpack lists are deserialized to Python
+    lists.  Otherwise they are deserialized to tuples.
+
+    `object_hook` is the same as in simplejson.  If it is not None, it should
+    be callable and Unpacker calls it with a dict argument after deserializing
+    a map.
+
+    `object_pairs_hook` is the same as in simplejson.  If it is not None, it
+    should be callable and Unpacker calls it with a list of key-value pairs
+    after deserializing a map.
+
+    `ext_hook` is callback for ext (User defined) type. It called with two
+    arguments: (code, bytes). default: `msgpack.ExtType`
+
+    `encoding` is the encoding used for decoding msgpack bytes.  If it is
+    None (default), msgpack bytes are deserialized to Python bytes.
+
+    `unicode_errors` is used for decoding bytes.
+
+    `max_buffer_size` limits the buffer size.  0 means INT_MAX (default).
+
+    Raises `BufferFull` exception when it is unsufficient.
+
+    You should set this parameter when unpacking data from an untrustred source.
+
+    example of streaming deserialization from file-like object::
+
+        unpacker = Unpacker(file_like)
+        for o in unpacker:
+            do_something(o)
+
+    example of streaming deserialization from socket::
+
+        unpacker = Unpacker()
+        while 1:
+            buf = sock.recv(1024*2)
+            if not buf:
+                break
+            unpacker.feed(buf)
+            for o in unpacker:
+                do_something(o)
+    """
+
+    def __init__(self, file_like=None, read_size=0, use_list=True,
+                 object_hook=None, object_pairs_hook=None, list_hook=None,
+                 encoding=None, unicode_errors='strict', max_buffer_size=0,
+                 ext_hook=ExtType):
+        if file_like is None:
+            self._fb_feeding = True
+        else:
+            if not callable(file_like.read):
+                raise TypeError("`file_like.read` must be callable")
+            self.file_like = file_like
+            self._fb_feeding = False
+        self._fb_buffers = []
+        self._fb_buf_o = 0
+        self._fb_buf_i = 0
+        self._fb_buf_n = 0
+        self._max_buffer_size = max_buffer_size or 2**31-1
+        if read_size > self._max_buffer_size:
+            raise ValueError("read_size must be smaller than max_buffer_size")
+        self._read_size = read_size or min(self._max_buffer_size, 2048)
+        self._encoding = encoding
+        self._unicode_errors = unicode_errors
+        self._use_list = use_list
+        self._list_hook = list_hook
+        self._object_hook = object_hook
+        self._object_pairs_hook = object_pairs_hook
+        self._ext_hook = ext_hook
+
+        if list_hook is not None and not callable(list_hook):
+            raise TypeError('`list_hook` is not callable')
+        if object_hook is not None and not callable(object_hook):
+            raise TypeError('`object_hook` is not callable')
+        if object_pairs_hook is not None and not callable(object_pairs_hook):
+            raise TypeError('`object_pairs_hook` is not callable')
+        if object_hook is not None and object_pairs_hook is not None:
+            raise TypeError("object_pairs_hook and object_hook are mutually "
+                            "exclusive")
+        if not callable(ext_hook):
+            raise TypeError("`ext_hook` is not callable")
+
+    def feed(self, next_bytes):
+        if isinstance(next_bytes, array.array):
+            next_bytes = next_bytes.tostring()
+        elif isinstance(next_bytes, bytearray):
+            next_bytes = bytes(next_bytes)
+        assert self._fb_feeding
+        if self._fb_buf_n + len(next_bytes) > self._max_buffer_size:
+            raise BufferFull
+        self._fb_buf_n += len(next_bytes)
+        self._fb_buffers.append(next_bytes)
+
+    def _fb_consume(self):
+        self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
+        if self._fb_buffers:
+            self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:]
+        self._fb_buf_o = 0
+        self._fb_buf_i = 0
+        self._fb_buf_n = sum(map(len, self._fb_buffers))
+
+    def _fb_got_extradata(self):
+        if self._fb_buf_i != len(self._fb_buffers):
+            return True
+        if self._fb_feeding:
+            return False
+        if not self.file_like:
+            return False
+        if self.file_like.read(1):
+            return True
+        return False
+
+    def __iter__(self):
+        return self
+
+    def read_bytes(self, n):
+        return self._fb_read(n)
+
+    def _fb_rollback(self):
+        self._fb_buf_i = 0
+        self._fb_buf_o = 0
+
+    def _fb_get_extradata(self):
+        bufs = self._fb_buffers[self._fb_buf_i:]
+        if bufs:
+            bufs[0] = bufs[0][self._fb_buf_o:]
+        return b''.join(bufs)
+
+    def _fb_read(self, n, write_bytes=None):
+        buffs = self._fb_buffers
+        if (write_bytes is None and self._fb_buf_i < len(buffs) and
+                self._fb_buf_o + n < len(buffs[self._fb_buf_i])):
+            self._fb_buf_o += n
+            return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o]
+
+        ret = b''
+        while len(ret) != n:
+            if self._fb_buf_i == len(buffs):
+                if self._fb_feeding:
+                    break
+                tmp = self.file_like.read(self._read_size)
+                if not tmp:
+                    break
+                buffs.append(tmp)
+                continue
+            sliced = n - len(ret)
+            ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced]
+            self._fb_buf_o += sliced
+            if self._fb_buf_o >= len(buffs[self._fb_buf_i]):
+                self._fb_buf_o = 0
+                self._fb_buf_i += 1
+        if len(ret) != n:
+            self._fb_rollback()
+            raise OutOfData
+        if write_bytes is not None:
+            write_bytes(ret)
+        return ret
+
+    def _read_header(self, execute=EX_CONSTRUCT, write_bytes=None):
+        typ = TYPE_IMMEDIATE
+        n = 0
+        obj = None
+        c = self._fb_read(1, write_bytes)
+        b = ord(c)
+        if   b & 0b10000000 == 0:
+            obj = b
+        elif b & 0b11100000 == 0b11100000:
+            obj = struct.unpack("b", c)[0]
+        elif b & 0b11100000 == 0b10100000:
+            n = b & 0b00011111
+            obj = self._fb_read(n, write_bytes)
+            typ = TYPE_RAW
+        elif b & 0b11110000 == 0b10010000:
+            n = b & 0b00001111
+            typ = TYPE_ARRAY
+        elif b & 0b11110000 == 0b10000000:
+            n = b & 0b00001111
+            typ = TYPE_MAP
+        elif b == 0xc0:
+            obj = None
+        elif b == 0xc2:
+            obj = False
+        elif b == 0xc3:
+            obj = True
+        elif b == 0xc4:
+            typ = TYPE_BIN
+            n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xc5:
+            typ = TYPE_BIN
+            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xc6:
+            typ = TYPE_BIN
+            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xc7:  # ext 8
+            typ = TYPE_EXT
+            L, n = struct.unpack('Bb', self._fb_read(2, write_bytes))
+            obj = self._fb_read(L, write_bytes)
+        elif b == 0xc8:  # ext 16
+            typ = TYPE_EXT
+            L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes))
+            obj = self._fb_read(L, write_bytes)
+        elif b == 0xc9:  # ext 32
+            typ = TYPE_EXT
+            L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes))
+            obj = self._fb_read(L, write_bytes)
+        elif b == 0xca:
+            obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0]
+        elif b == 0xcb:
+            obj = struct.unpack(">d", self._fb_read(8, write_bytes))[0]
+        elif b == 0xcc:
+            obj = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+        elif b == 0xcd:
+            obj = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+        elif b == 0xce:
+            obj = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+        elif b == 0xcf:
+            obj = struct.unpack(">Q", self._fb_read(8, write_bytes))[0]
+        elif b == 0xd0:
+            obj = struct.unpack("b", self._fb_read(1, write_bytes))[0]
+        elif b == 0xd1:
+            obj = struct.unpack(">h", self._fb_read(2, write_bytes))[0]
+        elif b == 0xd2:
+            obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0]
+        elif b == 0xd3:
+            obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0]
+        elif b == 0xd4:  # fixext 1
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes))
+        elif b == 0xd5:  # fixext 2
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes))
+        elif b == 0xd6:  # fixext 4
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes))
+        elif b == 0xd7:  # fixext 8
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes))
+        elif b == 0xd8:  # fixext 16
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes))
+        elif b == 0xd9:
+            typ = TYPE_RAW
+            n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xda:
+            typ = TYPE_RAW
+            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xdb:
+            typ = TYPE_RAW
+            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xdc:
+            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            typ = TYPE_ARRAY
+        elif b == 0xdd:
+            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            typ = TYPE_ARRAY
+        elif b == 0xde:
+            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            typ = TYPE_MAP
+        elif b == 0xdf:
+            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            typ = TYPE_MAP
+        else:
+            raise UnpackValueError("Unknown header: 0x%x" % b)
+        return typ, n, obj
+
+    def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
+        typ, n, obj = self._read_header(execute, write_bytes)
+
+        if execute == EX_READ_ARRAY_HEADER:
+            if typ != TYPE_ARRAY:
+                raise UnpackValueError("Expected array")
+            return n
+        if execute == EX_READ_MAP_HEADER:
+            if typ != TYPE_MAP:
+                raise UnpackValueError("Expected map")
+            return n
+        # TODO should we eliminate the recursion?
+        if typ == TYPE_ARRAY:
+            if execute == EX_SKIP:
+                for i in xrange(n):
+                    # TODO check whether we need to call `list_hook`
+                    self._fb_unpack(EX_SKIP, write_bytes)
+                return
+            ret = newlist_hint(n)
+            for i in xrange(n):
+                ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes))
+            if self._list_hook is not None:
+                ret = self._list_hook(ret)
+            # TODO is the interaction between `list_hook` and `use_list` ok?
+            return ret if self._use_list else tuple(ret)
+        if typ == TYPE_MAP:
+            if execute == EX_SKIP:
+                for i in xrange(n):
+                    # TODO check whether we need to call hooks
+                    self._fb_unpack(EX_SKIP, write_bytes)
+                    self._fb_unpack(EX_SKIP, write_bytes)
+                return
+            if self._object_pairs_hook is not None:
+                ret = self._object_pairs_hook(
+                    (self._fb_unpack(EX_CONSTRUCT, write_bytes),
+                     self._fb_unpack(EX_CONSTRUCT, write_bytes))
+                    for _ in xrange(n))
+            else:
+                ret = {}
+                for _ in xrange(n):
+                    key = self._fb_unpack(EX_CONSTRUCT, write_bytes)
+                    ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes)
+                if self._object_hook is not None:
+                    ret = self._object_hook(ret)
+            return ret
+        if execute == EX_SKIP:
+            return
+        if typ == TYPE_RAW:
+            if self._encoding is not None:
+                obj = obj.decode(self._encoding, self._unicode_errors)
+            return obj
+        if typ == TYPE_EXT:
+            return self._ext_hook(n, obj)
+        if typ == TYPE_BIN:
+            return obj
+        assert typ == TYPE_IMMEDIATE
+        return obj
+
+    def next(self):
+        try:
+            ret = self._fb_unpack(EX_CONSTRUCT, None)
+            self._fb_consume()
+            return ret
+        except OutOfData:
+            raise StopIteration
+    __next__ = next
+
+    def skip(self, write_bytes=None):
+        self._fb_unpack(EX_SKIP, write_bytes)
+        self._fb_consume()
+
+    def unpack(self, write_bytes=None):
+        ret = self._fb_unpack(EX_CONSTRUCT, write_bytes)
+        self._fb_consume()
+        return ret
+
+    def read_array_header(self, write_bytes=None):
+        ret = self._fb_unpack(EX_READ_ARRAY_HEADER, write_bytes)
+        self._fb_consume()
+        return ret
+
+    def read_map_header(self, write_bytes=None):
+        ret = self._fb_unpack(EX_READ_MAP_HEADER, write_bytes)
+        self._fb_consume()
+        return ret
+
+
+class Packer(object):
+    """
+    MessagePack Packer
+
+    usage:
+
+        packer = Packer()
+        astream.write(packer.pack(a))
+        astream.write(packer.pack(b))
+
+    Packer's constructor has some keyword arguments:
+
+    :param callable default:
+        Convert user type to builtin type that Packer supports.
+        See also simplejson's document.
+    :param str encoding:
+            Convert unicode to bytes with this encoding. (default: 'utf-8')
+    :param str unicode_errors:
+        Error handler for encoding unicode. (default: 'strict')
+    :param bool use_single_float:
+        Use single precision float type for float. (default: False)
+    :param bool autoreset:
+        Reset buffer after each pack and return it's content as `bytes`. (default: True).
+        If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+    :param bool use_bin_type:
+        Use bin type introduced in msgpack spec 2.0 for bytes.
+        It also enable str8 type for unicode.
+    """
+    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
+                 use_single_float=False, autoreset=True, use_bin_type=False):
+        self._use_float = use_single_float
+        self._autoreset = autoreset
+        self._use_bin_type = use_bin_type
+        self._encoding = encoding
+        self._unicode_errors = unicode_errors
+        self._buffer = StringIO()
+        if default is not None:
+            if not callable(default):
+                raise TypeError("default must be callable")
+        self._default = default
+
+    def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance):
+        default_used = False
+        while True:
+            if nest_limit < 0:
+                raise PackValueError("recursion limit exceeded")
+            if obj is None:
+                return self._buffer.write(b"\xc0")
+            if isinstance(obj, bool):
+                if obj:
+                    return self._buffer.write(b"\xc3")
+                return self._buffer.write(b"\xc2")
+            if isinstance(obj, int_types):
+                if 0 <= obj < 0x80:
+                    return self._buffer.write(struct.pack("B", obj))
+                if -0x20 <= obj < 0:
+                    return self._buffer.write(struct.pack("b", obj))
+                if 0x80 <= obj <= 0xff:
+                    return self._buffer.write(struct.pack("BB", 0xcc, obj))
+                if -0x80 <= obj < 0:
+                    return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
+                if 0xff < obj <= 0xffff:
+                    return self._buffer.write(struct.pack(">BH", 0xcd, obj))
+                if -0x8000 <= obj < -0x80:
+                    return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
+                if 0xffff < obj <= 0xffffffff:
+                    return self._buffer.write(struct.pack(">BI", 0xce, obj))
+                if -0x80000000 <= obj < -0x8000:
+                    return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
+                if 0xffffffff < obj <= 0xffffffffffffffff:
+                    return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
+                if -0x8000000000000000 <= obj < -0x80000000:
+                    return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
+                raise PackValueError("Integer value out of range")
+            if self._use_bin_type and isinstance(obj, bytes):
+                n = len(obj)
+                if n <= 0xff:
+                    self._buffer.write(struct.pack('>BB', 0xc4, n))
+                elif n <= 0xffff:
+                    self._buffer.write(struct.pack(">BH", 0xc5, n))
+                elif n <= 0xffffffff:
+                    self._buffer.write(struct.pack(">BI", 0xc6, n))
+                else:
+                    raise PackValueError("Bytes is too large")
+                return self._buffer.write(obj)
+            if isinstance(obj, (Unicode, bytes)):
+                if isinstance(obj, Unicode):
+                    if self._encoding is None:
+                        raise TypeError(
+                            "Can't encode unicode string: "
+                            "no encoding is specified")
+                    obj = obj.encode(self._encoding, self._unicode_errors)
+                n = len(obj)
+                if n <= 0x1f:
+                    self._buffer.write(struct.pack('B', 0xa0 + n))
+                elif self._use_bin_type and n <= 0xff:
+                    self._buffer.write(struct.pack('>BB', 0xd9, n))
+                elif n <= 0xffff:
+                    self._buffer.write(struct.pack(">BH", 0xda, n))
+                elif n <= 0xffffffff:
+                    self._buffer.write(struct.pack(">BI", 0xdb, n))
+                else:
+                    raise PackValueError("String is too large")
+                return self._buffer.write(obj)
+            if isinstance(obj, float):
+                if self._use_float:
+                    return self._buffer.write(struct.pack(">Bf", 0xca, obj))
+                return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
+            if isinstance(obj, ExtType):
+                code = obj.code
+                data = obj.data
+                assert isinstance(code, int)
+                assert isinstance(data, bytes)
+                L = len(data)
+                if L == 1:
+                    self._buffer.write(b'\xd4')
+                elif L == 2:
+                    self._buffer.write(b'\xd5')
+                elif L == 4:
+                    self._buffer.write(b'\xd6')
+                elif L == 8:
+                    self._buffer.write(b'\xd7')
+                elif L == 16:
+                    self._buffer.write(b'\xd8')
+                elif L <= 0xff:
+                    self._buffer.write(struct.pack(">BB", 0xc7, L))
+                elif L <= 0xffff:
+                    self._buffer.write(struct.pack(">BH", 0xc8, L))
+                else:
+                    self._buffer.write(struct.pack(">BI", 0xc9, L))
+                self._buffer.write(struct.pack("b", code))
+                self._buffer.write(data)
+                return
+            if isinstance(obj, (list, tuple)):
+                n = len(obj)
+                self._fb_pack_array_header(n)
+                for i in xrange(n):
+                    self._pack(obj[i], nest_limit - 1)
+                return
+            if isinstance(obj, dict):
+                return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj),
+                                               nest_limit - 1)
+            if not default_used and self._default is not None:
+                obj = self._default(obj)
+                default_used = 1
+                continue
+            raise TypeError("Cannot serialize %r" % obj)
+
+    def pack(self, obj):
+        self._pack(obj)
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self._buffer = StringIO(ret)
+        return ret
+
+    def pack_map_pairs(self, pairs):
+        self._fb_pack_map_pairs(len(pairs), pairs)
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self._buffer = StringIO(ret)
+        return ret
+
+    def pack_array_header(self, n):
+        if n >= 2**32:
+            raise ValueError
+        self._fb_pack_array_header(n)
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self._buffer = StringIO(ret)
+        return ret
+
+    def pack_map_header(self, n):
+        if n >= 2**32:
+            raise ValueError
+        self._fb_pack_map_header(n)
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self._buffer = StringIO(ret)
+        return ret
+
+    def pack_ext_type(self, typecode, data):
+        if not isinstance(typecode, int):
+            raise TypeError("typecode must have int type.")
+        if not 0 <= typecode <= 127:
+            raise ValueError("typecode should be 0-127")
+        if not isinstance(data, bytes):
+            raise TypeError("data must have bytes type")
+        L = len(data)
+        if L > 0xffffffff:
+            raise ValueError("Too large data")
+        if L == 1:
+            self._buffer.write(b'\xd4')
+        elif L == 2:
+            self._buffer.write(b'\xd5')
+        elif L == 4:
+            self._buffer.write(b'\xd6')
+        elif L == 8:
+            self._buffer.write(b'\xd7')
+        elif L == 16:
+            self._buffer.write(b'\xd8')
+        elif L <= 0xff:
+            self._buffer.write(b'\xc7' + struct.pack('B', L))
+        elif L <= 0xffff:
+            self._buffer.write(b'\xc8' + struct.pack('>H', L))
+        else:
+            self._buffer.write(b'\xc9' + struct.pack('>I', L))
+        self._buffer.write(struct.pack('B', typecode))
+        self._buffer.write(data)
+
+    def _fb_pack_array_header(self, n):
+        if n <= 0x0f:
+            return self._buffer.write(struct.pack('B', 0x90 + n))
+        if n <= 0xffff:
+            return self._buffer.write(struct.pack(">BH", 0xdc, n))
+        if n <= 0xffffffff:
+            return self._buffer.write(struct.pack(">BI", 0xdd, n))
+        raise PackValueError("Array is too large")
+
+    def _fb_pack_map_header(self, n):
+        if n <= 0x0f:
+            return self._buffer.write(struct.pack('B', 0x80 + n))
+        if n <= 0xffff:
+            return self._buffer.write(struct.pack(">BH", 0xde, n))
+        if n <= 0xffffffff:
+            return self._buffer.write(struct.pack(">BI", 0xdf, n))
+        raise PackValueError("Dict is too large")
+
+    def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
+        self._fb_pack_map_header(n)
+        for (k, v) in pairs:
+            self._pack(k, nest_limit - 1)
+            self._pack(v, nest_limit - 1)
+
+    def bytes(self):
+        return self._buffer.getvalue()
+
+    def reset(self):
+        self._buffer = StringIO()

utils/converters/msgpack/msgpack/pack.h

@@ -0,0 +1,103 @@
+/*
+ * MessagePack for Python packing routine
+ *
+ * Copyright (C) 2009 Naoki INADA
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#include <stddef.h>
+#include <stdlib.h>
+#include "sysdep.h"
+#include <limits.h>
+#include <string.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _MSC_VER
+#define inline __inline
+#endif
+
+typedef struct msgpack_packer {
+    char *buf;
+    size_t length;
+    size_t buf_size;
+    bool use_bin_type;
+} msgpack_packer;
+
+typedef struct Packer Packer;
+
+static inline int msgpack_pack_int(msgpack_packer* pk, int d);
+static inline int msgpack_pack_long(msgpack_packer* pk, long d);
+static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
+static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d);
+static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d);
+static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d);
+//static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
+
+static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d);
+static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d);
+static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d);
+static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d);
+static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d);
+static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d);
+static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d);
+static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d);
+
+static inline int msgpack_pack_float(msgpack_packer* pk, float d);
+static inline int msgpack_pack_double(msgpack_packer* pk, double d);
+
+static inline int msgpack_pack_nil(msgpack_packer* pk);
+static inline int msgpack_pack_true(msgpack_packer* pk);
+static inline int msgpack_pack_false(msgpack_packer* pk);
+
+static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
+
+static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
+
+static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
+static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l);
+static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
+
+static inline int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l);
+
+static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
+{
+    char* buf = pk->buf;
+    size_t bs = pk->buf_size;
+    size_t len = pk->length;
+
+    if (len + l > bs) {
+        bs = (len + l) * 2;
+        buf = (char*)realloc(buf, bs);
+        if (!buf) return -1;
+    }
+    memcpy(buf + len, data, l);
+    len += l;
+
+    pk->buf = buf;
+    pk->buf_size = bs;
+    pk->length = len;
+    return 0;
+}
+
+#define msgpack_pack_append_buffer(user, buf, len) \
+        return msgpack_pack_write(user, (const char*)buf, len)
+
+#include "pack_template.h"
+
+#ifdef __cplusplus
+}
+#endif

utils/converters/msgpack/msgpack/pack_template.h

@@ -0,0 +1,785 @@
+/*
+ * MessagePack packing routine template
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#if defined(__LITTLE_ENDIAN__)
+#define TAKE8_8(d)  ((uint8_t*)&d)[0]
+#define TAKE8_16(d) ((uint8_t*)&d)[0]
+#define TAKE8_32(d) ((uint8_t*)&d)[0]
+#define TAKE8_64(d) ((uint8_t*)&d)[0]
+#elif defined(__BIG_ENDIAN__)
+#define TAKE8_8(d)  ((uint8_t*)&d)[0]
+#define TAKE8_16(d) ((uint8_t*)&d)[1]
+#define TAKE8_32(d) ((uint8_t*)&d)[3]
+#define TAKE8_64(d) ((uint8_t*)&d)[7]
+#endif
+
+#ifndef msgpack_pack_append_buffer
+#error msgpack_pack_append_buffer callback is not defined
+#endif
+
+
+/*
+ * Integer
+ */
+
+#define msgpack_pack_real_uint8(x, d) \
+do { \
+    if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
+    } else { \
+        /* unsigned 8 */ \
+        unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \
+        msgpack_pack_append_buffer(x, buf, 2); \
+    } \
+} while(0)
+
+#define msgpack_pack_real_uint16(x, d) \
+do { \
+    if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
+    } else if(d < (1<<8)) { \
+        /* unsigned 8 */ \
+        unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
+        msgpack_pack_append_buffer(x, buf, 2); \
+    } else { \
+        /* unsigned 16 */ \
+        unsigned char buf[3]; \
+        buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+        msgpack_pack_append_buffer(x, buf, 3); \
+    } \
+} while(0)
+
+#define msgpack_pack_real_uint32(x, d) \
+do { \
+    if(d < (1<<8)) { \
+        if(d < (1<<7)) { \
+            /* fixnum */ \
+            msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
+        } else { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else { \
+        if(d < (1<<16)) { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* unsigned 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } \
+    } \
+} while(0)
+
+#define msgpack_pack_real_uint64(x, d) \
+do { \
+    if(d < (1ULL<<8)) { \
+        if(d < (1ULL<<7)) { \
+            /* fixnum */ \
+            msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
+        } else { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else { \
+        if(d < (1ULL<<16)) { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else if(d < (1ULL<<32)) { \
+            /* unsigned 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } else { \
+            /* unsigned 64 */ \
+            unsigned char buf[9]; \
+            buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
+            msgpack_pack_append_buffer(x, buf, 9); \
+        } \
+    } \
+} while(0)
+
+#define msgpack_pack_real_int8(x, d) \
+do { \
+    if(d < -(1<<5)) { \
+        /* signed 8 */ \
+        unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \
+        msgpack_pack_append_buffer(x, buf, 2); \
+    } else { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
+    } \
+} while(0)
+
+#define msgpack_pack_real_int16(x, d) \
+do { \
+    if(d < -(1<<5)) { \
+        if(d < -(1<<7)) { \
+            /* signed 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* signed 8 */ \
+            unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
+    } else { \
+        if(d < (1<<8)) { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } else { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } \
+    } \
+} while(0)
+
+#define msgpack_pack_real_int32(x, d) \
+do { \
+    if(d < -(1<<5)) { \
+        if(d < -(1<<15)) { \
+            /* signed 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } else if(d < -(1<<7)) { \
+            /* signed 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* signed 8 */ \
+            unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
+    } else { \
+        if(d < (1<<8)) { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } else if(d < (1<<16)) { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* unsigned 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } \
+    } \
+} while(0)
+
+#define msgpack_pack_real_int64(x, d) \
+do { \
+    if(d < -(1LL<<5)) { \
+        if(d < -(1LL<<15)) { \
+            if(d < -(1LL<<31)) { \
+                /* signed 64 */ \
+                unsigned char buf[9]; \
+                buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \
+                msgpack_pack_append_buffer(x, buf, 9); \
+            } else { \
+                /* signed 32 */ \
+                unsigned char buf[5]; \
+                buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
+                msgpack_pack_append_buffer(x, buf, 5); \
+            } \
+        } else { \
+            if(d < -(1<<7)) { \
+                /* signed 16 */ \
+                unsigned char buf[3]; \
+                buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
+                msgpack_pack_append_buffer(x, buf, 3); \
+            } else { \
+                /* signed 8 */ \
+                unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \
+                msgpack_pack_append_buffer(x, buf, 2); \
+            } \
+        } \
+    } else if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
+    } else { \
+        if(d < (1LL<<16)) { \
+            if(d < (1<<8)) { \
+                /* unsigned 8 */ \
+                unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
+                msgpack_pack_append_buffer(x, buf, 2); \
+            } else { \
+                /* unsigned 16 */ \
+                unsigned char buf[3]; \
+                buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+                msgpack_pack_append_buffer(x, buf, 3); \
+            } \
+        } else { \
+            if(d < (1LL<<32)) { \
+                /* unsigned 32 */ \
+                unsigned char buf[5]; \
+                buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+                msgpack_pack_append_buffer(x, buf, 5); \
+            } else { \
+                /* unsigned 64 */ \
+                unsigned char buf[9]; \
+                buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
+                msgpack_pack_append_buffer(x, buf, 9); \
+            } \
+        } \
+    } \
+} while(0)
+
+
+static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d)
+{
+    msgpack_pack_real_uint8(x, d);
+}
+
+static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d)
+{
+    msgpack_pack_real_uint16(x, d);
+}
+
+static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d)
+{
+    msgpack_pack_real_uint32(x, d);
+}
+
+static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d)
+{
+    msgpack_pack_real_uint64(x, d);
+}
+
+static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d)
+{
+    msgpack_pack_real_int8(x, d);
+}
+
+static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d)
+{
+    msgpack_pack_real_int16(x, d);
+}
+
+static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d)
+{
+    msgpack_pack_real_int32(x, d);
+}
+
+static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d)
+{
+    msgpack_pack_real_int64(x, d);
+}
+
+
+//#ifdef msgpack_pack_inline_func_cint
+
+static inline int msgpack_pack_short(msgpack_packer* x, short d)
+{
+#if defined(SIZEOF_SHORT)
+#if SIZEOF_SHORT == 2
+    msgpack_pack_real_int16(x, d);
+#elif SIZEOF_SHORT == 4
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(SHRT_MAX)
+#if SHRT_MAX == 0x7fff
+    msgpack_pack_real_int16(x, d);
+#elif SHRT_MAX == 0x7fffffff
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(short) == 2) {
+    msgpack_pack_real_int16(x, d);
+} else if(sizeof(short) == 4) {
+    msgpack_pack_real_int32(x, d);
+} else {
+    msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_int(msgpack_packer* x, int d)
+{
+#if defined(SIZEOF_INT)
+#if SIZEOF_INT == 2
+    msgpack_pack_real_int16(x, d);
+#elif SIZEOF_INT == 4
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(INT_MAX)
+#if INT_MAX == 0x7fff
+    msgpack_pack_real_int16(x, d);
+#elif INT_MAX == 0x7fffffff
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(int) == 2) {
+    msgpack_pack_real_int16(x, d);
+} else if(sizeof(int) == 4) {
+    msgpack_pack_real_int32(x, d);
+} else {
+    msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_long(msgpack_packer* x, long d)
+{
+#if defined(SIZEOF_LONG)
+#if SIZEOF_LONG == 2
+    msgpack_pack_real_int16(x, d);
+#elif SIZEOF_LONG == 4
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(LONG_MAX)
+#if LONG_MAX == 0x7fffL
+    msgpack_pack_real_int16(x, d);
+#elif LONG_MAX == 0x7fffffffL
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(long) == 2) {
+    msgpack_pack_real_int16(x, d);
+} else if(sizeof(long) == 4) {
+    msgpack_pack_real_int32(x, d);
+} else {
+    msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_long_long(msgpack_packer* x, long long d)
+{
+#if defined(SIZEOF_LONG_LONG)
+#if SIZEOF_LONG_LONG == 2
+    msgpack_pack_real_int16(x, d);
+#elif SIZEOF_LONG_LONG == 4
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(LLONG_MAX)
+#if LLONG_MAX == 0x7fffL
+    msgpack_pack_real_int16(x, d);
+#elif LLONG_MAX == 0x7fffffffL
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(long long) == 2) {
+    msgpack_pack_real_int16(x, d);
+} else if(sizeof(long long) == 4) {
+    msgpack_pack_real_int32(x, d);
+} else {
+    msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short d)
+{
+#if defined(SIZEOF_SHORT)
+#if SIZEOF_SHORT == 2
+    msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_SHORT == 4
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(USHRT_MAX)
+#if USHRT_MAX == 0xffffU
+    msgpack_pack_real_uint16(x, d);
+#elif USHRT_MAX == 0xffffffffU
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned short) == 2) {
+    msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned short) == 4) {
+    msgpack_pack_real_uint32(x, d);
+} else {
+    msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d)
+{
+#if defined(SIZEOF_INT)
+#if SIZEOF_INT == 2
+    msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_INT == 4
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(UINT_MAX)
+#if UINT_MAX == 0xffffU
+    msgpack_pack_real_uint16(x, d);
+#elif UINT_MAX == 0xffffffffU
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned int) == 2) {
+    msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned int) == 4) {
+    msgpack_pack_real_uint32(x, d);
+} else {
+    msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d)
+{
+#if defined(SIZEOF_LONG)
+#if SIZEOF_LONG == 2
+    msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_LONG == 4
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(ULONG_MAX)
+#if ULONG_MAX == 0xffffUL
+    msgpack_pack_real_uint16(x, d);
+#elif ULONG_MAX == 0xffffffffUL
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned long) == 2) {
+    msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned long) == 4) {
+    msgpack_pack_real_uint32(x, d);
+} else {
+    msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned long long d)
+{
+#if defined(SIZEOF_LONG_LONG)
+#if SIZEOF_LONG_LONG == 2
+    msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_LONG_LONG == 4
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(ULLONG_MAX)
+#if ULLONG_MAX == 0xffffUL
+    msgpack_pack_real_uint16(x, d);
+#elif ULLONG_MAX == 0xffffffffUL
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned long long) == 2) {
+    msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned long long) == 4) {
+    msgpack_pack_real_uint32(x, d);
+} else {
+    msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+//#undef msgpack_pack_inline_func_cint
+//#endif
+
+
+
+/*
+ * Float
+ */
+
+static inline int msgpack_pack_float(msgpack_packer* x, float d)
+{
+    union { float f; uint32_t i; } mem;
+    mem.f = d;
+    unsigned char buf[5];
+    buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i);
+    msgpack_pack_append_buffer(x, buf, 5);
+}
+
+static inline int msgpack_pack_double(msgpack_packer* x, double d)
+{
+    union { double f; uint64_t i; } mem;
+    mem.f = d;
+    unsigned char buf[9];
+    buf[0] = 0xcb;
+#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
+    // https://github.com/msgpack/msgpack-perl/pull/1
+    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
+#endif
+    _msgpack_store64(&buf[1], mem.i);
+    msgpack_pack_append_buffer(x, buf, 9);
+}
+
+
+/*
+ * Nil
+ */
+
+static inline int msgpack_pack_nil(msgpack_packer* x)
+{
+    static const unsigned char d = 0xc0;
+    msgpack_pack_append_buffer(x, &d, 1);
+}
+
+
+/*
+ * Boolean
+ */
+
+static inline int msgpack_pack_true(msgpack_packer* x)
+{
+    static const unsigned char d = 0xc3;
+    msgpack_pack_append_buffer(x, &d, 1);
+}
+
+static inline int msgpack_pack_false(msgpack_packer* x)
+{
+    static const unsigned char d = 0xc2;
+    msgpack_pack_append_buffer(x, &d, 1);
+}
+
+
+/*
+ * Array
+ */
+
+static inline int msgpack_pack_array(msgpack_packer* x, unsigned int n)
+{
+    if(n < 16) {
+        unsigned char d = 0x90 | n;
+        msgpack_pack_append_buffer(x, &d, 1);
+    } else if(n < 65536) {
+        unsigned char buf[3];
+        buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5];
+        buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
+
+/*
+ * Map
+ */
+
+static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n)
+{
+    if(n < 16) {
+        unsigned char d = 0x80 | n;
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
+    } else if(n < 65536) {
+        unsigned char buf[3];
+        buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5];
+        buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
+
+/*
+ * Raw
+ */
+
+static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
+{
+    if (l < 32) {
+        unsigned char d = 0xa0 | (uint8_t)l;
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
+    } else if (x->use_bin_type && l < 256) {  // str8 is new format introduced with bin.
+        unsigned char buf[2] = {0xd9, (uint8_t)l};
+        msgpack_pack_append_buffer(x, buf, 2);
+    } else if (l < 65536) {
+        unsigned char buf[3];
+        buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5];
+        buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
+/*
+ * bin
+ */
+static inline int msgpack_pack_bin(msgpack_packer *x, size_t l)
+{
+    if (!x->use_bin_type) {
+        return msgpack_pack_raw(x, l);
+    }
+    if (l < 256) {
+        unsigned char buf[2] = {0xc4, (unsigned char)l};
+        msgpack_pack_append_buffer(x, buf, 2);
+    } else if (l < 65536) {
+        unsigned char buf[3] = {0xc5};
+        _msgpack_store16(&buf[1], (uint16_t)l);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5] = {0xc6};
+        _msgpack_store32(&buf[1], (uint32_t)l);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
+static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l)
+{
+    if (l > 0) msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
+    return 0;
+}
+
+/*
+ * Ext
+ */
+static inline int msgpack_pack_ext(msgpack_packer* x, int8_t typecode, size_t l)
+{
+    if (l == 1) {
+        unsigned char buf[2];
+        buf[0] = 0xd4;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 2) {
+        unsigned char buf[2];
+        buf[0] = 0xd5;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 4) {
+        unsigned char buf[2];
+        buf[0] = 0xd6;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 8) {
+        unsigned char buf[2];
+        buf[0] = 0xd7;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 16) {
+        unsigned char buf[2];
+        buf[0] = 0xd8;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l < 256) {
+        unsigned char buf[3];
+        buf[0] = 0xc7;
+        buf[1] = l;
+        buf[2] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else if(l < 65536) {
+        unsigned char buf[4];
+        buf[0] = 0xc8;
+        _msgpack_store16(&buf[1], (uint16_t)l);
+        buf[3] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 4);
+    } else {
+        unsigned char buf[6];
+        buf[0] = 0xc9;
+        _msgpack_store32(&buf[1], (uint32_t)l);
+        buf[5] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 6);
+    }
+
+}
+
+
+
+#undef msgpack_pack_append_buffer
+
+#undef TAKE8_8
+#undef TAKE8_16
+#undef TAKE8_32
+#undef TAKE8_64
+
+#undef msgpack_pack_real_uint8
+#undef msgpack_pack_real_uint16
+#undef msgpack_pack_real_uint32
+#undef msgpack_pack_real_uint64
+#undef msgpack_pack_real_int8
+#undef msgpack_pack_real_int16
+#undef msgpack_pack_real_int32
+#undef msgpack_pack_real_int64

utils/converters/msgpack/msgpack/sysdep.h

@@ -0,0 +1,194 @@
+/*
+ * MessagePack system dependencies
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_SYSDEP_H__
+#define MSGPACK_SYSDEP_H__
+
+#include <stdlib.h>
+#include <stddef.h>
+#if defined(_MSC_VER) && _MSC_VER < 1600
+typedef __int8 int8_t;
+typedef unsigned __int8 uint8_t;
+typedef __int16 int16_t;
+typedef unsigned __int16 uint16_t;
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+#elif defined(_MSC_VER)  // && _MSC_VER >= 1600
+#include <stdint.h>
+#else
+#include <stdint.h>
+#include <stdbool.h>
+#endif
+
+#ifdef _WIN32
+#define _msgpack_atomic_counter_header <windows.h>
+typedef long _msgpack_atomic_counter_t;
+#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr)
+#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr)
+#elif defined(__GNUC__) && ((__GNUC__*10 + __GNUC_MINOR__) < 41)
+#define _msgpack_atomic_counter_header "gcc_atomic.h"
+#else
+typedef unsigned int _msgpack_atomic_counter_t;
+#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1)
+#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1)
+#endif
+
+#ifdef _WIN32
+
+#ifdef __cplusplus
+/* numeric_limits<T>::min,max */
+#ifdef max
+#undef max
+#endif
+#ifdef min
+#undef min
+#endif
+#endif
+
+#else
+#include <arpa/inet.h>  /* __BYTE_ORDER */
+#endif
+
+#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+#define __LITTLE_ENDIAN__
+#elif __BYTE_ORDER == __BIG_ENDIAN
+#define __BIG_ENDIAN__
+#elif _WIN32
+#define __LITTLE_ENDIAN__
+#endif
+#endif
+
+
+#ifdef __LITTLE_ENDIAN__
+
+#ifdef _WIN32
+#  if defined(ntohs)
+#    define _msgpack_be16(x) ntohs(x)
+#  elif defined(_byteswap_ushort) || (defined(_MSC_VER) && _MSC_VER >= 1400)
+#    define _msgpack_be16(x) ((uint16_t)_byteswap_ushort((unsigned short)x))
+#  else
+#    define _msgpack_be16(x) ( \
+        ((((uint16_t)x) <<  8) ) | \
+        ((((uint16_t)x) >>  8) ) )
+#  endif
+#else
+#  define _msgpack_be16(x) ntohs(x)
+#endif
+
+#ifdef _WIN32
+#  if defined(ntohl)
+#    define _msgpack_be32(x) ntohl(x)
+#  elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400)
+#    define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x))
+#  else
+#    define _msgpack_be32(x) \
+        ( ((((uint32_t)x) << 24)               ) | \
+          ((((uint32_t)x) <<  8) & 0x00ff0000U ) | \
+          ((((uint32_t)x) >>  8) & 0x0000ff00U ) | \
+          ((((uint32_t)x) >> 24)               ) )
+#  endif
+#else
+#  define _msgpack_be32(x) ntohl(x)
+#endif
+
+#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400)
+#  define _msgpack_be64(x) (_byteswap_uint64(x))
+#elif defined(bswap_64)
+#  define _msgpack_be64(x) bswap_64(x)
+#elif defined(__DARWIN_OSSwapInt64)
+#  define _msgpack_be64(x) __DARWIN_OSSwapInt64(x)
+#else
+#define _msgpack_be64(x) \
+    ( ((((uint64_t)x) << 56)                         ) | \
+      ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \
+      ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \
+      ((((uint64_t)x) <<  8) & 0x000000ff00000000ULL ) | \
+      ((((uint64_t)x) >>  8) & 0x00000000ff000000ULL ) | \
+      ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \
+      ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \
+      ((((uint64_t)x) >> 56)                         ) )
+#endif
+
+#define _msgpack_load16(cast, from) ((cast)( \
+        (((uint16_t)((uint8_t*)(from))[0]) << 8) | \
+        (((uint16_t)((uint8_t*)(from))[1])     ) ))
+
+#define _msgpack_load32(cast, from) ((cast)( \
+        (((uint32_t)((uint8_t*)(from))[0]) << 24) | \
+        (((uint32_t)((uint8_t*)(from))[1]) << 16) | \
+        (((uint32_t)((uint8_t*)(from))[2]) <<  8) | \
+        (((uint32_t)((uint8_t*)(from))[3])      ) ))
+
+#define _msgpack_load64(cast, from) ((cast)( \
+        (((uint64_t)((uint8_t*)(from))[0]) << 56) | \
+        (((uint64_t)((uint8_t*)(from))[1]) << 48) | \
+        (((uint64_t)((uint8_t*)(from))[2]) << 40) | \
+        (((uint64_t)((uint8_t*)(from))[3]) << 32) | \
+        (((uint64_t)((uint8_t*)(from))[4]) << 24) | \
+        (((uint64_t)((uint8_t*)(from))[5]) << 16) | \
+        (((uint64_t)((uint8_t*)(from))[6]) << 8)  | \
+        (((uint64_t)((uint8_t*)(from))[7])     )  ))
+
+#else
+
+#define _msgpack_be16(x) (x)
+#define _msgpack_be32(x) (x)
+#define _msgpack_be64(x) (x)
+
+#define _msgpack_load16(cast, from) ((cast)( \
+        (((uint16_t)((uint8_t*)from)[0]) << 8) | \
+        (((uint16_t)((uint8_t*)from)[1])     ) ))
+
+#define _msgpack_load32(cast, from) ((cast)( \
+        (((uint32_t)((uint8_t*)from)[0]) << 24) | \
+        (((uint32_t)((uint8_t*)from)[1]) << 16) | \
+        (((uint32_t)((uint8_t*)from)[2]) <<  8) | \
+        (((uint32_t)((uint8_t*)from)[3])      ) ))
+
+#define _msgpack_load64(cast, from) ((cast)( \
+        (((uint64_t)((uint8_t*)from)[0]) << 56) | \
+        (((uint64_t)((uint8_t*)from)[1]) << 48) | \
+        (((uint64_t)((uint8_t*)from)[2]) << 40) | \
+        (((uint64_t)((uint8_t*)from)[3]) << 32) | \
+        (((uint64_t)((uint8_t*)from)[4]) << 24) | \
+        (((uint64_t)((uint8_t*)from)[5]) << 16) | \
+        (((uint64_t)((uint8_t*)from)[6]) << 8)  | \
+        (((uint64_t)((uint8_t*)from)[7])     )  ))
+#endif
+
+
+#define _msgpack_store16(to, num) \
+    do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0)
+#define _msgpack_store32(to, num) \
+    do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0)
+#define _msgpack_store64(to, num) \
+    do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0)
+
+/*
+#define _msgpack_load16(cast, from) \
+    ({ cast val; memcpy(&val, (char*)from, 2); _msgpack_be16(val); })
+#define _msgpack_load32(cast, from) \
+    ({ cast val; memcpy(&val, (char*)from, 4); _msgpack_be32(val); })
+#define _msgpack_load64(cast, from) \
+    ({ cast val; memcpy(&val, (char*)from, 8); _msgpack_be64(val); })
+*/
+
+
+#endif /* msgpack/sysdep.h */

utils/converters/msgpack/msgpack/unpack.h

@@ -0,0 +1,263 @@
+/*
+ * MessagePack for Python unpacking routine
+ *
+ * Copyright (C) 2009 Naoki INADA
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#define MSGPACK_EMBED_STACK_SIZE  (1024)
+#include "unpack_define.h"
+
+typedef struct unpack_user {
+    int use_list;
+    PyObject *object_hook;
+    bool has_pairs_hook;
+    PyObject *list_hook;
+    PyObject *ext_hook;
+    const char *encoding;
+    const char *unicode_errors;
+} unpack_user;
+
+typedef PyObject* msgpack_unpack_object;
+struct unpack_context;
+typedef struct unpack_context unpack_context;
+typedef int (*execute_fn)(unpack_context *ctx, const char* data, size_t len, size_t* off);
+
+static inline msgpack_unpack_object unpack_callback_root(unpack_user* u)
+{
+    return NULL;
+}
+
+static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyInt_FromLong((long)d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o)
+{
+    return unpack_callback_uint16(u, d, o);
+}
+
+
+static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
+{
+    PyObject *p;
+#if UINT32_MAX > LONG_MAX
+    if (d > LONG_MAX) {
+        p = PyLong_FromUnsignedLong((unsigned long)d);
+    } else
+#endif
+    {
+        p = PyInt_FromLong((long)d);
+    }
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o)
+{
+    PyObject *p;
+    if (d > LONG_MAX) {
+        p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d);
+    } else {
+        p = PyInt_FromLong((long)d);
+    }
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyInt_FromLong(d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o)
+{
+    return unpack_callback_int32(u, d, o);
+}
+
+static inline int unpack_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o)
+{
+    return unpack_callback_int32(u, d, o);
+}
+
+static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o)
+{
+    PyObject *p;
+    if (d > LONG_MAX || d < LONG_MIN) {
+        p = PyLong_FromLongLong((unsigned PY_LONG_LONG)d);
+    } else {
+        p = PyInt_FromLong((long)d);
+    }
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_double(unpack_user* u, double d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyFloat_FromDouble(d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_float(unpack_user* u, float d, msgpack_unpack_object* o)
+{
+    return unpack_callback_double(u, d, o);
+}
+
+static inline int unpack_callback_nil(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_None); *o = Py_None; return 0; }
+
+static inline int unpack_callback_true(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_True); *o = Py_True; return 0; }
+
+static inline int unpack_callback_false(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_False); *o = Py_False; return 0; }
+
+static inline int unpack_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+{
+    PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n);
+
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o)
+{
+    if (u->use_list)
+        PyList_SET_ITEM(*c, current, o);
+    else
+        PyTuple_SET_ITEM(*c, current, o);
+    return 0;
+}
+
+static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
+{
+    if (u->list_hook) {
+        PyObject *new_c = PyObject_CallFunctionObjArgs(u->list_hook, *c, NULL);
+        if (!new_c)
+            return -1;
+        Py_DECREF(*c);
+        *c = new_c;
+    }
+    return 0;
+}
+
+static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+{
+    PyObject *p;
+    if (u->has_pairs_hook) {
+        p = PyList_New(n); // Or use tuple?
+    }
+    else {
+        p = PyDict_New();
+    }
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
+{
+    if (u->has_pairs_hook) {
+        msgpack_unpack_object item = PyTuple_Pack(2, k, v);
+        if (!item)
+            return -1;
+        Py_DECREF(k);
+        Py_DECREF(v);
+        PyList_SET_ITEM(*c, current, item);
+        return 0;
+    }
+    else if (PyDict_SetItem(*c, k, v) == 0) {
+        Py_DECREF(k);
+        Py_DECREF(v);
+        return 0;
+    }
+    return -1;
+}
+
+static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
+{
+    if (u->object_hook) {
+        PyObject *new_c = PyObject_CallFunctionObjArgs(u->object_hook, *c, NULL);
+        if (!new_c)
+            return -1;
+
+        Py_DECREF(*c);
+        *c = new_c;
+    }
+    return 0;
+}
+
+static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+{
+    PyObject *py;
+    if(u->encoding) {
+        py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors);
+    } else {
+        py = PyBytes_FromStringAndSize(p, l);
+    }
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
+static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+{
+    PyObject *py = PyBytes_FromStringAndSize(p, l);
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
+static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
+                                      unsigned int lenght, msgpack_unpack_object* o)
+{
+    PyObject *py;
+    int8_t typecode = (int8_t)*pos++;
+    if (!u->ext_hook) {
+        PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL");
+        return -1;
+    }
+    // length also includes the typecode, so the actual data is lenght-1
+#if PY_MAJOR_VERSION == 2
+    py = PyObject_CallFunction(u->ext_hook, "(is#)", typecode, pos, lenght-1);
+#else
+    py = PyObject_CallFunction(u->ext_hook, "(iy#)", typecode, pos, lenght-1);
+#endif
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
+#include "unpack_template.h"

utils/converters/msgpack/msgpack/unpack_define.h

@@ -0,0 +1,95 @@
+/*
+ * MessagePack unpacking routine template
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_UNPACK_DEFINE_H__
+#define MSGPACK_UNPACK_DEFINE_H__
+
+#include "msgpack/sysdep.h"
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <stdio.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#ifndef MSGPACK_EMBED_STACK_SIZE
+#define MSGPACK_EMBED_STACK_SIZE 32
+#endif
+
+
+// CS is first byte & 0x1f
+typedef enum {
+    CS_HEADER            = 0x00,  // nil
+
+    //CS_                = 0x01,
+    //CS_                = 0x02,  // false
+    //CS_                = 0x03,  // true
+
+    CS_BIN_8             = 0x04,
+    CS_BIN_16            = 0x05,
+    CS_BIN_32            = 0x06,
+
+    CS_EXT_8             = 0x07,
+    CS_EXT_16            = 0x08,
+    CS_EXT_32            = 0x09,
+
+    CS_FLOAT             = 0x0a,
+    CS_DOUBLE            = 0x0b,
+    CS_UINT_8            = 0x0c,
+    CS_UINT_16           = 0x0d,
+    CS_UINT_32           = 0x0e,
+    CS_UINT_64           = 0x0f,
+    CS_INT_8             = 0x10,
+    CS_INT_16            = 0x11,
+    CS_INT_32            = 0x12,
+    CS_INT_64            = 0x13,
+
+    //CS_FIXEXT1           = 0x14,
+    //CS_FIXEXT2           = 0x15,
+    //CS_FIXEXT4           = 0x16,
+    //CS_FIXEXT8           = 0x17,
+    //CS_FIXEXT16          = 0x18,
+
+    CS_RAW_8             = 0x19,
+    CS_RAW_16            = 0x1a,
+    CS_RAW_32            = 0x1b,
+    CS_ARRAY_16          = 0x1c,
+    CS_ARRAY_32          = 0x1d,
+    CS_MAP_16            = 0x1e,
+    CS_MAP_32            = 0x1f,
+
+    ACS_RAW_VALUE,
+    ACS_BIN_VALUE,
+    ACS_EXT_VALUE,
+} msgpack_unpack_state;
+
+
+typedef enum {
+    CT_ARRAY_ITEM,
+    CT_MAP_KEY,
+    CT_MAP_VALUE,
+} msgpack_container_type;
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* msgpack/unpack_define.h */

utils/converters/msgpack/msgpack/unpack_template.h

@@ -0,0 +1,475 @@
+/*
+ * MessagePack unpacking routine template
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#ifndef USE_CASE_RANGE
+#if !defined(_MSC_VER)
+#define USE_CASE_RANGE
+#endif
+#endif
+
+typedef struct unpack_stack {
+    PyObject* obj;
+    size_t size;
+    size_t count;
+    unsigned int ct;
+    PyObject* map_key;
+} unpack_stack;
+
+struct unpack_context {
+    unpack_user user;
+    unsigned int cs;
+    unsigned int trail;
+    unsigned int top;
+    /*
+    unpack_stack* stack;
+    unsigned int stack_size;
+    unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE];
+    */
+    unpack_stack stack[MSGPACK_EMBED_STACK_SIZE];
+};
+
+
+static inline void unpack_init(unpack_context* ctx)
+{
+    ctx->cs = CS_HEADER;
+    ctx->trail = 0;
+    ctx->top = 0;
+    /*
+    ctx->stack = ctx->embed_stack;
+    ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
+    */
+    ctx->stack[0].obj = unpack_callback_root(&ctx->user);
+}
+
+/*
+static inline void unpack_destroy(unpack_context* ctx)
+{
+    if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
+        free(ctx->stack);
+    }
+}
+*/
+
+static inline PyObject* unpack_data(unpack_context* ctx)
+{
+    return (ctx)->stack[0].obj;
+}
+
+
+template <bool construct>
+static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off)
+{
+    assert(len >= *off);
+
+    const unsigned char* p = (unsigned char*)data + *off;
+    const unsigned char* const pe = (unsigned char*)data + len;
+    const void* n = NULL;
+
+    unsigned int trail = ctx->trail;
+    unsigned int cs = ctx->cs;
+    unsigned int top = ctx->top;
+    unpack_stack* stack = ctx->stack;
+    /*
+    unsigned int stack_size = ctx->stack_size;
+    */
+    unpack_user* user = &ctx->user;
+
+    PyObject* obj;
+    unpack_stack* c = NULL;
+
+    int ret;
+
+#define construct_cb(name) \
+    construct && unpack_callback ## name
+
+#define push_simple_value(func) \
+    if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \
+    goto _push
+#define push_fixed_value(func, arg) \
+    if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \
+    goto _push
+#define push_variable_value(func, base, pos, len) \
+    if(construct_cb(func)(user, \
+        (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
+    goto _push
+
+#define again_fixed_trail(_cs, trail_len) \
+    trail = trail_len; \
+    cs = _cs; \
+    goto _fixed_trail_again
+#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
+    trail = trail_len; \
+    if(trail == 0) { goto ifzero; } \
+    cs = _cs; \
+    goto _fixed_trail_again
+
+#define start_container(func, count_, ct_) \
+    if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
+    if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
+    if((count_) == 0) { obj = stack[top].obj; \
+        if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \
+        goto _push; } \
+    stack[top].ct = ct_; \
+    stack[top].size  = count_; \
+    stack[top].count = 0; \
+    ++top; \
+    /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
+    /*printf("stack push %d\n", top);*/ \
+    /* FIXME \
+    if(top >= stack_size) { \
+        if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
+            size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \
+            size_t nsize = csize * 2; \
+            unpack_stack* tmp = (unpack_stack*)malloc(nsize); \
+            if(tmp == NULL) { goto _failed; } \
+            memcpy(tmp, ctx->stack, csize); \
+            ctx->stack = stack = tmp; \
+            ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
+        } else { \
+            size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \
+            unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \
+            if(tmp == NULL) { goto _failed; } \
+            ctx->stack = stack = tmp; \
+            ctx->stack_size = stack_size = stack_size * 2; \
+        } \
+    } \
+    */ \
+    goto _header_again
+
+#define NEXT_CS(p)  ((unsigned int)*p & 0x1f)
+
+#ifdef USE_CASE_RANGE
+#define SWITCH_RANGE_BEGIN     switch(*p) {
+#define SWITCH_RANGE(FROM, TO) case FROM ... TO:
+#define SWITCH_RANGE_DEFAULT   default:
+#define SWITCH_RANGE_END       }
+#else
+#define SWITCH_RANGE_BEGIN     { if(0) {
+#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) {
+#define SWITCH_RANGE_DEFAULT   } else {
+#define SWITCH_RANGE_END       } }
+#endif
+
+    if(p == pe) { goto _out; }
+    do {
+        switch(cs) {
+        case CS_HEADER:
+            SWITCH_RANGE_BEGIN
+            SWITCH_RANGE(0x00, 0x7f)  // Positive Fixnum
+                push_fixed_value(_uint8, *(uint8_t*)p);
+            SWITCH_RANGE(0xe0, 0xff)  // Negative Fixnum
+                push_fixed_value(_int8, *(int8_t*)p);
+            SWITCH_RANGE(0xc0, 0xdf)  // Variable
+                switch(*p) {
+                case 0xc0:  // nil
+                    push_simple_value(_nil);
+                //case 0xc1:  // never used
+                case 0xc2:  // false
+                    push_simple_value(_false);
+                case 0xc3:  // true
+                    push_simple_value(_true);
+                case 0xc4:  // bin 8
+                    again_fixed_trail(NEXT_CS(p), 1);
+                case 0xc5:  // bin 16
+                    again_fixed_trail(NEXT_CS(p), 2);
+                case 0xc6:  // bin 32
+                    again_fixed_trail(NEXT_CS(p), 4);
+                case 0xc7:  // ext 8
+                    again_fixed_trail(NEXT_CS(p), 1);
+                case 0xc8:  // ext 16
+                    again_fixed_trail(NEXT_CS(p), 2);
+                case 0xc9:  // ext 32
+                    again_fixed_trail(NEXT_CS(p), 4);
+                case 0xca:  // float
+                case 0xcb:  // double
+                case 0xcc:  // unsigned int  8
+                case 0xcd:  // unsigned int 16
+                case 0xce:  // unsigned int 32
+                case 0xcf:  // unsigned int 64
+                case 0xd0:  // signed int  8
+                case 0xd1:  // signed int 16
+                case 0xd2:  // signed int 32
+                case 0xd3:  // signed int 64
+                    again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
+                case 0xd4:  // fixext 1
+                case 0xd5:  // fixext 2
+                case 0xd6:  // fixext 4
+                case 0xd7:  // fixext 8
+                    again_fixed_trail_if_zero(ACS_EXT_VALUE, 
+                                              (1 << (((unsigned int)*p) & 0x03))+1,
+                                              _ext_zero);
+                case 0xd8:  // fixext 16
+                    again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero);
+                case 0xd9:  // str 8
+                    again_fixed_trail(NEXT_CS(p), 1);
+                case 0xda:  // raw 16
+                case 0xdb:  // raw 32
+                case 0xdc:  // array 16
+                case 0xdd:  // array 32
+                case 0xde:  // map 16
+                case 0xdf:  // map 32
+                    again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
+                default:
+                    goto _failed;
+                }
+            SWITCH_RANGE(0xa0, 0xbf)  // FixRaw
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
+            SWITCH_RANGE(0x90, 0x9f)  // FixArray
+                start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM);
+            SWITCH_RANGE(0x80, 0x8f)  // FixMap
+                start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
+
+            SWITCH_RANGE_DEFAULT
+                goto _failed;
+            SWITCH_RANGE_END
+            // end CS_HEADER
+
+
+        _fixed_trail_again:
+            ++p;
+
+        default:
+            if((size_t)(pe - p) < trail) { goto _out; }
+            n = p;  p += trail - 1;
+            switch(cs) {
+            case CS_EXT_8:
+                again_fixed_trail_if_zero(ACS_EXT_VALUE, *(uint8_t*)n+1, _ext_zero);
+            case CS_EXT_16:
+                again_fixed_trail_if_zero(ACS_EXT_VALUE,
+                                          _msgpack_load16(uint16_t,n)+1,
+                                          _ext_zero);
+            case CS_EXT_32:
+                again_fixed_trail_if_zero(ACS_EXT_VALUE,
+                                          _msgpack_load32(uint32_t,n)+1,
+                                          _ext_zero);
+            case CS_FLOAT: {
+                    union { uint32_t i; float f; } mem;
+                    mem.i = _msgpack_load32(uint32_t,n);
+                    push_fixed_value(_float, mem.f); }
+            case CS_DOUBLE: {
+                    union { uint64_t i; double f; } mem;
+                    mem.i = _msgpack_load64(uint64_t,n);
+#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
+                    // https://github.com/msgpack/msgpack-perl/pull/1
+                    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
+#endif
+                    push_fixed_value(_double, mem.f); }
+            case CS_UINT_8:
+                push_fixed_value(_uint8, *(uint8_t*)n);
+            case CS_UINT_16:
+                push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
+            case CS_UINT_32:
+                push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
+            case CS_UINT_64:
+                push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
+
+            case CS_INT_8:
+                push_fixed_value(_int8, *(int8_t*)n);
+            case CS_INT_16:
+                push_fixed_value(_int16, _msgpack_load16(int16_t,n));
+            case CS_INT_32:
+                push_fixed_value(_int32, _msgpack_load32(int32_t,n));
+            case CS_INT_64:
+                push_fixed_value(_int64, _msgpack_load64(int64_t,n));
+
+            case CS_BIN_8:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero);
+            case CS_BIN_16:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero);
+            case CS_BIN_32:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero);
+            case ACS_BIN_VALUE:
+            _bin_zero:
+                push_variable_value(_bin, data, n, trail);
+
+            case CS_RAW_8:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero);
+            case CS_RAW_16:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
+            case CS_RAW_32:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
+            case ACS_RAW_VALUE:
+            _raw_zero:
+                push_variable_value(_raw, data, n, trail);
+
+            case ACS_EXT_VALUE:
+            _ext_zero:
+                push_variable_value(_ext, data, n, trail);
+
+            case CS_ARRAY_16:
+                start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
+            case CS_ARRAY_32:
+                /* FIXME security guard */
+                start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
+
+            case CS_MAP_16:
+                start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
+            case CS_MAP_32:
+                /* FIXME security guard */
+                start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
+
+            default:
+                goto _failed;
+            }
+        }
+
+_push:
+    if(top == 0) { goto _finish; }
+    c = &stack[top-1];
+    switch(c->ct) {
+    case CT_ARRAY_ITEM:
+        if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
+        if(++c->count == c->size) {
+            obj = c->obj;
+            if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; }
+            --top;
+            /*printf("stack pop %d\n", top);*/
+            goto _push;
+        }
+        goto _header_again;
+    case CT_MAP_KEY:
+        c->map_key = obj;
+        c->ct = CT_MAP_VALUE;
+        goto _header_again;
+    case CT_MAP_VALUE:
+        if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+        if(++c->count == c->size) {
+            obj = c->obj;
+            if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
+            --top;
+            /*printf("stack pop %d\n", top);*/
+            goto _push;
+        }
+        c->ct = CT_MAP_KEY;
+        goto _header_again;
+
+    default:
+        goto _failed;
+    }
+
+_header_again:
+        cs = CS_HEADER;
+        ++p;
+    } while(p != pe);
+    goto _out;
+
+
+_finish:
+    if (!construct)
+        unpack_callback_nil(user, &obj);
+    stack[0].obj = obj;
+    ++p;
+    ret = 1;
+    /*printf("-- finish --\n"); */
+    goto _end;
+
+_failed:
+    /*printf("** FAILED **\n"); */
+    ret = -1;
+    goto _end;
+
+_out:
+    ret = 0;
+    goto _end;
+
+_end:
+    ctx->cs = cs;
+    ctx->trail = trail;
+    ctx->top = top;
+    *off = p - (const unsigned char*)data;
+
+    return ret;
+#undef construct_cb
+}
+
+#undef SWITCH_RANGE_BEGIN
+#undef SWITCH_RANGE
+#undef SWITCH_RANGE_DEFAULT
+#undef SWITCH_RANGE_END
+#undef push_simple_value
+#undef push_fixed_value
+#undef push_variable_value
+#undef again_fixed_trail
+#undef again_fixed_trail_if_zero
+#undef start_container
+
+template <unsigned int fixed_offset, unsigned int var_offset>
+static inline int unpack_container_header(unpack_context* ctx, const char* data, size_t len, size_t* off)
+{
+    assert(len >= *off);
+    uint32_t size;
+    const unsigned char *const p = (unsigned char*)data + *off;
+
+#define inc_offset(inc) \
+    if (len - *off < inc) \
+        return 0; \
+    *off += inc;
+
+    switch (*p) {
+    case var_offset:
+        inc_offset(3);
+        size = _msgpack_load16(uint16_t, p + 1);
+        break;
+    case var_offset + 1:
+        inc_offset(5);
+        size = _msgpack_load32(uint32_t, p + 1);
+        break;
+#ifdef USE_CASE_RANGE
+    case fixed_offset + 0x0 ... fixed_offset + 0xf:
+#else
+    case fixed_offset + 0x0:
+    case fixed_offset + 0x1:
+    case fixed_offset + 0x2:
+    case fixed_offset + 0x3:
+    case fixed_offset + 0x4:
+    case fixed_offset + 0x5:
+    case fixed_offset + 0x6:
+    case fixed_offset + 0x7:
+    case fixed_offset + 0x8:
+    case fixed_offset + 0x9:
+    case fixed_offset + 0xa:
+    case fixed_offset + 0xb:
+    case fixed_offset + 0xc:
+    case fixed_offset + 0xd:
+    case fixed_offset + 0xe:
+    case fixed_offset + 0xf:
+#endif
+        ++*off;
+        size = ((unsigned int)*p) & 0x0f;
+        break;
+    default:
+        PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
+        return -1;
+    }
+    unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj);
+    return 1;
+}
+
+#undef SWITCH_RANGE_BEGIN
+#undef SWITCH_RANGE
+#undef SWITCH_RANGE_DEFAULT
+#undef SWITCH_RANGE_END
+
+static const execute_fn unpack_construct = &unpack_execute<true>;
+static const execute_fn unpack_skip = &unpack_execute<false>;
+static const execute_fn read_array_header = &unpack_container_header<0x90, 0xdc>;
+static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>;
+
+#undef NEXT_CS
+
+/* vim: set ts=4 sw=4 sts=4 expandtab  */