fallback.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714
  1. """Fallback pure Python implementation of msgpack"""
  2. import sys
  3. import array
  4. import struct
  5. if sys.version_info[0] == 3:
  6. PY3 = True
  7. int_types = int
  8. Unicode = str
  9. xrange = range
  10. def dict_iteritems(d):
  11. return d.items()
  12. else:
  13. PY3 = False
  14. int_types = (int, long)
  15. Unicode = unicode
  16. def dict_iteritems(d):
  17. return d.iteritems()
  18. if hasattr(sys, 'pypy_version_info'):
  19. # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own
  20. # StringBuilder is fastest.
  21. from __pypy__ import newlist_hint
  22. from __pypy__.builders import StringBuilder
  23. USING_STRINGBUILDER = True
  24. class StringIO(object):
  25. def __init__(self, s=b''):
  26. if s:
  27. self.builder = StringBuilder(len(s))
  28. self.builder.append(s)
  29. else:
  30. self.builder = StringBuilder()
  31. def write(self, s):
  32. self.builder.append(s)
  33. def getvalue(self):
  34. return self.builder.build()
  35. else:
  36. USING_STRINGBUILDER = False
  37. from io import BytesIO as StringIO
  38. newlist_hint = lambda size: []
  39. from msgpack.exceptions import (
  40. BufferFull,
  41. OutOfData,
  42. UnpackValueError,
  43. PackValueError,
  44. ExtraData)
  45. from msgpack import ExtType
  46. EX_SKIP = 0
  47. EX_CONSTRUCT = 1
  48. EX_READ_ARRAY_HEADER = 2
  49. EX_READ_MAP_HEADER = 3
  50. TYPE_IMMEDIATE = 0
  51. TYPE_ARRAY = 1
  52. TYPE_MAP = 2
  53. TYPE_RAW = 3
  54. TYPE_BIN = 4
  55. TYPE_EXT = 5
  56. DEFAULT_RECURSE_LIMIT = 511
  57. def unpack(stream, **kwargs):
  58. """
  59. Unpack an object from `stream`.
  60. Raises `ExtraData` when `packed` contains extra bytes.
  61. See :class:`Unpacker` for options.
  62. """
  63. unpacker = Unpacker(stream, **kwargs)
  64. ret = unpacker._fb_unpack()
  65. if unpacker._fb_got_extradata():
  66. raise ExtraData(ret, unpacker._fb_get_extradata())
  67. return ret
  68. def unpackb(packed, **kwargs):
  69. """
  70. Unpack an object from `packed`.
  71. Raises `ExtraData` when `packed` contains extra bytes.
  72. See :class:`Unpacker` for options.
  73. """
  74. unpacker = Unpacker(None, **kwargs)
  75. unpacker.feed(packed)
  76. try:
  77. ret = unpacker._fb_unpack()
  78. except OutOfData:
  79. raise UnpackValueError("Data is not enough.")
  80. if unpacker._fb_got_extradata():
  81. raise ExtraData(ret, unpacker._fb_get_extradata())
  82. return ret
  83. class Unpacker(object):
  84. """
  85. Streaming unpacker.
  86. `file_like` is a file-like object having a `.read(n)` method.
  87. When `Unpacker` is initialized with a `file_like`, `.feed()` is not
  88. usable.
  89. `read_size` is used for `file_like.read(read_size)`.
  90. If `use_list` is True (default), msgpack lists are deserialized to Python
  91. lists. Otherwise they are deserialized to tuples.
  92. `object_hook` is the same as in simplejson. If it is not None, it should
  93. be callable and Unpacker calls it with a dict argument after deserializing
  94. a map.
  95. `object_pairs_hook` is the same as in simplejson. If it is not None, it
  96. should be callable and Unpacker calls it with a list of key-value pairs
  97. after deserializing a map.
  98. `ext_hook` is callback for ext (User defined) type. It called with two
  99. arguments: (code, bytes). default: `msgpack.ExtType`
  100. `encoding` is the encoding used for decoding msgpack bytes. If it is
  101. None (default), msgpack bytes are deserialized to Python bytes.
  102. `unicode_errors` is used for decoding bytes.
  103. `max_buffer_size` limits the buffer size. 0 means INT_MAX (default).
  104. Raises `BufferFull` exception when it is unsufficient.
  105. You should set this parameter when unpacking data from an untrustred source.
  106. example of streaming deserialization from file-like object::
  107. unpacker = Unpacker(file_like)
  108. for o in unpacker:
  109. do_something(o)
  110. example of streaming deserialization from socket::
  111. unpacker = Unpacker()
  112. while 1:
  113. buf = sock.recv(1024*2)
  114. if not buf:
  115. break
  116. unpacker.feed(buf)
  117. for o in unpacker:
  118. do_something(o)
  119. """
  120. def __init__(self, file_like=None, read_size=0, use_list=True,
  121. object_hook=None, object_pairs_hook=None, list_hook=None,
  122. encoding=None, unicode_errors='strict', max_buffer_size=0,
  123. ext_hook=ExtType):
  124. if file_like is None:
  125. self._fb_feeding = True
  126. else:
  127. if not callable(file_like.read):
  128. raise TypeError("`file_like.read` must be callable")
  129. self.file_like = file_like
  130. self._fb_feeding = False
  131. self._fb_buffers = []
  132. self._fb_buf_o = 0
  133. self._fb_buf_i = 0
  134. self._fb_buf_n = 0
  135. self._max_buffer_size = max_buffer_size or 2**31-1
  136. if read_size > self._max_buffer_size:
  137. raise ValueError("read_size must be smaller than max_buffer_size")
  138. self._read_size = read_size or min(self._max_buffer_size, 2048)
  139. self._encoding = encoding
  140. self._unicode_errors = unicode_errors
  141. self._use_list = use_list
  142. self._list_hook = list_hook
  143. self._object_hook = object_hook
  144. self._object_pairs_hook = object_pairs_hook
  145. self._ext_hook = ext_hook
  146. if list_hook is not None and not callable(list_hook):
  147. raise TypeError('`list_hook` is not callable')
  148. if object_hook is not None and not callable(object_hook):
  149. raise TypeError('`object_hook` is not callable')
  150. if object_pairs_hook is not None and not callable(object_pairs_hook):
  151. raise TypeError('`object_pairs_hook` is not callable')
  152. if object_hook is not None and object_pairs_hook is not None:
  153. raise TypeError("object_pairs_hook and object_hook are mutually "
  154. "exclusive")
  155. if not callable(ext_hook):
  156. raise TypeError("`ext_hook` is not callable")
  157. def feed(self, next_bytes):
  158. if isinstance(next_bytes, array.array):
  159. next_bytes = next_bytes.tostring()
  160. elif isinstance(next_bytes, bytearray):
  161. next_bytes = bytes(next_bytes)
  162. assert self._fb_feeding
  163. if self._fb_buf_n + len(next_bytes) > self._max_buffer_size:
  164. raise BufferFull
  165. self._fb_buf_n += len(next_bytes)
  166. self._fb_buffers.append(next_bytes)
  167. def _fb_consume(self):
  168. self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
  169. if self._fb_buffers:
  170. self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:]
  171. self._fb_buf_o = 0
  172. self._fb_buf_i = 0
  173. self._fb_buf_n = sum(map(len, self._fb_buffers))
  174. def _fb_got_extradata(self):
  175. if self._fb_buf_i != len(self._fb_buffers):
  176. return True
  177. if self._fb_feeding:
  178. return False
  179. if not self.file_like:
  180. return False
  181. if self.file_like.read(1):
  182. return True
  183. return False
  184. def __iter__(self):
  185. return self
  186. def read_bytes(self, n):
  187. return self._fb_read(n)
  188. def _fb_rollback(self):
  189. self._fb_buf_i = 0
  190. self._fb_buf_o = 0
  191. def _fb_get_extradata(self):
  192. bufs = self._fb_buffers[self._fb_buf_i:]
  193. if bufs:
  194. bufs[0] = bufs[0][self._fb_buf_o:]
  195. return b''.join(bufs)
  196. def _fb_read(self, n, write_bytes=None):
  197. buffs = self._fb_buffers
  198. if (write_bytes is None and self._fb_buf_i < len(buffs) and
  199. self._fb_buf_o + n < len(buffs[self._fb_buf_i])):
  200. self._fb_buf_o += n
  201. return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o]
  202. ret = b''
  203. while len(ret) != n:
  204. if self._fb_buf_i == len(buffs):
  205. if self._fb_feeding:
  206. break
  207. tmp = self.file_like.read(self._read_size)
  208. if not tmp:
  209. break
  210. buffs.append(tmp)
  211. continue
  212. sliced = n - len(ret)
  213. ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced]
  214. self._fb_buf_o += sliced
  215. if self._fb_buf_o >= len(buffs[self._fb_buf_i]):
  216. self._fb_buf_o = 0
  217. self._fb_buf_i += 1
  218. if len(ret) != n:
  219. self._fb_rollback()
  220. raise OutOfData
  221. if write_bytes is not None:
  222. write_bytes(ret)
  223. return ret
  224. def _read_header(self, execute=EX_CONSTRUCT, write_bytes=None):
  225. typ = TYPE_IMMEDIATE
  226. n = 0
  227. obj = None
  228. c = self._fb_read(1, write_bytes)
  229. b = ord(c)
  230. if b & 0b10000000 == 0:
  231. obj = b
  232. elif b & 0b11100000 == 0b11100000:
  233. obj = struct.unpack("b", c)[0]
  234. elif b & 0b11100000 == 0b10100000:
  235. n = b & 0b00011111
  236. obj = self._fb_read(n, write_bytes)
  237. typ = TYPE_RAW
  238. elif b & 0b11110000 == 0b10010000:
  239. n = b & 0b00001111
  240. typ = TYPE_ARRAY
  241. elif b & 0b11110000 == 0b10000000:
  242. n = b & 0b00001111
  243. typ = TYPE_MAP
  244. elif b == 0xc0:
  245. obj = None
  246. elif b == 0xc2:
  247. obj = False
  248. elif b == 0xc3:
  249. obj = True
  250. elif b == 0xc4:
  251. typ = TYPE_BIN
  252. n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
  253. obj = self._fb_read(n, write_bytes)
  254. elif b == 0xc5:
  255. typ = TYPE_BIN
  256. n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
  257. obj = self._fb_read(n, write_bytes)
  258. elif b == 0xc6:
  259. typ = TYPE_BIN
  260. n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
  261. obj = self._fb_read(n, write_bytes)
  262. elif b == 0xc7: # ext 8
  263. typ = TYPE_EXT
  264. L, n = struct.unpack('Bb', self._fb_read(2, write_bytes))
  265. obj = self._fb_read(L, write_bytes)
  266. elif b == 0xc8: # ext 16
  267. typ = TYPE_EXT
  268. L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes))
  269. obj = self._fb_read(L, write_bytes)
  270. elif b == 0xc9: # ext 32
  271. typ = TYPE_EXT
  272. L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes))
  273. obj = self._fb_read(L, write_bytes)
  274. elif b == 0xca:
  275. obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0]
  276. elif b == 0xcb:
  277. obj = struct.unpack(">d", self._fb_read(8, write_bytes))[0]
  278. elif b == 0xcc:
  279. obj = struct.unpack("B", self._fb_read(1, write_bytes))[0]
  280. elif b == 0xcd:
  281. obj = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
  282. elif b == 0xce:
  283. obj = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
  284. elif b == 0xcf:
  285. obj = struct.unpack(">Q", self._fb_read(8, write_bytes))[0]
  286. elif b == 0xd0:
  287. obj = struct.unpack("b", self._fb_read(1, write_bytes))[0]
  288. elif b == 0xd1:
  289. obj = struct.unpack(">h", self._fb_read(2, write_bytes))[0]
  290. elif b == 0xd2:
  291. obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0]
  292. elif b == 0xd3:
  293. obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0]
  294. elif b == 0xd4: # fixext 1
  295. typ = TYPE_EXT
  296. n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes))
  297. elif b == 0xd5: # fixext 2
  298. typ = TYPE_EXT
  299. n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes))
  300. elif b == 0xd6: # fixext 4
  301. typ = TYPE_EXT
  302. n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes))
  303. elif b == 0xd7: # fixext 8
  304. typ = TYPE_EXT
  305. n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes))
  306. elif b == 0xd8: # fixext 16
  307. typ = TYPE_EXT
  308. n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes))
  309. elif b == 0xd9:
  310. typ = TYPE_RAW
  311. n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
  312. obj = self._fb_read(n, write_bytes)
  313. elif b == 0xda:
  314. typ = TYPE_RAW
  315. n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
  316. obj = self._fb_read(n, write_bytes)
  317. elif b == 0xdb:
  318. typ = TYPE_RAW
  319. n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
  320. obj = self._fb_read(n, write_bytes)
  321. elif b == 0xdc:
  322. n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
  323. typ = TYPE_ARRAY
  324. elif b == 0xdd:
  325. n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
  326. typ = TYPE_ARRAY
  327. elif b == 0xde:
  328. n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
  329. typ = TYPE_MAP
  330. elif b == 0xdf:
  331. n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
  332. typ = TYPE_MAP
  333. else:
  334. raise UnpackValueError("Unknown header: 0x%x" % b)
  335. return typ, n, obj
  336. def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
  337. typ, n, obj = self._read_header(execute, write_bytes)
  338. if execute == EX_READ_ARRAY_HEADER:
  339. if typ != TYPE_ARRAY:
  340. raise UnpackValueError("Expected array")
  341. return n
  342. if execute == EX_READ_MAP_HEADER:
  343. if typ != TYPE_MAP:
  344. raise UnpackValueError("Expected map")
  345. return n
  346. # TODO should we eliminate the recursion?
  347. if typ == TYPE_ARRAY:
  348. if execute == EX_SKIP:
  349. for i in xrange(n):
  350. # TODO check whether we need to call `list_hook`
  351. self._fb_unpack(EX_SKIP, write_bytes)
  352. return
  353. ret = newlist_hint(n)
  354. for i in xrange(n):
  355. ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes))
  356. if self._list_hook is not None:
  357. ret = self._list_hook(ret)
  358. # TODO is the interaction between `list_hook` and `use_list` ok?
  359. return ret if self._use_list else tuple(ret)
  360. if typ == TYPE_MAP:
  361. if execute == EX_SKIP:
  362. for i in xrange(n):
  363. # TODO check whether we need to call hooks
  364. self._fb_unpack(EX_SKIP, write_bytes)
  365. self._fb_unpack(EX_SKIP, write_bytes)
  366. return
  367. if self._object_pairs_hook is not None:
  368. ret = self._object_pairs_hook(
  369. (self._fb_unpack(EX_CONSTRUCT, write_bytes),
  370. self._fb_unpack(EX_CONSTRUCT, write_bytes))
  371. for _ in xrange(n))
  372. else:
  373. ret = {}
  374. for _ in xrange(n):
  375. key = self._fb_unpack(EX_CONSTRUCT, write_bytes)
  376. ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes)
  377. if self._object_hook is not None:
  378. ret = self._object_hook(ret)
  379. return ret
  380. if execute == EX_SKIP:
  381. return
  382. if typ == TYPE_RAW:
  383. if self._encoding is not None:
  384. obj = obj.decode(self._encoding, self._unicode_errors)
  385. return obj
  386. if typ == TYPE_EXT:
  387. return self._ext_hook(n, obj)
  388. if typ == TYPE_BIN:
  389. return obj
  390. assert typ == TYPE_IMMEDIATE
  391. return obj
  392. def next(self):
  393. try:
  394. ret = self._fb_unpack(EX_CONSTRUCT, None)
  395. self._fb_consume()
  396. return ret
  397. except OutOfData:
  398. raise StopIteration
  399. __next__ = next
  400. def skip(self, write_bytes=None):
  401. self._fb_unpack(EX_SKIP, write_bytes)
  402. self._fb_consume()
  403. def unpack(self, write_bytes=None):
  404. ret = self._fb_unpack(EX_CONSTRUCT, write_bytes)
  405. self._fb_consume()
  406. return ret
  407. def read_array_header(self, write_bytes=None):
  408. ret = self._fb_unpack(EX_READ_ARRAY_HEADER, write_bytes)
  409. self._fb_consume()
  410. return ret
  411. def read_map_header(self, write_bytes=None):
  412. ret = self._fb_unpack(EX_READ_MAP_HEADER, write_bytes)
  413. self._fb_consume()
  414. return ret
  415. class Packer(object):
  416. """
  417. MessagePack Packer
  418. usage:
  419. packer = Packer()
  420. astream.write(packer.pack(a))
  421. astream.write(packer.pack(b))
  422. Packer's constructor has some keyword arguments:
  423. :param callable default:
  424. Convert user type to builtin type that Packer supports.
  425. See also simplejson's document.
  426. :param str encoding:
  427. Convert unicode to bytes with this encoding. (default: 'utf-8')
  428. :param str unicode_errors:
  429. Error handler for encoding unicode. (default: 'strict')
  430. :param bool use_single_float:
  431. Use single precision float type for float. (default: False)
  432. :param bool autoreset:
  433. Reset buffer after each pack and return it's content as `bytes`. (default: True).
  434. If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
  435. :param bool use_bin_type:
  436. Use bin type introduced in msgpack spec 2.0 for bytes.
  437. It also enable str8 type for unicode.
  438. """
  439. def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
  440. use_single_float=False, autoreset=True, use_bin_type=False):
  441. self._use_float = use_single_float
  442. self._autoreset = autoreset
  443. self._use_bin_type = use_bin_type
  444. self._encoding = encoding
  445. self._unicode_errors = unicode_errors
  446. self._buffer = StringIO()
  447. if default is not None:
  448. if not callable(default):
  449. raise TypeError("default must be callable")
  450. self._default = default
  451. def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance):
  452. default_used = False
  453. while True:
  454. if nest_limit < 0:
  455. raise PackValueError("recursion limit exceeded")
  456. if obj is None:
  457. return self._buffer.write(b"\xc0")
  458. if isinstance(obj, bool):
  459. if obj:
  460. return self._buffer.write(b"\xc3")
  461. return self._buffer.write(b"\xc2")
  462. if isinstance(obj, int_types):
  463. if 0 <= obj < 0x80:
  464. return self._buffer.write(struct.pack("B", obj))
  465. if -0x20 <= obj < 0:
  466. return self._buffer.write(struct.pack("b", obj))
  467. if 0x80 <= obj <= 0xff:
  468. return self._buffer.write(struct.pack("BB", 0xcc, obj))
  469. if -0x80 <= obj < 0:
  470. return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
  471. if 0xff < obj <= 0xffff:
  472. return self._buffer.write(struct.pack(">BH", 0xcd, obj))
  473. if -0x8000 <= obj < -0x80:
  474. return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
  475. if 0xffff < obj <= 0xffffffff:
  476. return self._buffer.write(struct.pack(">BI", 0xce, obj))
  477. if -0x80000000 <= obj < -0x8000:
  478. return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
  479. if 0xffffffff < obj <= 0xffffffffffffffff:
  480. return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
  481. if -0x8000000000000000 <= obj < -0x80000000:
  482. return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
  483. raise PackValueError("Integer value out of range")
  484. if self._use_bin_type and isinstance(obj, bytes):
  485. n = len(obj)
  486. if n <= 0xff:
  487. self._buffer.write(struct.pack('>BB', 0xc4, n))
  488. elif n <= 0xffff:
  489. self._buffer.write(struct.pack(">BH", 0xc5, n))
  490. elif n <= 0xffffffff:
  491. self._buffer.write(struct.pack(">BI", 0xc6, n))
  492. else:
  493. raise PackValueError("Bytes is too large")
  494. return self._buffer.write(obj)
  495. if isinstance(obj, (Unicode, bytes)):
  496. if isinstance(obj, Unicode):
  497. if self._encoding is None:
  498. raise TypeError(
  499. "Can't encode unicode string: "
  500. "no encoding is specified")
  501. obj = obj.encode(self._encoding, self._unicode_errors)
  502. n = len(obj)
  503. if n <= 0x1f:
  504. self._buffer.write(struct.pack('B', 0xa0 + n))
  505. elif self._use_bin_type and n <= 0xff:
  506. self._buffer.write(struct.pack('>BB', 0xd9, n))
  507. elif n <= 0xffff:
  508. self._buffer.write(struct.pack(">BH", 0xda, n))
  509. elif n <= 0xffffffff:
  510. self._buffer.write(struct.pack(">BI", 0xdb, n))
  511. else:
  512. raise PackValueError("String is too large")
  513. return self._buffer.write(obj)
  514. if isinstance(obj, float):
  515. if self._use_float:
  516. return self._buffer.write(struct.pack(">Bf", 0xca, obj))
  517. return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
  518. if isinstance(obj, ExtType):
  519. code = obj.code
  520. data = obj.data
  521. assert isinstance(code, int)
  522. assert isinstance(data, bytes)
  523. L = len(data)
  524. if L == 1:
  525. self._buffer.write(b'\xd4')
  526. elif L == 2:
  527. self._buffer.write(b'\xd5')
  528. elif L == 4:
  529. self._buffer.write(b'\xd6')
  530. elif L == 8:
  531. self._buffer.write(b'\xd7')
  532. elif L == 16:
  533. self._buffer.write(b'\xd8')
  534. elif L <= 0xff:
  535. self._buffer.write(struct.pack(">BB", 0xc7, L))
  536. elif L <= 0xffff:
  537. self._buffer.write(struct.pack(">BH", 0xc8, L))
  538. else:
  539. self._buffer.write(struct.pack(">BI", 0xc9, L))
  540. self._buffer.write(struct.pack("b", code))
  541. self._buffer.write(data)
  542. return
  543. if isinstance(obj, (list, tuple)):
  544. n = len(obj)
  545. self._fb_pack_array_header(n)
  546. for i in xrange(n):
  547. self._pack(obj[i], nest_limit - 1)
  548. return
  549. if isinstance(obj, dict):
  550. return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj),
  551. nest_limit - 1)
  552. if not default_used and self._default is not None:
  553. obj = self._default(obj)
  554. default_used = 1
  555. continue
  556. raise TypeError("Cannot serialize %r" % obj)
  557. def pack(self, obj):
  558. self._pack(obj)
  559. ret = self._buffer.getvalue()
  560. if self._autoreset:
  561. self._buffer = StringIO()
  562. elif USING_STRINGBUILDER:
  563. self._buffer = StringIO(ret)
  564. return ret
  565. def pack_map_pairs(self, pairs):
  566. self._fb_pack_map_pairs(len(pairs), pairs)
  567. ret = self._buffer.getvalue()
  568. if self._autoreset:
  569. self._buffer = StringIO()
  570. elif USING_STRINGBUILDER:
  571. self._buffer = StringIO(ret)
  572. return ret
  573. def pack_array_header(self, n):
  574. if n >= 2**32:
  575. raise ValueError
  576. self._fb_pack_array_header(n)
  577. ret = self._buffer.getvalue()
  578. if self._autoreset:
  579. self._buffer = StringIO()
  580. elif USING_STRINGBUILDER:
  581. self._buffer = StringIO(ret)
  582. return ret
  583. def pack_map_header(self, n):
  584. if n >= 2**32:
  585. raise ValueError
  586. self._fb_pack_map_header(n)
  587. ret = self._buffer.getvalue()
  588. if self._autoreset:
  589. self._buffer = StringIO()
  590. elif USING_STRINGBUILDER:
  591. self._buffer = StringIO(ret)
  592. return ret
  593. def pack_ext_type(self, typecode, data):
  594. if not isinstance(typecode, int):
  595. raise TypeError("typecode must have int type.")
  596. if not 0 <= typecode <= 127:
  597. raise ValueError("typecode should be 0-127")
  598. if not isinstance(data, bytes):
  599. raise TypeError("data must have bytes type")
  600. L = len(data)
  601. if L > 0xffffffff:
  602. raise ValueError("Too large data")
  603. if L == 1:
  604. self._buffer.write(b'\xd4')
  605. elif L == 2:
  606. self._buffer.write(b'\xd5')
  607. elif L == 4:
  608. self._buffer.write(b'\xd6')
  609. elif L == 8:
  610. self._buffer.write(b'\xd7')
  611. elif L == 16:
  612. self._buffer.write(b'\xd8')
  613. elif L <= 0xff:
  614. self._buffer.write(b'\xc7' + struct.pack('B', L))
  615. elif L <= 0xffff:
  616. self._buffer.write(b'\xc8' + struct.pack('>H', L))
  617. else:
  618. self._buffer.write(b'\xc9' + struct.pack('>I', L))
  619. self._buffer.write(struct.pack('B', typecode))
  620. self._buffer.write(data)
  621. def _fb_pack_array_header(self, n):
  622. if n <= 0x0f:
  623. return self._buffer.write(struct.pack('B', 0x90 + n))
  624. if n <= 0xffff:
  625. return self._buffer.write(struct.pack(">BH", 0xdc, n))
  626. if n <= 0xffffffff:
  627. return self._buffer.write(struct.pack(">BI", 0xdd, n))
  628. raise PackValueError("Array is too large")
  629. def _fb_pack_map_header(self, n):
  630. if n <= 0x0f:
  631. return self._buffer.write(struct.pack('B', 0x80 + n))
  632. if n <= 0xffff:
  633. return self._buffer.write(struct.pack(">BH", 0xde, n))
  634. if n <= 0xffffffff:
  635. return self._buffer.write(struct.pack(">BI", 0xdf, n))
  636. raise PackValueError("Dict is too large")
  637. def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
  638. self._fb_pack_map_header(n)
  639. for (k, v) in pairs:
  640. self._pack(k, nest_limit - 1)
  641. self._pack(v, nest_limit - 1)
  642. def bytes(self):
  643. return self._buffer.getvalue()
  644. def reset(self):
  645. self._buffer = StringIO()