snappy.h 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202
  1. // Copyright 2005 and onwards Google Inc.
  2. //
  3. // Redistribution and use in source and binary forms, with or without
  4. // modification, are permitted provided that the following conditions are
  5. // met:
  6. //
  7. // * Redistributions of source code must retain the above copyright
  8. // notice, this list of conditions and the following disclaimer.
  9. // * Redistributions in binary form must reproduce the above
  10. // copyright notice, this list of conditions and the following disclaimer
  11. // in the documentation and/or other materials provided with the
  12. // distribution.
  13. // * Neither the name of Google Inc. nor the names of its
  14. // contributors may be used to endorse or promote products derived from
  15. // this software without specific prior written permission.
  16. //
  17. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28. //
  29. // A light-weight compression algorithm. It is designed for speed of
  30. // compression and decompression, rather than for the utmost in space
  31. // savings.
  32. //
  33. // For getting better compression ratios when you are compressing data
  34. // with long repeated sequences or compressing data that is similar to
  35. // other data, while still compressing fast, you might look at first
  36. // using BMDiff and then compressing the output of BMDiff with
  37. // Snappy.
  38. #ifndef THIRD_PARTY_SNAPPY_SNAPPY_H__
  39. #define THIRD_PARTY_SNAPPY_SNAPPY_H__
  40. #include <stddef.h>
  41. //#include <string>
  42. #include "snappy-stubs-public.h"
  43. namespace snappy {
  44. class Source;
  45. class Sink;
  46. // ------------------------------------------------------------------------
  47. // Generic compression/decompression routines.
  48. // ------------------------------------------------------------------------
  49. // Compress the bytes read from "*source" and append to "*sink". Return the
  50. // number of bytes written.
  51. size_t Compress(Source* source, Sink* sink);
  52. // Find the uncompressed length of the given stream, as given by the header.
  53. // Note that the true length could deviate from this; the stream could e.g.
  54. // be truncated.
  55. //
  56. // Also note that this leaves "*source" in a state that is unsuitable for
  57. // further operations, such as RawUncompress(). You will need to rewind
  58. // or recreate the source yourself before attempting any further calls.
  59. bool GetUncompressedLength(Source* source, uint32* result);
  60. // ------------------------------------------------------------------------
  61. // Higher-level string based routines (should be sufficient for most users)
  62. // ------------------------------------------------------------------------
  63. // Sets "*output" to the compressed version of "input[0,input_length-1]".
  64. // Original contents of *output are lost.
  65. //
  66. // REQUIRES: "input[]" is not an alias of "*output".
  67. //size_t Compress(const char* input, size_t input_length, string* output);
  68. // Decompresses "compressed[0,compressed_length-1]" to "*uncompressed".
  69. // Original contents of "*uncompressed" are lost.
  70. //
  71. // REQUIRES: "compressed[]" is not an alias of "*uncompressed".
  72. //
  73. // returns false if the message is corrupted and could not be decompressed
  74. //bool Uncompress(const char* compressed, size_t compressed_length, string* uncompressed);
  75. // Decompresses "compressed" to "*uncompressed".
  76. //
  77. // returns false if the message is corrupted and could not be decompressed
  78. bool Uncompress(Source* compressed, Sink* uncompressed);
  79. // This routine uncompresses as much of the "compressed" as possible
  80. // into sink. It returns the number of valid bytes added to sink
  81. // (extra invalid bytes may have been added due to errors; the caller
  82. // should ignore those). The emitted data typically has length
  83. // GetUncompressedLength(), but may be shorter if an error is
  84. // encountered.
  85. size_t UncompressAsMuchAsPossible(Source* compressed, Sink* uncompressed);
  86. // ------------------------------------------------------------------------
  87. // Lower-level character array based routines. May be useful for
  88. // efficiency reasons in certain circumstances.
  89. // ------------------------------------------------------------------------
  90. // REQUIRES: "compressed" must point to an area of memory that is at
  91. // least "MaxCompressedLength(input_length)" bytes in length.
  92. //
  93. // Takes the data stored in "input[0..input_length]" and stores
  94. // it in the array pointed to by "compressed".
  95. //
  96. // "*compressed_length" is set to the length of the compressed output.
  97. //
  98. // Example:
  99. // char* output = new char[snappy::MaxCompressedLength(input_length)];
  100. // size_t output_length;
  101. // RawCompress(input, input_length, output, &output_length);
  102. // ... Process(output, output_length) ...
  103. // delete [] output;
  104. void RawCompress(const char* input,
  105. size_t input_length,
  106. char* compressed,
  107. size_t* compressed_length);
  108. // Given data in "compressed[0..compressed_length-1]" generated by
  109. // calling the Snappy::Compress routine, this routine
  110. // stores the uncompressed data to
  111. // uncompressed[0..GetUncompressedLength(compressed)-1]
  112. // returns false if the message is corrupted and could not be decrypted
  113. bool RawUncompress(const char* compressed, size_t compressed_length,
  114. char* uncompressed);
  115. // Given data from the byte source 'compressed' generated by calling
  116. // the Snappy::Compress routine, this routine stores the uncompressed
  117. // data to
  118. // uncompressed[0..GetUncompressedLength(compressed,compressed_length)-1]
  119. // returns false if the message is corrupted and could not be decrypted
  120. bool RawUncompress(Source* compressed, char* uncompressed);
  121. // Given data in "compressed[0..compressed_length-1]" generated by
  122. // calling the Snappy::Compress routine, this routine
  123. // stores the uncompressed data to the iovec "iov". The number of physical
  124. // buffers in "iov" is given by iov_cnt and their cumulative size
  125. // must be at least GetUncompressedLength(compressed). The individual buffers
  126. // in "iov" must not overlap with each other.
  127. //
  128. // returns false if the message is corrupted and could not be decrypted
  129. bool RawUncompressToIOVec(const char* compressed, size_t compressed_length,
  130. const struct iovec* iov, size_t iov_cnt);
  131. // Given data from the byte source 'compressed' generated by calling
  132. // the Snappy::Compress routine, this routine stores the uncompressed
  133. // data to the iovec "iov". The number of physical
  134. // buffers in "iov" is given by iov_cnt and their cumulative size
  135. // must be at least GetUncompressedLength(compressed). The individual buffers
  136. // in "iov" must not overlap with each other.
  137. //
  138. // returns false if the message is corrupted and could not be decrypted
  139. bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov,
  140. size_t iov_cnt);
  141. // Returns the maximal size of the compressed representation of
  142. // input data that is "source_bytes" bytes in length;
  143. size_t MaxCompressedLength(size_t source_bytes);
  144. // REQUIRES: "compressed[]" was produced by RawCompress() or Compress()
  145. // Returns true and stores the length of the uncompressed data in
  146. // *result normally. Returns false on parsing error.
  147. // This operation takes O(1) time.
  148. bool GetUncompressedLength(const char* compressed, size_t compressed_length,
  149. size_t* result);
  150. // Returns true iff the contents of "compressed[]" can be uncompressed
  151. // successfully. Does not return the uncompressed data. Takes
  152. // time proportional to compressed_length, but is usually at least
  153. // a factor of four faster than actual decompression.
  154. bool IsValidCompressedBuffer(const char* compressed,
  155. size_t compressed_length);
  156. // Returns true iff the contents of "compressed" can be uncompressed
  157. // successfully. Does not return the uncompressed data. Takes
  158. // time proportional to *compressed length, but is usually at least
  159. // a factor of four faster than actual decompression.
  160. // On success, consumes all of *compressed. On failure, consumes an
  161. // unspecified prefix of *compressed.
  162. bool IsValidCompressed(Source* compressed);
  163. // The size of a compression block. Note that many parts of the compression
  164. // code assumes that kBlockSize <= 65536; in particular, the hash table
  165. // can only store 16-bit offsets, and EmitCopy() also assumes the offset
  166. // is 65535 bytes or less. Note also that if you change this, it will
  167. // affect the framing format (see framing_format.txt).
  168. //
  169. // Note that there might be older data around that is compressed with larger
  170. // block sizes, so the decompression code should not rely on the
  171. // non-existence of long backreferences.
  172. static const int kBlockLog = 16;
  173. static const size_t kBlockSize = 1 << kBlockLog;
  174. static const int kMaxHashTableBits = 14;
  175. static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
  176. } // end namespace snappy
  177. #endif // THIRD_PARTY_SNAPPY_SNAPPY_H__