[tor-commits] [stegotorus/master] Simplify zpack.cc by not rebuffering and by using zlib's built-in gzip-format support.

20 Jul 2012

commit 1e6bd79436b04dcc4dfc56b4ca393a93a487b203
Author: Zack Weinberg zackw@cmu.edu
Date:   Mon Apr 30 14:58:15 2012 -0700
Simplify zpack.cc by not rebuffering and by using zlib's built-in gzip-format support.
---
 src/zpack.cc |  360 ++++++++++++++++------------------------------------------
 1 files changed, 100 insertions(+), 260 deletions(-)

diff --git a/src/zpack.cc b/src/zpack.cc
index c851f12..3ffd065 100644
--- a/src/zpack.cc
+++ b/src/zpack.cc
@@ -7,319 +7,159 @@
#include <limits>
-#define CHUNK 16384
-
-/* Compress from file source to file dest until EOF on source.
-   def() returns Z_OK on success, Z_MEM_ERROR if memory could not be
-   allocated for processing, Z_STREAM_ERROR if an invalid compression
-   level is supplied, Z_VERSION_ERROR if the version of zlib.h and the
-   version of the library linked do not match, or Z_ERRNO if there is
-   an error reading or writing the files. */
-
+// zlib doesn't believe in size_t. When size_t is bigger than uInt, we
+// theoretically could break operations up into uInt-sized chunks to
+// support the full range of size_t, but I doubt we will ever need to
+// compress, decompress, or crc32 more than 2^32 bytes in one
+// operation, so I'm not bothering.  -- zw, 2012
+//
+// The indirection through ZLIB_UINT_MAX makes some versions of gcc
+// not produce a 'comparison is always (true/false)' warning.
+const size_t ZLIB_UINT_MAX = std::numeric_limits<uInt>::max();
+const size_t ZLIB_CEILING = (SIZE_T_CEILING > ZLIB_UINT_MAX
+                             ? ZLIB_UINT_MAX : SIZE_T_CEILING);
+
+// Compress from 'source' to 'dest', producing 'zlib' (RFC 1950) format,
+// with compression level 'level'.
 ssize_t
 def(const uint8_t *source, size_t slen, uint8_t *dest, size_t dlen, int level)
 {
-  int ret, flush;
-  size_t have;
-  z_stream strm;
-  uint8_t in[CHUNK];
-  uint8_t out[CHUNK];
-  size_t dlen_orig = dlen;
-
-  if (slen > SIZE_T_CEILING || dlen > SIZE_T_CEILING)
+  if (slen > ZLIB_CEILING || dlen > ZLIB_CEILING)
     return -1;
-  /* allocate deflate state */
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
-  ret = deflateInit(&strm, level);
-  if (ret != Z_OK)
-    return ret;
-
-  /* compress until end of file */
-  do {
-    if (slen > CHUNK)
-      strm.avail_in = CHUNK;
-    else
-      strm.avail_in = slen;
-
-    memcpy (in, source, strm.avail_in);
-    slen = slen - strm.avail_in;
-    source = source + strm.avail_in;
-
-    flush = (slen == 0) ? Z_FINISH : Z_NO_FLUSH;
-    strm.next_in = in;
-
-    /* run deflate() on input until output buffer not full, finish
-       compression if all of source has been read in */
-    do {
-      strm.avail_out = CHUNK;
-      strm.next_out = out;
-      ret = deflate(&strm, flush);    /* no bad return value */
-      log_assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
-      have = CHUNK - strm.avail_out;
-
-      if (dlen < have) {
-        log_warn("dest buf too small - have %lu, need %lu",
-                 (unsigned long)dlen, (unsigned long)have);
-        return Z_ERRNO;
-      }
+  z_stream strm;
+  memset(&strm, 0, sizeof strm);
+  int ret = deflateInit(&strm, level);
+  if (ret != Z_OK) {
+    log_warn("compression failure (initialization): %s", strm.msg);
+    return -1;
+  }
-      memcpy(dest, out, have);
-      dest += have;
-      dlen = dlen - have;
-    } while (strm.avail_out == 0);
-    log_assert(strm.avail_in == 0);     /* all input will be used */
+  strm.next_in = const_cast<Bytef*>(source);
+  strm.avail_in = slen;
+  strm.next_out = dest;
+  strm.avail_out = dlen;
-    /* done when last data in file processed */
-  } while (flush != Z_FINISH);
-  log_assert(ret == Z_STREAM_END);        /* stream will be complete */
+  ret = deflate(&strm, Z_FINISH);
+  if (ret != Z_STREAM_END) {
+    log_warn("compression failure: %s", strm.msg);
+    deflateEnd(&strm);
+    return -1;
+  }
-  /* clean up and return */
   deflateEnd(&strm);
-  return (dlen_orig - dlen);
+  return strm.total_out;
 }
-/* Decompress from file source to file dest until stream ends or EOF.
-   inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be
-   allocated for processing, Z_DATA_ERROR if the deflate data is
-   invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and
-   the version of the library linked do not match, or Z_ERRNO if there
-   is an error reading or writing the files. */
-
+// Decompress 'zlib'-format data from 'source' to 'dest'.
 ssize_t
 inf(const uint8_t *source, size_t slen, uint8_t *dest, size_t dlen)
 {
-  int ret;
-  size_t have;
-  z_stream strm;
-  uint8_t in[CHUNK];
-  uint8_t out[CHUNK];
-  size_t dlen_orig = dlen;
-
-  if (slen > SIZE_T_CEILING || dlen > SIZE_T_CEILING)
+  if (slen > ZLIB_CEILING || dlen > ZLIB_CEILING)
     return -1;
/* allocate inflate state */
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
-  strm.avail_in = 0;
-  strm.next_in = Z_NULL;
-  ret = inflateInit(&strm);
-  if (ret != Z_OK)
-    return ret;
-
-  /* decompress until deflate stream ends or end of file */
-  do {
-    if (slen == 0)
-      break;
-
-    if (slen > CHUNK)
-      strm.avail_in = CHUNK;
-    else
-      strm.avail_in = slen;
-
-    memcpy(in, source, strm.avail_in);
-    slen = slen - strm.avail_in;
-    source = source + strm.avail_in;
-    strm.next_in = in;
-
-    /* run inflate() on input until output buffer not full */
-    do {
-      strm.avail_out = CHUNK;
-      strm.next_out = out;
-      ret = inflate(&strm, Z_NO_FLUSH);
-      log_assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
-      switch (ret) {
-      case Z_NEED_DICT:
-      case Z_DATA_ERROR:
-      case Z_MEM_ERROR:
-        inflateEnd(&strm);
-        return ret;
-      }
-      have = CHUNK - strm.avail_out;
-
-      if (dlen < have) {
-        log_warn("dest buf too small - have %lu, need %lu",
-                 (unsigned long)dlen, (unsigned long)have);
-        return Z_ERRNO;
-      }
-
-      memcpy(dest, out, have);
-      dest += have;
-      dlen = dlen - have;
+  z_stream strm;
+  memset(&strm, 0, sizeof strm);
+  int ret = inflateInit(&strm);
+  if (ret != Z_OK) {
+    log_warn("decompression failure (initialization): %s", strm.msg);
+    return -1;
+  }
-    } while (strm.avail_out == 0);
+  strm.next_in = const_cast<Bytef*>(source);
+  strm.avail_in = slen;
+  strm.next_out = dest;
+  strm.avail_out = dlen;
-    /* done when inflate() says it's done */
-  } while (ret != Z_STREAM_END);
+  ret = inflate(&strm, Z_FINISH);
+  if (ret != Z_STREAM_END) {
+    log_warn("decompression failure: %s", strm.msg);
+    inflateEnd(&strm);
+    return -1;
+  }
-  /* clean up and return */
   inflateEnd(&strm);
-
-  if (ret == Z_STREAM_END)
-    return dlen_orig - dlen;
-  return Z_DATA_ERROR;
+  return strm.total_out;
 }
-/* assumes that we know there is exactly 10 bytes of gzip header */
-
 ssize_t
 gzInflate(const uint8_t *source, size_t slen, uint8_t *dest, size_t dlen)
 {
-  int ret;
-  size_t have;
-  z_stream strm;
-  uint8_t in[CHUNK];
-  uint8_t out[CHUNK];
-  size_t dlen_orig = dlen;
-
-  if (slen > SIZE_T_CEILING || dlen > SIZE_T_CEILING)
+  if (slen > ZLIB_CEILING || dlen > ZLIB_CEILING)
     return -1;
-  /* allocate inflate state */
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
-  strm.avail_in = 0;
-  strm.next_in = Z_NULL;
-
-  ret = inflateInit2(&strm, -MAX_WBITS);
-  if (ret != Z_OK)
-    return ret;
-
-  source = source + 10;
-  slen -= 10;
-
-  /* decompress until deflate stream ends or end of file */
-  do {
-    if (slen == 0)
-      break;
-
-    if (slen > CHUNK)
-      strm.avail_in = CHUNK;
-    else
-      strm.avail_in = slen;
-
-    memcpy(in, source, strm.avail_in);
-    slen = slen - strm.avail_in;
-    source = source + strm.avail_in;
-    strm.next_in = in;
-
-    /* run inflate() on input until output buffer not full */
-    do {
-      strm.avail_out = CHUNK;
-      strm.next_out = out;
-      ret = inflate(&strm, Z_NO_FLUSH);
-      log_assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
-      switch (ret) {
-      case Z_NEED_DICT:
-        ret = Z_DATA_ERROR;     /* and fall through */
-      case Z_DATA_ERROR:
-      case Z_MEM_ERROR:
-        inflateEnd(&strm);
-        return ret;
-      }
-      have = CHUNK - strm.avail_out;
-
-      if (dlen < have) {
-        log_warn("dest buf too small - have %lu, need %lu",
-                 (unsigned long)dlen, (unsigned long)have);
-        return Z_ERRNO;
-      }
+  z_stream strm;
+  memset(&strm, 0, sizeof strm);
+  int ret = inflateInit2(&strm, MAX_WBITS|16); // 16 = decode gzip (only)
-      memcpy(dest, out, have);
-      dest += have;
-      dlen = dlen - have;
+  if (ret != Z_OK) {
+    log_warn("decompression failure (initialization): %s", strm.msg);
+    return -1;
+  }
-    } while (strm.avail_out == 0);
+  strm.next_in = const_cast<Bytef*>(source);
+  strm.avail_in = slen;
+  strm.next_out = dest;
+  strm.avail_out = dlen;
-    /* done when inflate() says it's done */
-  } while (ret != Z_STREAM_END);
+  ret = inflate(&strm, Z_FINISH);
+  if (ret != Z_STREAM_END) {
+    log_warn("decompression failure: %s", strm.msg);
+    inflateEnd(&strm);
+    return -1;
+  }
-  /* clean up and return */
   inflateEnd(&strm);
-
-  if (ret == Z_STREAM_END)
-    return dlen_orig - dlen;
-  return Z_DATA_ERROR;
+  return strm.total_out;
 }
ssize_t
 gzDeflate(const uint8_t *source, size_t slen, uint8_t *dest, size_t dlen,
           time_t mtime)
 {
-  uint32_t crc;
-  z_stream z;
-
-  if (slen > SIZE_T_CEILING || dlen > SIZE_T_CEILING)
+  if (slen > ZLIB_CEILING || dlen > ZLIB_CEILING)
     return -1;
-  z.zalloc = Z_NULL;
-  z.zfree = Z_NULL;
-  z.opaque = Z_NULL;
-
-  if (Z_OK != deflateInit2(&z,
-                           Z_DEFAULT_COMPRESSION,
-                           Z_DEFLATED,
-                           -MAX_WBITS,  /* supress zlib-header */
-                           8,
-                           Z_DEFAULT_STRATEGY))
+  z_stream strm;
+  memset(&strm, 0, sizeof strm);
+  int ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
+                         MAX_WBITS|16, // compress as gzip
+                         8, Z_DEFAULT_STRATEGY);
+  if (ret != Z_OK) {
+    log_warn("compression failure (initialization): %s", strm.msg);
     return -1;
+  }
-  z.next_in = const_cast<uint8_t*>(source);
-  z.avail_in = slen;
-  z.total_in = 0;
-
-  /* write gzip header */
-
-  dest[0] = 0x1f;
-  dest[1] = 0x8b;
-  dest[2] = Z_DEFLATED;
-  dest[3] = 0; /* options */
-  dest[4] = (mtime >>  0) & 0xff;
-  dest[5] = (mtime >>  8) & 0xff;
-  dest[6] = (mtime >> 16) & 0xff;
-  dest[7] = (mtime >> 24) & 0xff;
-  dest[8] = 0x00; /* extra flags */
-  dest[9] = 0xFF; /* unknown */
-
-  z.next_out = dest + 10;
-  z.avail_out = dlen - 10 - 8;
-  z.total_out = 0;
-
-  if (deflate(&z, Z_FINISH) != Z_STREAM_END) {
-    deflateEnd(&z);
+  gz_header gzh;
+  memset(&gzh, 0, sizeof gzh);
+  gzh.time = mtime;
+  gzh.os = 0xFF; // "unknown"
+  ret = deflateSetHeader(&strm, &gzh);
+  if (ret != Z_OK) {
+    log_warn("compression failure (initialization): %s", strm.msg);
     return -1;
   }
-  crc = generate_crc32c(source, slen);
-
-  dest = dest + 10 + z.total_out;
-  dest[0] = (crc >>  0) & 0xff;
-  dest[1] = (crc >>  8) & 0xff;
-  dest[2] = (crc >> 16) & 0xff;
-  dest[3] = (crc >> 24) & 0xff;
-  dest[4] = (z.total_in >>  0) & 0xff;
-  dest[5] = (z.total_in >>  8) & 0xff;
-  dest[6] = (z.total_in >> 16) & 0xff;
-  dest[7] = (z.total_in >> 24) & 0xff;
+  strm.next_in = const_cast<Bytef*>(source);
+  strm.avail_in = slen;
+  strm.next_out = dest;
+  strm.avail_out = dlen;
-  if (deflateEnd(&z) != Z_OK)
+  ret = deflate(&strm, Z_FINISH);
+  if (ret != Z_STREAM_END) {
+    log_warn("compression failure: %s", strm.msg);
+    deflateEnd(&strm);
     return -1;
-  return 10 + z.total_out + 8;
+  }
+
+  deflateEnd(&strm);
+  return strm.total_out;
 }
uint32_t
 generate_crc32c(const uint8_t *string, size_t length)
 {
-  // zlib doesn't believe in size_t. When size_t is bigger than uInt,
-  // we theoretically could break the operation up into uInt-sized
-  // chunks to support the full range of size_t, but I doubt we will
-  // ever need to crc32 more than 2^32 bytes, so I'm not bothering.
-  // -- zw, 2012
   log_assert(length <= std::numeric_limits<uInt>::max());
uLong crc = crc32(crc32(0, 0, 0), string, length);

    

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

[tor-commits] [stegotorus/master] Simplify zpack.cc by not rebuffering and by using zlib's built-in gzip-format support.