dovecot-2.2: lib-compression: Added support for liblzma (xz)
dovecot at dovecot.org
dovecot at dovecot.org
Sun Nov 24 23:02:51 EET 2013
details: http://hg.dovecot.org/dovecot-2.2/rev/81e486aacbc7
changeset: 17028:81e486aacbc7
user: Timo Sirainen <tss at iki.fi>
date: Sun Nov 24 23:02:13 2013 +0200
description:
lib-compression: Added support for liblzma (xz)
Annoyingly this is mainly copy&pasted [io]stream-bzlib, but I'm not sure if
it's worth the effort to try to create common functions for them.
diffstat:
configure.ac | 24 ++
src/lib-compression/Makefile.am | 2 +
src/lib-compression/compression.c | 16 +
src/lib-compression/istream-lzma.c | 342 +++++++++++++++++++++++++++++++++++++
src/lib-compression/istream-zlib.h | 1 +
src/lib-compression/ostream-lzma.c | 229 ++++++++++++++++++++++++
src/lib-compression/ostream-zlib.h | 1 +
7 files changed, 615 insertions(+), 0 deletions(-)
diffs (truncated from 693 to 300 lines):
diff -r ace36e525e1f -r 81e486aacbc7 configure.ac
--- a/configure.ac Sun Nov 24 20:19:48 2013 +0000
+++ b/configure.ac Sun Nov 24 23:02:13 2013 +0200
@@ -179,6 +179,11 @@
TEST_WITH(bzlib, $withval),
want_bzlib=auto)
+AC_ARG_WITH(lzma,
+AS_HELP_STRING([--with-lzma], [Build with LZMA compression support]),
+ TEST_WITH(lzma, $withval),
+ want_lzma=auto)
+
AC_ARG_WITH(libcap,
AS_HELP_STRING([--with-libcap], [Build with libcap support (Dropping capabilities).]),
TEST_WITH(libcap, $withval),
@@ -2657,6 +2662,25 @@
fi
])
fi
+
+if test "$want_lzma" != "no"; then
+ AC_CHECK_HEADER(lzma.h, [
+ AC_CHECK_LIB(lzma, lzma_stream_decoder, [
+ have_lzma=yes
+ have_compress_lib=yes
+ AC_DEFINE(HAVE_LZMA,, Define if you have lzma library)
+ COMPRESS_LIBS="$COMPRESS_LIBS -llzma"
+ ], [
+ if test "$want_lzma" = "yes"; then
+ AC_ERROR([Can't build with lzma support: liblzma not found])
+ fi
+ ])
+ ], [
+ if test "$want_lzma" = "yes"; then
+ AC_ERROR([Can't build with lzma support: lzma.h not found])
+ fi
+ ])
+fi
AC_SUBST(COMPRESS_LIBS)
AM_CONDITIONAL(BUILD_ZLIB_PLUGIN, test "$have_compress_lib" = "yes")
diff -r ace36e525e1f -r 81e486aacbc7 src/lib-compression/Makefile.am
--- a/src/lib-compression/Makefile.am Sun Nov 24 20:19:48 2013 +0000
+++ b/src/lib-compression/Makefile.am Sun Nov 24 23:02:13 2013 +0200
@@ -6,8 +6,10 @@
libcompression_la_SOURCES = \
compression.c \
+ istream-lzma.c \
istream-zlib.c \
istream-bzlib.c \
+ ostream-lzma.c \
ostream-zlib.c \
ostream-bzlib.c
libcompression_la_LIBADD = \
diff -r ace36e525e1f -r 81e486aacbc7 src/lib-compression/compression.c
--- a/src/lib-compression/compression.c Sun Nov 24 20:19:48 2013 +0000
+++ b/src/lib-compression/compression.c Sun Nov 24 23:02:13 2013 +0200
@@ -16,6 +16,10 @@
# define i_stream_create_bz2 NULL
# define o_stream_create_bz2 NULL
#endif
+#ifndef HAVE_LZMA
+# define i_stream_create_lzma NULL
+# define o_stream_create_lzma NULL
+#endif
static bool is_compressed_zlib(struct istream *input)
{
@@ -49,6 +53,16 @@
return memcmp(data + 4, "\x31\x41\x59\x26\x53\x59", 6) == 0;
}
+static bool is_compressed_xz(struct istream *input)
+{
+ const unsigned char *data;
+ size_t size;
+
+ if (i_stream_read_data(input, &data, &size, 6 - 1) <= 0)
+ return FALSE;
+ return memcmp(data, "\xfd\x37\x7a\x58\x5a", 6) == 0;
+}
+
const struct compression_handler *compression_lookup_handler(const char *name)
{
unsigned int i;
@@ -97,5 +111,7 @@
i_stream_create_bz2, o_stream_create_bz2 },
{ "deflate", NULL, NULL,
i_stream_create_deflate, o_stream_create_deflate },
+ { "xz", ".xz", is_compressed_xz,
+ i_stream_create_lzma, o_stream_create_lzma },
{ NULL, NULL, NULL, NULL, NULL }
};
diff -r ace36e525e1f -r 81e486aacbc7 src/lib-compression/istream-lzma.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-compression/istream-lzma.c Sun Nov 24 23:02:13 2013 +0200
@@ -0,0 +1,342 @@
+/* Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+
+#ifdef HAVE_LZMA
+
+#include "istream-private.h"
+#include "istream-zlib.h"
+#include <lzma.h>
+
+#define CHUNK_SIZE (1024*64)
+
+#define LZMA_MEMORY_LIMIT (1024*1024*80)
+
+struct lzma_istream {
+ struct istream_private istream;
+
+ lzma_stream strm;
+ uoff_t eof_offset, stream_size;
+ size_t high_pos;
+ struct stat last_parent_statbuf;
+
+ unsigned int log_errors:1;
+ unsigned int marked:1;
+ unsigned int strm_closed:1;
+};
+
+static void i_stream_lzma_close(struct iostream_private *stream,
+ bool close_parent)
+{
+ struct lzma_istream *zstream = (struct lzma_istream *)stream;
+
+ if (!zstream->strm_closed) {
+ lzma_end(&zstream->strm);
+ zstream->strm_closed = TRUE;
+ }
+ if (close_parent)
+ i_stream_close(zstream->istream.parent);
+}
+
+static void lzma_read_error(struct lzma_istream *zstream, const char *error)
+{
+ io_stream_set_error(&zstream->istream.iostream,
+ "lzma.read(%s): %s at %"PRIuUOFF_T,
+ i_stream_get_name(&zstream->istream.istream), error,
+ zstream->istream.abs_start_offset +
+ zstream->istream.istream.v_offset);
+ if (zstream->log_errors)
+ i_error("%s", zstream->istream.iostream.error);
+}
+
+static ssize_t i_stream_lzma_read(struct istream_private *stream)
+{
+ struct lzma_istream *zstream = (struct lzma_istream *)stream;
+ const unsigned char *data;
+ uoff_t high_offset;
+ size_t size, out_size;
+ lzma_ret ret;
+
+ high_offset = stream->istream.v_offset + (stream->pos - stream->skip);
+ if (zstream->eof_offset == high_offset) {
+ i_assert(zstream->high_pos == 0 ||
+ zstream->high_pos == stream->pos);
+ stream->istream.eof = TRUE;
+ return -1;
+ }
+
+ if (stream->pos < zstream->high_pos) {
+ /* we're here because we seeked back within the read buffer. */
+ ret = zstream->high_pos - stream->pos;
+ stream->pos = zstream->high_pos;
+ zstream->high_pos = 0;
+
+ if (zstream->eof_offset != (uoff_t)-1) {
+ high_offset = stream->istream.v_offset +
+ (stream->pos - stream->skip);
+ i_assert(zstream->eof_offset == high_offset);
+ stream->istream.eof = TRUE;
+ }
+ return ret;
+ }
+ zstream->high_pos = 0;
+
+ if (stream->pos + CHUNK_SIZE > stream->buffer_size) {
+ /* try to keep at least CHUNK_SIZE available */
+ if (!zstream->marked && stream->skip > 0) {
+ /* don't try to keep anything cached if we don't
+ have a seek mark. */
+ i_stream_compress(stream);
+ }
+ if (stream->max_buffer_size == 0 ||
+ stream->buffer_size < stream->max_buffer_size)
+ i_stream_grow_buffer(stream, CHUNK_SIZE);
+
+ if (stream->pos == stream->buffer_size) {
+ if (stream->skip > 0) {
+ /* lose our buffer cache */
+ i_stream_compress(stream);
+ }
+
+ if (stream->pos == stream->buffer_size)
+ return -2; /* buffer full */
+ }
+ }
+
+ if (i_stream_read_data(stream->parent, &data, &size, 0) < 0) {
+ if (stream->parent->stream_errno != 0) {
+ stream->istream.stream_errno =
+ stream->parent->stream_errno;
+ } else {
+ i_assert(stream->parent->eof);
+ lzma_read_error(zstream, "unexpected EOF");
+ stream->istream.stream_errno = EINVAL;
+ }
+ return -1;
+ }
+ if (size == 0) {
+ /* no more input */
+ i_assert(!stream->istream.blocking);
+ return 0;
+ }
+
+ zstream->strm.next_in = data;
+ zstream->strm.avail_in = size;
+
+ out_size = stream->buffer_size - stream->pos;
+ zstream->strm.next_out = stream->w_buffer + stream->pos;
+ zstream->strm.avail_out = out_size;
+ ret = lzma_code(&zstream->strm, LZMA_RUN);
+
+ out_size -= zstream->strm.avail_out;
+ stream->pos += out_size;
+
+ i_stream_skip(stream->parent, size - zstream->strm.avail_in);
+
+ switch (ret) {
+ case LZMA_OK:
+ break;
+ case LZMA_DATA_ERROR:
+ case LZMA_BUF_ERROR:
+ lzma_read_error(zstream, "corrupted data");
+ stream->istream.stream_errno = EINVAL;
+ return -1;
+ case LZMA_FORMAT_ERROR:
+ lzma_read_error(zstream, "wrong magic in header (not xz file?)");
+ stream->istream.stream_errno = EINVAL;
+ return -1;
+ case LZMA_OPTIONS_ERROR:
+ lzma_read_error(zstream, "Unsupported xz options");
+ stream->istream.stream_errno = EINVAL;
+ return -1;
+ case LZMA_MEM_ERROR:
+ i_fatal_status(FATAL_OUTOFMEM, "lzma.read(%s): Out of memory",
+ i_stream_get_name(&stream->istream));
+ case LZMA_STREAM_END:
+ zstream->eof_offset = stream->istream.v_offset +
+ (stream->pos - stream->skip);
+ zstream->stream_size = zstream->eof_offset;
+ if (out_size == 0) {
+ stream->istream.eof = TRUE;
+ return -1;
+ }
+ break;
+ default:
+ lzma_read_error(zstream, t_strdup_printf(
+ "lzma_code() failed with %d", ret));
+ stream->istream.stream_errno = EINVAL;
+ return -1;
+ }
+ if (out_size == 0) {
+ /* read more input */
+ return i_stream_lzma_read(stream);
+ }
+ return out_size;
+}
+
+static void i_stream_lzma_init(struct lzma_istream *zstream)
+{
+ lzma_ret ret;
+
+ ret = lzma_stream_decoder(&zstream->strm, LZMA_MEMORY_LIMIT,
+ LZMA_CONCATENATED);
+ switch (ret) {
+ case LZMA_OK:
+ break;
+ case LZMA_MEM_ERROR:
+ i_fatal_status(FATAL_OUTOFMEM, "lzma: Out of memory");
+ default:
+ i_fatal("lzma_stream_decoder() failed with ret=%d", ret);
+ }
+}
+
+static void i_stream_lzma_reset(struct lzma_istream *zstream)
+{
+ struct istream_private *stream = &zstream->istream;
+
+ i_stream_seek(stream->parent, stream->parent_start_offset);
+ zstream->eof_offset = (uoff_t)-1;
+ zstream->strm.next_in = NULL;
+ zstream->strm.avail_in = 0;
+
+ stream->parent_expected_offset = stream->parent_start_offset;
More information about the dovecot-cvs
mailing list