[llvm-commits] [parallel] CVS: llvm/runtime/zlib/contrib/gzappend/gzappend.c
Misha Brukman
brukman at cs.uiuc.edu
Mon Mar 1 18:12:10 PST 2004
Changes in directory llvm/runtime/zlib/contrib/gzappend:
gzappend.c added (r1.1.2.1)
---
Log message:
Merge from trunk
---
Diffs of the changes: (+500 -0)
Index: llvm/runtime/zlib/contrib/gzappend/gzappend.c
diff -c /dev/null llvm/runtime/zlib/contrib/gzappend/gzappend.c:1.1.2.1
*** /dev/null Mon Mar 1 17:58:58 2004
--- llvm/runtime/zlib/contrib/gzappend/gzappend.c Mon Mar 1 17:58:48 2004
***************
*** 0 ****
--- 1,500 ----
+ /* gzappend -- command to append to a gzip file
+
+ Copyright (C) 2003 Mark Adler, all rights reserved
+ version 1.1, 4 Nov 2003
+
+ This software is provided 'as-is', without any express or implied
+ warranty. In no event will the author be held liable for any damages
+ arising from the use of this software.
+
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely, subject to the following restrictions:
+
+ 1. The origin of this software must not be misrepresented; you must not
+ claim that you wrote the original software. If you use this software
+ in a product, an acknowledgment in the product documentation would be
+ appreciated but is not required.
+ 2. Altered source versions must be plainly marked as such, and must not be
+ misrepresented as being the original software.
+ 3. This notice may not be removed or altered from any source distribution.
+
+ Mark Adler madler at alumni.caltech.edu
+ */
+
+ /*
+ * Change history:
+ *
+ * 1.0 19 Oct 2003 - First version
+ * 1.1 4 Nov 2003 - Expand and clarify some comments and notes
+ * - Add version and copyright to help
+ * - Send help to stdout instead of stderr
+ * - Add some preemptive typecasts
+ * - Add L to constants in lseek() calls
+ * - Remove some debugging information in error messages
+ * - Use new data_type definition for zlib 1.2.1
+ * - Simplfy and unify file operations
+ * - Finish off gzip file in gztack()
+ * - Use deflatePrime() instead of adding empty blocks
+ * - Keep gzip file clean on appended file read errors
+ * - Use in-place rotate instead of auxiliary buffer
+ * (Why you ask? Because it was fun to write!)
+ */
+
+ /*
+ gzappend takes a gzip file and appends to it, compressing files from the
+ command line or data from stdin. The gzip file is written to directly, to
+ avoid copying that file, in case it's large. Note that this results in the
+ unfriendly behavior that if gzappend fails, the gzip file is corrupted.
+
+ This program was written to illustrate the use of the new Z_BLOCK option of
+ zlib 1.2.1's inflate() function. This option returns from inflate() at each
+ block boundary to facilitate locating and modifying the last block bit at
+ the start of the final deflate block. Also whether using Z_BLOCK or not,
+ another required feature of zlib 1.2.1 is that inflate() now provides the
+ number of unusued bits in the last input byte used. gzappend will not work
+ with versions of zlib earlier than 1.2.1.
+
+ gzappend first decompresses the gzip file internally, discarding all but
+ the last 32K of uncompressed data, and noting the location of the last block
+ bit and the number of unused bits in the last byte of the compressed data.
+ The gzip trailer containing the CRC-32 and length of the uncompressed data
+ is verified. This trailer will be later overwritten.
+
+ Then the last block bit is cleared by seeking back in the file and rewriting
+ the byte that contains it. Seeking forward, the last byte of the compressed
+ data is saved along with the number of unused bits to initialize deflate.
+
+ A deflate process is initialized, using the last 32K of the uncompressed
+ data from the gzip file to initialize the dictionary. If the total
+ uncompressed data was less than 32K, then all of it is used to initialize
+ the dictionary. The deflate output bit buffer is also initialized with the
+ last bits from the original deflate stream. From here on, the data to
+ append is simply compressed using deflate, and written to the gzip file.
+ When that is complete, the new CRC-32 and uncompressed length are written
+ as the trailer of the gzip file.
+ */
+
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+ #include <fcntl.h>
+ #include <unistd.h>
+ #include "zlib.h"
+
+ #define local static
+ #define LGCHUNK 14
+ #define CHUNK (1U << LGCHUNK)
+ #define DSIZE 32768U
+
+ /* print an error message and terminate with extreme prejudice */
+ local void bye(char *msg1, char *msg2)
+ {
+ fprintf(stderr, "gzappend error: %s%s\n", msg1, msg2);
+ exit(1);
+ }
+
+ /* return the greatest common divisor of a and b using Euclid's algorithm,
+ modified to be fast when one argument much greater than the other, and
+ coded to avoid unnecessary swapping */
+ local unsigned gcd(unsigned a, unsigned b)
+ {
+ unsigned c;
+
+ while (a && b)
+ if (a > b) {
+ c = b;
+ while (a - c >= c)
+ c <<= 1;
+ a -= c;
+ }
+ else {
+ c = a;
+ while (b - c >= c)
+ c <<= 1;
+ b -= c;
+ }
+ return a + b;
+ }
+
+ /* rotate list[0..len-1] left by rot positions, in place */
+ local void rotate(unsigned char *list, unsigned len, unsigned rot)
+ {
+ unsigned char tmp;
+ unsigned cycles;
+ unsigned char *start, *last, *to, *from;
+
+ /* normalize rot and handle degenerate cases */
+ if (len < 2) return;
+ if (rot >= len) rot %= len;
+ if (rot == 0) return;
+
+ /* pointer to last entry in list */
+ last = list + (len - 1);
+
+ /* do simple left shift by one */
+ if (rot == 1) {
+ tmp = *list;
+ memcpy(list, list + 1, len - 1);
+ *last = tmp;
+ return;
+ }
+
+ /* do simple right shift by one */
+ if (rot == len - 1) {
+ tmp = *last;
+ memmove(list + 1, list, len - 1);
+ *list = tmp;
+ return;
+ }
+
+ /* otherwise do rotate as a set of cycles in place */
+ cycles = gcd(len, rot); /* number of cycles */
+ do {
+ start = from = list + cycles; /* start index is arbitrary */
+ tmp = *from; /* save entry to be overwritten */
+ for (;;) {
+ to = from; /* next step in cycle */
+ from += rot; /* go right rot positions */
+ if (from > last) from -= len; /* (pointer better not wrap) */
+ if (from == start) break; /* all but one shifted */
+ *to = *from; /* shift left */
+ }
+ *to = tmp; /* complete the circle */
+ } while (--cycles);
+ }
+
+ /* structure for gzip file read operations */
+ typedef struct {
+ int fd; /* file descriptor */
+ int size; /* 1 << size is bytes in buf */
+ unsigned left; /* bytes available at next */
+ unsigned char *buf; /* buffer */
+ unsigned char *next; /* next byte in buffer */
+ char *name; /* file name for error messages */
+ } file;
+
+ /* reload buffer */
+ local int readin(file *in)
+ {
+ int len;
+
+ len = read(in->fd, in->buf, 1 << in->size);
+ if (len == -1) bye("error reading ", in->name);
+ in->left = (unsigned)len;
+ in->next = in->buf;
+ return len;
+ }
+
+ /* read from file in, exit if end-of-file */
+ local int readmore(file *in)
+ {
+ if (readin(in) == 0) bye("unexpected end of ", in->name);
+ return 0;
+ }
+
+ #define read1(in) (in->left == 0 ? readmore(in) : 0, \
+ in->left--, *(in->next)++)
+
+ /* skip over n bytes of in */
+ local void skip(file *in, unsigned n)
+ {
+ unsigned bypass;
+
+ if (n > in->left) {
+ n -= in->left;
+ bypass = n & ~((1U << in->size) - 1);
+ if (bypass) {
+ if (lseek(in->fd, (off_t)bypass, SEEK_CUR) == -1)
+ bye("seeking ", in->name);
+ n -= bypass;
+ }
+ readmore(in);
+ if (n > in->left)
+ bye("unexpected end of ", in->name);
+ }
+ in->left -= n;
+ in->next += n;
+ }
+
+ /* read a four-byte unsigned integer, little-endian, from in */
+ unsigned long read4(file *in)
+ {
+ unsigned long val;
+
+ val = read1(in);
+ val += (unsigned)read1(in) << 8;
+ val += (unsigned long)read1(in) << 16;
+ val += (unsigned long)read1(in) << 24;
+ return val;
+ }
+
+ /* skip over gzip header */
+ local void gzheader(file *in)
+ {
+ int flags;
+ unsigned n;
+
+ if (read1(in) != 31 || read1(in) != 139) bye(in->name, " not a gzip file");
+ if (read1(in) != 8) bye("unknown compression method in", in->name);
+ flags = read1(in);
+ if (flags & 0xe0) bye("unknown header flags set in", in->name);
+ skip(in, 6);
+ if (flags & 4) {
+ n = read1(in);
+ n += (unsigned)(read1(in)) << 8;
+ skip(in, n);
+ }
+ if (flags & 8) while (read1(in) != 0) ;
+ if (flags & 16) while (read1(in) != 0) ;
+ if (flags & 2) skip(in, 2);
+ }
+
+ /* decompress gzip file "name", return strm with a deflate stream ready to
+ continue compression of the data in the gzip file, and return a file
+ descriptor pointing to where to write the compressed data -- the deflate
+ stream is initialized to compress using level "level" */
+ local int gzscan(char *name, z_stream *strm, int level)
+ {
+ int ret, lastbit, left, full;
+ unsigned have;
+ unsigned long crc, tot;
+ unsigned char *window;
+ off_t lastoff, end;
+ file gz;
+
+ /* open gzip file */
+ gz.name = name;
+ gz.fd = open(name, O_RDWR, 0);
+ if (gz.fd == -1) bye("cannot open ", name);
+ gz.buf = malloc(CHUNK);
+ if (gz.buf == NULL) bye("out of memory", "");
+ gz.size = LGCHUNK;
+ gz.left = 0;
+
+ /* skip gzip header */
+ gzheader(&gz);
+
+ /* prepare to decompress */
+ window = malloc(DSIZE);
+ if (window == NULL) bye("out of memory", "");
+ strm->zalloc = Z_NULL;
+ strm->zfree = Z_NULL;
+ strm->opaque = Z_NULL;
+ ret = inflateInit2(strm, -15);
+ if (ret != Z_OK) bye("out of memory", " or library mismatch");
+
+ /* decompress the deflate stream, saving append information */
+ lastbit = 0;
+ lastoff = lseek(gz.fd, 0L, SEEK_CUR) - gz.left;
+ left = 0;
+ strm->avail_in = gz.left;
+ strm->next_in = gz.next;
+ crc = crc32(0L, Z_NULL, 0);
+ have = full = 0;
+ do {
+ /* if needed, get more input */
+ if (strm->avail_in == 0) {
+ readmore(&gz);
+ strm->avail_in = gz.left;
+ strm->next_in = gz.next;
+ }
+
+ /* set up output to next available section of sliding window */
+ strm->avail_out = DSIZE - have;
+ strm->next_out = window + have;
+
+ /* inflate and check for errors */
+ ret = inflate(strm, Z_BLOCK);
+ if (ret == Z_STREAM_ERROR) bye("internal stream error!", "");
+ if (ret == Z_MEM_ERROR) bye("out of memory", "");
+ if (ret == Z_DATA_ERROR)
+ bye("invalid compressed data--format violated in", name);
+
+ /* update crc and sliding window pointer */
+ crc = crc32(crc, window + have, DSIZE - have - strm->avail_out);
+ if (strm->avail_out)
+ have = DSIZE - strm->avail_out;
+ else {
+ have = 0;
+ full = 1;
+ }
+
+ /* process end of block */
+ if (strm->data_type & 128) {
+ if (strm->data_type & 64)
+ left = strm->data_type & 0x1f;
+ else {
+ lastbit = strm->data_type & 0x1f;
+ lastoff = lseek(gz.fd, 0L, SEEK_CUR) - strm->avail_in;
+ }
+ }
+ } while (ret != Z_STREAM_END);
+ inflateEnd(strm);
+ gz.left = strm->avail_in;
+ gz.next = strm->next_in;
+
+ /* save the location of the end of the compressed data */
+ end = lseek(gz.fd, 0L, SEEK_CUR) - gz.left;
+
+ /* check gzip trailer and save total for deflate */
+ if (crc != read4(&gz))
+ bye("invalid compressed data--crc mismatch in ", name);
+ tot = strm->total_out;
+ if ((tot & 0xffffffffUL) != read4(&gz))
+ bye("invalid compressed data--length mismatch in", name);
+
+ /* if not at end of file, warn */
+ if (gz.left || readin(&gz))
+ fprintf(stderr,
+ "gzappend warning: junk at end of gzip file overwritten\n");
+
+ /* clear last block bit */
+ lseek(gz.fd, lastoff - (lastbit != 0), SEEK_SET);
+ if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name);
+ *gz.buf = (unsigned char)(*gz.buf ^ (1 << ((8 - lastbit) & 7)));
+ lseek(gz.fd, -1L, SEEK_CUR);
+ if (write(gz.fd, gz.buf, 1) != 1) bye("writing after seek to ", name);
+
+ /* if window wrapped, build dictionary from window by rotating */
+ if (full) {
+ rotate(window, DSIZE, have);
+ have = DSIZE;
+ }
+
+ /* set up deflate stream with window, crc, total_in, and leftover bits */
+ ret = deflateInit2(strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
+ if (ret != Z_OK) bye("out of memory", "");
+ deflateSetDictionary(strm, window, have);
+ strm->adler = crc;
+ strm->total_in = tot;
+ if (left) {
+ lseek(gz.fd, --end, SEEK_SET);
+ if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name);
+ deflatePrime(strm, 8 - left, *gz.buf);
+ }
+ lseek(gz.fd, end, SEEK_SET);
+
+ /* clean up and return */
+ free(window);
+ free(gz.buf);
+ return gz.fd;
+ }
+
+ /* append file "name" to gzip file gd using deflate stream strm -- if last
+ is true, then finish off the deflate stream at the end */
+ local void gztack(char *name, int gd, z_stream *strm, int last)
+ {
+ int fd, len, ret;
+ unsigned left;
+ unsigned char *in, *out;
+
+ /* open file to compress and append */
+ fd = 0;
+ if (name != NULL) {
+ fd = open(name, O_RDONLY, 0);
+ if (fd == -1)
+ fprintf(stderr, "gzappend warning: %s not found, skipping ...\n",
+ name);
+ }
+
+ /* allocate buffers */
+ in = fd == -1 ? NULL : malloc(CHUNK);
+ out = malloc(CHUNK);
+ if (out == NULL) bye("out of memory", "");
+
+ /* compress input file and append to gzip file */
+ do {
+ /* get more input */
+ len = fd == -1 ? 0 : read(fd, in, CHUNK);
+ if (len == -1) {
+ fprintf(stderr,
+ "gzappend warning: error reading %s, skipping rest ...\n",
+ name);
+ len = 0;
+ }
+ strm->avail_in = (unsigned)len;
+ strm->next_in = in;
+ if (len) strm->adler = crc32(strm->adler, in, (unsigned)len);
+
+ /* compress and write all available output */
+ do {
+ strm->avail_out = CHUNK;
+ strm->next_out = out;
+ ret = deflate(strm, last && len == 0 ? Z_FINISH : Z_NO_FLUSH);
+ left = CHUNK - strm->avail_out;
+ while (left) {
+ len = write(gd, out + CHUNK - strm->avail_out - left, left);
+ if (len == -1) bye("writing gzip file", "");
+ left -= (unsigned)len;
+ }
+ } while (strm->avail_out == 0 && ret != Z_STREAM_END);
+ } while (len != 0);
+
+ /* write trailer after last entry */
+ if (last) {
+ deflateEnd(strm);
+ out[0] = (unsigned char)(strm->adler);
+ out[1] = (unsigned char)(strm->adler >> 8);
+ out[2] = (unsigned char)(strm->adler >> 16);
+ out[3] = (unsigned char)(strm->adler >> 24);
+ out[4] = (unsigned char)(strm->total_in);
+ out[5] = (unsigned char)(strm->total_in >> 8);
+ out[6] = (unsigned char)(strm->total_in >> 16);
+ out[7] = (unsigned char)(strm->total_in >> 24);
+ len = 8;
+ do {
+ ret = write(gd, out + 8 - len, len);
+ if (ret == -1) bye("writing gzip file", "");
+ len -= ret;
+ } while (len);
+ close(gd);
+ }
+
+ /* clean up and return */
+ free(out);
+ if (in != NULL) free(in);
+ if (fd > 0) close(fd);
+ }
+
+ /* process the compression level option if present, scan the gzip file, and
+ append the specified files, or append the data from stdin if no other file
+ names are provided on the command line -- the gzip file must be writable
+ and seekable */
+ int main(int argc, char **argv)
+ {
+ int gd, level;
+ z_stream strm;
+
+ /* ignore command name */
+ argv++;
+
+ /* provide usage if no arguments */
+ if (*argv == NULL) {
+ printf("gzappend 1.1 (4 Nov 2003) Copyright (C) 2003 Mark Adler\n");
+ printf(
+ "usage: gzappend [-level] file.gz [ addthis [ andthis ... ]]\n");
+ return 0;
+ }
+
+ /* set compression level */
+ level = Z_DEFAULT_COMPRESSION;
+ if (argv[0][0] == '-') {
+ if (argv[0][1] < '0' || argv[0][1] > '9' || argv[0][2] != 0)
+ bye("invalid compression level", "");
+ level = argv[0][1] - '0';
+ if (*++argv == NULL) bye("no gzip file name after options", "");
+ }
+
+ /* prepare to append to gzip file */
+ gd = gzscan(*argv++, &strm, level);
+
+ /* append files on command line, or from stdin if none */
+ if (*argv == NULL)
+ gztack(NULL, gd, &strm, 1);
+ else
+ do {
+ gztack(*argv, gd, &strm, argv[1] == NULL);
+ } while (*++argv != NULL);
+ return 0;
+ }
More information about the llvm-commits
mailing list