Taken from http://download.savannah.gnu.org/releases/lzip/busybox/ --- archival/Config.src +++ archival/Config.src @@ -21,6 +21,10 @@ bool "Make tar, rpm, modprobe etc understand .gz data" default y +config FEATURE_SEAMLESS_LZ + bool "Make tar, rpm, modprobe etc understand .lz data" + default y + config FEATURE_SEAMLESS_Z bool "Make tar, rpm, modprobe etc understand .Z data" default n # it is ancient --- archival/bbunzip.c +++ archival/bbunzip.c @@ -194,18 +194,31 @@ #if ENABLE_UNCOMPRESS \ || ENABLE_FEATURE_BZIP2_DECOMPRESS \ + || ENABLE_FEATURE_LZIP_DECOMPRESS \ || ENABLE_UNLZMA || ENABLE_LZCAT || ENABLE_LZMA \ || ENABLE_UNXZ || ENABLE_XZCAT || ENABLE_XZ static char* FAST_FUNC make_new_name_generic(char *filename, const char *expected_ext) { char *extension = strrchr(filename, '.'); - if (!extension || strcmp(extension + 1, expected_ext) != 0) { + + if (!extension) + return NULL; + + if (strcmp(extension + 1, expected_ext) == 0) { + *extension = '\0'; + } else if (extension[1] == 't' && strlen(expected_ext) >= 2 && + strcmp(extension + 2, expected_ext) == 0) { + filename = xstrdup(filename); + extension = strrchr(filename, '.'); + extension[2] = 'a'; + extension[3] = 'r'; + extension[4] = '\0'; + } else { /* Mimic GNU gunzip - "real" bunzip2 tries to */ /* unpack file anyway, to file.out */ return NULL; } - *extension = '\0'; return filename; } #endif @@ -451,6 +464,39 @@ } #endif +//usage:#define lunzip_trivial_usage +//usage: "[-cfkt] [FILE]..." +//usage:#define lunzip_full_usage "\n\n" +//usage: "Decompress FILEs (or stdin)\n" +//usage: "\n -c Write to stdout" +//usage: "\n -f Force" +//usage: "\n -k Keep input files" +//usage: "\n -t Test file integrity" + +//config:config LUNZIP +//config: bool "lunzip" +//config: default y +//config: select FEATURE_LZIP_DECOMPRESS +//config: help +//config: lunzip is used to decompress archives created by lzip. +//config: You can use the option '-t' to test the integrity of an archive, +//config: without decompressing it. +//config: + +//applet:IF_LUNZIP(APPLET(lunzip, BB_DIR_USR_BIN, BB_SUID_DROP)) +//kbuild:lib-$(CONFIG_LUNZIP) += bbunzip.o + +#if ENABLE_FEATURE_LZIP_DECOMPRESS +int lunzip_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; +int lunzip_main(int argc UNUSED_PARAM, char **argv) +{ + getopt32(argv, BBUNPK_OPTSTR "dt"); + argv += optind; + + return bbunpack(argv, unpack_lz_stream, make_new_name_generic, "lz"); +} +#endif + /* * Small lzma deflate implementation. --- archival/dpkg.c +++ archival/dpkg.c @@ -1505,6 +1505,9 @@ #if ENABLE_FEATURE_SEAMLESS_XZ llist_add_to(&(ar_handle->accept), (char*)"control.tar.xz"); #endif +#if ENABLE_FEATURE_SEAMLESS_LZ + llist_add_to(&(ar_handle->accept), (char*)"control.tar.lz"); +#endif /* Assign the tar handle as a subarchive of the ar handle */ ar_handle->dpkg__sub_archive = tar_handle; @@ -1532,6 +1535,9 @@ #if ENABLE_FEATURE_SEAMLESS_XZ llist_add_to(&(ar_handle->accept), (char*)"data.tar.xz"); #endif +#if ENABLE_FEATURE_SEAMLESS_LZ + llist_add_to(&(ar_handle->accept), (char*)"data.tar.lz"); +#endif /* Assign the tar handle as a subarchive of the ar handle */ ar_handle->dpkg__sub_archive = tar_handle; --- archival/dpkg_deb.c +++ archival/dpkg_deb.c @@ -69,6 +69,10 @@ llist_add_to(&ar_archive->accept, (char*)"data.tar.bz2"); llist_add_to(&control_tar_llist, (char*)"control.tar.bz2"); #endif +#if ENABLE_FEATURE_SEAMLESS_LZ + llist_add_to(&ar_archive->accept, (char*)"data.tar.lz"); + llist_add_to(&control_tar_llist, (char*)"control.tar.lz"); +#endif #if ENABLE_FEATURE_SEAMLESS_LZMA llist_add_to(&ar_archive->accept, (char*)"data.tar.lzma"); llist_add_to(&control_tar_llist, (char*)"control.tar.lzma"); --- archival/libarchive/Kbuild.src +++ archival/libarchive/Kbuild.src @@ -37,6 +37,7 @@ get_header_tar.o \ get_header_tar_gz.o \ get_header_tar_bz2.o \ + get_header_tar_lz.o \ get_header_tar_lzma.o \ get_header_tar_xz.o \ @@ -56,6 +57,8 @@ # 'bzip2 -d', bunzip2 or bzcat selects FEATURE_BZIP2_DECOMPRESS lib-$(CONFIG_FEATURE_BZIP2_DECOMPRESS) += open_transformer.o decompress_bunzip2.o lib-$(CONFIG_FEATURE_UNZIP_BZIP2) += open_transformer.o decompress_bunzip2.o +# 'lzip -d' or lunzip select FEATURE_LZIP_DECOMPRESS +lib-$(CONFIG_FEATURE_LZIP_DECOMPRESS) += open_transformer.o decompress_lunzip.o lib-$(CONFIG_UNLZMA) += open_transformer.o decompress_unlzma.o lib-$(CONFIG_LZCAT) += open_transformer.o decompress_unlzma.o lib-$(CONFIG_LZMA) += open_transformer.o decompress_unlzma.o @@ -72,6 +75,7 @@ lib-$(CONFIG_RPM) += open_transformer.o decompress_gunzip.o get_header_cpio.o lib-$(CONFIG_GZIP) += open_transformer.o lib-$(CONFIG_BZIP2) += open_transformer.o +lib-$(CONFIG_LZIP) += open_transformer.o lib-$(CONFIG_LZOP) += open_transformer.o lib-$(CONFIG_MAN) += open_transformer.o lib-$(CONFIG_SETFONT) += open_transformer.o @@ -87,6 +91,7 @@ lib-$(CONFIG_FEATURE_SEAMLESS_Z) += open_transformer.o decompress_uncompress.o lib-$(CONFIG_FEATURE_SEAMLESS_GZ) += open_transformer.o decompress_gunzip.o lib-$(CONFIG_FEATURE_SEAMLESS_BZ2) += open_transformer.o decompress_bunzip2.o +lib-$(CONFIG_FEATURE_SEAMLESS_LZ) += open_transformer.o decompress_lunzip.o lib-$(CONFIG_FEATURE_SEAMLESS_LZMA) += open_transformer.o decompress_unlzma.o lib-$(CONFIG_FEATURE_SEAMLESS_XZ) += open_transformer.o decompress_unxz.o lib-$(CONFIG_FEATURE_COMPRESS_USAGE) += open_transformer.o decompress_bunzip2.o --- archival/libarchive/decompress_lunzip.c +++ archival/libarchive/decompress_lunzip.c @@ -0,0 +1,493 @@ +/* + * lunzip implementation for busybox + * + * Copyright (C) 2012-2021 Antonio Diaz Diaz. + * + * Licensed under GPLv2 or later, see file LICENSE in this source tree. + */ + +#include "libbb.h" +#include "bb_archive.h" +#include "lzip.h" + +/* Some functions have been marked with __always_inline because xz does + it, giving the impression that unxz is much faster than lunzip. */ +#ifndef __always_inline +# ifdef __GNUC__ +# define __always_inline \ + inline __attribute__((__always_inline__)) +# else +# define __always_inline inline +# endif +#endif + + +enum { rd_buffer_size = 16384 }; + +struct Range_decoder { + unsigned long long partial_member_pos; + uint8_t *buffer; /* input buffer */ + int pos; /* current pos in buffer */ + int stream_pos; /* when reached, a new block must be read */ + uint32_t code; + uint32_t range; + int infd; /* input file descriptor */ + bool at_stream_end; +}; + + +static bool Rd_read_block(struct Range_decoder * const rdec) +{ + if (!rdec->at_stream_end) { + rdec->stream_pos = + full_read(rdec->infd, rdec->buffer, rd_buffer_size); + rdec->at_stream_end = (rdec->stream_pos < rd_buffer_size); + rdec->partial_member_pos += rdec->pos; + rdec->pos = 0; + } + return rdec->pos < rdec->stream_pos; +} + + +static bool Rd_init(struct Range_decoder * const rdec, const int ifd, + const bool magic_skipped) +{ + rdec->partial_member_pos = (magic_skipped ? 4 : 0); + rdec->buffer = (uint8_t *) malloc(rd_buffer_size); + if (!rdec->buffer) return false; + rdec->pos = 0; + rdec->stream_pos = 0; + rdec->code = 0; + rdec->range = 0xFFFFFFFFU; + rdec->infd = ifd; + rdec->at_stream_end = false; + return true; +} + +static __always_inline bool Rd_finished(struct Range_decoder * const rdec) +{ + return rdec->pos >= rdec->stream_pos && !Rd_read_block(rdec); +} + +static inline unsigned long long +Rd_member_position(const struct Range_decoder * const rdec) +{ + return rdec->partial_member_pos + rdec->pos; +} + +static inline void Rd_reset_member_position(struct Range_decoder * const rdec) +{ + rdec->partial_member_pos = 0; rdec->partial_member_pos -= rdec->pos; +} + +static __always_inline uint8_t Rd_get_byte(struct Range_decoder * const rdec) +{ + /* 0xFF avoids decoder error if member is truncated at EOS marker */ + if (Rd_finished(rdec)) return 0xFF; + return rdec->buffer[rdec->pos++]; +} + +static void Rd_load(struct Range_decoder * const rdec) +{ + int i; + rdec->code = 0; + for (i = 0; i < 5; ++i) + rdec->code = (rdec->code << 8) | Rd_get_byte(rdec); + rdec->range = 0xFFFFFFFFU; +} + +static __always_inline void Rd_normalize(struct Range_decoder * const rdec) +{ + if (rdec->range <= 0x00FFFFFFU) { + rdec->range <<= 8; + rdec->code = (rdec->code << 8) | Rd_get_byte(rdec); + } +} + +static unsigned Rd_decode(struct Range_decoder * const rdec, + const int num_bits) +{ + unsigned symbol = 0; + int i; + for (i = num_bits; i > 0; --i) { + bool bit; + Rd_normalize(rdec); + rdec->range >>= 1; + /* symbol <<= 1; */ + /* if(rdec->code >= rdec->range) { rdec->code -= rdec->range; symbol |= 1; } */ + bit = (rdec->code >= rdec->range); + symbol = (symbol << 1) + bit; + rdec->code -= rdec->range & (0U - bit); + } + return symbol; +} + +static __always_inline unsigned Rd_decode_bit(struct Range_decoder * const rdec, + Bit_model * const probability) +{ + uint32_t bound; + Rd_normalize(rdec); + bound = (rdec->range >> bit_model_total_bits) * *probability; + if (rdec->code < bound) { + rdec->range = bound; + *probability += (bit_model_total - *probability) >> bit_model_move_bits; + return 0; + } else { + rdec->range -= bound; + rdec->code -= bound; + *probability -= *probability >> bit_model_move_bits; + return 1; + } +} + +static __always_inline unsigned Rd_decode_tree(struct Range_decoder * const rdec, + Bit_model bm[], const int num_bits) +{ + unsigned symbol = 1; + int i; + for (i = num_bits; i > 0; --i) + symbol = (symbol << 1) | Rd_decode_bit(rdec, &bm[symbol]); + return symbol - (1 << num_bits); +} + +static __always_inline unsigned Rd_decode_tree_reversed(struct Range_decoder * const rdec, + Bit_model bm[], const int num_bits) +{ + unsigned model = 1; + unsigned symbol = 0; + int i; + for (i = 0; i < num_bits; ++i) { + const unsigned bit = Rd_decode_bit(rdec, &bm[model]); + model = (model << 1) + bit; + symbol |= (bit << i); + } + return symbol; +} + +static unsigned Rd_decode_matched(struct Range_decoder * const rdec, + Bit_model bm[], unsigned match_byte) +{ + unsigned symbol = 1; + unsigned mask = 0x100; + while(true) { + const unsigned match_bit = (match_byte <<= 1) & mask; + const unsigned bit = Rd_decode_bit(rdec, &bm[symbol+match_bit+mask]); + symbol = (symbol << 1) + bit; + if (symbol >= 0x100) return symbol & 0xFF; + mask &= ~(match_bit ^ (bit << 8)); /* if( match_bit != bit ) mask = 0; */ + } +} + +static __always_inline unsigned Rd_decode_len(struct Range_decoder * const rdec, + struct Len_model * const lm, + const int pos_state) +{ + if (Rd_decode_bit(rdec, &lm->choice1) == 0) + return Rd_decode_tree(rdec, lm->bm_low[pos_state], len_low_bits); + if (Rd_decode_bit(rdec, &lm->choice2) == 0) + return len_low_symbols + + Rd_decode_tree(rdec, lm->bm_mid[pos_state], len_mid_bits); + return len_low_symbols + len_mid_symbols + + Rd_decode_tree(rdec, lm->bm_high, len_high_bits); +} + + +struct LZ_decoder { + unsigned long long partial_data_pos; + struct Range_decoder *rdec; + unsigned dictionary_size; + uint8_t *buffer; /* output buffer */ + unsigned pos; /* current pos in buffer */ + unsigned stream_pos; /* first byte not yet written to file */ + uint32_t crc; + int outfd; /* output file descriptor */ + bool pos_wrapped; + bool write_error; +}; + +static void LZd_flush_data(struct LZ_decoder * const d) +{ + if (d->pos > d->stream_pos) { + const int size = d->pos - d->stream_pos; + d->crc = crc32_block_endian0(d->crc, d->buffer + d->stream_pos, + size, global_crc32_table); + if (d->outfd >= 0 && full_write(d->outfd, + d->buffer + d->stream_pos, size) != size) + d->write_error = true; + if (d->pos >= d->dictionary_size) { + d->partial_data_pos += d->pos; + d->pos = 0; + d->pos_wrapped = true; + } + d->stream_pos = d->pos; + } +} + +static __always_inline uint8_t LZd_peek_prev(const struct LZ_decoder * const d) +{ + const unsigned i = ((d->pos > 0) ? d->pos : d->dictionary_size) - 1; + return d->buffer[i]; +} + +static __always_inline uint8_t LZd_peek(const struct LZ_decoder * const d, + const unsigned distance) +{ + unsigned i = d->pos - distance - 1; + if (d->pos <= distance) i += d->dictionary_size; + return d->buffer[i]; +} + +static __always_inline void LZd_put_byte(struct LZ_decoder * const d, + const uint8_t b) +{ + d->buffer[d->pos] = b; + if (++d->pos >= d->dictionary_size) LZd_flush_data(d); +} + +static void LZd_copy_block(struct LZ_decoder * const d, + const unsigned distance, unsigned len) +{ + unsigned i = d->pos - distance - 1; + bool fast; + if (d->pos <= distance) { + i += d->dictionary_size; + fast = (len <= d->dictionary_size - i && len <= i - d->pos); + } + else + fast = (len < d->dictionary_size - d->pos && len <= d->pos - i); + if( fast ) { /* no wrap, no overlap */ + memcpy(d->buffer + d->pos, d->buffer + i, len); + d->pos += len; + } else for (; len > 0; --len) { + d->buffer[d->pos] = d->buffer[i]; + if (++d->pos >= d->dictionary_size) LZd_flush_data(d); + if (++i >= d->dictionary_size) i = 0; + } +} + +static bool LZd_init(struct LZ_decoder * const d, + struct Range_decoder * const rde, + const unsigned dict_size, const int ofd) +{ + d->partial_data_pos = 0; + d->rdec = rde; + d->dictionary_size = dict_size; + d->buffer = (uint8_t *) malloc(d->dictionary_size); + if (!d->buffer) return false; + d->pos = 0; + d->stream_pos = 0; + d->crc = 0xFFFFFFFFU; + d->outfd = ofd; + d->pos_wrapped = false; + d->write_error = false; + d->buffer[d->dictionary_size - 1] = 0; /* prev_byte of first byte */ + return true; +} + +static inline uint32_t LZd_crc(const struct LZ_decoder * const d) +{ + return d->crc ^ 0xFFFFFFFFU; +} + +static __always_inline unsigned long long +LZd_data_position(const struct LZ_decoder * const d) +{ + return d->partial_data_pos + d->pos; +} + + +static bool LZd_verify_trailer(struct LZ_decoder * const d) +{ + Lzip_trailer trailer; + int i = 0; + while (i < Lt_size) + trailer[i++] = Rd_get_byte(d->rdec); + + return (Lt_get_data_crc(trailer) == LZd_crc(d) && + Lt_get_data_size(trailer) == LZd_data_position(d) && + Lt_get_member_size(trailer) == Rd_member_position(d->rdec)); +} + + +/* Return value: -1 = write error, 0 = OK, 1 = data error. */ +static int LZd_decode_member(struct LZ_decoder * const d) +{ + struct Range_decoder * const rdec = d->rdec; + Bit_model bm_literal[1 << literal_context_bits][0x300]; + Bit_model bm_match[states][pos_states]; + Bit_model bm_rep[states]; + Bit_model bm_rep0[states]; + Bit_model bm_rep1[states]; + Bit_model bm_rep2[states]; + Bit_model bm_len[states][pos_states]; + Bit_model bm_dis_slot[len_states][1 << dis_slot_bits]; + Bit_model bm_dis[modeled_distances-end_dis_model+1]; + Bit_model bm_align[dis_align_size]; + struct Len_model match_len_model; + struct Len_model rep_len_model; + unsigned rep0 = 0; /* rep[0-3] latest four distances */ + unsigned rep1 = 0; /* used for efficient coding of */ + unsigned rep2 = 0; /* repeated distances */ + unsigned rep3 = 0; + State state = 0; + + Bm_array_init(bm_literal[0], (1 << literal_context_bits) * 0x300); + Bm_array_init(bm_match[0], states * pos_states); + Bm_array_init(bm_rep, states); + Bm_array_init(bm_rep0, states); + Bm_array_init(bm_rep1, states); + Bm_array_init(bm_rep2, states); + Bm_array_init(bm_len[0], states * pos_states); + Bm_array_init(bm_dis_slot[0], len_states * (1 << dis_slot_bits)); + Bm_array_init(bm_dis, modeled_distances - end_dis_model + 1); + Bm_array_init(bm_align, dis_align_size); + Lm_init(&match_len_model); + Lm_init(&rep_len_model); + + Rd_load(rdec); + while (!Rd_finished(rdec)) { + int len; + const int pos_state = LZd_data_position(d) & pos_state_mask; + if (Rd_decode_bit(rdec, &bm_match[state][pos_state]) == 0) { + /* literal byte */ + Bit_model * const bm = bm_literal[get_lit_state(LZd_peek_prev(d))]; + if (St_is_char(state)) { + state -= (state < 4) ? state : 3; + LZd_put_byte(d, Rd_decode_tree(rdec, bm, 8)); + } else { + state -= (state < 10) ? 3 : 6; + LZd_put_byte(d, Rd_decode_matched(rdec, bm, LZd_peek(d, rep0))); + } + continue; + } + /* match or repeated match */ + if (Rd_decode_bit(rdec, &bm_rep[state]) != 0) { + if (Rd_decode_bit(rdec, &bm_rep0[state]) == 0) { + if (Rd_decode_bit(rdec, &bm_len[state][pos_state]) == 0) { + state = St_set_short_rep(state); + LZd_put_byte(d, LZd_peek(d, rep0)); + continue; + } + } else { + unsigned distance; + if (Rd_decode_bit(rdec, &bm_rep1[state]) == 0) + distance = rep1; + else { + if (Rd_decode_bit(rdec, &bm_rep2[state]) == 0) + distance = rep2; + else { + distance = rep3; + rep3 = rep2; + } + rep2 = rep1; + } + rep1 = rep0; + rep0 = distance; + } + state = St_set_rep(state); + len = min_match_len + Rd_decode_len(rdec, &rep_len_model, pos_state); + } else { /* match */ + unsigned distance; + len = min_match_len + Rd_decode_len(rdec, &match_len_model, pos_state); + distance = Rd_decode_tree(rdec, bm_dis_slot[get_len_state(len)], 6); + if (distance >= start_dis_model) { + const unsigned dis_slot = distance; + const int direct_bits = (dis_slot >> 1) - 1; + distance = (2 | (dis_slot & 1)) << direct_bits; + if (dis_slot < end_dis_model) + distance += Rd_decode_tree_reversed(rdec, + bm_dis + (distance - dis_slot), direct_bits); + else { + distance += + Rd_decode(rdec, direct_bits - dis_align_bits) << dis_align_bits; + distance += Rd_decode_tree_reversed(rdec, bm_align, dis_align_bits); + if (distance == 0xFFFFFFFFU) { /* marker found */ + Rd_normalize(rdec); + LZd_flush_data(d); + if (d->write_error) return -1; + if (len == min_match_len && /* End Of Stream marker */ + LZd_verify_trailer(d)) + return 0; + return 1; + } + } + } + rep3 = rep2; rep2 = rep1; rep1 = rep0; rep0 = distance; + state = St_set_match(state); + if (rep0 >= d->dictionary_size || + (rep0 >= d->pos && !d->pos_wrapped)) { + LZd_flush_data(d); + return 1; + } + } + LZd_copy_block(d, rep0, len); + } + LZd_flush_data(d); + return 1; +} + + +IF_DESKTOP(long long) int FAST_FUNC +unpack_lz_stream(transformer_state_t *xstate) +{ + IF_DESKTOP(long long) int total = 0; + struct Range_decoder rdec; + bool first_member; + const bool magic_skipped = (xstate->signature_skipped != 0); + + if (!global_crc32_table) + global_crc32_new_table_le(); + + if (!Rd_init(&rdec, xstate->src_fd, magic_skipped)) + return -1; + + for (first_member = true;; first_member = false) { + int tmp = 0; + Lzip_header header; + struct LZ_decoder decoder; + + if (first_member && magic_skipped) { + Lh_set_magic(header); + tmp = 4; + } else { + Rd_reset_member_position(&rdec); + } + while (tmp < Lh_size) + header[tmp++] = Rd_get_byte(&rdec); + if (Rd_finished(&rdec)) { /* End Of File */ + if (first_member) { + bb_error_msg(bb_msg_read_error); + total = -1; + } + break; + } + tmp = Lh_get_dictionary_size(header); + if (!Lh_verify_magic(header) || tmp < min_dictionary_size || + tmp > max_dictionary_size) { + if (!first_member) + break; /* trailing garbage */ + bb_error_msg("invalid magic"); + total = -1; + break; + } + + if (!LZd_init(&decoder, &rdec, tmp, xstate->dst_fd)) { + bb_error_msg(bb_msg_memory_exhausted); + total = -1; + break; + } + tmp = LZd_decode_member(&decoder); + IF_DESKTOP(total += Rd_member_position(&rdec);) + free(decoder.buffer); + if (tmp != 0) { + if (tmp < 0) + bb_perror_msg(bb_msg_write_error); + else + bb_error_msg("corrupted data"); + total = -1; + break; + } + } + free(rdec.buffer); + return total; +} --- archival/libarchive/filter_accept_list_reassign.c +++ archival/libarchive/filter_accept_list_reassign.c @@ -43,6 +43,12 @@ archive_handle->dpkg__action_data_subarchive = get_header_tar_bz2; return EXIT_SUCCESS; } + if (ENABLE_FEATURE_SEAMLESS_LZ + && strcmp(name_ptr, "lz") == 0 + ) { + archive_handle->dpkg__action_data_subarchive = get_header_tar_lz; + return EXIT_SUCCESS; + } if (ENABLE_FEATURE_SEAMLESS_LZMA && strcmp(name_ptr, "lzma") == 0 ) { --- archival/libarchive/get_header_tar_lz.c +++ archival/libarchive/get_header_tar_lz.c @@ -0,0 +1,20 @@ +/* vi: set sw=4 ts=4: */ +/* + * Licensed under GPLv2 or later, see file LICENSE in this source tree. + */ +#include "libbb.h" +#include "bb_archive.h" + +char FAST_FUNC get_header_tar_lz(archive_handle_t *archive_handle) +{ + /* Can't lseek over pipes */ + archive_handle->seek = seek_by_read; + + fork_transformer_with_sig(archive_handle->src_fd, unpack_lz_stream, "lunzip"); + archive_handle->offset = 0; + while (get_header_tar(archive_handle) == EXIT_SUCCESS) + continue; + + /* Can only do one file at a time */ + return EXIT_FAILURE; +} --- archival/libarchive/lzip.h +++ archival/libarchive/lzip.h @@ -0,0 +1,249 @@ +/* Lzip - LZMA lossless data compressor + Copyright (C) 2008-2021 Antonio Diaz Diaz. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +typedef int State; + +enum { states = 12 }; + +static inline bool St_is_char(const State st) { return st < 7; } + +static inline State St_set_char(const State st) +{ + static const State next[states] = { 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5 }; + return next[st]; +} + +static inline State St_set_match(const State st) +{ + return ((st < 7) ? 7 : 10); +} + +static inline State St_set_rep(const State st) +{ + return ((st < 7) ? 8 : 11); +} + +static inline State St_set_short_rep(const State st) +{ + return ((st < 7) ? 9 : 11); +} + + +enum { + min_dictionary_bits = 12, + min_dictionary_size = 1 << min_dictionary_bits, /* >= modeled_distances */ + max_dictionary_bits = 29, + max_dictionary_size = 1 << max_dictionary_bits, + literal_context_bits = 3, + pos_state_bits = 2, + pos_states = 1 << pos_state_bits, + pos_state_mask = pos_states - 1, + + len_states = 4, + dis_slot_bits = 6, + start_dis_model = 4, + end_dis_model = 14, + modeled_distances = 1 << (end_dis_model / 2), /* 128 */ + dis_align_bits = 4, + dis_align_size = 1 << dis_align_bits, + + len_low_bits = 3, + len_mid_bits = 3, + len_high_bits = 8, + len_low_symbols = 1 << len_low_bits, + len_mid_symbols = 1 << len_mid_bits, + len_high_symbols = 1 << len_high_bits, + max_len_symbols = len_low_symbols + len_mid_symbols + len_high_symbols, + + min_match_len = 2, /* must be 2 */ + max_match_len = min_match_len + max_len_symbols - 1, /* 273 */ + min_match_len_limit = 5, + + lz_num_models = + ((1 << literal_context_bits) * 0x300) + + (2 * states * pos_states) + + (4 * states) + + (len_states * (1 << dis_slot_bits)) + + (modeled_distances - end_dis_model + 1) + + dis_align_size, +}; + +static inline int get_len_state(const int len) +{ + return MIN(len - min_match_len, len_states - 1); +} + +static inline int get_lit_state(const uint8_t prev_byte) +{ + return (prev_byte >> (8 - literal_context_bits)); +} + + +enum { bit_model_move_bits = 5, + bit_model_total_bits = 11, + bit_model_total = 1 << bit_model_total_bits +}; + +typedef int Bit_model; + +static inline void Bm_init(Bit_model * const probability) +{ + *probability = bit_model_total / 2; +} + +static inline void Bm_array_init(Bit_model bm[], const int size) +{ + int i; + for (i = 0; i < size; ++i) + Bm_init(&bm[i]); +} + +struct Len_model { + Bit_model choice1; + Bit_model choice2; + Bit_model bm_low[pos_states][len_low_symbols]; + Bit_model bm_mid[pos_states][len_mid_symbols]; + Bit_model bm_high[len_high_symbols]; +}; + +static inline void Lm_init(struct Len_model * const lm) +{ + Bm_init(&lm->choice1); + Bm_init(&lm->choice2); + Bm_array_init(lm->bm_low[0], pos_states * len_low_symbols); + Bm_array_init(lm->bm_mid[0], pos_states * len_mid_symbols); + Bm_array_init(lm->bm_high, len_high_symbols); +} + + +static inline int real_bits(unsigned value) +{ + int bits = 0; + while(value > 0) { value >>= 1; ++bits; } + return bits; +} + + +static const uint8_t lzip_magic[4] = { 0x4C, 0x5A, 0x49, 0x50 }; /* "LZIP" */ + +typedef uint8_t Lzip_header[6]; /* 0-3 magic bytes */ + /* 4 version */ + /* 5 coded_dict_size */ +enum { Lh_size = 6 }; + +static inline void Lh_set_magic(Lzip_header data) +{ + memcpy(data, lzip_magic, 4); + data[4] = 1; +} + +static inline bool Lh_verify_magic(const Lzip_header data) +{ + return (memcmp(data, lzip_magic, 4) == 0 && data[4] == 1); +} + +static inline unsigned Lh_get_dictionary_size(const Lzip_header data) +{ + unsigned sz = (1 << (data[5] & 0x1F)); + if (sz > min_dictionary_size) + sz -= (sz / 16) * ((data[5] >> 5) & 7); + return sz; +} + +static inline bool Lh_set_dictionary_size(Lzip_header data, const unsigned sz) +{ + if (sz < min_dictionary_size || sz > max_dictionary_size) return false; + data[5] = real_bits(sz - 1); + if (sz > min_dictionary_size) { + const unsigned base_size = 1 << data[5]; + const unsigned fraction = base_size / 16; + unsigned i; + for (i = 7; i >= 1; --i) + if (base_size - (i * fraction) >= sz) { + data[5] |= (i << 5); + break; + } + } + return true; +} + + +typedef uint8_t Lzip_trailer[20]; + /* 0-3 CRC32 of the uncompressed data */ + /* 4-11 size of the uncompressed data */ + /* 12-19 member size including header and trailer */ +enum { Lt_size = 20 }; + +static inline unsigned Lt_get_data_crc(const Lzip_trailer data) +{ + unsigned tmp = 0; + int i; + for (i = 3; i >= 0; --i) { + tmp <<= 8; + tmp += data[i]; + } + return tmp; +} + +static inline void Lt_set_data_crc(Lzip_trailer data, unsigned crc) +{ + int i; + for (i = 0; i <= 3; ++i) { + data[i] = (uint8_t)crc; + crc >>= 8; + } +} + +static inline unsigned long long Lt_get_data_size(const Lzip_trailer data) +{ + unsigned long long tmp = 0; + int i; + for (i = 11; i >= 4; --i) { + tmp <<= 8; + tmp += data[i]; + } + return tmp; +} + +static inline void Lt_set_data_size(Lzip_trailer data, unsigned long long sz) +{ + int i; + for (i = 4; i <= 11; ++i) { + data[i] = (uint8_t)sz; + sz >>= 8; + } +} + +static inline unsigned long long Lt_get_member_size(const Lzip_trailer data) +{ + unsigned long long tmp = 0; + int i; + for (i = 19; i >= 12; --i) { + tmp <<= 8; + tmp += data[i]; + } + return tmp; +} + +static inline void Lt_set_member_size(Lzip_trailer data, unsigned long long sz) +{ + int i; + for (i = 12; i <= 19; ++i) { + data[i] = (uint8_t)sz; + sz >>= 8; + } +} --- archival/libarchive/open_transformer.c +++ archival/libarchive/open_transformer.c @@ -189,6 +189,17 @@ USE_FOR_NOMMU(xstate->xformer_prog = "bunzip2";) goto found_magic; } + if (ENABLE_FEATURE_SEAMLESS_LZ + && xstate->magic.b16[0] == LZIP_MAGIC1 + ) { + xstate->signature_skipped = 4; + xread(fd, xstate->magic.b16, sizeof(xstate->magic.b16[0])); + if (xstate->magic.b16[0] == LZIP_MAGIC2) { + xstate->xformer = unpack_lz_stream; + USE_FOR_NOMMU(xstate->xformer_prog = "lunzip";) + goto found_magic; + } + } if (ENABLE_FEATURE_SEAMLESS_XZ && xstate->magic.b16[0] == XZ_MAGIC1 ) { @@ -207,6 +218,7 @@ if (fail_if_not_compressed) bb_simple_error_msg_and_die("no gzip" IF_FEATURE_SEAMLESS_BZ2("/bzip2") + IF_FEATURE_SEAMLESS_LZ("/lzip") IF_FEATURE_SEAMLESS_XZ("/xz") " magic"); --- archival/lzip.c +++ archival/lzip.c @@ -0,0 +1,1436 @@ +/* + * lzip implementation for busybox + * + * Copyright (C) 2012-2021 Antonio Diaz Diaz. + * + * Licensed under GPLv2 or later, see file LICENSE in this source tree. + */ + +//config:config LZIP +//config: bool "lzip" +//config: default y +//config: help +//config: lzip is a lossless data compressor with a user interface similar to +//config: the one of gzip or bzip2. lzip can compress about as fast as gzip or +//config: compress most files more than bzip2 (depending on compression level). +//config: Decompression speed is intermediate between gzip and bzip2. lzip has +//config: been designed, written, and tested with great care to replace gzip and +//config: bzip2 as the standard general-purpose compressed format for unix-like +//config: systems, especially for long-term archiving. +//config: +//config:config FEATURE_LZIP_DECOMPRESS +//config: bool "Enable decompression" +//config: default y +//config: depends on LZIP || LUNZIP +//config: help +//config: Enable options -d (--decompress) and -t (--test) for lzip. +//config: This will be automatically selected if lunzip is enabled. +//config: + +//applet:IF_LZIP(APPLET(lzip, BB_DIR_USR_BIN, BB_SUID_DROP)) + +//kbuild:lib-$(CONFIG_LZIP) += lzip.o + +//usage:#define lzip_trivial_usage +//usage: "[-123456789c" +//usage: IF_FEATURE_LZIP_DECOMPRESS("d") "fk" +//usage: IF_FEATURE_LZIP_DECOMPRESS("t") +//usage: "] [-m MATCH_LENGTH] [-s DICT_SIZE] [FILE]..." +//usage:#define lzip_full_usage "\n\n" +//usage: "Compress FILEs (or stdin) with lzip algorithm\n" +//usage: "\n -1..9 Compression level" +//usage: "\n -c Write to stdout" +//usage: IF_FEATURE_LZIP_DECOMPRESS("\n -d Decompress") +//usage: "\n -f Force" +//usage: "\n -k Keep input files" +//usage: "\n -m Match length limit [36]" +//usage: "\n -s Dictionary size limit [8MiB]" +//usage: IF_FEATURE_LZIP_DECOMPRESS("\n -t Test compressed file integrity") + + +#include "libbb.h" +#include "bb_archive.h" +#include "libarchive/lzip.h" + + +#if CHAR_BIT != 8 +#error "Environments where CHAR_BIT != 8 are not supported." +#endif + + +static void CRC32_update_byte(uint32_t * crc, const uint8_t byte) +{ + *crc = global_crc32_table[(*crc ^ byte) & 0xFF] ^ (*crc >> 8); +} + + +enum { max_num_trials = 1 << 12, + price_shift_bits = 6, + price_step_bits = 2, + price_step = 1 << price_step_bits +}; + + +static uint8_t * dis_slots; + +static void Dis_slots_init(void) +{ + int i, size, slot; + dis_slots = xmalloc((1 << 10) * sizeof dis_slots[0]); + + for (slot = 0; slot < 4; ++slot) dis_slots[slot] = slot; + for (i = 4, size = 2, slot = 4; slot < 20; slot += 2) { + memset(&dis_slots[i], slot, size); + memset(&dis_slots[i + size], slot + 1, size); + size <<= 1; + i += size; + } +} + +static uint8_t get_slot(const unsigned dis) +{ + if (dis < (1 << 10)) return dis_slots[dis]; + if (dis < (1 << 19)) return dis_slots[dis>> 9] + 18; + if (dis < (1 << 28)) return dis_slots[dis>>18] + 36; + return dis_slots[dis>>27] + 54; +} + + +static int * prob_prices; + +static void Prob_prices_init(void) +{ + int i, j; + prob_prices = + xmalloc((bit_model_total >> price_step_bits) * sizeof prob_prices[0]); + for( i = 0; i < bit_model_total >> price_step_bits; ++i ) { + unsigned val = ( i * price_step ) + ( price_step / 2 ); + int bits = 0; /* base 2 logarithm of val */ + for( j = 0; j < price_shift_bits; ++j ) { + val = val * val; + bits <<= 1; + while( val >= 1 << 16 ) { val >>= 1; ++bits; } + } + bits += 15; /* remaining bits in val */ + prob_prices[i] = ( bit_model_total_bits << price_shift_bits ) - bits; + } +} + +static inline int get_price(const int probability) +{ + return prob_prices[probability >> price_step_bits]; +} + + +static inline int price0(const Bit_model probability) +{ + return get_price(probability); +} + +static inline int price1(const Bit_model probability) +{ + return get_price(bit_model_total - probability); +} + +static int price_bit(const Bit_model bm, const bool bit) +{ + return (bit ? price1(bm) : price0(bm)); +} + + +static int price_symbol(const Bit_model bm[], int symbol, const int num_bits) +{ + int price = 0; + symbol |= (1 << num_bits); + while (symbol > 1) { + const bool bit = symbol & 1; + symbol >>= 1; + price += price_bit(bm[symbol], bit); + } + return price; +} + + +static int price_symbol_reversed(const Bit_model bm[], int symbol, + const int num_bits) +{ + int price = 0; + int model = 1; + int i; + for (i = num_bits; i > 0; --i) { + const bool bit = symbol & 1; + symbol >>= 1; + price += price_bit(bm[model], bit); + model = (model << 1) | bit; + } + return price; +} + + +static int price_matched(const Bit_model bm[], unsigned symbol, + unsigned match_byte) +{ + int price = 0; + unsigned mask = 0x100; + symbol |= mask; + + while (true) { + const unsigned match_bit = (match_byte <<= 1) & mask; + const bool bit = (symbol <<= 1) & 0x100; + price += price_bit( bm[(symbol>>9)+match_bit+mask], bit ); + if (symbol >= 0x10000) return price; + mask &= ~(match_bit ^ symbol); /* if( match_bit != bit ) mask = 0; */ + } +} + + +enum { /* bytes to keep in buffer before dictionary */ + before_size = max_num_trials, + /* bytes to keep in buffer after pos */ + after_size = ( 2 * max_match_len ) + 1, + num_prev_positions3 = 1 << 16, + num_prev_positions2 = 1 << 10, + num_prev_positions23 = num_prev_positions2 + num_prev_positions3 +}; + +struct Matchfinder { + unsigned long long partial_data_pos; + uint8_t *buffer; /* input buffer */ + int32_t *prev_positions; /* 1 + last seen position of key. else 0 */ + int32_t *pos_array; /* tree of previous positions of key */ + int match_len_limit; + int buffer_size; + int dictionary_size; /* bytes to keep in buffer before pos */ + int pos; /* current pos in buffer */ + int cyclic_pos; /* current pos in dictionary */ + int stream_pos; /* first byte not yet read from file */ + int pos_limit; /* when reached, a new block must be read */ + int key4_mask; + int num_prev_positions; /* size of prev_positions */ + int pos_array_size; + int cycles; + bool at_stream_end; /* stream_pos shows real end of file */ +}; + +static bool Mf_read_block(struct Matchfinder * const mf) +{ + if (!mf->at_stream_end && mf->stream_pos < mf->buffer_size) { + const int size = mf->buffer_size - mf->stream_pos; + const int rd = full_read(STDIN_FILENO, + mf->buffer + mf->stream_pos, size); + mf->stream_pos += rd; + if (rd < size) { + mf->at_stream_end = true; + mf->pos_limit = mf->buffer_size; + } + } + return mf->pos < mf->stream_pos; +} + +static void Mf_normalize_pos(struct Matchfinder * const mf) +{ + int i; + const int offset = mf->pos - before_size - mf->dictionary_size; + const int size = mf->stream_pos - offset; + if (mf->at_stream_end) return; + memmove(mf->buffer, mf->buffer + offset, size); + mf->partial_data_pos += offset; + mf->pos -= offset; /* pos = before_size + dictionary_size */ + mf->stream_pos -= offset; + for (i = 0; i < mf->num_prev_positions; ++i) + mf->prev_positions[i] -= MIN(mf->prev_positions[i], offset); + for (i = 0; i < mf->pos_array_size; ++i) + mf->pos_array[i] -= MIN(mf->pos_array[i], offset); + Mf_read_block(mf); +} + +static bool Mf_init(struct Matchfinder * const mf, const int dict_size, + const int match_len_limit) +{ + const int buffer_size_limit = (2 * dict_size) + before_size + after_size; + unsigned size; + int i; + + mf->partial_data_pos = 0; + mf->match_len_limit = match_len_limit; + mf->pos = 0; + mf->cyclic_pos = 0; + mf->stream_pos = 0; + mf->cycles = (match_len_limit < max_match_len) ? + 16 + (match_len_limit / 2) : 256; + mf->at_stream_end = false; + + mf->buffer_size = MAX(65536, dict_size); + mf->buffer = (uint8_t *) malloc(mf->buffer_size); + if (!mf->buffer) return false; + if (Mf_read_block(mf) && !mf->at_stream_end && + mf->buffer_size < buffer_size_limit) { + uint8_t *tmp; + mf->buffer_size = buffer_size_limit; + tmp = (uint8_t *) realloc(mf->buffer, mf->buffer_size); + if (!tmp) { free(mf->buffer); return false; } + mf->buffer = tmp; + Mf_read_block(mf); + } + if (mf->at_stream_end && mf->stream_pos < dict_size) + mf->dictionary_size = MAX(min_dictionary_size, mf->stream_pos); + else + mf->dictionary_size = dict_size; + mf->pos_limit = mf->buffer_size; + if (!mf->at_stream_end) mf->pos_limit -= after_size; + size = 1 << MAX(16, real_bits(mf->dictionary_size - 1) - 2); + if(mf->dictionary_size > 1 << 26) /* 64 MiB */ + size >>= 1; + mf->key4_mask = size - 1; + size += num_prev_positions23; + mf->num_prev_positions = size; + + mf->pos_array_size = 2 * (mf->dictionary_size + 1); + size += mf->pos_array_size; + if (size * sizeof mf->prev_positions[0] <= size) mf->prev_positions = 0; + else mf->prev_positions = + (int32_t *) malloc(size * sizeof mf->prev_positions[0]); + if (!mf->prev_positions) { free(mf->buffer); return false; } + mf->pos_array = mf->prev_positions + mf->num_prev_positions; + for (i = 0; i < mf->num_prev_positions; ++i) mf->prev_positions[i] = 0; + return true; +} + +static void Mf_free(struct Matchfinder * const mf) +{ + free(mf->prev_positions); + free(mf->buffer); +} + +static inline uint8_t Mf_peek(const struct Matchfinder * const mf, + const int distance) +{ + return mf->buffer[mf->pos-distance]; +} + +static inline int Mf_available_bytes(const struct Matchfinder * const mf) +{ + return mf->stream_pos - mf->pos; +} + +static inline unsigned long long +Mf_data_position(const struct Matchfinder * const mf) +{ + return mf->partial_data_pos + mf->pos; +} + +static inline bool Mf_finished(const struct Matchfinder * const mf) +{ + return mf->at_stream_end && mf->pos >= mf->stream_pos; +} + +static inline const uint8_t * +Mf_ptr_to_current_pos(const struct Matchfinder * const mf) +{ + return mf->buffer + mf->pos; +} + +static int Mf_true_match_len(const struct Matchfinder * const mf, + const int index, const int distance) +{ + const uint8_t * const data = mf->buffer + mf->pos; + int i = index; + const int len_limit = MIN(Mf_available_bytes(mf), max_match_len); + while (i < len_limit && data[i - distance] == data[i]) ++i; + return i; +} + +static void Mf_move_pos(struct Matchfinder * const mf) +{ + if (++mf->cyclic_pos > mf->dictionary_size) mf->cyclic_pos = 0; + if (++mf->pos >= mf->pos_limit) Mf_normalize_pos(mf); +} + +static int Mf_longest_match_len(struct Matchfinder * const mf, + int * const distances) +{ + int32_t *ptr0 = mf->pos_array + (mf->cyclic_pos << 1); + int32_t *ptr1 = ptr0 + 1; + int32_t *newptr; + int len = 0, len0 = 0, len1 = 0; + int maxlen = min_match_len - 1; + const int pos1 = mf->pos + 1; + const int min_pos = (mf->pos > mf->dictionary_size) ? + mf->pos - mf->dictionary_size : 0; + const uint8_t * const data = Mf_ptr_to_current_pos(mf); + int count, key2, key3, key4, newpos1; + unsigned tmp; + int len_limit = mf->match_len_limit; + + if (len_limit > Mf_available_bytes(mf)) { + len_limit = Mf_available_bytes(mf); + if (len_limit < 4) return 0; + } + + tmp = global_crc32_table[data[0]] ^ data[1]; + key2 = tmp & (num_prev_positions2 - 1); + tmp ^= ((unsigned) data[2] << 8); + key3 = num_prev_positions2 + (tmp & (num_prev_positions3 - 1)); + key4 = num_prev_positions2 + num_prev_positions3 + + ((tmp ^ (global_crc32_table[data[3]] << 5)) & mf->key4_mask); + + if (distances) { + int np2 = mf->prev_positions[key2]; + int np3 = mf->prev_positions[key3]; + if (np2 > min_pos && mf->buffer[np2-1] == data[0]) { + distances[2] = mf->pos - np2; + maxlen = 2; + } else + distances[2] = 0x7FFFFFFF; + if (np2 != np3 && np3 > min_pos && mf->buffer[np3-1] == data[0]) { + distances[3] = mf->pos - np3; + maxlen = 3; + } else + distances[3] = 0x7FFFFFFF; + distances[4] = 0x7FFFFFFF; + } + + mf->prev_positions[key2] = pos1; + mf->prev_positions[key3] = pos1; + newpos1 = mf->prev_positions[key4]; + mf->prev_positions[key4] = pos1; + + for (count = mf->cycles;;) { + int delta; + if (newpos1 <= min_pos || --count < 0) { + *ptr0 = *ptr1 = 0; + break; + } + delta = pos1 - newpos1; + newptr = mf->pos_array + + ((mf->cyclic_pos - delta + + ((mf->cyclic_pos >= delta) ? 0 : mf->dictionary_size + 1)) << 1); + + if (data[len-delta] == data[len]) { + while (++len < len_limit && data[len-delta] == data[len]) {} + if (distances && maxlen < len) + while (maxlen < len) + distances[++maxlen] = delta - 1; + + if (len >= len_limit) { + *ptr0 = newptr[0]; + *ptr1 = newptr[1]; + break; + } + } + if (data[len-delta] < data[len]) { + *ptr0 = newpos1; + ptr0 = newptr + 1; + newpos1 = *ptr0; + len0 = len; + if (len1 < len) len = len1; + } else { + *ptr1 = newpos1; + ptr1 = newptr; + newpos1 = *ptr1; + len1 = len; + if (len0 < len) len = len0; + } + } + if (distances) { + if (distances[3] > distances[4]) + distances[3] = distances[4]; + if (distances[2] > distances[3]) + distances[2] = distances[3]; + } + return maxlen; +} + + +enum { re_buffer_size = 65536 }; + +struct Range_encoder { + uint64_t low; + unsigned long long partial_member_pos; + uint8_t *buffer; /* output buffer */ + int pos; /* current pos in buffer */ + uint32_t range; + unsigned ff_count; + uint8_t cache; + bool write_error; +}; + +static void Re_flush_data(struct Range_encoder * const renc) +{ + if (renc->pos > 0) { + if (full_write(STDOUT_FILENO, renc->buffer, renc->pos) != renc->pos) + renc->write_error = true; + renc->partial_member_pos += renc->pos; + renc->pos = 0; + } +} + +static void Re_put_byte(struct Range_encoder * const renc, const uint8_t b) +{ + renc->buffer[renc->pos] = b; + if (++renc->pos >= re_buffer_size) Re_flush_data(renc); +} + +static void Re_shift_low(struct Range_encoder * const renc) +{ + if(renc->low >> 24 != 0xFF) { + const bool carry = (renc->low > 0xFFFFFFFFU); + Re_put_byte(renc, renc->cache + carry); + for (; renc->ff_count > 0; --renc->ff_count) + Re_put_byte(renc, 0xFF + carry); + renc->cache = renc->low >> 24; + } else + ++renc->ff_count; + renc->low = (renc->low & 0x00FFFFFFU) << 8; +} + +static bool Re_init(struct Range_encoder * const renc) +{ + renc->low = 0; + renc->partial_member_pos = 0; + renc->buffer = (uint8_t *) malloc(re_buffer_size); + if (!renc->buffer) return false; + renc->pos = 0; + renc->range = 0xFFFFFFFFU; + renc->ff_count = 0; + renc->cache = 0; + renc->write_error = false; + return true; +} + +static inline void Re_free(struct Range_encoder * const renc) +{ + free(renc->buffer); +} + +static inline unsigned long long +Re_member_position(const struct Range_encoder * const renc) +{ + return renc->partial_member_pos + renc->pos + renc->ff_count; +} + +static void Re_flush(struct Range_encoder * const renc) +{ + int i; + for (i = 0; i < 5; ++i) Re_shift_low(renc); +} + +static void Re_encode(struct Range_encoder * const renc, + const int symbol, const int num_bits) +{ + unsigned mask; + for (mask = 1 << ( num_bits - 1 ); mask > 0; mask >>= 1) { + renc->range >>= 1; + if (symbol &mask) renc->low += renc->range; + if (renc->range <= 0x00FFFFFFU) { + renc->range <<= 8; + Re_shift_low(renc); + } + } +} + +static void Re_encode_bit(struct Range_encoder * const renc, + Bit_model * const probability, const bool bit) +{ + const uint32_t bound = (renc->range >> bit_model_total_bits) * *probability; + if (!bit) { + renc->range = bound; + *probability += (bit_model_total - *probability) >> bit_model_move_bits; + } else { + renc->low += bound; + renc->range -= bound; + *probability -= *probability >> bit_model_move_bits; + } + if (renc->range <= 0x00FFFFFFU) { + renc->range <<= 8; + Re_shift_low(renc); + } +} + +static void Re_encode_tree(struct Range_encoder * const renc, + Bit_model bm[], const int symbol, + const int num_bits) +{ + int model = 1; + int i; + for (i = num_bits - 1; i >= 0; --i) { + const bool bit = (symbol >> i) & 1; + Re_encode_bit(renc, &bm[model], bit); + model = (model << 1) | bit; + } +} + +static void Re_encode_tree_reversed(struct Range_encoder * const renc, + Bit_model bm[], int symbol, const int num_bits) +{ + int model = 1; + int i; + for (i = num_bits; i > 0; --i) { + const bool bit = symbol & 1; + symbol >>= 1; + Re_encode_bit(renc, &bm[model], bit); + model = (model << 1) | bit; + } +} + +static void Re_encode_matched(struct Range_encoder * const renc, + Bit_model bm[], unsigned symbol, + unsigned match_byte) +{ + unsigned mask = 0x100; + symbol |= mask; + + while (true) { + const unsigned match_bit = (match_byte <<= 1) & mask; + const bool bit = (symbol <<= 1) & 0x100; + Re_encode_bit( renc, &bm[(symbol>>9)+match_bit+mask], bit ); + if (symbol >= 0x10000) break; + mask &= ~(match_bit ^ symbol); /* if( match_bit != bit ) mask = 0; */ + } +} + +static void Re_encode_len( struct Range_encoder * const renc, + struct Len_model * const lm, + int symbol, const int pos_state ) +{ + bool bit = ( ( symbol -= min_match_len ) >= len_low_symbols ); + Re_encode_bit( renc, &lm->choice1, bit ); + if( !bit ) + Re_encode_tree( renc, lm->bm_low[pos_state], symbol, len_low_bits ); + else { + bit = ((symbol -= len_low_symbols) >= len_mid_symbols ); + Re_encode_bit( renc, &lm->choice2, bit ); + if( !bit ) + Re_encode_tree( renc, lm->bm_mid[pos_state], + symbol, len_mid_bits ); + else + Re_encode_tree( renc, lm->bm_high, + symbol - len_mid_symbols, len_high_bits ); + } +} + + +struct Len_encoder { + struct Len_model lm; + int len_symbols; + int prices[pos_states][max_len_symbols]; + int counters[pos_states]; +}; + +static void Lee_update_prices(struct Len_encoder * const le, const int pos_state) +{ + int * const pps = le->prices[pos_state]; + int tmp = price0(le->lm.choice1); + int len = 0; + + for (; len < len_low_symbols && len < le->len_symbols; ++len) + pps[len] = tmp + + price_symbol(le->lm.bm_low[pos_state], len, len_low_bits); + tmp = price1(le->lm.choice1); + for (; len < len_low_symbols + len_mid_symbols && len < le->len_symbols; ++len) + pps[len] = tmp + price0(le->lm.choice2) + + price_symbol(le->lm.bm_mid[pos_state], + len - len_low_symbols, len_mid_bits); + for (; len < le->len_symbols; ++len) + /* using 4 slots per value makes "Lee_price" faster */ + le->prices[3][len] = le->prices[2][len] = + le->prices[1][len] = le->prices[0][len] = + tmp + price1(le->lm.choice2) + + price_symbol(le->lm.bm_high, + len - len_low_symbols - len_mid_symbols, + len_high_bits); + le->counters[pos_state] = le->len_symbols; +} + +static void Lee_init(struct Len_encoder * const le, const int len_limit) +{ + int i; + Lm_init(&le->lm); + le->len_symbols = len_limit + 1 - min_match_len; + for (i = 0; i < pos_states; ++i) Lee_update_prices(le, i); +} + +static void Lee_encode(struct Len_encoder * const le, + struct Range_encoder * const renc, + int symbol, const int pos_state) +{ + Re_encode_len(renc, &le->lm, symbol, pos_state); + if (--le->counters[pos_state] <= 0) + Lee_update_prices(le, pos_state); +} + +static int Lee_price(const struct Len_encoder * const le, + const int symbol, const int pos_state) +{ + return le->prices[pos_state][symbol - min_match_len]; +} + + +enum { infinite_price = 0x0FFFFFFF, + num_rep_distances = 4 /* must be 4 */ +}; + +struct Trial { + State state; + int price; /* dual use var; cumulative price, match length */ + int dis; /* rep index or match distance. (-1 for literal) */ + int prev_index; /* index of prev trial in trials[] */ + int reps[num_rep_distances]; +}; + +static void Tr_update(struct Trial * const trial, const int pr, + const int distance, const int p_i) +{ + if (pr < trial->price) { + trial->price = pr; + trial->dis = distance; + trial->prev_index = p_i; + } +} + + +struct LZ_encoder { + int longest_match_found; + uint32_t crc; + + Bit_model bm_literal[1 << literal_context_bits][0x300]; + Bit_model bm_match[states][pos_states]; + Bit_model bm_rep[states]; + Bit_model bm_rep0[states]; + Bit_model bm_rep1[states]; + Bit_model bm_rep2[states]; + Bit_model bm_len[states][pos_states]; + Bit_model bm_dis_slot[len_states][1 << dis_slot_bits]; + Bit_model bm_dis[modeled_distances-end_dis_model+1]; + Bit_model bm_align[dis_align_size]; + + struct Matchfinder *matchfinder; + struct Range_encoder renc; + struct Len_encoder match_len_encoder; + struct Len_encoder rep_len_encoder; + + int match_distances[max_match_len+1]; + struct Trial trials[max_num_trials]; + + int dis_slot_prices[len_states][2*max_dictionary_bits]; + int dis_prices[len_states][modeled_distances]; + int align_prices[dis_align_size]; + int align_price_count; + int num_dis_slots; +}; + +static void LZe_fill_align_prices(struct LZ_encoder * const e) +{ + int i; + for (i = 0; i < dis_align_size; ++i) + e->align_prices[i] = + price_symbol_reversed(e->bm_align, i, dis_align_bits); + e->align_price_count = dis_align_size; +} + +static bool LZe_init(struct LZ_encoder * const e, + struct Matchfinder * const mf, const Lzip_header header) +{ + int i; + e->longest_match_found = 0; + e->crc = 0xFFFFFFFFU; + Bm_array_init(&e->bm_literal[0][0], lz_num_models); + e->matchfinder = mf; + if (!Re_init(&e->renc)) return false; + Lee_init(&e->match_len_encoder, e->matchfinder->match_len_limit); + Lee_init(&e->rep_len_encoder, e->matchfinder->match_len_limit); + LZe_fill_align_prices(e); + e->num_dis_slots = 2 * real_bits(e->matchfinder->dictionary_size - 1); + e->trials[1].prev_index = 0; + for (i = 0; i < Lh_size; ++i) + Re_put_byte(&e->renc, header[i]); + return true; +} + +static inline void LZe_free(struct LZ_encoder * const e) +{ + Re_free(&e->renc); +} + +static inline unsigned LZe_crc(const struct LZ_encoder * const e) +{ + return e->crc ^ 0xFFFFFFFFU; +} + + /* move-to-front dis in/into reps; do nothing if( dis4 <= 0 ) */ +static void mtf_reps(const int dis4, int reps[num_rep_distances]) +{ + if (dis4 >= num_rep_distances) { /* match */ + reps[3] = reps[2]; reps[2] = reps[1]; reps[1] = reps[0]; + reps[0] = dis4 - num_rep_distances; + } else if (dis4 > 0) { /* repeated match */ + const int distance = reps[dis4]; + int i; for (i = dis4; i > 0; --i) + reps[i] = reps[i - 1]; + reps[0] = distance; + } +} + +static int LZe_price_shortrep(const struct LZ_encoder * const e, + const State state, const int pos_state) +{ + return price0(e->bm_rep0[state]) + price0(e->bm_len[state][pos_state]); +} + +static int LZe_price_rep(const struct LZ_encoder * const e, const int rep, + const State state, const int pos_state) +{ + int price; + if (rep == 0) + return price0(e->bm_rep0[state]) + + price1(e->bm_len[state][pos_state]); + price = price1(e->bm_rep0[state]); + if (rep == 1) + price += price0(e->bm_rep1[state]); + else { + price += price1(e->bm_rep1[state]); + price += price_bit(e->bm_rep2[state], rep - 2); + } + return price; +} + +static int LZe_price_dis(const struct LZ_encoder * const e, + const int dis, const int len_state) +{ + if (dis < modeled_distances) + return e->dis_prices[len_state][dis]; + else + return e->dis_slot_prices[len_state][get_slot(dis)] + + e->align_prices[dis & (dis_align_size - 1)]; +} + +static int LZe_price_pair(const struct LZ_encoder * const e, + const int dis, const int len, + const int pos_state) +{ + if (len <= min_match_len && dis >= modeled_distances) + return infinite_price; + return Lee_price(&e->match_len_encoder, len, pos_state) + + LZe_price_dis(e, dis, get_len_state(len)); +} + +static int LZe_price_literal(const struct LZ_encoder * const e, + uint8_t prev_byte, uint8_t symbol) +{ + return price_symbol(e->bm_literal[get_lit_state(prev_byte)], symbol, 8); +} + +static int LZe_price_matched(const struct LZ_encoder * const e, + uint8_t prev_byte, uint8_t symbol, + uint8_t match_byte) +{ + return price_matched(e->bm_literal[get_lit_state(prev_byte)], symbol, + match_byte); +} + +static void LZe_encode_literal(struct LZ_encoder * const e, + uint8_t prev_byte, uint8_t symbol) +{ + Re_encode_tree(&e->renc, + e->bm_literal[get_lit_state(prev_byte)], symbol, 8); +} + +static void LZe_encode_matched(struct LZ_encoder * const e, + uint8_t prev_byte, uint8_t symbol, + uint8_t match_byte) +{ + Re_encode_matched(&e->renc, e->bm_literal[get_lit_state(prev_byte)], + symbol, match_byte); +} + +static void LZe_encode_pair(struct LZ_encoder * const e, + const unsigned dis, const int len, + const int pos_state) +{ + const int dis_slot = get_slot(dis); + Lee_encode(&e->match_len_encoder, &e->renc, len, pos_state); + Re_encode_tree(&e->renc, e->bm_dis_slot[get_len_state(len)], dis_slot, + dis_slot_bits); + + if (dis_slot >= start_dis_model) { + const int direct_bits = (dis_slot >> 1) - 1; + const unsigned base = (2 | (dis_slot & 1)) << direct_bits; + const unsigned direct_dis = dis - base; + + if (dis_slot < end_dis_model) + Re_encode_tree_reversed(&e->renc, + e->bm_dis + (base - dis_slot), + direct_dis, direct_bits); + else { + Re_encode(&e->renc, direct_dis >> dis_align_bits, + direct_bits - dis_align_bits); + Re_encode_tree_reversed(&e->renc, e->bm_align, + direct_dis, dis_align_bits); + if (--e->align_price_count <= 0) + LZe_fill_align_prices(e); + } + } +} + +static int LZe_read_match_distances(struct LZ_encoder * const e) +{ + int len = Mf_longest_match_len(e->matchfinder, e->match_distances); + if (len == e->matchfinder->match_len_limit && len < max_match_len) + len = Mf_true_match_len(e->matchfinder, len, + e->match_distances[len] + 1); + return len; +} + +static void LZe_move_pos(struct LZ_encoder * const e, int n) +{ + while (true) { + Mf_move_pos(e->matchfinder); + if( --n <= 0 ) break; + Mf_longest_match_len(e->matchfinder, 0); + } +} + +static void LZe_backward(struct LZ_encoder * const e, int cur) +{ + int * const dis = &e->trials[cur].dis; + while (cur > 0) { + const int prev_index = e->trials[cur].prev_index; + struct Trial * const prev_trial = &e->trials[prev_index]; + prev_trial->price = cur - prev_index; /* len */ + cur = *dis; + *dis = prev_trial->dis; + prev_trial->dis = cur; + cur = prev_index; + } +} + + /* End Of Stream mark => (dis == 0xFFFFFFFFU, len == min_match_len) */ +static void LZe_full_flush(struct LZ_encoder * const e, const State state) +{ + int i; + const int pos_state = Mf_data_position(e->matchfinder) & pos_state_mask; + Lzip_trailer trailer; + Re_encode_bit(&e->renc, &e->bm_match[state][pos_state], 1); + Re_encode_bit(&e->renc, &e->bm_rep[state], 0); + LZe_encode_pair(e, 0xFFFFFFFFU, min_match_len, pos_state); + Re_flush(&e->renc); + Lt_set_data_crc(trailer, LZe_crc(e)); + Lt_set_data_size(trailer, Mf_data_position(e->matchfinder)); + Lt_set_member_size(trailer, Re_member_position(&e->renc) + Lt_size); + for (i = 0; i < Lt_size; ++i) + Re_put_byte(&e->renc, trailer[i]); + Re_flush_data(&e->renc); +} + + +static void LZe_update_distance_prices(struct LZ_encoder * const e) +{ + int dis, len_state; + for (dis = start_dis_model; dis < modeled_distances; ++dis) { + const int dis_slot = dis_slots[dis]; + const int direct_bits = (dis_slot >> 1) - 1; + const int base = (2 | (dis_slot & 1)) << direct_bits; + const int price = + price_symbol_reversed(e->bm_dis + (base - dis_slot), + dis - base, direct_bits); + for (len_state = 0; len_state < len_states; ++len_state) + e->dis_prices[len_state][dis] = price; + } + + for (len_state = 0; len_state < len_states; ++len_state) { + int * const dsp = e->dis_slot_prices[len_state]; + int * const dp = e->dis_prices[len_state]; + const Bit_model * const bmds = e->bm_dis_slot[len_state]; + int slot = 0; + for (; slot < end_dis_model; ++slot) + dsp[slot] = price_symbol(bmds, slot, dis_slot_bits); + for (; slot < e->num_dis_slots; ++slot) + dsp[slot] = price_symbol(bmds, slot, dis_slot_bits) + + ((((slot >> 1) - 1) - dis_align_bits) << price_shift_bits); + + for (dis = 0; dis < start_dis_model; ++dis) + dp[dis] = dsp[dis]; + for (; dis < modeled_distances; ++dis) + dp[dis] += dsp[dis_slots[dis]]; + } +} + + +/* Returns the number of bytes advanced (ahead). + trials[0]..trials[ahead-1] contain the steps to encode. + ( trials[0].dis == -1 && trials[0].price == 1 ) means literal. + A match/rep longer or equal than match_len_limit finishes the sequence. +*/ +static int LZe_sequence_optimizer(struct LZ_encoder * const e, + const int reps[num_rep_distances], + const State state) +{ + int main_len, i, rep, cur = 0, num_trials; + int replens[num_rep_distances]; + int rep_index = 0; + + if (e->longest_match_found > 0) { /* from previous call */ + main_len = e->longest_match_found; + e->longest_match_found = 0; + } else + main_len = LZe_read_match_distances(e); + + for (i = 0; i < num_rep_distances; ++i) { + replens[i] = Mf_true_match_len(e->matchfinder, 0, reps[i] + 1); + if (replens[i] > replens[rep_index]) rep_index = i; + } + if (replens[rep_index] >= e->matchfinder->match_len_limit) { + e->trials[0].dis = rep_index; + e->trials[0].price = replens[rep_index]; + LZe_move_pos(e, replens[rep_index]); + return replens[rep_index]; + } + + if (main_len >= e->matchfinder->match_len_limit) { + e->trials[0].dis = + e->match_distances[e->matchfinder->match_len_limit] + + num_rep_distances; + e->trials[0].price = main_len; + LZe_move_pos(e, main_len); + return main_len; + } + + { + const int pos_state = Mf_data_position(e->matchfinder) & pos_state_mask; + const int match_price = price1(e->bm_match[state][pos_state]); + const int rep_match_price = match_price + price1(e->bm_rep[state]); + const uint8_t prev_byte = Mf_peek(e->matchfinder, 1); + const uint8_t cur_byte = Mf_peek(e->matchfinder, 0); + const uint8_t match_byte = Mf_peek(e->matchfinder, reps[0] + 1); + + e->trials[1].price = price0(e->bm_match[state][pos_state]); + if (St_is_char(state)) + e->trials[1].price += + LZe_price_literal(e, prev_byte, cur_byte); + else + e->trials[1].price += + LZe_price_matched(e, prev_byte, cur_byte, match_byte); + e->trials[1].dis = -1; /* literal */ + + if (match_byte == cur_byte) + Tr_update(&e->trials[1], rep_match_price + + LZe_price_shortrep(e, state, pos_state), 0, 0); + + if (main_len < min_match_len) { + e->trials[0].dis = e->trials[1].dis; + e->trials[0].price = 1; + Mf_move_pos(e->matchfinder); + return 1; + } + + e->trials[0].state = state; + for (i = 0; i < num_rep_distances; ++i) + e->trials[0].reps[i] = reps[i]; + + if (main_len <= replens[rep_index]) { + int len; + + main_len = replens[rep_index]; + for (len = min_match_len; len <= main_len; ++len) + e->trials[len].price = infinite_price; + } else { + int len; + const int normal_match_price = + match_price + price0(e->bm_rep[state]); + for (len = min_match_len; len <= main_len; ++len) { + e->trials[len].dis = + e->match_distances[len] + num_rep_distances; + e->trials[len].prev_index = 0; + e->trials[len].price = normal_match_price + + LZe_price_pair(e, e->match_distances[len], + len, pos_state); + } + } + + for (rep = 0; rep < num_rep_distances; ++rep) { + const int price = rep_match_price + + LZe_price_rep(e, rep, state, pos_state); + int len; + for (len = min_match_len; len <= replens[rep]; ++len) + Tr_update(&e->trials[len], price + + Lee_price(&e->rep_len_encoder, len, pos_state), + rep, 0); + } + } + + num_trials = main_len; + + while (true) { /* price optimization loop */ + struct Trial *cur_trial, *next_trial; + int newlen, pos_state, prev_index, len_limit; + int next_price, match_price, rep_match_price; + uint8_t prev_byte, cur_byte, match_byte; + + Mf_move_pos(e->matchfinder); + if (++cur >= num_trials) { /* no more initialized trials */ + LZe_backward(e, cur); + return cur; + } + newlen = LZe_read_match_distances(e); + if (newlen >= e->matchfinder->match_len_limit) { + e->longest_match_found = newlen; + LZe_backward(e, cur); + return cur; + } + + /* give final values to current trial */ + cur_trial = &e->trials[cur]; + prev_index = cur_trial->prev_index; + cur_trial->state = e->trials[prev_index].state; + + for (i = 0; i < num_rep_distances; ++i) + cur_trial->reps[i] = e->trials[prev_index].reps[i]; + + if (prev_index == cur - 1) { + if (cur_trial->dis == 0) + cur_trial->state = St_set_short_rep(cur_trial->state); + else + cur_trial->state = St_set_char(cur_trial->state); + } else { + if (cur_trial->dis < num_rep_distances) + cur_trial->state = St_set_rep(cur_trial->state); + else + cur_trial->state = St_set_match(cur_trial->state); + mtf_reps(cur_trial->dis, cur_trial->reps); + } + + pos_state = Mf_data_position(e->matchfinder) & pos_state_mask; + prev_byte = Mf_peek(e->matchfinder, 1); + cur_byte = Mf_peek(e->matchfinder, 0); + match_byte = Mf_peek(e->matchfinder, cur_trial->reps[0] + 1); + + next_price = cur_trial->price + + price0(e->bm_match[cur_trial->state][pos_state]); + if (St_is_char(cur_trial->state)) + next_price += LZe_price_literal(e, prev_byte, cur_byte); + else + next_price += LZe_price_matched(e, prev_byte, cur_byte, + match_byte); + /* try last updates to next trial */ + next_trial = &e->trials[cur + 1]; + + Tr_update(next_trial, next_price, -1, cur); /* literal */ + + match_price = cur_trial->price + + price1(e->bm_match[cur_trial->state][pos_state]); + rep_match_price = match_price + price1(e->bm_rep[cur_trial->state]); + + if (match_byte == cur_byte && next_trial->dis != 0) + Tr_update(next_trial, rep_match_price + + LZe_price_shortrep(e, cur_trial->state, + pos_state), 0, cur); + + len_limit = MIN(MIN(max_num_trials - 1 - cur, + Mf_available_bytes(e->matchfinder)), + e->matchfinder->match_len_limit); + if (len_limit < min_match_len) continue; + + for (rep = 0; rep < num_rep_distances; ++rep) { + const int dis = cur_trial->reps[rep] + 1; + int len = 0; + const uint8_t * const data = + Mf_ptr_to_current_pos(e->matchfinder); + while (len < len_limit && data[len] == data[len - dis]) + ++len; + if (len >= min_match_len) { + const int price = rep_match_price + + LZe_price_rep(e, rep, cur_trial->state, pos_state); + while (num_trials < cur + len) + e->trials[++num_trials].price = infinite_price; + for (; len >= min_match_len; --len) + Tr_update(&e->trials[cur + len], price + + Lee_price(&e->rep_len_encoder, len, + pos_state), rep, cur); + } + } + + if (newlen <= len_limit && + (newlen > min_match_len || + (newlen == min_match_len && + e->match_distances[min_match_len] < modeled_distances))) { + const int normal_match_price = match_price + + price0(e->bm_rep[cur_trial->state]); + int len; + int dis = e->match_distances[min_match_len]; + int len_state = get_len_state(min_match_len); + int dis_price = infinite_price; + + while (num_trials < cur + newlen) + e->trials[++num_trials].price = infinite_price; + + if (dis < modeled_distances) + Tr_update(&e->trials[cur + min_match_len], + normal_match_price + + e->dis_prices[len_state][dis] + + Lee_price(&e->match_len_encoder, + min_match_len, pos_state), + dis + num_rep_distances, cur); + + for (len = min_match_len + 1; len <= newlen; ++len) { + if (dis != e->match_distances[len] || + len_state < len_states - 1) { + dis = e->match_distances[len]; + len_state = get_len_state(len); + dis_price = LZe_price_dis(e, dis, len_state); + } + Tr_update(&e->trials[cur + len], + normal_match_price + dis_price + + Lee_price(&e->match_len_encoder, len, pos_state), + dis + num_rep_distances, cur); + } + } + } +} + + +static bool LZe_encode_member(struct LZ_encoder * const e) +{ + const int dis_price_count = + (e->matchfinder->match_len_limit > 12) ? 512 : 2048; + int dis_price_counter = 0; + int ahead, i; + int reps[num_rep_distances]; + State state = 0; + for (i = 0; i < num_rep_distances; ++i) reps[i] = 0; + + if (!Mf_finished(e->matchfinder)) { /* encode first byte */ + const uint8_t prev_byte = 0; + const uint8_t cur_byte = Mf_peek(e->matchfinder, 0); + Re_encode_bit(&e->renc, &e->bm_match[state][0], 0); + LZe_encode_literal(e, prev_byte, cur_byte); + CRC32_update_byte(&e->crc, cur_byte); + Mf_longest_match_len(e->matchfinder, 0); + Mf_move_pos(e->matchfinder); + } + + while (!Mf_finished(e->matchfinder)) { + if (dis_price_counter <= 0) { + LZe_update_distance_prices(e); + dis_price_counter = dis_price_count; + } + + ahead = LZe_sequence_optimizer(e, reps, state); + dis_price_counter -= ahead; + + for (i = 0; ahead > 0;) { + const int pos_state = + (Mf_data_position(e->matchfinder) - ahead) & pos_state_mask; + const int dis = e->trials[i].dis; + const int len = e->trials[i].price; + + bool bit = (dis < 0); + Re_encode_bit(&e->renc, &e->bm_match[state][pos_state], !bit); + if (bit) { /* literal byte */ + const uint8_t prev_byte = Mf_peek(e->matchfinder, ahead + 1); + const uint8_t cur_byte = Mf_peek(e->matchfinder, ahead); + CRC32_update_byte(&e->crc, cur_byte); + if (St_is_char(state)) + LZe_encode_literal(e, prev_byte, cur_byte); + else { + const uint8_t match_byte = + Mf_peek(e->matchfinder, ahead + reps[0] + 1); + LZe_encode_matched(e, prev_byte, cur_byte, match_byte); + } + state = St_set_char(state); + } else { /* match or repeated match */ + + e->crc = crc32_block_endian0(e->crc, + Mf_ptr_to_current_pos(e->matchfinder) - ahead, + len, global_crc32_table); + mtf_reps(dis, reps); + bit = (dis < num_rep_distances); + Re_encode_bit(&e->renc, &e->bm_rep[state], bit); + if (bit) { /* repeated match */ + bit = (dis == 0); + Re_encode_bit(&e->renc, &e->bm_rep0[state], !bit); + if (bit) + Re_encode_bit(&e->renc, &e->bm_len[state][pos_state], len > 1); + else { + Re_encode_bit(&e->renc, &e->bm_rep1[state], dis > 1); + if (dis > 1) + Re_encode_bit(&e->renc, &e->bm_rep2[state], dis > 2); + } + if (len == 1) + state = St_set_short_rep(state); + else { + Lee_encode(&e->rep_len_encoder, + &e->renc, len, pos_state); + state = St_set_rep(state); + } + } else { /* match */ + LZe_encode_pair(e, dis - num_rep_distances, len, pos_state); + state = St_set_match(state); + } + } + ahead -= len; + i += len; + } + } + LZe_full_flush(e, state); + return !e->renc.write_error; +} + + +struct Lzma_options { + int dictionary_size; /* 4KiB..512MiB */ + int match_len_limit; /* 5..273 */ +} encoder_options; + + +static int getnum(const char * const ptr, const int llimit, const int ulimit) +{ + long result; + char *tail; + errno = 0; + result = strtol(ptr, &tail, 0); + if (tail == ptr || errno) + goto error; + if (tail[0]) { + int factor = (tail[1] == 'i') ? 1024 : 1000; + int exponent = 0, i; + + switch (tail[0]) { + case 'M': + exponent = 2; + break; + case 'K': + if (factor == 1024) { + exponent = 1; + break; + } + goto error; + case 'k': + if (factor == 1000) { + exponent = 1; + break; + } + default: + goto error; + } + for (i = 0; i < exponent; ++i) { + if (LONG_MAX / factor >= labs(result)) + result *= factor; + else + goto error; + } + } + if (result >= llimit && result <= ulimit) + return result; + error: + bb_error_msg_and_die("invalid number"); +} + + +static int get_dict_size(const char * const arg) +{ + char *tail; + long bits = strtol(arg, &tail, 0); + if (bits >= min_dictionary_bits && + bits <= max_dictionary_bits && *tail == 0) + return (1 << bits); + return getnum(arg, min_dictionary_size, max_dictionary_size); +} + + +static IF_DESKTOP(long long) int FAST_FUNC pack_lzip(transformer_state_t *xstate UNUSED_PARAM) +{ + int retval = 0; + Lzip_header header; + struct Matchfinder matchfinder; + struct LZ_encoder * encoder; + + Lh_set_magic(header); + if (!Lh_set_dictionary_size(header, encoder_options.dictionary_size) || + encoder_options.match_len_limit < min_match_len_limit || + encoder_options.match_len_limit > max_match_len) + bb_error_msg_and_die("internal error"); + + if (!Mf_init(&matchfinder, Lh_get_dictionary_size(header), + encoder_options.match_len_limit)) { + bb_error_msg(bb_msg_memory_exhausted); + return -1; + } + Lh_set_dictionary_size(header, matchfinder.dictionary_size); + + encoder = malloc(sizeof(struct LZ_encoder)); + if (!encoder || !LZe_init(encoder, &matchfinder, header)) { + bb_error_msg(bb_msg_memory_exhausted); + retval = -1; + } else { + if (!LZe_encode_member(encoder)) { + bb_perror_msg(bb_msg_write_error); + retval = -1; + } + LZe_free(encoder); + } + free(encoder); + Mf_free(&matchfinder); + return retval; +} + + +int lzip_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; +#if ENABLE_FEATURE_LZIP_DECOMPRESS +int lzip_main(int argc, char **argv) +#else +int lzip_main(int argc UNUSED_PARAM, char **argv) +#endif +{ + /* Mapping from gzip/bzip2 style 1..9 compression modes + to the corresponding LZMA compression modes. */ + const struct Lzma_options option_mapping[] = { + {1 << 16, 16}, /* -0 */ + {1 << 20, 5}, /* -1 */ + {3 << 19, 6}, /* -2 */ + {1 << 21, 8}, /* -3 */ + {3 << 20, 12}, /* -4 */ + {1 << 22, 20}, /* -5 */ + {1 << 23, 36}, /* -6 */ + {1 << 24, 68}, /* -7 */ + {3 << 23, 132}, /* -8 */ + {1 << 25, 273} /* -9 */ + }; + int i; + char *m_arg; + char *s_arg; + /* Must match BBUNPK_foo constants in include/bb_archive.h! */ + unsigned opt = getopt32(argv, BBUNPK_OPTSTR IF_FEATURE_LZIP_DECOMPRESS("dt") "0123456789Fm:s:", &m_arg, &s_arg); + +#if ENABLE_FEATURE_LZIP_DECOMPRESS /* lunzip_main may not be visible... */ + if (opt & (BBUNPK_OPT_DECOMPRESS|BBUNPK_OPT_TEST)) /* -d and/or -t */ + return lunzip_main(argc, argv); +#endif + /* drop BBUNPK_OPTSTR "dt" bits */ + opt >>= (BBUNPK_OPTSTRLEN IF_FEATURE_LZIP_DECOMPRESS(+ 2)); + + encoder_options = option_mapping[6]; /* default = "-6" */ + + for (i = 9; i >= 7; --i) + if (opt & (1 << i)) + encoder_options = option_mapping[i]; + for (i = 0; i <= 6; ++i) + if (opt & (1 << i)) + encoder_options = option_mapping[i]; + if (opt & (1 << 11)) /* -m */ + encoder_options.match_len_limit = + getnum(m_arg, min_match_len_limit, max_match_len); + if (opt & (1 << 12)) /* -s */ + encoder_options.dictionary_size = get_dict_size(s_arg); + /* end process options */ + option_mask32 &= BBUNPK_OPTSTRMASK; /* retain only -cfkvq */ + + /* Initialize the CRC32 table */ + global_crc32_new_table_le(); + if (!dis_slots) { + Dis_slots_init(); + Prob_prices_init(); + } + + argv += optind; + return bbunpack(argv, pack_lzip, append_ext, "lz"); +} --- archival/tar.c +++ archival/tar.c @@ -43,7 +43,7 @@ //config:config FEATURE_TAR_AUTODETECT //config: bool "Autodetect compressed tarballs" //config: default y -//config: depends on TAR && (FEATURE_SEAMLESS_Z || FEATURE_SEAMLESS_GZ || FEATURE_SEAMLESS_BZ2 || FEATURE_SEAMLESS_LZMA || FEATURE_SEAMLESS_XZ) +//config: depends on TAR && (FEATURE_SEAMLESS_Z || FEATURE_SEAMLESS_GZ || FEATURE_SEAMLESS_BZ2 || FEATURE_SEAMLESS_LZ || FEATURE_SEAMLESS_LZMA || FEATURE_SEAMLESS_XZ) //config: help //config: With this option tar can automatically detect compressed //config: tarballs. Currently it works only on files (not pipes etc). @@ -770,6 +770,7 @@ //usage: IF_FEATURE_SEAMLESS_GZ("z") //usage: IF_FEATURE_SEAMLESS_XZ("J") //usage: IF_FEATURE_SEAMLESS_BZ2("j") +//usage: IF_FEATURE_SEAMLESS_LZ("y") //usage: "a" //usage: IF_FEATURE_TAR_CREATE("h") //usage: IF_FEATURE_TAR_NOPRESERVE_TIME("m") @@ -809,6 +810,9 @@ //usage: IF_FEATURE_SEAMLESS_BZ2( //usage: "\n -j (De)compress using bzip2" //usage: ) +//usage: IF_FEATURE_SEAMLESS_LZ( +//usage: "\n -y (De)compress using lzip" +//usage: ) //usage: IF_FEATURE_SEAMLESS_LZMA( //usage: IF_FEATURE_TAR_LONG_OPTIONS( //usage: "\n --lzma (De)compress using lzma" @@ -849,7 +853,8 @@ IF_FEATURE_TAR_FROM( OPTBIT_EXCLUDE_FROM,) IF_FEATURE_SEAMLESS_GZ( OPTBIT_GZIP ,) IF_FEATURE_SEAMLESS_XZ( OPTBIT_XZ ,) - IF_FEATURE_SEAMLESS_Z( OPTBIT_COMPRESS ,) // 16th bit + IF_FEATURE_SEAMLESS_LZ( OPTBIT_LZIP ,) // 16th bit + IF_FEATURE_SEAMLESS_Z( OPTBIT_COMPRESS ,) OPTBIT_AUTOCOMPRESS_BY_EXT, IF_FEATURE_TAR_NOPRESERVE_TIME(OPTBIT_NOPRESERVE_TIME,) #if ENABLE_FEATURE_TAR_LONG_OPTIONS @@ -877,6 +882,7 @@ OPT_EXCLUDE_FROM = IF_FEATURE_TAR_FROM( (1 << OPTBIT_EXCLUDE_FROM)) + 0, // X OPT_GZIP = IF_FEATURE_SEAMLESS_GZ( (1 << OPTBIT_GZIP )) + 0, // z OPT_XZ = IF_FEATURE_SEAMLESS_XZ( (1 << OPTBIT_XZ )) + 0, // J + OPT_LZIP = IF_FEATURE_SEAMLESS_LZ( (1 << OPTBIT_LZIP )) + 0, // y OPT_COMPRESS = IF_FEATURE_SEAMLESS_Z( (1 << OPTBIT_COMPRESS )) + 0, // Z OPT_AUTOCOMPRESS_BY_EXT = 1 << OPTBIT_AUTOCOMPRESS_BY_EXT, // a OPT_NOPRESERVE_TIME = IF_FEATURE_TAR_NOPRESERVE_TIME((1 << OPTBIT_NOPRESERVE_TIME)) + 0, // m @@ -888,7 +894,7 @@ OPT_NOPRESERVE_PERM = IF_FEATURE_TAR_LONG_OPTIONS((1 << OPTBIT_NOPRESERVE_PERM)) + 0, // no-same-permissions OPT_OVERWRITE = IF_FEATURE_TAR_LONG_OPTIONS((1 << OPTBIT_OVERWRITE )) + 0, // overwrite - OPT_ANY_COMPRESS = (OPT_BZIP2 | OPT_LZMA | OPT_GZIP | OPT_XZ | OPT_COMPRESS), + OPT_ANY_COMPRESS = (OPT_BZIP2 | OPT_LZMA | OPT_GZIP | OPT_XZ | OPT_LZIP | OPT_COMPRESS), }; #if ENABLE_FEATURE_TAR_LONG_OPTIONS static const char tar_longopts[] ALIGN1 = @@ -918,6 +924,9 @@ # if ENABLE_FEATURE_SEAMLESS_GZ "gzip\0" No_argument "z" # endif +# if ENABLE_FEATURE_SEAMLESS_LZ + "lzip\0" No_argument "y" +# endif # if ENABLE_FEATURE_SEAMLESS_XZ "xz\0" No_argument "J" # endif @@ -1023,6 +1032,7 @@ IF_FEATURE_TAR_FROM( "T:*X:*") IF_FEATURE_SEAMLESS_GZ( "z" ) IF_FEATURE_SEAMLESS_XZ( "J" ) + IF_FEATURE_SEAMLESS_LZ( "y" ) IF_FEATURE_SEAMLESS_Z( "Z" ) "a" IF_FEATURE_TAR_NOPRESERVE_TIME("m") @@ -1072,6 +1082,7 @@ showopt(OPT_EXCLUDE_FROM ); showopt(OPT_GZIP ); showopt(OPT_XZ ); + showopt(OPT_LZIP ); showopt(OPT_COMPRESS ); showopt(OPT_AUTOCOMPRESS_BY_EXT); showopt(OPT_NOPRESERVE_TIME ); @@ -1229,6 +1240,8 @@ zipMode = "gzip"; if (opt & OPT_BZIP2) zipMode = "bzip2"; + if (opt & OPT_LZIP) + zipMode = "lzip"; if (opt & OPT_LZMA) zipMode = "lzma"; if (opt & OPT_XZ) @@ -1265,6 +1278,10 @@ USE_FOR_MMU(IF_FEATURE_SEAMLESS_BZ2(xformer = unpack_bz2_stream;)) USE_FOR_NOMMU(xformer_prog = "bunzip2";) } + if (opt & OPT_LZIP) { + USE_FOR_MMU(IF_FEATURE_SEAMLESS_LZ(xformer = unpack_lz_stream;)) + USE_FOR_NOMMU(xformer_prog = "lunzip";) + } if (opt & OPT_LZMA) { USE_FOR_MMU(IF_FEATURE_SEAMLESS_LZMA(xformer = unpack_lzma_stream;)) USE_FOR_NOMMU(xformer_prog = "unlzma";) --- configs/TEST_nommu_defconfig +++ configs/TEST_nommu_defconfig @@ -107,6 +107,7 @@ CONFIG_FEATURE_SEAMLESS_LZMA=y CONFIG_FEATURE_SEAMLESS_BZ2=y CONFIG_FEATURE_SEAMLESS_GZ=y +CONFIG_FEATURE_SEAMLESS_LZ=y CONFIG_FEATURE_SEAMLESS_Z=y CONFIG_AR=y CONFIG_FEATURE_AR_LONG_FILENAMES=y @@ -120,6 +121,8 @@ CONFIG_GUNZIP=y CONFIG_GZIP=y CONFIG_FEATURE_GZIP_LONG_OPTIONS=y +CONFIG_LUNZIP=y +CONFIG_LZIP=y CONFIG_LZOP=y CONFIG_LZOP_COMPR_HIGH=y CONFIG_RPM2CPIO=y --- configs/TEST_noprintf_defconfig +++ configs/TEST_noprintf_defconfig @@ -118,6 +118,7 @@ CONFIG_FEATURE_SEAMLESS_LZMA=y CONFIG_FEATURE_SEAMLESS_BZ2=y CONFIG_FEATURE_SEAMLESS_GZ=y +CONFIG_FEATURE_SEAMLESS_LZ=y CONFIG_FEATURE_SEAMLESS_Z=y # CONFIG_AR is not set # CONFIG_FEATURE_AR_LONG_FILENAMES is not set @@ -132,6 +133,8 @@ # CONFIG_GUNZIP is not set # CONFIG_GZIP is not set # CONFIG_FEATURE_GZIP_LONG_OPTIONS is not set +# CONFIG_LUNZIP is not set +# CONFIG_LZIP is not set # CONFIG_LZOP is not set # CONFIG_LZOP_COMPR_HIGH is not set # CONFIG_RPM2CPIO is not set --- configs/TEST_rh9_defconfig +++ configs/TEST_rh9_defconfig @@ -116,6 +116,7 @@ CONFIG_FEATURE_SEAMLESS_LZMA=y CONFIG_FEATURE_SEAMLESS_BZ2=y CONFIG_FEATURE_SEAMLESS_GZ=y +CONFIG_FEATURE_SEAMLESS_LZ=y CONFIG_FEATURE_SEAMLESS_Z=y CONFIG_AR=y CONFIG_FEATURE_AR_LONG_FILENAMES=y @@ -130,6 +131,8 @@ CONFIG_GUNZIP=y CONFIG_GZIP=y CONFIG_FEATURE_GZIP_LONG_OPTIONS=y +CONFIG_LUNZIP=y +CONFIG_LZIP=y CONFIG_LZOP=y # CONFIG_LZOP_COMPR_HIGH is not set CONFIG_RPM2CPIO=y --- configs/android2_defconfig +++ configs/android2_defconfig @@ -120,6 +120,7 @@ CONFIG_FEATURE_SEAMLESS_LZMA=y CONFIG_FEATURE_SEAMLESS_BZ2=y CONFIG_FEATURE_SEAMLESS_GZ=y +CONFIG_FEATURE_SEAMLESS_LZ=y CONFIG_FEATURE_SEAMLESS_Z=y CONFIG_AR=y CONFIG_FEATURE_AR_LONG_FILENAMES=y @@ -134,6 +135,8 @@ CONFIG_GUNZIP=y CONFIG_GZIP=y # CONFIG_FEATURE_GZIP_LONG_OPTIONS is not set +CONFIG_LUNZIP=y +CONFIG_LZIP=y CONFIG_LZOP=y CONFIG_LZOP_COMPR_HIGH=y CONFIG_RPM2CPIO=y --- configs/android_defconfig +++ configs/android_defconfig @@ -140,6 +140,7 @@ CONFIG_FEATURE_SEAMLESS_LZMA=y CONFIG_FEATURE_SEAMLESS_BZ2=y CONFIG_FEATURE_SEAMLESS_GZ=y +CONFIG_FEATURE_SEAMLESS_LZ=y CONFIG_FEATURE_SEAMLESS_Z=y CONFIG_AR=y CONFIG_FEATURE_AR_LONG_FILENAMES=y @@ -155,6 +156,8 @@ CONFIG_GZIP=y # CONFIG_FEATURE_GZIP_LONG_OPTIONS is not set CONFIG_GZIP_FAST=0 +CONFIG_LUNZIP=y +CONFIG_LZIP=y CONFIG_LZOP=y CONFIG_LZOP_COMPR_HIGH=y CONFIG_RPM2CPIO=y --- configs/android_ndk_defconfig +++ configs/android_ndk_defconfig @@ -130,6 +130,7 @@ CONFIG_FEATURE_SEAMLESS_LZMA=y CONFIG_FEATURE_SEAMLESS_BZ2=y CONFIG_FEATURE_SEAMLESS_GZ=y +CONFIG_FEATURE_SEAMLESS_LZ=y CONFIG_FEATURE_SEAMLESS_Z=y CONFIG_AR=y CONFIG_FEATURE_AR_LONG_FILENAMES=y @@ -152,6 +153,8 @@ # CONFIG_FEATURE_GZIP_LONG_OPTIONS is not set CONFIG_GZIP_FAST=0 # CONFIG_FEATURE_GZIP_LEVELS is not set +CONFIG_LUNZIP=y +CONFIG_LZIP=y CONFIG_LZOP=y CONFIG_LZOP_COMPR_HIGH=y CONFIG_RPM=y --- configs/cygwin_defconfig +++ configs/cygwin_defconfig @@ -120,6 +120,7 @@ CONFIG_FEATURE_SEAMLESS_LZMA=y CONFIG_FEATURE_SEAMLESS_BZ2=y CONFIG_FEATURE_SEAMLESS_GZ=y +CONFIG_FEATURE_SEAMLESS_LZ=y # CONFIG_FEATURE_SEAMLESS_Z is not set # CONFIG_AR is not set # CONFIG_FEATURE_AR_LONG_FILENAMES is not set @@ -134,6 +135,8 @@ CONFIG_GUNZIP=y CONFIG_GZIP=y CONFIG_FEATURE_GZIP_LONG_OPTIONS=y +CONFIG_LUNZIP=y +CONFIG_LZIP=y CONFIG_LZOP=y # CONFIG_LZOP_COMPR_HIGH is not set CONFIG_RPM2CPIO=y --- configs/freebsd_defconfig +++ configs/freebsd_defconfig @@ -118,6 +118,7 @@ CONFIG_FEATURE_SEAMLESS_LZMA=y CONFIG_FEATURE_SEAMLESS_BZ2=y CONFIG_FEATURE_SEAMLESS_GZ=y +CONFIG_FEATURE_SEAMLESS_LZ=y CONFIG_FEATURE_SEAMLESS_Z=y CONFIG_AR=y CONFIG_FEATURE_AR_LONG_FILENAMES=y @@ -132,6 +133,8 @@ CONFIG_GUNZIP=y CONFIG_GZIP=y CONFIG_FEATURE_GZIP_LONG_OPTIONS=y +CONFIG_LUNZIP=y +CONFIG_LZIP=y CONFIG_LZOP=y # CONFIG_LZOP_COMPR_HIGH is not set CONFIG_RPM2CPIO=y --- include/bb_archive.h +++ include/bb_archive.h @@ -9,6 +9,8 @@ COMPRESS_MAGIC = 0x1f9d, GZIP_MAGIC = 0x1f8b, BZIP2_MAGIC = 256 * 'B' + 'Z', + LZIP_MAGIC1 = 0x4C5A, + LZIP_MAGIC2 = 0x4950, /* .xz signature: 0xfd, '7', 'z', 'X', 'Z', 0x00 */ /* More info at: http://tukaani.org/xz/xz-file-format.txt */ XZ_MAGIC1 = 256 * 0xfd + '7', @@ -21,6 +23,8 @@ COMPRESS_MAGIC = 0x9d1f, GZIP_MAGIC = 0x8b1f, BZIP2_MAGIC = 'B' + 'Z' * 256, + LZIP_MAGIC1 = 0x5A4C, + LZIP_MAGIC2 = 0x5049, XZ_MAGIC1 = 0xfd + '7' * 256, XZ_MAGIC2 = 'z' + ('X' + ('Z' + 0 * 256) * 256) * 256, XZ_MAGIC1a = 0xfd + ('7' + ('z' + 'X' * 256) * 256) * 256, @@ -196,6 +200,7 @@ char get_header_tar_gz(archive_handle_t *archive_handle) FAST_FUNC; char get_header_tar_xz(archive_handle_t *archive_handle) FAST_FUNC; char get_header_tar_bz2(archive_handle_t *archive_handle) FAST_FUNC; +char get_header_tar_lz(archive_handle_t *archive_handle) FAST_FUNC; char get_header_tar_lzma(archive_handle_t *archive_handle) FAST_FUNC; char get_header_tar_xz(archive_handle_t *archive_handle) FAST_FUNC; @@ -252,6 +257,7 @@ IF_DESKTOP(long long) int unpack_Z_stream(transformer_state_t *xstate) FAST_FUNC; IF_DESKTOP(long long) int unpack_gz_stream(transformer_state_t *xstate) FAST_FUNC; IF_DESKTOP(long long) int unpack_bz2_stream(transformer_state_t *xstate) FAST_FUNC; +IF_DESKTOP(long long) int unpack_lz_stream(transformer_state_t *xstate) FAST_FUNC; IF_DESKTOP(long long) int unpack_lzma_stream(transformer_state_t *xstate) FAST_FUNC; IF_DESKTOP(long long) int unpack_xz_stream(transformer_state_t *xstate) FAST_FUNC; --- include/libbb.h +++ include/libbb.h @@ -980,6 +980,7 @@ #define SEAMLESS_COMPRESSION (0 \ || ENABLE_FEATURE_SEAMLESS_XZ \ || ENABLE_FEATURE_SEAMLESS_LZMA \ + || ENABLE_FEATURE_SEAMLESS_LZ \ || ENABLE_FEATURE_SEAMLESS_BZ2 \ || ENABLE_FEATURE_SEAMLESS_GZ \ || ENABLE_FEATURE_SEAMLESS_Z) @@ -1513,6 +1514,7 @@ /* Don't need IF_xxx() guard for these */ int gunzip_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; int bunzip2_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; +int lunzip_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; #if ENABLE_ROUTE void bb_displayroutes(int noresolve, int netstatfmt) FAST_FUNC; --- miscutils/man.c +++ miscutils/man.c @@ -180,6 +180,11 @@ if (run_pipe(filename_with_zext, man, level)) return 1; #endif +#if ENABLE_FEATURE_SEAMLESS_LZ + strcpy(ext, "lz"); + if (run_pipe(filename_with_zext, man, level)) + return 1; +#endif #if ENABLE_FEATURE_SEAMLESS_BZ2 strcpy(ext, "bz2"); if (run_pipe(filename_with_zext, man, level)) --- scripts/fix_ws.sh +++ scripts/fix_ws.sh @@ -19,6 +19,7 @@ | while read name; do test "YES" = "${name/*.bz2/YES}" && continue test "YES" = "${name/*.gz/YES}" && continue + test "YES" = "${name/*.lz/YES}" && continue test "YES" = "${name/*.png/YES}" && continue test "YES" = "${name/*.gif/YES}" && continue test "YES" = "${name/*.jpg/YES}" && continue --- testsuite/bunzip2.tests +++ testsuite/bunzip2.tests @@ -9,6 +9,9 @@ elif test "${0##*/}" = "bunzip2.tests"; then unpack=bunzip2 ext=bz2 +elif test "${0##*/}" = "lunzip.tests"; then + unpack=lunzip + ext=lz else echo "WTF? argv0='$0'" exit 1 @@ -35,6 +38,13 @@ $ECHO -ne "\x17\x72\x45\x38\x50\x90\x5b\xb8\xe8\xa3" } +hello_lz() { +# Lzipped "HELLO\n" +$ECHO -ne "\x4c\x5a\x49\x50\x01\x0c\x00\x24\x11\x45\xcf\x72\xcd\x3d\x3a\xdf" +$ECHO -ne "\xff\xff\xdd\x12\x00\x00\x6e\xd7\xac\xfd\x06\x00\x00\x00\x00\x00" +$ECHO -ne "\x00\x00\x2a\x00\x00\x00\x00\x00\x00\x00" +} + # We had bunzip2 error on this .bz2 file (fixed in rev 22521) test1_bz2() { @@ -530,6 +540,12 @@ expected="ok\n" prep; check "$unpack: delete src" "${bb}$unpack t2.$ext; test ! -f t2.$ext && echo ok" +expected="ok\n" +rm -f t1.* +hello_$ext > t1.t$ext +check "$unpack: replace .t$ext --> .tar" \ +"${bb}$unpack t1.t$ext && test -f t1.tar && test ! -f t1.t$ext && echo ok" + ) rm -rf testdir --- testsuite/bzcat.tests +++ testsuite/bzcat.tests @@ -30,10 +30,18 @@ $ECHO -ne "\x17\x72\x45\x38\x50\x90\x5b\xb8\xe8\xa3" } +hello_lz() { + # Lzipped "HELLO\n" + $ECHO -ne "\x4c\x5a\x49\x50\x01\x0c\x00\x24\x11\x45\xcf\x72\xcd\x3d\x3a\xdf" + $ECHO -ne "\xff\xff\xdd\x12\x00\x00\x6e\xd7\xac\xfd\x06\x00\x00\x00\x00\x00" + $ECHO -ne "\x00\x00\x2a\x00\x00\x00\x00\x00\x00\x00" +} + test x"$CONFIG_ZCAT" = x"y" && \ for ext in \ `test x"$CONFIG_FEATURE_SEAMLESS_GZ" = x"y" && echo gz` \ `test x"$CONFIG_FEATURE_SEAMLESS_BZ2" = x"y" && echo bz2` \ + `test x"$CONFIG_FEATURE_SEAMLESS_LZ" = x"y" && echo lz` \ `test x"$CONFIG_FEATURE_SEAMLESS_Z" = x"y" && echo Z` do prep() { --- testsuite/lunzip.tests +++ testsuite/lunzip.tests @@ -0,0 +1,3 @@ +#!/bin/sh + +. ./bunzip2.tests --- testsuite/lzip.tests +++ testsuite/lzip.tests @@ -0,0 +1,58 @@ +#!/bin/sh +# Copyright (C) 2012-2021 Antonio Diaz Diaz. +# Licensed under GPLv2 or later, see file LICENSE in this source tree. + +. ./testing.sh + +rm -rf lzip.testdir 2>/dev/null +mkdir lzip.testdir + +# Lzipped "a" +$ECHO -ne "\x4c\x5a\x49\x50\x01\x0c\x00\x30\xc1\xfb\xff\xff\xff\xe0\x00\x00\ +\x00\x43\xbe\xb7\xe8\x01\x00\x00\x00\x00\x00\x00\x00\x25\x00\x00\ +\x00\x00\x00\x00\x00" > lzip.testdir/a.lz + +# Lzipped zero-length file +$ECHO -ne "\x4c\x5a\x49\x50\x01\x0c\x00\x83\xff\xfb\xff\xff\xc0\x00\x00\x00\ +\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x24\x00\x00\x00\ +\x00\x00\x00\x00" > lzip.testdir/zero.lz + + +# testing "test name" "commands" "expected result" "file input" "stdin" + +testing "lzip reads from standard input" \ +"busybox lzip | cmp lzip.testdir/a.lz -" "" "" "a" + +testing "lzip accepts single minus" \ +"busybox lzip - | cmp lzip.testdir/a.lz -" "" "" "a" + +testing "lzip compresses a zero-length file" \ +"> zero ; busybox lzip -c zero | cmp lzip.testdir/zero.lz -" "" "" "" + +testing "lzip replaces original file" \ +"$ECHO foo > lzip.testdir/foo +rm -f lzip.testdir/foo.lz 2>/dev/null +busybox lzip lzip.testdir/foo +test -f lzip.testdir/foo.lz && test ! -f lzip.testdir/foo && echo ok" \ +"ok\n" "" "" + +testing "lzip replaces multiple files" \ +"$ECHO foo > lzip.testdir/foo +$ECHO bar > lzip.testdir/bar +rm -f lzip.testdir/foo.lz 2>/dev/null +rm -f lzip.testdir/bar.lz 2>/dev/null +busybox lzip lzip.testdir/foo lzip.testdir/bar +test -f lzip.testdir/foo.lz && test ! -f lzip.testdir/foo && echo ok1 +test -f lzip.testdir/bar.lz && test ! -f lzip.testdir/bar && echo ok2" \ +"ok1\nok2\n" "" "" + +testing "lzip -c does not remove original file" \ +"$ECHO foo > lzip.testdir/foo +busybox lzip -c lzip.testdir/foo >/dev/null +test -f lzip.testdir/foo && echo ok" "ok\n" "" "" + + +# Clean up +rm -rf lzip.testdir 2>/dev/null + +exit $FAILCOUNT