Commit 32643ecd authored by longpanda's avatar longpanda
Browse files

compress 7za.exe

parent d2e59305
......@@ -1110,11 +1110,31 @@ static BOOL check_tar_archive(const char *archive, CHAR *tarName)
return FALSE;
}
static UCHAR *g_unxz_buffer = NULL;
static int g_unxz_len = 0;
static void unxz_error(char *x)
{
Log("%s", x);
}
static int unxz_flush(void *src, unsigned int size)
{
memcpy(g_unxz_buffer + g_unxz_len, src, size);
g_unxz_len += (int)size;
return (int)size;
}
static int DecompressInjectionArchive(const char *archive, DWORD PhyDrive)
{
int rc = 1;
int writelen = 0;
UCHAR *Buffer = NULL;
UCHAR *RawBuffer = NULL;
BOOL bRet;
DWORD dwBytes;
DWORD dwSize;
HANDLE hDrive;
HANDLE hOut;
DWORD flags = CREATE_NO_WINDOW;
......@@ -1154,11 +1174,40 @@ static int DecompressInjectionArchive(const char *archive, DWORD PhyDrive)
{
if (g_system_bit == 64)
{
CopyFileFromFatDisk("/ventoy/7z/64/7za.exe", "ventoy\\7za.exe");
CopyFileFromFatDisk("/ventoy/7z/64/7za.xz", "ventoy\\7za.xz");
}
else
{
CopyFileFromFatDisk("/ventoy/7z/32/7za.xz", "ventoy\\7za.xz");
}
ReadWholeFile2Buf("ventoy\\7za.xz", &Buffer, &dwSize);
Log("7za.xz file size:%u", dwSize);
RawBuffer = malloc(SIZE_1MB * 4);
if (RawBuffer)
{
g_unxz_buffer = RawBuffer;
g_unxz_len = 0;
unxz(Buffer, (int)dwSize, NULL, unxz_flush, NULL, &writelen, unxz_error);
if (writelen == (int)dwSize)
{
Log("Decompress success 7za.xz(%u) ---> 7za.exe(%d)", dwSize, g_unxz_len);
}
else
{
Log("Decompress failed 7za.xz(%u) ---> 7za.exe(%u)", dwSize, dwSize);
}
SaveBuffer2File("ventoy\\7za.exe", RawBuffer, (DWORD)g_unxz_len);
g_unxz_buffer = NULL;
g_unxz_len = 0;
free(RawBuffer);
}
else
{
CopyFileFromFatDisk("/ventoy/7z/32/7za.exe", "ventoy\\7za.exe");
Log("Failed to alloc 4MB memory");
}
sprintf_s(StrBuf, sizeof(StrBuf), "ventoy\\7za.exe x -y -aoa -oX:\\ %s", archive);
......
......@@ -153,4 +153,10 @@ typedef struct VTOY_GPT_INFO
#define LASTERR GetLastError()
int unxz(unsigned char *in, int in_size,
int(*fill)(void *dest, unsigned int size),
int(*flush)(void *src, unsigned int size),
unsigned char *out, int *in_used,
void(*error)(char *x));
#endif
......@@ -69,22 +69,22 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<TargetName>$(ProjectName)32</TargetName>
<IncludePath>$(ProjectDir)\fat_io_lib;$(IncludePath)</IncludePath>
<IncludePath>$(ProjectDir)\fat_io_lib;$(ProjectDir)\xz-embedded-20130513\linux\include;$(ProjectDir)\xz-embedded-20130513\linux\include\linux;$(ProjectDir)\xz-embedded-20130513\userspace;$(IncludePath)</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<TargetName>$(ProjectName)64</TargetName>
<IncludePath>$(ProjectDir)\fat_io_lib;$(IncludePath)</IncludePath>
<IncludePath>$(ProjectDir)\fat_io_lib;$(ProjectDir)\xz-embedded-20130513\linux\include;$(ProjectDir)\xz-embedded-20130513\linux\include\linux;$(ProjectDir)\xz-embedded-20130513\userspace;$(IncludePath)</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<TargetName>$(ProjectName)32</TargetName>
<IncludePath>$(ProjectDir)\fat_io_lib;$(IncludePath)</IncludePath>
<IncludePath>$(ProjectDir)\fat_io_lib;$(ProjectDir)\xz-embedded-20130513\linux\include;$(ProjectDir)\xz-embedded-20130513\linux\include\linux;$(ProjectDir)\xz-embedded-20130513\userspace;$(IncludePath)</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<TargetName>$(ProjectName)64</TargetName>
<IncludePath>$(ProjectDir)\fat_io_lib;$(IncludePath)</IncludePath>
<IncludePath>$(ProjectDir)\fat_io_lib;$(ProjectDir)\xz-embedded-20130513\linux\include;$(ProjectDir)\xz-embedded-20130513\linux\include\linux;$(ProjectDir)\xz-embedded-20130513\userspace;$(IncludePath)</IncludePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
......@@ -92,7 +92,7 @@
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>VTOY_32;FATFS_INC_FORMAT_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>STATIC=static;INIT=;VTOY_32;FATFS_INC_FORMAT_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
......@@ -108,7 +108,7 @@
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>VTOY_64;FATFS_INC_FORMAT_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>STATIC=static;INIT=;VTOY_64;FATFS_INC_FORMAT_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
......@@ -126,7 +126,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>VTOY_BIT=32;FATFS_INC_FORMAT_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>STATIC=static;INIT=;VTOY_BIT=32;FATFS_INC_FORMAT_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
</ClCompile>
......@@ -146,7 +146,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>VTOY_BIT=64;FATFS_INC_FORMAT_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>STATIC=static;INIT=;VTOY_BIT=64;FATFS_INC_FORMAT_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
</ClCompile>
......@@ -169,6 +169,7 @@
<ClCompile Include="fat_io_lib\fat_table.c" />
<ClCompile Include="fat_io_lib\fat_write.c" />
<ClCompile Include="vtoyjump.c" />
<ClCompile Include="xz-embedded-20130513\linux\lib\decompress_unxz.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="fat_io_lib\fat_access.h" />
......
......@@ -42,6 +42,9 @@
<ClCompile Include="fat_io_lib\fat_write.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="xz-embedded-20130513\linux\lib\decompress_unxz.c">
<Filter>源文件</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="vtoyjump.h">
......
Licensing of XZ Embedded
========================
All the files in this package have been written by Lasse Collin
and/or Igor Pavlov. All these files have been put into the
public domain. You can do whatever you want with these files.
As usual, this software is provided "as is", without any warranty.
XZ Embedded
===========
XZ Embedded is a relatively small, limited implementation of the .xz
file format. Currently only decoding is implemented.
XZ Embedded was written for use in the Linux kernel, but the code can
be easily used in other environments too, including regular userspace
applications. See userspace/xzminidec.c for an example program.
This README contains information that is useful only when the copy
of XZ Embedded isn't part of the Linux kernel tree. You should also
read linux/Documentation/xz.txt even if you aren't using XZ Embedded
as part of Linux; information in that file is not repeated in this
README.
Compiling the Linux kernel module
The xz_dec module depends on crc32 module, so make sure that you have
it enabled (CONFIG_CRC32).
Building the xz_dec and xz_dec_test modules without support for BCJ
filters:
cd linux/lib/xz
make -C /path/to/kernel/source \
KCPPFLAGS=-I"$(pwd)/../../include" M="$(pwd)" \
CONFIG_XZ_DEC=m CONFIG_XZ_DEC_TEST=m
Building the xz_dec and xz_dec_test modules with support for BCJ
filters:
cd linux/lib/xz
make -C /path/to/kernel/source \
KCPPFLAGS=-I"$(pwd)/../../include" M="$(pwd)" \
CONFIG_XZ_DEC=m CONFIG_XZ_DEC_TEST=m CONFIG_XZ_DEC_BCJ=y \
CONFIG_XZ_DEC_X86=y CONFIG_XZ_DEC_POWERPC=y \
CONFIG_XZ_DEC_IA64=y CONFIG_XZ_DEC_ARM=y \
CONFIG_XZ_DEC_ARMTHUMB=y CONFIG_XZ_DEC_SPARC=y
If you want only one or a few of the BCJ filters, omit the appropriate
variables. CONFIG_XZ_DEC_BCJ=y is always required to build the support
code shared between all BCJ filters.
Most people don't need the xz_dec_test module. You can skip building
it by omitting CONFIG_XZ_DEC_TEST=m from the make command line.
Compiler requirements
XZ Embedded should compile as either GNU-C89 (used in the Linux
kernel) or with any C99 compiler. Getting the code to compile with
non-GNU C89 compiler or a C++ compiler should be quite easy as
long as there is a data type for unsigned 64-bit integer (or the
code is modified not to support large files, which needs some more
care than just using 32-bit integer instead of 64-bit).
If you use GCC, try to use a recent version. For example, on x86-32,
xz_dec_lzma2.c compiled with GCC 3.3.6 is 15-25 % slower than when
compiled with GCC 4.3.3.
Embedding into userspace applications
To embed the XZ decoder, copy the following files into a single
directory in your source code tree:
linux/include/linux/xz.h
linux/lib/xz/xz_crc32.c
linux/lib/xz/xz_dec_lzma2.c
linux/lib/xz/xz_dec_stream.c
linux/lib/xz/xz_lzma2.h
linux/lib/xz/xz_private.h
linux/lib/xz/xz_stream.h
userspace/xz_config.h
Alternatively, xz.h may be placed into a different directory but then
that directory must be in the compiler include path when compiling
the .c files.
Your code should use only the functions declared in xz.h. The rest of
the .h files are meant only for internal use in XZ Embedded.
You may want to modify xz_config.h to be more suitable for your build
environment. Probably you should at least skim through it even if the
default file works as is.
Integrity check support
XZ Embedded always supports the integrity check types None and
CRC32. Support for CRC64 is optional. SHA-256 is currently not
supported in XZ Embedded although the .xz format does support it.
The xz tool from XZ Utils uses CRC64 by default, but CRC32 is usually
enough in embedded systems to keep the code size smaller.
If you want support for CRC64, you need to copy linux/lib/xz/xz_crc64.c
into your application, and #define XZ_USE_CRC64 in xz_config.h or in
compiler flags.
When using the internal CRC32 or CRC64, their lookup tables need to be
initialized with xz_crc32_init() and xz_crc64_init(), respectively.
See xz.h for details.
To use external CRC32 or CRC64 code instead of the code from
xz_crc32.c or xz_crc64.c, the following #defines may be used
in xz_config.h or in compiler flags:
#define XZ_INTERNAL_CRC32 0
#define XZ_INTERNAL_CRC64 0
Then it is up to you to provide compatible xz_crc32() or xz_crc64()
functions.
If the .xz file being decompressed uses an integrity check type that
isn't supported by XZ Embedded, it is treated as an error and the
file cannot be decompressed. For multi-call mode, this can be modified
by #defining XZ_DEC_ANY_CHECK. Then xz_dec_run() will return
XZ_UNSUPPORTED_CHECK when unsupported check type is detected. After
that decompression can be continued normally except that the
integrity check won't be verified. In single-call mode there's
no way to continue decoding, so XZ_DEC_ANY_CHECK is almost useless
in single-call mode.
BCJ filter support
If you want support for one or more BCJ filters, you need to copy also
linux/lib/xz/xz_dec_bcj.c into your application, and use appropriate
#defines in xz_config.h or in compiler flags. You don't need these
#defines in the code that just uses XZ Embedded via xz.h, but having
them always #defined doesn't hurt either.
#define Instruction set BCJ filter endianness
XZ_DEC_X86 x86-32 or x86-64 Little endian only
XZ_DEC_POWERPC PowerPC Big endian only
XZ_DEC_IA64 Itanium (IA-64) Big or little endian
XZ_DEC_ARM ARM Little endian only
XZ_DEC_ARMTHUMB ARM-Thumb Little endian only
XZ_DEC_SPARC SPARC Big or little endian
While some architectures are (partially) bi-endian, the endianness
setting doesn't change the endianness of the instructions on all
architectures. That's why Itanium and SPARC filters work for both big
and little endian executables (Itanium has little endian instructions
and SPARC has big endian instructions).
There currently is no filter for little endian PowerPC or big endian
ARM or ARM-Thumb. Implementing filters for them can be considered if
there is a need for such filters in real-world applications.
Notes about shared libraries
If you are including XZ Embedded into a shared library, you very
probably should rename the xz_* functions to prevent symbol
conflicts in case your library is linked against some other library
or application that also has XZ Embedded in it (which may even be
a different version of XZ Embedded). TODO: Provide an easy way
to do this.
Please don't create a shared library of XZ Embedded itself unless
it is fine to rebuild everything depending on that shared library
everytime you upgrade to a newer version of XZ Embedded. There are
no API or ABI stability guarantees between different versions of
XZ Embedded.
XZ data compression in Linux
============================
Introduction
XZ is a general purpose data compression format with high compression
ratio and relatively fast decompression. The primary compression
algorithm (filter) is LZMA2. Additional filters can be used to improve
compression ratio even further. E.g. Branch/Call/Jump (BCJ) filters
improve compression ratio of executable data.
The XZ decompressor in Linux is called XZ Embedded. It supports
the LZMA2 filter and optionally also BCJ filters. CRC32 is supported
for integrity checking. The home page of XZ Embedded is at
<http://tukaani.org/xz/embedded.html>, where you can find the
latest version and also information about using the code outside
the Linux kernel.
For userspace, XZ Utils provide a zlib-like compression library
and a gzip-like command line tool. XZ Utils can be downloaded from
<http://tukaani.org/xz/>.
XZ related components in the kernel
The xz_dec module provides XZ decompressor with single-call (buffer
to buffer) and multi-call (stateful) APIs. The usage of the xz_dec
module is documented in include/linux/xz.h.
The xz_dec_test module is for testing xz_dec. xz_dec_test is not
useful unless you are hacking the XZ decompressor. xz_dec_test
allocates a char device major dynamically to which one can write
.xz files from userspace. The decompressed output is thrown away.
Keep an eye on dmesg to see diagnostics printed by xz_dec_test.
See the xz_dec_test source code for the details.
For decompressing the kernel image, initramfs, and initrd, there
is a wrapper function in lib/decompress_unxz.c. Its API is the
same as in other decompress_*.c files, which is defined in
include/linux/decompress/generic.h.
scripts/xz_wrap.sh is a wrapper for the xz command line tool found
from XZ Utils. The wrapper sets compression options to values suitable
for compressing the kernel image.
For kernel makefiles, two commands are provided for use with
$(call if_needed). The kernel image should be compressed with
$(call if_needed,xzkern) which will use a BCJ filter and a big LZMA2
dictionary. It will also append a four-byte trailer containing the
uncompressed size of the file, which is needed by the boot code.
Other things should be compressed with $(call if_needed,xzmisc)
which will use no BCJ filter and 1 MiB LZMA2 dictionary.
Notes on compression options
Since the XZ Embedded supports only streams with no integrity check or
CRC32, make sure that you don't use some other integrity check type
when encoding files that are supposed to be decoded by the kernel. With
liblzma, you need to use either LZMA_CHECK_NONE or LZMA_CHECK_CRC32
when encoding. With the xz command line tool, use --check=none or
--check=crc32.
Using CRC32 is strongly recommended unless there is some other layer
which will verify the integrity of the uncompressed data anyway.
Double checking the integrity would probably be waste of CPU cycles.
Note that the headers will always have a CRC32 which will be validated
by the decoder; you can only change the integrity check type (or
disable it) for the actual uncompressed data.
In userspace, LZMA2 is typically used with dictionary sizes of several
megabytes. The decoder needs to have the dictionary in RAM, thus big
dictionaries cannot be used for files that are intended to be decoded
by the kernel. 1 MiB is probably the maximum reasonable dictionary
size for in-kernel use (maybe more is OK for initramfs). The presets
in XZ Utils may not be optimal when creating files for the kernel,
so don't hesitate to use custom settings. Example:
xz --check=crc32 --lzma2=dict=512KiB inputfile
An exception to above dictionary size limitation is when the decoder
is used in single-call mode. Decompressing the kernel itself is an
example of this situation. In single-call mode, the memory usage
doesn't depend on the dictionary size, and it is perfectly fine to
use a big dictionary: for maximum compression, the dictionary should
be at least as big as the uncompressed data itself.
Future plans
Creating a limited XZ encoder may be considered if people think it is
useful. LZMA2 is slower to compress than e.g. Deflate or LZO even at
the fastest settings, so it isn't clear if LZMA2 encoder is wanted
into the kernel.
Support for limited random-access reading is planned for the
decompression code. I don't know if it could have any use in the
kernel, but I know that it would be useful in some embedded projects
outside the Linux kernel.
Conformance to the .xz file format specification
There are a couple of corner cases where things have been simplified
at expense of detecting errors as early as possible. These should not
matter in practice all, since they don't cause security issues. But
it is good to know this if testing the code e.g. with the test files
from XZ Utils.
Reporting bugs
Before reporting a bug, please check that it's not fixed already
at upstream. See <http://tukaani.org/xz/embedded.html> to get the
latest code.
Report bugs to <lasse.collin@tukaani.org> or visit #tukaani on
Freenode and talk to Larhzu. I don't actively read LKML or other
kernel-related mailing lists, so if there's something I should know,
you should email to me personally or use IRC.
Don't bother Igor Pavlov with questions about the XZ implementation
in the kernel or about XZ Utils. While these two implementations
include essential code that is directly based on Igor Pavlov's code,
these implementations aren't maintained nor supported by him.
/*
* Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd
*
* Author: Lasse Collin <lasse.collin@tukaani.org>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
#ifndef DECOMPRESS_UNXZ_H
#define DECOMPRESS_UNXZ_H
int unxz(unsigned char *in, int in_size,
int (*fill)(void *dest, unsigned int size),
int (*flush)(void *src, unsigned int size),
unsigned char *out, int *in_used,
void (*error)(char *x));
#endif
/*
* XZ decompressor
*
* Authors: Lasse Collin <lasse.collin@tukaani.org>
* Igor Pavlov <http://7-zip.org/>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
#ifndef XZ_H
#define XZ_H
#ifdef __KERNEL__
# include <linux/stddef.h>
# include <linux/types.h>
#else
# include <stddef.h>
# include <stdint.h>
#endif
#ifdef __cplusplus
extern "C" {
#endif
/* In Linux, this is used to make extern functions static when needed. */
#ifndef XZ_EXTERN
# define XZ_EXTERN extern
#endif
/**
* enum xz_mode - Operation mode
*
* @XZ_SINGLE: Single-call mode. This uses less RAM than
* than multi-call modes, because the LZMA2
* dictionary doesn't need to be allocated as
* part of the decoder state. All required data
* structures are allocated at initialization,
* so xz_dec_run() cannot return XZ_MEM_ERROR.
* @XZ_PREALLOC: Multi-call mode with preallocated LZMA2
* dictionary buffer. All data structures are
* allocated at initialization, so xz_dec_run()
* cannot return XZ_MEM_ERROR.
* @XZ_DYNALLOC: Multi-call mode. The LZMA2 dictionary is
* allocated once the required size has been
* parsed from the stream headers. If the
* allocation fails, xz_dec_run() will return
* XZ_MEM_ERROR.
*
* It is possible to enable support only for a subset of the above
* modes at compile time by defining XZ_DEC_SINGLE, XZ_DEC_PREALLOC,
* or XZ_DEC_DYNALLOC. The xz_dec kernel module is always compiled
* with support for all operation modes, but the preboot code may
* be built with fewer features to minimize code size.
*/
enum xz_mode {
XZ_SINGLE,
XZ_PREALLOC,
XZ_DYNALLOC
};
/**
* enum xz_ret - Return codes
* @XZ_OK: Everything is OK so far. More input or more
* output space is required to continue. This
* return code is possible only in multi-call mode
* (XZ_PREALLOC or XZ_DYNALLOC).
* @XZ_STREAM_END: Operation finished successfully.
* @XZ_UNSUPPORTED_CHECK: Integrity check type is not supported. Decoding
* is still possible in multi-call mode by simply
* calling xz_dec_run() again.
* Note that this return value is used only if
* XZ_DEC_ANY_CHECK was defined at build time,
* which is not used in the kernel. Unsupported
* check types return XZ_OPTIONS_ERROR if
* XZ_DEC_ANY_CHECK was not defined at build time.
* @XZ_MEM_ERROR: Allocating memory failed. This return code is
* possible only if the decoder was initialized
* with XZ_DYNALLOC. The amount of memory that was
* tried to be allocated was no more than the
* dict_max argument given to xz_dec_init().
* @XZ_MEMLIMIT_ERROR: A bigger LZMA2 dictionary would be needed than
* allowed by the dict_max argument given to
* xz_dec_init(). This return value is possible
* only in multi-call mode (XZ_PREALLOC or
* XZ_DYNALLOC); the single-call mode (XZ_SINGLE)
* ignores the dict_max argument.
* @XZ_FORMAT_ERROR: File format was not recognized (wrong magic
* bytes).
* @XZ_OPTIONS_ERROR: This implementation doesn't support the requested
* compression options. In the decoder this means
* that the header CRC32 matches, but the header
* itself specifies something that we don't support.
* @XZ_DATA_ERROR: Compressed data is corrupt.
* @XZ_BUF_ERROR: Cannot make any progress. Details are slightly
* different between multi-call and single-call
* mode; more information below.
*
* In multi-call mode, XZ_BUF_ERROR is returned when two consecutive calls
* to XZ code cannot consume any input and cannot produce any new output.
* This happens when there is no new input available, or the output buffer
* is full while at least one output byte is still pending. Assuming your
* code is not buggy, you can get this error only when decoding a compressed
* stream that is truncated or otherwise corrupt.
*
* In single-call mode, XZ_BUF_ERROR is returned only when the output buffer
* is too small or the compressed input is corrupt in a way that makes the
* decoder produce more output than the caller expected. When it is
* (relatively) clear that the compressed input is truncated, XZ_DATA_ERROR
* is used instead of XZ_BUF_ERROR.
*/
enum xz_ret {
XZ_OK,
XZ_STREAM_END,
XZ_UNSUPPORTED_CHECK,
XZ_MEM_ERROR,
XZ_MEMLIMIT_ERROR,
XZ_FORMAT_ERROR,
XZ_OPTIONS_ERROR,
XZ_DATA_ERROR,
XZ_BUF_ERROR
};
/**
* struct xz_buf - Passing input and output buffers to XZ code
* @in: Beginning of the input buffer. This may be NULL if and only
* if in_pos is equal to in_size.
* @in_pos: Current position in the input buffer. This must not exceed
* in_size.
* @in_size: Size of the input buffer
* @out: Beginning of the output buffer. This may be NULL if and only
* if out_pos is equal to out_size.
* @out_pos: Current position in the output buffer. This must not exceed
* out_size.
* @out_size: Size of the output buffer
*
* Only the contents of the output buffer from out[out_pos] onward, and
* the variables in_pos and out_pos are modified by the XZ code.
*/
struct xz_buf {
const uint8_t *in;
size_t in_pos;
size_t in_size;
uint8_t *out;
size_t out_pos;
size_t out_size;
};
/**
* struct xz_dec - Opaque type to hold the XZ decoder state
*/
struct xz_dec;
/**
* xz_dec_init() - Allocate and initialize a XZ decoder state
* @mode: Operation mode
* @dict_max: Maximum size of the LZMA2 dictionary (history buffer) for
* multi-call decoding. This is ignored in single-call mode
* (mode == XZ_SINGLE). LZMA2 dictionary is always 2^n bytes
* or 2^n + 2^(n-1) bytes (the latter sizes are less common
* in practice), so other values for dict_max don't make sense.
* In the kernel, dictionary sizes of 64 KiB, 128 KiB, 256 KiB,
* 512 KiB, and 1 MiB are probably the only reasonable values,
* except for kernel and initramfs images where a bigger
* dictionary can be fine and useful.
*
* Single-call mode (XZ_SINGLE): xz_dec_run() decodes the whole stream at
* once. The caller must provide enough output space or the decoding will
* fail. The output space is used as the dictionary buffer, which is why
* there is no need to allocate the dictionary as part of the decoder's
* internal state.
*
* Because the output buffer is used as the workspace, streams encoded using
* a big dictionary are not a problem in single-call mode. It is enough that
* the output buffer is big enough to hold the actual uncompressed data; it
* can be smaller than the dictionary size stored in the stream headers.
*
* Multi-call mode with preallocated dictionary (XZ_PREALLOC): dict_max bytes
* of memory is preallocated for the LZMA2 dictionary. This way there is no
* risk that xz_dec_run() could run out of memory, since xz_dec_run() will
* never allocate any memory. Instead, if the preallocated dictionary is too
* small for decoding the given input stream, xz_dec_run() will return
* XZ_MEMLIMIT_ERROR. Thus, it is important to know what kind of data will be
* decoded to avoid allocating excessive amount of memory for the dictionary.
*
* Multi-call mode with dynamically allocated dictionary (XZ_DYNALLOC):
* dict_max specifies the maximum allowed dictionary size that xz_dec_run()
* may allocate once it has parsed the dictionary size from the stream
* headers. This way excessive allocations can be avoided while still
* limiting the maximum memory usage to a sane value to prevent running the
* system out of memory when decompressing streams from untrusted sources.
*
* On success, xz_dec_init() returns a pointer to struct xz_dec, which is
* ready to be used with xz_dec_run(). If memory allocation fails,
* xz_dec_init() returns NULL.
*/
XZ_EXTERN struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max);
/**
* xz_dec_run() - Run the XZ decoder
* @s: Decoder state allocated using xz_dec_init()
* @b: Input and output buffers
*
* The possible return values depend on build options and operation mode.
* See enum xz_ret for details.
*
* Note that if an error occurs in single-call mode (return value is not
* XZ_STREAM_END), b->in_pos and b->out_pos are not modified and the
* contents of the output buffer from b->out[b->out_pos] onward are
* undefined. This is true even after XZ_BUF_ERROR, because with some filter
* chains, there may be a second pass over the output buffer, and this pass
* cannot be properly done if the output buffer is truncated. Thus, you
* cannot give the single-call decoder a too small buffer and then expect to
* get that amount valid data from the beginning of the stream. You must use
* the multi-call decoder if you don't want to uncompress the whole stream.
*/
XZ_EXTERN enum xz_ret xz_dec_run(struct xz_dec *s, struct xz_buf *b);
/**
* xz_dec_reset() - Reset an already allocated decoder state
* @s: Decoder state allocated using xz_dec_init()
*
* This function can be used to reset the multi-call decoder state without
* freeing and reallocating memory with xz_dec_end() and xz_dec_init().
*
* In single-call mode, xz_dec_reset() is always called in the beginning of
* xz_dec_run(). Thus, explicit call to xz_dec_reset() is useful only in
* multi-call mode.
*/
XZ_EXTERN void xz_dec_reset(struct xz_dec *s);
/**
* xz_dec_end() - Free the memory allocated for the decoder state
* @s: Decoder state allocated using xz_dec_init(). If s is NULL,
* this function does nothing.
*/
XZ_EXTERN void xz_dec_end(struct xz_dec *s);
/*
* Standalone build (userspace build or in-kernel build for boot time use)
* needs a CRC32 implementation. For normal in-kernel use, kernel's own
* CRC32 module is used instead, and users of this module don't need to
* care about the functions below.
*/
#ifndef XZ_INTERNAL_CRC32
# ifdef __KERNEL__
# define XZ_INTERNAL_CRC32 0
# else
# define XZ_INTERNAL_CRC32 1
# endif
#endif
/*
* If CRC64 support has been enabled with XZ_USE_CRC64, a CRC64
* implementation is needed too.
*/
#ifndef XZ_USE_CRC64
# undef XZ_INTERNAL_CRC64
# define XZ_INTERNAL_CRC64 0
#endif
#ifndef XZ_INTERNAL_CRC64
# ifdef __KERNEL__
# error Using CRC64 in the kernel has not been implemented.
# else
# define XZ_INTERNAL_CRC64 1
# endif
#endif
#if XZ_INTERNAL_CRC32
/*
* This must be called before any other xz_* function to initialize
* the CRC32 lookup table.
*/
XZ_EXTERN void xz_crc32_init(void);
/*
* Update CRC32 value using the polynomial from IEEE-802.3. To start a new
* calculation, the third argument must be zero. To continue the calculation,
* the previously returned value is passed as the third argument.
*/
XZ_EXTERN uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc);
#endif
#if XZ_INTERNAL_CRC64
/*
* This must be called before any other xz_* function (except xz_crc32_init())
* to initialize the CRC64 lookup table.
*/
XZ_EXTERN void xz_crc64_init(void);
/*
* Update CRC64 value using the polynomial from ECMA-182. To start a new
* calculation, the third argument must be zero. To continue the calculation,
* the previously returned value is passed as the third argument.
*/
XZ_EXTERN uint64_t xz_crc64(const uint8_t *buf, size_t size, uint64_t crc);
#endif
#ifdef __cplusplus
}
#endif
#endif
/*
* Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd
*
* Author: Lasse Collin <lasse.collin@tukaani.org>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
/*
* Important notes about in-place decompression
*
* At least on x86, the kernel is decompressed in place: the compressed data
* is placed to the end of the output buffer, and the decompressor overwrites
* most of the compressed data. There must be enough safety margin to
* guarantee that the write position is always behind the read position.
*
* The safety margin for XZ with LZMA2 or BCJ+LZMA2 is calculated below.
* Note that the margin with XZ is bigger than with Deflate (gzip)!
*
* The worst case for in-place decompression is that the beginning of
* the file is compressed extremely well, and the rest of the file is
* uncompressible. Thus, we must look for worst-case expansion when the
* compressor is encoding uncompressible data.
*
* The structure of the .xz file in case of a compresed kernel is as follows.
* Sizes (as bytes) of the fields are in parenthesis.
*
* Stream Header (12)
* Block Header:
* Block Header (8-12)
* Compressed Data (N)
* Block Padding (0-3)
* CRC32 (4)
* Index (8-20)
* Stream Footer (12)
*
* Normally there is exactly one Block, but let's assume that there are
* 2-4 Blocks just in case. Because Stream Header and also Block Header
* of the first Block don't make the decompressor produce any uncompressed
* data, we can ignore them from our calculations. Block Headers of possible
* additional Blocks have to be taken into account still. With these
* assumptions, it is safe to assume that the total header overhead is
* less than 128 bytes.
*
* Compressed Data contains LZMA2 or BCJ+LZMA2 encoded data. Since BCJ
* doesn't change the size of the data, it is enough to calculate the
* safety margin for LZMA2.
*
* LZMA2 stores the data in chunks. Each chunk has a header whose size is
* a maximum of 6 bytes, but to get round 2^n numbers, let's assume that
* the maximum chunk header size is 8 bytes. After the chunk header, there
* may be up to 64 KiB of actual payload in the chunk. Often the payload is
* quite a bit smaller though; to be safe, let's assume that an average
* chunk has only 32 KiB of payload.
*
* The maximum uncompressed size of the payload is 2 MiB. The minimum
* uncompressed size of the payload is in practice never less than the
* payload size itself. The LZMA2 format would allow uncompressed size
* to be less than the payload size, but no sane compressor creates such
* files. LZMA2 supports storing uncompressible data in uncompressed form,
* so there's never a need to create payloads whose uncompressed size is
* smaller than the compressed size.
*
* The assumption, that the uncompressed size of the payload is never
* smaller than the payload itself, is valid only when talking about
* the payload as a whole. It is possible that the payload has parts where
* the decompressor consumes more input than it produces output. Calculating
* the worst case for this would be tricky. Instead of trying to do that,
* let's simply make sure that the decompressor never overwrites any bytes
* of the payload which it is currently reading.
*
* Now we have enough information to calculate the safety margin. We need
* - 128 bytes for the .xz file format headers;
* - 8 bytes per every 32 KiB of uncompressed size (one LZMA2 chunk header
* per chunk, each chunk having average payload size of 32 KiB); and
* - 64 KiB (biggest possible LZMA2 chunk payload size) to make sure that
* the decompressor never overwrites anything from the LZMA2 chunk
* payload it is currently reading.
*
* We get the following formula:
*
* safety_margin = 128 + uncompressed_size * 8 / 32768 + 65536
* = 128 + (uncompressed_size >> 12) + 65536
*
* For comparison, according to arch/x86/boot/compressed/misc.c, the
* equivalent formula for Deflate is this:
*
* safety_margin = 18 + (uncompressed_size >> 12) + 32768
*
* Thus, when updating Deflate-only in-place kernel decompressor to
* support XZ, the fixed overhead has to be increased from 18+32768 bytes
* to 128+65536 bytes.
*/
/*
* STATIC is defined to "static" if we are being built for kernel
* decompression (pre-boot code). <linux/decompress/mm.h> will define
* STATIC to empty if it wasn't already defined. Since we will need to
* know later if we are being used for kernel decompression, we define
* XZ_PREBOOT here.
*/
#ifdef STATIC
# define XZ_PREBOOT
#endif
#ifdef __KERNEL__
# include <linux/decompress/mm.h>
#endif
#define XZ_EXTERN STATIC
#ifndef XZ_PREBOOT
# include <linux/slab.h>
# include <linux/xz.h>
#else
/*
* Use the internal CRC32 code instead of kernel's CRC32 module, which
* is not available in early phase of booting.
*/
#define XZ_INTERNAL_CRC32 1
/*
* For boot time use, we enable only the BCJ filter of the current
* architecture or none if no BCJ filter is available for the architecture.
*/
#ifdef CONFIG_X86
# define XZ_DEC_X86
#endif
#ifdef CONFIG_PPC
# define XZ_DEC_POWERPC
#endif
#ifdef CONFIG_ARM
# define XZ_DEC_ARM
#endif
#ifdef CONFIG_IA64
# define XZ_DEC_IA64
#endif
#ifdef CONFIG_SPARC
# define XZ_DEC_SPARC
#endif
/*
* This will get the basic headers so that memeq() and others
* can be defined.
*/
#include "xz/xz_private.h"
/*
* Replace the normal allocation functions with the versions from
* <linux/decompress/mm.h>. vfree() needs to support vfree(NULL)
* when XZ_DYNALLOC is used, but the pre-boot free() doesn't support it.
* Workaround it here because the other decompressors don't need it.
*/
#undef kmalloc
#undef kfree
#undef vmalloc
#undef vfree
#define kmalloc(size, flags) malloc(size)
#define kfree(ptr) free(ptr)
#define vmalloc(size) malloc(size)
#define vfree(ptr) do { if (ptr != NULL) free(ptr); } while (0)
/*
* FIXME: Not all basic memory functions are provided in architecture-specific
* files (yet). We define our own versions here for now, but this should be
* only a temporary solution.
*
* memeq and memzero are not used much and any remotely sane implementation
* is fast enough. memcpy/memmove speed matters in multi-call mode, but
* the kernel image is decompressed in single-call mode, in which only
* memcpy speed can matter and only if there is a lot of uncompressible data
* (LZMA2 stores uncompressible chunks in uncompressed form). Thus, the
* functions below should just be kept small; it's probably not worth
* optimizing for speed.
*/
#ifndef memeq
static bool memeq(const void *a, const void *b, size_t size)
{
const uint8_t *x = a;
const uint8_t *y = b;
size_t i;
for (i = 0; i < size; ++i)
if (x[i] != y[i])
return false;
return true;
}
#endif
#ifndef memzero
static void memzero(void *buf, size_t size)
{
uint8_t *b = buf;
uint8_t *e = b + size;
while (b != e)
*b++ = '\0';
}
#endif
#if 0
/* Not static to avoid a conflict with the prototype in the Linux headers. */
void *memmove(void *dest, const void *src, size_t size)
{
uint8_t *d = dest;
const uint8_t *s = src;
size_t i;
if (d < s) {
for (i = 0; i < size; ++i)
d[i] = s[i];
} else if (d > s) {
i = size;
while (i-- > 0)
d[i] = s[i];
}
return dest;
}
#endif
/*
* Since we need memmove anyway, would use it as memcpy too.
* Commented out for now to avoid breaking things.
*/
/*
#ifndef memcpy
# define memcpy memmove
#endif
*/
#include "xz/xz_crc32.c"
#include "xz/xz_dec_stream.c"
#include "xz/xz_dec_lzma2.c"
#include "xz/xz_dec_bcj.c"
#endif /* XZ_PREBOOT */
/* Size of the input and output buffers in multi-call mode */
#define XZ_IOBUF_SIZE 4096
/*
* This function implements the API defined in <linux/decompress/generic.h>.
*
* This wrapper will automatically choose single-call or multi-call mode
* of the native XZ decoder API. The single-call mode can be used only when
* both input and output buffers are available as a single chunk, i.e. when
* fill() and flush() won't be used.
*/
int INIT unxz(unsigned char *in, int in_size,
int (*fill)(void *dest, unsigned int size),
int (*flush)(void *src, unsigned int size),
unsigned char *out, int *in_used,
void (*error)(char *x))
{
struct xz_buf b;
struct xz_dec *s;
enum xz_ret ret;
bool must_free_in = false;
#if XZ_INTERNAL_CRC32
xz_crc32_init();
#endif
if (in_used != NULL)
*in_used = 0;
if (fill == NULL && flush == NULL)
s = xz_dec_init(XZ_SINGLE, 0);
else
s = xz_dec_init(XZ_DYNALLOC, (uint32_t)-1);
if (s == NULL)
goto error_alloc_state;
if (flush == NULL) {
b.out = out;
b.out_size = (size_t)-1;
} else {
b.out_size = XZ_IOBUF_SIZE;
b.out = malloc(XZ_IOBUF_SIZE);
if (b.out == NULL)
goto error_alloc_out;
}
if (in == NULL) {
must_free_in = true;
in = malloc(XZ_IOBUF_SIZE);
if (in == NULL)
goto error_alloc_in;
}
b.in = in;
b.in_pos = 0;
b.in_size = in_size;
b.out_pos = 0;
if (fill == NULL && flush == NULL) {
ret = xz_dec_run(s, &b);
} else {
do {
if (b.in_pos == b.in_size && fill != NULL) {
if (in_used != NULL)
*in_used += (int)(b.in_pos);
b.in_pos = 0;
in_size = fill(in, XZ_IOBUF_SIZE);
if (in_size < 0) {
/*
* This isn't an optimal error code
* but it probably isn't worth making
* a new one either.
*/
ret = XZ_BUF_ERROR;
break;
}
b.in_size = in_size;
}
ret = xz_dec_run(s, &b);
if (flush != NULL && (b.out_pos == b.out_size
|| (ret != XZ_OK && b.out_pos > 0))) {
/*
* Setting ret here may hide an error
* returned by xz_dec_run(), but probably
* it's not too bad.
*/
if (flush(b.out, (unsigned int)(b.out_pos)) != (int)b.out_pos)
ret = XZ_BUF_ERROR;
b.out_pos = 0;
}
} while (ret == XZ_OK);
if (must_free_in)
free(in);
if (flush != NULL)
free(b.out);
}
if (in_used != NULL)
*in_used += (int)(b.in_pos);
xz_dec_end(s);
switch (ret) {
case XZ_STREAM_END:
return 0;
case XZ_MEM_ERROR:
/* This can occur only in multi-call mode. */
error("XZ decompressor ran out of memory");
break;
case XZ_FORMAT_ERROR:
error("Input is not in the XZ format (wrong magic bytes)");
break;
case XZ_OPTIONS_ERROR:
error("Input was encoded with settings that are not "
"supported by this XZ decoder");
break;
case XZ_DATA_ERROR:
case XZ_BUF_ERROR:
error("XZ-compressed data is corrupt");
break;
default:
error("Bug in the XZ decompressor");
break;
}
return -1;
error_alloc_in:
if (flush != NULL)
free(b.out);
error_alloc_out:
xz_dec_end(s);
error_alloc_state:
error("XZ decompressor ran out of memory");
return -1;
}
/*
* This macro is used by architecture-specific files to decompress
* the kernel image.
*/
#define decompress unxz
config XZ_DEC
tristate "XZ decompression support"
select CRC32
help
LZMA2 compression algorithm and BCJ filters are supported using
the .xz file format as the container. For integrity checking,
CRC32 is supported. See Documentation/xz.txt for more information.
if XZ_DEC
config XZ_DEC_X86
bool "x86 BCJ filter decoder"
default y if X86
select XZ_DEC_BCJ
config XZ_DEC_POWERPC
bool "PowerPC BCJ filter decoder"
default y if PPC
select XZ_DEC_BCJ
config XZ_DEC_IA64
bool "IA-64 BCJ filter decoder"
default y if IA64
select XZ_DEC_BCJ
config XZ_DEC_ARM
bool "ARM BCJ filter decoder"
default y if ARM
select XZ_DEC_BCJ
config XZ_DEC_ARMTHUMB
bool "ARM-Thumb BCJ filter decoder"
default y if (ARM && ARM_THUMB)
select XZ_DEC_BCJ
config XZ_DEC_SPARC
bool "SPARC BCJ filter decoder"
default y if SPARC
select XZ_DEC_BCJ
endif
config XZ_DEC_BCJ
bool
default n
config XZ_DEC_TEST
tristate "XZ decompressor tester"
default n
depends on XZ_DEC
help
This allows passing .xz files to the in-kernel XZ decoder via
a character special file. It calculates CRC32 of the decompressed
data and writes diagnostics to the system log.
Unless you are developing the XZ decoder, you don't need this
and should say N.
obj-$(CONFIG_XZ_DEC) += xz_dec.o
xz_dec-y := xz_dec_syms.o xz_dec_stream.o xz_dec_lzma2.o
xz_dec-$(CONFIG_XZ_DEC_BCJ) += xz_dec_bcj.o
obj-$(CONFIG_XZ_DEC_TEST) += xz_dec_test.o
/*
* CRC32 using the polynomial from IEEE-802.3
*
* Authors: Lasse Collin <lasse.collin@tukaani.org>
* Igor Pavlov <http://7-zip.org/>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
/*
* This is not the fastest implementation, but it is pretty compact.
* The fastest versions of xz_crc32() on modern CPUs without hardware
* accelerated CRC instruction are 3-5 times as fast as this version,
* but they are bigger and use more memory for the lookup table.
*/
#include "xz_private.h"
/*
* STATIC_RW_DATA is used in the pre-boot environment on some architectures.
* See <linux/decompress/mm.h> for details.
*/
#ifndef STATIC_RW_DATA
# define STATIC_RW_DATA static
#endif
STATIC_RW_DATA uint32_t xz_crc32_table[256];
XZ_EXTERN void xz_crc32_init(void)
{
const uint32_t poly = 0xEDB88320;
uint32_t i;
uint32_t j;
uint32_t r;
for (i = 0; i < 256; ++i) {
r = i;
for (j = 0; j < 8; ++j)
r = (r >> 1) ^ (poly & ~((r & 1) - 1));
xz_crc32_table[i] = r;
}
return;
}
XZ_EXTERN uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc)
{
crc = ~crc;
while (size != 0) {
crc = xz_crc32_table[*buf++ ^ (crc & 0xFF)] ^ (crc >> 8);
--size;
}
return ~crc;
}
/*
* CRC64 using the polynomial from ECMA-182
*
* This file is similar to xz_crc32.c. See the comments there.
*
* Authors: Lasse Collin <lasse.collin@tukaani.org>
* Igor Pavlov <http://7-zip.org/>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
#include "xz_private.h"
#ifndef STATIC_RW_DATA
# define STATIC_RW_DATA static
#endif
STATIC_RW_DATA uint64_t xz_crc64_table[256];
XZ_EXTERN void xz_crc64_init(void)
{
const uint64_t poly = 0xC96C5795D7870F42;
uint32_t i;
uint32_t j;
uint64_t r;
for (i = 0; i < 256; ++i) {
r = i;
for (j = 0; j < 8; ++j)
r = (r >> 1) ^ (poly & ~((r & 1) - 1));
xz_crc64_table[i] = r;
}
return;
}
XZ_EXTERN uint64_t xz_crc64(const uint8_t *buf, size_t size, uint64_t crc)
{
crc = ~crc;
while (size != 0) {
crc = xz_crc64_table[*buf++ ^ (crc & 0xFF)] ^ (crc >> 8);
--size;
}
return ~crc;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment