--- /dev/null
+/*\r
+ LZ4c - LZ4 Compression CLI program \r
+ Copyright (C) Yann Collet 2011-2013\r
+ GPL v2 License\r
+\r
+ This program is free software; you can redistribute it and/or modify\r
+ it under the terms of the GNU General Public License as published by\r
+ the Free Software Foundation; either version 2 of the License, or\r
+ (at your option) any later version.\r
+\r
+ This program is distributed in the hope that it will be useful,\r
+ but WITHOUT ANY WARRANTY; without even the implied warranty of\r
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\r
+ GNU General Public License for more details.\r
+\r
+ You should have received a copy of the GNU General Public License along\r
+ with this program; if not, write to the Free Software Foundation, Inc.,\r
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\r
+\r
+ You can contact the author at :\r
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html\r
+ - LZ4 source repository : http://code.google.com/p/lz4/\r
+*/\r
+/*\r
+ Note : this is stand-alone program.\r
+ It is not part of LZ4 compression library, it is a user program of LZ4 library.\r
+ The license of LZ4 library is BSD.\r
+ The license of xxHash library is BSD.\r
+ The license of this compression CLI program is GPLv2.\r
+*/\r
+\r
+//**************************************\r
+// Compiler Options\r
+//**************************************\r
+// Disable some Visual warning messages\r
+#ifdef _MSC_VER // Visual Studio\r
+# define _CRT_SECURE_NO_WARNINGS\r
+# define _CRT_SECURE_NO_DEPRECATE // VS2005\r
+# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant\r
+#endif\r
+\r
+\r
+//****************************\r
+// Includes\r
+//****************************\r
+#include <stdio.h> // fprintf, fopen, fread, _fileno(?)\r
+#include <stdlib.h> // malloc\r
+#include <string.h> // strcmp\r
+#include <time.h> // clock\r
+#ifdef _WIN32\r
+#include <io.h> // _setmode\r
+#include <fcntl.h> // _O_BINARY\r
+#endif\r
+#include "lz4.h"\r
+#include "lz4hc.h"\r
+#include "bench.h"\r
+#include "xxhash.h"\r
+\r
+\r
+//**************************************\r
+// Compiler-specific functions\r
+//**************************************\r
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)\r
+\r
+#if defined(_MSC_VER) // Visual Studio\r
+# define swap32 _byteswap_ulong\r
+#elif GCC_VERSION >= 403\r
+# define swap32 __builtin_bswap32\r
+#else\r
+ static inline unsigned int swap32(unsigned int x) {\r
+ return ((x << 24) & 0xff000000 ) |\r
+ ((x << 8) & 0x00ff0000 ) |\r
+ ((x >> 8) & 0x0000ff00 ) |\r
+ ((x >> 24) & 0x000000ff );\r
+ }\r
+#endif\r
+\r
+\r
+//****************************\r
+// Constants\r
+//****************************\r
+#define COMPRESSOR_NAME "LZ4 Compression CLI"\r
+#define COMPRESSOR_VERSION ""\r
+#define COMPILED __DATE__\r
+#define AUTHOR "Yann Collet"\r
+#define EXTENSION ".lz4"\r
+#define WELCOME_MESSAGE "*** %s %s, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, AUTHOR, COMPILED\r
+\r
+#define KB *(1U<<10)\r
+#define MB *(1U<<20)\r
+#define GB *(1U<<30)\r
+\r
+#define _1BIT 0x01\r
+#define _2BITS 0x03\r
+#define _3BITS 0x07\r
+#define _4BITS 0x0F\r
+#define _8BITS 0xFF\r
+\r
+#define MAGICNUMBER_SIZE 4\r
+#define LZ4S_MAGICNUMBER 0x184D2204\r
+#define LZ4S_SKIPPABLE0 0x184D2A50\r
+#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0\r
+#define LEGACY_MAGICNUMBER 0x184C2102\r
+\r
+#define CACHELINE 64\r
+#define LEGACY_BLOCKSIZE (8 MB)\r
+#define MIN_STREAM_BUFSIZE (1 MB + 64 KB)\r
+#define LZ4S_BLOCKSIZEID_DEFAULT 7\r
+#define LZ4S_CHECKSUM_SEED 0\r
+#define LZ4S_EOS 0\r
+#define LZ4S_MAXHEADERSIZE (4+2+8+4+1)\r
+\r
+\r
+//**************************************\r
+// Architecture Macros\r
+//**************************************\r
+static const int one = 1;\r
+#define CPU_LITTLE_ENDIAN (*(char*)(&one))\r
+#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN)\r
+#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i))\r
+\r
+\r
+//**************************************\r
+// Macros\r
+//**************************************\r
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)\r
+\r
+\r
+//**************************************\r
+// Special input/output\r
+//**************************************\r
+#define NULL_INPUT "null"\r
+char stdinmark[] = "stdin";\r
+char stdoutmark[] = "stdout";\r
+#ifdef _WIN32\r
+char nulmark[] = "nul";\r
+#else\r
+char nulmark[] = "/dev/null";\r
+#endif\r
+\r
+\r
+//**************************************\r
+// Local Parameters\r
+//**************************************\r
+static int overwrite = 0;\r
+static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;\r
+static int blockChecksum = 0;\r
+static int streamChecksum = 1;\r
+static int blockIndependence = 1;\r
+\r
+//**************************************\r
+// Exceptions\r
+//**************************************\r
+#define DEBUG 0\r
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);\r
+#define EXM_THROW(error, ...) \\r
+{ \\r
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \\r
+ DISPLAY("Error %i : ", error); \\r
+ DISPLAY(__VA_ARGS__); \\r
+ DISPLAY("\n"); \\r
+ exit(error); \\r
+}\r
+\r
+\r
+\r
+//****************************\r
+// Functions\r
+//****************************\r
+int usage(char* exename)\r
+{\r
+ DISPLAY( "Usage :\n");\r
+ DISPLAY( " %s [arg] input [output]\n", exename);\r
+ DISPLAY( "Arguments :\n");\r
+ DISPLAY( " -c0/-c : Fast compression (default) \n");\r
+ DISPLAY( " -c1/-hc: High compression \n");\r
+ DISPLAY( " -d : decompression \n");\r
+ DISPLAY( " -y : overwrite without prompting \n");\r
+ DISPLAY( " -H : Help (this text + advanced options)\n");\r
+ return 0;\r
+}\r
+\r
+int usage_advanced()\r
+{\r
+ DISPLAY( "\nAdvanced options :\n");\r
+ DISPLAY( " -t : test compressed file \n");\r
+ DISPLAY( " -B# : Block size [4-7](default : 7)\n");\r
+ DISPLAY( " -BD : Block dependency (improve compression ratio)\n");\r
+ DISPLAY( " -BX : enable block checksum (default:disabled)\n");\r
+ DISPLAY( " -Sx : disable stream checksum (default:enabled)\n");\r
+ DISPLAY( " -b# : benchmark files, using # [0-1] compression level\n");\r
+ DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");\r
+ DISPLAY( "input : can be 'stdin' (pipe) or a filename\n");\r
+ DISPLAY( "output : can be 'stdout'(pipe) or a filename or 'null'\n");\r
+ DISPLAY( " example 1 : lz4c -hc stdin compressedfile.lz4\n");\r
+ DISPLAY( " example 2 : lz4c -hcyB4D filename \n");\r
+ return 0;\r
+}\r
+\r
+int badusage(char* exename)\r
+{\r
+ DISPLAY("Wrong parameters\n");\r
+ usage(exename);\r
+ return 0;\r
+}\r
+\r
+\r
+static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }\r
+static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; }\r
+static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; }\r
+\r
+\r
+int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput)\r
+{\r
+\r
+ if (!strcmp (input_filename, stdinmark)) \r
+ {\r
+ DISPLAY( "Using stdin for input\n");\r
+ *pfinput = stdin;\r
+#ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows\r
+ _setmode( _fileno( stdin ), _O_BINARY );\r
+#endif\r
+ } \r
+ else \r
+ {\r
+ *pfinput = fopen(input_filename, "rb");\r
+ }\r
+\r
+ if (!strcmp (output_filename, stdoutmark)) \r
+ {\r
+ DISPLAY( "Using stdout for output\n");\r
+ *pfoutput = stdout;\r
+#ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows\r
+ _setmode( _fileno( stdout ), _O_BINARY );\r
+#endif\r
+ } \r
+ else \r
+ {\r
+ // Check if destination file already exists\r
+ *pfoutput=0;\r
+ if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" );\r
+ if (*pfoutput!=0) \r
+ { \r
+ char ch;\r
+ fclose(*pfoutput); \r
+ DISPLAY( "Warning : %s already exists\n", output_filename); \r
+ if (!overwrite)\r
+ {\r
+ DISPLAY( "Overwrite ? (Y/N) : ");\r
+ ch = (char)getchar();\r
+ if (ch!='Y') EXM_THROW(11, "Operation aborted : %s already exists", output_filename);\r
+ }\r
+ }\r
+ *pfoutput = fopen( output_filename, "wb" );\r
+ }\r
+\r
+ if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);\r
+ if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); \r
+\r
+ return 0;\r
+}\r
+\r
+\r
+\r
+int legacy_compress_file(char* input_filename, char* output_filename, int compressionlevel)\r
+{\r
+ int (*compressionFunction)(const char*, char*, int);\r
+ unsigned long long filesize = 0;\r
+ unsigned long long compressedfilesize = MAGICNUMBER_SIZE;\r
+ char* in_buff;\r
+ char* out_buff;\r
+ FILE* finput;\r
+ FILE* foutput;\r
+ int displayLevel = (compressionlevel>0);\r
+ clock_t start, end;\r
+ size_t sizeCheck;\r
+\r
+\r
+ // Init\r
+ switch (compressionlevel)\r
+ {\r
+ case 0 : compressionFunction = LZ4_compress; break;\r
+ case 1 : compressionFunction = LZ4_compressHC; break;\r
+ default: compressionFunction = LZ4_compress;\r
+ }\r
+ start = clock();\r
+ get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
+\r
+ // Allocate Memory\r
+ in_buff = (char*)malloc(LEGACY_BLOCKSIZE);\r
+ out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));\r
+ if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");\r
+\r
+ // Write Archive Header\r
+ *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER);\r
+ sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);\r
+ if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header");\r
+\r
+ // Main Loop\r
+ while (1)\r
+ {\r
+ unsigned int outSize;\r
+ // Read Block\r
+ int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);\r
+ if( inSize<=0 ) break;\r
+ filesize += inSize;\r
+ if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20));\r
+\r
+ // Compress Block\r
+ outSize = compressionFunction(in_buff, out_buff+4, inSize);\r
+ compressedfilesize += outSize+4;\r
+ if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100);\r
+\r
+ // Write Block\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);\r
+ sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);\r
+ if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block");\r
+ }\r
+\r
+ // Status\r
+ end = clock();\r
+ DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",\r
+ (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);\r
+ {\r
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;\r
+ DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);\r
+ }\r
+\r
+ // Close & Free\r
+ free(in_buff);\r
+ free(out_buff);\r
+ fclose(finput);\r
+ fclose(foutput);\r
+\r
+ return 0;\r
+}\r
+\r
+\r
+int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)\r
+{\r
+ void* (*initFunction) (const char*);\r
+ int (*compressionFunction)(void*, const char*, char*, int, int);\r
+ char* (*translateFunction) (void*);\r
+ int (*freeFunction) (void*);\r
+ void* ctx;\r
+ unsigned long long filesize = 0;\r
+ unsigned long long compressedfilesize = 0;\r
+ unsigned int checkbits;\r
+ char* in_buff, *in_start, *in_end;\r
+ char* out_buff;\r
+ FILE* finput;\r
+ FILE* foutput;\r
+ int errorcode;\r
+ int displayLevel = (compressionlevel>0);\r
+ clock_t start, end;\r
+ unsigned int blockSize, inputBufferSize;\r
+ size_t sizeCheck, header_size;\r
+ void* streamChecksumState=NULL;\r
+\r
+\r
+ // Init\r
+ start = clock();\r
+ switch (compressionlevel)\r
+ {\r
+ case 0 :\r
+ case 1 :\r
+ default:\r
+ initFunction = LZ4_createHC;\r
+ compressionFunction = LZ4_compressHC_limitedOutput_continue;\r
+ translateFunction = LZ4_slideInputBufferHC;\r
+ freeFunction = LZ4_freeHC;\r
+ }\r
+ errorcode = get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
+ if (errorcode) return errorcode;\r
+ blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);\r
+\r
+ // Allocate Memory\r
+ inputBufferSize = blockSize + 64 KB;\r
+ if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;\r
+ in_buff = (char*)malloc(inputBufferSize);\r
+ out_buff = (char*)malloc(blockSize+CACHELINE);\r
+ if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");\r
+ in_start = in_buff; in_end = in_buff + inputBufferSize;\r
+ if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);\r
+ ctx = initFunction(in_buff);\r
+\r
+ // Write Archive Header\r
+ *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention\r
+ *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01')\r
+ *(out_buff+4) |= (blockIndependence & _1BIT) << 5;\r
+ *(out_buff+4) |= (blockChecksum & _1BIT) << 4;\r
+ *(out_buff+4) |= (streamChecksum & _1BIT) << 2;\r
+ *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4);\r
+ checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED);\r
+ checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);\r
+ *(out_buff+6) = (unsigned char) checkbits;\r
+ header_size = 7;\r
+ sizeCheck = fwrite(out_buff, 1, header_size, foutput);\r
+ if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");\r
+ compressedfilesize += header_size;\r
+\r
+ // Main Loop\r
+ while (1)\r
+ {\r
+ unsigned int outSize;\r
+ unsigned int inSize;\r
+ // Read Block\r
+ if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx);\r
+ inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);\r
+ if( inSize<=0 ) break; // No more input : end of compression\r
+ filesize += inSize;\r
+ if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20));\r
+ if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize);\r
+\r
+ // Compress Block\r
+ outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1);\r
+ if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;\r
+ if (blockChecksum) compressedfilesize+=4;\r
+ if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100);\r
+\r
+ // Write Block\r
+ if (outSize > 0)\r
+ {\r
+ unsigned int checksum;\r
+ int sizeToWrite;\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);\r
+ if (blockChecksum)\r
+ {\r
+ checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);\r
+ * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);\r
+ }\r
+ sizeToWrite = 4 + outSize + (4*blockChecksum);\r
+ sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);\r
+ if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");\r
+\r
+ }\r
+ else // Copy Original\r
+ {\r
+ unsigned int checksum;\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");\r
+ sizeCheck = fwrite(in_start, 1, inSize, foutput);\r
+ if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");\r
+ if (blockChecksum)\r
+ {\r
+ checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED);\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");\r
+ }\r
+ }\r
+ in_start += inSize;\r
+ }\r
+\r
+ // End of Stream mark\r
+ * (unsigned int*) out_buff = LZ4S_EOS;\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");\r
+ compressedfilesize += 4;\r
+ if (streamChecksum)\r
+ {\r
+ unsigned int checksum = XXH32_digest(streamChecksumState);\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");\r
+ compressedfilesize += 4;\r
+ }\r
+\r
+ // Status\r
+ end = clock();\r
+ DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",\r
+ (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);\r
+ {\r
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;\r
+ DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);\r
+ }\r
+\r
+ // Close & Free\r
+ freeFunction(ctx);\r
+ free(in_buff);\r
+ free(out_buff);\r
+ fclose(finput);\r
+ fclose(foutput);\r
+\r
+ return 0;\r
+}\r
+\r
+\r
+int compress_file(char* input_filename, char* output_filename, int compressionlevel)\r
+{\r
+ int (*compressionFunction)(const char*, char*, int, int);\r
+ unsigned long long filesize = 0;\r
+ unsigned long long compressedfilesize = 0;\r
+ unsigned int checkbits;\r
+ char* in_buff;\r
+ char* out_buff;\r
+ FILE* finput;\r
+ FILE* foutput;\r
+ int errorcode;\r
+ int displayLevel = (compressionlevel>0);\r
+ clock_t start, end;\r
+ int blockSize;\r
+ size_t sizeCheck, header_size;\r
+ void* streamChecksumState=NULL;\r
+\r
+ // Branch out\r
+ if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionlevel);\r
+\r
+ // Init\r
+ start = clock();\r
+ switch (compressionlevel)\r
+ {\r
+ case 0 : compressionFunction = LZ4_compress_limitedOutput; break;\r
+ case 1 : compressionFunction = LZ4_compressHC_limitedOutput; break;\r
+ default: compressionFunction = LZ4_compress_limitedOutput;\r
+ }\r
+ errorcode = get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
+ if (errorcode) return errorcode;\r
+ blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);\r
+\r
+ // Allocate Memory\r
+ in_buff = (char*)malloc(blockSize);\r
+ out_buff = (char*)malloc(blockSize+CACHELINE);\r
+ if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");\r
+ if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);\r
+\r
+ // Write Archive Header\r
+ *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention\r
+ *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01')\r
+ *(out_buff+4) |= (blockIndependence & _1BIT) << 5;\r
+ *(out_buff+4) |= (blockChecksum & _1BIT) << 4;\r
+ *(out_buff+4) |= (streamChecksum & _1BIT) << 2;\r
+ *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4);\r
+ checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED);\r
+ checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);\r
+ *(out_buff+6) = (unsigned char) checkbits;\r
+ header_size = 7;\r
+ sizeCheck = fwrite(out_buff, 1, header_size, foutput);\r
+ if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");\r
+ compressedfilesize += header_size;\r
+\r
+ // Main Loop\r
+ while (1)\r
+ {\r
+ unsigned int outSize;\r
+ // Read Block\r
+ unsigned int inSize = (unsigned int) fread(in_buff, (size_t)1, (size_t)blockSize, finput);\r
+ if( inSize<=0 ) break; // No more input : end of compression\r
+ filesize += inSize;\r
+ if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20));\r
+ if (streamChecksum) XXH32_update(streamChecksumState, in_buff, inSize);\r
+\r
+ // Compress Block\r
+ outSize = compressionFunction(in_buff, out_buff+4, inSize, inSize-1);\r
+ if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;\r
+ if (blockChecksum) compressedfilesize+=4;\r
+ if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100);\r
+\r
+ // Write Block\r
+ if (outSize > 0)\r
+ {\r
+ unsigned int checksum;\r
+ int sizeToWrite;\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);\r
+ if (blockChecksum)\r
+ {\r
+ checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);\r
+ * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);\r
+ }\r
+ sizeToWrite = 4 + outSize + (4*blockChecksum);\r
+ sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);\r
+ if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");\r
+\r
+ }\r
+ else // Copy Original\r
+ {\r
+ unsigned int checksum;\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");\r
+ sizeCheck = fwrite(in_buff, 1, inSize, foutput);\r
+ if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");\r
+ if (blockChecksum)\r
+ {\r
+ checksum = XXH32(in_buff, inSize, LZ4S_CHECKSUM_SEED);\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");\r
+ }\r
+ }\r
+ }\r
+\r
+ // End of Stream mark\r
+ * (unsigned int*) out_buff = LZ4S_EOS;\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");\r
+ compressedfilesize += 4;\r
+ if (streamChecksum)\r
+ {\r
+ unsigned int checksum = XXH32_digest(streamChecksumState);\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");\r
+ compressedfilesize += 4;\r
+ }\r
+\r
+ // Status\r
+ end = clock();\r
+ DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",\r
+ (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);\r
+ {\r
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;\r
+ DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);\r
+ }\r
+\r
+ // Close & Free\r
+ free(in_buff);\r
+ free(out_buff);\r
+ fclose(finput);\r
+ fclose(foutput);\r
+\r
+ return 0;\r
+}\r
+\r
+\r
+unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)\r
+{\r
+ unsigned long long filesize = 0;\r
+ char* in_buff;\r
+ char* out_buff;\r
+ size_t uselessRet;\r
+ int sinkint;\r
+ unsigned int blockSize;\r
+ size_t sizeCheck;\r
+\r
+\r
+ // Allocate Memory\r
+ in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));\r
+ out_buff = (char*)malloc(LEGACY_BLOCKSIZE);\r
+ if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");\r
+\r
+ // Main Loop\r
+ while (1)\r
+ {\r
+ // Block Size\r
+ uselessRet = fread(&blockSize, 1, 4, finput);\r
+ if( uselessRet==0 ) break; // Nothing to read : file read is completed\r
+ blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian\r
+ if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) \r
+ { // Cannot read next block : maybe new stream ?\r
+ fseek(finput, -4, SEEK_CUR);\r
+ break;\r
+ }\r
+\r
+ // Read Block\r
+ uselessRet = fread(in_buff, 1, blockSize, finput);\r
+\r
+ // Decode Block\r
+ sinkint = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);\r
+ if (sinkint < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !");\r
+ filesize += sinkint;\r
+\r
+ // Write Block\r
+ sizeCheck = fwrite(out_buff, 1, sinkint, foutput);\r
+ if (sizeCheck != (size_t)sinkint) EXM_THROW(53, "Write error : cannot write decoded block into output\n");\r
+ }\r
+\r
+ // Free\r
+ free(in_buff);\r
+ free(out_buff);\r
+\r
+ return filesize;\r
+}\r
+\r
+\r
+unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)\r
+{\r
+ unsigned long long filesize = 0;\r
+ char* in_buff;\r
+ char* out_buff, *out_start, *out_end;\r
+ unsigned char descriptor[LZ4S_MAXHEADERSIZE];\r
+ size_t nbReadBytes;\r
+ int decodedBytes=0;\r
+ unsigned int maxBlockSize;\r
+ size_t sizeCheck;\r
+ int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag;\r
+ void* streamChecksumState=NULL;\r
+ int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe;\r
+ unsigned int prefix64k = 0;\r
+\r
+ // Decode stream descriptor\r
+ nbReadBytes = fread(descriptor, 1, 3, finput);\r
+ if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header");\r
+ {\r
+ int version = (descriptor[0] >> 6) & _2BITS;\r
+ int streamSize = (descriptor[0] >> 3) & _1BIT;\r
+ int reserved1 = (descriptor[0] >> 1) & _1BIT;\r
+ int dictionary = (descriptor[0] >> 0) & _1BIT;\r
+\r
+ int reserved2 = (descriptor[1] >> 7) & _1BIT;\r
+ int blockSizeId = (descriptor[1] >> 4) & _3BITS;\r
+ int reserved3 = (descriptor[1] >> 0) & _4BITS;\r
+ int checkBits = (descriptor[2] >> 0) & _8BITS;\r
+ int checkBits_xxh32;\r
+\r
+ blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT;\r
+ blockChecksumFlag = (descriptor[0] >> 4) & _1BIT;\r
+ streamChecksumFlag= (descriptor[0] >> 2) & _1BIT;\r
+\r
+ if (version != 1) EXM_THROW(62, "Wrong version number");\r
+ if (streamSize == 1) EXM_THROW(64, "Does not support stream size"); \r
+ if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits");\r
+ if (dictionary == 1) EXM_THROW(66, "Does not support dictionary"); \r
+ if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits");\r
+ if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size"); \r
+ if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits");\r
+ maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId);\r
+ // Checkbits verification\r
+ descriptor[1] &= 0xF0;\r
+ checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED);\r
+ checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32);\r
+ if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected");\r
+ }\r
+\r
+ if (!blockIndependenceFlag)\r
+ {\r
+ decompressionFunction = LZ4_decompress_safe_withPrefix64k;\r
+ prefix64k = 64 KB;\r
+ }\r
+\r
+ // Allocate Memory\r
+ {\r
+ unsigned int outbuffSize = prefix64k+maxBlockSize;\r
+ in_buff = (char*)malloc(maxBlockSize);\r
+ if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE;\r
+ out_buff = (char*)malloc(outbuffSize); \r
+ out_end = out_buff + outbuffSize;\r
+ out_start = out_buff + prefix64k;\r
+ if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");\r
+ }\r
+ if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);\r
+\r
+ // Main Loop\r
+ while (1)\r
+ {\r
+ unsigned int blockSize, uncompressedFlag;\r
+\r
+ // Block Size\r
+ nbReadBytes = fread(&blockSize, 1, 4, finput);\r
+ if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size");\r
+ if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed\r
+ blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian\r
+ uncompressedFlag = blockSize >> 31;\r
+ blockSize &= 0x7FFFFFFF;\r
+ if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size");\r
+\r
+ // Read Block\r
+ nbReadBytes = fread(in_buff, 1, blockSize, finput);\r
+ if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" );\r
+\r
+ // Check Block\r
+ if (blockChecksumFlag)\r
+ {\r
+ unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED);\r
+ unsigned int readChecksum;\r
+ sizeCheck = fread(&readChecksum, 1, 4, finput);\r
+ if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size");\r
+ readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian\r
+ if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected");\r
+ }\r
+\r
+ if (uncompressedFlag)\r
+ {\r
+ // Write uncompressed Block\r
+ sizeCheck = fwrite(in_buff, 1, blockSize, foutput);\r
+ if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block");\r
+ filesize += blockSize;\r
+ if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);\r
+ if (!blockIndependenceFlag)\r
+ {\r
+ if (blockSize >= prefix64k)\r
+ {\r
+ memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks\r
+ out_start = out_buff + prefix64k;\r
+ continue;\r
+ }\r
+ else\r
+ {\r
+ memcpy(out_start, in_buff, blockSize);\r
+ }\r
+ }\r
+ }\r
+ else\r
+ {\r
+ // Decode Block\r
+ decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize);\r
+ if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");\r
+ filesize += decodedBytes;\r
+ if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes);\r
+\r
+ // Write Block\r
+ sizeCheck = fwrite(out_start, 1, decodedBytes, foutput);\r
+ if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n");\r
+ }\r
+\r
+ if (!blockIndependenceFlag)\r
+ {\r
+ out_start += decodedBytes;\r
+ if (out_start + maxBlockSize > out_end) \r
+ {\r
+ memcpy(out_buff, out_start - prefix64k, prefix64k); \r
+ out_start = out_buff + prefix64k; \r
+ }\r
+ }\r
+ }\r
+\r
+ // Stream Checksum\r
+ if (streamChecksumFlag)\r
+ {\r
+ unsigned int checksum = XXH32_digest(streamChecksumState);\r
+ unsigned int readChecksum;\r
+ sizeCheck = fread(&readChecksum, 1, 4, finput);\r
+ if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum");\r
+ readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian\r
+ if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected");\r
+ }\r
+\r
+ // Free\r
+ free(in_buff);\r
+ free(out_buff);\r
+\r
+ return filesize;\r
+}\r
+\r
+\r
+unsigned long long selectDecoder( FILE* finput, FILE* foutput)\r
+{\r
+ unsigned int magicNumber, size;\r
+ int errorNb;\r
+ size_t nbReadBytes;\r
+\r
+ // Check Archive Header\r
+ nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput);\r
+ if (nbReadBytes==0) return 0; // EOF\r
+ if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable");\r
+ magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format\r
+ if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers\r
+\r
+ switch(magicNumber)\r
+ {\r
+ case LZ4S_MAGICNUMBER:\r
+ return decodeLZ4S(finput, foutput);\r
+ case LEGACY_MAGICNUMBER:\r
+ DISPLAY("Detected : Legacy format \n");\r
+ return decodeLegacyStream(finput, foutput);\r
+ case LZ4S_SKIPPABLE0:\r
+ nbReadBytes = fread(&size, 1, 4, finput);\r
+ if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");\r
+ size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format\r
+ errorNb = fseek(finput, size, SEEK_CUR);\r
+ if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");\r
+ return selectDecoder(finput, foutput);\r
+ default:\r
+ if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream\r
+ DISPLAY("Stream followed by unrecognized data\n");\r
+ return 0;\r
+ }\r
+}\r
+\r
+\r
+int decodeFile(char* input_filename, char* output_filename)\r
+{\r
+ unsigned long long filesize = 0, decodedSize=0;\r
+ FILE* finput;\r
+ FILE* foutput;\r
+ clock_t start, end;\r
+\r
+\r
+ // Init\r
+ start = clock();\r
+ get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
+\r
+ // Loop over multiple streams\r
+ do \r
+ {\r
+ decodedSize = selectDecoder(finput, foutput);\r
+ filesize += decodedSize;\r
+ } while (decodedSize);\r
+\r
+ // Final Status\r
+ end = clock();\r
+ DISPLAY( "Successfully decoded %llu bytes \n", filesize);\r
+ {\r
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;\r
+ DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);\r
+ }\r
+\r
+ // Close\r
+ fclose(finput);\r
+ fclose(foutput);\r
+\r
+ // Error status = OK\r
+ return 0;\r
+}\r
+\r
+\r
+int main(int argc, char** argv)\r
+{\r
+ int i,\r
+ cLevel=0,\r
+ decode=0,\r
+ bench=0,\r
+ filenamesStart=2,\r
+ legacy_format=0;\r
+ char* exename=argv[0];\r
+ char* input_filename=0;\r
+ char* output_filename=0;\r
+ char nullinput[] = NULL_INPUT;\r
+ char extension[] = EXTENSION;\r
+\r
+ // Welcome message\r
+ DISPLAY( WELCOME_MESSAGE);\r
+\r
+ if (argc<2) { badusage(exename); return 1; }\r
+\r
+ for(i=1; i<argc; i++)\r
+ {\r
+ char* argument = argv[i];\r
+\r
+ if(!argument) continue; // Protection if argument empty\r
+\r
+ // Decode command (note : aggregated commands are allowed)\r
+ if (argument[0]=='-')\r
+ {\r
+ while (argument[1]!=0)\r
+ {\r
+ argument ++;\r
+\r
+ switch(argument[0])\r
+ {\r
+ // Display help on usage\r
+ case 'H': usage(exename); usage_advanced(); return 0;\r
+\r
+ // Compression (default)\r
+ case 'c': if ((argument[1] >='0') && (argument[1] <='1')) { cLevel=argument[1] - '0'; argument++; } break;\r
+ case 'h': if (argument[1]=='c') { cLevel=1; argument++; } break;\r
+\r
+ // Use Legacy format (hidden option)\r
+ case 'l': legacy_format=1; break;\r
+\r
+ // Decoding\r
+ case 'd': decode=1; break;\r
+\r
+ // Test\r
+ case 't': decode=1; output_filename=nulmark; break;\r
+\r
+ // Modify Block Properties\r
+ case 'B':\r
+ while (argument[1]!=0)\r
+ switch(argument[1])\r
+ {\r
+ case '4':\r
+ case '5':\r
+ case '6':\r
+ case '7':\r
+ { \r
+ int B = argument[1] - '0'; \r
+ int S = 1 << (8 + 2*B); \r
+ BMK_SetBlocksize(S); \r
+ blockSizeId = B;\r
+ argument++;\r
+ break;\r
+ }\r
+ case 'D': blockIndependence = 0, argument++; break;\r
+ case 'X': blockChecksum = 1, argument ++; break;\r
+ default : goto _exit_blockProperties;\r
+ }\r
+_exit_blockProperties:\r
+ break;\r
+\r
+ // Modify Stream properties\r
+ case 'S': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(exename); return 1; }\r
+\r
+ // Bench\r
+ case 'b': bench=1; \r
+ if ((argument[1] >='0') && (argument[1] <='1')) { cLevel=argument[1] - '0'; argument++; } \r
+ break;\r
+\r
+ // Modify Nb Iterations (benchmark only)\r
+ case 'i': \r
+ if ((argument[1] >='1') && (argument[1] <='9'))\r
+ {\r
+ int iters = argument[1] - '0'; \r
+ BMK_SetNbIterations(iters); \r
+ argument++;\r
+ }\r
+ break;\r
+\r
+ // Pause at the end (benchmark only) (hidden option)\r
+ case 'p': BMK_SetPause(); break;\r
+\r
+ // Overwrite\r
+ case 'y': overwrite=1; break;\r
+\r
+ // Unrecognised command\r
+ default : badusage(exename); return 1;\r
+ }\r
+ }\r
+ continue;\r
+ }\r
+\r
+ // first provided filename is input\r
+ if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }\r
+\r
+ // second provided filename is output\r
+ if (!output_filename)\r
+ {\r
+ output_filename=argument;\r
+ if (!strcmp (output_filename, nullinput)) output_filename = nulmark;\r
+ continue;\r
+ }\r
+ }\r
+\r
+ // No input filename ==> Error\r
+ if(!input_filename) { badusage(exename); return 1; }\r
+\r
+ if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel);\r
+\r
+ // No output filename ==> build one automatically (when possible)\r
+ if (!output_filename) \r
+ { \r
+ if (!decode) // compression\r
+ {\r
+ int i=0, l=0;\r
+ while (input_filename[l]!=0) l++;\r
+ output_filename = (char*)calloc(1,l+5);\r
+ for (i=0;i<l;i++) output_filename[i] = input_filename[i];\r
+ for (i=l;i<l+4;i++) output_filename[i] = extension[i-l];\r
+ }\r
+ else // decompression (input file must respect format extension ".lz4")\r
+ {\r
+ int inl=0,outl;\r
+ while (input_filename[inl]!=0) inl++;\r
+ output_filename = (char*)calloc(1,inl+1);\r
+ for (outl=0;outl<inl;outl++) output_filename[outl] = input_filename[outl];\r
+ if (inl>4)\r
+ while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0;\r
+ if (outl != inl-5) output_filename = NULL;\r
+ }\r
+ if (!output_filename) { badusage(exename); return 1; }\r
+ }\r
+\r
+ if (decode) return decodeFile(input_filename, output_filename);\r
+\r
+ // compression is default action\r
+ if (legacy_format)\r
+ {\r
+ DISPLAY("! Generating compressed LZ4 using Legacy format (deprecated !) ! \n");\r
+ return legacy_compress_file(input_filename, output_filename, cLevel); \r
+ }\r
+ else\r
+ {\r
+ return compress_file(input_filename, output_filename, cLevel); \r
+ }\r
+}\r
projects / c11concurrency-benchmarks.git / blobdiff