2 bench.c - Demo program to benchmark open-source compression algorithm
\r
3 Copyright (C) Yann Collet 2012-2013
\r
6 This program is free software; you can redistribute it and/or modify
\r
7 it under the terms of the GNU General Public License as published by
\r
8 the Free Software Foundation; either version 2 of the License, or
\r
9 (at your option) any later version.
\r
11 This program is distributed in the hope that it will be useful,
\r
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 GNU General Public License for more details.
\r
16 You should have received a copy of the GNU General Public License along
\r
17 with this program; if not, write to the Free Software Foundation, Inc.,
\r
18 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
\r
20 You can contact the author at :
\r
21 - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
\r
22 - LZ4 source repository : http://code.google.com/p/lz4/
\r
25 //**************************************
\r
27 //**************************************
\r
28 // Disable some Visual warning messages
\r
29 #define _CRT_SECURE_NO_WARNINGS
\r
30 #define _CRT_SECURE_NO_DEPRECATE // VS2005
\r
32 // Unix Large Files support (>4GB)
\r
33 #if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
\r
34 # define _LARGEFILE_SOURCE
\r
35 # define _FILE_OFFSET_BITS 64
\r
36 #elif ! defined(__LP64__) // No point defining Large file for 64 bit
\r
37 # define _LARGEFILE64_SOURCE
\r
40 // S_ISREG & gettimeofday() are not supported by MSVC
\r
41 #if defined(_MSC_VER)
\r
42 # define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
\r
43 # define BMK_LEGACY_TIMER 1
\r
46 // GCC does not support _rotl outside of Windows
\r
47 #if !defined(_WIN32)
\r
48 # define _rotl(x,r) ((x << r) | (x >> (32 - r)))
\r
52 //**************************************
\r
54 //**************************************
\r
55 #include <stdlib.h> // malloc
\r
56 #include <stdio.h> // fprintf, fopen, ftello64
\r
57 #include <sys/types.h> // stat64
\r
58 #include <sys/stat.h> // stat64
\r
60 // Use ftime() if gettimeofday() is not available on your target
\r
61 #if defined(BMK_LEGACY_TIMER)
\r
62 # include <sys/timeb.h> // timeb, ftime
\r
64 # include <sys/time.h> // gettimeofday
\r
68 #define COMPRESSOR0 LZ4_compress
\r
70 #define COMPRESSOR1 LZ4_compressHC
\r
71 #define DEFAULTCOMPRESSOR COMPRESSOR0
\r
76 //**************************************
\r
78 //**************************************
\r
79 #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
\r
80 # include <stdint.h>
\r
81 typedef uint8_t BYTE;
\r
82 typedef uint16_t U16;
\r
83 typedef uint32_t U32;
\r
84 typedef int32_t S32;
\r
85 typedef uint64_t U64;
\r
87 typedef unsigned char BYTE;
\r
88 typedef unsigned short U16;
\r
89 typedef unsigned int U32;
\r
90 typedef signed int S32;
\r
91 typedef unsigned long long U64;
\r
95 //****************************
\r
97 //****************************
\r
98 #define COMPRESSOR_NAME "Full LZ4 speed analyzer"
\r
99 #define COMPRESSOR_VERSION ""
\r
100 #define COMPILED __DATE__
\r
101 #define AUTHOR "Yann Collet"
\r
102 #define WELCOME_MESSAGE "*** %s %s, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, AUTHOR, COMPILED
\r
105 #define TIMELOOP 2500
\r
107 #define KNUTH 2654435761U
\r
108 #define MAX_MEM (1984<<20)
\r
109 #define DEFAULT_CHUNKSIZE (4<<20)
\r
112 //**************************************
\r
113 // Local structures
\r
114 //**************************************
\r
115 struct chunkParameters
\r
119 char* compressedBuffer;
\r
121 int compressedSize;
\r
125 //**************************************
\r
127 //**************************************
\r
128 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
\r
132 //**************************************
\r
133 // Benchmark Parameters
\r
134 //**************************************
\r
135 static int chunkSize = DEFAULT_CHUNKSIZE;
\r
136 static int nbIterations = NBLOOPS;
\r
137 static int BMK_pause = 0;
\r
138 static int compressionTest = 1;
\r
139 static int decompressionTest = 1;
\r
141 void BMK_SetBlocksize(int bsize)
\r
144 DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10);
\r
147 void BMK_SetNbIterations(int nbLoops)
\r
149 nbIterations = nbLoops;
\r
150 DISPLAY("- %i iterations -\n", nbIterations);
\r
153 void BMK_SetPause()
\r
158 //*********************************************************
\r
159 // Private functions
\r
160 //*********************************************************
\r
162 #if defined(BMK_LEGACY_TIMER)
\r
164 static int BMK_GetMilliStart()
\r
166 // Based on Legacy ftime()
\r
167 // Rolls over every ~ 12.1 days (0x100000/24/60/60)
\r
168 // Use GetMilliSpan to correct for rollover
\r
172 nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
\r
178 static int BMK_GetMilliStart()
\r
180 // Based on newer gettimeofday()
\r
181 // Use GetMilliSpan to correct for rollover
\r
184 gettimeofday(&tv, NULL);
\r
185 nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
\r
192 static int BMK_GetMilliSpan( int nTimeStart )
\r
194 int nSpan = BMK_GetMilliStart() - nTimeStart;
\r
196 nSpan += 0x100000 * 1000;
\r
201 static size_t BMK_findMaxMem(U64 requiredMem)
\r
203 size_t step = (64U<<20); // 64 MB
\r
204 BYTE* testmem=NULL;
\r
206 requiredMem = (((requiredMem >> 25) + 1) << 26);
\r
207 if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
\r
209 requiredMem += 2*step;
\r
212 requiredMem -= step;
\r
213 testmem = (BYTE*) malloc ((size_t)requiredMem);
\r
217 return (size_t) (requiredMem - step);
\r
221 static U64 BMK_GetFileSize(char* infilename)
\r
224 #if defined(_MSC_VER)
\r
225 struct _stat64 statbuf;
\r
226 r = _stat64(infilename, &statbuf);
\r
228 struct stat statbuf;
\r
229 r = stat(infilename, &statbuf);
\r
231 if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...
\r
232 return (U64)statbuf.st_size;
\r
236 //*********************************************************
\r
238 //*********************************************************
\r
240 static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
\r
242 return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
\r
245 static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
\r
247 return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
\r
250 static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
\r
253 LZ4_decompress_fast(in, out, outSize);
\r
257 static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
\r
260 LZ4_decompress_fast_withPrefix64k(in, out, outSize);
\r
264 static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
\r
266 return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
\r
269 int fullSpeedBench(char** fileNamesTable, int nbFiles)
\r
275 size_t benchedSize;
\r
280 char* compressed_buff; int compressed_buff_size;
\r
281 struct chunkParameters* chunkP;
\r
283 # define NB_COMPRESSION_ALGORITHMS 4
\r
284 static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput", "LZ4_compressHC", "LZ4_compressHC_limitedOutput" };
\r
285 double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0};
\r
286 double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0};
\r
287 # define NB_DECOMPRESSION_ALGORITHMS 5
\r
288 static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_partial" };
\r
289 double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0};
\r
294 // Loop for each file
\r
295 while (fileIdx<nbFiles)
\r
297 // Check file existence
\r
298 infilename = fileNamesTable[fileIdx++];
\r
299 fileIn = fopen( infilename, "rb" );
\r
302 DISPLAY( "Pb opening %s\n", infilename);
\r
306 // Memory allocation & restrictions
\r
307 largefilesize = BMK_GetFileSize(infilename);
\r
308 benchedSize = (size_t) BMK_findMaxMem(largefilesize) / 2;
\r
309 if ((U64)benchedSize > largefilesize) benchedSize = (size_t)largefilesize;
\r
310 if (benchedSize < largefilesize)
\r
312 DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", infilename, (int)(benchedSize>>20));
\r
316 chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters));
\r
317 orig_buff = (char*) malloc((size_t)benchedSize);
\r
318 nbChunks = (int) (benchedSize / chunkSize) + 1;
\r
319 maxCChunkSize = LZ4_compressBound(chunkSize);
\r
320 compressed_buff_size = nbChunks * maxCChunkSize;
\r
321 compressed_buff = (char*)malloc((size_t)compressed_buff_size);
\r
324 if(!orig_buff || !compressed_buff)
\r
326 DISPLAY("\nError: not enough memory!\n");
\r
328 free(compressed_buff);
\r
333 // Init chunks data
\r
336 size_t remaining = benchedSize;
\r
337 char* in = orig_buff;
\r
338 char* out = compressed_buff;
\r
339 for (i=0; i<nbChunks; i++)
\r
342 chunkP[i].origBuffer = in; in += chunkSize;
\r
343 if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
\r
344 chunkP[i].compressedBuffer = out; out += maxCChunkSize;
\r
345 chunkP[i].compressedSize = 0;
\r
349 // Fill input buffer
\r
350 DISPLAY("Loading %s... \r", infilename);
\r
351 readSize = fread(orig_buff, 1, benchedSize, fileIn);
\r
354 if(readSize != benchedSize)
\r
356 DISPLAY("\nError: problem reading file '%s' !! \n", infilename);
\r
358 free(compressed_buff);
\r
362 // Calculating input Checksum
\r
363 crcc = XXH32(orig_buff, (unsigned int)benchedSize,0);
\r
368 int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb;
\r
372 DISPLAY("\r%79s\r", "");
\r
373 DISPLAY(" %s : \n", infilename);
\r
375 // Compression Algorithms
\r
376 for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
\r
378 char* cName = compressionNames[cAlgNb];
\r
379 int (*compressionFunction)(const char*, char*, int);
\r
380 double bestTime = 100000000.;
\r
384 case 0: compressionFunction = LZ4_compress; break;
\r
385 case 1: compressionFunction = local_LZ4_compress_limitedOutput; break;
\r
386 case 2: compressionFunction = LZ4_compressHC; break;
\r
387 case 3: compressionFunction = local_LZ4_compressHC_limitedOutput; break;
\r
388 default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); return 1;
\r
391 for (loopNb = 1; loopNb <= nbIterations; loopNb++)
\r
393 double averageTime;
\r
396 DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize);
\r
397 { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory
\r
400 milliTime = BMK_GetMilliStart();
\r
401 while(BMK_GetMilliStart() == milliTime);
\r
402 milliTime = BMK_GetMilliStart();
\r
403 while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
\r
405 for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
\r
407 chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
\r
408 if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1);
\r
412 milliTime = BMK_GetMilliSpan(milliTime);
\r
414 averageTime = (double)milliTime / nb_loops;
\r
415 if (averageTime < bestTime) bestTime = averageTime;
\r
416 cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
\r
417 ratio = (double)cSize/(double)benchedSize*100.;
\r
418 DISPLAY("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
\r
422 DISPLAY("%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
\r
424 DISPLAY("%-21.21s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
\r
426 totalCTime[cAlgNb] += bestTime;
\r
427 totalCSize[cAlgNb] += cSize;
\r
430 // Prepare layout for decompression
\r
431 for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
\r
433 chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
\r
434 if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1);
\r
436 { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking
\r
438 // Decompression Algorithms
\r
439 for (dAlgNb=0; (dAlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++)
\r
441 char* dName = decompressionNames[dAlgNb];
\r
442 int (*decompressionFunction)(const char*, char*, int, int);
\r
443 double bestTime = 100000000.;
\r
447 case 0: decompressionFunction = local_LZ4_decompress_fast; break;
\r
448 case 1: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; break;
\r
449 case 2: decompressionFunction = LZ4_decompress_safe; break;
\r
450 case 3: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break;
\r
451 case 4: decompressionFunction = local_LZ4_decompress_safe_partial; break;
\r
452 default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); return 1;
\r
455 for (loopNb = 1; loopNb <= nbIterations; loopNb++)
\r
457 double averageTime;
\r
460 DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, dName, (int)benchedSize);
\r
463 milliTime = BMK_GetMilliStart();
\r
464 while(BMK_GetMilliStart() == milliTime);
\r
465 milliTime = BMK_GetMilliStart();
\r
466 while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
\r
468 for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
\r
470 int decodedSize = decompressionFunction(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize);
\r
471 if (chunkP[chunkNb].origSize != decodedSize) DISPLAY("ERROR ! %s() == %i != %i !! \n", dName, decodedSize, chunkP[chunkNb].origSize), exit(1);
\r
475 milliTime = BMK_GetMilliSpan(milliTime);
\r
477 averageTime = (double)milliTime / nb_loops;
\r
478 if (averageTime < bestTime) bestTime = averageTime;
\r
480 DISPLAY("%1i-%-19.19s : %9i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
\r
484 crcd = XXH32(orig_buff, (int)benchedSize, 0);
\r
485 if (crcc!=crcd) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", infilename, (unsigned)crcc, (unsigned)crcd); exit(1); }
\r
486 DISPLAY("%-21.21s : %9i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
\r
488 totalDTime[dAlgNb] += bestTime;
\r
491 totals += benchedSize;
\r
495 free(compressed_buff);
\r
503 DISPLAY(" ** TOTAL ** : \n");
\r
504 for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++)
\r
506 char* cName = compressionNames[AlgNb];
\r
507 DISPLAY("%-21.21s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.);
\r
509 for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++)
\r
511 char* dName = decompressionNames[AlgNb];
\r
512 DISPLAY("%-21.21s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.);
\r
516 if (BMK_pause) { printf("press enter...\n"); getchar(); }
\r
522 int usage(char* exename)
\r
524 DISPLAY( "Usage :\n");
\r
525 DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
\r
526 DISPLAY( "Arguments :\n");
\r
527 DISPLAY( " -c : compression tests only\n");
\r
528 DISPLAY( " -d : decompression tests only\n");
\r
529 DISPLAY( " -H : Help (this text + advanced options)\n");
\r
533 int usage_advanced()
\r
535 DISPLAY( "\nAdvanced options :\n");
\r
536 DISPLAY( " -B# : Block size [4-7](default : 7)\n");
\r
537 //DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
\r
538 DISPLAY( " -i# : iteration loops [1-9](default : 6)\n");
\r
542 int badusage(char* exename)
\r
544 DISPLAY("Wrong parameters\n");
\r
549 int main(int argc, char** argv)
\r
553 char* exename=argv[0];
\r
554 char* input_filename=0;
\r
557 DISPLAY( WELCOME_MESSAGE);
\r
559 if (argc<2) { badusage(exename); return 1; }
\r
561 for(i=1; i<argc; i++)
\r
563 char* argument = argv[i];
\r
565 if(!argument) continue; // Protection if argument empty
\r
567 // Decode command (note : aggregated commands are allowed)
\r
568 if (argument[0]=='-')
\r
570 while (argument[1]!=0)
\r
574 switch(argument[0])
\r
576 // Select compression algorithm only
\r
577 case 'c': decompressionTest = 0; break;
\r
579 // Select decompression algorithm only
\r
580 case 'd': compressionTest = 0; break;
\r
582 // Display help on usage
\r
583 case 'H': usage(exename); usage_advanced(); return 0;
\r
585 // Modify Block Properties
\r
587 while (argument[1]!=0)
\r
588 switch(argument[1])
\r
595 int B = argument[1] - '0';
\r
596 int S = 1 << (8 + 2*B);
\r
597 BMK_SetBlocksize(S);
\r
601 case 'D': argument++; break;
\r
602 default : goto _exit_blockProperties;
\r
604 _exit_blockProperties:
\r
607 // Modify Nb Iterations
\r
609 if ((argument[1] >='1') && (argument[1] <='9'))
\r
611 int iters = argument[1] - '0';
\r
612 BMK_SetNbIterations(iters);
\r
617 // Pause at the end (hidden option)
\r
618 case 'p': BMK_SetPause(); break;
\r
620 // Unrecognised command
\r
621 default : badusage(exename); return 1;
\r
627 // first provided filename is input
\r
628 if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
\r
632 // No input filename ==> Error
\r
633 if(!input_filename) { badusage(exename); return 1; }
\r
635 return fullSpeedBench(argv+filenamesStart, argc-filenamesStart);
\r