ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add basic tracing functionality

Browse Source
This commit is contained in:
Nick Terrell
2021-02-05 16:28:52 -08:00
parent e926e9c4c3
commit 54a4998a80
20 changed files with 521 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+4 -4
View File
@@ -314,16 +314,16 @@ zstd-pgo :
## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
zstd-small: CFLAGS = -Os -s
zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT)
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE $^ -o $@$(EXT)
zstd-decompress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_DECOMPRESS_C) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
zstd-compress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS
zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE
zstd-dictBuilder: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
programs/zstdcli.c
+13
View File
@@ -42,6 +42,9 @@
#ifndef ZSTD_NODICT
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
#endif
#ifndef ZSTD_NOTRACE
# include "zstdcli_trace.h"
#endif
#include "../lib/zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
@@ -186,6 +189,10 @@ static void usage_advanced(const char* programName)
DISPLAYOUT( "--[no-]check : during decompression, ignore/validate checksums in compressed frame (default: validate).");
#endif
#endif /* ZSTD_NOCOMPRESS */
#ifndef ZSTD_NOTRACE
DISPLAYOUT( "--trace FILE : log tracing information to FILE.");
#endif
DISPLAYOUT( "\n");
DISPLAYOUT( "-- : All arguments after \"--\" are treated as files \n");
@@ -918,6 +925,9 @@ int main(int const argCount, const char* argv[])
if (longCommandWArg(&argument, "--output-dir-flat")) { NEXT_FIELD(outDirName); continue; }
#ifdef UTIL_HAS_MIRRORFILELIST
if (longCommandWArg(&argument, "--output-dir-mirror")) { NEXT_FIELD(outMirroredDirName); continue; }
#endif
#ifndef ZSTD_NOTRACE
if (longCommandWArg(&argument, "--trace")) { char const* traceFile; NEXT_FIELD(traceFile); TRACE_enable(traceFile); continue; }
#endif
if (longCommandWArg(&argument, "--patch-from")) { NEXT_FIELD(patchFromDictFileName); continue; }
if (longCommandWArg(&argument, "--long")) {
@@ -1397,6 +1407,9 @@ _end:
if (main_pause) waitEnter();
UTIL_freeFileNamesTable(filenames);
UTIL_freeFileNamesTable(file_of_names);
#ifndef ZSTD_NOTRACE
TRACE_finish();
#endif
return operationResult;
}
programs/zstdcli_trace.c
+192
View File
@@ -0,0 +1,192 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "zstdcli_trace.h"
#include <stdio.h>
#include <stdlib.h>
#include "timefn.h"
#include "util.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "../lib/zstd.h"
/* We depend on the trace header to avoid duplicating the ZSTD_trace struct.
* But, we check the version so it is compatible with dynamic linking.
*/
#include "../lib/common/zstd_trace.h"
/* We only use macros from threading.h so it is compatible with dynamic linking */
#include "../lib/common/threading.h"
#if ZSTD_TRACE
static FILE* g_traceFile = NULL;
static int g_mutexInit = 0;
static ZSTD_pthread_mutex_t g_mutex;
void TRACE_enable(char const* filename)
{
int const writeHeader = !UTIL_isRegularFile(filename);
if (g_traceFile)
fclose(g_traceFile);
g_traceFile = fopen(filename, "a");
if (g_traceFile && writeHeader) {
/* Fields:
* algorithm
* version
* method
* streaming
* level
* workers
* dictionary size
* uncompressed size
* compressed size
* duration nanos
* compression ratio
* speed MB/s
*/
fprintf(g_traceFile, "Algorithm, Version, Method, Mode, Level, Workers, Dictionary Size, Uncompressed Size, Compressed Size, Duration Nanos, Compression Ratio, Speed MB/s\n");
}
if (!g_mutexInit) {
if (!ZSTD_pthread_mutex_init(&g_mutex, NULL)) {
g_mutexInit = 1;
} else {
TRACE_finish();
}
}
}
void TRACE_finish(void)
{
if (g_traceFile) {
fclose(g_traceFile);
}
g_traceFile = NULL;
if (g_mutexInit) {
ZSTD_pthread_mutex_destroy(&g_mutex);
g_mutexInit = 0;
}
}
static void TRACE_log(char const* method, PTime duration, ZSTD_trace const* trace)
{
int level = 0;
int workers = 0;
double const ratio = (double)trace->uncompressedSize / (double)trace->compressedSize;
double const speed = ((double)trace->uncompressedSize * 1000) / (double)duration;
if (trace->params) {
ZSTD_CCtxParams_getParameter(trace->params, ZSTD_c_compressionLevel, &level);
ZSTD_CCtxParams_getParameter(trace->params, ZSTD_c_nbWorkers, &workers);
}
assert(g_traceFile != NULL);
/* Fields:
* algorithm
* version
* method
* streaming
* level
* workers
* dictionary size
* uncompressed size
* compressed size
* duration nanos
* compression ratio
* speed MB/s
*/
fprintf(g_traceFile,
"zstd, %u, %s, %s, %d, %d, %llu, %llu, %llu, %llu, %.2f, %.2f\n",
trace->version,
method,
trace->streaming ? "streaming" : "single-pass",
level,
workers,
(unsigned long long)trace->dictionarySize,
(unsigned long long)trace->uncompressedSize,
(unsigned long long)trace->compressedSize,
(unsigned long long)duration,
ratio,
speed);
}
static ZSTD_CCtx const* g_cctx;
static ZSTD_DCtx const* g_dctx;
static UTIL_time_t g_begin = UTIL_TIME_INITIALIZER;
/**
* These symbols override the weak symbols provided by the library.
*/
int ZSTD_trace_compress_begin(ZSTD_CCtx const* cctx)
{
int enabled = 0;
if (g_traceFile == NULL)
return 0;
ZSTD_pthread_mutex_lock(&g_mutex);
if (g_cctx == NULL) {
g_cctx = cctx;
g_dctx = NULL;
g_begin = UTIL_getTime();
enabled = 1;
}
ZSTD_pthread_mutex_unlock(&g_mutex);
return enabled;
}
void ZSTD_trace_compress_end(ZSTD_CCtx const* cctx, ZSTD_trace const* trace)
{
assert(g_traceFile != NULL);
ZSTD_pthread_mutex_lock(&g_mutex);
assert(g_cctx == cctx);
assert(g_dctx == NULL);
if (cctx == g_cctx && trace->version == ZSTD_VERSION_NUMBER)
TRACE_log("compress", UTIL_clockSpanNano(g_begin), trace);
g_cctx = NULL;
ZSTD_pthread_mutex_unlock(&g_mutex);
}
int ZSTD_trace_decompress_begin(ZSTD_DCtx const* dctx)
{
int enabled = 0;
if (g_traceFile == NULL)
return 0;
ZSTD_pthread_mutex_lock(&g_mutex);
if (g_dctx == NULL) {
g_cctx = NULL;
g_dctx = dctx;
g_begin = UTIL_getTime();
enabled = 1;
}
ZSTD_pthread_mutex_unlock(&g_mutex);
return enabled;
}
void ZSTD_trace_decompress_end(ZSTD_DCtx const* dctx, ZSTD_trace const* trace)
{
assert(g_traceFile != NULL);
ZSTD_pthread_mutex_lock(&g_mutex);
assert(g_cctx == NULL);
assert(g_dctx == dctx);
if (dctx == g_dctx && trace->version == ZSTD_VERSION_NUMBER)
TRACE_log("decompress", UTIL_clockSpanNano(g_begin), trace);
g_dctx = NULL;
ZSTD_pthread_mutex_unlock(&g_mutex);
}
#else /* ZSTD_TRACE */
void TRACE_enable(char const* filename)
{
(void)filename;
}
void TRACE_finish(void) {}
#endif /* ZSTD_TRACE */
programs/zstdcli_trace.h
+24
View File
@@ -0,0 +1,24 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZSTDCLI_TRACE_H
#define ZSTDCLI_TRACE_H
/**
* Enable tracing - log to filename.
*/
void TRACE_enable(char const* filename);
/**
* Shut down the tracing library.
*/
void TRACE_finish(void);
#endif /* ZSTDCLI_TRACE_H */