diff --git a/cli/xxhsum.1.md b/cli/xxhsum.1.md index c66b6b93..8f005c28 100644 --- a/cli/xxhsum.1.md +++ b/cli/xxhsum.1.md @@ -5,6 +5,7 @@ SYNOPSIS -------- `xxhsum` [*OPTION*]... [*FILE*]... + `xxhsum -b` [*OPTION*]... `xxh32sum` is equivalent to `xxhsum -H0`, @@ -56,16 +57,23 @@ OPTIONS * `-h`, `--help`: Displays help and exits -### The following options are useful only when verifying checksums (-c): +### The following options are useful only when using lists in *FILE* (--check and --files-from): * `-c`, `--check` *FILE*: Read xxHash sums from *FILE* and check them +* `--files-from` *FILE*: + Read filenames from *FILE* and generate hashes for them. + Valid *FILE*s have one filename per line, which can include embedded spaces, etc with no need for quotes, escapes, etc. + Note that a line commencing with '\\' will enable the convention used in the encoding of filenames against output hashes, + whereby subsequent \\\\, \n and \r seqeuences are converted to the single + character 0x5C, 0x0A and 0x0D respectively. + * `-q`, `--quiet`: - Don't print OK for each successfully verified file + Don't print OK for each successfully verified hash (only for --check) * `--strict`: - Return an error code if any line in the file is invalid, + Return an error code if any line in *FILE** is invalid, not just if some checksums are wrong. This policy is disabled by default, though UI will prompt an informational message @@ -75,7 +83,7 @@ OPTIONS Don't output anything. Status code shows success. * `-w`, `--warn`: - Emit a warning message about each improperly formatted checksum line. + Emit a warning message about each improperly formatted line in *FILE*. ### The following options are useful only benchmark purpose: @@ -119,6 +127,19 @@ Read xxHash sums from specific files and check them $ xxhsum -c xyz.xxh32 qux.xxh64 +Produce a list of files, then generate hashes for that list + + $ find . -type f -name '*.[ch]' > c-files.txt + $ xxhsum --files-from c-files.txt + +Read the list of files from standard input to avoid the need for an intermediate file + + $ find . -type f -name '*.[ch]' | xxhsum --files-from - + +Note that if shell expansion, length of argument list, clarity of use of spaces in filenames, etc allow it then the same output as the previous example can be generated like this + + $ xxhsum `find . -name '*.[ch]'` + Benchmark xxHash algorithm. By default, `xxhsum` benchmarks xxHash main variants on a synthetic sample of 100 KB, diff --git a/cli/xxhsum.c b/cli/xxhsum.c index 4c0d372d..dfa41efd 100644 --- a/cli/xxhsum.c +++ b/cli/xxhsum.c @@ -390,12 +390,20 @@ typedef enum { display_gnu, display_bsd } Display_convention; typedef void (*XSUM_displayLine_f)(const char*, const void*, AlgoSelected); /* line display signature */ +typedef enum { + LineStatus_hashOk, + LineStatus_hashFailed, + LineStatus_failedToOpen, + LineStatus_isDirectory, + LineStatus_memoryError +} LineStatus; + static XSUM_displayLine_f XSUM_kDisplayLine_fTable[2][2] = { { XSUM_printLine_GNU, XSUM_printLine_GNU_LE }, { XSUM_printLine_BSD, XSUM_printLine_BSD_LE } }; -static int XSUM_hashFile(const char* fileName, +static LineStatus XSUM_hashFile(const char* fileName, const AlgoSelected hashType, const Display_endianness displayEndianness, const Display_convention convention) @@ -414,13 +422,11 @@ static int XSUM_hashFile(const char* fileName, XSUM_setBinaryMode(stdin); } else { if (XSUM_isDirectory(fileName)) { - XSUM_log("xxhsum: %s: Is a directory \n", fileName); - return 1; + return LineStatus_isDirectory; } inFile = XSUM_fopen( fileName, "rb" ); if (inFile==NULL) { - XSUM_log("Error: Could not open '%s': %s. \n", fileName, strerror(errno)); - return 1; + return LineStatus_failedToOpen; } } /* Memory allocation & streaming */ @@ -428,7 +434,7 @@ static int XSUM_hashFile(const char* fileName, if (buffer == NULL) { XSUM_log("\nError: Out of memory.\n"); fclose(inFile); - return 1; + return LineStatus_memoryError; } /* Stream file & update hash */ @@ -469,7 +475,7 @@ static int XSUM_hashFile(const char* fileName, assert(0); /* not possible */ } - return 0; + return LineStatus_hashOk; } @@ -485,11 +491,53 @@ static int XSUM_hashFiles(const char* fnList[], int fnTotal, int fnNb; int result = 0; - if (fnTotal==0) - return XSUM_hashFile(stdinName, hashType, displayEndianness, convention); + if (fnTotal == 0) + { + LineStatus filestatus = XSUM_hashFile(stdinName, hashType, displayEndianness, convention); + switch (filestatus) + { + case LineStatus_hashOk: + case LineStatus_hashFailed: + break; + case LineStatus_isDirectory: + XSUM_log("xxhsum: %s: Is a directory \n", stdinName); + break; + case LineStatus_failedToOpen: + XSUM_log("Error: Could not open '%s': %s. \n", stdinName, strerror(errno)); + break; + case LineStatus_memoryError: + XSUM_log("\nError: Out of memory.\n"); + break; + } + + if (filestatus != LineStatus_hashOk) + result = 1; + } + + + for (fnNb = 0; fnNb < fnTotal; fnNb++) + { + LineStatus filestatus = XSUM_hashFile(fnList[fnNb], hashType, displayEndianness, convention); + switch (filestatus) + { + case LineStatus_hashOk: + case LineStatus_hashFailed: + break; + case LineStatus_isDirectory: + XSUM_log("xxhsum: %s: Is a directory \n", fnList[fnNb]); + break; + case LineStatus_failedToOpen: + XSUM_log("Error: Could not open '%s': %s. \n", fnList[fnNb], strerror(errno)); + break; + case LineStatus_memoryError: + XSUM_log("\nError: Out of memory.\n"); + break; + } + + if (filestatus != LineStatus_hashOk) + result = 1; + } - for (fnNb=0; fnNbquit = 1; break; + case LineStatus_memoryError: + case LineStatus_isDirectory: + assert(0); /* Never happens on these paths */ + break; + case LineStatus_failedToOpen: if (XSUM_parseFileArg->ignoreMissing) { report->nMissing++; @@ -1094,6 +1141,250 @@ static int XSUM_checkFiles(const char* fnList[], int fnTotal, } +/* +* +* Parse single filename from list to generate hashes for. +* Returns ParseLine_invalidFormat if the filename is not well formatted. +* Returns ParseLine_ok if the filename is parsed successfully. +*/ +static ParseLineResult XSUM_parseGenLine(ParsedLine * parsedLine, + char* filename) +{ + if (XSUM_lineNeedsUnescape(filename)) { + size_t filenameLen; + ++filename; + filenameLen = strlen(filename); + + if (XSUM_filenameUnescape(filename, filenameLen) == NULL) { + parsedLine->filename = NULL; + return ParseLine_invalidFormat; + } + } + + parsedLine->filename = filename; + + return ParseLine_ok; +} + +/* + * Parse gen source file. + */ +static void XSUM_parseGenFile1(ParseFileArg* XSUM_parseGenArg, + AlgoSelected hashType, + Display_endianness displayEndianness, + Display_convention convention) +{ + const char* const inFileName = XSUM_parseGenArg->inFileName; + ParseFileReport* const report = &XSUM_parseGenArg->report; + + unsigned long lineNumber = 0; + memset(report, 0, sizeof(*report)); + + while (!report->quit) { + LineStatus lineStatus = LineStatus_hashFailed; + ParsedLine parsedLine; + memset(&parsedLine, 0, sizeof(parsedLine)); + + lineNumber++; + if (lineNumber == 0) { + /* This is unlikely happen, but md5sum.c has this error check. */ + XSUM_log("%s: Error: Too many generate lines\n", inFileName); + report->quit = 1; + break; + } + + { GetLineResult const XSUM_getLineResult = XSUM_getLine(&XSUM_parseGenArg->lineBuf, + &XSUM_parseGenArg->lineMax, + XSUM_parseGenArg->inFile); + + /* Ignore comment lines */ + if (XSUM_getLineResult == GetLine_comment) { + continue; + } + + if (XSUM_getLineResult != GetLine_ok) { + if (XSUM_getLineResult == GetLine_eof) break; + + switch (XSUM_getLineResult) + { + case GetLine_ok: + case GetLine_comment: + case GetLine_eof: + /* These cases never happen. See above XSUM_getLineResult related "if"s. + They exist just for make gcc's -Wswitch-enum happy. */ + assert(0); + break; + + default: + XSUM_log("%s:%lu: Error: Unknown error.\n", inFileName, lineNumber); + break; + + case GetLine_exceedMaxLineLength: + XSUM_log("%s:%lu: Error: Line too long.\n", inFileName, lineNumber); + break; + + case GetLine_outOfMemory: + XSUM_log("%s:%lu: Error: Out of memory.\n", inFileName, lineNumber); + break; + } + report->quit = 1; + break; + } } + + if (XSUM_parseGenLine(&parsedLine, XSUM_parseGenArg->lineBuf) != ParseLine_ok) { + report->nImproperlyFormattedLines++; + if (XSUM_parseGenArg->warn) { + XSUM_log("%s:%lu: Error: Improperly formatted line.\n", + inFileName, lineNumber); + } + continue; + } + + report->nProperlyFormattedLines++; + + lineStatus = XSUM_hashFile(parsedLine.filename, hashType, displayEndianness, convention); + + switch (lineStatus) + { + default: + XSUM_log("%s: Error: Unknown error.\n", parsedLine.filename); + report->quit = 1; + break; + + case LineStatus_memoryError: + XSUM_log("\nError: Out of memory.\n"); + break; + + case LineStatus_failedToOpen: + case LineStatus_isDirectory: + if (XSUM_parseGenArg->ignoreMissing) { + report->nMissing++; + } + else { + report->nOpenOrReadFailures++; + if (!XSUM_parseGenArg->statusOnly) { + XSUM_output( + lineStatus == LineStatus_failedToOpen ? + "%s:%lu: Could not open or read '%s': %s.\n" : + "%s:%lu: Target is a directory '%s'.\n", /* Leaves errno argument unconsumed */ + inFileName, lineNumber, parsedLine.filename, strerror(errno)); + } + } + break; + + case LineStatus_hashOk: + case LineStatus_hashFailed: + break; + } + } /* while (!report->quit) */ +} + + +/* Parse text file for list of targets. + */ +static int XSUM_generateFile(const char* inFileName, + AlgoSelected hashType, + Display_endianness displayEndianness, + Display_convention convention, + XSUM_U32 statusOnly, + XSUM_U32 ignoreMissing, + XSUM_U32 warn) +{ + int result = 0; + FILE* inFile = NULL; + ParseFileArg XSUM_parseGenArgBody; + ParseFileArg* const XSUM_parseGenArg = &XSUM_parseGenArgBody; + ParseFileReport* const report = &XSUM_parseGenArg->report; + + /* note: stdinName is special constant pointer. It is not a string. */ + if (inFileName == stdinName) { + /* + * Note: Since we expect text input for xxhash -c mode, + * we don't set binary mode for stdin. + */ + inFileName = stdinFileName; /* "stdin" */ + inFile = stdin; + } + else { + inFile = XSUM_fopen(inFileName, "rt"); + } + + if (inFile == NULL) { + XSUM_log("Error: Could not open '%s': %s\n", inFileName, strerror(errno)); + return 0; + } + + XSUM_parseGenArg->inFileName = inFileName; + XSUM_parseGenArg->inFile = inFile; + XSUM_parseGenArg->lineMax = DEFAULT_LINE_LENGTH; + XSUM_parseGenArg->lineBuf = (char*)malloc((size_t)XSUM_parseGenArg->lineMax); + XSUM_parseGenArg->blockSize = 64 * 1024; + XSUM_parseGenArg->blockBuf = (char*)malloc(XSUM_parseGenArg->blockSize); + XSUM_parseGenArg->statusOnly = statusOnly; + XSUM_parseGenArg->ignoreMissing = ignoreMissing; + XSUM_parseGenArg->warn = warn; + + if ((XSUM_parseGenArg->lineBuf == NULL) + || (XSUM_parseGenArg->blockBuf == NULL)) { + XSUM_log("Error: : memory allocation failed \n"); + exit(1); + } + XSUM_parseGenFile1(XSUM_parseGenArg, hashType, displayEndianness, convention); + + free(XSUM_parseGenArg->blockBuf); + free(XSUM_parseGenArg->lineBuf); + + if (inFile != stdin) fclose(inFile); + + /* Show error/warning messages. All messages are copied from md5sum.c + */ + if (report->nProperlyFormattedLines == 0) { + XSUM_log("%s: no properly formatted filename lines found\n", inFileName); + } + if (report->nImproperlyFormattedLines) { + XSUM_output("%lu %s improperly formatted\n" + , report->nImproperlyFormattedLines + , report->nImproperlyFormattedLines == 1 ? "line is" : "lines are"); + } + if (report->nOpenOrReadFailures) { + XSUM_output("%lu listed %s could not be read\n" + , report->nOpenOrReadFailures + , report->nOpenOrReadFailures == 1 ? "file" : "files"); + } + /* Result (exit) code logic is copied from + * gnu coreutils/src/md5sum.c digest_check() */ + result = report->nProperlyFormattedLines != 0 + && report->nOpenOrReadFailures == 0 + && (report->nImproperlyFormattedLines == 0) + && report->quit == 0; + + return result; +} + +static int XSUM_generateFiles(const char* fnList[], int fnTotal, + AlgoSelected hashType, + Display_endianness displayEndianness, + Display_convention convention, + XSUM_U32 statusOnly, + XSUM_U32 ignoreMissing, + XSUM_U32 warn) +{ + int ok = 1; + + /* Special case for stdinName "-", + * note: stdinName is not a string. It's special pointer. */ + if (fnTotal == 0) { + ok &= XSUM_generateFile(stdinName, hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn); + } + else { + int fnNb; + for (fnNb = 0; fnNb < fnTotal; fnNb++) + ok &= XSUM_generateFile(fnList[fnNb], hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn); + } + return ok ? 0 : 1; +} + + /* ******************************************************** * Main **********************************************************/ @@ -1101,17 +1392,18 @@ static int XSUM_checkFiles(const char* fnList[], int fnTotal, static int XSUM_usage(const char* exename) { XSUM_log( WELCOME_MESSAGE(exename) ); - XSUM_log( "Print or verify checksums using fast non-cryptographic algorithm xxHash \n\n" ); + XSUM_log( "Create or verify checksums using fast non-cryptographic algorithm xxHash \n\n" ); XSUM_log( "Usage: %s [options] [files] \n\n", exename); XSUM_log( "When no filename provided or when '-' is provided, uses stdin as input. \n"); XSUM_log( "\nOptions: \n"); - XSUM_log( " -H# select an xxhash algorithm (default: %i) \n", (int)g_defaultAlgo); - XSUM_log( " 0: XXH32 \n"); - XSUM_log( " 1: XXH64 \n"); - XSUM_log( " 2: XXH128 (also called XXH3_128bits) \n"); - XSUM_log( " 3: XXH3 (also called XXH3_64bits) \n"); - XSUM_log( " -c, --check read xxHash checksum from [files] and check them \n"); - XSUM_log( " -h, --help display a long help page about advanced options \n"); + XSUM_log( " -H# select an xxhash algorithm (default: %i) \n", (int)g_defaultAlgo); + XSUM_log( " 0: XXH32 \n"); + XSUM_log( " 1: XXH64 \n"); + XSUM_log( " 2: XXH128 (also called XXH3_128bits) \n"); + XSUM_log( " 3: XXH3 (also called XXH3_64bits) \n"); + XSUM_log( " -c, --check read xxHash checksum from [files] and check them \n"); + XSUM_log( " --files-from generate hashes for files listed in [files] \n"); + XSUM_log( " -h, --help display a long help page about advanced options \n"); return 0; } @@ -1129,11 +1421,11 @@ static int XSUM_usage_advanced(const char* exename) XSUM_log( " -i# Number of times to run the benchmark (default: %i) \n", NBLOOPS_DEFAULT); XSUM_log( " -q, --quiet Don't display version header in benchmark mode \n"); XSUM_log( "\n"); - XSUM_log( "The following five options are useful only when verifying checksums (-c): \n"); - XSUM_log( " -q, --quiet Don't print OK for each successfully verified file \n"); + XSUM_log( "The following five options are useful only when using lists in [files] to verify or generate checksums: \n"); + XSUM_log( " -q, --quiet Don't print OK for each successfully verified hash \n"); XSUM_log( " --status Don't output anything, status code shows success \n"); - XSUM_log( " --strict Exit non-zero for improperly formatted checksum lines \n"); - XSUM_log( " --warn Warn about improperly formatted checksum lines \n"); + XSUM_log( " --strict Exit non-zero for improperly formatted lines in [files] \n"); + XSUM_log( " --warn Warn about improperly formatted lines in [files] \n"); XSUM_log( " --ignore-missing Don't fail or report status for missing files \n"); return 0; } @@ -1239,6 +1531,7 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) assert(argument != NULL); if (!strcmp(argument, "--check")) { fileCheckMode = 1; continue; } + if (!strcmp(argument, "--files-from")) { fileCheckMode = 2; continue; } if (!strcmp(argument, "--benchmark-all")) { benchmarkMode = 1; selectBenchIDs = kBenchAll; continue; } if (!strcmp(argument, "--bench-all")) { benchmarkMode = 1; selectBenchIDs = kBenchAll; continue; } if (!strcmp(argument, "--quiet")) { XSUM_logLevel--; continue; } @@ -1300,6 +1593,12 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) argument++; break; + /* Generate hash mode (tar style short form of --files-from) + case 'T': + fileCheckMode = 2; + argument++; + break; */ + /* Warning mode (file check mode only, alias of "--warning") */ case 'w': warn=1; @@ -1361,9 +1660,11 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) return XSUM_badusage(exename); if (filenamesStart==0) filenamesStart = argc; - if (fileCheckMode) { + if (fileCheckMode == 1) { return XSUM_checkFiles(argv+filenamesStart, argc-filenamesStart, displayEndianness, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/, algoBitmask); + } else if (fileCheckMode == 2) { + return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, statusOnly, ignoreMissing, warn); } else { return XSUM_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianness, convention); }