Add WebAssembly SIMD128 implementation and Node.JS support

.github/workflows/ci.yml

@@ -208,6 +208,59 @@ jobs:
         make -C tests/bench
 
 
+  ubuntu-wasm:
+    name: Ubuntu Node ${{ matrix.node-version }} WebAssembly/asm.js tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [16.x, 17.x, 18.x]
+
+    env:
+      EM_VERSION: 3.1.33 # TODO: more emsdk versions
+      EM_CACHE_FOLDER: emsdk-cache-${{ matrix.node-version }}
+      CC: emcc
+
+    steps:
+    - uses: actions/checkout@v3 # https://github.com/actions/checkout
+
+    - name: Setup cache
+      id: cache-system-libraries
+      uses: actions/cache@v3
+      with:
+        path: ${{env.EM_CACHE_FOLDER}}
+        key: em${{env.EM_VERSION}}-node${{ matrix.node-version }}-${{ runner.os }}
+
+    - name: Setup emsdk
+      uses: mymindstorm/setup-emsdk@v12
+      with:
+        version: ${{env.EM_VERSION}}
+        actions-cache-folder: ${{env.EM_CACHE_FOLDER}}
+
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Environment info
+      run: |
+        echo && node -p '`node version: ${process.versions.node}, v8 version: ${process.versions.v8}`'
+        echo && emcc --version
+        echo && make -v
+        echo && cat /proc/cpuinfo || echo /proc/cpuinfo is not present
+
+    - name: Scalar code path
+      run: |
+        CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" RUN_ENV="node" NODE_JS=1 make clean check
+
+    - name: SIMD128 (via NEON SIMDe) code path (XXH_VECTOR=XXH_NEON)
+      run: |
+        CPPFLAGS="-DXXH_VECTOR=XXH_NEON -msimd128" RUN_ENV="node" NODE_JS=1 make clean check
+
+    - name: Scalar asm.js (-sWASM=0)
+      run: |
+        CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" RUN_ENV="node" NODE_JS=1 LDFLAGS="-sWASM=0" make clean check
+
+
   ubuntu-misc:
     name: Linux x64 misc tests
     runs-on: ubuntu-latest

Makefile

@@ -55,6 +55,13 @@ else
 EXT =
 endif
 
+ifeq ($(NODE_JS),1)
+    # Link in unrestricted filesystem support
+    LDFLAGS += -sNODERAWFS
+    # Set flag to fix isatty() support
+    CPPFLAGS += -DXSUM_NODE_JS=1
+endif
+
 # OS X linker doesn't support -soname, and use different extension
 # see: https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
 ifeq ($(UNAME), Darwin)
@@ -174,6 +181,7 @@ clean:  ## remove all build artifacts
 	$(Q)$(RM) -r *.dSYM   # Mac OS-X specific
 	$(Q)$(RM) core *.o *.obj *.$(SHARED_EXT) *.$(SHARED_EXT).* *.a libxxhash.pc
 	$(Q)$(RM) xxhsum$(EXT) xxhsum32$(EXT) xxhsum_inlinedXXH$(EXT) dispatch$(EXT)
+	$(Q)$(RM) xxhsum.wasm xxhsum.js xxhsum.html
 	$(Q)$(RM) xxh32sum$(EXT) xxh64sum$(EXT) xxh128sum$(EXT)
 	$(Q)$(RM) $(XXHSUM_SRC_DIR)/*.o $(XXHSUM_SRC_DIR)/*.obj
 	$(MAKE) -C tests clean
@@ -191,6 +199,7 @@ clean:  ## remove all build artifacts
 .PHONY: check
 check: xxhsum   ## basic tests for xxhsum CLI, set RUN_ENV for emulated environments
 	# stdin
+	# If you get "Wrong parameters" on Emscripten+Node.js, recompile with `NODE_JS=1`
 	$(RUN_ENV) ./xxhsum$(EXT) < xxhash.c
 	# multiple files
 	$(RUN_ENV) ./xxhsum$(EXT) xxhash.*

README.md

@@ -154,6 +154,7 @@ The following macros can be set at compilation time to modify libxxhash's behavi
 When compiling the Command Line Interface `xxhsum` using `make`, the following environment variables can also be set :
 - `DISPATCH=1` : use `xxh_x86dispatch.c`, to automatically select between `scalar`, `sse2`, `avx2` or `avx512` instruction set at runtime, depending on local host. This option is only valid for `x86`/`x64` systems.
 - `XXH_1ST_SPEED_TARGET` : select an initial speed target, expressed in MB/s, for the first speed test in benchmark mode. Benchmark will adjust the target at subsequent iterations, but the first test is made "blindly" by targeting this speed. Currently conservatively set to 10 MB/s, to support very slow (emulated) platforms.
+- `NODE_JS=1` : When compiling `xxhsum` for Node.js with Emscripten, this links the `NODERAWFS` library for unrestricted filesystem access and patches `isatty` to make the command line utility correctly detect the terminal. This does make the binary specific to Node.js.
 
 ### Building xxHash - Using vcpkg
 

cli/xsum_arch.h

@@ -151,6 +151,12 @@
 #  define XSUM_ARCH "s390x"
 #elif defined(__s390__)
 #  define XSUM_ARCH "s390"
+#elif defined(__wasm__) || defined(__asmjs__) || defined(__EMSCRIPTEN__)
+#  if defined(__wasm_simd128__)
+#    define XSUM_ARCH "wasm/asmjs + simd128"
+#  else
+#    define XSUM_ARCH "wasm/asmjs"
+#  endif
 #else
 #  define XSUM_ARCH "unknown"
 #endif

cli/xsum_os_specific.c

@@ -39,7 +39,32 @@
     typedef struct stat XSUM_stat_t;
 #endif
 
-#if (defined(__linux__) && (XSUM_PLATFORM_POSIX_VERSION >= 1)) \
+#if defined(__EMSCRIPTEN__) && defined(XSUM_NODE_JS)
+#  include <unistd.h>   /* isatty */
+#  include <emscripten.h> /* EM_ASM_INT */
+
+/* The Emscripten SDK does not properly detect when the standard streams
+ * are piped to node.js, and there does not seem to be any way to tell in
+ * plain C. To work around it, inline JavaScript is used to call Node's
+ * isatty() function. */
+static int XSUM_IS_CONSOLE(FILE* stdStream)
+{
+    /* https://github.com/iliakan/detect-node */
+    int is_node = EM_ASM_INT((
+        return (Object.prototype.toString.call(
+            typeof process !== 'undefined' ? process : 0
+        ) == '[object process]') | 0
+    ));
+    if (is_node) {
+        return EM_ASM_INT(
+            return require('node:tty').isatty($0),
+            fileno(stdStream)
+        );
+    } else {
+        return isatty(fileno(stdStream));
+    }
+}
+#elif defined(__EMSCRIPTEN__) || (defined(__linux__) && (XSUM_PLATFORM_POSIX_VERSION >= 1)) \
  || (XSUM_PLATFORM_POSIX_VERSION >= 200112L) \
  || defined(__DJGPP__) \
  || defined(__MSYS__) \

xxh_x86dispatch.c

@@ -56,6 +56,14 @@ extern "C" {
 #  error "Dispatching is currently only supported on x86 and x86_64."
 #endif
 
+#ifndef XXH_HAS_INCLUDE
+#  ifdef __has_include
+#    define XXH_HAS_INCLUDE(x) __has_include(x)
+#  else
+#    define XXH_HAS_INCLUDE(x) 0
+#  endif
+#endif
+
 /*!
  * @def XXH_X86DISPATCH_ALLOW_AVX
  * @brief Disables the AVX sanity check.
@@ -85,12 +93,6 @@ extern "C" {
 #  error "Do not compile xxh_x86dispatch.c with AVX enabled! See the comment above."
 #endif
 
-#ifdef __has_include
-#  define XXH_HAS_INCLUDE(header) __has_include(header)
-#else
-#  define XXH_HAS_INCLUDE(header) 0
-#endif
-
 /*!
  * @def XXH_DISPATCH_SCALAR
  * @brief Enables/dispatching the scalar code path.
@@ -107,7 +109,7 @@ extern "C" {
 #ifndef XXH_DISPATCH_SCALAR
 #  if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \
      || defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \
-     || defined(__ANDROID__) || defined(__APPLEv__) /* Android or macOS */
+     || defined(__ANDROID__) || defined(__APPLE__) /* Android or macOS */
 #     define XXH_DISPATCH_SCALAR 0 /* disable */
 #  else
 #     define XXH_DISPATCH_SCALAR 1