init

node_modules/msgpackr-extract/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Kris Zyp
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

node_modules/msgpackr-extract/README.md

@@ -0,0 +1,5 @@
+## Summary
+This module is designed to do fast and efficient native/C-level extraction of strings from MessagePack binary data. This works by calling `extractStrings(buffer, start, end)`, and it will extract strings by doing partial MessagePack parsing, and scanning to find the string data in the range specified in the buffer. It will return an array of strings that it finds. When it finds strings that can be represented with latin-1/one-byte strings (and important V8 optimization), it will attempt return a continuous string of MessagePack data that contains multiple sub-strings, so the decoder can slice off strings by offset. When a string contains non-latin characters, and must be represented as a two-byte string, this will always be returned as the string alone without combination with any other strings. The extractor will return an array of a maximum of 256 strings. The decoder can call the extractStrings again, with a new offset to continue extracting more strings as necessary.
+
+## License
+MIT

node_modules/msgpackr-extract/bin/download-prebuilds.js

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+
+const { dirname } = require('path');
+const { fileURLToPath } = require('url');
+const { exec } = require('child_process');
+
+process.chdir(dirname(__dirname));
+exec('prebuildify-ci download', (error, stdout, stderr) => {
+	console.error(stderr);
+	console.log(stdout);
+	if (error?.code)
+		process.exit(error.code);
+});

node_modules/msgpackr-extract/binding.gyp

@@ -0,0 +1,63 @@
+{
+  "variables": {
+      "os_linux_compiler%": "gcc",
+      "enable_v8%": "true",
+      "enable_pointer_compression%": "false",
+      "build_v8_with_gn": "false"
+  },
+  "conditions": [
+    ['OS=="win"', {
+      "variables": {
+        "enable_v8%": "<!(echo %ENABLE_V8_FUNCTIONS%)",
+      }
+    }],
+    ['OS!="win"', {
+      "variables": {
+        "enable_v8%": "<!(echo $ENABLE_V8_FUNCTIONS)",
+      }
+    }]
+  ],
+  "targets": [
+    {
+      "target_name": "extract",
+      "sources": [
+        "src/extract.cpp",
+      ],
+      "defines": [ "NAPI_DISABLE_CPP_EXCEPTIONS" ],
+      "conditions": [
+        ["OS=='linux'", {
+          "variables": {
+            "gcc_version" : "<!(<(os_linux_compiler) -dumpversion | cut -d '.' -f 1)",
+          },
+          "cflags_cc": [
+            "-fPIC",
+            "-fvisibility=hidden",
+            "-fvisibility-inlines-hidden",
+          ],
+          "conditions": [
+            ["gcc_version>=7", {
+              "cflags": [
+                "-Wimplicit-fallthrough=2",
+              ],
+            }],
+          ],
+          "ldflags": [
+            "-fPIC",
+            "-fvisibility=hidden"
+          ],
+          "cflags": [
+            "-fPIC",
+            "-fvisibility=hidden",
+            "-O3"
+          ],
+        }],
+        ["enable_v8!='false'", {
+          "defines": ["ENABLE_V8_API=1"]
+        }],
+        ["enable_pointer_compression=='true'", {
+          "defines": ["V8_COMPRESS_POINTERS", "V8_COMPRESS_POINTERS_IN_ISOLATE_CAGE"],
+        }],
+      ],
+    }
+  ]
+}

node_modules/msgpackr-extract/index.js

@@ -0,0 +1 @@
+module.exports = require('node-gyp-build-optional-packages')(__dirname);

node_modules/msgpackr-extract/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "msgpackr-extract",
+  "author": "Kris Zyp",
+  "version": "3.0.3",
+  "description": "Node addon for string extraction for msgpackr",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "http://github.com/kriszyp/msgpackr-extract"
+  },
+  "scripts": {
+    "install": "node-gyp-build-optional-packages",
+    "recompile": "node-gyp rebuild",
+    "before-publish": "prebuildify-ci download && node set-optional-deps.cjs",
+    "prebuild": "prebuildify-platform-packages --target 20.14.0",
+    "prebuild-win32": "prebuildify-platform-packages --target 20.14.0 && set ENABLE_V8_FUNCTIONS=false&& prebuildify-platform-packages --platform-packages --napi --target 20.14.0",
+    "prebuild-libc": "prebuildify-platform-packages --tag-libc --target 20.14.0 && prebuildify-platform-packages --platform-packages --napi --tag-libc --target 22.2.0 && ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --platform-packages --napi --tag-libc --target 20.14.0",
+    "prebuild-libc-napi": "ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --platform-packages --napi --tag-libc --target 20.14.0",
+    "prebuild-libc-alpine": "prebuildify-cross --image alpine --tag-libc --target 20.14.0",
+    "publish-all": "cd prebuilds/win32-x64 && npm publish --access public && cd ../darwin-x64 && npm publish --access public && cd ../darwin-arm64 && npm publish --access public && cd ../linux-x64 && npm publish --access public && cd ../linux-arm64 && npm publish --access public  && cd ../linux-arm && npm publish --access public && cd ../.. && npm publish --access public",
+    "test": "node ./index.js"
+  },
+  "main": "./index.js",
+  "gypfile": true,
+  "dependencies": {
+    "node-gyp-build-optional-packages": "5.2.2"
+  },
+  "files": [
+    "index.js",
+    "/src",
+    "/*.gyp",
+    "/bin"
+  ],
+  "bin": {
+    "download-msgpackr-prebuilds": "./bin/download-prebuilds.js"
+  },
+  "optionalDependencies": {
+    "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
+    "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
+    "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
+    "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
+    "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
+    "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
+  },
+  "devDependencies": {
+    "prebuildify-platform-packages": "5.0.4",
+    "prebuildify-ci": "^1.0.5",
+    "prebuildify-cross": "5.0.0"
+  }
+}

node_modules/msgpackr-extract/src/extract.cpp

@@ -0,0 +1,274 @@
+/*
+This is responsible for extracting the strings, in bulk, from a MessagePack buffer. Creating strings from buffers can
+be one of the biggest performance bottlenecks of parsing, but creating an array of extracting strings all at once
+provides much better performance. This will parse and produce up to 256 strings at once .The JS parser can call this multiple
+times as necessary to get more strings. This must be partially capable of parsing MessagePack so it can know where to
+find the string tokens and determine their position and length. All strings are decoded as UTF-8.
+*/
+#include <node_api.h>
+#if ENABLE_V8_API
+#include <v8.h>
+#endif
+
+#ifndef thread_local
+#ifdef __GNUC__
+# define thread_local __thread
+#elif __STDC_VERSION__ >= 201112L
+# define thread_local _Thread_local
+#elif defined(_MSC_VER)
+# define thread_local __declspec(thread)
+#else
+# define thread_local
+#endif
+#endif
+
+const int MAX_TARGET_SIZE = 255;
+const uint32_t UNEXPECTED_END = 0xffffffff;
+typedef uint32_t (*token_handler)(uint8_t* source, uint32_t position, uint32_t size);
+token_handler tokenTable[256] = {};
+napi_value unexpectedEnd(napi_env env) {
+	napi_value returnValue;
+	napi_get_undefined(env, &returnValue);
+	napi_throw_type_error(env, NULL, "Unexpected end of buffer reading string");
+	return returnValue;
+}
+
+class Extractor {
+public:
+	// napi_ref targetArray; // could consider reenabling this optimization for napi
+	bool hasTargetArray = false;
+	uint8_t* source;
+	uint32_t position = 0;
+	uint32_t writePosition = 0;
+	uint32_t stringStart = 0;
+	uint32_t lastStringEnd = 0;
+
+	void readString(napi_env env, uint32_t length, bool allowStringBlocks, napi_value* target) {
+		uint32_t start = position;
+		uint32_t end = position + length;
+		if (allowStringBlocks) { // for larger strings, we don't bother to check every character for being latin, and just go right to creating a new string
+			while(position < end) {
+				if (source[position] < 0x80) // ensure we character is latin and can be decoded as one byte
+					position++;
+				else {
+					break;
+				}
+			}
+		}
+		if (position < end) {
+			// non-latin character
+			if (lastStringEnd) {
+				napi_value value;
+				napi_create_string_latin1(env, (const char*) source + stringStart, lastStringEnd - stringStart, &value);
+				target[writePosition++] = value;
+				lastStringEnd = 0;
+			}
+			// use standard utf-8 conversion
+			napi_value value;
+			napi_create_string_utf8(env, (const char*) source + start, (int) length, &value);
+			target[writePosition++] = value;
+			position = end;
+			return;
+		}
+
+		if (lastStringEnd) {
+			if (start - lastStringEnd > 40 || end - stringStart > 6000) {
+				napi_value value;
+				napi_create_string_latin1(env, (const char*) source + stringStart, lastStringEnd - stringStart, &value);
+				target[writePosition++] = value;
+				stringStart = start;
+			}
+		} else {
+			stringStart = start;
+		}
+		lastStringEnd = end;
+	}
+
+	napi_value extractStrings(napi_env env, uint32_t startingPosition, uint32_t size, uint8_t* inputSource) {
+		napi_value target[MAX_TARGET_SIZE + 1]; // leave one for the queued string
+		writePosition = 0;
+		lastStringEnd = 0;
+		position = startingPosition;
+		source = inputSource;
+		while (position < size) {
+			uint8_t token = source[position++];
+			if (token < 0xa0) {
+				// all one byte tokens
+			} else if (token < 0xc0) {
+				// fixstr, we want to convert this
+				token -= 0xa0;
+				if (token + position > size) {
+					return unexpectedEnd(env);
+				}
+				readString(env, token, true, target);
+				if (writePosition >= MAX_TARGET_SIZE)
+					break;
+			} else if (token <= 0xdb && token >= 0xd9) {
+				if (token == 0xd9) { //str 8
+					if (position >= size) {
+						return unexpectedEnd(env);
+					}
+					uint32_t length = source[position++];
+					if (length + position > size) {
+						return unexpectedEnd(env);
+					}
+					readString(env,length, true, target);
+				} else if (token == 0xda) { //str 16
+					if (2 + position > size) {
+						return unexpectedEnd(env);
+					}
+					uint32_t length = source[position++] << 8;
+					length += source[position++];
+					if (length + position > size) {
+						return unexpectedEnd(env);
+					}
+					readString(env,length, false, target);
+				} else { //str 32
+					if (4 + position > size) {
+						return unexpectedEnd(env);
+					}
+					uint32_t length = source[position++] << 24;
+					length += source[position++] << 16;
+					length += source[position++] << 8;
+					length += source[position++];
+					if (length + position > size) {
+						return unexpectedEnd(env);
+					}
+					readString(env, length, false, target);
+				}
+				if (writePosition >= MAX_TARGET_SIZE)
+					break;
+			} else {
+				auto handle = tokenTable[token];
+				if ((size_t ) handle < 20) {
+					position += (size_t ) handle;
+				} else {
+					position = tokenTable[token](source, position, size);
+					if (position == UNEXPECTED_END) {
+						return unexpectedEnd(env);
+					}
+				}
+			}
+		}
+
+		if (lastStringEnd) {
+			napi_value value;
+			napi_create_string_latin1(env, (const char*) source + stringStart, lastStringEnd - stringStart, &value);
+			if (writePosition == 0) {
+				return value;
+			}
+			target[writePosition++] = value;
+		} else if (writePosition == 1) {
+			return target[0];
+		}
+		napi_value array;
+		#if ENABLE_V8_API
+		v8::Local<v8::Array> v8Array = v8::Array::New(v8::Isolate::GetCurrent(), (v8::Local<v8::Value>*) target, writePosition);
+		memcpy(&array, &v8Array, sizeof(array));
+		#else
+		napi_create_array_with_length(env, writePosition, &array);
+		for (uint32_t i = 0; i < writePosition; i++) {
+			napi_set_element(env, array, i, target[i]);
+		}
+		#endif
+		return array;
+	}
+};
+
+void setupTokenTable() {
+	for (int i = 0; i < 256; i++) {
+		tokenTable[i] = nullptr;
+	}
+	// uint8, int8
+	tokenTable[0xcc] = tokenTable[0xd0] = (token_handler) 1;
+	// uint16, int16, array 16, map 16, fixext 1
+	tokenTable[0xcd] = tokenTable[0xd1] = tokenTable[0xdc] = tokenTable[0xde] = tokenTable[0xd4] = (token_handler) 2;
+	// fixext 16
+	tokenTable[0xd5] = (token_handler) 3;
+	// uint32, int32, float32, array 32, map 32
+	tokenTable[0xce] = tokenTable[0xd2] = tokenTable[0xca] = tokenTable[0xdd] = tokenTable[0xdf] = (token_handler) 4;
+	// fixext 4
+	tokenTable[0xd6] = (token_handler) 5;
+	// uint64, int64, float64, fixext 8
+	tokenTable[0xcf] = tokenTable[0xd3] = tokenTable[0xcb] = (token_handler) 8;
+	// fixext 8
+	tokenTable[0xd7] = (token_handler) 9;
+	// fixext 16
+	tokenTable[0xd8] = (token_handler) 17;
+	// bin 8
+	tokenTable[0xc4] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
+		if (position >= size) {
+			return UNEXPECTED_END;
+		}
+		uint32_t length = source[position++];
+		return position + length;
+	});
+	// bin 16
+	tokenTable[0xc5] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
+		if (position + 2 > size) {
+			return UNEXPECTED_END;
+		}
+		uint32_t length = source[position++] << 8;
+		length += source[position++];
+		return position + length;
+	});
+	// bin 32
+	tokenTable[0xc6] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
+		if (position + 4 > size)
+			return UNEXPECTED_END;
+		uint32_t length = source[position++] << 24;
+		length += source[position++] << 16;
+		length += source[position++] << 8;
+		length += source[position++];
+		return position + length;
+	});
+	// ext 8
+	tokenTable[0xc7] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
+		if (position >= size)
+			return UNEXPECTED_END;
+		uint32_t length = source[position++];
+		position++;
+		return position + length;
+	});
+	// ext 16
+	tokenTable[0xc8] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
+		if (position + 2 > size)
+			return UNEXPECTED_END;
+		uint32_t length = source[position++] << 8;
+		length += source[position++];
+		position++;
+		return position + length;
+	});
+	// ext 32
+	tokenTable[0xc9] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
+		if (position + 4 > size)
+			return UNEXPECTED_END;
+		uint32_t length = source[position++] << 24;
+		length += source[position++] << 16;
+		length += source[position++] << 8;
+		length += source[position++];
+		position++;
+		return position + length;
+	});
+}
+static thread_local Extractor* extractor;
+napi_value extractStrings(napi_env env, napi_callback_info info) {
+	size_t argc = 3;
+	napi_value args[3];
+	napi_get_cb_info(env, info, &argc, args, NULL, NULL);
+	uint32_t position;
+	uint32_t size;
+	napi_get_value_uint32(env, args[0], &position);
+	napi_get_value_uint32(env, args[1], &size);
+	uint8_t* source;
+	size_t buffer_size;
+	napi_get_arraybuffer_info(env, args[2], (void**) &source, &buffer_size);
+	return extractor->extractStrings(env, position, size, source);
+}
+#define EXPORT_NAPI_FUNCTION(name, func) { napi_property_descriptor desc = { name, 0, func, 0, 0, 0, (napi_property_attributes) (napi_writable | napi_configurable), 0 }; napi_define_properties(env, exports, 1, &desc); }
+NAPI_MODULE_INIT() {
+	extractor = new Extractor(); // create our thread-local extractor
+	setupTokenTable();
+	EXPORT_NAPI_FUNCTION("extractStrings", extractStrings);
+	return exports;
+}