cernont/pyrofetes-frontend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
CHEVALLIER Abel
2025-11-13 16:23:22 +01:00
parent de9c515a47
commit cb235644dc
34924 changed files with 3811102 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

234
node_modules/lmdb/src/compression.cpp generated vendored Normal file
View File

@@ -0,0 +1,234 @@
#include "lz4.h"
#include "lmdb-js.h"
#include <atomic>
using namespace Napi;
thread_local LZ4_stream_t* Compression::stream = nullptr;
Compression::Compression(const CallbackInfo& info) : ObjectWrap<Compression>(info) {
unsigned int compressionThreshold = 1000;
char* dictionary = nullptr;
size_t dictSize = 0;
unsigned int startingOffset = 0;
if (info[0].IsObject()) {
auto dictionaryOption = info[0].As<Object>().Get("dictionary");
if (!dictionaryOption.IsUndefined()) {
if (!dictionaryOption.IsTypedArray()) {
throwError(info.Env(), "Dictionary must be a buffer");
return;
}
napi_get_buffer_info(info.Env(), dictionaryOption, (void**) &dictionary, &dictSize);
dictSize = (dictSize >> 3) << 3; // make sure it is word-aligned
}
auto thresholdOption = info[0].As<Object>().Get("threshold");
if (thresholdOption.IsNumber())
compressionThreshold = thresholdOption.As<Number>();
auto offsetOption = info[0].As<Object>().Get("startingOffset");
if (offsetOption.IsNumber())
startingOffset = offsetOption.As<Number>();
}
this->startingOffset = startingOffset;
this->dictionary = this->compressDictionary = dictionary;
this->dictionarySize = dictSize;
this->decompressTarget = dictionary + dictSize;
this->decompressSize = 0;
this->acceleration = 1;
this->compressionThreshold = compressionThreshold;
info.This().As<Object>().Set("address", Number::New(info.Env(), (double) (size_t) this));
}
Napi::Value Compression::setBuffer(const CallbackInfo& info) {
size_t length;
napi_get_arraybuffer_info(info.Env(), info[0], (void**) &this->decompressTarget, &length);
unsigned int offset = info[1].As<Number>();
this->decompressTarget += offset;
this->decompressSize = info[2].As<Number>();
this->dictionarySize = info[4].As<Number>();
napi_get_buffer_info(info.Env(), info[3], (void**) &this->dictionary, &length);
return info.Env().Undefined();
}
void Compression::decompress(MDB_val& data, bool &isValid, bool canAllocate) {
uint32_t uncompressedLength;
int compressionHeaderSize;
uint32_t compressedLength = data.mv_size;
void* originalData = data.mv_data;
unsigned char* charData = (unsigned char*) data.mv_data + startingOffset;
if (charData[0] == 254) {
uncompressedLength = ((uint32_t)charData[1] << 16) | ((uint32_t)charData[2] << 8) | (uint32_t)charData[3];
compressionHeaderSize = 4;
}
else if (charData[0] == 255) {
uncompressedLength = ((uint32_t)charData[4] << 24) | ((uint32_t)charData[5] << 16) | ((uint32_t)charData[6] << 8) | (uint32_t)charData[7];
compressionHeaderSize = 8;
}
else {
fprintf(stderr, "Unknown status byte %u\n", charData[0]);
//if (canAllocate)
// Nan::ThrowError("Unknown status byte");
isValid = false;
return;
}
//fprintf(stdout, "compressed size %u uncompressedLength %u, target size %u, first byte %u\n", data.mv_size, uncompressedLength + startingOffset, decompressSize, charData[compressionHeaderSize]);
data.mv_data = decompressTarget;
data.mv_size = uncompressedLength + startingOffset;
//TODO: For larger blocks with known encoding, it might make sense to allocate space for it and use an ExternalString
if (uncompressedLength + startingOffset > decompressSize) {
isValid = false;
return;
}
int written = LZ4_decompress_safe_usingDict(
(char*)charData + compressionHeaderSize, decompressTarget + startingOffset,
compressedLength - compressionHeaderSize - startingOffset, decompressSize - startingOffset,
dictionary, dictionarySize);
//fprintf(stdout, "first uncompressed byte %X %X %X %X %X %X\n", uncompressedData[0], uncompressedData[1], uncompressedData[2], uncompressedData[3], uncompressedData[4], uncompressedData[5]);
if (written < 0) {
fprintf(stderr, "Failed to decompress data %u %u bytes:\n", compressionHeaderSize, uncompressedLength);
for (uint32_t i = 0; i < compressedLength; i++) {
fprintf(stderr, "%u ", charData[i]);
}
//if (canAllocate)
// Nan::ThrowError("Failed to decompress data");
isValid = false;
return;
}
if (startingOffset)
memcpy(decompressTarget, originalData, startingOffset);
isValid = true;
}
int Compression::compressInstruction(EnvWrap* env, double* compressionAddress) {
MDB_val value;
value.mv_data = (void*)((size_t) * (compressionAddress - 1));
value.mv_size = *(((uint32_t*)compressionAddress) - 3);
argtokey_callback_t compressedData = compress(&value, nullptr);
if (compressedData) {
*(((uint32_t*)compressionAddress) - 3) = value.mv_size;
*((size_t*)(compressionAddress - 1)) = (size_t)value.mv_data;
int64_t status = std::atomic_exchange((std::atomic<int64_t>*) compressionAddress, (int64_t) 0);
if (status == 1 && env) {
pthread_mutex_lock(env->writingLock);
pthread_cond_signal(env->writingCond);
pthread_mutex_unlock(env->writingLock);
//fprintf(stderr, "sent compression completion signal\n");
}
//fprintf(stdout, "compressed to %p %u %u %p\n", value.mv_data, value.mv_size, status, env);
return 0;
} else {
fprintf(stdout, "failed to compress\n");
return 1;
}
}
argtokey_callback_t Compression::compress(MDB_val* value, void (*freeValue)(MDB_val&)) {
size_t dataLength = value->mv_size - startingOffset;
char* data = (char*)value->mv_data;
if (value->mv_size < compressionThreshold && !(value->mv_size > startingOffset && ((uint8_t*)data)[startingOffset] >= 250))
return freeValue; // don't compress if less than threshold (but we must compress if the first byte is the compression indicator)
bool longSize = dataLength >= 0x1000000;
int prefixSize = (longSize ? 8 : 4) + startingOffset;
int maxCompressedSize = LZ4_COMPRESSBOUND(dataLength);
char* compressed = new char[maxCompressedSize + prefixSize];
//fprintf(stdout, "compressing %u\n", dataLength);
if (!stream)
stream = LZ4_createStream();
LZ4_loadDict(stream, compressDictionary, dictionarySize);
int compressedSize = LZ4_compress_fast_continue(stream, data + startingOffset, compressed + prefixSize, dataLength, maxCompressedSize, acceleration);
if (compressedSize > 0) {
if (startingOffset > 0) // copy the uncompressed prefix
memcpy(compressed, data, startingOffset);
if (freeValue)
freeValue(*value);
uint8_t* compressedData = (uint8_t*)compressed + startingOffset;
if (longSize) {
compressedData[0] = 255;
compressedData[2] = (uint8_t)(dataLength >> 40u);
compressedData[3] = (uint8_t)(dataLength >> 32u);
compressedData[4] = (uint8_t)(dataLength >> 24u);
compressedData[5] = (uint8_t)(dataLength >> 16u);
compressedData[6] = (uint8_t)(dataLength >> 8u);
compressedData[7] = (uint8_t)dataLength;
}
else {
compressedData[0] = 254;
compressedData[1] = (uint8_t)(dataLength >> 16u);
compressedData[2] = (uint8_t)(dataLength >> 8u);
compressedData[3] = (uint8_t)dataLength;
}
value->mv_size = compressedSize + prefixSize;
value->mv_data = compressed;
return ([](MDB_val &value) -> void {
delete[] (char*)value.mv_data;
});
}
else {
delete[] compressed;
return nullptr;
}
}
class CompressionWorker : public AsyncWorker {
public:
CompressionWorker(EnvWrap* env, double* compressionAddress, const Function& callback)
: AsyncWorker(callback), env(env), compressionAddress(compressionAddress) {}
void Execute() {
uint64_t compressionPointer;
compressionPointer = std::atomic_exchange((std::atomic<int64_t>*) compressionAddress, (int64_t) 2);
if (compressionPointer > 1) {
Compression* compression = (Compression*)(size_t) * ((double*)&compressionPointer);
compression->compressInstruction(env, compressionAddress);
}
}
void OnOK() {
// don't actually call the callback, no need
}
private:
EnvWrap* env;
double* compressionAddress;
};
NAPI_FUNCTION(EnvWrap::compress) {
ARGS(3)
GET_INT64_ARG(0);
EnvWrap* ew = (EnvWrap*) i64;
napi_get_value_int64(env, args[1], &i64);
double* compressionAddress = (double*) i64;
CompressionWorker* worker = new CompressionWorker(ew, (double*) compressionAddress, Function(env, args[2]));
worker->Queue();
RETURN_UNDEFINED;
}
void Compression::setupExports(Napi::Env env, Object exports) {
Function CompressionClass = DefineClass(env, "Compression", {
Compression::InstanceMethod("setBuffer", &Compression::setBuffer),
});
exports.Set("Compression", CompressionClass);
// compressionTpl->InstanceTemplate()->SetInternalFieldCount(1);
}
// This file contains code from the node-lmdb project
// Copyright (c) 2013-2017 Timur Kristóf
// Copyright (c) 2021 Kristopher Tate
// Licensed to you under the terms of the MIT license
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

398
node_modules/lmdb/src/cursor.cpp generated vendored Normal file
View File

@@ -0,0 +1,398 @@
#include "lmdb-js.h"
#include <string.h>
using namespace Napi;
const int INCLUDE_VALUES = 0x100;
const int REVERSE = 0x400;
const int VALUES_FOR_KEY = 0x800;
const int ONLY_COUNT = 0x1000;
const int RENEW_CURSOR = 0x2000;
const int EXACT_MATCH = 0x4000;
const int INCLUSIVE_END = 0x8000;
const int EXCLUSIVE_START = 0x10000;
CursorWrap::CursorWrap(const CallbackInfo& info) : Napi::ObjectWrap<CursorWrap>(info) {
this->keyType = LmdbKeyType::StringKey;
this->freeKey = nullptr;
this->endKey.mv_size = 0; // indicates no end key (yet)
if (info.Length() < 1) {
throwError(info.Env(), "Wrong number of arguments");
return;
}
DbiWrap *dw;
napi_unwrap(info.Env(), info[0], (void**)&dw);
int64_t tw_address = 0;
napi_get_value_int64(info.Env(), info[1], &tw_address);
// Open the cursor
MDB_cursor *cursor;
MDB_txn *txn = dw->ew->getReadTxn(tw_address);
int rc = mdb_cursor_open(txn, dw->dbi, &cursor);
if (rc != 0) {
throwLmdbError(info.Env(), rc);
return;
}
info.This().As<Object>().Set("address", Number::New(info.Env(), (size_t) this));
this->cursor = cursor;
this->dw = dw;
this->txn = txn;
this->keyType = keyType;
}
CursorWrap::~CursorWrap() {
if (this->cursor) {
// Don't close cursor here, it is possible that the environment may already be closed, which causes it to crash
//mdb_cursor_close(this->cursor);
}
if (this->freeKey) {
this->freeKey(this->key);
}
}
Value CursorWrap::close(const CallbackInfo& info) {
if (this->cursor) {
mdb_cursor_close(this->cursor);
this->cursor = nullptr;
}
return info.Env().Undefined();
}
Value CursorWrap::del(const CallbackInfo& info) {
int flags = 0;
if (info.Length() == 1) {
if (!info[0].IsObject()) {
return throwError(info.Env(), "cursor.del: Invalid options argument. It should be an object.");
}
auto options = info[0].As<Object>();
setFlagFromValue(&flags, MDB_NODUPDATA, "noDupData", false, options);
}
int rc = mdb_cursor_del(this->cursor, flags);
if (rc != 0) {
return throwLmdbError(info.Env(), rc);
}
return info.Env().Undefined();
}
int CursorWrap::returnEntry(int lastRC, MDB_val &key, MDB_val &data) {
if (lastRC) {
if (lastRC == MDB_NOTFOUND)
return 0;
else {
return lastRC > 0 ? -lastRC : lastRC;
}
}
if (endKey.mv_size > 0) {
int comparison;
if (flags & VALUES_FOR_KEY)
comparison = mdb_dcmp(txn, dw->dbi, &endKey, &data);
else
comparison = mdb_cmp(txn, dw->dbi, &endKey, &key);
if ((flags & REVERSE) ? comparison >= 0 : (comparison <= 0)) {
if (!((flags & INCLUSIVE_END) && comparison == 0))
return 0;
}
}
char* keyBuffer = dw->ew->keyBuffer;
if (flags & INCLUDE_VALUES) {
int result = getVersionAndUncompress(data, dw);
bool fits = true;
if (result) {
fits = valToBinaryFast(data, dw); // it fit in the global/compression-target buffer
}
#if ENABLE_V8_API
if (fits || result == 2 || data.mv_size < SHARED_BUFFER_THRESHOLD) {// if it was decompressed
#endif
*((uint32_t*)keyBuffer) = data.mv_size;
*((uint32_t*)(keyBuffer + 4)) = 0; // buffer id of 0
#if ENABLE_V8_API
} else {
EnvWrap::toSharedBuffer(dw->ew->env, (uint32_t*) dw->ew->keyBuffer, data);
}
#endif
}
if (!(flags & VALUES_FOR_KEY)) {
memcpy(keyBuffer + 32, key.mv_data, key.mv_size);
*(keyBuffer + 32 + key.mv_size) = 0; // make sure it is null terminated for the sake of better ordered-binary performance
}
return key.mv_size;
}
const int START_ADDRESS_POSITION = 4064;
int32_t CursorWrap::doPosition(uint32_t offset, uint32_t keySize, uint64_t endKeyAddress) {
//char* keyBuffer = dw->ew->keyBuffer;
MDB_val key, data;
int rc;
if (dw->ew->env == nullptr) {
return MDB_BAD_TXN;
}
if (flags & RENEW_CURSOR) { // TODO: check the txn_id to determine if we need to renew
rc = mdb_cursor_renew(txn = dw->ew->getReadTxn(), cursor);
if (rc) {
if (rc > 0)
rc = -rc;
return rc;
}
}
if (endKeyAddress) {
uint32_t* keyBuffer = (uint32_t*) endKeyAddress;
endKey.mv_size = *keyBuffer;
endKey.mv_data = (char*)(keyBuffer + 1);
} else
endKey.mv_size = 0;
iteratingOp = (flags & REVERSE) ?
(flags & INCLUDE_VALUES) ?
(flags & VALUES_FOR_KEY) ? MDB_PREV_DUP : MDB_PREV :
MDB_PREV_NODUP :
(flags & INCLUDE_VALUES) ?
(flags & VALUES_FOR_KEY) ? MDB_NEXT_DUP : MDB_NEXT :
MDB_NEXT_NODUP;
key.mv_size = keySize;
key.mv_data = dw->ew->keyBuffer;
if (keySize == 0) {
rc = mdb_cursor_get(cursor, &key, &data, flags & REVERSE ? MDB_LAST : MDB_FIRST);
} else {
if (flags & VALUES_FOR_KEY) { // only values for this key
// take the next part of the key buffer as a pointer to starting data
uint32_t* startValueBuffer = (uint32_t*)(size_t)(*(double*)(dw->ew->keyBuffer + START_ADDRESS_POSITION));
data.mv_size = endKeyAddress ? *((uint32_t*)startValueBuffer) : 0;
data.mv_data = startValueBuffer + 1;
MDB_val startValue;
if (flags & EXCLUSIVE_START)
startValue = data; // save it for comparison
if (flags & REVERSE) {// reverse through values
startValue = data; // save it for comparison
rc = mdb_cursor_get(cursor, &key, &data, data.mv_size ? MDB_GET_BOTH_RANGE : MDB_SET_KEY);
if (rc) {
if (startValue.mv_size) {
// value specified, but not found, so find key and go to last item
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY);
if (!rc)
rc = mdb_cursor_get(cursor, &key, &data, MDB_LAST_DUP);
} // else just couldn't find the key
} else { // found entry
if (startValue.mv_size == 0) // no value specified, so go to last value
rc = mdb_cursor_get(cursor, &key, &data, MDB_LAST_DUP);
else if (mdb_dcmp(txn, dw->dbi, &startValue, &data)) // the range found the next value *after* the start
rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV_DUP);
}
} else // forward, just do a get by range
rc = mdb_cursor_get(cursor, &key, &data, data.mv_size ?
(flags & EXACT_MATCH) ? MDB_GET_BOTH : MDB_GET_BOTH_RANGE : MDB_SET_KEY);
if (rc == MDB_NOTFOUND)
return 0;
if (flags & ONLY_COUNT && (!endKeyAddress || (flags & EXACT_MATCH))) {
size_t count;
rc = mdb_cursor_count(cursor, &count);
if (rc)
return rc > 0 ? -rc : rc;
return count;
}
if (flags & EXCLUSIVE_START) {
while(!rc) {
if (mdb_dcmp(txn, dw->dbi, &startValue, &data))
break;
rc = mdb_cursor_get(cursor, &key, &data, iteratingOp);
}
}
} else {
MDB_val firstKey;
if (flags & EXCLUSIVE_START)
firstKey = key; // save it for comparison
if (flags & REVERSE) {// reverse
firstKey = key; // save it for comparison
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_RANGE);
if (rc)
rc = mdb_cursor_get(cursor, &key, &data, MDB_LAST);
else if (mdb_cmp(txn, dw->dbi, &firstKey, &key)) // the range found the next entry *after* the start
rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV);
else if (dw->flags & MDB_DUPSORT)
// we need to go to the last value of this key
rc = mdb_cursor_get(cursor, &key, &data, MDB_LAST_DUP);
} else // forward, just do a get by range
rc = mdb_cursor_get(cursor, &key, &data, (flags & EXACT_MATCH) ? MDB_SET_KEY : MDB_SET_RANGE);
if (flags & EXCLUSIVE_START) {
while(!rc) {
if (mdb_cmp(txn, dw->dbi, &firstKey, &key))
break;
rc = mdb_cursor_get(cursor, &key, &data, iteratingOp);
}
}
}
}
while (offset-- > 0 && !rc) {
rc = mdb_cursor_get(cursor, &key, &data, iteratingOp);
}
if (flags & ONLY_COUNT) {
uint32_t count = 0;
bool useCursorCount = false;
// if we are in a dupsort database, and we are iterating over all entries, we can just count all the values for each key
if (dw->flags & MDB_DUPSORT) {
if (iteratingOp == MDB_PREV) {
iteratingOp = MDB_PREV_NODUP;
useCursorCount = true;
}
if (iteratingOp == MDB_NEXT) {
iteratingOp = MDB_NEXT_NODUP;
useCursorCount = true;
}
}
while (!rc) {
if (endKey.mv_size > 0) {
int comparison;
if (flags & VALUES_FOR_KEY)
comparison = mdb_dcmp(txn, dw->dbi, &endKey, &data);
else
comparison = mdb_cmp(txn, dw->dbi, &endKey, &key);
if ((flags & REVERSE) ? comparison >= 0 : (comparison <=0)) {
if (!((flags & INCLUSIVE_END) && comparison == 0))
return count;
}
}
if (useCursorCount) {
size_t countForKey;
rc = mdb_cursor_count(cursor, &countForKey);
if (rc) {
if (rc > 0)
rc = -rc;
return rc;
}
count += countForKey;
} else
count++;
rc = mdb_cursor_get(cursor, &key, &data, iteratingOp);
}
return count;
}
// TODO: Handle count?
return returnEntry(rc, key, data);
}
NAPI_FUNCTION(position) {
ARGS(5)
GET_INT64_ARG(0);
CursorWrap* cw = (CursorWrap*) i64;
GET_UINT32_ARG(cw->flags, 1);
uint32_t offset;
GET_UINT32_ARG(offset, 2);
uint32_t keySize;
GET_UINT32_ARG(keySize, 3);
napi_get_value_int64(env, args[4], &i64);
int64_t endKeyAddress = i64;
int32_t result = cw->doPosition(offset, keySize, endKeyAddress);
RETURN_INT32(result);
}
int32_t positionFFI(double cwPointer, uint32_t flags, uint32_t offset, uint32_t keySize, uint64_t endKeyAddress) {
CursorWrap* cw = (CursorWrap*) (size_t) cwPointer;
DbiWrap* dw = cw->dw;
dw->getFast = true;
cw->flags = flags;
return cw->doPosition(offset, keySize, endKeyAddress);
}
NAPI_FUNCTION(iterate) {
ARGS(1)
GET_INT64_ARG(0);
CursorWrap* cw = (CursorWrap*) i64;
MDB_val key, data;
int rc;
if (cw->dw->ew->env == nullptr) rc = MDB_BAD_TXN;
else
rc = mdb_cursor_get(cw->cursor, &key, &data, cw->iteratingOp);
RETURN_INT32(cw->returnEntry(rc, key, data));
}
int32_t iterateFFI(double cwPointer) {
CursorWrap* cw = (CursorWrap*) (size_t) cwPointer;
DbiWrap* dw = cw->dw;
dw->getFast = true;
MDB_val key, data;
if (cw->dw->ew->env == nullptr)
return MDB_BAD_TXN;
int rc = mdb_cursor_get(cw->cursor, &key, &data, cw->iteratingOp);
return cw->returnEntry(rc, key, data);
}
NAPI_FUNCTION(getCurrentValue) {
ARGS(1)
GET_INT64_ARG(0);
CursorWrap* cw = (CursorWrap*) i64;
MDB_val key, data;
int rc = mdb_cursor_get(cw->cursor, &key, &data, MDB_GET_CURRENT);
RETURN_INT32(cw->returnEntry(rc, key, data));
}
napi_finalize noopCursor = [](napi_env, void *, void *) {
// Data belongs to LMDB, we shouldn't free it here
};
NAPI_FUNCTION(getCurrentShared) {
ARGS(1)
GET_INT64_ARG(0);
CursorWrap* cw = (CursorWrap*) i64;
MDB_val key, data;
int rc = mdb_cursor_get(cw->cursor, &key, &data, MDB_GET_CURRENT);
if (rc)
RETURN_INT32(cw->returnEntry(rc, key, data));
getVersionAndUncompress(data, cw->dw);
napi_create_external_buffer(env, data.mv_size,
(char*) data.mv_data, noopCursor, nullptr, &returnValue);
return returnValue;
}
NAPI_FUNCTION(renew) {
ARGS(1)
GET_INT64_ARG(0);
CursorWrap* cw = (CursorWrap*) i64;
mdb_cursor_renew(cw->txn = cw->dw->ew->getReadTxn(), cw->cursor);
RETURN_UNDEFINED;
}
void CursorWrap::setupExports(Napi::Env env, Object exports) {
// CursorWrap: Prepare constructor template
Function CursorClass = DefineClass(env, "Cursor", {
// CursorWrap: Add functions to the prototype
CursorWrap::InstanceMethod("close", &CursorWrap::close),
CursorWrap::InstanceMethod("del", &CursorWrap::del),
});
EXPORT_NAPI_FUNCTION("position", position);
EXPORT_NAPI_FUNCTION("iterate", iterate);
EXPORT_NAPI_FUNCTION("getCurrentValue", getCurrentValue);
EXPORT_NAPI_FUNCTION("getCurrentShared", getCurrentShared);
EXPORT_NAPI_FUNCTION("renew", renew);
EXPORT_FUNCTION_ADDRESS("positionPtr", positionFFI);
EXPORT_FUNCTION_ADDRESS("iteratePtr", iterateFFI);
exports.Set("Cursor", CursorClass);
// cursorTpl->InstanceTemplate()->SetInternalFieldCount(1);
}
// This file contains code from the node-lmdb project
// Copyright (c) 2013-2017 Timur Kristóf
// Copyright (c) 2021 Kristopher Tate
// Licensed to you under the terms of the MIT license
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

419
node_modules/lmdb/src/dbi.cpp generated vendored Normal file
View File

@@ -0,0 +1,419 @@
#include "lmdb-js.h"
#include <cstdio>
using namespace Napi;
void setFlagFromValue(int *flags, int flag, const char *name, bool defaultValue, Object options);
DbiWrap::DbiWrap(const Napi::CallbackInfo& info) : ObjectWrap<DbiWrap>(info) {
this->dbi = 0;
this->keyType = LmdbKeyType::DefaultKey;
this->compression = nullptr;
this->isOpen = false;
this->getFast = false;
this->ew = nullptr;
EnvWrap *ew;
napi_unwrap(info.Env(), info[0], (void**) &ew);
this->env = ew->env;
this->ew = ew;
int flags = info[1].As<Number>();
char* nameBytes;
std::string name;
if (info[2].IsString()) {
name = info[2].As<String>().Utf8Value();
nameBytes = (char*) name.c_str();
} else
nameBytes = nullptr;
LmdbKeyType keyType = (LmdbKeyType) info[3].As<Number>().Int32Value();
Compression* compression;
if (info[4].IsObject())
napi_unwrap(info.Env(), info[4], (void**) &compression);
else
compression = nullptr;
int rc = this->open(flags, nameBytes, flags & HAS_VERSIONS,
keyType, compression);
//if (nameBytes)
//delete nameBytes;
if (rc) {
if (rc == MDB_NOTFOUND)
this->dbi = (MDB_dbi) 0xffffffff;
else {
//delete this;
throwLmdbError(info.Env(), rc);
return;
}
}
info.This().As<Object>().Set("dbi", Number::New(info.Env(), this->dbi));
info.This().As<Object>().Set("address", Number::New(info.Env(), (size_t) this));
}
DbiWrap::~DbiWrap() {
// Imagine the following JS:
// ------------------------
// var dbi1 = env.openDbi({ name: "hello" });
// var dbi2 = env.openDbi({ name: "hello" });
// dbi1.close();
// txn.putString(dbi2, "world");
// -----
// The above DbiWrap objects would both wrap the same MDB_dbi, and if closing the first one called mdb_dbi_close,
// that'd also render the second DbiWrap instance unusable.
//
// For this reason, we will never call mdb_dbi_close
// NOTE: according to LMDB authors, it is perfectly fine if mdb_dbi_close is never called on an MDB_dbi
}
int DbiWrap::open(int flags, char* name, bool hasVersions, LmdbKeyType keyType, Compression* compression) {
MDB_txn* txn = ew->getReadTxn();
this->hasVersions = hasVersions;
this->compression = compression;
this->keyType = keyType;
#ifndef MDB_RPAGE_CACHE
flags &= ~0x100; // remove use versions flag for older lmdb
#endif
this->flags = flags;
int rc = txn ? mdb_dbi_open(txn, name, flags, &this->dbi) : EINVAL;
if (rc)
return rc;
this->isOpen = true;
if (keyType == LmdbKeyType::DefaultKey && name) { // use the fast compare, but can't do it if we have db table/names mixed in
mdb_set_compare(txn, dbi, compareFast);
}
return 0;
}
Value DbiWrap::close(const Napi::CallbackInfo& info) {
if (this->isOpen) {
mdb_dbi_close(this->env, this->dbi);
this->isOpen = false;
this->ew = nullptr;
}
else {
return throwError(info.Env(), "The Dbi is not open, you can't close it.");
}
return info.Env().Undefined();
}
Value DbiWrap::drop(const Napi::CallbackInfo& info) {
int del = 1;
int rc;
if (!this->isOpen) {
return throwError(info.Env(), "The Dbi is not open, you can't drop it.");
}
// Check if the database should be deleted
if (info.Length() == 1 && info[0].IsObject()) {
Napi::Object options = info[0].As<Object>();
// Just free pages
Napi::Value opt = options.Get("justFreePages");
del = opt.IsBoolean() ? !opt.As<Boolean>().Value() : 1;
}
// Drop database
rc = mdb_drop(ew->writeTxn->txn, dbi, del);
if (rc != 0) {
return throwLmdbError(info.Env(), rc);
}
// Only close database if del == 1
if (del == 1) {
isOpen = false;
ew = nullptr;
}
return info.Env().Undefined();
}
Value DbiWrap::stat(const Napi::CallbackInfo& info) {
MDB_stat stat;
mdb_stat(this->ew->getReadTxn(), dbi, &stat);
Object stats = Object::New(info.Env());
stats.Set("pageSize", Number::New(info.Env(), stat.ms_psize));
stats.Set("treeDepth", Number::New(info.Env(), stat.ms_depth));
stats.Set("treeBranchPageCount", Number::New(info.Env(), stat.ms_branch_pages));
stats.Set("treeLeafPageCount", Number::New(info.Env(), stat.ms_leaf_pages));
stats.Set("entryCount", Number::New(info.Env(), stat.ms_entries));
stats.Set("overflowPages", Number::New(info.Env(), stat.ms_overflow_pages));
return stats;
}
int32_t DbiWrap::doGetByBinary(uint32_t keySize, uint32_t ifNotTxnId, int64_t txnWrapAddress) {
char* keyBuffer = ew->keyBuffer;
MDB_txn* txn = ew->getReadTxn(txnWrapAddress);
MDB_val key, data;
key.mv_size = keySize;
key.mv_data = (void*) keyBuffer;
uint32_t* currentTxnId = (uint32_t*) (keyBuffer + 32);
#ifdef MDB_RPAGE_CACHE
int result = mdb_get_with_txn(txn, dbi, &key, &data, (mdb_size_t*) currentTxnId);
#else
int result = mdb_get(txn, dbi, &key, &data);
#endif
if (result) {
if (result > 0)
return -result;
return result;
}
#ifdef MDB_RPAGE_CACHE
if (ifNotTxnId && ifNotTxnId == *currentTxnId)
return -30004;
#endif
result = getVersionAndUncompress(data, this);
bool fits = true;
if (result) {
fits = valToBinaryFast(data, this); // it fits in the global/compression-target buffer
}
#if ENABLE_V8_API
// TODO: We may want to enable this for Bun as well, since it probably doesn't have the same
// shared pointer problems that V8 does
if (fits || result == 2 || data.mv_size < SHARED_BUFFER_THRESHOLD) {// result = 2 if it was decompressed
#endif
if (data.mv_size < 0x80000000)
return data.mv_size;
*((uint32_t*)keyBuffer) = data.mv_size;
return -30000;
#if ENABLE_V8_API
} else {
return EnvWrap::toSharedBuffer(ew->env, (uint32_t*) ew->keyBuffer, data);
}
#endif
}
NAPI_FUNCTION(directWrite) {
ARGS(5)
GET_INT64_ARG(0);
DbiWrap* dw = (DbiWrap*) i64;
uint32_t keySize;
GET_UINT32_ARG(keySize, 1);
uint32_t offset;
GET_UINT32_ARG(offset, 2);
uint32_t dataSize;
GET_UINT32_ARG(dataSize, 3);
int64_t txnAddress = 0;
napi_status status = napi_get_value_int64(env, args[4], &txnAddress);
if (dw->hasVersions) offset += 8;
EnvWrap* ew = dw->ew;
char* keyBuffer = ew->keyBuffer;
MDB_txn* txn = ew->getReadTxn(txnAddress);
MDB_val key, data;
key.mv_size = keySize;
key.mv_data = (void*) keyBuffer;
data.mv_size = dataSize;
data.mv_data = (void*) (keyBuffer + (((keySize >> 3) + 1) << 3));
#ifdef MDB_RPAGE_CACHE
int result = mdb_direct_write(txn, dw->dbi, &key, offset, &data);
#else
int result = -1;
#endif
RETURN_INT32(result);
}
NAPI_FUNCTION(getByBinary) {
ARGS(4)
GET_INT64_ARG(0);
DbiWrap* dw = (DbiWrap*) i64;
uint32_t keySize;
GET_UINT32_ARG(keySize, 1);
uint32_t ifNotTxnId;
GET_UINT32_ARG(ifNotTxnId, 2);
int64_t txnAddress = 0;
napi_status status = napi_get_value_int64(env, args[3], &txnAddress);
RETURN_INT32(dw->doGetByBinary(keySize, ifNotTxnId, txnAddress));
}
uint32_t getByBinaryFFI(double dwPointer, uint32_t keySize, uint32_t ifNotTxnId, uint64_t txnAddress) {
DbiWrap* dw = (DbiWrap*) (size_t) dwPointer;
return dw->doGetByBinary(keySize, ifNotTxnId, txnAddress);
}
napi_finalize noopDbi = [](napi_env, void *, void *) {
// Data belongs to LMDB, we shouldn't free it here
};
NAPI_FUNCTION(getSharedByBinary) {
ARGS(2)
GET_INT64_ARG(0);
DbiWrap* dw = (DbiWrap*) i64;
uint32_t keySize;
GET_UINT32_ARG(keySize, 1);
MDB_val key;
MDB_val data;
key.mv_size = keySize;
key.mv_data = (void*) dw->ew->keyBuffer;
MDB_txn* txn = dw->ew->getReadTxn();
int rc = mdb_get(txn, dw->dbi, &key, &data);
if (rc) {
if (rc == MDB_NOTFOUND) {
RETURN_UNDEFINED;
} else
return throwLmdbError(env, rc);
}
rc = getVersionAndUncompress(data, dw);
napi_create_external_buffer(env, data.mv_size,
(char*) data.mv_data, noopDbi, nullptr, &returnValue);
return returnValue;
}
NAPI_FUNCTION(getStringByBinary) {
ARGS(3)
GET_INT64_ARG(0);
DbiWrap* dw = (DbiWrap*) i64;
uint32_t keySize;
GET_UINT32_ARG(keySize, 1);
int64_t txnAddress = 0;
napi_status status = napi_get_value_int64(env, args[2], &txnAddress);
MDB_val key;
MDB_val data;
key.mv_size = keySize;
key.mv_data = (void*) dw->ew->keyBuffer;
MDB_txn* txn = dw->ew->getReadTxn(txnAddress);
int rc = mdb_get(txn, dw->dbi, &key, &data);
if (rc) {
if (rc == MDB_NOTFOUND) {
RETURN_UNDEFINED;
} else
return throwLmdbError(env, rc);
}
rc = getVersionAndUncompress(data, dw);
if (rc)
napi_create_string_utf8(env, (char*) data.mv_data, data.mv_size, &returnValue);
else
napi_create_int32(env, data.mv_size, &returnValue);
return returnValue;
}
int DbiWrap::prefetch(uint32_t* keys) {
MDB_txn* txn = ExtendedEnv::getPrefetchReadTxn(ew->env);
MDB_val key;
MDB_val data;
unsigned int flags;
mdb_dbi_flags(txn, dbi, &flags);
bool findAllValues = flags & MDB_DUPSORT;
int effected = 0;
bool findDataValue = false;
MDB_cursor *cursor;
int rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
ExtendedEnv::donePrefetchReadTxn(txn);
return rc;
}
while((key.mv_size = *keys++) > 0) {
if (key.mv_size == 0xffffffff) {
// it is a pointer to a new buffer
keys = (uint32_t*) (size_t) *((double*) keys); // read as a double pointer
key.mv_size = *keys++;
if (key.mv_size == 0)
break;
}
if (key.mv_size & 0x80000000) {
// indicator of using a data value combination (with dupSort), to be followed by the corresponding key
data.mv_size = key.mv_size & 0x7fffffff;
data.mv_data = (void *) keys;
keys += (data.mv_size + 12) >> 2;
findDataValue = true;
findAllValues = false;
continue;
}
// else standard key
key.mv_data = (void *) keys;
keys += (key.mv_size + 12) >> 2;
int rc = mdb_cursor_get(cursor, &key, &data, findDataValue ? MDB_GET_BOTH : MDB_SET_KEY);
findDataValue = false;
while (!rc) {
// access one byte from each of the pages to ensure they are in the OS cache,
// potentially triggering the hard page fault in this thread
int pages = (data.mv_size + 0xfff) >> 12;
// TODO: Adjust this for the page headers, I believe that makes the first page slightly less 4KB.
for (int i = 0; i < pages; i++) {
effected += *(((uint8_t*)data.mv_data) + (i << 12));
}
if (findAllValues) // in dupsort databases, access the rest of the values
rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_DUP);
else
rc = 1; // done
}
}
mdb_cursor_close(cursor);
ExtendedEnv::donePrefetchReadTxn(txn);
return effected;
}
class PrefetchWorker : public AsyncWorker {
public:
PrefetchWorker(DbiWrap* dw, uint32_t* keys, const Function& callback)
: AsyncWorker(callback), dw(dw), keys(keys) {}
void Execute() {
dw->prefetch(keys);
}
void OnOK() {
napi_value result; // we use direct napi call here because node-addon-api interface with throw a fatal error if a worker thread is terminating
napi_call_function(Env(), Env().Undefined(), Callback().Value(), 0, {}, &result);
}
void OnError(const Error& e) {
napi_value result; // we use direct napi call here because node-addon-api interface with throw a fatal error if a worker thread is terminating
napi_value arg = e.Value();
napi_call_function(Env(), Env().Undefined(), Callback().Value(), 1, &arg, &result);
}
private:
DbiWrap* dw;
uint32_t* keys;
};
NAPI_FUNCTION(prefetchNapi) {
ARGS(3)
GET_INT64_ARG(0);
DbiWrap* dw = (DbiWrap*) i64;
napi_get_value_int64(env, args[1], &i64);
uint32_t* keys = (uint32_t*) i64;
PrefetchWorker* worker = new PrefetchWorker(dw, keys, Function(env, args[2]));
worker->Queue();
RETURN_UNDEFINED;
}
void DbiWrap::setupExports(Napi::Env env, Object exports) {
Function DbiClass = DefineClass(env, "Dbi", {
// DbiWrap: Prepare constructor template
// DbiWrap: Add functions to the prototype
DbiWrap::InstanceMethod("close", &DbiWrap::close),
DbiWrap::InstanceMethod("drop", &DbiWrap::drop),
DbiWrap::InstanceMethod("stat", &DbiWrap::stat),
});
exports.Set("Dbi", DbiClass);
EXPORT_NAPI_FUNCTION("directWrite", directWrite);
EXPORT_NAPI_FUNCTION("getByBinary", getByBinary);
EXPORT_NAPI_FUNCTION("prefetch", prefetchNapi);
EXPORT_NAPI_FUNCTION("getStringByBinary", getStringByBinary);
EXPORT_NAPI_FUNCTION("getSharedByBinary", getSharedByBinary);
EXPORT_FUNCTION_ADDRESS("getByBinaryPtr", getByBinaryFFI);
// TODO: wrap mdb_stat too
}
// This file contains code from the node-lmdb project
// Copyright (c) 2013-2017 Timur Kristóf
// Copyright (c) 2021 Kristopher Tate
// Licensed to you under the terms of the MIT license
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

1383
node_modules/lmdb/src/env.cpp generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

64
node_modules/lmdb/src/lmdb-js.cpp generated vendored Normal file
View File

@@ -0,0 +1,64 @@
#include "lmdb-js.h"
using namespace Napi;
int Logging::initLogging() {
char* logging = getenv("LMDB_JS_LOGGING");
if (logging)
fprintf(stderr, "Start logging for lmdb-js\n");
return !!logging;
}
int Logging::debugLogging = Logging::initLogging();
Object InitAll(Napi::Env env, Object exports) {
if (Logging::debugLogging)
fprintf(stderr, "Start initialization\n");
// Initializes the module
// Export Env as constructor for EnvWrap
EnvWrap::setupExports(env, exports);
// Export Cursor as constructor for CursorWrap
CursorWrap::setupExports(env, exports);
TxnWrap::setupExports(env, exports);
DbiWrap::setupExports(env, exports);
CursorWrap::setupExports(env, exports);
Compression::setupExports(env, exports);
// Export misc things
setupExportMisc(env, exports);
if (Logging::debugLogging)
fprintf(stderr, "Finished initialization\n");
return exports;
}
NAPI_MODULE_INIT() {
Value exp = Value::From(env, exports);
return InitAll(env, exp.As<Object>());
}
#ifndef _WIN32
extern "C" void node_module_register(void* m) {
//fprintf(stderr, "This is just a dummy function to be called if node isn't there so deno can load this module\n");
}
#endif
// This file contains code from the node-lmdb project
// Copyright (c) 2013-2017 Timur Kristóf
// Licensed to you under the terms of the MIT license
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

687
node_modules/lmdb/src/lmdb-js.h generated vendored Normal file
View File

@@ -0,0 +1,687 @@
#ifndef NODE_LMDB_H
#define NODE_LMDB_H
#include <vector>
#include <unordered_map>
#include <algorithm>
#include <ctime>
#include <napi.h>
#include <node_api.h>
#if ENABLE_V8_API
#include <v8.h>
#endif
#include "lmdb.h"
#include "lz4.h"
#ifdef MDB_RPAGE_CACHE
#include "chacha8.h"
#endif
using namespace Napi;
// set the threshold of when to use shared buffers (for uncompressed entries larger than this value)
const size_t SHARED_BUFFER_THRESHOLD = 0x1000000;
const uint32_t SPECIAL_WRITE = 0x10101;
const uint32_t REPLACE_WITH_TIMESTAMP_FLAG = 0x1000000;
const uint32_t REPLACE_WITH_TIMESTAMP = 0x1010101;
const uint32_t DIRECT_WRITE = 0x2000000;
#ifndef __CPTHREAD_H__
#define __CPTHREAD_H__
#ifdef _WIN32
# include <windows.h>
#else
# include <pthread.h>
#endif
#ifdef _WIN32
typedef CRITICAL_SECTION pthread_mutex_t;
typedef void pthread_mutexattr_t;
typedef void pthread_condattr_t;
typedef HANDLE pthread_t;
typedef CONDITION_VARIABLE pthread_cond_t;
#endif
#ifndef mdb_size_t
typedef size_t mdb_size_t;
#endif
#define NAPI_FUNCTION(name) napi_value name(napi_env env, napi_callback_info info)
#define ARGS(count) napi_value returnValue;\
size_t argc = count;\
napi_value args[count];\
napi_get_cb_info(env, info, &argc, args, NULL, NULL);
#define GET_UINT32_ARG(target, position) napi_get_value_uint32(env, args[position], (uint32_t*) &target)
#define GET_INT32_ARG(target, position) napi_get_value_int32(env, args[position], (int32_t*) &target)
#define GET_INT64_ARG(position)\
int64_t i64;\
napi_get_value_int64(env, args[position], &i64);
#define RETURN_UINT32(value) { napi_create_uint32(env, value, &returnValue); return returnValue; }
#define RETURN_INT32(value) { napi_create_int32(env, value, &returnValue); return returnValue; }
#define RETURN_UNDEFINED { napi_get_undefined(env, &returnValue); return returnValue; }
#define THROW_ERROR(message) { napi_throw_error(env, NULL, message); napi_get_undefined(env, &returnValue); return returnValue; }
#define EXPORT_NAPI_FUNCTION(name, func) { napi_property_descriptor desc = { name, 0, func, 0, 0, 0, (napi_property_attributes) (napi_writable | napi_configurable), 0 };\
napi_define_properties(env, exports, 1, &desc); }
#define EXPORT_FUNCTION_ADDRESS(name, func) { \
napi_value address;\
void* f = (void*) func;\
napi_create_double(env, *((double*) &f), &address);\
napi_property_descriptor desc = { name, 0, 0, 0, 0, address, (napi_property_attributes) (napi_writable | napi_configurable), 0 };\
napi_define_properties(env, exports, 1, &desc); }
#ifdef _WIN32
const uint64_t TICKS_PER_SECOND = 1000;
int pthread_mutex_init(pthread_mutex_t *mutex, pthread_mutexattr_t *attr);
int pthread_mutex_destroy(pthread_mutex_t *mutex);
int pthread_mutex_lock(pthread_mutex_t *mutex);
int pthread_mutex_unlock(pthread_mutex_t *mutex);
int pthread_cond_destroy(pthread_cond_t *cond);
int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex);
int pthread_cond_signal(pthread_cond_t *cond);
int pthread_cond_broadcast(pthread_cond_t *cond);
#else
const uint64_t TICKS_PER_SECOND = 1000000000;
#endif
uint64_t get_time64();
uint64_t next_time_double();
uint64_t last_time_double();
int cond_init(pthread_cond_t *cond);
int cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, uint64_t ns);
// if we need to add others: https://stackoverflow.com/questions/41770887/cross-platform-definition-of-byteswap-uint64-and-byteswap-ulong/46137633#46137633
#ifdef _WIN32
#define bswap_64(x) _byteswap_uint64(x)
#elif defined(__APPLE__)
#include <libkern/OSByteOrder.h>
#define bswap_64(x) OSSwapInt64(x)
#else
#include <byteswap.h> // bswap_64
#endif
#endif /* __CPTHREAD_H__ */
class Logging {
public:
static int debugLogging;
static int initLogging();
};
enum class LmdbKeyType {
// Invalid key (used internally by lmdb-js)
InvalidKey = -1,
// Default key (used internally by lmdb-js)
DefaultKey = 0,
// UCS-2/UTF-16 with zero terminator - Appears to V8 as string
StringKey = 1,
// LMDB fixed size integer key with 32 bit keys - Appearts to V8 as an Uint32
Uint32Key = 2,
// LMDB default key format - Appears to V8 as node::Buffer
BinaryKey = 3,
};
enum class KeyCreation {
Reset = 0,
Continue = 1,
InArray = 2,
};
const int THEAD_MEMORY_THRESHOLD = 4000;
class TxnWrap;
class DbiWrap;
class EnvWrap;
class CursorWrap;
class Compression;
// Exports misc stuff to the module
void setupExportMisc(Env env, Object exports);
// Helper callback
typedef void (*argtokey_callback_t)(MDB_val &key);
void consoleLog(Value val);
void consoleLog(const char *msg);
void consoleLogN(int n);
void setFlagFromValue(int *flags, int flag, const char *name, bool defaultValue, Object options);
void writeValueToEntry(const Value &str, MDB_val *val);
LmdbKeyType keyTypeFromOptions(const Value &val, LmdbKeyType defaultKeyType = LmdbKeyType::DefaultKey);
int getVersionAndUncompress(MDB_val &data, DbiWrap* dw);
int compareFast(const MDB_val *a, const MDB_val *b);
napi_value setGlobalBuffer(napi_env env, napi_callback_info info);
napi_value lmdbError(napi_env env, napi_callback_info info);
napi_value createBufferForAddress(napi_env env, napi_callback_info info);
napi_value getBufferAddress(napi_env env, napi_callback_info info);
napi_value getAddress(napi_env env, napi_callback_info info);
napi_value detachBuffer(napi_env env, napi_callback_info info);
napi_value enableThreadSafeCalls(napi_env env, napi_callback_info info);
napi_value startRead(napi_env env, napi_callback_info info);
napi_value setReadCallback(napi_env env, napi_callback_info info);
Value getAddress(const CallbackInfo& info);
Value lmdbNativeFunctions(const CallbackInfo& info);
napi_value enableDirectV8(napi_env env, napi_callback_info info);
#ifndef thread_local
#ifdef __GNUC__
# define thread_local __thread
#elif __STDC_VERSION__ >= 201112L
# define thread_local _Thread_local
#elif defined(_MSC_VER)
# define thread_local __declspec(thread)
#else
# define thread_local
#endif
#endif
bool valToBinaryFast(MDB_val &data, DbiWrap* dw);
Value valToUtf8(Env env, MDB_val &data);
Value valToString(MDB_val &data);
Value valToStringUnsafe(MDB_val &data);
Value valToBinary(MDB_val &data);
Value valToBinaryUnsafe(MDB_val &data, DbiWrap* dw, Env env);
int putWithVersion(MDB_txn* txn,
MDB_dbi dbi,
MDB_val * key,
MDB_val * data,
unsigned int flags, double version);
Napi::Value throwLmdbError(Napi::Env env, int rc);
Napi::Value throwError(Napi::Env env, const char* message);
class TxnWrap;
class DbiWrap;
class EnvWrap;
class CursorWrap;
class SharedEnv {
public:
MDB_env* env;
uint64_t dev;
uint64_t inode;
int count;
bool hasWrites;
};
const int INTERRUPT_BATCH = 9998;
const int WORKER_WAITING = 9997;
const int RESTART_WORKER_TXN = 9999;
const int RESUME_BATCH = 9996;
const int USER_HAS_LOCK = 9995;
const int SEPARATE_FLUSHED = 1;
const int DELETE_ON_CLOSE = 2;
const int OPEN_FAILED = 0x10000;
class WriteWorker {
public:
WriteWorker(MDB_env* env, EnvWrap* envForTxn, uint32_t* instructions);
void Write();
MDB_txn* txn;
MDB_txn* AcquireTxn(int* flags);
void UnlockTxn();
int WaitForCallbacks(MDB_txn** txn, bool allowCommit, uint32_t* target);
virtual void SendUpdate();
int interruptionStatus;
bool finishedProgress;
int resultCode;
bool hasError;
napi_ref callback;
napi_async_work work;
napi_threadsafe_function progress;
EnvWrap* envForTxn;
virtual ~WriteWorker();
uint32_t* instructions;
int progressStatus;
MDB_env* env;
static int DoWrites(MDB_txn* txn, EnvWrap* envForTxn, uint32_t* instruction, WriteWorker* worker);
static bool threadSafeCallsEnabled;
};
class TxnTracked {
public:
TxnTracked(MDB_txn *txn, unsigned int flags);
~TxnTracked();
unsigned int flags;
MDB_txn *txn;
TxnTracked *parent;
};
typedef void* (get_shared_buffers_t)();
/*
`Env`
Represents a database environment.
(Wrapper for `MDB_env`)
*/
typedef struct env_tracking_t {
pthread_mutex_t* envsLock;
std::vector<SharedEnv> envs;
get_shared_buffers_t* getSharedBuffers;
} env_tracking_t;
typedef struct buffer_info_t { // definition of a buffer that is available/used in JS
int32_t id;
bool isSharedMap;
char* end;
MDB_env* env;
napi_ref ref;
} buffer_info_t;
typedef struct js_buffers_t { // there is one instance of this for each JS (worker) thread, holding all the active buffers
std::unordered_map<char*, buffer_info_t> buffers;
int nextId;
pthread_mutex_t modification_lock;
} js_buffers_t;
typedef struct callback_holder_t {
EnvWrap* ew;
std::vector<napi_threadsafe_function> callbacks;
} callback_holder_t;
typedef struct user_buffer_t {
MDB_val buffer;
std::vector<napi_threadsafe_function> callbacks;
} user_buffer_t;
class ExtendedEnv {
public:
ExtendedEnv();
~ExtendedEnv();
static MDB_txn* prefetchTxns[20];
static pthread_mutex_t* prefetchTxnsLock;
std::unordered_map<std::string, callback_holder_t> lockCallbacks;
std::unordered_map<std::string, user_buffer_t> userSharedBuffers;
pthread_mutex_t locksModificationLock;
pthread_mutex_t userBuffersLock;
uint64_t lastTime; // actually encoded as double
uint64_t previousTime; // actually encoded as double
napi_value getUserSharedBuffer(std::string key, napi_value default_buffer, napi_value func, bool has_callback, napi_env env, EnvWrap* ew);
bool notifyUserCallbacks(std::string key);
bool attemptLock(std::string key, napi_env env, napi_value func, bool has_callback, EnvWrap* ew);
bool unlock(std::string key, bool only_check);
uint64_t getNextTime();
uint64_t getLastTime();
static MDB_txn* getPrefetchReadTxn(MDB_env* env);
static void donePrefetchReadTxn(MDB_txn* txn);
static void removeReadTxns(MDB_env* env);
};
class EnvWrap : public ObjectWrap<EnvWrap> {
private:
// List of open read transactions
std::vector<TxnWrap*> readTxns;
static env_tracking_t* initTracking();
napi_env napiEnv;
// compression settings and space
Compression *compression;
static thread_local std::vector<EnvWrap*>* openEnvWraps;
// Cleans up stray transactions
void cleanupStrayTxns();
void consolidateTxns();
static void cleanupEnvWraps(void* data);
friend class TxnWrap;
friend class DbiWrap;
public:
EnvWrap(const CallbackInfo&);
~EnvWrap();
// The wrapped object
MDB_env *env;
// Current write transaction
static thread_local js_buffers_t* sharedBuffers;
static env_tracking_t* envTracking;
TxnWrap *currentWriteTxn;
TxnTracked *writeTxn;
pthread_mutex_t* writingLock;
pthread_cond_t* writingCond;
std::vector<AsyncWorker*> workers;
#if ENABLE_V8_API
static std::unordered_map<void*, std::shared_ptr<v8::BackingStore>> backingStores;
#endif
MDB_txn* currentReadTxn;
WriteWorker* writeWorker;
bool readTxnRenewed;
bool hasWrites;
uint64_t timeTxnWaiting;
unsigned int jsFlags;
char* keyBuffer;
int pageSize;
time_t lastReaderCheck;
MDB_txn* getReadTxn(int64_t tw_address = 0);
// Sets up exports for the Env constructor
static void setupExports(Napi::Env env, Object exports);
void closeEnv(bool hasLock = false);
int openEnv(int flags, int jsFlags, const char* path, char* keyBuffer, Compression* compression, int maxDbs,
int maxReaders, mdb_size_t mapSize, int pageSize, unsigned int max_free_to_load, unsigned int max_free_to_retain, char* encryptionKey);
/*
Gets statistics about the database environment.
*/
Napi::Value stat(const CallbackInfo& info);
/*
Gets statistics about the free space database
*/
Napi::Value freeStat(const CallbackInfo& info);
/*
Gets information about the database environment.
*/
Napi::Value info(const CallbackInfo& info);
/*
Check for stale readers
*/
Napi::Value readerCheck(const CallbackInfo& info);
/*
Print a list of readers
*/
Napi::Value readerList(const CallbackInfo& info);
/*
Opens the database environment with the specified options. The options will be used to configure the environment before opening it.
(Wrapper for `mdb_env_open`)
Parameters:
* Options object that contains possible configuration options.
Possible options are:
* maxDbs: the maximum number of named databases you can have in the environment (default is 1)
* maxReaders: the maximum number of concurrent readers of the environment (default is 126)
* mapSize: maximal size of the memory map (the full environment) in bytes (default is 10485760 bytes)
* path: path to the database environment
*/
Napi::Value open(const CallbackInfo& info);
Napi::Value getMaxKeySize(const CallbackInfo& info);
/*
Copies the database environment to a file.
(Wrapper for `mdb_env_copy2`)
Parameters:
* path - Path to the target file
* compact (optional) - Copy using compact setting
* callback - Callback when finished (this is performed asynchronously)
*/
Napi::Value copy(const CallbackInfo& info);
/*
Closes the database environment.
(Wrapper for `mdb_env_close`)
*/
Napi::Value close(const CallbackInfo& info);
/*
Starts a new transaction in the environment.
(Wrapper for `mdb_txn_begin`)
Parameters:
* Options object that contains possible configuration options.
Possible options are:
* readOnly: if true, the transaction is read-only
*/
Napi::Value beginTxn(const CallbackInfo& info);
Napi::Value commitTxn(const CallbackInfo& info);
Napi::Value abortTxn(const CallbackInfo& info);
Napi::Value getWriteTxnId(const CallbackInfo& info);
/*
Flushes all data to the disk asynchronously.
(Asynchronous wrapper for `mdb_env_sync`)
Parameters:
* Callback to be executed after the sync is complete.
*/
Napi::Value sync(const CallbackInfo& info);
/*
Performs a set of operations asynchronously, automatically wrapping it in its own transaction
Parameters:
* Callback to be executed after the sync is complete.
*/
Napi::Value startWriting(const CallbackInfo& info);
Napi::Value resumeWriting(const CallbackInfo& info);
static napi_value compress(napi_env env, napi_callback_info info);
static napi_value write(napi_env env, napi_callback_info info);
static napi_value onExit(napi_env env, napi_callback_info info);
static int32_t toSharedBuffer(MDB_env* env, uint32_t* keyBuffer, MDB_val data);
};
const int TXN_ABORTABLE = 1;
const int TXN_SYNCHRONOUS_COMMIT = 2;
const int TXN_FROM_WORKER = 4;
/*
`Txn`
Represents a transaction running on a database environment.
(Wrapper for `MDB_txn`)
*/
class TxnWrap : public ObjectWrap<TxnWrap> {
private:
// Reference to the MDB_env of the wrapped MDB_txn
MDB_env *env;
// Environment wrapper of the current transaction
EnvWrap *ew;
// parent TW, if it is exists
TxnWrap *parentTw;
// Flags used with mdb_txn_begin
unsigned int flags;
friend class CursorWrap;
friend class DbiWrap;
friend class EnvWrap;
public:
TxnWrap(const CallbackInfo& info);
~TxnWrap();
// The wrapped object
MDB_txn *txn;
// Remove the current TxnWrap from its EnvWrap
void removeFromEnvWrap();
int begin(EnvWrap *ew, unsigned int flags);
MDB_env* getEnv();
/*
Commits the transaction.
(Wrapper for `mdb_txn_commit`)
*/
Napi::Value commit(const CallbackInfo& info);
/*
Aborts the transaction.
(Wrapper for `mdb_txn_abort`)
*/
Napi::Value abort(const CallbackInfo& info);
/*
Aborts a read-only transaction but makes it renewable with `renew`.
(Wrapper for `mdb_txn_reset`)
*/
void reset();
/*
Renews a read-only transaction after it has been reset.
(Wrapper for `mdb_txn_renew`)
*/
Napi::Value renew(const CallbackInfo& info);
static void setupExports(Napi::Env env, Object exports);
};
const int HAS_VERSIONS = 0x100;
/*
`Dbi`
Represents a database instance in an environment.
(Wrapper for `MDB_dbi`)
*/
class DbiWrap : public ObjectWrap<DbiWrap> {
public:
// Tells how keys should be treated
LmdbKeyType keyType;
// Stores flags set when opened
int flags;
// The wrapped object
MDB_dbi dbi;
// Reference to the MDB_env of the wrapped MDB_dbi
MDB_env *env;
// The EnvWrap object of the current Dbi
EnvWrap *ew;
// Whether the Dbi was opened successfully
bool isOpen;
// compression settings and space
Compression* compression;
// versions stored in data
bool hasVersions;
// current unsafe buffer for this db
bool getFast;
friend class TxnWrap;
friend class CursorWrap;
friend class EnvWrap;
DbiWrap(const CallbackInfo& info);
~DbiWrap();
/*
Closes the database instance.
Wrapper for `mdb_dbi_close`)
*/
Napi::Value close(const CallbackInfo& info);
/*
Drops the database instance, either deleting it completely (default) or just freeing its pages.
Parameters:
* Options object that contains possible configuration options.
Possible options are:
* justFreePages - indicates that the database pages need to be freed but the database shouldn't be deleted
*/
Napi::Value drop(const CallbackInfo& info);
Napi::Value stat(const CallbackInfo& info);
int prefetch(uint32_t* keys);
int open(int flags, char* name, bool hasVersions, LmdbKeyType keyType, Compression* compression);
int32_t doGetByBinary(uint32_t keySize, uint32_t ifNotTxnId, int64_t txnAddress);
static void setupExports(Napi::Env env, Object exports);
};
class Compression : public ObjectWrap<Compression> {
public:
char* dictionary; // dictionary to use to decompress
char* compressDictionary; // separate dictionary to use to compress since the decompression dictionary can move around in the main thread
unsigned int dictionarySize;
char* decompressTarget;
unsigned int decompressSize;
unsigned int compressionThreshold;
unsigned int startingOffset; // compression can be configured to start compression at a certain offset, so header bytes are left uncompressed.
// compression acceleration (defaults to 1)
int acceleration;
static thread_local LZ4_stream_t* stream;
void decompress(MDB_val& data, bool &isValid, bool canAllocate);
argtokey_callback_t compress(MDB_val* value, argtokey_callback_t freeValue);
int compressInstruction(EnvWrap* env, double* compressionAddress);
Napi::Value ctor(const CallbackInfo& info);
Napi::Value setBuffer(const CallbackInfo& info);
Compression(const CallbackInfo& info);
friend class EnvWrap;
friend class DbiWrap;
//NAN_METHOD(Compression::startCompressing);
static void setupExports(Napi::Env env, Object exports);
};
/*
`Cursor`
Represents a cursor instance that is assigned to a transaction and a database instance
(Wrapper for `MDB_cursor`)
*/
class CursorWrap : public ObjectWrap<CursorWrap> {
private:
// Key/data pair where the cursor is at, and ending key
MDB_val key, data, endKey;
// Free function for the current key
argtokey_callback_t freeKey;
public:
MDB_cursor_op iteratingOp;
MDB_cursor *cursor;
// Stores how key is represented
LmdbKeyType keyType;
int flags;
DbiWrap *dw;
MDB_txn *txn;
// The wrapped object
CursorWrap(MDB_cursor* cursor);
CursorWrap(const CallbackInfo& info);
~CursorWrap();
// Sets up exports for the Cursor constructor
static void setupExports(Napi::Env env, Object exports);
/*
Closes the cursor.
(Wrapper for `mdb_cursor_close`)
Parameters:
* Transaction object
* Database instance object
*/
Napi::Value close(const CallbackInfo& info);
/*
Deletes the key/data pair to which the cursor refers.
(Wrapper for `mdb_cursor_del`)
*/
Napi::Value del(const CallbackInfo& info);
int returnEntry(int lastRC, MDB_val &key, MDB_val &data);
int32_t doPosition(uint32_t offset, uint32_t keySize, uint64_t endKeyAddress);
//Value getStringByBinary(const CallbackInfo& info);
};
#endif // NODE_LMDB_H
// Portions of this file are from node-lmdb
// Copyright (c) 2013-2017 Timur Kristóf
// Licensed to you under the terms of the MIT license
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

623
node_modules/lmdb/src/misc.cpp generated vendored Normal file
View File

@@ -0,0 +1,623 @@
#include "lmdb-js.h"
#include <atomic>
#include <string.h>
#include <stdio.h>
#include <node_version.h>
#include <time.h>
using namespace Napi;
static thread_local char* globalUnsafePtr;
static thread_local size_t globalUnsafeSize;
void setupExportMisc(Napi::Env env, Object exports) {
Object versionObj = Object::New(env);
int major, minor, patch;
char *str = mdb_version(&major, &minor, &patch);
versionObj.Set("versionString", String::New(env, str));
versionObj.Set("major", Number::New(env, major));
versionObj.Set("minor", Number::New(env, minor));
versionObj.Set("patch", Number::New(env, patch));
#if ENABLE_V8_API
versionObj.Set("nodeCompiledVersion", Number::New(env, NODE_MAJOR_VERSION));
#endif
exports.Set("version", versionObj);
EXPORT_NAPI_FUNCTION("setGlobalBuffer", setGlobalBuffer)
EXPORT_NAPI_FUNCTION("lmdbError", lmdbError)
EXPORT_NAPI_FUNCTION("enableDirectV8", enableDirectV8)
EXPORT_NAPI_FUNCTION("createBufferForAddress", createBufferForAddress);
EXPORT_NAPI_FUNCTION("getAddress", getAddress);
EXPORT_NAPI_FUNCTION("getBufferAddress", getBufferAddress);
EXPORT_NAPI_FUNCTION("detachBuffer", detachBuffer);
EXPORT_NAPI_FUNCTION("startRead", startRead);
EXPORT_NAPI_FUNCTION("setReadCallback", setReadCallback);
EXPORT_NAPI_FUNCTION("enableThreadSafeCalls", enableThreadSafeCalls);
napi_value globalBuffer;
napi_create_buffer(env, SHARED_BUFFER_THRESHOLD, (void**) &globalUnsafePtr, &globalBuffer);
globalUnsafeSize = SHARED_BUFFER_THRESHOLD;
exports.Set("globalBuffer", Object(env, globalBuffer));
}
void setFlagFromValue(int *flags, int flag, const char *name, bool defaultValue, Object options) {
Value opt = options.Get(name);
if (opt.IsBoolean() ? opt.As<Boolean>().Value() : defaultValue)
*flags |= flag;
}
/*
Value valToStringUnsafe(MDB_val &data) {
auto resource = new CustomExternalOneByteStringResource(&data);
auto str = Nan::New<v8::String>(resource);
return str.ToLocalChecked();
}*/
Value valToUtf8(Env env, MDB_val &data) {
return String::New(env, (const char*) data.mv_data, data.mv_size);
}
Value valToString(Env env, MDB_val &data) {
// UTF-16 buffer
const uint16_t *buffer = reinterpret_cast<const uint16_t*>(data.mv_data);
// Number of UTF-16 code points
size_t n = data.mv_size / sizeof(uint16_t);
// Check zero termination
if (n < 1 || buffer[n - 1] != 0) {
return throwError(env, "Invalid zero-terminated UTF-16 string");
}
size_t length = n - 1;
return String::New(env, (const char16_t*)data.mv_data, length);
}
bool valToBinaryFast(MDB_val &data, DbiWrap* dw) {
Compression* compression = dw->compression;
if (compression) {
if (data.mv_data == compression->decompressTarget) {
// already decompressed to the target, nothing more to do
} else {
if (data.mv_size > compression->decompressSize) {
// indicates we could not copy, won't fit
return false;
}
// copy into the buffer target
memcpy(compression->decompressTarget, data.mv_data, data.mv_size);
}
} else {
if (data.mv_size > globalUnsafeSize) {
// indicates we could not copy, won't fit
return false;
}
memcpy(globalUnsafePtr, data.mv_data, data.mv_size);
}
return true;
}
Value valToBinaryUnsafe(MDB_val &data, DbiWrap* dw, Env env) {
valToBinaryFast(data, dw);
return Number::New(env, data.mv_size);
}
int getVersionAndUncompress(MDB_val &data, DbiWrap* dw) {
//fprintf(stdout, "uncompressing %u\n", compressionThreshold);
unsigned char* charData = (unsigned char*) data.mv_data;
if (dw->hasVersions) {
memcpy((dw->ew->keyBuffer + 16), charData, 8);
// fprintf(stderr, "getVersion %u\n", lastVersion);
charData = charData + 8;
data.mv_data = charData;
data.mv_size -= 8;
}
if (data.mv_size == 0) {
return 1;// successFunc(data);
}
unsigned char statusByte = (dw->compression && dw->compression->startingOffset < data.mv_size)
? charData[dw->compression->startingOffset] : 0;
//fprintf(stdout, "uncompressing status %X\n", statusByte);
if (statusByte >= 250) {
bool isValid;
dw->compression->decompress(data, isValid, !dw->getFast);
return isValid ? 2 : 0;
}
return 1;
}
NAPI_FUNCTION(lmdbError) {
ARGS(1)
int32_t error_code;
GET_INT32_ARG(error_code, 0);
return throwLmdbError(env, error_code);
}
NAPI_FUNCTION(setGlobalBuffer) {
ARGS(1)
napi_get_buffer_info(env, args[0], (void**) &globalUnsafePtr, &globalUnsafeSize);
RETURN_UNDEFINED
}
/*Value getBufferForAddress) {
char* address = (char*) (size_t) Nan::To<v8::Number>(info[0]).ToLocalChecked()->Value();
std::unique_ptr<v8::BackingStore> backing = v8::ArrayBuffer::NewBackingStore(
address, 0x100000000, [](void*, size_t, void*){}, nullptr);
auto array_buffer = v8::ArrayBuffer::New(Isolate::GetCurrent(), std::move(backing));
info.GetReturnValue().Set(array_buffer);
}*/
NAPI_FUNCTION(createBufferForAddress) {
ARGS(2)
GET_INT64_ARG(0)
void* data = (void*) i64;
uint32_t length;
GET_UINT32_ARG(length, 1);
napi_create_external_buffer(env, length, data, nullptr, nullptr, &returnValue);
return returnValue;
}
NAPI_FUNCTION(getAddress) {
ARGS(1)
void* data;
size_t length;
napi_get_arraybuffer_info(env, args[0], &data, &length);
napi_create_double(env, (double) (size_t) data, &returnValue);
return returnValue;
}
NAPI_FUNCTION(getBufferAddress) {
ARGS(1)
void* data;
size_t length;
napi_get_buffer_info(env, args[0], &data, &length);
napi_create_double(env, (double) (size_t) data, &returnValue);
return returnValue;
}
NAPI_FUNCTION(detachBuffer) {
ARGS(1)
#if (NAPI_VERSION > 6)
napi_detach_arraybuffer(env, args[0]);
#endif
RETURN_UNDEFINED;
}
class ReadWorker : public AsyncWorker {
public:
ReadWorker(uint32_t* start, const Function& callback)
: AsyncWorker(callback), start(start) {}
void Execute() {
uint32_t instruction;
uint32_t* gets = start;
while((instruction = std::atomic_exchange((std::atomic<uint32_t>*)(gets + 2), (uint32_t)0xf0000000))) {
MDB_val key;
key.mv_size = *(gets + 3);
MDB_dbi dbi = (MDB_dbi) (instruction & 0xffff);
MDB_val data;
MDB_txn* txn = (MDB_txn*) (size_t) *((double*)gets);
unsigned int flags;
mdb_dbi_flags(txn, dbi, &flags);
bool dupSort = flags & MDB_DUPSORT;
int effected = 0;
MDB_cursor *cursor;
int rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc)
return SetError(mdb_strerror(rc));
key.mv_data = (void*) gets;
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY);
MDB_env* env = mdb_txn_env(txn);
*(gets + 3) = data.mv_size;
*((double*)gets) = (double) (size_t) data.mv_data;
gets += (key.mv_size + 28) >> 2;
while (!rc) {
// access one byte from each of the pages to ensure they are in the OS cache,
// potentially triggering the hard page fault in this thread
int pages = (data.mv_size + 0xfff) >> 12;
// TODO: Adjust this for the page headers, I believe that makes the first page slightly less 4KB.
for (int i = 0; i < pages; i++) {
effected += *(((uint8_t*)data.mv_data) + (i << 12));
}
if (dupSort) // in dupsort databases, access the rest of the values
rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_DUP);
else
rc = 1; // done
}
mdb_cursor_close(cursor);
}
}
void OnOK() {
// TODO: For each entry, find the shared buffer
uint32_t* gets = start;
// EnvWrap::toSharedBuffer();
Callback().Call({Env().Null()});
}
private:
uint32_t* start;
};
typedef struct { // this is read results data buffer that is actively being used by a JS thread (read results are written to it)
int id;
char* data;
uint32_t offset;
uint32_t size;
} read_results_buffer_t;
static thread_local std::unordered_map<void*, read_results_buffer_t*>* buffersByWorker;
typedef struct {
napi_ref callback;
napi_ref resource;
} read_callback_t;
static int next_buffer_id = -1;
typedef struct {
uint32_t* instructionAddress;
uint32_t callback_id;
napi_async_work work;
//napi_deferred deferred;
js_buffers_t* buffers;
} read_instruction_t;
const uint32_t ZERO = 0;
void do_read(napi_env nenv, void* instruction_pointer) {
read_instruction_t* readInstruction = (read_instruction_t*) instruction_pointer;
//fprintf(stderr, "lock %p\n", &readInstruction->buffers->modification_lock);
uint32_t* instruction = readInstruction->instructionAddress;
MDB_val key;
key.mv_size = *(instruction + 3);
MDB_dbi dbi = (MDB_dbi) (*(instruction + 2) & 0xffff) ;
MDB_val data;
TxnWrap* tw = (TxnWrap*) (size_t) *((double*)instruction);
MDB_txn* txn = tw->txn;
mdb_txn_renew(txn);
unsigned int flags;
mdb_dbi_flags(txn, dbi, &flags);
bool dupSort = flags & MDB_DUPSORT;
int effected = 0;
MDB_cursor *cursor;
MDB_env* env = mdb_txn_env(txn);
int rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
*instruction = rc;
return;
}
key.mv_data = (void*) (instruction + 4);
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY);
*(instruction + 3) = data.mv_size;
//instruction += (key.mv_size + 28) >> 2;
while (!rc) {
// access one byte from each of the pages to ensure they are in the OS cache,
// potentially triggering the hard page fault in this thread
int pages = (data.mv_size + 0xfff) >> 12;
// TODO: Adjust this for the page headers, I believe that makes the first page slightly less than 4KB.
for (int i = 0; i < pages; i++) {
effected += *(((uint8_t*)data.mv_data) + (i << 12));
}
if (dupSort) // in dupsort databases, access the rest of the values
rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_DUP);
else
rc = 1; // done
}
*instruction = rc;
unsigned int env_flags = 0;
mdb_env_get_flags(env, &env_flags);
if (data.mv_size > 4096
#ifdef MDB_RPAGE_CACHE
&& !(env_flags & MDB_REMAP_CHUNKS)
#endif
) {
EnvWrap::toSharedBuffer(env, instruction, data);
*((double*)instruction) = (double) (size_t) data.mv_data;
} else {
if (!buffersByWorker)
buffersByWorker = new std::unordered_map<void*, read_results_buffer_t*>;
read_results_buffer_t* read_buffer;
auto buffer_search = buffersByWorker->find(readInstruction->buffers);
if (buffer_search == buffersByWorker->end()) {
// create new one
buffersByWorker->emplace(readInstruction->buffers, read_buffer = new read_results_buffer_t);
read_buffer->size = 0;
read_buffer->offset = 0; // force it re-malloc
} else
read_buffer = buffer_search->second;
if ((int) read_buffer->size - (int) read_buffer->offset - 4 < (int) data.mv_size) {
size_t size = 0x40000;// 256KB
read_buffer->data = (char*) malloc(size);
read_buffer->size = size;
read_buffer->offset = 0;
buffer_info_t buffer_info;
buffer_info.end = read_buffer->data + size;
buffer_info.env = nullptr;
buffer_info.isSharedMap = false;
pthread_mutex_lock(&readInstruction->buffers->modification_lock);
buffer_info.id = read_buffer->id = readInstruction->buffers->nextId++;
readInstruction->buffers->buffers.emplace(read_buffer->data, buffer_info);
pthread_mutex_unlock(&readInstruction->buffers->modification_lock);
}
auto position = (uint32_t*) (read_buffer->data + read_buffer->offset);
memcpy(position, data.mv_data, data.mv_size);
position += (data.mv_size + 7) >> 2;
*(instruction + 1) = read_buffer->id;
*(instruction + 2) = read_buffer->offset;
read_buffer->offset = (char*)position - read_buffer->data;
}
mdb_cursor_close(cursor);
//fprintf(stderr, "unlock %p\n", &readInstruction->buffers->modification_lock);
}
static thread_local napi_ref* read_callback;
void read_complete(napi_env env, napi_status status, void* data) {
read_instruction_t* readInstruction = (read_instruction_t*) data;
napi_value callback;
napi_get_reference_value(env, *read_callback, &callback);
//uint32_t count;
napi_value result;
napi_value callback_id;
napi_create_int32(env, readInstruction->callback_id, &callback_id);
status = napi_call_function(env, callback, callback, 1, &callback_id, &result);
napi_delete_async_work(env, readInstruction->work);
delete readInstruction;
//napi_resolve_deferred(env, readInstruction->deferred, resolution);
}
NAPI_FUNCTION(enableThreadSafeCalls) {
WriteWorker::threadSafeCallsEnabled = true;
napi_value returnValue;
RETURN_UNDEFINED;
}
NAPI_FUNCTION(setReadCallback) {
ARGS(1)
read_callback = new napi_ref;
napi_create_reference(env, args[0], 1, read_callback);
RETURN_UNDEFINED;
}
NAPI_FUNCTION(startRead) {
ARGS(4)
GET_INT64_ARG(0);
uint32_t* instructionAddress = (uint32_t*) i64;
read_instruction_t* readInstruction = new read_instruction_t;
readInstruction->instructionAddress = instructionAddress;
uint32_t callback_id;
GET_UINT32_ARG(callback_id, 1);
//napi_create_reference(env, args[1], 1, &readInstruction->callback);
readInstruction->callback_id = callback_id;
readInstruction->buffers = EnvWrap::sharedBuffers;
napi_status status;
status = napi_create_async_work(env, args[2], args[3], do_read, read_complete, readInstruction, &readInstruction->work);
status = napi_queue_async_work(env, readInstruction->work);
RETURN_UNDEFINED;
}/*
NAPI_FUNCTION(nextRead) {
ARGS(1)
uint32_t offset;
GET_UINT32_ARG(offset, 0);
uint32_t* instructionAddress = (uint32_t*) currentReadAddress + offset;
read_callback_t* callback = lastCallback;
uint32_t count;
napi_reference_ref(callback->env, callback->callback, &count);
read_instruction_t* readInstruction = new read_instruction_t;
readInstruction->instructionAddress = instructionAddress;
readInstruction->callback = callback;
napi_async_work work;
napi_create_async_work(callback->env, nullptr, readInstruction, &work);
napi_queue_async_work(env, work);
ReadWorker* worker = new ReadWorker(instructionAddress, Function(env, args[1]));
worker->Queue();
RETURN_UNDEFINED;
}*/
Value lmdbNativeFunctions(const CallbackInfo& info) {
// no-op, just doing this to give a label to the native functions
return info.Env().Undefined();
}
Napi::Value throwLmdbError(Napi::Env env, int rc) {
if (rc < 0 && !(rc < -30700 && rc > -30800))
rc = -rc;
Error error = Error::New(env, mdb_strerror(rc));
error.Set("code", Number::New(env, rc));
error.ThrowAsJavaScriptException();
return env.Undefined();
}
Napi::Value throwError(Napi::Env env, const char* message) {
Error::New(env, message).ThrowAsJavaScriptException();
return env.Undefined();
}
const int ASSIGN_NEXT_TIMESTAMP = 0;
const int ASSIGN_LAST_TIMESTAMP = 1;
const int ASSIGN_NEXT_TIMESTAMP_AND_RECORD_PREVIOUS = 2;
const int ASSIGN_PREVIOUS_TIMESTAMP = 3;
int putWithVersion(MDB_txn * txn,
MDB_dbi dbi,
MDB_val * key,
MDB_val * data,
unsigned int flags, double version) {
// leave 8 header bytes available for version and copy in with reserved memory
char* source_data = (char*) data->mv_data;
int size = data->mv_size;
data->mv_size = size + 8;
int rc = mdb_put(txn, dbi, key, data, flags | MDB_RESERVE);
if (rc == 0) {
// if put is successful, data->mv_data will point into the database where we copy the data to
memcpy((char*) data->mv_data + 8, source_data, size);
memcpy(data->mv_data, &version, 8);
//*((double*) data->mv_data) = version; // this doesn't work on ARM v7 because it is not (guaranteed) memory-aligned
}
data->mv_data = source_data; // restore this so that if it points to data that needs to be freed, it points to the right place
return rc;
}
static uint64_t last_time; // actually encoded as double
#ifdef _WIN32
int pthread_mutex_init(pthread_mutex_t *mutex, pthread_mutexattr_t *attr)
{
(void)attr;
if (mutex == NULL)
return 1;
InitializeCriticalSection(mutex);
return 0;
}
int pthread_mutex_destroy(pthread_mutex_t *mutex)
{
if (mutex == NULL)
return 1;
DeleteCriticalSection(mutex);
return 0;
}
int pthread_mutex_lock(pthread_mutex_t *mutex)
{
if (mutex == NULL)
return 1;
EnterCriticalSection(mutex);
return 0;
}
int pthread_mutex_unlock(pthread_mutex_t *mutex)
{
if (mutex == NULL)
return 1;
LeaveCriticalSection(mutex);
return 0;
}
int cond_init(pthread_cond_t *cond)
{
if (cond == NULL)
return 1;
InitializeConditionVariable(cond);
return 0;
}
int pthread_cond_destroy(pthread_cond_t *cond)
{
/* Windows does not have a destroy for conditionals */
(void)cond;
return 0;
}
int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
{
if (cond == NULL || mutex == NULL)
return 1;
if (!SleepConditionVariableCS(cond, mutex, INFINITE))
return 1;
return 0;
}
int cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, uint64_t ms)
{
if (cond == NULL || mutex == NULL)
return 1;
if (!SleepConditionVariableCS(cond, mutex, ms))
return 1;
return 0;
}
int pthread_cond_signal(pthread_cond_t *cond)
{
if (cond == NULL)
return 1;
WakeConditionVariable(cond);
return 0;
}
int pthread_cond_broadcast(pthread_cond_t *cond)
{
if (cond == NULL)
return 1;
WakeAllConditionVariable(cond);
return 0;
}
uint64_t get_time64() {
return GetTickCount64();
}
// from: https://github.com/wadey/node-microtime/blob/master/src/microtime.cc#L19
uint64_t next_time_double() {
FILETIME ft;
GetSystemTimePreciseAsFileTime(&ft);
unsigned long long t = ft.dwHighDateTime;
t <<= 32;
t |= ft.dwLowDateTime;
t /= 10;
t -= 11644473600000000ULL;
double next_time = (double)t/ 1000;
return *((uint64_t*)&next_time);
}
#else
int cond_init(pthread_cond_t *cond) {
pthread_condattr_t attr;
pthread_condattr_init( &attr);
#if defined(__linux)
// only tested in linux, not available on macos
pthread_condattr_setclock( &attr, CLOCK_MONOTONIC);
#endif
return pthread_cond_init(cond, &attr);
}
int cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, uint64_t cms)
{
struct timespec ts;
#if defined(__linux)
clock_gettime(CLOCK_MONOTONIC, &ts);
#else
// without being able to set the clock for condition/mutexes, need to use default realtime clock on macos
clock_gettime(CLOCK_REALTIME, &ts);
#endif
uint64_t ns = ts.tv_nsec + cms * 10000;
ts.tv_sec += ns / 1000000000;
ts.tv_nsec = ns % 1000000000;
return pthread_cond_timedwait(cond, mutex, &ts);
}
uint64_t get_time64() {
struct timespec time;
clock_gettime(CLOCK_MONOTONIC, &time);
return time.tv_sec * 1000000000ll + time.tv_nsec;
}
uint64_t next_time_double() {
struct timespec time;
clock_gettime(CLOCK_REALTIME, &time);
double next_time = (double)time.tv_sec * 1000 + (double)time.tv_nsec / 1000000;
return *((uint64_t*)&next_time);
}
#endif
// This file contains code from the node-lmdb project
// Copyright (c) 2013-2017 Timur Kristóf
// Copyright (c) 2021 Kristopher Tate
// Licensed to you under the terms of the MIT license
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

50
node_modules/lmdb/src/ordered-binary.cpp generated vendored Normal file
View File

@@ -0,0 +1,50 @@
#include "lmdb-js.h"
#ifdef _WIN32
#define ntohl _byteswap_ulong
#define htonl _byteswap_ulong
#else
#include <arpa/inet.h>
#endif
// compare items by 32-bit comparison, a is user provided and assumed to be zero terminated/padded
// which allows us to do the full 32-bit comparisons safely
int compareFast(const MDB_val *a, const MDB_val *b) {
uint32_t* dataA = (uint32_t*) a->mv_data;
uint32_t* dataB = (uint32_t*) b->mv_data;
size_t remaining = b->mv_size;
uint32_t aVal, bVal;
while(remaining >= 4) {
aVal = ntohl(*dataA);
bVal = ntohl(*dataB);
if (aVal > bVal)
return 1;
if (aVal < bVal)
return -1;
/*diff = (int64_t) ntohl(*dataA) - (int64_t) ntohl(*dataB);
if (diff)
return diff;*/
dataA++;
dataB++;
remaining -= 4;
}
if (remaining) {
if (remaining == 1) {
aVal = *((uint8_t*) dataA);
bVal = *((uint8_t*) dataB);
} else {
aVal = ntohl(*dataA);
#if defined(__BYTE_ORDER__)&&(__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
bVal = remaining == 2 ? *dataB & 0xffff0000 : *dataB & 0xffffff00;
#else
bVal = remaining == 2 ? (*((uint8_t*) dataB) << 24) + (*((uint8_t*) dataB + 1) << 16) :
ntohl(*dataB & 0x00ffffff);
#endif
}
if (aVal > bVal)
return 1;
if (aVal < bVal)
return -1;
}
return a->mv_size - b->mv_size;
}

211
node_modules/lmdb/src/txn.cpp generated vendored Normal file
View File

@@ -0,0 +1,211 @@
#include "lmdb-js.h"
using namespace Napi;
TxnTracked::TxnTracked(MDB_txn *txn, unsigned int flags) {
this->txn = txn;
this->flags = flags;
parent = nullptr;
}
TxnTracked::~TxnTracked() {
this->txn = nullptr;
}
TxnWrap::TxnWrap(const Napi::CallbackInfo& info) : ObjectWrap<TxnWrap>(info) {
EnvWrap *ew;
napi_unwrap(info.Env(), info[0], (void**)&ew);
if (ew->env == nullptr) throwError(info.Env(), "Attempt to start a transaction on a database environment that is closed");
int flags = 0;
TxnWrap *parentTw;
if (info[1].IsBoolean() && ew->writeWorker) { // this is from a transaction callback
txn = ew->writeWorker->AcquireTxn(&flags);
parentTw = nullptr;
} else {
if (info[1].IsObject()) {
Object options = info[1].As<Object>();
// Get flags from options
setFlagFromValue(&flags, MDB_RDONLY, "readOnly", false, options);
} else if (info[1].IsNumber()) {
flags = info[1].As<Number>();
}
MDB_txn *parentTxn;
if (info[2].IsObject()) {
napi_unwrap(info.Env(), info[2], (void**) &parentTw);
parentTxn = parentTw->txn;
} else {
parentTxn = nullptr;
parentTw = nullptr;
// Check existence of current write transaction
if (0 == (flags & MDB_RDONLY)) {
if (ew->currentWriteTxn != nullptr) {
throwError(info.Env(), "You have already opened a write transaction in the current process, can't open a second one.");
return;
}
//fprintf(stderr, "begin sync txn");
auto writeWorker = ew->writeWorker;
if (writeWorker) {
parentTxn = writeWorker->AcquireTxn(&flags); // see if we have a paused transaction
// else we create a child transaction from the current batch transaction. TODO: Except in WRITEMAP mode, where we need to indicate that the transaction should not be committed
}
}
}
//fprintf(stderr, "txn_begin from txn.cpp %u %p\n", flags, parentTxn);
if ((flags & MDB_RDONLY) && parentTxn) {
// if a txn is passed in, we check to see if it is up-to-date and can be reused
MDB_envinfo stat;
mdb_env_info(ew->env, &stat);
if (mdb_txn_id(parentTxn) == stat.me_last_txnid) {
txn = nullptr;
info.This().As<Object>().Set("address", Number::New(info.Env(), 0));
return;
}
parentTw = nullptr;
parentTxn = nullptr;
}
int rc = mdb_txn_begin(ew->env, parentTxn, flags, &txn);
if (rc == MDB_READERS_FULL) { // try again after reader check, in case a dead process frees a slot
int dead;
mdb_reader_check(ew->env, &dead);
ew->consolidateTxns();
rc = mdb_txn_begin(ew->env, parentTxn, flags, &txn);
}
if (rc != 0) {
txn = nullptr;
throwLmdbError(info.Env(), rc);
return;
}
}
// Set the current write transaction
if (0 == (flags & MDB_RDONLY)) {
ew->currentWriteTxn = this;
}
else {
ew->readTxns.push_back(this);
ew->currentReadTxn = txn;
}
this->parentTw = parentTw;
this->flags = flags;
this->ew = ew;
this->env = ew->env;
info.This().As<Object>().Set("address", Number::New(info.Env(), (size_t) this));
}
TxnWrap::~TxnWrap() {
// Close if not closed already
if (this->txn) {
mdb_txn_abort(txn);
this->removeFromEnvWrap();
}
}
void TxnWrap::removeFromEnvWrap() {
if (this->ew) {
if (this->ew->currentWriteTxn == this) {
this->ew->currentWriteTxn = this->parentTw;
}
else {
auto it = std::find(ew->readTxns.begin(), ew->readTxns.end(), this);
if (it != ew->readTxns.end()) {
ew->readTxns.erase(it);
}
}
this->ew = nullptr;
}
this->txn = nullptr;
}
Value TxnWrap::commit(const Napi::CallbackInfo& info) {
// this should only be used for committing read-only txns
if (!this->txn) {
return throwError(info.Env(), "The transaction is already closed.");
}
int rc = mdb_txn_commit(this->txn);
this->removeFromEnvWrap();
if (rc != 0) {
return throwLmdbError(info.Env(), rc);
}
return info.Env().Undefined();
}
Value TxnWrap::abort(const Napi::CallbackInfo& info) {
if (!this->txn) {
return throwError(info.Env(), "The transaction is already closed.");
}
mdb_txn_abort(this->txn);
this->removeFromEnvWrap();
return info.Env().Undefined();
}
NAPI_FUNCTION(resetTxn) {
ARGS(1)
GET_INT64_ARG(0);
TxnWrap* tw = (TxnWrap*) i64;
if (!tw->txn || !tw->getEnv()) {
THROW_ERROR("The transaction is already closed.");
} else {
tw->reset();
RETURN_UNDEFINED;
}
}
void resetTxnFFI(double twPointer) {
TxnWrap* tw = (TxnWrap*) (size_t) twPointer;
if (tw->txn && tw->getEnv()) tw->reset();
}
void TxnWrap::reset() {
ew->readTxnRenewed = false;
mdb_txn_reset(txn);
}
Value TxnWrap::renew(const Napi::CallbackInfo& info) {
if (!this->txn || !this->ew->env) {
return throwError(info.Env(), "The transaction is already closed.");
}
int rc = mdb_txn_renew(this->txn);
if (rc != 0 && rc != EINVAL) { // EINVAL means the txn is already renewed
return throwLmdbError(info.Env(), rc);
}
return info.Env().Undefined();
}
MDB_env* TxnWrap::getEnv() {
return this->ew->env;
}
void TxnWrap::setupExports(Napi::Env env, Object exports) {
// TxnWrap: Prepare constructor template
Function TxnClass = DefineClass(env, "Txn", {
// TxnWrap: Add functions to the prototype
TxnWrap::InstanceMethod("commit", &TxnWrap::commit),
TxnWrap::InstanceMethod("abort", &TxnWrap::abort),
TxnWrap::InstanceMethod("renew", &TxnWrap::renew),
});
exports.Set("Txn", TxnClass);
EXPORT_NAPI_FUNCTION("resetTxn", resetTxn);
EXPORT_FUNCTION_ADDRESS("resetTxnPtr", resetTxnFFI);
//txnTpl->InstanceTemplate()->SetInternalFieldCount(1);
}
// This file contains code from the node-lmdb project
// Copyright (c) 2013-2017 Timur Kristóf
// Copyright (c) 2021 Kristopher Tate
// Licensed to you under the terms of the MIT license
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

170
node_modules/lmdb/src/v8-functions.cpp generated vendored Normal file
View File

@@ -0,0 +1,170 @@
#include "lmdb-js.h"
#if ENABLE_V8_API
#include <string.h>
#include <stdio.h>
#include <v8.h>
#include <node.h>
#if NODE_VERSION_AT_LEAST(16,6,0)
#if NODE_VERSION_AT_LEAST(17,0,0)
#include "../dependencies/v8/v8-fast-api-calls.h"
#else
#include "../dependencies/v8/v8-fast-api-calls-v16.h"
#endif
#endif
using namespace v8;
int32_t getByBinaryFast(Local<v8::Object> instance, double dwPointer, uint32_t keySize, uint32_t ifNotTxnId, int64_t txnAddress) {
DbiWrap* dw = (DbiWrap*) (size_t) dwPointer;
return dw->doGetByBinary(keySize, ifNotTxnId, txnAddress);
}
//class NanWrap : public Nan::ObjectWrap {};
void getByBinaryV8(const FunctionCallbackInfo<v8::Value>& info) {
Isolate* isolate = Isolate::GetCurrent();
auto context = isolate->GetCurrentContext();
DbiWrap* dw = (DbiWrap*) (size_t) info[0]->NumberValue(context).FromJust();
info.GetReturnValue().Set(v8::Number::New(isolate, dw->doGetByBinary(
info[1]->Uint32Value(context).FromJust(),
info[2]->Uint32Value(context).FromJust(),
info[3]->NumberValue(context).FromJust())));
}
int32_t positionFast(Local<v8::Object> instance, double cwPointer, uint32_t flags, uint32_t offset, uint32_t keySize, uint64_t endKeyAddress) {
CursorWrap* cw = (CursorWrap*) (size_t) cwPointer;
DbiWrap* dw = cw->dw;
dw->getFast = true;
cw->flags = flags;
return cw->doPosition(offset, keySize, endKeyAddress);
}
void positionV8(const FunctionCallbackInfo<v8::Value>& info) {
Isolate* isolate = Isolate::GetCurrent();
CursorWrap* cw = (CursorWrap*) (size_t) info[0]->NumberValue(isolate->GetCurrentContext()).FromJust();
cw->flags = info[1]->Uint32Value(isolate->GetCurrentContext()).FromJust();
uint32_t offset = info[2]->Uint32Value(isolate->GetCurrentContext()).FromJust();
uint32_t keySize = info[3]->Uint32Value(isolate->GetCurrentContext()).FromJust();
uint64_t endKeyAddress = info[4]->IntegerValue(isolate->GetCurrentContext()).FromJust();
info.GetReturnValue().Set(v8::Number::New(isolate, cw->doPosition(offset, keySize, endKeyAddress)));
}
int32_t iterateFast(Local<v8::Object> instance, double cwPointer) {
CursorWrap* cw = (CursorWrap*) (size_t) cwPointer;
DbiWrap* dw = cw->dw;
dw->getFast = true;
MDB_val key, data;
int rc = mdb_cursor_get(cw->cursor, &key, &data, cw->iteratingOp);
return cw->returnEntry(rc, key, data);
}
void iterateV8(const FunctionCallbackInfo<v8::Value>& info) {
Isolate* isolate = Isolate::GetCurrent();
CursorWrap* cw = (CursorWrap*) (size_t) info[0]->NumberValue(isolate->GetCurrentContext()).FromJust();
DbiWrap* dw = cw->dw;
dw->getFast = true;
MDB_val key, data;
int rc = mdb_cursor_get(cw->cursor, &key, &data, cw->iteratingOp);
info.GetReturnValue().Set(v8::Number::New(isolate, cw->returnEntry(rc, key, data)));
}
int32_t writeFast(Local<v8::Object> instance, double ewPointer, uint64_t instructionAddress) {
EnvWrap* ew = (EnvWrap*) (size_t) ewPointer;
int rc;
if (instructionAddress)
rc = WriteWorker::DoWrites(ew->writeTxn->txn, ew, (uint32_t*)instructionAddress, nullptr);
else {
pthread_cond_signal(ew->writingCond);
rc = 0;
}
return rc;
// if (rc && !(rc == MDB_KEYEXIST || rc == MDB_NOTFOUND))
// options.fallback = true;
}
void writeV8(const v8::FunctionCallbackInfo<v8::Value>& info) {
Isolate* isolate = Isolate::GetCurrent();
EnvWrap* ew = (EnvWrap*) (size_t) info[0]->NumberValue(isolate->GetCurrentContext()).FromJust();
uint64_t instructionAddress = info[1]->IntegerValue(isolate->GetCurrentContext()).FromJust();
int rc;
if (instructionAddress)
rc = WriteWorker::DoWrites(ew->writeTxn->txn, ew, (uint32_t*)instructionAddress, nullptr);
else {
pthread_cond_signal(ew->writingCond);
rc = 0;
}
info.GetReturnValue().Set(v8::Number::New(isolate, rc));
}
void resetTxnFast(Local<v8::Object> instance, double twPointer) {
TxnWrap* tw = (TxnWrap*) (size_t) twPointer;
tw->reset();
}
void resetTxnV8(const FunctionCallbackInfo<v8::Value>& info) {
Isolate* isolate = Isolate::GetCurrent();
TxnWrap* tw = (TxnWrap*) (size_t) info[0]->NumberValue(isolate->GetCurrentContext()).FromJust();
tw->reset();
}
void noopFast(Local<v8::Object> instance) {
}
void noopV8(const FunctionCallbackInfo<v8::Value>& info) {
}
void clearKeptObjects(const FunctionCallbackInfo<v8::Value>& info) {
#if NODE_VERSION_AT_LEAST(14,0,0)
v8::Isolate::GetCurrent()->ClearKeptObjects();
#endif
}
void detachBuffer(const FunctionCallbackInfo<v8::Value>& info) {
#if NODE_VERSION_AT_LEAST(12,0,0)
v8::Local<v8::ArrayBuffer> v8Buffer = Local<v8::ArrayBuffer>::Cast(info[0]);
v8Buffer->Detach();
#endif
}
#endif
#define EXPORT_FAST(exportName, slowName, fastName) {\
auto fast = CFunction::Make(fastName);\
exports->Set(isolate->GetCurrentContext(), v8::String::NewFromUtf8(isolate, exportName, NewStringType::kInternalized).ToLocalChecked(), FunctionTemplate::New(\
isolate, slowName, Local<v8::Value>(),\
Local<Signature>(), 0, ConstructorBehavior::kThrow,\
SideEffectType::kHasNoSideEffect, &fast)->GetFunction(isolate->GetCurrentContext()).ToLocalChecked());\
}
#define EXPORT_FUNCTION(exportName, funcName) \
exports->Set(isolate->GetCurrentContext(), v8::String::NewFromUtf8(isolate, exportName, NewStringType::kInternalized).ToLocalChecked(), FunctionTemplate::New(\
isolate, funcName, Local<v8::Value>(),\
Local<Signature>(), 0, ConstructorBehavior::kThrow,\
SideEffectType::kHasNoSideEffect)->GetFunction(isolate->GetCurrentContext()).ToLocalChecked());
NAPI_FUNCTION(enableDirectV8) {
ARGS(2)
#if ENABLE_V8_API
Isolate* isolate = Isolate::GetCurrent();
napi_value exportsValue = args[0];
Local<v8::Object> exports;
memcpy((void*) &exports, (void*) &exportsValue, sizeof(exportsValue));
#if NODE_VERSION_AT_LEAST(16,6,1)
bool useFastApi;
napi_get_value_bool(env, args[1], &useFastApi);
if (useFastApi) {
EXPORT_FAST("getByBinary", getByBinaryV8, getByBinaryFast);
EXPORT_FAST("position", positionV8, positionFast);
EXPORT_FAST("iterate", iterateV8, iterateFast);
EXPORT_FAST("write", writeV8, writeFast);
EXPORT_FAST("resetTxn", resetTxnV8, resetTxnFast);
EXPORT_FAST("noop", noopV8, noopFast);
} else {
#endif
EXPORT_FUNCTION("getByBinary", getByBinaryV8);
EXPORT_FUNCTION("position", positionV8);
EXPORT_FUNCTION("iterate", iterateV8);
EXPORT_FUNCTION("write", writeV8);
EXPORT_FUNCTION("noop", noopV8);
#if NODE_VERSION_AT_LEAST(16,6,1)
}
#endif
EXPORT_FUNCTION("clearKeptObjects", clearKeptObjects);
EXPORT_FUNCTION("detachBuffer", detachBuffer);
#endif
RETURN_UNDEFINED
}

35
node_modules/lmdb/src/windows.c generated vendored Normal file
View File

@@ -0,0 +1,35 @@
#ifdef _WIN32
#include <windows.h>
#include <synchapi.h>
static int initializeMemoryPriority = 1;
static MEMORY_PRIORITY_INFORMATION lowMemPriority;
static MEMORY_PRIORITY_INFORMATION normalMemPriority;
int lowerMemoryPriority(int priority) {
if (initializeMemoryPriority) {
GetThreadInformation(GetCurrentThread(), ThreadMemoryPriority, &normalMemPriority, sizeof(normalMemPriority));
// fprintf(stderr, "initialized memory %u setting to %u\n", normalMemPriority.MemoryPriority, priority);
ZeroMemory(&lowMemPriority, sizeof(lowMemPriority));
lowMemPriority.MemoryPriority = priority;
initializeMemoryPriority = 0;
}
void* instruction;
void* pointer;
WaitOnAddress(instruction, pointer, 8, INFINITE);
return SetThreadInformation(GetCurrentThread(), ThreadMemoryPriority, &lowMemPriority, sizeof(lowMemPriority));
}
int setProcessMemoryPriority(int priority) {
if (initializeMemoryPriority) {
GetThreadInformation(GetCurrentThread(), ThreadMemoryPriority, &normalMemPriority, sizeof(normalMemPriority));
// fprintf(stderr, "initialized memory %u setting to %u\n", normalMemPriority.MemoryPriority, priority);
ZeroMemory(&lowMemPriority, sizeof(lowMemPriority));
lowMemPriority.MemoryPriority = priority;
initializeMemoryPriority = 0;
}
return SetProcessInformation(GetCurrentProcess(), ProcessMemoryPriority, &lowMemPriority, sizeof(lowMemPriority));
}
int restoreMemoryPriority() {
return SetThreadInformation(GetCurrentThread(), ThreadMemoryPriority, &normalMemPriority, sizeof(normalMemPriority));
}
#endif

641
node_modules/lmdb/src/writer.cpp generated vendored Normal file
View File

@@ -0,0 +1,641 @@
/* write instructions
0-3 flags
4-7 dbi
8-11 key-size
12 ... key followed by at least 2 32-bit zeros
4 value-size
8 bytes: value pointer (or value itself)
8 compressor pointer?
8 bytes (optional): conditional version
8 bytes (optional): version
inline value?
*/
#include "lmdb-js.h"
#include <atomic>
#include <ctime>
#ifndef _WIN32
#include <unistd.h>
#endif
#ifdef _WIN32
#define ntohl _byteswap_ulong
#define htonl _byteswap_ulong
#else
#include <arpa/inet.h>
#endif
// flags:
const uint32_t NO_INSTRUCTION_YET = 0;
const int PUT = 15;
const int DEL = 13;
const int DEL_VALUE = 14;
const int START_CONDITION_BLOCK = 4;
//const int START_CONDITION_VALUE_BLOCK = 6;
const int START_BLOCK = 1;
const int BLOCK_END = 2;
const int POINTER_NEXT = 3;
const int USER_CALLBACK = 8;
const int USER_CALLBACK_STRICT_ORDER = 0x100000;
const int DROP_DB = 12;
const int HAS_KEY = 4;
const int HAS_VALUE = 2;
const int CONDITIONAL = 8;
const int CONDITIONAL_VERSION = 0x100;
const int CONDITIONAL_VERSION_LESS_THAN = 0x800;
const int CONDITIONAL_ALLOW_NOTFOUND = 0x1000;
const int ASSIGN_TIMESTAMP = 0x2000;
const int SET_VERSION = 0x200;
//const int HAS_INLINE_VALUE = 0x400;
const int COMPRESSIBLE = 0x100000;
const int DELETE_DATABASE = 0x400;
const int TXN_HAD_ERROR = 0x40000000;
const int TXN_DELIMITER = 0x8000000;
const int TXN_COMMITTED = 0x10000000;
//const int TXN_FLUSHED = 0x20000000;
const int WAITING_OPERATION = 0x2000000;
const int IF_NO_EXISTS = MDB_NOOVERWRITE; //0x10;
// result codes:
const int FAILED_CONDITION = 0x4000000;
const int FINISHED_OPERATION = 0x1000000;
const double ANY_VERSION = 3.542694326329068e-103; // special marker for any version
WriteWorker::~WriteWorker() {
// TODO: Make sure this runs on the JS main thread, or we need to move it
if (envForTxn->writeWorker == this)
envForTxn->writeWorker = nullptr;
}
WriteWorker::WriteWorker(MDB_env* env, EnvWrap* envForTxn, uint32_t* instructions)
: envForTxn(envForTxn),
instructions(instructions),
env(env) {
//fprintf(stdout, "nextCompressibleArg %p\n", nextCompressibleArg);
interruptionStatus = 0;
resultCode = 0;
txn = nullptr;
}
void WriteWorker::SendUpdate() {
if (WriteWorker::threadSafeCallsEnabled)
napi_call_threadsafe_function(progress, nullptr, napi_tsfn_blocking);
}
MDB_txn* WriteWorker::AcquireTxn(int* flags) {
bool commitSynchronously = *flags & TXN_SYNCHRONOUS_COMMIT;
// TODO: if the conditionDepth is 0, we could allow the current worker's txn to be continued, committed and restarted
pthread_mutex_lock(envForTxn->writingLock);
retry:
if (commitSynchronously && interruptionStatus == WORKER_WAITING) {
interruptionStatus = INTERRUPT_BATCH;
pthread_cond_signal(envForTxn->writingCond);
pthread_cond_wait(envForTxn->writingCond, envForTxn->writingLock);
if (interruptionStatus == RESTART_WORKER_TXN) {
*flags |= TXN_FROM_WORKER;
return nullptr;
} else if (interruptionStatus == WORKER_WAITING || interruptionStatus == INTERRUPT_BATCH) {
interruptionStatus = WORKER_WAITING;
goto retry;
} else {
return nullptr;
}
} else {
//if (interruptionStatus == RESTART_WORKER_TXN)
// pthread_cond_wait(envForTxn->writingCond, envForTxn->writingLock);
interruptionStatus = USER_HAS_LOCK;
*flags |= TXN_FROM_WORKER;
//if (txn)
//fprintf(stderr, "acquire lock from worker %p %u\n", txn, commitSynchronously);
return txn;
}
}
void WriteWorker::UnlockTxn() {
interruptionStatus = 0;
pthread_cond_signal(envForTxn->writingCond);
pthread_mutex_unlock(envForTxn->writingLock);
}
int WriteWorker::WaitForCallbacks(MDB_txn** txn, bool allowCommit, uint32_t* target) {
int rc;
if (!finishedProgress)
SendUpdate();
pthread_cond_signal(envForTxn->writingCond);
interruptionStatus = WORKER_WAITING;
uint64_t start;
unsigned int envFlags;
mdb_env_get_flags(env, &envFlags);
#ifdef MDB_TRACK_METRICS
if (envFlags & MDB_TRACK_METRICS)
start = get_time64();
#endif
if (target) {
uint64_t delay = 1;
do {
cond_timedwait(envForTxn->writingCond, envForTxn->writingLock, delay);
delay = delay << 1ll;
if ((*target & 0xf) || (allowCommit && finishedProgress)) {
// we are in position to continue writing or commit, so forward progress can be made without interrupting yet
#ifdef MDB_TRACK_METRICS
if (envFlags & MDB_TRACK_METRICS)
envForTxn->timeTxnWaiting += get_time64() - start;
#endif
interruptionStatus = 0;
return 0;
}
} while(interruptionStatus != INTERRUPT_BATCH);
} else {
pthread_cond_wait(envForTxn->writingCond, envForTxn->writingLock);
}
#ifdef MDB_TRACK_METRICS
if (envFlags & MDB_TRACK_METRICS)
envForTxn->timeTxnWaiting += get_time64() - start;
#endif
if (interruptionStatus == INTERRUPT_BATCH) { // interrupted by JS code that wants to run a synchronous transaction
interruptionStatus = RESTART_WORKER_TXN;
rc = mdb_txn_commit(*txn);
#ifdef MDB_EMPTY_TXN
if (rc == MDB_EMPTY_TXN)
rc = 0;
#endif
if (rc == 0) {
// wait again until the sync transaction is completed
this->txn = *txn = nullptr;
pthread_cond_signal(envForTxn->writingCond);
pthread_cond_wait(envForTxn->writingCond, envForTxn->writingLock);
// now restart our transaction
rc = mdb_txn_begin(env, nullptr, 0, txn);
this->txn = *txn;
//fprintf(stderr, "Restarted txn after interruption\n");
interruptionStatus = 0;
}
if (rc != 0) {
fprintf(stdout, "wfc unlock due to error %u\n", rc);
return rc;
}
} else
interruptionStatus = 0;
return 0;
}
int WriteWorker::DoWrites(MDB_txn* txn, EnvWrap* envForTxn, uint32_t* instruction, WriteWorker* worker) {
MDB_val key, value;
int rc = 0;
int conditionDepth = 0;
int validatedDepth = 0;
double conditionalVersion, setVersion = 0;
bool overlappedWord = !!worker;
uint32_t* start;
do {
next_inst: start = instruction++;
uint32_t flags = *start;
MDB_dbi dbi = 0;
//fprintf(stderr, "do %u %u\n", flags, get_time64());
bool validated = conditionDepth == validatedDepth;
if (flags & 0xc0c0) {
fprintf(stderr, "Unknown flag bits %u %p\n", flags, start);
fprintf(stderr, "flags after message %u\n", *start);
worker->resultCode = 22;
abort();
}
if (flags & HAS_KEY) {
// a key based instruction, get the key
dbi = (MDB_dbi) *instruction++;
key.mv_size = *instruction++;
key.mv_data = instruction;
instruction = (uint32_t*) (((size_t) instruction + key.mv_size + 16) & (~7));
if (flags & HAS_VALUE) {
if (flags & COMPRESSIBLE) {
int64_t status = -1;
status = std::atomic_exchange((std::atomic<int64_t>*)(instruction + 2), (int64_t)1);
if (status == 2) {
//fprintf(stderr, "wait on compression %p\n", instruction);
worker->interruptionStatus = WORKER_WAITING;
do {
pthread_cond_wait(envForTxn->writingCond, envForTxn->writingLock);
} while (std::atomic_load((std::atomic<int64_t>*)(instruction + 2)));
worker->interruptionStatus = 0;
} else if (status > 2) {
//fprintf(stderr, "doing the compression ourselves\n");
((Compression*) (size_t) *((double*)&status))->compressInstruction(nullptr, (double*) (instruction + 2));
} // else status is 0 and compression is done
// compressed
value.mv_data = (void*)(size_t) * ((size_t*)instruction);
if ((size_t)value.mv_data > 0x1000000000000)
fprintf(stderr, "compression not completed %p %i\n", value.mv_data, (int) status);
value.mv_size = *(instruction - 1);
instruction += 4; // skip compression pointers
} else {
value.mv_data = (void*)(size_t) * ((double*)instruction);
value.mv_size = *(instruction - 1);
instruction += 2;
}
}
if (flags & CONDITIONAL_VERSION) {
conditionalVersion = *((double*) instruction);
instruction += 2;
MDB_val conditionalValue;
rc = mdb_get(txn, dbi, &key, &conditionalValue);
if (rc) {
// not found counts as version 0, so this is acceptable for conditional less than,
// otherwise does not validate
if (rc == MDB_NOTFOUND)
validated = flags & CONDITIONAL_ALLOW_NOTFOUND;
else
worker->resultCode = rc;
} else if (conditionalVersion != ANY_VERSION) {
double version;
memcpy(&version, conditionalValue.mv_data, 8);
validated = validated && ((flags & CONDITIONAL_VERSION_LESS_THAN) ? version <= conditionalVersion : (version == conditionalVersion));
}
}
if (flags & SET_VERSION) {
setVersion = *((double*) instruction);
instruction += 2;
}
if ((flags & IF_NO_EXISTS) && (flags & START_CONDITION_BLOCK)) {
rc = mdb_get(txn, dbi, &key, &value);
if (!rc)
validated = false;
else if (rc == MDB_NOTFOUND)
validated = true;
else
worker->resultCode = rc;
}
} else
instruction++;
//fprintf(stderr, "instr flags %p %p %u\n", start, flags, conditionDepth);
if (validated || !(flags & CONDITIONAL)) {
switch (flags & 0xf) {
case NO_INSTRUCTION_YET:
instruction -= 2; // reset back to the previous flag as the current instruction
rc = 0;
// in windows InterlockedCompareExchange might be faster
if (!worker->finishedProgress || conditionDepth) {
if (std::atomic_compare_exchange_strong((std::atomic<uint32_t>*) start,
(uint32_t*) &flags,
(uint32_t)WAITING_OPERATION)) {
worker->WaitForCallbacks(&txn, conditionDepth == 0, start);
}
goto next_inst;
} else {
if (std::atomic_compare_exchange_strong((std::atomic<uint32_t>*) start,
(uint32_t*) &flags,
(uint32_t)TXN_DELIMITER)) {
worker->instructions = start;
return 0;
} else
goto next_inst;
}
case BLOCK_END:
conditionDepth--;
if (validatedDepth > conditionDepth)
validatedDepth--;
if (conditionDepth < 0) {
fprintf(stderr, "Negative condition depth");
}
goto next_inst;
case PUT:
#ifdef MDB_OVERLAPPINGSYNC
if (flags & ASSIGN_TIMESTAMP) {
if ((*(uint64_t*)key.mv_data & 0xfffffffful) == REPLACE_WITH_TIMESTAMP) {
ExtendedEnv* extended_env = (ExtendedEnv*) mdb_env_get_userctx(envForTxn->env);
*(uint64_t*)key.mv_data = ((*(uint64_t*)key.mv_data >> 32) & 0x1) ?
extended_env->getLastTime() : extended_env->getNextTime();
}
uint64_t first_word = *(uint64_t*)value.mv_data;
// 0 assign new time
// 1 assign last assigned time
// 3 assign last recorded previous time
// 4 record previous time
if ((first_word & 0xffffff) == SPECIAL_WRITE) {
if (first_word & REPLACE_WITH_TIMESTAMP_FLAG) {
ExtendedEnv* extended_env = (ExtendedEnv*) mdb_env_get_userctx(envForTxn->env);
uint32_t next_32 = first_word >> 32;
if (next_32 & 4) {
// preserve last timestamp
MDB_val last_data;
rc = mdb_get(txn, dbi, &key, &last_data);
if (rc) break;
if (flags & SET_VERSION) last_data.mv_data = (char *) last_data.mv_data + 8;
extended_env->previousTime = *(uint64_t *) last_data.mv_data;
//fprintf(stderr, "previous time %llx \n", previous_time);
}
uint64_t timestamp = (next_32 & 1) ? (next_32 & 2) ? extended_env->previousTime : extended_env->getLastTime()
: extended_env->getNextTime();
if (first_word & DIRECT_WRITE) {
// write to second word, which is used by the direct write
*((uint64_t *) value.mv_data + 1) = timestamp ^ (next_32 >> 8);
first_word = first_word & 0xffffffff; // clear out the offset so it is just zero (always must be at the beginning)
} else
*(uint64_t *) value.mv_data = timestamp ^ (next_32 >> 8);
//fprintf(stderr, "set time %llx \n", timestamp);
}
if (first_word & DIRECT_WRITE) {
// direct in-place write
unsigned int offset = first_word >> 32;
if (flags & SET_VERSION)
offset += 8;
MDB_val bytes_to_write;
bytes_to_write.mv_data = (char*)value.mv_data + 8;
bytes_to_write.mv_size = value.mv_size - 8;
#ifdef MDB_RPAGE_CACHE
rc = mdb_direct_write(txn, dbi, &key, offset, &bytes_to_write);
if (!rc) break; // success
#endif
// if no success, this means we probably weren't able to write to a single
// word safely, so we need to do a real put
MDB_val last_data;
rc = mdb_get(txn, dbi, &key, &last_data);
if (rc) break; // failed to get
bytes_to_write.mv_size = last_data.mv_size;
// attempt a put, using reserve (so we can efficiently copy data in)
rc = mdb_put(txn, dbi, &key, &bytes_to_write, (flags & (MDB_NOOVERWRITE | MDB_NODUPDATA | MDB_APPEND | MDB_APPENDDUP)) | MDB_RESERVE);
if (!rc) {
// copy the existing data
memcpy(bytes_to_write.mv_data, last_data.mv_data, last_data.mv_size);
// copy the changes
memcpy((char*)bytes_to_write.mv_data + offset, (char*)value.mv_data + 8, value.mv_size - 8);
}
break; // done
}
}
}
#endif
if (flags & SET_VERSION)
rc = putWithVersion(txn, dbi, &key, &value, flags & (MDB_NOOVERWRITE | MDB_NODUPDATA | MDB_APPEND | MDB_APPENDDUP), setVersion);
else
rc = mdb_put(txn, dbi, &key, &value, flags & (MDB_NOOVERWRITE | MDB_NODUPDATA | MDB_APPEND | MDB_APPENDDUP));
if (flags & COMPRESSIBLE)
delete value.mv_data;
break;
case DEL:
rc = mdb_del(txn, dbi, &key, nullptr);
break;
case DEL_VALUE:
rc = mdb_del(txn, dbi, &key, &value);
if (flags & COMPRESSIBLE)
delete value.mv_data;
break;
case START_BLOCK: case START_CONDITION_BLOCK:
rc = validated ? 0 : MDB_NOTFOUND;
if (validated)
validatedDepth++;
conditionDepth++;
break;
case USER_CALLBACK:
worker->finishedProgress = false;
worker->progressStatus = 2;
rc = 0;
if (flags & USER_CALLBACK_STRICT_ORDER) {
std::atomic_fetch_or((std::atomic<uint32_t>*) start, (uint32_t) FINISHED_OPERATION); // mark it as finished so it is processed
while (!worker->finishedProgress) {
worker->WaitForCallbacks(&txn, conditionDepth == 0, nullptr);
}
}
break;
case DROP_DB:
rc = mdb_drop(txn, dbi, (flags & DELETE_DATABASE) ? 1 : 0);
break;
case POINTER_NEXT:
instruction = (uint32_t*)(size_t) * ((double*)instruction);
goto next_inst;
default:
fprintf(stderr, "Unknown flags %u %p\n", flags, start);
fprintf(stderr, "flags after message %u\n", *start);
worker->resultCode = 22;
abort();
}
if (rc) {
if (!(rc == MDB_KEYEXIST || rc == MDB_NOTFOUND)) {
if (worker) {
worker->resultCode = rc;
} else {
return rc;
}
}
flags = FINISHED_OPERATION | FAILED_CONDITION;
}
else
flags = FINISHED_OPERATION;
} else
flags = FINISHED_OPERATION | FAILED_CONDITION;
//fprintf(stderr, "finished flag %p\n", flags);
if (overlappedWord) {
std::atomic_fetch_or((std::atomic<uint32_t>*) start, flags);
overlappedWord = false;
} else
*start |= flags;
} while(worker); // keep iterating in async/multiple-instruction mode, just one instruction in sync mode
return rc;
}
bool WriteWorker::threadSafeCallsEnabled = false;
void txn_callback(const void* data, int finished) {
auto worker = (WriteWorker*) data;
if (finished) {
// we don't want to release our lock until *after* the txn lock is released to give other threads a better chance
// at executing next
if (worker->txn) {
if (!(*worker->instructions & TXN_DELIMITER))
fprintf(stderr, "in txn_callback not valid %p\n", *worker->instructions);
worker->txn = nullptr;
worker->interruptionStatus = 0;
pthread_cond_signal(worker->envForTxn->writingCond); // in case there a sync txn waiting for us
pthread_mutex_unlock(worker->envForTxn->writingLock);
}
} else // transaction is visible (to readers), but not unlocked
worker->SendUpdate();
}
void do_write(napi_env env, void* data) {
auto worker = (WriteWorker*) data;
worker->Write();
napi_release_threadsafe_function(worker->progress, napi_tsfn_abort);
}
const int READER_CHECK_INTERVAL = 600; // ten minutes
void WriteWorker::Write() {
int rc;
finishedProgress = true;
unsigned int envFlags;
mdb_env_get_flags(env, &envFlags);
time_t now = time(0);
if (now - envForTxn->lastReaderCheck > READER_CHECK_INTERVAL) {
int dead;
mdb_reader_check(env, &dead);
envForTxn->lastReaderCheck = now;
}
pthread_mutex_lock(envForTxn->writingLock);
if (!env) return;// already closed
#ifndef _WIN32
int retries = 0;
retry:
#endif
rc = mdb_txn_begin(env, nullptr,
#ifdef MDB_OVERLAPPINGSYNC
(envForTxn->jsFlags & MDB_OVERLAPPINGSYNC) ? MDB_NOSYNC :
#endif
0, &txn);
#if !defined(_WIN32) && defined(MDB_RPAGE_CACHE)
if (rc == MDB_LOCK_FAILURE) {
if (retries++ < 4) {
sleep(1);
goto retry;
}
}
#endif
if (rc != 0) {
resultCode = rc;
return;
}
uint32_t* start = instructions;
rc = DoWrites(txn, envForTxn, instructions, this);
uint32_t txnId = (uint32_t) mdb_txn_id(txn);
if (!(*instructions & TXN_DELIMITER))
fprintf(stderr, "after writes %p %p NOT still valid %p\n", start, instructions, *instructions);
progressStatus = 1;
#ifdef MDB_OVERLAPPINGSYNC
if (envForTxn->jsFlags & MDB_OVERLAPPINGSYNC) {
mdb_txn_set_callback(txn, txn_callback, this);
}
#endif
bool had_changes = false;
if (rc || resultCode) {
fprintf(stderr, "do_write error %u %u\n", rc, resultCode);
mdb_txn_abort(txn);
} else {
rc = mdb_txn_commit(txn);
#ifdef MDB_EMPTY_TXN
if (rc == MDB_EMPTY_TXN)
rc = 0;
else {
had_changes = true;
}
#else
had_changes = true;
#endif
}
if (!(*instructions & TXN_DELIMITER))
fprintf(stderr, "end write %p, next start %p NOT still valid %p\n", start, instructions, *instructions);
txn_callback(this, 1);
if (rc || resultCode) {
std::atomic_fetch_or((std::atomic<uint32_t>*) instructions, (uint32_t) TXN_HAD_ERROR);
if (rc)
resultCode = rc ? rc : resultCode;
return;
}
*(instructions - 1) = txnId;
std::atomic_fetch_or((std::atomic<uint32_t>*) instructions, (uint32_t) TXN_COMMITTED);
if (had_changes) {
ExtendedEnv* extended_env = (ExtendedEnv*) mdb_env_get_userctx(env);
if (extended_env) extended_env->notifyUserCallbacks(std::string("__committed__"));
}
}
void write_progress(napi_env env,
napi_value js_callback,
void* context,
void* data) {
if (!js_callback)
return;
auto worker = (WriteWorker*) context;
napi_value result;
napi_value undefined;
napi_value arg;
napi_create_int32(env, worker->progressStatus, &arg);
napi_get_undefined(env, &undefined);
if (worker->progressStatus == 1) {
napi_call_function(env, undefined, js_callback, 1, &arg, &result);
return;
}
if (worker->finishedProgress)
return;
auto envForTxn = worker->envForTxn;
pthread_mutex_lock(envForTxn->writingLock);
while(!worker->txn) // possible to jump in after an interrupted txn here
pthread_cond_wait(envForTxn->writingCond, envForTxn->writingLock);
envForTxn->writeTxn = new TxnTracked(worker->txn, 0);
worker->finishedProgress = true;
napi_create_int32(env, worker->progressStatus, &arg);
napi_call_function(env, undefined, js_callback, 1, &arg, &result);
bool is_async = false;
napi_get_value_bool(env, result, &is_async);
if (!is_async) {
delete envForTxn->writeTxn;
envForTxn->writeTxn = nullptr;
pthread_cond_signal(envForTxn->writingCond);
pthread_mutex_unlock(envForTxn->writingLock);
}
}
void writes_complete(napi_env env,
napi_status status,
void* data) {
auto worker = (WriteWorker*) data;
worker->finishedProgress = true;
napi_value result, arg; // we use direct napi call here because node-addon-api interface with throw a fatal error if a worker thread is terminating, and bun doesn't support escapable scopes yet
napi_create_int32(env, worker->resultCode, &arg);
napi_value callback;
napi_get_reference_value(env, worker->callback, &callback);
napi_call_function(env, callback, callback, 1, &arg, &result);
napi_delete_reference(env, worker->callback);
napi_delete_async_work(env, worker->work);
delete worker;
}
Value EnvWrap::resumeWriting(const Napi::CallbackInfo& info) {
// if we had async txns, now we resume
delete writeTxn;
writeTxn = nullptr;
pthread_cond_signal(writingCond);
pthread_mutex_unlock(writingLock);
return info.Env().Undefined();
}
Value EnvWrap::startWriting(const Napi::CallbackInfo& info) {
napi_env n_env = info.Env();
if (!this->env) {
return throwError(info.Env(), "The environment is already closed.");
}
hasWrites = true;
size_t instructionAddress = info[0].As<Number>().Int64Value();
napi_value resource;
napi_status status;
status = napi_create_object(n_env, &resource);
napi_value resource_name;
status = napi_create_string_latin1(n_env, "write", NAPI_AUTO_LENGTH, &resource_name);
auto worker = new WriteWorker(this->env, this, (uint32_t*) instructionAddress);
this->writeWorker = worker;
napi_create_reference(n_env, info[1].As<Function>(), 1, &worker->callback);
status = napi_create_async_work(n_env, resource, resource_name, do_write, writes_complete, worker, &worker->work);
if (status != napi_ok) abort();
napi_create_threadsafe_function(n_env, info[1].As<Function>(), resource, resource_name, 0, 1, nullptr, nullptr, worker, write_progress, &worker->progress);
status = napi_queue_async_work(n_env, worker->work);
if (status != napi_ok) abort();
return info.Env().Undefined();
}
NAPI_FUNCTION(EnvWrap::write) {
ARGS(2)
GET_INT64_ARG(0);
EnvWrap* ew = (EnvWrap*) i64;
if (!ew->env) {
napi_throw_error(env, nullptr, "The environment is already closed.");
RETURN_UNDEFINED;
}
ew->hasWrites = true;
napi_get_value_int64(env, args[1], &i64);
uint32_t* instructionAddress = (uint32_t*) i64;
int rc = 0;
if (instructionAddress)
rc = WriteWorker::DoWrites(ew->writeTxn->txn, ew, instructionAddress, nullptr);
else if (ew->writeWorker) {
pthread_cond_signal(ew->writingCond);
}
if (rc && !(rc == MDB_KEYEXIST || rc == MDB_NOTFOUND)) {
throwLmdbError(env, rc);
RETURN_UNDEFINED;
}
RETURN_UNDEFINED;
}