From e4cf6e98e52b570baf77af52f253b8a97ae70f4d Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Sat, 4 Feb 2023 11:28:23 +1100 Subject: [PATCH 01/21] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8830cb89..20b02b04 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ org.lmdbjava lmdbjava - 0.8.3 + 0.9.0-SNAPSHOT jar LmdbJava Low latency Java API for the ultra-fast, embedded Symas Lightning Database (LMDB) @@ -206,7 +206,7 @@ scm:git:git@github.com:${github.org}/${github.repo}.git scm:git:git@github.com:${github.org}/${github.repo}.git git@github.com:${github.org}/${github.repo}.git - lmdbjava-0.8.3 + HEAD GitHub Issues From 3524995b97c86e0cd66c64ce648693cad9feaa0c Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Sat, 4 Feb 2023 15:23:49 +1100 Subject: [PATCH 02/21] Refactor Comparator handling (fixes #199) --- src/main/java/org/lmdbjava/BufferProxy.java | 15 ++-- .../java/org/lmdbjava/ByteArrayProxy.java | 7 +- src/main/java/org/lmdbjava/ByteBufProxy.java | 7 +- .../java/org/lmdbjava/ByteBufferProxy.java | 20 ++++- src/main/java/org/lmdbjava/Dbi.java | 54 +++--------- .../java/org/lmdbjava/DirectBufferProxy.java | 7 +- src/main/java/org/lmdbjava/Env.java | 88 ++++++++++++++----- src/main/java/org/lmdbjava/Txn.java | 15 ---- .../java/org/lmdbjava/ComparatorTest.java | 18 ++-- .../java/org/lmdbjava/CursorIterableTest.java | 22 +++-- src/test/java/org/lmdbjava/DbiTest.java | 14 +-- 11 files changed, 157 insertions(+), 110 deletions(-) diff --git a/src/main/java/org/lmdbjava/BufferProxy.java b/src/main/java/org/lmdbjava/BufferProxy.java index bd58d7b6..aff732bc 100644 --- a/src/main/java/org/lmdbjava/BufferProxy.java +++ b/src/main/java/org/lmdbjava/BufferProxy.java @@ -22,6 +22,8 @@ import static java.lang.Long.BYTES; +import java.util.Comparator; + import jnr.ffi.Pointer; /** @@ -61,17 +63,16 @@ public abstract class BufferProxy { protected abstract T allocate(); /** - * Compare the two buffers. + * Get a suitable default {@link Comparator} given the provided flags. * *

- * Implemented as a protected method to discourage use of the buffer proxy - * in collections etc (given by design it wraps a temporary value only). + * The provided comparator must strictly match the lexicographical order of + * keys in the native LMDB database. * - * @param o1 left operand - * @param o2 right operand - * @return as per {@link Comparable} + * @param flags for the database + * @return a comparator that can be used (never null) */ - protected abstract int compare(T o1, T o2); + protected abstract Comparator getComparator(DbiFlags... flags); /** * Deallocate a buffer that was previously provided by {@link #allocate()}. diff --git a/src/main/java/org/lmdbjava/ByteArrayProxy.java b/src/main/java/org/lmdbjava/ByteArrayProxy.java index f6ddbef2..3fe8184a 100644 --- a/src/main/java/org/lmdbjava/ByteArrayProxy.java +++ b/src/main/java/org/lmdbjava/ByteArrayProxy.java @@ -24,6 +24,7 @@ import static org.lmdbjava.Library.RUNTIME; import java.util.Arrays; +import java.util.Comparator; import jnr.ffi.Pointer; import jnr.ffi.provider.MemoryManager; @@ -78,7 +79,6 @@ protected byte[] allocate() { return new byte[0]; } - @Override protected int compare(final byte[] o1, final byte[] o2) { return compareArrays(o1, o2); } @@ -93,6 +93,11 @@ protected byte[] getBytes(final byte[] buffer) { return Arrays.copyOf(buffer, buffer.length); } + @Override + protected Comparator getComparator(final DbiFlags... flags) { + return this::compare; + } + @Override protected void in(final byte[] buffer, final Pointer ptr, final long ptrAddr) { diff --git a/src/main/java/org/lmdbjava/ByteBufProxy.java b/src/main/java/org/lmdbjava/ByteBufProxy.java index d8bbc452..26351676 100644 --- a/src/main/java/org/lmdbjava/ByteBufProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufProxy.java @@ -25,6 +25,7 @@ import static org.lmdbjava.UnsafeAccess.UNSAFE; import java.lang.reflect.Field; +import java.util.Comparator; import io.netty.buffer.ByteBuf; import io.netty.buffer.PooledByteBufAllocator; @@ -106,11 +107,15 @@ protected ByteBuf allocate() { throw new IllegalStateException("Netty buffer must be " + NAME); } - @Override protected int compare(final ByteBuf o1, final ByteBuf o2) { return o1.compareTo(o2); } + @Override + protected Comparator getComparator(final DbiFlags... flags) { + return this::compare; + } + @Override protected void deallocate(final ByteBuf buff) { buff.release(); diff --git a/src/main/java/org/lmdbjava/ByteBufferProxy.java b/src/main/java/org/lmdbjava/ByteBufferProxy.java index 0d1781b3..8eb95da8 100644 --- a/src/main/java/org/lmdbjava/ByteBufferProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufferProxy.java @@ -26,13 +26,17 @@ import static java.nio.ByteOrder.BIG_ENDIAN; import static java.nio.ByteOrder.LITTLE_ENDIAN; import static java.util.Objects.requireNonNull; +import static org.lmdbjava.DbiFlags.MDB_INTEGERKEY; import static org.lmdbjava.Env.SHOULD_CHECK; +import static org.lmdbjava.MaskedFlag.isSet; +import static org.lmdbjava.MaskedFlag.mask; import static org.lmdbjava.UnsafeAccess.UNSAFE; import java.lang.reflect.Field; import java.nio.Buffer; import java.nio.ByteBuffer; import java.util.ArrayDeque; +import java.util.Comparator; import jnr.ffi.Pointer; @@ -189,7 +193,21 @@ protected final ByteBuffer allocate() { } @Override - protected final int compare(final ByteBuffer o1, final ByteBuffer o2) { + protected Comparator getComparator(final DbiFlags... flags) { + final int flagInt = mask(flags); + if (isSet(flagInt, MDB_INTEGERKEY)) { + return this::compareCustom; + } + return this::compareDefault; + } + + protected final int compareDefault(final ByteBuffer o1, + final ByteBuffer o2) { + return o1.compareTo(o2); + } + + protected final int compareCustom(final ByteBuffer o1, + final ByteBuffer o2) { return compareBuff(o1, o2); } diff --git a/src/main/java/org/lmdbjava/Dbi.java b/src/main/java/org/lmdbjava/Dbi.java index 3f36bbeb..d0eb0a14 100644 --- a/src/main/java/org/lmdbjava/Dbi.java +++ b/src/main/java/org/lmdbjava/Dbi.java @@ -56,38 +56,35 @@ public final class Dbi { private final ComparatorCallback ccb; private boolean cleaned; - private final Comparator compFunc; + private final Comparator comparator; private final Env env; private final byte[] name; - private final BufferProxy proxy; private final Pointer ptr; Dbi(final Env env, final Txn txn, final byte[] name, - final Comparator comparator, final DbiFlags... flags) { + final Comparator comparator, final boolean nativeCb, + final BufferProxy proxy, final DbiFlags... flags) { this.env = env; this.name = name == null ? null : Arrays.copyOf(name, name.length); + this.comparator = comparator; final int flagsMask = mask(flags); final Pointer dbiPtr = allocateDirect(RUNTIME, ADDRESS); checkRc(LIB.mdb_dbi_open(txn.pointer(), name, flagsMask, dbiPtr)); ptr = dbiPtr.getPointer(0); - if (comparator == null) { - proxy = null; - compFunc = null; - ccb = null; - } else { - this.proxy = txn.getProxy(); - this.compFunc = comparator; + if (nativeCb) { this.ccb = (keyA, keyB) -> { final T compKeyA = proxy.allocate(); final T compKeyB = proxy.allocate(); proxy.out(compKeyA, keyA, keyA.address()); proxy.out(compKeyB, keyB, keyB.address()); - final int result = compFunc.compare(compKeyA, compKeyB); + final int result = this.comparator.compare(compKeyA, compKeyB); proxy.deallocate(compKeyA); proxy.deallocate(compKeyB); return result; }; LIB.mdb_set_compare(txn.pointer(), ptr, ccb); + } else { + ccb = null; } } @@ -265,51 +262,20 @@ public CursorIterable iterate(final Txn txn) { } /** - * Iterate the database in accordance with the provided {@link KeyRange} and - * default {@link Comparator}. + * Iterate the database in accordance with the provided {@link KeyRange}. * * @param txn transaction handle (not null; not committed) * @param range range of acceptable keys (not null) * @return iterator (never null) */ public CursorIterable iterate(final Txn txn, final KeyRange range) { - return iterate(txn, range, null); - } - - /** - * Iterate the database in accordance with the provided {@link KeyRange} and - * {@link Comparator}. - * - *

- * If a comparator is provided, it must reflect the same ordering as LMDB uses - * for cursor operations (eg first, next, last, previous etc). - * - *

- * If a null comparator is provided, any comparator provided when opening the - * database is used. If no database comparator was specified, the buffer's - * default comparator is used. Such buffer comparators reflect LMDB's default - * lexicographical order. - * - * @param txn transaction handle (not null; not committed) - * @param range range of acceptable keys (not null) - * @param comparator custom comparator for keys (may be null) - * @return iterator (never null) - */ - public CursorIterable iterate(final Txn txn, final KeyRange range, - final Comparator comparator) { if (SHOULD_CHECK) { requireNonNull(txn); requireNonNull(range); env.checkNotClosed(); txn.checkReady(); } - final Comparator useComp; - if (comparator == null) { - useComp = compFunc == null ? txn.comparator() : compFunc; - } else { - useComp = comparator; - } - return new CursorIterable<>(txn, this, range, useComp); + return new CursorIterable<>(txn, this, range, comparator); } /* diff --git a/src/main/java/org/lmdbjava/DirectBufferProxy.java b/src/main/java/org/lmdbjava/DirectBufferProxy.java index ac46dec5..62b49095 100644 --- a/src/main/java/org/lmdbjava/DirectBufferProxy.java +++ b/src/main/java/org/lmdbjava/DirectBufferProxy.java @@ -28,6 +28,7 @@ import java.nio.ByteBuffer; import java.util.ArrayDeque; +import java.util.Comparator; import jnr.ffi.Pointer; import org.agrona.DirectBuffer; @@ -111,7 +112,6 @@ protected DirectBuffer allocate() { } } - @Override protected int compare(final DirectBuffer o1, final DirectBuffer o2) { return compareBuff(o1, o2); } @@ -129,6 +129,11 @@ protected byte[] getBytes(final DirectBuffer buffer) { return dest; } + @Override + protected Comparator getComparator(final DbiFlags... flags) { + return this::compare; + } + @Override protected void in(final DirectBuffer buffer, final Pointer ptr, final long ptrAddr) { diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index 45db5064..66ab7cda 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -258,43 +258,79 @@ public boolean isReadOnly() { } /** - * Convenience method that opens a {@link Dbi} with a UTF-8 database name. + * Convenience method that opens a {@link Dbi} with a UTF-8 database name and + * default {@link Comparator} that is not invoked from native code. * * @param name name of the database (or null if no name is required) * @param flags to open the database with * @return a database that is ready to use */ public Dbi openDbi(final String name, final DbiFlags... flags) { - return openDbi(name, null, flags); + final byte[] nameBytes = name == null ? null : name.getBytes(UTF_8); + return openDbi(nameBytes, null, false, flags); } /** - * Convenience method that opens a {@link Dbi} with a UTF-8 database name. + * Convenience method that opens a {@link Dbi} with a UTF-8 database name and + * associated {@link Comparator} that is not invoked from native code. * * @param name name of the database (or null if no name is required) - * @param comparator custom comparator callback (or null to use LMDB default) + * @param comparator custom comparator callback (or null to use default) * @param flags to open the database with * @return a database that is ready to use */ public Dbi openDbi(final String name, final Comparator comparator, final DbiFlags... flags) { final byte[] nameBytes = name == null ? null : name.getBytes(UTF_8); - return openDbi(nameBytes, comparator, flags); + return openDbi(nameBytes, comparator, false, flags); + } + + /** + * Convenience method that opens a {@link Dbi} with a UTF-8 database name and + * associated {@link Comparator} that may be invoked from native code if + * specified. + * + * @param name name of the database (or null if no name is required) + * @param comparator custom comparator callback (or null to use default) + * @param nativeCb whether native code calls back to the Java comparator + * @param flags to open the database with + * @return a database that is ready to use + */ + public Dbi openDbi(final String name, final Comparator comparator, + final boolean nativeCb, final DbiFlags... flags) { + final byte[] nameBytes = name == null ? null : name.getBytes(UTF_8); + return openDbi(nameBytes, comparator, nativeCb, flags); } /** - * Convenience method that opens a {@link Dbi}. + * Convenience method that opens a {@link Dbi} with a default + * {@link Comparator} that is not invoked from native code. * * @param name name of the database (or null if no name is required) * @param flags to open the database with * @return a database that is ready to use */ public Dbi openDbi(final byte[] name, final DbiFlags... flags) { - return openDbi(name, null, flags); + return openDbi(name, null, false, flags); + } + + /** + * Convenience method that opens a {@link Dbi} with an associated + * {@link Comparator} that is not invoked from native code. + * + * @param name name of the database (or null if no name is required) + * @param comparator custom comparator callback (or null to use LMDB default) + * @param flags to open the database with + * @return a database that is ready to use + */ + public Dbi openDbi(final byte[] name, final Comparator comparator, + final DbiFlags... flags) { + return openDbi(name, comparator, false, flags); } /** - * Convenience method that opens a {@link Dbi} inside a private transaction. + * Convenience method that opens a {@link Dbi} with an associated + * {@link Comparator} that may be invoked from native code if specified. * *

* This method will automatically commit the private transaction before @@ -303,13 +339,14 @@ public Dbi openDbi(final byte[] name, final DbiFlags... flags) { * * @param name name of the database (or null if no name is required) * @param comparator custom comparator callback (or null to use LMDB default) + * @param nativeCb whether native code calls back to the Java comparator * @param flags to open the database with * @return a database that is ready to use */ public Dbi openDbi(final byte[] name, final Comparator comparator, - final DbiFlags... flags) { + final boolean nativeCb, final DbiFlags... flags) { try (Txn txn = readOnly ? txnRead() : txnWrite()) { - final Dbi dbi = openDbi(txn, name, comparator, flags); + final Dbi dbi = openDbi(txn, name, comparator, nativeCb, flags); txn.commit(); // even RO Txns require a commit to retain Dbi in Env return dbi; } @@ -323,14 +360,18 @@ public Dbi openDbi(final byte[] name, final Comparator comparator, * to retain the Dbi in the Env. * *

- * If a custom comparator is specified, this comparator is called from LMDB - * any time it needs to compare two keys. The comparator must be used any time - * any time this database is opened, otherwise database corruption may occur. - * The custom comparator will also be used whenever a {@link CursorIterable} - * is created from the returned {@link Dbi}. If a custom comparator is not - * specified, LMDB's native default lexicographical order is used. The default - * comparator is typically more efficient (as there is no need for the native - * library to call back into Java for the comparator result). + * A {@link Comparator} may be provided when calling this method. Such + * comparator is primarily used by {@link CursorIterable} instances. A + * secondary (but uncommon) use of the comparator is to act as a callback from + * the native library if nativeCb is true. This is + * usually avoided due to the overhead of native code calling back into Java. + * It is instead highly recommended to set the correct {@link DbiFlag}s to + * allow the native library to correctly order the intended keys. + * + *

+ * A default comparator will be provided if null is passed as the + * comparator. If a custom comparator is provided, it must strictly match the + * lexicographical order of keys in the native LMDB database. * *

* This method (and its overloaded convenience variants) must not be called @@ -339,17 +380,24 @@ public Dbi openDbi(final byte[] name, final Comparator comparator, * @param txn transaction to use (required; not closed) * @param name name of the database (or null if no name is required) * @param comparator custom comparator callback (or null to use LMDB default) + * @param nativeCb whether native code should call back to the comparator * @param flags to open the database with * @return a database that is ready to use */ public Dbi openDbi(final Txn txn, final byte[] name, - final Comparator comparator, + final Comparator comparator, final boolean nativeCb, final DbiFlags... flags) { if (SHOULD_CHECK) { requireNonNull(txn); txn.checkReady(); } - return new Dbi<>(this, txn, name, comparator, flags); + final Comparator useComparator; + if (comparator == null) { + useComparator = proxy.getComparator(flags); + } else { + useComparator = comparator; + } + return new Dbi<>(this, txn, name, useComparator, nativeCb, proxy, flags); } /** diff --git a/src/main/java/org/lmdbjava/Txn.java b/src/main/java/org/lmdbjava/Txn.java index f3071152..b6d28917 100644 --- a/src/main/java/org/lmdbjava/Txn.java +++ b/src/main/java/org/lmdbjava/Txn.java @@ -34,8 +34,6 @@ import static org.lmdbjava.Txn.State.RESET; import static org.lmdbjava.TxnFlags.MDB_RDONLY_TXN; -import java.util.Comparator; - import jnr.ffi.Pointer; /** @@ -225,19 +223,6 @@ void checkWritesAllowed() { } } - Comparator comparator() { - return proxy::compare; - } - - /** - * Obtain the buffer proxy. - * - * @return proxy (never null) - */ - BufferProxy getProxy() { - return proxy; - } - /** * Return the state of the transaction. * diff --git a/src/test/java/org/lmdbjava/ComparatorTest.java b/src/test/java/org/lmdbjava/ComparatorTest.java index 77c5ca37..f21604ee 100644 --- a/src/test/java/org/lmdbjava/ComparatorTest.java +++ b/src/test/java/org/lmdbjava/ComparatorTest.java @@ -25,6 +25,7 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; import static org.lmdbjava.ByteArrayProxy.PROXY_BA; +import static org.lmdbjava.ByteBufProxy.PROXY_NETTY; import static org.lmdbjava.ByteBufferProxy.PROXY_OPTIMAL; import static org.lmdbjava.ComparatorTest.ComparatorResult.EQUAL_TO; import static org.lmdbjava.ComparatorTest.ComparatorResult.GREATER_THAN; @@ -146,7 +147,8 @@ private static class ByteArrayRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { - return PROXY_BA.compare(o1, o2); + final Comparator c = PROXY_BA.getComparator(); + return c.compare(o1, o2); } } @@ -157,23 +159,25 @@ private static class ByteBufferRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { + final Comparator c = PROXY_OPTIMAL.getComparator(); + // Convert arrays to buffers that are larger than the array, with // limit set at the array length. One buffer bigger than the other. ByteBuffer o1b = arrayToBuffer(o1, o1.length * 3); ByteBuffer o2b = arrayToBuffer(o2, o2.length * 2); - final int result = PROXY_OPTIMAL.compare(o1b, o2b); + final int result = c.compare(o1b, o2b); // Now swap which buffer is bigger o1b = arrayToBuffer(o1, o1.length * 2); o2b = arrayToBuffer(o2, o2.length * 3); - final int result2 = PROXY_OPTIMAL.compare(o1b, o2b); + final int result2 = c.compare(o1b, o2b); assertThat(result2, is(result)); // Now try with buffers sized to the array. o1b = ByteBuffer.wrap(o1); o2b = ByteBuffer.wrap(o2); - final int result3 = PROXY_OPTIMAL.compare(o1b, o2b); + final int result3 = c.compare(o1b, o2b); assertThat(result3, is(result)); @@ -201,7 +205,8 @@ private static class DirectBufferRunner implements ComparatorRunner { public int compare(final byte[] o1, final byte[] o2) { final DirectBuffer o1b = new UnsafeBuffer(o1); final DirectBuffer o2b = new UnsafeBuffer(o2); - return PROXY_DB.compare(o1b, o2b); + final Comparator c = PROXY_DB.getComparator(); + return c.compare(o1b, o2b); } } @@ -240,7 +245,8 @@ public int compare(final byte[] o1, final byte[] o2) { final ByteBuf o2b = DEFAULT.directBuffer(o2.length); o1b.writeBytes(o1); o2b.writeBytes(o2); - return ByteBufProxy.PROXY_NETTY.compare(o1b, o2b); + final Comparator c = PROXY_NETTY.getComparator(); + return c.compare(o1b, o2b); } } diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index af7ca27f..e08e9a0c 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -131,10 +131,14 @@ public void before() throws IOException { .setMaxDbs(1) .open(path, POSIX_MODE, MDB_NOSUBDIR); db = env.openDbi(DB_1, MDB_CREATE); + populateDatabase(db); + } + + private void populateDatabase(final Dbi dbi) { list = new LinkedList<>(); list.addAll(asList(2, 3, 4, 5, 6, 7, 8, 9)); try (Txn txn = env.txnWrite()) { - final Cursor c = db.openCursor(txn); + final Cursor c = dbi.openCursor(txn); c.put(bb(2), bb(3), MDB_NOOVERWRITE); c.put(bb(4), bb(5)); c.put(bb(6), bb(7)); @@ -267,8 +271,10 @@ public void openClosedBackwardTestWithGuava() { bb2.reset(); return guava.compare(array1, array2); }; - verify(openClosedBackward(bb(7), bb(2)), comparator, 6, 4, 2); - verify(openClosedBackward(bb(8), bb(4)), comparator, 6, 4); + final Dbi guavaDbi = env.openDbi(DB_1, comparator, MDB_CREATE); + populateDatabase(guavaDbi); + verify(openClosedBackward(bb(7), bb(2)), guavaDbi, 6, 4, 2); + verify(openClosedBackward(bb(8), bb(4)), guavaDbi, 6, 4); } @Test @@ -356,17 +362,17 @@ public void forEachRemainingWithClosedEnvTest() { } } - private void verify(final KeyRange range, final int... expected) { - verify(range, null, expected); + private void verify(final KeyRange range, + final int... expected) { + verify(range, db, expected); } private void verify(final KeyRange range, - final Comparator comparator, - final int... expected) { + final Dbi dbi, final int... expected) { final List results = new ArrayList<>(); try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn, range, comparator)) { + CursorIterable c = dbi.iterate(txn, range)) { for (final KeyVal kv : c) { final int key = kv.key().getInt(); final int val = kv.val().getInt(); diff --git a/src/test/java/org/lmdbjava/DbiTest.java b/src/test/java/org/lmdbjava/DbiTest.java index 6db5a9b8..8646aec2 100644 --- a/src/test/java/org/lmdbjava/DbiTest.java +++ b/src/test/java/org/lmdbjava/DbiTest.java @@ -43,6 +43,7 @@ import static org.lmdbjava.ByteBufferProxy.PROXY_OPTIMAL; import static org.lmdbjava.DbiFlags.MDB_CREATE; import static org.lmdbjava.DbiFlags.MDB_DUPSORT; +import static org.lmdbjava.DbiFlags.MDB_INTEGERKEY; import static org.lmdbjava.DbiFlags.MDB_REVERSEKEY; import static org.lmdbjava.Env.create; import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR; @@ -117,13 +118,13 @@ public void close() { @Test public void customComparator() { final Comparator reverseOrder = (o1, o2) -> { - final int lexicalOrder = ByteBufferProxy.PROXY_OPTIMAL.compare(o1, o2); - if (lexicalOrder == 0) { + final int lexical = PROXY_OPTIMAL.getComparator().compare(o1, o2); + if (lexical == 0) { return 0; } - return lexicalOrder * -1; + return lexical * -1; }; - final Dbi db = env.openDbi(DB_1, reverseOrder, MDB_CREATE); + final Dbi db = env.openDbi(DB_1, reverseOrder, true, MDB_CREATE); try (Txn txn = env.txnWrite()) { assertThat(db.put(txn, bb(2), bb(3)), is(true)); assertThat(db.put(txn, bb(4), bb(6)), is(true)); @@ -150,8 +151,9 @@ public void dbOpenMaxDatabases() { @Test public void dbiWithComparatorThreadSafety() { - final Dbi db = env.openDbi(DB_1, PROXY_OPTIMAL::compare, - MDB_CREATE); + final DbiFlags[] flags = new DbiFlags[] {MDB_CREATE, MDB_INTEGERKEY}; + final Comparator c = PROXY_OPTIMAL.getComparator(flags); + final Dbi db = env.openDbi(DB_1, c, true, flags); final List keys = range(0, 1_000).boxed().collect(toList()); From 0dc67805a55341d24e49db342c51ff2aecb02f50 Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Sat, 4 Feb 2023 15:30:18 +1100 Subject: [PATCH 03/21] JavaDoc corrections (#199) --- src/main/java/org/lmdbjava/CursorIterable.java | 6 ------ src/main/java/org/lmdbjava/Env.java | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/main/java/org/lmdbjava/CursorIterable.java b/src/main/java/org/lmdbjava/CursorIterable.java index 681f805b..39a6d58c 100644 --- a/src/main/java/org/lmdbjava/CursorIterable.java +++ b/src/main/java/org/lmdbjava/CursorIterable.java @@ -41,12 +41,6 @@ *

* An instance will create and close its own cursor. * - *

- * If iterating over keys stored with {@link DbiFlags#MDB_INTEGERKEY} you must - * provide a Java comparator when constructing the {@link Dbi} or this class. It - * is more efficient to use a comparator only with this class, as this avoids - * LMDB calling back into Java code to perform the integer key comparison. - * * @param buffer type */ public final class CursorIterable implements diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index 66ab7cda..0a4c034b 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -365,7 +365,7 @@ public Dbi openDbi(final byte[] name, final Comparator comparator, * secondary (but uncommon) use of the comparator is to act as a callback from * the native library if nativeCb is true. This is * usually avoided due to the overhead of native code calling back into Java. - * It is instead highly recommended to set the correct {@link DbiFlag}s to + * It is instead highly recommended to set the correct {@link DbiFlags} to * allow the native library to correctly order the intended keys. * *

From 860c827e0b377cb55f6967ff66777428603456a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Lamar=C3=A3o?= Date: Wed, 8 Mar 2023 22:29:36 -0300 Subject: [PATCH 04/21] Define manifest attribute Automatic-Module-Name. (#213) --- pom.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pom.xml b/pom.xml index 20b02b04..09b36edb 100644 --- a/pom.xml +++ b/pom.xml @@ -117,6 +117,17 @@ org.apache.maven.plugins maven-enforcer-plugin + + org.apache.maven.plugins + maven-jar-plugin + + + + org.lmdbjava + + + + org.apache.maven.plugins maven-pmd-plugin From 9cf97a5cd28cbf996a1b206148f586626982880b Mon Sep 17 00:00:00 2001 From: Martin Hristov Date: Thu, 9 Mar 2023 03:31:27 +0200 Subject: [PATCH 05/21] Library.java : fix mdb_reader_check api, Env.java : readerCheck impl (#212) * Library.java : fix mdb_reader_check api, Env.java : readerCheck impl * whitespace in Library.java * upd * update Env.java * remove tabs --- src/main/java/org/lmdbjava/Env.java | 12 ++++++++++++ src/main/java/org/lmdbjava/Library.java | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index 0a4c034b..db8b0f4a 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -41,6 +41,7 @@ import java.util.List; import jnr.ffi.Pointer; +import jnr.ffi.byref.IntByReference; import jnr.ffi.byref.PointerByReference; import org.lmdbjava.Library.MDB_envinfo; import org.lmdbjava.Library.MDB_stat; @@ -500,6 +501,15 @@ private void validatePath(final File path) { validateDirectoryEmpty(path); } + + /* Check for stale entries in the reader lock table. */ + public int readerCheck() { + final IntByReference resultPtr = new IntByReference(); + checkRc(LIB.mdb_reader_check(ptr, resultPtr)); + return resultPtr.intValue(); + } + + /** * Object has already been closed and the operation is therefore prohibited. */ @@ -530,6 +540,7 @@ public AlreadyOpenException() { } } + /** * Builder for configuring and opening Env. * @@ -595,6 +606,7 @@ public Env open(final File path, final EnvFlags... flags) { return open(path, 0664, flags); } + /** * Sets the map size. * diff --git a/src/main/java/org/lmdbjava/Library.java b/src/main/java/org/lmdbjava/Library.java index 44b17492..e48def73 100644 --- a/src/main/java/org/lmdbjava/Library.java +++ b/src/main/java/org/lmdbjava/Library.java @@ -291,7 +291,7 @@ int mdb_put(@In Pointer txn, @In Pointer dbi, @In Pointer key, @In Pointer data, int flags); - int mdb_reader_check(@In Pointer env, int dead); + int mdb_reader_check(@In Pointer env, @Out IntByReference dead); int mdb_set_compare(@In Pointer txn, @In Pointer dbi, ComparatorCallback cb); From 0c636f1a2718de9d3e01c8afd04699a162265539 Mon Sep 17 00:00:00 2001 From: Ian Kang Date: Thu, 23 Mar 2023 16:16:16 -0400 Subject: [PATCH 06/21] Hold strong reference to all buffers passed to LmdbJava (fixes #207) --- pom.xml | 8 ++ src/main/java/org/lmdbjava/Cursor.java | 7 ++ src/main/java/org/lmdbjava/Dbi.java | 6 + src/main/java/org/lmdbjava/ReferenceUtil.java | 55 +++++++++ .../org/lmdbjava/GarbageCollectionTest.java | 111 ++++++++++++++++++ 5 files changed, 187 insertions(+) create mode 100644 src/main/java/org/lmdbjava/ReferenceUtil.java create mode 100644 src/test/java/org/lmdbjava/GarbageCollectionTest.java diff --git a/pom.xml b/pom.xml index 09b36edb..fa025026 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,12 @@ 0.9.29-1 true + + org.mockito + mockito-inline + 4.11.0 + test + @@ -110,6 +116,8 @@ com.github.jnr:jffi + org.mockito:mockito-core + org.mockito:mockito-inline diff --git a/src/main/java/org/lmdbjava/Cursor.java b/src/main/java/org/lmdbjava/Cursor.java index aa98a9ea..ed7c1848 100644 --- a/src/main/java/org/lmdbjava/Cursor.java +++ b/src/main/java/org/lmdbjava/Cursor.java @@ -162,6 +162,7 @@ public boolean get(final T key, final T data, final SeekOp op) { checkRc(rc); kv.keyOut(); kv.valOut(); + ReferenceUtil.reachabilityFence0(key); return true; } @@ -192,6 +193,7 @@ public boolean get(final T key, final GetOp op) { checkRc(rc); kv.keyOut(); kv.valOut(); + ReferenceUtil.reachabilityFence0(key); return true; } @@ -266,6 +268,8 @@ public boolean put(final T key, final T val, final PutFlags... op) { return false; } checkRc(rc); + ReferenceUtil.reachabilityFence0(key); + ReferenceUtil.reachabilityFence0(val); return true; } @@ -304,6 +308,8 @@ public void putMultiple(final T key, final T val, final int elements, final int rc = LIB.mdb_cursor_put(ptrCursor, txn.kv().pointerKey(), dataPtr, mask); checkRc(rc); + ReferenceUtil.reachabilityFence0(key); + ReferenceUtil.reachabilityFence0(val); } /** @@ -362,6 +368,7 @@ public T reserve(final T key, final int size, final PutFlags... op) { checkRc(LIB.mdb_cursor_put(ptrCursor, kv.pointerKey(), kv.pointerVal(), flags)); kv.valOut(); + ReferenceUtil.reachabilityFence0(key); return val(); } diff --git a/src/main/java/org/lmdbjava/Dbi.java b/src/main/java/org/lmdbjava/Dbi.java index d0eb0a14..ef8ec315 100644 --- a/src/main/java/org/lmdbjava/Dbi.java +++ b/src/main/java/org/lmdbjava/Dbi.java @@ -171,6 +171,8 @@ public boolean delete(final Txn txn, final T key, final T val) { return false; } checkRc(rc); + ReferenceUtil.reachabilityFence0(key); + ReferenceUtil.reachabilityFence0(val); return true; } @@ -239,6 +241,7 @@ public T get(final Txn txn, final T key) { return null; } checkRc(rc); + ReferenceUtil.reachabilityFence0(key); return txn.kv().valOut(); // marked as out in LMDB C docs } @@ -386,6 +389,8 @@ public boolean put(final Txn txn, final T key, final T val, return false; } checkRc(rc); + ReferenceUtil.reachabilityFence0(key); + ReferenceUtil.reachabilityFence0(val); return true; } @@ -421,6 +426,7 @@ public T reserve(final Txn txn, final T key, final int size, checkRc(LIB.mdb_put(txn.pointer(), ptr, txn.kv().pointerKey(), txn.kv() .pointerVal(), flags)); txn.kv().valOut(); // marked as in,out in LMDB C docs + ReferenceUtil.reachabilityFence0(key); return txn.val(); } diff --git a/src/main/java/org/lmdbjava/ReferenceUtil.java b/src/main/java/org/lmdbjava/ReferenceUtil.java new file mode 100644 index 00000000..0002a509 --- /dev/null +++ b/src/main/java/org/lmdbjava/ReferenceUtil.java @@ -0,0 +1,55 @@ +/*- + * #%L + * LmdbJava + * %% + * Copyright (C) 2016 - 2023 The LmdbJava Open Source Project + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.lmdbjava; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +public final class ReferenceUtil { + /** + * Ensures that the object referenced by the given reference remains + * strongly reachable, + * regardless of any prior actions of the program that might otherwise cause + * the object to become unreachable; thus, the referenced object is not + * reclaimable by garbage collection at least until after the invocation of + * this method. + * + *

Recent versions of the JDK have a nasty habit of prematurely deciding objects are unreachable. + * see: https://stackoverflow.com/questions/26642153/finalize-called-on-strongly-reachable-object-in-java-8 + * The Java 9 method Reference.reachabilityFence offers a solution to this problem. + * + *

This method is always implemented as a synchronization on {@code ref}, not as + * {@code Reference.reachabilityFence} for consistency across platforms and to allow building on JDK 6-8. + * It is the caller's responsibility to ensure that this synchronization will not cause deadlock. + * + * @param ref the reference. If {@code null}, this method has no effect. + * @see https://github.com/netty/netty/pull/8410 + */ + @SuppressFBWarnings({"ESync_EMPTY_SYNC", "UC_USELESS_VOID_METHOD"}) + public static void reachabilityFence0(Object ref) { + if (ref != null) { + synchronized (ref) { + // Empty synchronized is ok: https://stackoverflow.com/a/31933260/1151521 + } + } + } + + private ReferenceUtil() {} +} diff --git a/src/test/java/org/lmdbjava/GarbageCollectionTest.java b/src/test/java/org/lmdbjava/GarbageCollectionTest.java new file mode 100644 index 00000000..aa8a8856 --- /dev/null +++ b/src/test/java/org/lmdbjava/GarbageCollectionTest.java @@ -0,0 +1,111 @@ +/*- + * #%L + * LmdbJava + * %% + * Copyright (C) 2016 - 2023 The LmdbJava Open Source Project + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.lmdbjava; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; + +import static java.nio.ByteBuffer.allocateDirect; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; +import static org.lmdbjava.DbiFlags.MDB_CREATE; +import static org.lmdbjava.Env.create; + +@SuppressFBWarnings({"DM_GC", "RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT"}) +@SuppressWarnings("PMD.DoNotCallGarbageCollectionExplicitly") +public class GarbageCollectionTest { + + private static final String DB_NAME = "my DB"; + private static final String KEY_PREFIX = "Uncorruptedkey"; + private static final String VAL_PREFIX = "Uncorruptedval"; + + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); + private void putBuffer(Dbi db, Txn txn, int i) { + ByteBuffer key = allocateDirect(24); + ByteBuffer val = allocateDirect(24); + key.put((KEY_PREFIX+i).getBytes(UTF_8)).flip(); + val.put((VAL_PREFIX+i).getBytes(UTF_8)).flip(); + db.put(txn, key, val); + } + @Test + public void buffersNotGarbageCollectedTest() throws IOException { + final File path = tmp.newFolder(); + + final Env env = create() + .setMapSize(2_085_760_999) + .setMaxDbs(1) + .open(path); + final Dbi db = env.openDbi(DB_NAME, MDB_CREATE); + + // Trigger compilation and whatnot + try (Txn txn = env.txnWrite()) { + for (int i = 0; i < 5000; i++) { + putBuffer(db, txn, i); + } + txn.commit(); + } + // Call gc before writing to lmdb and after last reference to buffer by changing the behavior of mask + try (MockedStatic mockedStatic = Mockito.mockStatic(MaskedFlag.class)) { + mockedStatic.when(MaskedFlag::mask).thenAnswer(invocationOnMock -> { + System.gc(); + return 0; + }); + try (Txn txn = env.txnWrite()) { + for (int i = 0; i < 1000; i++) { + putBuffer(db, txn, i); + } + txn.commit(); + } + } + + // Find corrupt keys + try (Txn txn = env.txnRead()) { + try (Cursor c = db.openCursor(txn)) { + if (c.first()) { + do { + byte[] rkey = new byte[c.key().remaining()]; + c.key().get(rkey); + byte[] rval = new byte[c.val().remaining()]; + c.val().get(rval); + String skey = new String(rkey, UTF_8); + String sval = new String(rval, UTF_8); + if (!skey.startsWith("Uncorruptedkey")) { + fail("Found corrupt key " + skey); + } + if (!sval.startsWith("Uncorruptedval")) { + fail("Found corrupt val " + sval); + } + } while (c.next()); + } + } + } + env.close(); + } +} From 288bb09f00fdfe8f4d2d062aabc33e5dcbef8af8 Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Mon, 24 Apr 2023 10:17:49 +1000 Subject: [PATCH 07/21] Zig cross-compilation of native libraries (#217) This new approach provides a number of benefits: 1. The LmdbJava Native project can be archived. This project required coordination with LmdbJava module releases and added extra artifacts to Maven Central simply to wrap native libraries. The native project also required a great deal of Maven configuration and it was always tedious to support additional platforms. Using Zig eliminates the need to run QEMU emulators in builds to support unusual platforms etc. 2. Zig supports numerous cross-compilation targets straight out of the box. A full list is available via zig targets | jq -r '.libc[]' and this presently lists 64 on my machine. This is likely to accommodate most LmdbJava platform support requests we are likely to receive. 3. Platform naming conventions have now been standardised based on the Zig target name. Support for individual build chains or specific processors is now simplified and much more transparent. 4. The GitHub Action has been amended to perform cross-compilation under Linux and upload the resulting native artifacts for later build steps. The later build steps run the Verifier on a native VM where available (eg Windows, Mac OS) and this therefore tests the cross-compiled libraries. 5. Target name resolution logic has been refactored and externalised in its own class with corresponding unit tests to ensure corner cases are duly considered and any bugs more easily reproduced and permanently rectified. This change is backwards compatible with users who used (and may continue to use) the lmdbjava.native.lib system property. --- .github/workflows/maven.yml | 27 +++ .gitignore | 1 + README.md | 20 ++- cross-compile.sh | 27 +++ pom.xml | 45 ----- src/main/java/org/lmdbjava/Library.java | 52 +----- src/main/java/org/lmdbjava/TargetName.java | 156 ++++++++++++++++++ src/main/resources/org/lmdbjava/.gitignore | 2 + .../java/org/lmdbjava/TargetNameTest.java | 74 +++++++++ 9 files changed, 310 insertions(+), 94 deletions(-) create mode 100755 cross-compile.sh create mode 100644 src/main/java/org/lmdbjava/TargetName.java create mode 100644 src/main/resources/org/lmdbjava/.gitignore create mode 100644 src/test/java/org/lmdbjava/TargetNameTest.java diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 85d4385f..4c3ac069 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -22,14 +22,29 @@ jobs: java-version: 17 cache: maven + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + + - name: Cross compile using Zig + run: ./cross-compile.sh + - name: Build with Maven run: mvn -B verify + - name: Store built native libraries for later jobs + uses: actions/upload-artifact@v3 + with: + name: native-libraries + path: | + src/main/resources/org/lmdbjava/*.so + src/main/resources/org/lmdbjava/*.dll + - name: Upload code coverage to Codecov uses: codecov/codecov-action@v3 compatibility-checks: name: Java ${{ matrix.java }} on ${{ matrix.os }} Compatibility + needs: [build] runs-on: ${{ matrix.os }} strategy: @@ -48,6 +63,12 @@ jobs: java-version: ${{ matrix.java }} cache: maven + - name: Fetch built native libraries + uses: actions/download-artifact@v3 + with: + name: native-libraries + path: src/main/resources/org/lmdbjava + - name: Execute verifier run: mvn -B test -Dtest=VerifierTest -DverificationSeconds=10 @@ -81,6 +102,12 @@ jobs: gpg-private-key: ${{ secrets.gpg_private_key }} gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + + - name: Cross compile using Zig + run: ./cross-compile.sh + - name: Publish Maven package run: mvn -B -Possrh-deploy deploy -DskipTests env: diff --git a/.gitignore b/.gitignore index fe6430cb..0c771329 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ dependency-reduced-pom.xml gpg-sign.json mvn-sync.json secrets.tar +lmdb diff --git a/README.md b/README.md index 450f814f..16ff78ae 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ * Modern, idiomatic Java API (including iterators, key ranges, enums, exceptions etc) * Nothing to install (the JAR embeds the latest LMDB libraries for Linux, OS X and Windows) * Buffer agnostic (Java `ByteBuffer`, Agrona `DirectBuffer`, Netty `ByteBuf`, your own buffer) -* 100% stock-standard, officially-released, widely-tested LMDB C code ([no extra](https://github.com/lmdbjava/native) C/JNI code) +* 100% stock-standard, officially-released, widely-tested LMDB C code (no extra C/JNI code) * Low latency design (allocation-free; buffer pools; optional checks can be easily disabled in production etc) * Mature code (commenced in 2016) and used for heavy production workloads (eg > 500 TB of HFT data) * Actively maintained and with a "Zero Bug Policy" before every release (see [issues](https://github.com/lmdbjava/lmdbjava/issues)) @@ -55,6 +55,24 @@ We're happy to help you use LmdbJava. Simply [open a GitHub issue](https://github.com/lmdbjava/lmdbjava/issues) if you have any questions. +### Building + +This project uses [Zig](https://ziglang.org/) to cross-compile the LMDB native +library for all supported architectures. To locally build LmdbJava you must +firstly install a recent version of Zig and then execute the project's +[cross-compile.sh](https://github.com/lmdbjava/lmdbjava/tree/master/cross-compile.sh) +script. This only needs to be repeated when the `cross-compile.sh` script is +updated (eg following a new official release of the upstream LMDB library). + +If you do not wish to install Zig and/or use an operating system which cannot +easily execute the `cross-compile.sh` script, you can download the compiled +LMDB native library for your platform from a location of your choice and set the +`lmdbjava.native.lib` system property to the resulting file system system +location. Possible sources of a compiled LMDB native library include operating +system package managers, running `cross-compile.sh` on a supported system, or +copying it from the `org/lmdbjava` directory of any recent, officially released +LmdbJava JAR. + ### Contributing Contributions are welcome! Please see the [Contributing Guidelines](CONTRIBUTING.md). diff --git a/cross-compile.sh b/cross-compile.sh new file mode 100755 index 00000000..1ebf92f6 --- /dev/null +++ b/cross-compile.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -o errexit + +rm -rf lmdb +git clone --depth 1 --branch LMDB_0.9.29 https://github.com/LMDB/lmdb.git +pushd lmdb/libraries/liblmdb +trap popd SIGINT + +# zig targets | jq -r '.libc[]' +for target in aarch64-linux-gnu \ + aarch64-macos-none \ + x86_64-linux-gnu \ + x86_64-macos-none \ + x86_64-windows-gnu +do + echo "##### Building $target ####" + make -e clean liblmdb.so CC="zig cc -target $target" AR="zig ar" + if [[ "$target" == *-windows-* ]]; then + extension="dll" + else + extension="so" + fi + cp -v liblmdb.so ../../../src/main/resources/org/lmdbjava/$target.$extension +done + +ls -l ../../../src/main/resources/org/lmdbjava diff --git a/pom.xml b/pom.xml index 09b36edb..4c8f03ba 100644 --- a/pom.xml +++ b/pom.xml @@ -70,24 +70,6 @@ org.hamcrest hamcrest - - org.lmdbjava - lmdbjava-native-linux-x86_64 - 0.9.29-1 - true - - - org.lmdbjava - lmdbjava-native-osx-x86_64 - 0.9.29-1 - true - - - org.lmdbjava - lmdbjava-native-windows-x86_64 - 0.9.29-1 - true - @@ -103,11 +85,6 @@ org.apache.maven.plugins maven-dependency-plugin - - org.lmdbjava:lmdbjava-native-linux-x86_64 - org.lmdbjava:lmdbjava-native-windows-x86_64 - org.lmdbjava:lmdbjava-native-osx-x86_64 - com.github.jnr:jffi @@ -132,28 +109,6 @@ org.apache.maven.plugins maven-pmd-plugin - - org.apache.maven.plugins - maven-shade-plugin - - - lmdbjava-shade - - shade - - package - - - - org.lmdbjava:lmdbjava-native-linux-x86_64 - org.lmdbjava:lmdbjava-native-windows-x86_64 - org.lmdbjava:lmdbjava-native-osx-x86_64 - - - - - - org.apache.maven.plugins maven-surefire-plugin diff --git a/src/main/java/org/lmdbjava/Library.java b/src/main/java/org/lmdbjava/Library.java index e48def73..22308600 100644 --- a/src/main/java/org/lmdbjava/Library.java +++ b/src/main/java/org/lmdbjava/Library.java @@ -21,11 +21,8 @@ package org.lmdbjava; import static java.io.File.createTempFile; -import static java.lang.Boolean.getBoolean; import static java.lang.System.getProperty; import static java.lang.Thread.currentThread; -import static java.util.Locale.ENGLISH; -import static java.util.Objects.nonNull; import static java.util.Objects.requireNonNull; import static jnr.ffi.LibraryLoader.create; import static jnr.ffi.Runtime.getRuntime; @@ -55,35 +52,15 @@ */ final class Library { - /** - * Java system property name that can be set to disable automatic extraction - * of the LMDB system library from the LmdbJava JAR. This may be desirable if - * an operating system-provided LMDB system library is preferred (eg operating - * system package management, vendor support, special compiler flags, security - * auditing, profile guided optimization builds, faster startup time by - * avoiding the library copy etc). - */ - public static final String DISABLE_EXTRACT_PROP = "lmdbjava.disable.extract"; /** * Java system property name that can be set to the path of an existing * directory into which the LMDB system library will be extracted from the * LmdbJava JAR. If unspecified the LMDB system library is extracted to the * java.io.tmpdir. Ignored if the LMDB system library is not * being extracted from the LmdbJava JAR (as would be the case if other - * system properties defined in Library have been set). + * system properties defined in TargetName have been set). */ public static final String LMDB_EXTRACT_DIR_PROP = "lmdbjava.extract.dir"; - /** - * Java system property name that can be set to provide a custom path to a - * external LMDB system library. If set, the system property - * DISABLE_EXTRACT_PROP will be overridden. - */ - public static final String LMDB_NATIVE_LIB_PROP = "lmdbjava.native.lib"; - /** - * Indicates whether automatic extraction of the LMDB system library is - * permitted. - */ - public static final boolean SHOULD_EXTRACT = !getBoolean(DISABLE_EXTRACT_PROP); /** * Indicates the directory where the LMDB system library will be extracted. */ @@ -91,35 +68,14 @@ final class Library { getProperty("java.io.tmpdir")); static final Lmdb LIB; static final jnr.ffi.Runtime RUNTIME; - /** - * Indicates whether external LMDB system library is provided. - */ - static final boolean SHOULD_USE_LIB = nonNull( - getProperty(LMDB_NATIVE_LIB_PROP)); - private static final String LIB_NAME = "lmdb"; static { final String libToLoad; - final String arch = getProperty("os.arch"); - final boolean arch64 = "x64".equals(arch) || "amd64".equals(arch) - || "x86_64".equals(arch); - - final String os = getProperty("os.name"); - final boolean linux = os.toLowerCase(ENGLISH).startsWith("linux"); - final boolean osx = os.startsWith("Mac OS X"); - final boolean windows = os.startsWith("Windows"); - - if (SHOULD_USE_LIB) { - libToLoad = getProperty(LMDB_NATIVE_LIB_PROP); - } else if (SHOULD_EXTRACT && arch64 && linux) { - libToLoad = extract("org/lmdbjava/lmdbjava-native-linux-x86_64.so"); - } else if (SHOULD_EXTRACT && arch64 && osx) { - libToLoad = extract("org/lmdbjava/lmdbjava-native-osx-x86_64.dylib"); - } else if (SHOULD_EXTRACT && arch64 && windows) { - libToLoad = extract("org/lmdbjava/lmdbjava-native-windows-x86_64.dll"); + if (TargetName.IS_EXTERNAL) { + libToLoad = TargetName.RESOLVED_FILENAME; } else { - libToLoad = LIB_NAME; + libToLoad = extract(TargetName.RESOLVED_FILENAME); } LIB = create(Lmdb.class).load(libToLoad); diff --git a/src/main/java/org/lmdbjava/TargetName.java b/src/main/java/org/lmdbjava/TargetName.java new file mode 100644 index 00000000..3f8692a6 --- /dev/null +++ b/src/main/java/org/lmdbjava/TargetName.java @@ -0,0 +1,156 @@ +/*- + * #%L + * LmdbJava + * %% + * Copyright (C) 2016 - 2023 The LmdbJava Open Source Project + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.lmdbjava; + +import static java.lang.System.getProperty; +import static java.util.Locale.ENGLISH; + +/** + * Determines the name of the target LMDB native library. + * + *

+ * Users will typically use an LMDB native library that is embedded within the + * LmdbJava JAR. Embedded libraries are built by a Zig cross-compilation step as + * part of the release process. The naming convention reflects the Zig target + * name plus a common filename extension. This simplifies support for future Zig + * targets (eg with different toolchains etc). + * + *

+ * Users can set two system properties to override the automatic resolution of + * an embedded library. Setting {@link #LMDB_NATIVE_LIB_PROP} will force use of + * that external LMDB library. Setting {@link #LMDB_EMBEDDED_LIB_PROP} will + * force use of that embedded LMDB library. If both are set, the former property + * will take precedence. Most users do not need to set either property. + */ +public final class TargetName { + + /** + * True if the resolved native filename is an external file (conversely false + * indicates the file should be considered a classpath resource). + */ + public static final boolean IS_EXTERNAL; + + /** + * Java system property name that can be set to override the embedded library + * that will be used. This is likely to required if automatic resolution fails + * but the user still prefers to use an LmdbJava-bundled library. + */ + public static final String LMDB_EMBEDDED_LIB_PROP = "lmdbjava.embedded.lib"; + /** + * Java system property name that can be set to provide a custom path to an + * external LMDB system library. This path must include the classpath prefix + * (usually org/lmdbjava). + */ + public static final String LMDB_NATIVE_LIB_PROP = "lmdbjava.native.lib"; + /** + * Resolved target native filename or fully-qualified classpath location. + */ + public static final String RESOLVED_FILENAME; + private static final String ARCH = getProperty("os.arch"); + private static final String EMBED = getProperty(LMDB_EMBEDDED_LIB_PROP); + private static final String EXTERNAL = getProperty(LMDB_NATIVE_LIB_PROP); + private static final String OS = getProperty("os.name"); + + static { + IS_EXTERNAL = isExternal(EXTERNAL); + RESOLVED_FILENAME = resolveFilename(EXTERNAL, EMBED, ARCH, OS); + } + + private TargetName() { + } + + public static String resolveExtension(final String os) { + return check(os, "Windows") ? "dll" : "so"; + } + + static boolean isExternal(final String external) { + return external != null && !external.isEmpty(); + } + + static String resolveFilename(final String external, final String embed, + final String arch, final String os) { + if (external != null && !external.isEmpty()) { + return external; + } + + if (embed != null && !embed.isEmpty()) { + return embed; + } + + return "org/lmdbjava/" + resolveArch(arch) + "-" + resolveOs(os) + "-" + + resolveToolchain(os) + "." + resolveExtension(os); + } + + /** + * Case insensitively checks whether the passed string starts with any of the + * candidate strings. + * + * @param string the string being checked + * @param candidates one or more candidate strings + * @return true if the string starts with any of the candidates + */ + private static boolean check(final String string, + final String... candidates) { + if (string == null) { + return false; + } + + final String strLower = string.toLowerCase(ENGLISH); + for (final String c : candidates) { + if (strLower.startsWith(c.toLowerCase(ENGLISH))) { + return true; + } + } + return false; + } + + private static String err(final String reason) { + return reason + " (please set system property " + LMDB_NATIVE_LIB_PROP + + " to the path of an external LMDB native library or property " + + LMDB_EMBEDDED_LIB_PROP + " to the name of an LmdbJava embedded" + + " library; os.arch='" + ARCH + "' os.name='" + OS + "')"; + } + + private static String resolveArch(final String arch) { + if (check(arch, "aarch64")) { + return "aarch64"; + } else if (check(arch, "x86_64", "amd64")) { + return "x86_64"; + } + throw new UnsupportedOperationException(err("Unsupported os.arch")); + } + + private static String resolveOs(final String os) { + if (check(os, "Linux")) { + return "linux"; + } else if (check(os, "Mac OS")) { + return "macos"; + } else if (check(os, "Windows")) { + return "windows"; + } + throw new UnsupportedOperationException(err("Unsupported os.name")); + } + + private static String resolveToolchain(final String os) { + return check(os, "Mac OS") ? "none" : "gnu"; + } + +} diff --git a/src/main/resources/org/lmdbjava/.gitignore b/src/main/resources/org/lmdbjava/.gitignore new file mode 100644 index 00000000..661f98b3 --- /dev/null +++ b/src/main/resources/org/lmdbjava/.gitignore @@ -0,0 +1,2 @@ +*.so +*.dll diff --git a/src/test/java/org/lmdbjava/TargetNameTest.java b/src/test/java/org/lmdbjava/TargetNameTest.java new file mode 100644 index 00000000..eec38233 --- /dev/null +++ b/src/test/java/org/lmdbjava/TargetNameTest.java @@ -0,0 +1,74 @@ +/*- + * #%L + * LmdbJava + * %% + * Copyright (C) 2016 - 2023 The LmdbJava Open Source Project + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package org.lmdbjava; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.lmdbjava.TargetName.isExternal; +import static org.lmdbjava.TargetName.resolveFilename; +import static org.lmdbjava.TestUtils.invokePrivateConstructor; + +import org.junit.Test; + +/** + * Test {@link TargetName}. + */ +public final class TargetNameTest { + + private static final String NONE = ""; + + @Test + public void coverPrivateConstructors() { + invokePrivateConstructor(TargetName.class); + } + + @Test + public void customEmbedded() { + assertThat(resolveFilename(NONE, "x/y.so", NONE, NONE), is("x/y.so")); + assertThat(isExternal(NONE), is(false)); + } + + @Test + public void embeddedNameResolution() { + embed("aarch64-linux-gnu.so", "aarch64", "Linux"); + embed("aarch64-macos-none.so", "aarch64", "Mac OS"); + embed("x86_64-linux-gnu.so", "x86_64", "Linux"); + embed("x86_64-macos-none.so", "x86_64", "Mac OS"); + embed("x86_64-windows-gnu.dll", "x86_64", "Windows"); + } + + @Test + public void externalLibrary() { + assertThat(resolveFilename("/l.so", NONE, NONE, NONE), is("/l.so")); + assertThat(TargetName.isExternal("/l.so"), is(true)); + } + + @Test + public void externalTakesPriority() { + assertThat(resolveFilename("/lm.so", "x/y.so", NONE, NONE), is("/lm.so")); + assertThat(isExternal("/lm.so"), is(true)); + } + + private void embed(final String lib, final String arch, final String os) { + assertThat(resolveFilename(NONE, NONE, arch, os), is("org/lmdbjava/" + lib)); + assertThat(isExternal(NONE), is(false)); + } +} From d87bc4b2cef38c28f9917d5f52cc1d4d962c08ea Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Mon, 24 Apr 2023 12:55:11 +1000 Subject: [PATCH 08/21] JavaDoc corrections (follows #217) --- src/main/java/org/lmdbjava/TargetName.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/lmdbjava/TargetName.java b/src/main/java/org/lmdbjava/TargetName.java index 3f8692a6..48411044 100644 --- a/src/main/java/org/lmdbjava/TargetName.java +++ b/src/main/java/org/lmdbjava/TargetName.java @@ -50,14 +50,14 @@ public final class TargetName { /** * Java system property name that can be set to override the embedded library - * that will be used. This is likely to required if automatic resolution fails - * but the user still prefers to use an LmdbJava-bundled library. + * that will be used. This is likely to be required if automatic resolution + * fails but the user still prefers to use an LmdbJava-bundled library. This + * path must include the classpath prefix (usually org/lmdbjava). */ public static final String LMDB_EMBEDDED_LIB_PROP = "lmdbjava.embedded.lib"; /** * Java system property name that can be set to provide a custom path to an - * external LMDB system library. This path must include the classpath prefix - * (usually org/lmdbjava). + * external LMDB system library. */ public static final String LMDB_NATIVE_LIB_PROP = "lmdbjava.native.lib"; /** From 98c233d110fb3876d1f3d7421bca86e514af3586 Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Mon, 24 Apr 2023 13:22:19 +1000 Subject: [PATCH 09/21] Remove hard-coded package name resolution (follows #217) --- src/main/java/org/lmdbjava/TargetName.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/lmdbjava/TargetName.java b/src/main/java/org/lmdbjava/TargetName.java index 48411044..446ea5b5 100644 --- a/src/main/java/org/lmdbjava/TargetName.java +++ b/src/main/java/org/lmdbjava/TargetName.java @@ -95,7 +95,8 @@ static String resolveFilename(final String external, final String embed, return embed; } - return "org/lmdbjava/" + resolveArch(arch) + "-" + resolveOs(os) + "-" + final String pkg = TargetName.class.getPackage().getName().replace('.', '/'); + return pkg + "/" + resolveArch(arch) + "-" + resolveOs(os) + "-" + resolveToolchain(os) + "." + resolveExtension(os); } From cbdaee61fce707aa33fae2738fd8653a7f78c04c Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Mon, 24 Apr 2023 14:25:20 +1000 Subject: [PATCH 10/21] Update formatting, documentation and limit GC test write frequency The GarbageCollectionTest performing 1,000 GC operations caused a substantial increase in test execution time. Trial and error found commenting out the new ReferenceUtil.reachabilityFence0 references resulted in the failures being consistently detected with a value of 250. Nevertheless even a value of 250 would result in the new test being approximately 3 times longer in duration than any other test. To resolve this the default value was reduced to 50 and a system property introduced to allow any arbitrary value to be used. In turn the GitHub Action was amended to test with 1,000 GC-inducing writes as per the original code. As an aside on OpenJDK Java 20 the mere presence of a call to ReferenceUtil.reachabilityFence0 was sufficient to pass the tests (even with high values such as 2,000) despite there being no method body remaining in reachabilityFence0. As such it is unverified whether the synchronization is actually required for the strong reachability. This is consistent with https://github.com/apache/datasketches-memory/issues/91#issuecomment-444294778. In any event synchronization has been retained given it is the more conservative approach. --- .github/workflows/maven.yml | 2 +- src/main/java/org/lmdbjava/ReferenceUtil.java | 68 +++++---- .../org/lmdbjava/GarbageCollectionTest.java | 144 +++++++++--------- 3 files changed, 116 insertions(+), 98 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 4c3ac069..2a4f691c 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -29,7 +29,7 @@ jobs: run: ./cross-compile.sh - name: Build with Maven - run: mvn -B verify + run: mvn -B verify -DgcRecordWrites=1000 - name: Store built native libraries for later jobs uses: actions/upload-artifact@v3 diff --git a/src/main/java/org/lmdbjava/ReferenceUtil.java b/src/main/java/org/lmdbjava/ReferenceUtil.java index 0002a509..36bf9247 100644 --- a/src/main/java/org/lmdbjava/ReferenceUtil.java +++ b/src/main/java/org/lmdbjava/ReferenceUtil.java @@ -22,34 +22,46 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +/** + * Supports creating strong references in manner compatible with Java 8. + */ public final class ReferenceUtil { - /** - * Ensures that the object referenced by the given reference remains - * strongly reachable, - * regardless of any prior actions of the program that might otherwise cause - * the object to become unreachable; thus, the referenced object is not - * reclaimable by garbage collection at least until after the invocation of - * this method. - * - *

Recent versions of the JDK have a nasty habit of prematurely deciding objects are unreachable. - * see: https://stackoverflow.com/questions/26642153/finalize-called-on-strongly-reachable-object-in-java-8 - * The Java 9 method Reference.reachabilityFence offers a solution to this problem. - * - *

This method is always implemented as a synchronization on {@code ref}, not as - * {@code Reference.reachabilityFence} for consistency across platforms and to allow building on JDK 6-8. - * It is the caller's responsibility to ensure that this synchronization will not cause deadlock. - * - * @param ref the reference. If {@code null}, this method has no effect. - * @see https://github.com/netty/netty/pull/8410 - */ - @SuppressFBWarnings({"ESync_EMPTY_SYNC", "UC_USELESS_VOID_METHOD"}) - public static void reachabilityFence0(Object ref) { - if (ref != null) { - synchronized (ref) { - // Empty synchronized is ok: https://stackoverflow.com/a/31933260/1151521 - } - } - } - private ReferenceUtil() {} + private ReferenceUtil() { + } + + /** + * Ensures that the object referenced by the given reference remains + * strongly reachable, regardless of any prior actions of the program + * that might otherwise cause the object to become unreachable. Thus, the + * referenced object is not reclaimable by garbage collection at least until + * after the invocation of this method. + * + *

+ * Recent versions of the JDK have a nasty habit of prematurely deciding + * objects are unreachable (eg + * StackOverflow question + * 26642153. + * + *

+ * java.lang.ref.Reference.reachabilityFence offers a solution to + * this problem, but it was only introduced in Java 9. LmdbJava presently + * supports Java 8 and therefore this method provides an alternative. + * + *

+ * This method is always implemented as a synchronization on {@code ref}. + * It is the caller's responsibility to ensure that this synchronization + * will not cause deadlock. + * + * @param ref the reference (null is acceptable but has no effect) + * @see Netty PR 8410 + */ + @SuppressFBWarnings({"ESync_EMPTY_SYNC", "UC_USELESS_VOID_METHOD"}) + public static void reachabilityFence0(final Object ref) { + if (ref != null) { + synchronized (ref) { + // Empty synchronized is ok: https://stackoverflow.com/a/31933260/1151521 + } + } + } } diff --git a/src/test/java/org/lmdbjava/GarbageCollectionTest.java b/src/test/java/org/lmdbjava/GarbageCollectionTest.java index aa8a8856..49d56b1d 100644 --- a/src/test/java/org/lmdbjava/GarbageCollectionTest.java +++ b/src/test/java/org/lmdbjava/GarbageCollectionTest.java @@ -20,6 +20,16 @@ package org.lmdbjava; +import static java.nio.ByteBuffer.allocateDirect; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; +import static org.lmdbjava.DbiFlags.MDB_CREATE; +import static org.lmdbjava.Env.create; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; + import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.junit.Rule; import org.junit.Test; @@ -27,85 +37,81 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; - -import static java.nio.ByteBuffer.allocateDirect; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.fail; -import static org.lmdbjava.DbiFlags.MDB_CREATE; -import static org.lmdbjava.Env.create; - @SuppressFBWarnings({"DM_GC", "RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT"}) @SuppressWarnings("PMD.DoNotCallGarbageCollectionExplicitly") public class GarbageCollectionTest { - private static final String DB_NAME = "my DB"; - private static final String KEY_PREFIX = "Uncorruptedkey"; - private static final String VAL_PREFIX = "Uncorruptedval"; + private static final String DB_NAME = "my DB"; + private static final String KEY_PREFIX = "Uncorruptedkey"; + private static final String VAL_PREFIX = "Uncorruptedval"; - @Rule - public final TemporaryFolder tmp = new TemporaryFolder(); - private void putBuffer(Dbi db, Txn txn, int i) { - ByteBuffer key = allocateDirect(24); - ByteBuffer val = allocateDirect(24); - key.put((KEY_PREFIX+i).getBytes(UTF_8)).flip(); - val.put((VAL_PREFIX+i).getBytes(UTF_8)).flip(); - db.put(txn, key, val); - } - @Test - public void buffersNotGarbageCollectedTest() throws IOException { - final File path = tmp.newFolder(); + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); - final Env env = create() - .setMapSize(2_085_760_999) - .setMaxDbs(1) - .open(path); - final Dbi db = env.openDbi(DB_NAME, MDB_CREATE); + @Test + public void buffersNotGarbageCollectedTest() throws IOException { + final File path = tmp.newFolder(); + try (Env env = create() + .setMapSize(2_085_760_999) + .setMaxDbs(1) + .open(path)) { + final Dbi db = env.openDbi(DB_NAME, MDB_CREATE); - // Trigger compilation and whatnot - try (Txn txn = env.txnWrite()) { - for (int i = 0; i < 5000; i++) { - putBuffer(db, txn, i); - } - txn.commit(); + try (Txn txn = env.txnWrite()) { + for (int i = 0; i < 5_000; i++) { + putBuffer(db, txn, i); } - // Call gc before writing to lmdb and after last reference to buffer by changing the behavior of mask - try (MockedStatic mockedStatic = Mockito.mockStatic(MaskedFlag.class)) { - mockedStatic.when(MaskedFlag::mask).thenAnswer(invocationOnMock -> { - System.gc(); - return 0; - }); - try (Txn txn = env.txnWrite()) { - for (int i = 0; i < 1000; i++) { - putBuffer(db, txn, i); - } - txn.commit(); - } + txn.commit(); + } + + // Call GC before writing to LMDB and after last reference to buffer by + // changing the behavior of mask + try (MockedStatic mockedStatic = Mockito.mockStatic( + MaskedFlag.class)) { + mockedStatic.when(MaskedFlag::mask).thenAnswer(invocationOnMock -> { + System.gc(); + return 0; + }); + final int gcRecordWrites = Integer.getInteger("gcRecordWrites", 50); + try (Txn txn = env.txnWrite()) { + for (int i = 0; i < gcRecordWrites; i++) { + putBuffer(db, txn, i); + } + txn.commit(); } + } - // Find corrupt keys - try (Txn txn = env.txnRead()) { - try (Cursor c = db.openCursor(txn)) { - if (c.first()) { - do { - byte[] rkey = new byte[c.key().remaining()]; - c.key().get(rkey); - byte[] rval = new byte[c.val().remaining()]; - c.val().get(rval); - String skey = new String(rkey, UTF_8); - String sval = new String(rval, UTF_8); - if (!skey.startsWith("Uncorruptedkey")) { - fail("Found corrupt key " + skey); - } - if (!sval.startsWith("Uncorruptedval")) { - fail("Found corrupt val " + sval); - } - } while (c.next()); - } - } + // Find corrupt keys + try (Txn txn = env.txnRead()) { + try (Cursor c = db.openCursor(txn)) { + if (c.first()) { + do { + final byte[] rkey = new byte[c.key().remaining()]; + c.key().get(rkey); + final byte[] rval = new byte[c.val().remaining()]; + c.val().get(rval); + final String skey = new String(rkey, UTF_8); + final String sval = new String(rval, UTF_8); + if (!skey.startsWith("Uncorruptedkey")) { + fail("Found corrupt key " + skey); + } + if (!sval.startsWith("Uncorruptedval")) { + fail("Found corrupt val " + sval); + } + } while (c.next()); + } } - env.close(); + } } + } + + private void putBuffer(final Dbi db, final Txn txn, + final int i) { + final ByteBuffer key = allocateDirect(24); + final ByteBuffer val = allocateDirect(24); + key.put((KEY_PREFIX + i).getBytes(UTF_8)).flip(); + val.put((VAL_PREFIX + i).getBytes(UTF_8)).flip(); + db.put(txn, key, val); + } + } From 838a8a0e2663a5469f450dea4471054ee138bf8f Mon Sep 17 00:00:00 2001 From: Mark Wardle Date: Tue, 25 Apr 2023 10:47:34 +0100 Subject: [PATCH 11/21] Fix build on aarch64 (e.g. Apple Silicon) (#218) * Increase map size to multiple of larger page size Most architectures such as x86 and x86-64 use 4KiB, while aarch64 systems can use 4, 16 or 64KiB. Apple Silicon machines currently have 16KiB page sizes. The LMDB documentation states that map size should be a multiple of the page size, so a 100KiB default map size for tests fails for systems with a page size that is not 4 KiB. This commit simply increases the map size for tests. Of course, this applies only to these tests, as in production, users can choose the appropriate map size for their requirements and system architecture. It is not straightforward to determine architecture page size at runtime. Otherwise, it might be more appropriate to dynamically determine a minimum map size. * Tweak test of page size The options for determining architecture page size are a) environment variables, b) unsafe and undocumented JDK internals or c) looking to get answer using FFI. Given this test is asking checking the statistics returned by LMDB, on balance it is reasonable to simply validate we're getting a multiple of 4096 in the statistics report. * Fix test for cut-off given large map sizes --- src/test/java/org/lmdbjava/CursorIterableTest.java | 2 +- src/test/java/org/lmdbjava/DbiTest.java | 2 +- src/test/java/org/lmdbjava/EnvTest.java | 11 ++++++----- src/test/java/org/lmdbjava/TxnTest.java | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index e08e9a0c..db28405a 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -126,7 +126,7 @@ public void atMostTest() { public void before() throws IOException { final File path = tmp.newFile(); env = create() - .setMapSize(KIBIBYTES.toBytes(100)) + .setMapSize(KIBIBYTES.toBytes(256)) .setMaxReaders(1) .setMaxDbs(1) .open(path, POSIX_MODE, MDB_NOSUBDIR); diff --git a/src/test/java/org/lmdbjava/DbiTest.java b/src/test/java/org/lmdbjava/DbiTest.java index 8646aec2..5a8cf549 100644 --- a/src/test/java/org/lmdbjava/DbiTest.java +++ b/src/test/java/org/lmdbjava/DbiTest.java @@ -460,7 +460,7 @@ public void stats() { assertThat(stat.entries, is(3L)); assertThat(stat.leafPages, is(1L)); assertThat(stat.overflowPages, is(0L)); - assertThat(stat.pageSize, is(4_096)); + assertThat(stat.pageSize % 4_096, is(0)); } @Test(expected = MapFullException.class) diff --git a/src/test/java/org/lmdbjava/EnvTest.java b/src/test/java/org/lmdbjava/EnvTest.java index f26d42f3..46d8766f 100644 --- a/src/test/java/org/lmdbjava/EnvTest.java +++ b/src/test/java/org/lmdbjava/EnvTest.java @@ -20,6 +20,7 @@ package org.lmdbjava; +import static com.jakewharton.byteunits.BinaryByteUnit.KIBIBYTES; import static com.jakewharton.byteunits.BinaryByteUnit.MEBIBYTES; import static java.nio.ByteBuffer.allocateDirect; import static org.hamcrest.CoreMatchers.containsString; @@ -338,7 +339,7 @@ public void setMapSize() throws IOException { final Random rnd = new Random(); try (Env env = create() .setMaxReaders(1) - .setMapSize(50_000) + .setMapSize(KIBIBYTES.toBytes(256)) .setMaxDbs(1) .open(path)) { final Dbi db = env.openDbi(DB_1, MDB_CREATE); @@ -346,7 +347,7 @@ public void setMapSize() throws IOException { db.put(bb(1), bb(42)); boolean mapFullExThrown = false; try { - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 70; i++) { rnd.nextBytes(k); key.clear(); key.put(k).flip(); @@ -358,7 +359,7 @@ public void setMapSize() throws IOException { } assertThat(mapFullExThrown, is(true)); - env.setMapSize(500_000); + env.setMapSize(KIBIBYTES.toBytes(512)); try (Txn roTxn = env.txnRead()) { assertThat(db.get(roTxn, bb(1)).getInt(), is(42)); @@ -366,7 +367,7 @@ public void setMapSize() throws IOException { mapFullExThrown = false; try { - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 70; i++) { rnd.nextBytes(k); key.clear(); key.put(k).flip(); @@ -393,7 +394,7 @@ public void stats() throws IOException { assertThat(stat.entries, is(0L)); assertThat(stat.leafPages, is(0L)); assertThat(stat.overflowPages, is(0L)); - assertThat(stat.pageSize, is(4_096)); + assertThat(stat.pageSize % 4_096, is(0)); assertThat(stat.toString(), containsString("pageSize=")); } } diff --git a/src/test/java/org/lmdbjava/TxnTest.java b/src/test/java/org/lmdbjava/TxnTest.java index c115be0b..775a10a6 100644 --- a/src/test/java/org/lmdbjava/TxnTest.java +++ b/src/test/java/org/lmdbjava/TxnTest.java @@ -84,7 +84,7 @@ public void after() { public void before() throws IOException { path = tmp.newFile(); env = create() - .setMapSize(KIBIBYTES.toBytes(100)) + .setMapSize(KIBIBYTES.toBytes(256)) .setMaxReaders(1) .setMaxDbs(2) .open(path, POSIX_MODE, MDB_NOSUBDIR); From 790212695d21a2a8c38acb7c652eb49cf2295502 Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Tue, 22 Aug 2023 16:23:33 +1000 Subject: [PATCH 12/21] private static final classes to be final (checkstyle) --- src/test/java/org/lmdbjava/ComparatorTest.java | 14 +++++++------- src/test/java/org/lmdbjava/KeyRangeTest.java | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/java/org/lmdbjava/ComparatorTest.java b/src/test/java/org/lmdbjava/ComparatorTest.java index f21604ee..8371a1ae 100644 --- a/src/test/java/org/lmdbjava/ComparatorTest.java +++ b/src/test/java/org/lmdbjava/ComparatorTest.java @@ -143,7 +143,7 @@ public void equalBuffers() { /** * Tests {@link ByteArrayProxy}. */ - private static class ByteArrayRunner implements ComparatorRunner { + private static final class ByteArrayRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { @@ -155,7 +155,7 @@ public int compare(final byte[] o1, final byte[] o2) { /** * Tests {@link ByteBufferProxy}. */ - private static class ByteBufferRunner implements ComparatorRunner { + private static final class ByteBufferRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { @@ -199,7 +199,7 @@ private ByteBuffer arrayToBuffer(final byte[] arr, final int bufferCapacity) { /** * Tests {@link DirectBufferProxy}. */ - private static class DirectBufferRunner implements ComparatorRunner { + private static final class DirectBufferRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { @@ -213,7 +213,7 @@ public int compare(final byte[] o1, final byte[] o2) { /** * Tests using Guava's {@link SignedBytes} comparator. */ - private static class GuavaSignedBytes implements ComparatorRunner { + private static final class GuavaSignedBytes implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { @@ -225,7 +225,7 @@ public int compare(final byte[] o1, final byte[] o2) { /** * Tests using Guava's {@link UnsignedBytes} comparator. */ - private static class GuavaUnsignedBytes implements ComparatorRunner { + private static final class GuavaUnsignedBytes implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { @@ -237,7 +237,7 @@ public int compare(final byte[] o1, final byte[] o2) { /** * Tests {@link ByteBufProxy}. */ - private static class NettyRunner implements ComparatorRunner { + private static final class NettyRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { @@ -254,7 +254,7 @@ public int compare(final byte[] o1, final byte[] o2) { * Tests {@link String} by providing a reference implementation of what a * comparator involving ASCII-encoded bytes should return. */ - private static class StringRunner implements ComparatorRunner { + private static final class StringRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { diff --git a/src/test/java/org/lmdbjava/KeyRangeTest.java b/src/test/java/org/lmdbjava/KeyRangeTest.java index 1b4384f9..3ea8c146 100644 --- a/src/test/java/org/lmdbjava/KeyRangeTest.java +++ b/src/test/java/org/lmdbjava/KeyRangeTest.java @@ -233,7 +233,7 @@ private void verify(final KeyRange range, final int... expected) { * We use Integer rather than the primitive to represent a * null buffer. */ - private static class FakeCursor { + private static final class FakeCursor { private static final int[] KEYS = new int[]{2, 4, 6, 8}; private int position; From c5f5689394364dea6a75771d4e5a2a0787345db2 Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Tue, 22 Aug 2023 16:23:56 +1000 Subject: [PATCH 13/21] Acegi Standard Project 0.7.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 57abcdcd..4f4706ee 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ au.com.acegi acegi-standard-project - 0.6.7 + 0.7.0 org.lmdbjava lmdbjava From e3c33f7649b6676dee94b34799af9d24e436664a Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Tue, 22 Aug 2023 16:25:31 +1000 Subject: [PATCH 14/21] Test on Java 20 instead of Java 19 --- .github/workflows/maven.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 2a4f691c..c7df2ed3 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -50,7 +50,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - java: [8, 11, 17, 19] + java: [8, 11, 17, 20] steps: - name: Check out Git repository diff --git a/README.md b/README.md index 16ff78ae..168e046e 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ * Mature code (commenced in 2016) and used for heavy production workloads (eg > 500 TB of HFT data) * Actively maintained and with a "Zero Bug Policy" before every release (see [issues](https://github.com/lmdbjava/lmdbjava/issues)) * Available from [Maven Central](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.lmdbjava%22%20AND%20a%3A%22lmdbjava%22) and [OSS Sonatype Snapshots](https://oss.sonatype.org/content/repositories/snapshots/org/lmdbjava/lmdbjava) -* [Continuous integration](https://github.com/lmdbjava/lmdbjava/actions) testing on Linux, Windows and macOS with Java 8, 11, 17 and 19 +* [Continuous integration](https://github.com/lmdbjava/lmdbjava/actions) testing on Linux, Windows and macOS with Java 8, 11, 17 and 20 ### Performance From 9b65ffdf142278fd7da4fc18db8e86c8174bb4eb Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Tue, 22 Aug 2023 16:40:27 +1000 Subject: [PATCH 15/21] Duplicate Finder Maven Plugin only to run on Java 11+ --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 4f4706ee..429e183a 100644 --- a/pom.xml +++ b/pom.xml @@ -125,10 +125,6 @@ false - - org.basepom.maven - duplicate-finder-maven-plugin - org.codehaus.mojo buildnumber-maven-plugin @@ -225,6 +221,10 @@ org.apache.maven.plugins maven-checkstyle-plugin + + org.basepom.maven + duplicate-finder-maven-plugin + From 1cc7e6c9e4c7e7a1e4dab6012e2d002b9bd05fba Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Mon, 4 Dec 2023 18:42:08 +1100 Subject: [PATCH 16/21] Test on Java 21 --- .github/workflows/maven.yml | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index c7df2ed3..32668e32 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -19,7 +19,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: zulu - java-version: 17 + java-version: 21 cache: maven - name: Install Zig @@ -50,7 +50,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - java: [8, 11, 17, 20] + java: [8, 11, 17, 21] steps: - name: Check out Git repository diff --git a/README.md b/README.md index 168e046e..048e57bc 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ * Mature code (commenced in 2016) and used for heavy production workloads (eg > 500 TB of HFT data) * Actively maintained and with a "Zero Bug Policy" before every release (see [issues](https://github.com/lmdbjava/lmdbjava/issues)) * Available from [Maven Central](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.lmdbjava%22%20AND%20a%3A%22lmdbjava%22) and [OSS Sonatype Snapshots](https://oss.sonatype.org/content/repositories/snapshots/org/lmdbjava/lmdbjava) -* [Continuous integration](https://github.com/lmdbjava/lmdbjava/actions) testing on Linux, Windows and macOS with Java 8, 11, 17 and 20 +* [Continuous integration](https://github.com/lmdbjava/lmdbjava/actions) testing on Linux, Windows and macOS with Java 8, 11, 17 and 21 ### Performance From 844c5195e40e1ab629b87633e857522101a958c4 Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Tue, 5 Dec 2023 11:36:39 +1100 Subject: [PATCH 17/21] Update to JNR-FFI 2.2.15 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 429e183a..1ea89d76 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ com.github.jnr jnr-ffi - 2.2.13 + 2.2.15 com.google.code.findbugs From bdccf3bc8593ad15342995124ef69bb5233275fd Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Tue, 5 Dec 2023 11:36:47 +1100 Subject: [PATCH 18/21] Update to Guava 32.1.3-JRE --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1ea89d76..542f1b43 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ com.google.guava guava - 31.1-jre + 32.1.3-jre test From 7ae7754a627c7d8153de4e907e2163be430cb03d Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Tue, 5 Dec 2023 11:37:02 +1100 Subject: [PATCH 19/21] Update to Netty Buffer 4.1.101.Final --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 542f1b43..205ef433 100644 --- a/pom.xml +++ b/pom.xml @@ -53,7 +53,7 @@ io.netty netty-buffer - 4.1.87.Final + 4.1.101.Final true From 699b735de002dab61f68441279b30ecdb09f09e3 Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Tue, 5 Dec 2023 11:37:13 +1100 Subject: [PATCH 20/21] Update to Agrona 1.20.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 205ef433..1946e3e9 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ org.agrona agrona - 1.17.1 + 1.20.0 true From f17b63f0ab8c90946d30f360a75b9e3107886a0c Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Tue, 5 Dec 2023 12:34:17 +1100 Subject: [PATCH 21/21] [maven-release-plugin] prepare release lmdbjava-0.9.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1946e3e9..52bc82f9 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ org.lmdbjava lmdbjava - 0.9.0-SNAPSHOT + 0.9.0 jar LmdbJava Low latency Java API for the ultra-fast, embedded Symas Lightning Database (LMDB) @@ -176,7 +176,7 @@ scm:git:git@github.com:${github.org}/${github.repo}.git scm:git:git@github.com:${github.org}/${github.repo}.git git@github.com:${github.org}/${github.repo}.git - HEAD + lmdbjava-0.9.0 GitHub Issues