From 9c2f31a6d7709c511e8bfb9011c218d8ac111bb9 Mon Sep 17 00:00:00 2001 From: Mark Wardle Date: Tue, 25 Apr 2023 07:36:28 +0100 Subject: [PATCH 1/3] Increase map size to multiple of larger page size Most architectures such as x86 and x86-64 use 4KiB, while aarch64 systems can use 4, 16 or 64KiB. Apple Silicon machines currently have 16KiB page sizes. The LMDB documentation states that map size should be a multiple of the page size, so a 100KiB default map size for tests fails for systems with a page size that is not 4 KiB. This commit simply increases the map size for tests. Of course, this applies only to these tests, as in production, users can choose the appropriate map size for their requirements and system architecture. It is not straightforward to determine architecture page size at runtime. Otherwise, it might be more appropriate to dynamically determine a minimum map size. --- src/test/java/org/lmdbjava/CursorIterableTest.java | 2 +- src/test/java/org/lmdbjava/EnvTest.java | 3 ++- src/test/java/org/lmdbjava/TxnTest.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index e08e9a0c..db28405a 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -126,7 +126,7 @@ public void atMostTest() { public void before() throws IOException { final File path = tmp.newFile(); env = create() - .setMapSize(KIBIBYTES.toBytes(100)) + .setMapSize(KIBIBYTES.toBytes(256)) .setMaxReaders(1) .setMaxDbs(1) .open(path, POSIX_MODE, MDB_NOSUBDIR); diff --git a/src/test/java/org/lmdbjava/EnvTest.java b/src/test/java/org/lmdbjava/EnvTest.java index f26d42f3..40776243 100644 --- a/src/test/java/org/lmdbjava/EnvTest.java +++ b/src/test/java/org/lmdbjava/EnvTest.java @@ -20,6 +20,7 @@ package org.lmdbjava; +import static com.jakewharton.byteunits.BinaryByteUnit.KIBIBYTES; import static com.jakewharton.byteunits.BinaryByteUnit.MEBIBYTES; import static java.nio.ByteBuffer.allocateDirect; import static org.hamcrest.CoreMatchers.containsString; @@ -338,7 +339,7 @@ public void setMapSize() throws IOException { final Random rnd = new Random(); try (Env env = create() .setMaxReaders(1) - .setMapSize(50_000) + .setMapSize(KIBIBYTES.toBytes(256)) .setMaxDbs(1) .open(path)) { final Dbi db = env.openDbi(DB_1, MDB_CREATE); diff --git a/src/test/java/org/lmdbjava/TxnTest.java b/src/test/java/org/lmdbjava/TxnTest.java index c115be0b..775a10a6 100644 --- a/src/test/java/org/lmdbjava/TxnTest.java +++ b/src/test/java/org/lmdbjava/TxnTest.java @@ -84,7 +84,7 @@ public void after() { public void before() throws IOException { path = tmp.newFile(); env = create() - .setMapSize(KIBIBYTES.toBytes(100)) + .setMapSize(KIBIBYTES.toBytes(256)) .setMaxReaders(1) .setMaxDbs(2) .open(path, POSIX_MODE, MDB_NOSUBDIR); From 39ecbb8f08793e9e73abc8ba24a5d366a83977bf Mon Sep 17 00:00:00 2001 From: Mark Wardle Date: Tue, 25 Apr 2023 07:43:30 +0100 Subject: [PATCH 2/3] Tweak test of page size The options for determining architecture page size are a) environment variables, b) unsafe and undocumented JDK internals or c) looking to get answer using FFI. Given this test is asking checking the statistics returned by LMDB, on balance it is reasonable to simply validate we're getting a multiple of 4096 in the statistics report. --- src/test/java/org/lmdbjava/DbiTest.java | 2 +- src/test/java/org/lmdbjava/EnvTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/lmdbjava/DbiTest.java b/src/test/java/org/lmdbjava/DbiTest.java index 8646aec2..5a8cf549 100644 --- a/src/test/java/org/lmdbjava/DbiTest.java +++ b/src/test/java/org/lmdbjava/DbiTest.java @@ -460,7 +460,7 @@ public void stats() { assertThat(stat.entries, is(3L)); assertThat(stat.leafPages, is(1L)); assertThat(stat.overflowPages, is(0L)); - assertThat(stat.pageSize, is(4_096)); + assertThat(stat.pageSize % 4_096, is(0)); } @Test(expected = MapFullException.class) diff --git a/src/test/java/org/lmdbjava/EnvTest.java b/src/test/java/org/lmdbjava/EnvTest.java index 40776243..9c4d786e 100644 --- a/src/test/java/org/lmdbjava/EnvTest.java +++ b/src/test/java/org/lmdbjava/EnvTest.java @@ -394,7 +394,7 @@ public void stats() throws IOException { assertThat(stat.entries, is(0L)); assertThat(stat.leafPages, is(0L)); assertThat(stat.overflowPages, is(0L)); - assertThat(stat.pageSize, is(4_096)); + assertThat(stat.pageSize % 4_096, is(0)); assertThat(stat.toString(), containsString("pageSize=")); } } From cff21e76e8e543b6130481cc68be796c082f46c6 Mon Sep 17 00:00:00 2001 From: Mark Wardle Date: Tue, 25 Apr 2023 09:16:08 +0100 Subject: [PATCH 3/3] Fix test for cut-off given large map sizes --- src/test/java/org/lmdbjava/EnvTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/lmdbjava/EnvTest.java b/src/test/java/org/lmdbjava/EnvTest.java index 9c4d786e..46d8766f 100644 --- a/src/test/java/org/lmdbjava/EnvTest.java +++ b/src/test/java/org/lmdbjava/EnvTest.java @@ -347,7 +347,7 @@ public void setMapSize() throws IOException { db.put(bb(1), bb(42)); boolean mapFullExThrown = false; try { - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 70; i++) { rnd.nextBytes(k); key.clear(); key.put(k).flip(); @@ -359,7 +359,7 @@ public void setMapSize() throws IOException { } assertThat(mapFullExThrown, is(true)); - env.setMapSize(500_000); + env.setMapSize(KIBIBYTES.toBytes(512)); try (Txn roTxn = env.txnRead()) { assertThat(db.get(roTxn, bb(1)).getInt(), is(42)); @@ -367,7 +367,7 @@ public void setMapSize() throws IOException { mapFullExThrown = false; try { - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 70; i++) { rnd.nextBytes(k); key.clear(); key.put(k).flip();