diff --git a/pom.xml b/pom.xml index 1e8874a0..4c961c46 100644 --- a/pom.xml +++ b/pom.xml @@ -216,4 +216,126 @@ GitHub Actions https://github.com/${github.org}/${github.repo}/actions + + + linux-aarch64 + + + Linux + unix + aarch64 + + + + + org.lmdbjava + lmdbjava-native-linux-aarch64 + 0.9.24-2-SNAPSHOT + true + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.lmdbjava:lmdbjava-native-linux-aarch64 + org.lmdbjava:lmdbjava-native-linux-x86_64 + org.lmdbjava:lmdbjava-native-windows-x86_64 + org.lmdbjava:lmdbjava-native-osx-x86_64 + + + com.github.jnr:jffi + + + + + org.apache.maven.plugins + maven-shade-plugin + + + lmdbjava-shade + + shade + + package + + + + org.lmdbjava:lmdbjava-native-linux-aarch64 + org.lmdbjava:lmdbjava-native-linux-x86_64 + org.lmdbjava:lmdbjava-native-windows-x86_64 + org.lmdbjava:lmdbjava-native-osx-x86_64 + + + + + + + + + + + linux-ppc64le + + + Linux + unix + ppc64le + + + + + org.lmdbjava + lmdbjava-native-linux-ppc64le + 0.9.24-2-SNAPSHOT + true + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.lmdbjava:lmdbjava-native-linux-ppc64le + org.lmdbjava:lmdbjava-native-linux-x86_64 + org.lmdbjava:lmdbjava-native-windows-x86_64 + org.lmdbjava:lmdbjava-native-osx-x86_64 + + + com.github.jnr:jffi + + + + + org.apache.maven.plugins + maven-shade-plugin + + + lmdbjava-shade + + shade + + package + + + + org.lmdbjava:lmdbjava-native-linux-ppc64le + org.lmdbjava:lmdbjava-native-linux-x86_64 + org.lmdbjava:lmdbjava-native-windows-x86_64 + org.lmdbjava:lmdbjava-native-osx-x86_64 + + + + + + + + + + diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index 7634a342..4ca70b8b 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -195,7 +195,8 @@ public List getDbiNames() { * @param mapSize the new size, in bytes */ public void setMapSize(final long mapSize) { - checkRc(LIB.mdb_env_set_mapsize(ptr, mapSize)); + // A number divisible by the number of pages is expected. + checkRc(LIB.mdb_env_set_mapsize(ptr, Builder.alignToWholePages(mapSize))); } /** @@ -417,6 +418,25 @@ private void validatePath(final File path) { validateDirectoryEmpty(path); } + public static int getPageSize() { + int pageSize = 0; + for (final String s: new String[]{ + System.getenv().get("PAGE_SIZE"), + System.getProperty("PAGE_SIZE")}) { + if (s != null && s.length() > 0) { + pageSize = Integer.parseInt(s); + break; + } + } + // This move is closely dependent on this logic: + // @see https://github.com/LMDB/lmdb/blob/LMDB_0.9.24/libraries/liblmdb/mdb.c#L498:L514 + if (pageSize >= 32_768) { + return 32_768; + } + // Silently defaulting to 4096 + return 4_096; + } + /** * Object has already been closed and the operation is therefore prohibited. */ @@ -454,8 +474,10 @@ public AlreadyOpenException() { */ public static final class Builder { + static final int PAGE_SIZE = getPageSize(); static final int MAX_READERS_DEFAULT = 126; - private long mapSize = 1_024 * 1_024; + // On a system with pages 4096 bytes big, this translates to default max map size of 4 MB + private long mapSize = PAGE_SIZE * 1_024; private int maxDbs = 1; private int maxReaders = MAX_READERS_DEFAULT; private boolean opened; @@ -486,6 +508,8 @@ public Env open(final File path, final int mode, checkRc(LIB.mdb_env_create(envPtr)); final Pointer ptr = envPtr.getValue(); try { + // This call actually takes a number divisible by the page size, + // not an arbitrary number of bytes. checkRc(LIB.mdb_env_set_mapsize(ptr, mapSize)); checkRc(LIB.mdb_env_set_maxdbs(ptr, maxDbs)); checkRc(LIB.mdb_env_set_maxreaders(ptr, maxReaders)); @@ -525,10 +549,26 @@ public Builder setMapSize(final long mapSize) { if (mapSize < 0) { throw new IllegalArgumentException("Negative value; overflow?"); } - this.mapSize = mapSize; + // A number divisible by the number of pages is expected. + this.mapSize = alignToWholePages(mapSize); return this; } + /** + * Aligns the allocated amount to whole pages. + * Note this is related to: + * https://github.com/LMDB/lmdb/blob/LMDB_0.9.24/libraries/liblmdb/mdb.c#L498:L514 + * https://github.com/LMDB/lmdb/blob/LMDB_0.9.24/libraries/liblmdb/lmdb.h#L813:L845 + * @param sizeb size in bytes + * @return size in bytes + */ + public static long alignToWholePages(final long sizeb) { + if ((sizeb % Builder.PAGE_SIZE) != 0) { + return (sizeb / Builder.PAGE_SIZE + 1) * Builder.PAGE_SIZE; + } + return sizeb; + } + /** * Sets the maximum number of databases (ie {@link Dbi}s permitted. * diff --git a/src/main/java/org/lmdbjava/Library.java b/src/main/java/org/lmdbjava/Library.java index 18d3e717..7ee71fea 100644 --- a/src/main/java/org/lmdbjava/Library.java +++ b/src/main/java/org/lmdbjava/Library.java @@ -29,6 +29,7 @@ import static java.util.Objects.requireNonNull; import static jnr.ffi.LibraryLoader.create; import static jnr.ffi.Runtime.getRuntime; +import static org.lmdbjava.Env.Builder.PAGE_SIZE; import java.io.File; import java.io.IOException; @@ -102,8 +103,12 @@ final class Library { final String libToLoad; final String arch = getProperty("os.arch"); - final boolean arch64 = "x64".equals(arch) || "amd64".equals(arch) + final boolean x64 = "x64".equals(arch) || "amd64".equals(arch) || "x86_64".equals(arch); + final boolean aarch64 = "aarch64".equals(arch); + final boolean ppc64le = "ppc64le".equals(arch); + + // final boolean arch64 = x64 || aarch64 || ppc64le; final String os = getProperty("os.name"); final boolean linux = os.toLowerCase(ENGLISH).startsWith("linux"); @@ -112,11 +117,15 @@ final class Library { if (SHOULD_USE_LIB) { libToLoad = getProperty(LMDB_NATIVE_LIB_PROP); - } else if (SHOULD_EXTRACT && arch64 && linux) { + } else if (SHOULD_EXTRACT && aarch64 && linux) { + libToLoad = extract("org/lmdbjava/lmdbjava-native-linux-aarch64.so"); + } else if (SHOULD_EXTRACT && ppc64le && linux) { + libToLoad = extract("org/lmdbjava/lmdbjava-native-linux-ppc64le.so"); + } else if (SHOULD_EXTRACT && x64 && linux) { libToLoad = extract("org/lmdbjava/lmdbjava-native-linux-x86_64.so"); - } else if (SHOULD_EXTRACT && arch64 && osx) { + } else if (SHOULD_EXTRACT && x64 && osx) { libToLoad = extract("org/lmdbjava/lmdbjava-native-osx-x86_64.dylib"); - } else if (SHOULD_EXTRACT && arch64 && windows) { + } else if (SHOULD_EXTRACT && x64 && windows) { libToLoad = extract("org/lmdbjava/lmdbjava-native-windows-x86_64.dll"); } else { libToLoad = LIB_NAME; @@ -145,7 +154,7 @@ private static String extract(final String name) { OutputStream out = Files.newOutputStream(file.toPath())) { requireNonNull(in, "Classpath resource not found"); int bytes; - final byte[] buffer = new byte[4_096]; + final byte[] buffer = new byte[PAGE_SIZE]; while (-1 != (bytes = in.read(buffer))) { out.write(buffer, 0, bytes); } diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index cc6cd0b6..8fc648b9 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -125,7 +125,14 @@ public void atMostTest() { public void before() throws IOException { final File path = tmp.newFile(); env = create() - .setMapSize(KIBIBYTES.toBytes(100)) + // 100 KB triggers Environment mapsize reached (-30792) on Txn.commit(Txn.java:110) + // on ppc64le and aarch64 systems with 65536 b big OS pages, i.e. 32768 b LMDB DB pages. + // 289 KB is just enough to hold the data without triggering the error. + // On a system with 4096 b OS pages, it translates to: + // Asked for : 295936 bytes, got 299008, which is 73 DB pages. + // On a system with 65536 n OS pages, it translates to: + // Asked for : 295936 bytes, got 327680, which is 10 DB pages. + .setMapSize(KIBIBYTES.toBytes(289)) .setMaxReaders(1) .setMaxDbs(1) .open(path, POSIX_MODE, MDB_NOSUBDIR); diff --git a/src/test/java/org/lmdbjava/DbiTest.java b/src/test/java/org/lmdbjava/DbiTest.java index b9bf453c..6c89612c 100644 --- a/src/test/java/org/lmdbjava/DbiTest.java +++ b/src/test/java/org/lmdbjava/DbiTest.java @@ -44,6 +44,7 @@ import static org.lmdbjava.DbiFlags.MDB_CREATE; import static org.lmdbjava.DbiFlags.MDB_DUPSORT; import static org.lmdbjava.DbiFlags.MDB_REVERSEKEY; +import static org.lmdbjava.Env.Builder.PAGE_SIZE; import static org.lmdbjava.Env.create; import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR; import static org.lmdbjava.GetOp.MDB_SET_KEY; @@ -457,7 +458,7 @@ public void stats() { assertThat(stat.entries, is(3L)); assertThat(stat.leafPages, is(1L)); assertThat(stat.overflowPages, is(0L)); - assertThat(stat.pageSize, is(4_096)); + assertThat(stat.pageSize, is(PAGE_SIZE)); } @Test(expected = MapFullException.class) diff --git a/src/test/java/org/lmdbjava/EnvTest.java b/src/test/java/org/lmdbjava/EnvTest.java index 3e9ece95..e131425d 100644 --- a/src/test/java/org/lmdbjava/EnvTest.java +++ b/src/test/java/org/lmdbjava/EnvTest.java @@ -30,6 +30,8 @@ import static org.lmdbjava.CopyFlags.MDB_CP_COMPACT; import static org.lmdbjava.DbiFlags.MDB_CREATE; import static org.lmdbjava.Env.Builder.MAX_READERS_DEFAULT; +import static org.lmdbjava.Env.Builder.PAGE_SIZE; +import static org.lmdbjava.Env.Builder.alignToWholePages; import static org.lmdbjava.Env.create; import static org.lmdbjava.Env.open; import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR; @@ -68,7 +70,8 @@ public void byteUnit() throws IOException { .setMapSize(MEBIBYTES.toBytes(1)) .open(path, MDB_NOSUBDIR)) { final EnvInfo info = env.info(); - assertThat(info.mapSize, is(MEBIBYTES.toBytes(1))); + // Allocation happens page by page, not in arbitrary chunks. + assertThat(info.mapSize, is(alignToWholePages(MEBIBYTES.toBytes(1)))); } } @@ -277,7 +280,7 @@ public void info() throws IOException { assertThat(info.lastPageNumber, is(1L)); assertThat(info.lastTransactionId, is(0L)); assertThat(info.mapAddress, is(0L)); - assertThat(info.mapSize, is(123_456L)); + assertThat(info.mapSize, is(alignToWholePages(123_456L))); assertThat(info.maxReaders, is(4)); assertThat(info.numReaders, is(0)); assertThat(info.toString(), containsString("maxReaders=")); @@ -335,7 +338,10 @@ public void setMapSize() throws IOException { final Random rnd = new Random(); try (Env env = create() .setMaxReaders(1) - .setMapSize(50_000) + // 50_000 was enough on system with OS and DB pages of 4096 b. + // On systems with OS pages 65536 b, i.e. DB pages of 32768 b, it is 196609 b. + // It translates to: Asked for: 196609 bytes, got 229376, which is 7 DB pages. + .setMapSize(PAGE_SIZE > 4096 ? 196_609 : 50_000) .setMaxDbs(1) .open(path)) { final Dbi db = env.openDbi(DB_1, MDB_CREATE); @@ -390,7 +396,7 @@ public void stats() throws IOException { assertThat(stat.entries, is(0L)); assertThat(stat.leafPages, is(0L)); assertThat(stat.overflowPages, is(0L)); - assertThat(stat.pageSize, is(4_096)); + assertThat(stat.pageSize, is(PAGE_SIZE)); assertThat(stat.toString(), containsString("pageSize=")); } } diff --git a/src/test/java/org/lmdbjava/TxnTest.java b/src/test/java/org/lmdbjava/TxnTest.java index dda4694e..bd9712ec 100644 --- a/src/test/java/org/lmdbjava/TxnTest.java +++ b/src/test/java/org/lmdbjava/TxnTest.java @@ -84,7 +84,10 @@ public void after() { public void before() throws IOException { path = tmp.newFile(); env = create() - .setMapSize(KIBIBYTES.toBytes(100)) + // 100 was enough on systems with OS and DB pages of 4096 b. + // For systems with OS pages 65536 b big, i.e. 32768 b DB pages, + // 353 KB is the minimum for this test. + .setMapSize(KIBIBYTES.toBytes(353)) .setMaxReaders(1) .setMaxDbs(2) .open(path, POSIX_MODE, MDB_NOSUBDIR);