Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Ports to aarch64 and ppc64le systems with OS pages 65536 bytes big #165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Karm wants to merge 2 commits into from
Closed

Ports to aarch64 and ppc64le systems with OS pages 65536 bytes big #165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bc0ccbc
Added usage of aarch64 build of native llmdb
judovana Aug 13, 2020
dacfdc8
Porting to systems with large pages
Karm Sep 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 122 additions & 0 deletions 122 pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,4 +216,126 @@
<system>GitHub Actions</system>
<url>https://github.com/${github.org}/${github.repo}/actions</url>
</ciManagement>
<profiles>
<profile>
<id>linux-aarch64</id>
<activation>
<os>
<name>Linux</name>
<family>unix</family>
<arch>aarch64</arch>
</os>
</activation>
<dependencies>
<dependency>
<groupId>org.lmdbjava</groupId>
<artifactId>lmdbjava-native-linux-aarch64</artifactId>
<version>0.9.24-2-SNAPSHOT</version>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<usedDependencies>
<usedDependency>org.lmdbjava:lmdbjava-native-linux-aarch64</usedDependency>
<usedDependency>org.lmdbjava:lmdbjava-native-linux-x86_64</usedDependency>
<usedDependency>org.lmdbjava:lmdbjava-native-windows-x86_64</usedDependency>
<usedDependency>org.lmdbjava:lmdbjava-native-osx-x86_64</usedDependency>
</usedDependencies>
<ignoredDependencies>
<ignoredDependency>com.github.jnr:jffi</ignoredDependency>
</ignoredDependencies>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<id>lmdbjava-shade</id>
<goals>
<goal>shade</goal>
</goals>
<phase>package</phase>
<configuration>
<artifactSet>
<includes>
<include>org.lmdbjava:lmdbjava-native-linux-aarch64</include>
<include>org.lmdbjava:lmdbjava-native-linux-x86_64</include>
<include>org.lmdbjava:lmdbjava-native-windows-x86_64</include>
<include>org.lmdbjava:lmdbjava-native-osx-x86_64</include>
</includes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>linux-ppc64le</id>
<activation>
<os>
<name>Linux</name>
<family>unix</family>
<arch>ppc64le</arch>
</os>
</activation>
<dependencies>
<dependency>
<groupId>org.lmdbjava</groupId>
<artifactId>lmdbjava-native-linux-ppc64le</artifactId>
<version>0.9.24-2-SNAPSHOT</version>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<usedDependencies>
<usedDependency>org.lmdbjava:lmdbjava-native-linux-ppc64le</usedDependency>
<usedDependency>org.lmdbjava:lmdbjava-native-linux-x86_64</usedDependency>
<usedDependency>org.lmdbjava:lmdbjava-native-windows-x86_64</usedDependency>
<usedDependency>org.lmdbjava:lmdbjava-native-osx-x86_64</usedDependency>
</usedDependencies>
<ignoredDependencies>
<ignoredDependency>com.github.jnr:jffi</ignoredDependency>
</ignoredDependencies>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<id>lmdbjava-shade</id>
<goals>
<goal>shade</goal>
</goals>
<phase>package</phase>
<configuration>
<artifactSet>
<includes>
<include>org.lmdbjava:lmdbjava-native-linux-ppc64le</include>
<include>org.lmdbjava:lmdbjava-native-linux-x86_64</include>
<include>org.lmdbjava:lmdbjava-native-windows-x86_64</include>
<include>org.lmdbjava:lmdbjava-native-osx-x86_64</include>
</includes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
46 changes: 43 additions & 3 deletions 46 src/main/java/org/lmdbjava/Env.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,8 @@ public List<byte[]> getDbiNames() {
* @param mapSize the new size, in bytes
*/
public void setMapSize(final long mapSize) {
checkRc(LIB.mdb_env_set_mapsize(ptr, mapSize));
// A number divisible by the number of pages is expected.
checkRc(LIB.mdb_env_set_mapsize(ptr, Builder.alignToWholePages(mapSize)));
}

/**
Expand Down Expand Up @@ -417,6 +418,25 @@ private void validatePath(final File path) {
validateDirectoryEmpty(path);
}

public static int getPageSize() {
int pageSize = 0;
for (final String s: new String[]{
System.getenv().get("PAGE_SIZE"),
System.getProperty("PAGE_SIZE")}) {
if (s != null && s.length() > 0) {
pageSize = Integer.parseInt(s);
break;
}
}
// This move is closely dependent on this logic:
// @see https://github.com/LMDB/lmdb/blob/LMDB_0.9.24/libraries/liblmdb/mdb.c#L498:L514
if (pageSize >= 32_768) {
return 32_768;
}
// Silently defaulting to 4096
return 4_096;
}

/**
* Object has already been closed and the operation is therefore prohibited.
*/
Expand Down Expand Up @@ -454,8 +474,10 @@ public AlreadyOpenException() {
*/
public static final class Builder<T> {

static final int PAGE_SIZE = getPageSize();
static final int MAX_READERS_DEFAULT = 126;
private long mapSize = 1_024 * 1_024;
// On a system with pages 4096 bytes big, this translates to default max map size of 4 MB
private long mapSize = PAGE_SIZE * 1_024;
private int maxDbs = 1;
private int maxReaders = MAX_READERS_DEFAULT;
private boolean opened;
Expand Down Expand Up @@ -486,6 +508,8 @@ public Env<T> open(final File path, final int mode,
checkRc(LIB.mdb_env_create(envPtr));
final Pointer ptr = envPtr.getValue();
try {
// This call actually takes a number divisible by the page size,
// not an arbitrary number of bytes.
checkRc(LIB.mdb_env_set_mapsize(ptr, mapSize));
checkRc(LIB.mdb_env_set_maxdbs(ptr, maxDbs));
checkRc(LIB.mdb_env_set_maxreaders(ptr, maxReaders));
Expand Down Expand Up @@ -525,10 +549,26 @@ public Builder<T> setMapSize(final long mapSize) {
if (mapSize < 0) {
throw new IllegalArgumentException("Negative value; overflow?");
}
this.mapSize = mapSize;
// A number divisible by the number of pages is expected.
this.mapSize = alignToWholePages(mapSize);
return this;
}

/**
* Aligns the allocated amount to whole pages.
* Note this is related to:
* https://github.com/LMDB/lmdb/blob/LMDB_0.9.24/libraries/liblmdb/mdb.c#L498:L514
* https://github.com/LMDB/lmdb/blob/LMDB_0.9.24/libraries/liblmdb/lmdb.h#L813:L845
* @param sizeb size in bytes
* @return size in bytes
*/
public static long alignToWholePages(final long sizeb) {
if ((sizeb % Builder.PAGE_SIZE) != 0) {
return (sizeb / Builder.PAGE_SIZE + 1) * Builder.PAGE_SIZE;
}
return sizeb;
}

/**
* Sets the maximum number of databases (ie {@link Dbi}s permitted.
*
Expand Down
19 changes: 14 additions & 5 deletions 19 src/main/java/org/lmdbjava/Library.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import static java.util.Objects.requireNonNull;
import static jnr.ffi.LibraryLoader.create;
import static jnr.ffi.Runtime.getRuntime;
import static org.lmdbjava.Env.Builder.PAGE_SIZE;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -102,8 +103,12 @@ final class Library {
final String libToLoad;

final String arch = getProperty("os.arch");
final boolean arch64 = "x64".equals(arch) || "amd64".equals(arch)
final boolean x64 = "x64".equals(arch) || "amd64".equals(arch)
|| "x86_64".equals(arch);
final boolean aarch64 = "aarch64".equals(arch);
final boolean ppc64le = "ppc64le".equals(arch);

// final boolean arch64 = x64 || aarch64 || ppc64le;

final String os = getProperty("os.name");
final boolean linux = os.toLowerCase(ENGLISH).startsWith("linux");
Expand All @@ -112,11 +117,15 @@ final class Library {

if (SHOULD_USE_LIB) {
libToLoad = getProperty(LMDB_NATIVE_LIB_PROP);
} else if (SHOULD_EXTRACT && arch64 && linux) {
} else if (SHOULD_EXTRACT && aarch64 && linux) {
libToLoad = extract("org/lmdbjava/lmdbjava-native-linux-aarch64.so");
} else if (SHOULD_EXTRACT && ppc64le && linux) {
libToLoad = extract("org/lmdbjava/lmdbjava-native-linux-ppc64le.so");
} else if (SHOULD_EXTRACT && x64 && linux) {
libToLoad = extract("org/lmdbjava/lmdbjava-native-linux-x86_64.so");
} else if (SHOULD_EXTRACT && arch64 && osx) {
} else if (SHOULD_EXTRACT && x64 && osx) {
libToLoad = extract("org/lmdbjava/lmdbjava-native-osx-x86_64.dylib");
} else if (SHOULD_EXTRACT && arch64 && windows) {
} else if (SHOULD_EXTRACT && x64 && windows) {
libToLoad = extract("org/lmdbjava/lmdbjava-native-windows-x86_64.dll");
} else {
libToLoad = LIB_NAME;
Expand Down Expand Up @@ -145,7 +154,7 @@ private static String extract(final String name) {
OutputStream out = Files.newOutputStream(file.toPath())) {
requireNonNull(in, "Classpath resource not found");
int bytes;
final byte[] buffer = new byte[4_096];
final byte[] buffer = new byte[PAGE_SIZE];
while (-1 != (bytes = in.read(buffer))) {
out.write(buffer, 0, bytes);
}
Expand Down
9 changes: 8 additions & 1 deletion 9 src/test/java/org/lmdbjava/CursorIterableTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,14 @@ public void atMostTest() {
public void before() throws IOException {
final File path = tmp.newFile();
env = create()
.setMapSize(KIBIBYTES.toBytes(100))
// 100 KB triggers Environment mapsize reached (-30792) on Txn.commit(Txn.java:110)
// on ppc64le and aarch64 systems with 65536 b big OS pages, i.e. 32768 b LMDB DB pages.
// 289 KB is just enough to hold the data without triggering the error.
// On a system with 4096 b OS pages, it translates to:
// Asked for : 295936 bytes, got 299008, which is 73 DB pages.
// On a system with 65536 n OS pages, it translates to:
// Asked for : 295936 bytes, got 327680, which is 10 DB pages.
.setMapSize(KIBIBYTES.toBytes(289))
.setMaxReaders(1)
.setMaxDbs(1)
.open(path, POSIX_MODE, MDB_NOSUBDIR);
Expand Down
3 changes: 2 additions & 1 deletion 3 src/test/java/org/lmdbjava/DbiTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import static org.lmdbjava.DbiFlags.MDB_CREATE;
import static org.lmdbjava.DbiFlags.MDB_DUPSORT;
import static org.lmdbjava.DbiFlags.MDB_REVERSEKEY;
import static org.lmdbjava.Env.Builder.PAGE_SIZE;
import static org.lmdbjava.Env.create;
import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR;
import static org.lmdbjava.GetOp.MDB_SET_KEY;
Expand Down Expand Up @@ -457,7 +458,7 @@ public void stats() {
assertThat(stat.entries, is(3L));
assertThat(stat.leafPages, is(1L));
assertThat(stat.overflowPages, is(0L));
assertThat(stat.pageSize, is(4_096));
assertThat(stat.pageSize, is(PAGE_SIZE));
}

@Test(expected = MapFullException.class)
Expand Down
14 changes: 10 additions & 4 deletions 14 src/test/java/org/lmdbjava/EnvTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
import static org.lmdbjava.CopyFlags.MDB_CP_COMPACT;
import static org.lmdbjava.DbiFlags.MDB_CREATE;
import static org.lmdbjava.Env.Builder.MAX_READERS_DEFAULT;
import static org.lmdbjava.Env.Builder.PAGE_SIZE;
import static org.lmdbjava.Env.Builder.alignToWholePages;
import static org.lmdbjava.Env.create;
import static org.lmdbjava.Env.open;
import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR;
Expand Down Expand Up @@ -68,7 +70,8 @@ public void byteUnit() throws IOException {
.setMapSize(MEBIBYTES.toBytes(1))
.open(path, MDB_NOSUBDIR)) {
final EnvInfo info = env.info();
assertThat(info.mapSize, is(MEBIBYTES.toBytes(1)));
// Allocation happens page by page, not in arbitrary chunks.
assertThat(info.mapSize, is(alignToWholePages(MEBIBYTES.toBytes(1))));
}
}

Expand Down Expand Up @@ -277,7 +280,7 @@ public void info() throws IOException {
assertThat(info.lastPageNumber, is(1L));
assertThat(info.lastTransactionId, is(0L));
assertThat(info.mapAddress, is(0L));
assertThat(info.mapSize, is(123_456L));
assertThat(info.mapSize, is(alignToWholePages(123_456L)));
assertThat(info.maxReaders, is(4));
assertThat(info.numReaders, is(0));
assertThat(info.toString(), containsString("maxReaders="));
Expand Down Expand Up @@ -335,7 +338,10 @@ public void setMapSize() throws IOException {
final Random rnd = new Random();
try (Env<ByteBuffer> env = create()
.setMaxReaders(1)
.setMapSize(50_000)
// 50_000 was enough on system with OS and DB pages of 4096 b.
// On systems with OS pages 65536 b, i.e. DB pages of 32768 b, it is 196609 b.
// It translates to: Asked for: 196609 bytes, got 229376, which is 7 DB pages.
.setMapSize(PAGE_SIZE > 4096 ? 196_609 : 50_000)
.setMaxDbs(1)
.open(path)) {
final Dbi<ByteBuffer> db = env.openDbi(DB_1, MDB_CREATE);
Expand Down Expand Up @@ -390,7 +396,7 @@ public void stats() throws IOException {
assertThat(stat.entries, is(0L));
assertThat(stat.leafPages, is(0L));
assertThat(stat.overflowPages, is(0L));
assertThat(stat.pageSize, is(4_096));
assertThat(stat.pageSize, is(PAGE_SIZE));
assertThat(stat.toString(), containsString("pageSize="));
}
}
Expand Down
5 changes: 4 additions & 1 deletion 5 src/test/java/org/lmdbjava/TxnTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,10 @@ public void after() {
public void before() throws IOException {
path = tmp.newFile();
env = create()
.setMapSize(KIBIBYTES.toBytes(100))
// 100 was enough on systems with OS and DB pages of 4096 b.
// For systems with OS pages 65536 b big, i.e. 32768 b DB pages,
// 353 KB is the minimum for this test.
.setMapSize(KIBIBYTES.toBytes(353))
.setMaxReaders(1)
.setMaxDbs(2)
.open(path, POSIX_MODE, MDB_NOSUBDIR);
Expand Down
Morty Proxy This is a proxified and sanitized view of the page, visit original site.