Skip to content

Commit f895068

Browse files
andrii0lomakinandrii0lomakin
committed
Issue #7812, if we detect that we run under container we leave only 256 MB free
1 parent 323dc73 commit f895068

File tree

17 files changed

+65
-51
lines changed

17 files changed

+65
-51
lines changed

‎Jenkinsfile

100644100755
-21
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,6 @@ node("master") {
1313
}
1414

1515
try {
16-
// stage('Compile on Java7') {
17-
// docker.image("${mvnJdk7Image}")
18-
// .inside("${env.VOLUMES}") {
19-
// sh "${mvnHome}/bin/mvn --batch-mode -V -U clean compile -Dmaven.test.failure.ignore=true -Dsurefire.useFile=false"
20-
// }
21-
// }
22-
2316
stage('Run tests on Java7') {
2417
docker.image("${mvnJdk7Image}")
2518
.inside("${env.VOLUMES}") {
@@ -35,18 +28,6 @@ node("master") {
3528
}
3629
}
3730

38-
// stage('Run QA/Integration tests on Java8') {
39-
// docker.image("${mvnJdk8Image}")
40-
// .inside("${env.VOLUMES}") {
41-
// try {
42-
// sh "${mvnHome}/bin/mvn -f distribution/pom.xml clean install -Pqa"
43-
// } finally {
44-
// junit allowEmptyResults: true, testResults: '**/target/failsafe-reports/TEST-*.xml'
45-
//
46-
// }
47-
// }
48-
// }
49-
5031
stage('Publish Javadoc') {
5132
docker.image("${mvnJdk8Image}")
5233
.inside("${env.VOLUMES}") {
@@ -57,8 +38,6 @@ node("master") {
5738

5839
stage("Downstream projects") {
5940
build job: "orientdb-spatial-multibranch/${env.BRANCH_NAME}", wait: false
60-
//excluded: too long
61-
//build job: "orientdb-enterprise-multibranch/${env.BRANCH_NAME}", wait: false
6241
build job: "orientdb-security-multibranch/${env.BRANCH_NAME}", wait: false
6342
build job: "orientdb-neo4j-importer-multibranch/${env.BRANCH_NAME}", wait: false
6443
build job: "orientdb-teleporter-multibranch/${env.BRANCH_NAME}", wait: false

‎client/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
<osgi.export>com.orientechnologies.orient.client.*</osgi.export>
4242
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
4343
<argLine>
44+
-Xmx1024m
4445
-XX:MaxDirectMemorySize=512g
4546
-Dmemory.directMemory.trackMode=true
4647
-Djava.util.logging.manager=com.orientechnologies.common.log.OLogManager$ShutdownLogManager

‎core/pom.xml

+1-2
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,7 @@
4949
</osgi.export>
5050
<jna.version>4.0.0</jna.version>
5151
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
52-
<argLine>-ea -Xmx3072m -XX:MaxDirectMemorySize=512g
53-
-Dstorage.diskCache.bufferSize=4096
52+
<argLine>-ea -Xmx1024m -XX:MaxDirectMemorySize=512g
5453
-Dindex.flushAfterCreate=false
5554
-Dstorage.makeFullCheckpointAfterCreate=false
5655
-Dstorage.makeFullCheckpointAfterOpen=false

‎core/src/main/java/com/orientechnologies/common/jna/ONative.java

+19-3
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,10 @@ private ONative() {
5050
/**
5151
* @param printSteps Print all steps of discovering of memory limit in the log with {@code INFO} level.
5252
*
53-
* @return Amount of memory which are allowed to be consumed by application, if value <= 0 then limit is not set.
53+
* @return Amount of memory which are allowed to be consumed by application, and detects whether OrientDB instance is running
54+
* inside container. If <code>null</code> is returned then it was impossible to detect amount of memory on machine.
5455
*/
55-
public long getMemoryLimit(boolean printSteps) {
56+
public MemoryLimitResult getMemoryLimit(boolean printSteps) {
5657
//Perform several steps here:
5758
//1. Fetch physical size available on machine
5859
//2. Fetch soft limit
@@ -61,6 +62,7 @@ public long getMemoryLimit(boolean printSteps) {
6162
//5. Return the minimal value from the list of results
6263

6364
long memoryLimit = getPhysicalMemorySize();
65+
boolean insideContainer = false;
6466

6567
if (printSteps) {
6668
OLogManager.instance()
@@ -105,6 +107,7 @@ public long getMemoryLimit(boolean printSteps) {
105107
+ "process is running in container, will try to read root '%s' memory cgroup data", memoryCGroup, memoryCGroupRoot);
106108

107109
memoryCGroup = new File(memoryCGroupRoot);
110+
insideContainer = true;
108111
}
109112

110113
final long softMemoryLimit = fetchCGroupSoftMemoryLimit(memoryCGroup, printSteps);
@@ -124,7 +127,10 @@ public long getMemoryLimit(boolean printSteps) {
124127
OLogManager.instance().infoNoDb(this, "Memory limit for current process is not set");
125128
}
126129

127-
return memoryLimit;
130+
if (memoryLimit <= 0)
131+
return null;
132+
133+
return new MemoryLimitResult(memoryLimit, insideContainer);
128134
}
129135

130136
private long updateMemoryLimit(long memoryLimit, long newMemoryLimit) {
@@ -400,4 +406,14 @@ private static long convertToGB(long bytes) {
400406

401407
return bytes / (1024 * 1024 * 1024);
402408
}
409+
410+
public final class MemoryLimitResult {
411+
public final long memoryLimit;
412+
public final boolean insideContainer;
413+
414+
public MemoryLimitResult(long memoryLimit, boolean insideContainer) {
415+
this.memoryLimit = memoryLimit;
416+
this.insideContainer = insideContainer;
417+
}
418+
}
403419
}

‎core/src/main/java/com/orientechnologies/common/util/OMemory.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -99,14 +99,14 @@ private static long getMaxCacheMemorySize() {
9999
* Checks the direct memory configuration and emits a warning if configuration is invalid.
100100
*/
101101
public static void checkDirectMemoryConfiguration() {
102-
final long physicalMemory = ONative.instance().getMemoryLimit(false);
102+
final ONative.MemoryLimitResult physicalMemory = ONative.instance().getMemoryLimit(false);
103103
final long maxDirectMemory = getConfiguredMaxDirectMemory();
104104

105105
if (maxDirectMemory == -1) {
106-
if (physicalMemory > 0)
106+
if (physicalMemory != null)
107107
OLogManager.instance().warnNoDb(OMemory.class, "MaxDirectMemorySize JVM option is not set or has invalid value, "
108-
+ "that may cause out of memory errors. Please set the -XX:MaxDirectMemorySize=" + physicalMemory / (1024 * 1024)
109-
+ "m option when you start the JVM.");
108+
+ "that may cause out of memory errors. Please set the -XX:MaxDirectMemorySize=" + physicalMemory.memoryLimit / (1024
109+
* 1024) + "m option when you start the JVM.");
110110
else
111111
OLogManager.instance().warnNoDb(OMemory.class, "MaxDirectMemorySize JVM option is not set or has invalid value, "
112112
+ "that may cause out of memory errors. Please set the -XX:MaxDirectMemorySize=<SIZE>m JVM option "
@@ -123,7 +123,7 @@ public static void checkDirectMemoryConfiguration() {
123123
public static void checkCacheMemoryConfiguration() {
124124
final long maxHeapSize = Runtime.getRuntime().maxMemory();
125125
final long maxCacheSize = getMaxCacheMemorySize();
126-
final long physicalMemory = ONative.instance().getMemoryLimit(false);
126+
final ONative.MemoryLimitResult physicalMemory = ONative.instance().getMemoryLimit(false);
127127
final long maxDirectMemory = getConfiguredMaxDirectMemory();
128128

129129
if (maxDirectMemory != -1 && maxCacheSize > maxDirectMemory)
@@ -133,7 +133,7 @@ public static void checkCacheMemoryConfiguration() {
133133
+ "maximum direct memory size or storage.diskCache.bufferSize OrientDB option to lower memory requirements of the "
134134
+ "cache.");
135135

136-
if (maxHeapSize != Long.MAX_VALUE && physicalMemory > 0 && maxHeapSize + maxCacheSize > physicalMemory)
136+
if (maxHeapSize != Long.MAX_VALUE && physicalMemory != null && maxHeapSize + maxCacheSize > physicalMemory.memoryLimit)
137137
OLogManager.instance().warnNoDb(OMemory.class,
138138
"The sum of the configured JVM maximum heap size (" + maxHeapSize + " bytes) " + "and the OrientDB maximum cache size ("
139139
+ maxCacheSize + " bytes) is larger than the available physical memory size " + "(" + physicalMemory

‎core/src/main/java/com/orientechnologies/orient/core/engine/OMemoryAndLocalPaginatedEnginesInitializer.java

+20-8
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,12 @@ else if (jvmMaxMemory > 512 * OFileUtils.MEGABYTE)
8484
}
8585

8686
private void configureDefaultDiskCacheSize() {
87-
final long osMemory = ONative.instance().getMemoryLimit(true);
87+
final ONative.MemoryLimitResult osMemory = ONative.instance().getMemoryLimit(true);
88+
if (osMemory == null) {
89+
OLogManager.instance().warnNoDb(this, "Can not determine amount of memory installed on machine, default values will be used");
90+
return;
91+
}
92+
8893
final long jvmMaxMemory = OMemory.getCappedRuntimeMaxMemory(2L * 1024 * 1024 * 1024 /* 2GB */);
8994
final long maxDirectMemory = OMemory.getConfiguredMaxDirectMemory();
9095

@@ -94,7 +99,7 @@ private void configureDefaultDiskCacheSize() {
9499
OLogManager.instance().infoNoDb(this,
95100
"OrientDB auto-config DISKCACHE=%,dMB (heap=%,dMB direct=%,dMB os=%,dMB), assuming maximum direct memory size "
96101
+ "equals to maximum JVM heap size", diskCacheInMB, jvmMaxMemory / 1024 / 1024, jvmMaxMemory / 1024 / 1024,
97-
osMemory / 1024 / 1024);
102+
osMemory.memoryLimit / 1024 / 1024);
98103
OGlobalConfiguration.DISK_CACHE_SIZE.setValue(diskCacheInMB);
99104
OGlobalConfiguration.MEMORY_CHUNK_SIZE
100105
.setValue(Math.min(diskCacheInMB * 1024 * 1024, OGlobalConfiguration.MEMORY_CHUNK_SIZE.getValueAsLong()));
@@ -103,13 +108,20 @@ private void configureDefaultDiskCacheSize() {
103108

104109
final long maxDirectMemoryInMB = maxDirectMemory / 1024 / 1024;
105110

106-
// DISK-CACHE IN MB = OS MEMORY - MAX HEAP JVM MEMORY - 2 GB
107-
long diskCacheInMB = (osMemory - jvmMaxMemory) / (1024 * 1024) - 2 * 1024;
111+
long diskCacheInMB;
112+
if (osMemory.insideContainer) {
113+
//DISK-CACHE IN MB = OS MEMORY - MAX HEAP JVM MEMORY - 256 MB
114+
diskCacheInMB = (osMemory.memoryLimit - jvmMaxMemory) / (1024 * 1024) - 256;
115+
} else {
116+
// DISK-CACHE IN MB = OS MEMORY - MAX HEAP JVM MEMORY - 2 GB
117+
diskCacheInMB = (osMemory.memoryLimit - jvmMaxMemory) / (1024 * 1024) - 2 * 1024;
118+
}
119+
108120
if (diskCacheInMB > 0) {
109121
diskCacheInMB = Math.min(diskCacheInMB, maxDirectMemoryInMB);
110122
OLogManager.instance()
111123
.infoNoDb(this, "OrientDB auto-config DISKCACHE=%,dMB (heap=%,dMB direct=%,dMB os=%,dMB)", diskCacheInMB,
112-
jvmMaxMemory / 1024 / 1024, maxDirectMemoryInMB, osMemory / 1024 / 1024);
124+
jvmMaxMemory / 1024 / 1024, maxDirectMemoryInMB, osMemory.memoryLimit / 1024 / 1024);
113125

114126
OGlobalConfiguration.DISK_CACHE_SIZE.setValue(diskCacheInMB);
115127
OGlobalConfiguration.MEMORY_CHUNK_SIZE
@@ -119,14 +131,14 @@ private void configureDefaultDiskCacheSize() {
119131
diskCacheInMB = Math.min(O2QCache.MIN_CACHE_SIZE, maxDirectMemoryInMB);
120132
OLogManager.instance().warnNoDb(this,
121133
"Not enough physical memory available for DISKCACHE: %,dMB (heap=%,dMB direct=%,dMB). Set lower Maximum Heap (-Xmx "
122-
+ "setting on JVM) and restart OrientDB. Now running with DISKCACHE=" + diskCacheInMB + "MB", osMemory / 1024 / 1024,
123-
jvmMaxMemory / 1024 / 1024, maxDirectMemoryInMB);
134+
+ "setting on JVM) and restart OrientDB. Now running with DISKCACHE=" + diskCacheInMB + "MB",
135+
osMemory.memoryLimit / 1024 / 1024, jvmMaxMemory / 1024 / 1024, maxDirectMemoryInMB);
124136
OGlobalConfiguration.DISK_CACHE_SIZE.setValue(diskCacheInMB);
125137
OGlobalConfiguration.MEMORY_CHUNK_SIZE
126138
.setValue(Math.min(diskCacheInMB * 1024 * 1024, OGlobalConfiguration.MEMORY_CHUNK_SIZE.getValueAsLong()));
127139

128140
OLogManager.instance().infoNoDb(this, "OrientDB config DISKCACHE=%,dMB (heap=%,dMB direct=%,dMB os=%,dMB)", diskCacheInMB,
129-
jvmMaxMemory / 1024 / 1024, maxDirectMemoryInMB, osMemory / 1024 / 1024);
141+
jvmMaxMemory / 1024 / 1024, maxDirectMemoryInMB, osMemory.memoryLimit / 1024 / 1024);
130142
}
131143
}
132144
}

‎distributed/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
<osgi.import>com.hazelcast.*;resolution:=optional,com.tinkerpop.*;resolution:=optional,*</osgi.import>
4545
<osgi.export>com.orientechnologies.orient.server.hazelcast.*,com.orientechnologies.orient.server.distributed.impl.*</osgi.export>
4646
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
47-
<argLine>-ea -Xmx2048m -XX:MaxDirectMemorySize=512g -Dstorage.diskCache.bufferSize=4096 -Dindex.flushAfterCreate=false
47+
<argLine>-ea -Xmx1024m -XX:MaxDirectMemorySize=512g -Dindex.flushAfterCreate=false
4848
-Dstorage.makeFullCheckpointAfterCreate=false
4949
-Dmemory.directMemory.trackMode=true
5050
-Dstorage.makeFullCheckpointAfterOpen=false

‎distribution/pom.xml

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@
159159
<executable>${java.home}/bin/java</executable>
160160
<workingDirectory>${project.basedir}</workingDirectory>
161161
<arguments>
162+
<argument>-Xmx1024m</argument>
162163
<argument>-XX:MaxDirectMemorySize=512g</argument>
163164
<argument>-classpath</argument>
164165
<classpath/>

‎etl/pom.xml

+1-2
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,7 @@
3434

3535
<properties>
3636
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
37-
<argLine>-ea -Xmx3072m -XX:MaxDirectMemorySize=512g
38-
-Dstorage.diskCache.bufferSize=4096
37+
<argLine>-ea -Xmx1024m -XX:MaxDirectMemorySize=512g
3938
-Dindex.flushAfterCreate=false
4039
-Dstorage.makeFullCheckpointAfterCreate=false
4140
-Dstorage.makeFullCheckpointAfterOpen=false

‎graphdb/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
</osgi.export>
4545

4646
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
47-
<argLine>-ea -Xmx3G -XX:MaxDirectMemorySize=512g -Xss2048k -Dstorage.diskCache.bufferSize=4096
47+
<argLine>-ea -Xmx1024m -XX:MaxDirectMemorySize=512g
4848
-Dindex.flushAfterCreate=false
4949
-Dmemory.directMemory.trackMode=true
5050
-Dstorage.makeFullCheckpointAfterCreate=false

‎jdbc/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636

3737
<properties>
3838
<argLine>
39+
-Xmx1024m
3940
-XX:MaxDirectMemorySize=512g
4041
-Dmemory.directMemory.trackMode=true
4142
-Djava.util.logging.manager=com.orientechnologies.common.log.OLogManager$ShutdownLogManager

‎lucene/pom.xml

+1-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,7 @@
3737
<lucene.version>5.3.2</lucene.version>
3838
<geotools.version>13.2</geotools.version>
3939
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
40-
<argLine>-ea -XX:MaxDirectMemorySize=512g -Xmx2048m
41-
-Dstorage.diskCache.bufferSize=4096
40+
<argLine>-ea -XX:MaxDirectMemorySize=512g -Xmx1024m
4241
-Dindex.flushAfterCreate=false
4342
-Dstorage.makeFullCheckpointAfterCreate=false
4443
-Dstorage.makeFullCheckpointAfterClusterCreate=false -Dstorage.wal.syncOnPageFlush=false

‎object/pom.xml

+1-2
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,7 @@
119119
<import.package>*</import.package>
120120
<export.package>com.orientechnologies.orient.object.*</export.package>
121121
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
122-
<argLine>-ea -Xmx2048m -XX:MaxDirectMemorySize=512g
123-
-Dstorage.diskCache.bufferSize=4096
122+
<argLine>-ea -Xmx1024m -XX:MaxDirectMemorySize=512g
124123
-Dmemory.directMemory.trackMode=true
125124
-Dstorage.makeFullCheckpointAfterCreate=false
126125
-Dstorage.makeFullCheckpointAfterOpen=false

‎server/pom.xml

+1-2
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,8 @@
3737
<osgi.export>com.orientechnologies.orient.server.*</osgi.export>
3838
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
3939
<argLine>-ea
40-
-Xmx2048m
40+
-Xmx1024m
4141
-XX:MaxDirectMemorySize=512g
42-
-Dstorage.diskCache.bufferSize=4096
4342
-Dsecurity.userPasswordSaltIterations=10
4443
-Dmemory.directMemory.trackMode=true
4544
-Djava.util.logging.manager=com.orientechnologies.common.log.OLogManager$ShutdownLogManager

‎test-commons/pom.xml

100644100755
+8
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@
1414
<name>OrientDB Test Commons</name>
1515
<properties>
1616
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
17+
<argLine>
18+
-ea
19+
-Xmx1024m
20+
-XX:MaxDirectMemorySize=512g
21+
-Dmemory.directMemory.trackMode=true
22+
-Djava.util.logging.manager=com.orientechnologies.common.log.OLogManager$ShutdownLogManager
23+
-Dstorage.diskCache.checksumMode=storeAndThrow
24+
</argLine>
1725
</properties>
1826

1927
<dependencies>

‎tests/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
<properties>
2929
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
3030
<argLine>
31-
-Xmx3072m
31+
-Xmx1024m
3232
-XX:MaxDirectMemorySize=512g
3333
-Dmemory.directMemory.trackMode=true
3434
-Djava.util.logging.manager=com.orientechnologies.common.log.OLogManager$ShutdownLogManager

‎tools/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
<jar.manifest.mainclass>com.orientechnologies.orient.console.OConsoleDatabaseApp</jar.manifest.mainclass>
4545
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
4646
<argLine>
47+
-Xmx1024m
4748
-XX:MaxDirectMemorySize=512g
4849
-Dmemory.directMemory.trackMode=true
4950
-Djava.util.logging.manager=com.orientechnologies.common.log.OLogManager$ShutdownLogManager

0 commit comments

Comments
 (0)