Skip to content

Commit 4e857c3

Browse files
moschemosche
authored
Support entitlements in internal cluster tests (#130710) (#131455)
* To prevent an implicit grant-all if storing node homes inside the Java temp dir, the temporary folder of ESTestCase is configured separately from the Java temp dir in internalClusterTests (by means of the system property tempDir, see TestRuleTemporaryFilesCleanup) * Move ReloadingDatabasesWhilePerformingGeoLookupsIT from internalClusterTest to test, file permissions in internalClusterTest are stricter on the lucene tempDir
1 parent 36440ef commit 4e857c3

File tree

12 files changed

+148
-55
lines changed

12 files changed

+148
-55
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

Lines changed: 41 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.io.File;
3535
import java.util.List;
3636
import java.util.Map;
37+
import java.util.Set;
3738
import java.util.stream.Stream;
3839

3940
import javax.inject.Inject;
@@ -50,6 +51,8 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
5051

5152
public static final String DUMP_OUTPUT_ON_FAILURE_PROP_NAME = "dumpOutputOnFailure";
5253

54+
public static final Set<String> TEST_TASKS_WITH_ENTITLEMENTS = Set.of("test", "internalClusterTest");
55+
5356
@Inject
5457
protected abstract ProviderFactory getProviderFactory();
5558

@@ -174,14 +177,23 @@ public void execute(Task t) {
174177
nonInputProperties.systemProperty("workspace.dir", Util.locateElasticsearchWorkspace(project.getGradle()));
175178
// we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
176179
nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
180+
if (test.getName().equals("internalClusterTest")) {
181+
// configure a node home directory independent of the Java temp dir so that entitlements can be properly enforced
182+
nonInputProperties.systemProperty("tempDir", test.getWorkingDir().toPath().resolve("nodesTemp"));
183+
}
177184

178185
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
179186
SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME);
180187
SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
181-
if ("test".equals(test.getName()) && mainSourceSet != null && testSourceSet != null) {
188+
SourceSet internalClusterTestSourceSet = sourceSets.findByName("internalClusterTest");
189+
190+
if (TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) && mainSourceSet != null && testSourceSet != null) {
182191
FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
183192
FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
184-
FileCollection testOnlyFiles = testRuntime.minus(mainRuntime);
193+
FileCollection internalClusterTestRuntime = ("internalClusterTest".equals(test.getName())
194+
&& internalClusterTestSourceSet != null) ? internalClusterTestSourceSet.getRuntimeClasspath() : project.files();
195+
FileCollection testOnlyFiles = testRuntime.plus(internalClusterTestRuntime).minus(mainRuntime);
196+
185197
test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath()));
186198
}
187199

@@ -241,14 +253,15 @@ public void execute(Task t) {
241253
* Computes and sets the {@code --patch-module=java.base} and {@code --add-opens=java.base} JVM command line options.
242254
*/
243255
private void configureJavaBaseModuleOptions(Project project) {
244-
project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
245-
FileCollection patchedImmutableCollections = patchedImmutableCollections(project);
256+
project.getTasks().withType(Test.class).configureEach(test -> {
257+
// patch immutable collections only for "test" task
258+
FileCollection patchedImmutableCollections = test.getName().equals("test") ? patchedImmutableCollections(project) : null;
246259
if (patchedImmutableCollections != null) {
247260
test.getInputs().files(patchedImmutableCollections);
248261
test.systemProperty("tests.hackImmutableCollections", "true");
249262
}
250263

251-
FileCollection entitlementBridge = entitlementBridge(project);
264+
FileCollection entitlementBridge = TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()) ? entitlementBridge(project) : null;
252265
if (entitlementBridge != null) {
253266
test.getInputs().files(entitlementBridge);
254267
}
@@ -312,27 +325,30 @@ private static void configureEntitlements(Project project) {
312325
}
313326
FileCollection bridgeFiles = bridgeConfig;
314327

315-
project.getTasks().withType(Test.class).configureEach(test -> {
316-
// See also SystemJvmOptions.maybeAttachEntitlementAgent.
317-
318-
// Agent
319-
if (agentFiles.isEmpty() == false) {
320-
test.getInputs().files(agentFiles);
321-
test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
322-
test.systemProperty("jdk.attach.allowAttachSelf", true);
323-
}
328+
project.getTasks()
329+
.withType(Test.class)
330+
.matching(test -> TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()))
331+
.configureEach(test -> {
332+
// See also SystemJvmOptions.maybeAttachEntitlementAgent.
333+
334+
// Agent
335+
if (agentFiles.isEmpty() == false) {
336+
test.getInputs().files(agentFiles);
337+
test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
338+
test.systemProperty("jdk.attach.allowAttachSelf", true);
339+
}
324340

325-
// Bridge
326-
if (bridgeFiles.isEmpty() == false) {
327-
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
328-
test.getInputs().files(bridgeFiles);
329-
// Tests may not be modular, but the JDK still is
330-
test.jvmArgs(
331-
"--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
332-
+ modulesContainingEntitlementInstrumentation
333-
);
334-
}
335-
});
341+
// Bridge
342+
if (bridgeFiles.isEmpty() == false) {
343+
String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
344+
test.getInputs().files(bridgeFiles);
345+
// Tests may not be modular, but the JDK still is
346+
test.jvmArgs(
347+
"--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
348+
+ modulesContainingEntitlementInstrumentation
349+
);
350+
}
351+
});
336352
}
337353

338354
}

‎build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,12 @@ public void apply(Project project) {
5858
});
5959

6060
if (project.getRootProject().getName().equals("elasticsearch")) {
61-
project.getTasks().withType(Test.class).matching(test -> List.of("test").contains(test.getName())).configureEach(test -> {
62-
test.systemProperty("es.entitlement.enableForTests", "true");
63-
});
61+
project.getTasks()
62+
.withType(Test.class)
63+
.matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
64+
.configureEach(test -> {
65+
test.systemProperty("es.entitlement.enableForTests", "true");
66+
});
6467
}
6568
}
6669
}

‎libs/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ configure(childProjects.values()) {
5050
// Omit oddball libraries that aren't in server.
5151
def nonServerLibs = ['plugin-scanner']
5252
if (false == nonServerLibs.contains(project.name)) {
53-
project.getTasks().withType(Test.class).matching(test -> ['test'].contains(test.name)).configureEach(test -> {
53+
project.getTasks().withType(Test.class).matching(test -> ['test', 'internalClusterTest'].contains(test.name)).configureEach(test -> {
5454
test.systemProperty('es.entitlement.enableForTests', 'true')
5555
})
5656
}

‎modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java renamed to ‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
// 'WindowsFS.checkDeleteAccess(...)').
5858
}
5959
)
60-
public class ReloadingDatabasesWhilePerformingGeoLookupsIT extends ESTestCase {
60+
public class ReloadingDatabasesWhilePerformingGeoLookupsTests extends ESTestCase {
6161

6262
/**
6363
* This tests essentially verifies that a Maxmind database reader doesn't fail with:

‎server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.gateway.PersistedClusterStateService;
2525
import org.elasticsearch.node.Node;
2626
import org.elasticsearch.test.ESIntegTestCase;
27+
import org.elasticsearch.test.ESTestCase;
2728
import org.elasticsearch.test.InternalTestCluster;
2829

2930
import java.io.IOException;
@@ -39,6 +40,7 @@
3940
import static org.hamcrest.Matchers.notNullValue;
4041

4142
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
43+
@ESTestCase.WithoutEntitlements // CLI tools don't run with entitlements enforced
4244
public class UnsafeBootstrapAndDetachCommandIT extends ESIntegTestCase {
4345

4446
private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, boolean abort) throws Exception {

‎test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.elasticsearch.common.network.IfConfig;
1515
import org.elasticsearch.common.settings.Settings;
1616
import org.elasticsearch.core.Booleans;
17-
import org.elasticsearch.core.Nullable;
1817
import org.elasticsearch.core.PathUtils;
1918
import org.elasticsearch.entitlement.bootstrap.TestEntitlementBootstrap;
2019
import org.elasticsearch.jdk.JarHell;
@@ -76,20 +75,12 @@ public class BootstrapForTesting {
7675

7776
// Fire up entitlements
7877
try {
79-
TestEntitlementBootstrap.bootstrap(javaTmpDir, maybePath(System.getProperty("tests.config")));
78+
TestEntitlementBootstrap.bootstrap(javaTmpDir);
8079
} catch (IOException e) {
8180
throw new IllegalStateException(e.getClass().getSimpleName() + " while initializing entitlements for tests", e);
8281
}
8382
}
8483

85-
private static @Nullable Path maybePath(String str) {
86-
if (str == null) {
87-
return null;
88-
} else {
89-
return PathUtils.get(str);
90-
}
91-
}
92-
9384
// does nothing, just easy way to make sure the class is loaded.
9485
public static void ensureInitialized() {}
9586
}

‎test/framework/src/main/java/org/elasticsearch/entitlement/bootstrap/TestEntitlementBootstrap.java

Lines changed: 72 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@
1212
import org.elasticsearch.bootstrap.TestBuildInfo;
1313
import org.elasticsearch.bootstrap.TestBuildInfoParser;
1414
import org.elasticsearch.bootstrap.TestScopeResolver;
15+
import org.elasticsearch.common.settings.Settings;
1516
import org.elasticsearch.core.Booleans;
1617
import org.elasticsearch.core.Nullable;
1718
import org.elasticsearch.core.PathUtils;
1819
import org.elasticsearch.core.Strings;
1920
import org.elasticsearch.core.SuppressForbidden;
2021
import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
2122
import org.elasticsearch.entitlement.runtime.policy.PathLookup;
23+
import org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir;
2224
import org.elasticsearch.entitlement.runtime.policy.Policy;
2325
import org.elasticsearch.entitlement.runtime.policy.PolicyParser;
2426
import org.elasticsearch.entitlement.runtime.policy.TestPathLookup;
@@ -32,39 +34,106 @@
3234
import java.net.URI;
3335
import java.net.URL;
3436
import java.nio.file.Path;
37+
import java.nio.file.Paths;
3538
import java.util.ArrayList;
3639
import java.util.Arrays;
40+
import java.util.Collection;
3741
import java.util.HashMap;
42+
import java.util.HashSet;
3843
import java.util.List;
3944
import java.util.Map;
4045
import java.util.Set;
4146
import java.util.TreeSet;
47+
import java.util.concurrent.ConcurrentHashMap;
48+
import java.util.function.BiFunction;
49+
import java.util.function.Consumer;
4250

4351
import static java.util.stream.Collectors.toCollection;
4452
import static java.util.stream.Collectors.toSet;
45-
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.CONFIG;
4653
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.TEMP;
54+
import static org.elasticsearch.env.Environment.PATH_DATA_SETTING;
55+
import static org.elasticsearch.env.Environment.PATH_HOME_SETTING;
56+
import static org.elasticsearch.env.Environment.PATH_REPO_SETTING;
4757

4858
public class TestEntitlementBootstrap {
4959

5060
private static final Logger logger = LogManager.getLogger(TestEntitlementBootstrap.class);
5161

62+
private static Map<BaseDir, Collection<Path>> baseDirPaths = new ConcurrentHashMap<>();
5263
private static TestPolicyManager policyManager;
5364

5465
/**
5566
* Activates entitlement checking in tests.
5667
*/
57-
public static void bootstrap(@Nullable Path tempDir, @Nullable Path configDir) throws IOException {
68+
public static void bootstrap(@Nullable Path tempDir) throws IOException {
5869
if (isEnabledForTest() == false) {
5970
return;
6071
}
61-
TestPathLookup pathLookup = new TestPathLookup(Map.of(TEMP, zeroOrOne(tempDir), CONFIG, zeroOrOne(configDir)));
72+
var previousTempDir = baseDirPaths.put(TEMP, zeroOrOne(tempDir));
73+
assert previousTempDir == null : "Test entitlement bootstrap called multiple times";
74+
TestPathLookup pathLookup = new TestPathLookup(baseDirPaths);
6275
policyManager = createPolicyManager(pathLookup);
6376
EntitlementInitialization.initializeArgs = new EntitlementInitialization.InitializeArgs(pathLookup, Set.of(), policyManager);
6477
logger.debug("Loading entitlement agent");
6578
EntitlementBootstrap.loadAgent(EntitlementBootstrap.findAgentJar(), EntitlementInitialization.class.getName());
6679
}
6780

81+
public static void registerNodeBaseDirs(Settings settings, Path configPath) {
82+
if (policyManager == null) {
83+
return;
84+
}
85+
Path homeDir = absolutePath(PATH_HOME_SETTING.get(settings));
86+
Path configDir = configPath != null ? configPath : homeDir.resolve("config");
87+
Collection<Path> dataDirs = dataDirs(settings, homeDir);
88+
Collection<Path> repoDirs = repoDirs(settings);
89+
logger.debug("Registering node dirs: config [{}], dataDirs [{}], repoDirs [{}]", configDir, dataDirs, repoDirs);
90+
baseDirPaths.compute(BaseDir.CONFIG, baseDirModifier(paths -> paths.add(configDir)));
91+
baseDirPaths.compute(BaseDir.DATA, baseDirModifier(paths -> paths.addAll(dataDirs)));
92+
baseDirPaths.compute(BaseDir.SHARED_REPO, baseDirModifier(paths -> paths.addAll(repoDirs)));
93+
policyManager.reset();
94+
}
95+
96+
public static void unregisterNodeBaseDirs(Settings settings, Path configPath) {
97+
if (policyManager == null) {
98+
return;
99+
}
100+
Path homeDir = absolutePath(PATH_HOME_SETTING.get(settings));
101+
Path configDir = configPath != null ? configPath : homeDir.resolve("config");
102+
Collection<Path> dataDirs = dataDirs(settings, homeDir);
103+
Collection<Path> repoDirs = repoDirs(settings);
104+
logger.debug("Unregistering node dirs: config [{}], dataDirs [{}], repoDirs [{}]", configDir, dataDirs, repoDirs);
105+
baseDirPaths.compute(BaseDir.CONFIG, baseDirModifier(paths -> paths.remove(configDir)));
106+
baseDirPaths.compute(BaseDir.DATA, baseDirModifier(paths -> paths.removeAll(dataDirs)));
107+
baseDirPaths.compute(BaseDir.SHARED_REPO, baseDirModifier(paths -> paths.removeAll(repoDirs)));
108+
policyManager.reset();
109+
}
110+
111+
private static Collection<Path> dataDirs(Settings settings, Path homeDir) {
112+
List<String> dataDirs = PATH_DATA_SETTING.get(settings);
113+
return dataDirs.isEmpty()
114+
? List.of(homeDir.resolve("data"))
115+
: dataDirs.stream().map(TestEntitlementBootstrap::absolutePath).toList();
116+
}
117+
118+
private static Collection<Path> repoDirs(Settings settings) {
119+
return PATH_REPO_SETTING.get(settings).stream().map(TestEntitlementBootstrap::absolutePath).toList();
120+
}
121+
122+
private static BiFunction<BaseDir, Collection<Path>, Collection<Path>> baseDirModifier(Consumer<Collection<Path>> consumer) {
123+
return (BaseDir baseDir, Collection<Path> paths) -> {
124+
if (paths == null) {
125+
paths = new HashSet<>();
126+
}
127+
consumer.accept(paths);
128+
return paths;
129+
};
130+
}
131+
132+
@SuppressForbidden(reason = "must be resolved using the default file system, rather then the mocked test file system")
133+
private static Path absolutePath(String path) {
134+
return Paths.get(path).toAbsolutePath().normalize();
135+
}
136+
68137
private static <T> List<T> zeroOrOne(T item) {
69138
if (item == null) {
70139
return List.of();

‎test/framework/src/main/java/org/elasticsearch/node/MockNode.java

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.common.util.MockBigArrays;
2525
import org.elasticsearch.common.util.MockPageCacheRecycler;
2626
import org.elasticsearch.common.util.PageCacheRecycler;
27+
import org.elasticsearch.entitlement.bootstrap.TestEntitlementBootstrap;
2728
import org.elasticsearch.env.Environment;
2829
import org.elasticsearch.http.HttpServerTransport;
2930
import org.elasticsearch.indices.ExecutorSelector;
@@ -53,6 +54,7 @@
5354
import org.elasticsearch.transport.TransportService;
5455
import org.elasticsearch.transport.TransportSettings;
5556

57+
import java.io.IOException;
5658
import java.nio.file.Path;
5759
import java.util.Collection;
5860
import java.util.Collections;
@@ -250,16 +252,7 @@ public MockNode(
250252
final Path configPath,
251253
final boolean forbidPrivateIndexSettings
252254
) {
253-
this(
254-
InternalSettingsPreparer.prepareEnvironment(
255-
Settings.builder().put(TransportSettings.PORT.getKey(), ESTestCase.getPortRange()).put(settings).build(),
256-
Collections.emptyMap(),
257-
configPath,
258-
() -> "mock_ node"
259-
),
260-
classpathPlugins,
261-
forbidPrivateIndexSettings
262-
);
255+
this(prepareEnvironment(settings, configPath), classpathPlugins, forbidPrivateIndexSettings);
263256
}
264257

265258
private MockNode(
@@ -278,6 +271,25 @@ PluginsService newPluginService(Environment environment, PluginsLoader pluginsLo
278271
this.classpathPlugins = classpathPlugins;
279272
}
280273

274+
private static Environment prepareEnvironment(final Settings settings, final Path configPath) {
275+
TestEntitlementBootstrap.registerNodeBaseDirs(settings, configPath);
276+
return InternalSettingsPreparer.prepareEnvironment(
277+
Settings.builder().put(TransportSettings.PORT.getKey(), ESTestCase.getPortRange()).put(settings).build(),
278+
Collections.emptyMap(),
279+
configPath,
280+
() -> "mock_ node"
281+
);
282+
}
283+
284+
@Override
285+
public synchronized void close() throws IOException {
286+
try {
287+
super.close();
288+
} finally {
289+
TestEntitlementBootstrap.unregisterNodeBaseDirs(getEnvironment().settings(), getEnvironment().configDir());
290+
}
291+
}
292+
281293
/**
282294
* The classpath plugins this node was constructed with.
283295
*/

‎test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,6 @@
285285
* </ul>
286286
*/
287287
@LuceneTestCase.SuppressFileSystems("ExtrasFS") // doesn't work with potential multi data path from test cluster yet
288-
@ESTestCase.WithoutEntitlements // ES-12042
289288
public abstract class ESIntegTestCase extends ESTestCase {
290289

291290
/** node names of the corresponding clusters will start with these prefixes */

‎test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,6 @@
9090
* A test that keep a singleton node started for all tests that can be used to get
9191
* references to Guice injectors in unit tests.
9292
*/
93-
@ESTestCase.WithoutEntitlements // ES-12042
9493
public abstract class ESSingleNodeTestCase extends ESTestCase {
9594

9695
private static Node NODE = null;

0 commit comments

Comments
 (0)