From 8c7ae6eee01f1249150f44c5f12a348fbe192312 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 22 Jul 2025 09:35:59 +1000 Subject: [PATCH 1/7] Updated RestoreService and cascading changes --- .../TransportRestoreSnapshotAction.java | 37 +-- .../elasticsearch/node/NodeConstruction.java | 3 +- .../service/FileSettingsService.java | 4 +- .../snapshots/RestoreService.java | 210 +++++++++++------- .../service/FileSettingsServiceTests.java | 5 +- .../snapshots/RestoreServiceTests.java | 30 ++- .../snapshots/SnapshotResiliencyTests.java | 12 +- .../xpack/ccr/CcrRepositoryIT.java | 3 +- .../ccr/action/TransportPutFollowAction.java | 8 +- .../elasticsearch/xpack/CcrIntegTestCase.java | 65 +++--- 10 files changed, 237 insertions(+), 140 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index 76fd2da37151d..47d1bf94644ff 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.snapshots.RestoreService; @@ -29,6 +30,7 @@ public class TransportRestoreSnapshotAction extends TransportMasterNodeAction { public static final ActionType TYPE = new ActionType<>("cluster:admin/snapshot/restore"); private final RestoreService restoreService; + private final ProjectResolver projectResolver; @Inject public TransportRestoreSnapshotAction( @@ -36,7 +38,8 @@ public TransportRestoreSnapshotAction( ClusterService clusterService, ThreadPool threadPool, RestoreService restoreService, - ActionFilters actionFilters + ActionFilters actionFilters, + ProjectResolver projectResolver ) { super( TYPE.name(), @@ -49,13 +52,15 @@ public TransportRestoreSnapshotAction( threadPool.executor(ThreadPool.Names.SNAPSHOT_META) ); this.restoreService = restoreService; + this.projectResolver = projectResolver; } @Override protected ClusterBlockException checkBlock(RestoreSnapshotRequest request, ClusterState state) { // Restoring a snapshot might change the global state and create/change an index, // so we need to check for METADATA_WRITE and WRITE blocks - ClusterBlockException blockException = state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + ClusterBlockException blockException = state.blocks() + .globalBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_WRITE); if (blockException != null) { return blockException; } @@ -70,17 +75,21 @@ protected void masterOperation( final ClusterState state, final ActionListener listener ) { - restoreService.restoreSnapshot(request, listener.delegateFailure((delegatedListener, restoreCompletionResponse) -> { - if (restoreCompletionResponse.restoreInfo() == null && request.waitForCompletion()) { - RestoreClusterStateListener.createAndRegisterListener( - clusterService, - restoreCompletionResponse, - delegatedListener, - threadPool.getThreadContext() - ); - } else { - delegatedListener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.restoreInfo())); - } - })); + restoreService.restoreSnapshot( + projectResolver.getProjectId(), + request, + listener.delegateFailure((delegatedListener, restoreCompletionResponse) -> { + if (restoreCompletionResponse.restoreInfo() == null && request.waitForCompletion()) { + RestoreClusterStateListener.createAndRegisterListener( + clusterService, + restoreCompletionResponse, + delegatedListener, + threadPool.getThreadContext() + ); + } else { + delegatedListener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.restoreInfo())); + } + }) + ); } } diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index bb28ed4a8aff5..3b9471eff9cb0 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -1149,7 +1149,8 @@ public Map queryFields() { systemIndices, indicesService, fileSettingsService, - threadPool + threadPool, + projectResolver.supportsMultipleProjects() ); DiscoveryModule discoveryModule = createDiscoveryModule( diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java index 76cf7ef1b6947..f6f2131f79d3d 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; @@ -126,8 +127,9 @@ public Path watchedFile() { * file based settings from the cluster state. * @param clusterState the cluster state before snapshot restore * @param mdBuilder the current metadata builder for the new cluster state + * @param projectId the project associated with the restore */ - public void handleSnapshotRestore(ClusterState clusterState, Metadata.Builder mdBuilder) { + public void handleSnapshotRestore(ClusterState clusterState, Metadata.Builder mdBuilder, ProjectId projectId) { assert clusterState.nodes().isLocalNodeElectedMaster(); ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(NAMESPACE); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 1680bfa3a59a0..4c0253b7a3826 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -118,6 +118,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; +import static org.elasticsearch.repositories.ProjectRepo.projectRepoString; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY; @@ -130,7 +131,7 @@ *

* Restore operation is performed in several stages. *

- * First {@link #restoreSnapshot(RestoreSnapshotRequest, org.elasticsearch.action.ActionListener)} + * First {@link #restoreSnapshot(ProjectId, RestoreSnapshotRequest, org.elasticsearch.action.ActionListener)} * method reads information about snapshot and metadata from repository. In update cluster state task it checks restore * preconditions, restores global state if needed, creates {@link RestoreInProgress} record with list of shards that needs * to be restored and adds this shard to the routing table using {@link RoutingTable.Builder#addAsRestore} method. @@ -190,6 +191,7 @@ public final class RestoreService implements ClusterStateApplier { private final MetadataCreateIndexService createIndexService; private final IndexMetadataVerifier indexMetadataVerifier; + private final boolean deserializeProjectMetadata; private final ShardLimitValidator shardLimitValidator; @@ -217,13 +219,15 @@ public RestoreService( SystemIndices systemIndices, IndicesService indicesService, FileSettingsService fileSettingsService, - ThreadPool threadPool + ThreadPool threadPool, + boolean deserializeProjectMetadata ) { this.clusterService = clusterService; this.repositoriesService = repositoriesService; this.allocationService = allocationService; this.createIndexService = createIndexService; this.indexMetadataVerifier = indexMetadataVerifier; + this.deserializeProjectMetadata = deserializeProjectMetadata; if (DiscoveryNode.isMasterNode(clusterService.getSettings())) { clusterService.addStateApplier(this); } @@ -242,31 +246,51 @@ public RestoreService( /** * Restores snapshot specified in the restore request. * + * @param projectId project for the restore * @param request restore request * @param listener restore listener */ - public void restoreSnapshot(final RestoreSnapshotRequest request, final ActionListener listener) { - restoreSnapshot(request, listener, (clusterState, builder) -> {}); + public void restoreSnapshot( + final ProjectId projectId, + final RestoreSnapshotRequest request, + final ActionListener listener + ) { + restoreSnapshot(projectId, request, listener, (clusterState, builder) -> {}); } /** * Restores snapshot specified in the restore request. * + * @param projectId project for the restore * @param request restore request * @param listener restore listener * @param updater handler that allows callers to make modifications to {@link Metadata} * in the same cluster state update as the restore operation */ public void restoreSnapshot( + final ProjectId projectId, final RestoreSnapshotRequest request, final ActionListener listener, final BiConsumer updater ) { assert Repository.assertSnapshotMetaThread(); + if (clusterService.state().metadata().hasProject(projectId) == false) { + listener.onFailure( + new SnapshotRestoreException(request.repository(), request.snapshot(), "project [" + projectId + "] does not exist") + ); + return; + } + // Try and fill in any missing repository UUIDs in case they're needed during the restore final var repositoryUuidRefreshStep = SubscribableListener.newForked( - l -> refreshRepositoryUuids(refreshRepositoryUuidOnRestore, repositoriesService, () -> l.onResponse(null), snapshotMetaExecutor) + l -> refreshRepositoryUuids( + refreshRepositoryUuidOnRestore, + projectId, + repositoriesService, + () -> l.onResponse(null), + snapshotMetaExecutor + ) ); // AtomicReference just so we have somewhere to hold these objects, there's no interesting concurrency here @@ -277,7 +301,7 @@ public void restoreSnapshot( .newForked(repositorySetListener -> { // do this within newForked for exception handling - repositoryRef.set(repositoriesService.repository(request.repository())); + repositoryRef.set(repositoriesService.repository(projectId, request.repository())); repositorySetListener.onResponse(null); }) @@ -320,7 +344,14 @@ public void restoreSnapshot( ) .addListener(listener.delegateResponse((delegate, e) -> { - logger.warn(() -> "[" + request.repository() + ":" + request.snapshot() + "] failed to restore snapshot", e); + logger.warn( + () -> "[" + + projectRepoString(projectId, request.repository()) + + ":" + + request.snapshot() + + "] failed to restore snapshot", + e + ); delegate.onFailure(e); })); } @@ -350,16 +381,17 @@ private void startRestore( assert Repository.assertSnapshotMetaThread(); final SnapshotId snapshotId = snapshotInfo.snapshotId(); final String repositoryName = repository.getMetadata().name(); - final Snapshot snapshot = new Snapshot(repositoryName, snapshotId); + final Snapshot snapshot = new Snapshot(snapshotInfo.projectId(), repositoryName, snapshotId); // Make sure that we can restore from this snapshot validateSnapshotRestorable(request, repository.getMetadata(), snapshotInfo, repositoriesService.getPreRestoreVersionChecks()); + final var projectId = snapshotInfo.projectId(); // Get the global state if necessary Metadata globalMetadata = null; final Metadata.Builder metadataBuilder; if (request.includeGlobalState()) { - globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId, false); + globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId, deserializeProjectMetadata); metadataBuilder = Metadata.builder(globalMetadata); } else { metadataBuilder = Metadata.builder(); @@ -389,7 +421,7 @@ private void startRestore( logger.warn( () -> format( "Restoring snapshot[%s] skipping feature [%s] because it is not available in this cluster", - snapshotInfo.snapshotId(), + snapshotInfo.snapshot(), featureName ) ); @@ -483,8 +515,6 @@ private void startRestore( final Set explicitlyRequestedSystemIndices = new HashSet<>(); - @FixForMultiProject - final ProjectId projectId = ProjectId.DEFAULT; ProjectMetadata.Builder projectBuilder = metadataBuilder.getProject(projectId); if (projectBuilder == null) { projectBuilder = ProjectMetadata.builder(projectId); @@ -504,6 +534,7 @@ private void startRestore( : "it should be impossible to reach this point with explicitly requested system indices, but got: " + explicitlyRequestedSystemIndices; + projectBuilder.dataStreams(dataStreamsToRestore, dataStreamAliasesToRestore); // Now we can start the actual restore process by adding shards to be recovered in the cluster state // and updating cluster metadata (global and index) as needed submitUnbatchedTask( @@ -523,7 +554,7 @@ private void startRestore( repositoryData ), snapshotInfo, - metadataBuilder.dataStreams(dataStreamsToRestore, dataStreamAliasesToRestore).build(), + metadataBuilder.build(), dataStreamsToRestore.values(), updater, clusterService.getSettings(), @@ -537,10 +568,10 @@ private void validateDataStreamTemplatesExistAndWarnIfMissing( SnapshotInfo snapshotInfo, Metadata globalMetadata ) { - + final var projectId = snapshotInfo.projectId(); Stream streams = Stream.concat( - clusterService.state().metadata().getProject().templatesV2().values().stream(), - globalMetadata == null ? Stream.empty() : globalMetadata.getProject().templatesV2().values().stream() + clusterService.state().metadata().getProject(projectId).templatesV2().values().stream(), + globalMetadata == null ? Stream.empty() : globalMetadata.getProject(projectId).templatesV2().values().stream() ); Set templatePatterns = streams.filter(cit -> cit.getDataStreamTemplate() != null) @@ -562,7 +593,7 @@ static void warnIfIndexTemplateMissing( String warningMessage = format( "Snapshot [%s] contains data stream [%s] but custer does not have a matching index template. This will cause" + " rollover to fail until a matching index template is created", - snapshotInfo.snapshotId(), + snapshotInfo.snapshot(), name ); logger.warn(() -> warningMessage); @@ -585,6 +616,7 @@ private void setRefreshRepositoryUuidOnRestore(boolean refreshRepositoryUuidOnRe * {@link BlobStoreRepository} with a missing UUID. * * @param enabled If {@code false} this method completes the listener immediately + * @param projectId project for the restore * @param repositoriesService Supplies the repositories to check * @param onCompletion Action that is executed when all repositories have been refreshed. * @param responseExecutor Executor on which to execute {@code onCompletion} if not using the calling thread. @@ -592,6 +624,7 @@ private void setRefreshRepositoryUuidOnRestore(boolean refreshRepositoryUuidOnRe // Exposed for tests static void refreshRepositoryUuids( boolean enabled, + ProjectId projectId, RepositoriesService repositoriesService, Runnable onCompletion, Executor responseExecutor @@ -602,8 +635,6 @@ static void refreshRepositoryUuids( return; } - @FixForMultiProject(description = "resolve the actual projectId, ES-10228") - final var projectId = ProjectId.DEFAULT; for (Repository repository : repositoriesService.getProjectRepositories(projectId).values()) { // We only care about BlobStoreRepositories because they're the only ones that can contain a searchable snapshot, and we // only care about ones with missing UUIDs. It's possible to have the UUID change from under us if, e.g., the repository was @@ -631,7 +662,7 @@ private boolean isSystemIndex(IndexMetadata indexMetadata) { return indexMetadata.isSystem() || systemIndices.isSystemName(indexMetadata.getIndex().getName()); } - private static Tuple, Map> getDataStreamsToRestore( + private Tuple, Map> getDataStreamsToRestore( Repository repository, SnapshotId snapshotId, SnapshotInfo snapshotInfo, @@ -652,9 +683,10 @@ private static Tuple, Map> getD dataStreamAliases = Map.of(); } else { if (globalMetadata == null) { - globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId, false); + globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId, deserializeProjectMetadata); } - final Map dataStreamsInSnapshot = globalMetadata.getProject().dataStreams(); + final ProjectId projectId = snapshotInfo.projectId(); + final Map dataStreamsInSnapshot = globalMetadata.getProject(projectId).dataStreams(); allDataStreams = Maps.newMapWithExpectedSize(requestedDataStreams.size()); Map systemDataStreams = new HashMap<>(); for (String requestedDataStream : requestedDataStreams) { @@ -682,7 +714,7 @@ private static Tuple, Map> getD } if (includeAliases || systemDataStreams.isEmpty() == false) { dataStreamAliases = new HashMap<>(); - final Map dataStreamAliasesInSnapshot = globalMetadata.getProject().dataStreamAliases(); + final Map dataStreamAliasesInSnapshot = globalMetadata.getProject(projectId).dataStreamAliases(); Map dataStreamsWithAliases = includeAliases ? allDataStreams : systemDataStreams; for (DataStreamAlias alias : dataStreamAliasesInSnapshot.values()) { DataStreamAlias copy = alias.intersect(dataStreamsWithAliases.keySet()::contains); @@ -766,11 +798,11 @@ private Map> getFeatureStatesToRestore( * Resolves a set of index names that currently exist in the cluster that are part of a feature state which is about to be restored, * and should therefore be removed prior to restoring those feature states from the snapshot. * - * @param currentState The current cluster state + * @param projectMetadata The current project metadata * @param featureStatesToRestore A set of feature state names that are about to be restored * @return A set of index names that should be removed based on the feature states being restored */ - private Set resolveSystemIndicesToDelete(ClusterState currentState, Set featureStatesToRestore) { + private Set resolveSystemIndicesToDelete(ProjectMetadata projectMetadata, Set featureStatesToRestore) { if (featureStatesToRestore == null) { return Collections.emptySet(); } @@ -779,11 +811,10 @@ private Set resolveSystemIndicesToDelete(ClusterState currentState, Set feature.getIndexDescriptors().stream()) - .flatMap(descriptor -> descriptor.getMatchingIndices(currentState.metadata().getProject()).stream()) + .flatMap(descriptor -> descriptor.getMatchingIndices(projectMetadata).stream()) .map(indexName -> { - assert currentState.metadata().getProject().hasIndex(indexName) - : "index [" + indexName + "] not found in metadata but must be present"; - return currentState.metadata().getProject().indices().get(indexName).getIndex(); + assert projectMetadata.hasIndex(indexName) : "index [" + indexName + "] not found in metadata but must be present"; + return projectMetadata.indices().get(indexName).getIndex(); }) .collect(Collectors.toUnmodifiableSet()); } @@ -792,11 +823,11 @@ private Set resolveSystemIndicesToDelete(ClusterState currentState, Set resolveSystemDataStreamsToDelete(ClusterState currentState, Set featureStatesToRestore) { + private Set resolveSystemDataStreamsToDelete(ProjectMetadata projectMetadata, Set featureStatesToRestore) { if (featureStatesToRestore == null) { return Collections.emptySet(); } @@ -806,25 +837,25 @@ private Set resolveSystemDataStreamsToDelete(ClusterState currentSta .filter(Objects::nonNull) // Features that aren't present on this node will be warned about in `getFeatureStatesToRestore` .flatMap(feature -> feature.getDataStreamDescriptors().stream()) .map(SystemDataStreamDescriptor::getDataStreamName) - .filter(datastreamName -> currentState.metadata().getProject().dataStreams().containsKey(datastreamName)) - .map(dataStreamName -> currentState.metadata().getProject().dataStreams().get(dataStreamName)) + .filter(datastreamName -> projectMetadata.dataStreams().containsKey(datastreamName)) + .map(dataStreamName -> projectMetadata.dataStreams().get(dataStreamName)) .collect(Collectors.toUnmodifiableSet()); } // visible for testing - static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metadata, RestoreSnapshotRequest request) { + static DataStream updateDataStream(DataStream dataStream, ProjectMetadata.Builder projectMetadata, RestoreSnapshotRequest request) { String dataStreamName = dataStream.getName(); if (request.renamePattern() != null && request.renameReplacement() != null) { dataStreamName = dataStreamName.replaceAll(request.renamePattern(), request.renameReplacement()); } List updatedIndices = dataStream.getIndices() .stream() - .map(i -> metadata.get(renameIndex(i.getName(), request, true, false)).getIndex()) + .map(i -> projectMetadata.get(renameIndex(i.getName(), request, true, false)).getIndex()) .toList(); List updatedFailureIndices = dataStream.getFailureComponent() .getIndices() .stream() - .map(i -> metadata.get(renameIndex(i.getName(), request, false, true)).getIndex()) + .map(i -> projectMetadata.get(renameIndex(i.getName(), request, false, true)).getIndex()) .toList(); return dataStream.copy() .setName(dataStreamName) @@ -1093,13 +1124,13 @@ static void validateSnapshotRestorable( ) { if (snapshotInfo.state().restorable() == false) { throw new SnapshotRestoreException( - new Snapshot(repository.name(), snapshotInfo.snapshotId()), + new Snapshot(snapshotInfo.projectId(), repository.name(), snapshotInfo.snapshotId()), "unsupported snapshot state [" + snapshotInfo.state() + "]" ); } if (IndexVersion.current().before(snapshotInfo.version())) { throw new SnapshotRestoreException( - new Snapshot(repository.name(), snapshotInfo.snapshotId()), + new Snapshot(snapshotInfo.projectId(), repository.name(), snapshotInfo.snapshotId()), "the snapshot was created with version [" + snapshotInfo.version().toReleaseVersion() + "] which is higher than the version of this node [" @@ -1107,11 +1138,11 @@ static void validateSnapshotRestorable( + "]" ); } - Snapshot snapshot = new Snapshot(repository.name(), snapshotInfo.snapshotId()); + Snapshot snapshot = new Snapshot(snapshotInfo.projectId(), repository.name(), snapshotInfo.snapshotId()); preRestoreVersionChecks.forEach(c -> c.accept(snapshot, snapshotInfo.version())); if (request.includeGlobalState() && snapshotInfo.includeGlobalState() == Boolean.FALSE) { throw new SnapshotRestoreException( - new Snapshot(repository.name(), snapshotInfo.snapshotId()), + new Snapshot(snapshotInfo.projectId(), repository.name(), snapshotInfo.snapshotId()), "cannot restore global state since the snapshot was created without global state" ); } @@ -1249,7 +1280,7 @@ private static IndexMetadata updateIndexSettings( format( "cannot change value of [%s] when restoring searchable snapshot [%s:%s] as index %s", SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, - snapshot.getRepository(), + projectRepoString(snapshot.getProjectId(), snapshot.getRepository()), snapshot.getSnapshotId().getName(), indexMetadata.getIndex() ) @@ -1370,20 +1401,26 @@ private final class RestoreSnapshotStateTask extends ClusterStateUpdateTask { @Override public ClusterState execute(ClusterState currentState) { + final ProjectId projectId = snapshot.getProjectId(); + + if (currentState.metadata().hasProject(projectId) == false) { + throw new SnapshotRestoreException(snapshot, "project [" + projectId + "] does not exist"); + } + // Check if the snapshot to restore is currently being deleted ensureSnapshotNotDeleted(currentState); // Clear out all existing indices which fall within a system index pattern being restored currentState = MetadataDeleteIndexService.deleteIndices( - currentState, - resolveSystemIndicesToDelete(currentState, featureStatesToRestore), + currentState.projectState(projectId), + resolveSystemIndicesToDelete(currentState.metadata().getProject(projectId), featureStatesToRestore), settings ); // Clear out all existing system data streams currentState = MetadataDataStreamsService.deleteDataStreams( - currentState.projectState(), - resolveSystemDataStreamsToDelete(currentState, featureStatesToRestore), + currentState.projectState(projectId), + resolveSystemDataStreamsToDelete(currentState.metadata().getProject(projectId), featureStatesToRestore), settings ); @@ -1392,12 +1429,10 @@ public ClusterState execute(ClusterState currentState) { // Updating cluster state final Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata()); - @FixForMultiProject - final ProjectMetadata project = currentState.metadata().getProject(); final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()); final RoutingTable.Builder rtBuilder = RoutingTable.builder( allocationService.getShardRoutingRoleStrategy(), - currentState.routingTable() + currentState.routingTable(projectId) ); final Map shards = new HashMap<>(); @@ -1407,7 +1442,7 @@ public ClusterState execute(ClusterState currentState) { final String localNodeId = clusterService.state().nodes().getLocalNodeId(); for (Map.Entry indexEntry : indicesToRestore.entrySet()) { final IndexId index = indexEntry.getValue(); - final IndexMetadata originalIndexMetadata = metadata.getProject().index(index.getName()); + final IndexMetadata originalIndexMetadata = metadata.getProject(projectId).index(index.getName()); repositoriesService.getPreRestoreVersionChecks() .forEach(check -> check.accept(snapshot, originalIndexMetadata.getCreationVersion())); IndexMetadata snapshotIndexMetadata = updateIndexSettings( @@ -1430,7 +1465,7 @@ public ClusterState execute(ClusterState currentState) { throw new SnapshotRestoreException(snapshot, "cannot restore index [" + index + "] because it cannot be upgraded", ex); } final String renamedIndexName = indexEntry.getKey(); - final IndexMetadata currentIndexMetadata = currentState.metadata().getProject().index(renamedIndexName); + final IndexMetadata currentIndexMetadata = currentState.metadata().getProject(projectId).index(renamedIndexName); final SnapshotRecoverySource recoverySource = new SnapshotRecoverySource( restoreUUID, snapshot, @@ -1447,8 +1482,8 @@ public ClusterState execute(ClusterState currentState) { // Index doesn't exist - create it and start recovery // Make sure that the index we are about to create has a valid name ensureValidIndexName( - currentState.metadata().getProject(), - currentState.routingTable(), + currentState.metadata().getProject(projectId), + currentState.routingTable(projectId), snapshotIndexMetadata, renamedIndexName ); @@ -1472,7 +1507,7 @@ && isSystemIndex(snapshotIndexMetadata) == false) { populateIgnoredShards(index.getName(), ignoreShards); } rtBuilder.addAsNewRestore(updatedIndexMetadata, recoverySource, ignoreShards); - blocks.addBlocks(updatedIndexMetadata); + blocks.addBlocks(projectId, updatedIndexMetadata); } else { // Index exists and it's closed - open it in metadata and start recovery validateExistingClosedIndex(currentIndexMetadata, snapshotIndexMetadata, renamedIndexName, partial); @@ -1492,10 +1527,10 @@ && isSystemIndex(snapshotIndexMetadata) == false) { } updatedIndexMetadata = indexMdBuilder.build(); rtBuilder.addAsRestore(updatedIndexMetadata, recoverySource); - blocks.updateBlocks(updatedIndexMetadata); + blocks.updateBlocks(projectId, updatedIndexMetadata); } - mdBuilder.getProject(project.id()).put(updatedIndexMetadata, true); + mdBuilder.getProject(projectId).put(updatedIndexMetadata, true); final Index renamedIndex = updatedIndexMetadata.getIndex(); for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) { shards.put( @@ -1528,12 +1563,12 @@ && isSystemIndex(snapshotIndexMetadata) == false) { ); } - applyDataStreamRestores(currentState, mdBuilder); + applyDataStreamRestores(currentState, mdBuilder, projectId); // Restore global state if needed if (request.includeGlobalState()) { - applyGlobalStateRestore(currentState, mdBuilder); - fileSettingsService.handleSnapshotRestore(currentState, mdBuilder); + applyGlobalStateRestore(currentState, mdBuilder, projectId); + fileSettingsService.handleSnapshotRestore(currentState, mdBuilder, projectId); } if (completed(shards)) { @@ -1547,36 +1582,43 @@ && isSystemIndex(snapshotIndexMetadata) == false) { } updater.accept(currentState, mdBuilder); - final ClusterState updatedClusterState = builder.metadata(mdBuilder).blocks(blocks).routingTable(rtBuilder.build()).build(); + final ClusterState updatedClusterState = builder.metadata(mdBuilder) + .blocks(blocks) + .putRoutingTable(projectId, rtBuilder.build()) + .build(); if (searchableSnapshotsIndices.isEmpty() == false) { ensureSearchableSnapshotsRestorable(updatedClusterState, snapshotInfo, searchableSnapshotsIndices); } return allocationService.reroute(updatedClusterState, "restored snapshot [" + snapshot + "]", listener.reroute()); } - private void applyDataStreamRestores(ClusterState currentState, Metadata.Builder mdBuilder) { - final Map updatedDataStreams = new HashMap<>(currentState.metadata().getProject().dataStreams()); + private void applyDataStreamRestores(ClusterState currentState, Metadata.Builder mdBuilder, ProjectId projectId) { + final Map updatedDataStreams = new HashMap<>(currentState.metadata().getProject(projectId).dataStreams()); updatedDataStreams.putAll( dataStreamsToRestore.stream() - .map(ds -> updateDataStream(ds, mdBuilder, request)) + .map(ds -> updateDataStream(ds, mdBuilder.getProject(projectId), request)) .collect(Collectors.toMap(DataStream::getName, Function.identity())) ); final Map updatedDataStreamAliases = new HashMap<>( - currentState.metadata().getProject().dataStreamAliases() + currentState.metadata().getProject(projectId).dataStreamAliases() ); - for (DataStreamAlias alias : metadata.getProject().dataStreamAliases().values()) { + for (DataStreamAlias alias : metadata.getProject(projectId).dataStreamAliases().values()) { // Merge data stream alias from snapshot with an existing data stream aliases in target cluster: updatedDataStreamAliases.compute( alias.getName(), (key, previous) -> alias.restore(previous, request.renamePattern(), request.renameReplacement()) ); } - mdBuilder.dataStreams(updatedDataStreams, updatedDataStreamAliases); + mdBuilder.getProject(projectId).dataStreams(updatedDataStreams, updatedDataStreamAliases); } private void ensureSnapshotNotDeleted(ClusterState currentState) { SnapshotDeletionsInProgress deletionsInProgress = SnapshotDeletionsInProgress.get(currentState); - if (deletionsInProgress.getEntries().stream().anyMatch(entry -> entry.snapshots().contains(snapshot.getSnapshotId()))) { + if (deletionsInProgress.getEntries() + .stream() + .anyMatch( + entry -> entry.projectId().equals(snapshot.getProjectId()) && entry.snapshots().contains(snapshot.getSnapshotId()) + )) { throw new ConcurrentSnapshotExecutionException( snapshot, "cannot restore a snapshot while a snapshot deletion is in-progress [" + deletionsInProgress.getEntries().get(0) + "]" @@ -1584,10 +1626,14 @@ private void ensureSnapshotNotDeleted(ClusterState currentState) { } } - private void applyGlobalStateRestore(ClusterState currentState, Metadata.Builder mdBuilder) { - @FixForMultiProject - final var projectBuilder = mdBuilder.getProject(ProjectId.DEFAULT); + private void applyGlobalStateRestore(ClusterState currentState, Metadata.Builder mdBuilder, ProjectId projectId) { + final var projectBuilder = mdBuilder.getProject(projectId); if (metadata.persistentSettings() != null) { + assert deserializeProjectMetadata == false || metadata.persistentSettings().isEmpty() + : "Inconsistent deserializeProjectMetadata [" + + deserializeProjectMetadata + + "] and cluster level persistent settings " + + metadata.persistentSettings(); Settings settings = metadata.persistentSettings(); if (request.skipOperatorOnlyState()) { // Skip any operator-only settings from the snapshot. This happens when operator privileges are enabled @@ -1608,9 +1654,9 @@ private void applyGlobalStateRestore(ClusterState currentState, Metadata.Builder clusterSettings.validateUpdate(settings); mdBuilder.persistentSettings(settings); } - if (metadata.getProject().templates() != null) { + if (metadata.getProject(projectId).templates() != null) { // TODO: Should all existing templates be deleted first? - for (IndexTemplateMetadata cursor : metadata.getProject().templates().values()) { + for (IndexTemplateMetadata cursor : metadata.getProject(projectId).templates().values()) { projectBuilder.put(cursor); } } @@ -1621,6 +1667,11 @@ private void applyGlobalStateRestore(ClusterState currentState, Metadata.Builder // restore customs from the snapshot if (metadata.customs() != null) { + assert deserializeProjectMetadata == false || metadata.persistentSettings().isEmpty() + : "Inconsistent deserializeProjectMetadata [" + + deserializeProjectMetadata + + "] and cluster level customs " + + metadata.customs(); for (var entry : metadata.customs().entrySet()) { if (entry.getValue().isRestorable()) { // TODO: Check request.skipOperatorOnly for Autoscaling policies (NonRestorableCustom) @@ -1630,8 +1681,8 @@ private void applyGlobalStateRestore(ClusterState currentState, Metadata.Builder } } } - if (metadata.getProject().customs() != null) { - for (var entry : metadata.getProject().customs().entrySet()) { + if (metadata.getProject(projectId).customs() != null) { + for (var entry : metadata.getProject(projectId).customs().entrySet()) { if (entry.getValue().isRestorable()) { // Also, don't restore data streams here, we already added them to the metadata builder above projectBuilder.putCustom(entry.getKey(), entry.getValue()); @@ -1914,9 +1965,9 @@ private static void ensureSearchableSnapshotsRestorable( final SnapshotInfo snapshotInfo, final Set indices ) { - final Metadata metadata = currentState.metadata(); + final ProjectMetadata projectMetadata = currentState.metadata().getProject(snapshotInfo.projectId()); for (Index index : indices) { - final Settings indexSettings = metadata.getProject().getIndexSafe(index).getSettings(); + final Settings indexSettings = projectMetadata.getIndexSafe(index).getSettings(); assert "snapshot".equals(INDEX_STORE_TYPE_SETTING.get(indexSettings)) : "not a snapshot backed index: " + index; final String repositoryUuid = indexSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY); @@ -1926,8 +1977,7 @@ private static void ensureSearchableSnapshotsRestorable( final boolean deleteSnapshot = indexSettings.getAsBoolean(SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, false); if (deleteSnapshot && snapshotInfo.indices().size() != 1 && Objects.equals(snapshotUuid, snapshotInfo.snapshotId().getUUID())) { throw new SnapshotRestoreException( - repositoryName, - snapshotInfo.snapshotId().getName(), + snapshotInfo.snapshot(), format( "cannot mount snapshot [%s/%s:%s] as index [%s] with the deletion of snapshot on index removal enabled " + "[index.store.snapshot.delete_searchable_snapshot: true]; snapshot contains [%d] indices instead of 1.", @@ -1940,7 +1990,7 @@ private static void ensureSearchableSnapshotsRestorable( ); } - for (IndexMetadata other : metadata.getProject()) { + for (IndexMetadata other : projectMetadata) { if (other.getIndex().equals(index)) { continue; // do not check the searchable snapshot index against itself } @@ -1952,6 +2002,10 @@ private static void ensureSearchableSnapshotsRestorable( if (Objects.equals(snapshotUuid, otherSnapshotUuid) == false) { continue; // other index is backed by a different snapshot, skip } + @FixForMultiProject( + description = "Repo UUID might be null and repo name is not sufficiently unique. " + + "But searchable snapshots may not be supported in multi-project. See also ES-12138" + ) final String otherRepositoryUuid = otherSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY); final String otherRepositoryName = otherSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY); if (matchRepository(repositoryUuid, repositoryName, otherRepositoryUuid, otherRepositoryName) == false) { diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java index 868d70ea6928a..7e8e262f954dc 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NodeConnectionsService; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; @@ -452,7 +453,7 @@ public void testHandleSnapshotRestoreClearsMetadata() throws Exception { .build(); Metadata.Builder metadata = Metadata.builder(state.metadata()); - fileSettingsService.handleSnapshotRestore(state, metadata); + fileSettingsService.handleSnapshotRestore(state, metadata, ProjectId.DEFAULT); assertThat(metadata.build().reservedStateMetadata(), anEmptyMap()); } @@ -475,7 +476,7 @@ public void testHandleSnapshotRestoreResetsMetadata() throws Exception { .build(); Metadata.Builder metadata = Metadata.builder(); - fileSettingsService.handleSnapshotRestore(state, metadata); + fileSettingsService.handleSnapshotRestore(state, metadata, ProjectId.DEFAULT); assertThat( metadata.build().reservedStateMetadata(), diff --git a/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java index e45dc4e3d8cd5..6551484257aba 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java @@ -14,8 +14,8 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; @@ -62,7 +62,10 @@ public void testWarnIfIndexTemplateMissingSkipsSystemDataStreams() throws Except var dataStream = DataStream.builder(dataStreamName, indices).setSystem(true).setHidden(true).build(); var dataStreamsToRestore = Map.of(dataStreamName, dataStream); var templatePatterns = Set.of("matches_none"); - var snapshotInfo = createSnapshotInfo(new Snapshot("repository", new SnapshotId("name", "uuid")), Boolean.FALSE); + var snapshotInfo = createSnapshotInfo( + new Snapshot(randomProjectIdOrDefault(), "repository", new SnapshotId("name", "uuid")), + Boolean.FALSE + ); RestoreService.warnIfIndexTemplateMissing(dataStreamsToRestore, templatePatterns, snapshotInfo); @@ -80,7 +83,10 @@ public void testWarnIfIndexTemplateMissing() throws Exception { var dataStream = DataStream.builder(dataStreamName, indices).build(); var dataStreamsToRestore = Map.of(dataStreamName, dataStream); var templatePatterns = Set.of("matches_none"); - var snapshotInfo = createSnapshotInfo(new Snapshot("repository", new SnapshotId("name", "uuid")), Boolean.FALSE); + var snapshotInfo = createSnapshotInfo( + new Snapshot(randomProjectIdOrDefault(), "repository", new SnapshotId("name", "uuid")), + Boolean.FALSE + ); RestoreService.warnIfIndexTemplateMissing(dataStreamsToRestore, templatePatterns, snapshotInfo); @@ -88,7 +94,7 @@ public void testWarnIfIndexTemplateMissing() throws Exception { format( "Snapshot [%s] contains data stream [%s] but custer does not have a matching index template. This will cause" + " rollover to fail until a matching index template is created", - snapshotInfo.snapshotId(), + snapshotInfo.snapshot(), dataStreamName ) ); @@ -104,7 +110,7 @@ public void testUpdateDataStream() { DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, indices, failureIndices); - Metadata.Builder metadata = mock(Metadata.Builder.class); + ProjectMetadata.Builder metadata = mock(ProjectMetadata.Builder.class); IndexMetadata backingIndexMetadata = mock(IndexMetadata.class); when(metadata.get(eq(backingIndexName))).thenReturn(backingIndexMetadata); @@ -139,7 +145,7 @@ public void testUpdateDataStreamRename() { DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, indices, failureIndices); - Metadata.Builder metadata = mock(Metadata.Builder.class); + ProjectMetadata.Builder metadata = mock(ProjectMetadata.Builder.class); IndexMetadata backingIndexMetadata = mock(IndexMetadata.class); when(metadata.get(eq(renamedBackingIndexName))).thenReturn(backingIndexMetadata); @@ -175,7 +181,7 @@ public void testPrefixNotChanged() { DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, indices, failureIndices); - Metadata.Builder metadata = mock(Metadata.Builder.class); + ProjectMetadata.Builder metadata = mock(ProjectMetadata.Builder.class); IndexMetadata indexMetadata = mock(IndexMetadata.class); when(metadata.get(eq(renamedBackingIndexName))).thenReturn(indexMetadata); @@ -209,6 +215,7 @@ public void testRefreshRepositoryUuidsDoesNothingIfDisabled() { final AtomicBoolean called = new AtomicBoolean(); RestoreService.refreshRepositoryUuids( false, + randomProjectIdOrDefault(), repositoriesService, () -> assertTrue(called.compareAndSet(false, true)), EsExecutors.DIRECT_EXECUTOR_SERVICE @@ -259,11 +266,13 @@ public void testRefreshRepositoryUuidsRefreshesAsNeeded() { } } + final ProjectId projectId = randomProjectIdOrDefault(); final RepositoriesService repositoriesService = mock(RepositoriesService.class); - when(repositoriesService.getProjectRepositories(eq(ProjectId.DEFAULT))).thenReturn(repositories); + when(repositoriesService.getProjectRepositories(eq(projectId))).thenReturn(repositories); final AtomicBoolean completed = new AtomicBoolean(); RestoreService.refreshRepositoryUuids( true, + projectId, repositoriesService, () -> assertTrue(completed.compareAndSet(false, true)), EsExecutors.DIRECT_EXECUTOR_SERVICE @@ -277,7 +286,8 @@ public void testNotAllowToRestoreGlobalStateFromSnapshotWithoutOne() { var request = new RestoreSnapshotRequest(TEST_REQUEST_TIMEOUT).includeGlobalState(true); var repository = new RepositoryMetadata("name", "type", Settings.EMPTY); - var snapshot = new Snapshot("repository", new SnapshotId("name", "uuid")); + final ProjectId projectId = randomProjectIdOrDefault(); + var snapshot = new Snapshot(projectId, "repository", new SnapshotId("name", "uuid")); var snapshotInfo = createSnapshotInfo(snapshot, Boolean.FALSE); @@ -287,7 +297,7 @@ public void testNotAllowToRestoreGlobalStateFromSnapshotWithoutOne() { ); assertThat( exception.getMessage(), - equalTo("[default:name:name/uuid] cannot restore global state since the snapshot was created without global state") + equalTo("[" + projectId + ":name:name/uuid] cannot restore global state since the snapshot was created without global state") ); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 81f4f743d821a..8be366c8144fc 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -2702,7 +2702,8 @@ public boolean clusterHasFeature(ClusterState state, NodeFeature feature) { EmptySystemIndices.INSTANCE, indicesService, mock(FileSettingsService.class), - threadPool + threadPool, + false ); actions.put( TransportPutMappingAction.TYPE, @@ -2755,7 +2756,14 @@ public boolean clusterHasFeature(ClusterState state, NodeFeature feature) { ); actions.put( TransportRestoreSnapshotAction.TYPE, - new TransportRestoreSnapshotAction(transportService, clusterService, threadPool, restoreService, actionFilters) + new TransportRestoreSnapshotAction( + transportService, + clusterService, + threadPool, + restoreService, + actionFilters, + TestProjectResolvers.DEFAULT_PROJECT_ONLY + ) ); actions.put( TransportDeleteIndexAction.TYPE, diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index 6c56724a6f20a..803596668171d 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; @@ -607,7 +608,7 @@ public void testCcrRepositoryFetchesSnapshotShardSizeFromIndexShardStoreStats() .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true) ) .quiet(true); - restoreService.restoreSnapshot(restoreRequest, ActionListener.noop()); + restoreService.restoreSnapshot(ProjectId.DEFAULT, restoreRequest, ActionListener.noop()); waitForRestoreInProgress.get(30L, TimeUnit.SECONDS); clusterService.removeListener(listener); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index 1ccf945e106c0..5b3ed6d3ed9d4 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -24,11 +24,13 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.injection.guice.Inject; @@ -240,8 +242,12 @@ private void createFollowerIndex( mdBuilder.put(updatedDataStream); }; } + @FixForMultiProject( + description = "CCR may not be in scope for multi-project though we haven't made the decision explicitly yet. See also ES-12139" + ) + final ProjectId projectId = ProjectId.DEFAULT; threadPool.executor(ThreadPool.Names.SNAPSHOT_META) - .execute(ActionRunnable.wrap(delegatelistener, l -> restoreService.restoreSnapshot(restoreRequest, l, updater))); + .execute(ActionRunnable.wrap(delegatelistener, l -> restoreService.restoreSnapshot(projectId, restoreRequest, l, updater))); } private void afterRestoreStarted( diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 644a2a074ac59..b037e7204a906 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -825,36 +826,40 @@ protected PlainActionFuture startRestore( RestoreSnapshotRequest restoreSnapshotRequest ) { final var future = new PlainActionFuture(); - restoreService.restoreSnapshot(restoreSnapshotRequest, future.delegateFailure((delegate, restoreCompletionResponse) -> { - assertNull(restoreCompletionResponse.restoreInfo()); - // this would only be non-null if the restore was a no-op, but that would be a test bug - final Snapshot snapshot = restoreCompletionResponse.snapshot(); - final String uuid = restoreCompletionResponse.uuid(); - final ClusterStateListener clusterStateListener = new ClusterStateListener() { - @Override - public void clusterChanged(ClusterChangedEvent changedEvent) { - final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), uuid); - final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), uuid); - - assertNotNull(prevEntry); - // prevEntry could be null if there was a master failover and (due to batching) we missed the cluster state update - // that completed the restore, but that doesn't happen in these tests - if (newEntry == null) { - clusterService.removeListener(this); - Map shards = prevEntry.shards(); - RestoreInfo ri = new RestoreInfo( - prevEntry.snapshot().getSnapshotId().getName(), - prevEntry.indices(), - shards.size(), - shards.size() - RestoreService.failedShards(shards) - ); - logger.debug("restore of [{}] completed", snapshot); - delegate.onResponse(ri); - } // else restore not completed yet, wait for next cluster state update - } - }; - clusterService.addListener(clusterStateListener); - })); + restoreService.restoreSnapshot( + ProjectId.DEFAULT, + restoreSnapshotRequest, + future.delegateFailure((delegate, restoreCompletionResponse) -> { + assertNull(restoreCompletionResponse.restoreInfo()); + // this would only be non-null if the restore was a no-op, but that would be a test bug + final Snapshot snapshot = restoreCompletionResponse.snapshot(); + final String uuid = restoreCompletionResponse.uuid(); + final ClusterStateListener clusterStateListener = new ClusterStateListener() { + @Override + public void clusterChanged(ClusterChangedEvent changedEvent) { + final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), uuid); + final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), uuid); + + assertNotNull(prevEntry); + // prevEntry could be null if there was a master failover and (due to batching) we missed the cluster state update + // that completed the restore, but that doesn't happen in these tests + if (newEntry == null) { + clusterService.removeListener(this); + Map shards = prevEntry.shards(); + RestoreInfo ri = new RestoreInfo( + prevEntry.snapshot().getSnapshotId().getName(), + prevEntry.indices(), + shards.size(), + shards.size() - RestoreService.failedShards(shards) + ); + logger.debug("restore of [{}] completed", snapshot); + delegate.onResponse(ri); + } // else restore not completed yet, wait for next cluster state update + } + }; + clusterService.addListener(clusterStateListener); + }) + ); return future; } From b90426f661e12f848e6daac7c6f8d2fb24cdb2e9 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 22 Jul 2025 12:34:09 +1000 Subject: [PATCH 2/7] fix internal snapshots info service --- .../InternalSnapshotsInfoService.java | 5 +- .../InternalSnapshotsInfoServiceTests.java | 50 +++++++++++-------- .../cluster/ESAllocationTestCase.java | 20 ++++++++ .../test/ClusterServiceUtils.java | 10 ++++ 4 files changed, 63 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/InternalSnapshotsInfoService.java b/server/src/main/java/org/elasticsearch/snapshots/InternalSnapshotsInfoService.java index 5f9afca87c41b..febd60fa4226c 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/InternalSnapshotsInfoService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/InternalSnapshotsInfoService.java @@ -211,7 +211,10 @@ private class FetchingSnapshotShardSizeRunnable extends AbstractRunnable { @Override protected void doRun() throws Exception { - final Repository repository = repositoriesService.repository(snapshotShard.snapshot.getRepository()); + final Repository repository = repositoriesService.repository( + snapshotShard.snapshot().getProjectId(), + snapshotShard.snapshot.getRepository() + ); logger.debug("fetching snapshot shard size for {}", snapshotShard); final long snapshotShardSize = repository.getShardSnapshotStatus( diff --git a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java index eebc999e0e570..4c8075f4375ac 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java @@ -17,6 +17,8 @@ import org.elasticsearch.cluster.TestShardRoutingRoleStrategies; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -82,13 +84,15 @@ public class InternalSnapshotsInfoServiceTests extends ESTestCase { private ClusterService clusterService; private RepositoriesService repositoriesService; private RerouteService rerouteService; + private ProjectId projectId; @Before @Override public void setUp() throws Exception { super.setUp(); threadPool = new TestThreadPool(getTestName()); - clusterService = ClusterServiceUtils.createClusterService(threadPool); + projectId = randomProjectIdOrDefault(); + clusterService = ClusterServiceUtils.createClusterService(threadPool, projectId); repositoriesService = mock(RepositoriesService.class); rerouteService = (reason, priority, listener) -> listener.onResponse(null); } @@ -138,7 +142,7 @@ public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotI return IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0L, expectedShardSizes[shardId.id()], null); } }; - when(repositoriesService.repository("_repo")).thenReturn(mockRepository); + when(repositoriesService.repository(projectId, "_repo")).thenReturn(mockRepository); applyClusterState("add-unassigned-shards", clusterState -> addUnassignedShards(clusterState, indexName, numberOfShards)); waitForMaxActiveGenericThreads(Math.min(numberOfShards, maxConcurrentFetches)); @@ -163,7 +167,7 @@ public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotI final SnapshotShardSizeInfo snapshotShardSizeInfo = snapshotsInfoService.snapshotShardSizes(); for (int i = 0; i < numberOfShards; i++) { - final ShardRouting shardRouting = clusterService.state().routingTable().index(indexName).shard(i).primaryShard(); + final ShardRouting shardRouting = clusterService.state().routingTable(projectId).index(indexName).shard(i).primaryShard(); assertThat(snapshotShardSizeInfo.getShardSize(shardRouting), equalTo(expectedShardSizes[i])); assertThat(snapshotShardSizeInfo.getShardSize(shardRouting, Long.MIN_VALUE), equalTo(expectedShardSizes[i])); } @@ -196,7 +200,7 @@ public void testErroneousSnapshotShardSizes() throws Exception { @Override public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) { final InternalSnapshotsInfoService.SnapshotShard snapshotShard = new InternalSnapshotsInfoService.SnapshotShard( - new Snapshot("_repo", snapshotId), + new Snapshot(projectId, "_repo", snapshotId), indexId, shardId ); @@ -210,7 +214,7 @@ public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotI } } }; - when(repositoriesService.repository("_repo")).thenReturn(mockRepository); + when(repositoriesService.repository(projectId, "_repo")).thenReturn(mockRepository); final Thread addSnapshotRestoreIndicesThread = new Thread(() -> { int remainingShards = maxShardsToCreate; @@ -249,7 +253,7 @@ public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotI for (Map.Entry snapshotShard : results.entrySet()) { final ShardId shardId = snapshotShard.getKey().shardId(); final ShardRouting shardRouting = clusterService.state() - .routingTable() + .routingTable(projectId) .index(shardId.getIndexName()) .shard(shardId.id()) .primaryShard(); @@ -286,7 +290,7 @@ public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotI return IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0L, randomNonNegativeLong(), null); } }; - when(repositoriesService.repository("_repo")).thenReturn(mockRepository); + when(repositoriesService.repository(projectId, "_repo")).thenReturn(mockRepository); for (int i = 0; i < randomIntBetween(1, 10); i++) { final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); @@ -320,13 +324,13 @@ public void testCleanUpSnapshotShardSizes() throws Exception { @Override public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) { if (randomBoolean()) { - throw new SnapshotException(new Snapshot("_repo", snapshotId), "simulated"); + throw new SnapshotException(new Snapshot(projectId, "_repo", snapshotId), "simulated"); } else { return IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0L, randomNonNegativeLong(), null); } } }; - when(repositoriesService.repository("_repo")).thenReturn(mockRepository); + when(repositoriesService.repository(projectId, "_repo")).thenReturn(mockRepository); final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( Settings.EMPTY, @@ -367,7 +371,12 @@ public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotI ); applyClusterState( "starting shards for " + indexName, - clusterState -> ESAllocationTestCase.startInitializingShardsAndReroute(allocationService, clusterState, indexName) + clusterState -> ESAllocationTestCase.startInitializingShardsAndReroute( + allocationService, + clusterState, + projectId, + indexName + ) ); RoutingNodes routingNodes = clusterService.state().getRoutingNodes(); assertTrue(RoutingNodesHelper.shardsWithState(routingNodes, ShardRoutingState.UNASSIGNED).isEmpty()); @@ -375,7 +384,7 @@ public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotI } else { // simulate deletion of the index applyClusterState("delete index " + indexName, clusterState -> deleteIndex(clusterState, indexName)); - assertFalse(clusterService.state().metadata().getProject().hasIndex(indexName)); + assertFalse(clusterService.state().metadata().getProject(projectId).hasIndex(indexName)); } assertThat(snapshotsInfoService.numberOfKnownSnapshotShardSizes(), equalTo(0)); @@ -405,7 +414,7 @@ private void waitForMaxActiveGenericThreads(final int nbActive) throws Exception } private ClusterState addUnassignedShards(final ClusterState currentState, String indexName, int numberOfShards) { - assertThat(currentState.metadata().getProject().hasIndex(indexName), is(false)); + assertThat(currentState.metadata().getProject(projectId).hasIndex(indexName), is(false)); final IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) .settings(indexSettings(IndexVersion.current(), numberOfShards, 0).put(SETTING_CREATION_DATE, System.currentTimeMillis())); @@ -414,21 +423,20 @@ private ClusterState addUnassignedShards(final ClusterState currentState, String indexMetadataBuilder.putInSyncAllocationIds(i, Collections.singleton(AllocationId.newInitializing().getId())); } - final Metadata.Builder metadata = Metadata.builder(currentState.metadata()) - .put(indexMetadataBuilder.build(), true) - .generateClusterUuidIfNeeded(); + final Metadata.Builder metadata = Metadata.builder(currentState.metadata()).generateClusterUuidIfNeeded(); + final ProjectMetadata.Builder projectMetadata = metadata.getProject(projectId).put(indexMetadataBuilder.build(), true); final RecoverySource.SnapshotRecoverySource recoverySource = new RecoverySource.SnapshotRecoverySource( UUIDs.randomBase64UUID(random()), - new Snapshot("_repo", new SnapshotId(randomAlphaOfLength(5), UUIDs.randomBase64UUID(random()))), + new Snapshot(projectId, "_repo", new SnapshotId(randomAlphaOfLength(5), UUIDs.randomBase64UUID(random()))), IndexVersion.current(), new IndexId(indexName, UUIDs.randomBase64UUID(random())) ); - final IndexMetadata indexMetadata = metadata.get(indexName); + final IndexMetadata indexMetadata = projectMetadata.get(indexName); final Index index = indexMetadata.getIndex(); - final RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable()); + final RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable(projectId)); routingTable.add( IndexRoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, index) .initializeAsNewRestore(indexMetadata, recoverySource, new HashSet<>()) @@ -454,7 +462,7 @@ private ClusterState addUnassignedShards(final ClusterState currentState, String return ClusterState.builder(currentState) .putCustom(RestoreInProgress.TYPE, restores.build()) - .routingTable(routingTable.build()) + .putRoutingTable(projectId, routingTable.build()) .metadata(metadata) .build(); } @@ -470,8 +478,8 @@ private ClusterState demoteMasterNode(final ClusterState currentState) { private ClusterState deleteIndex(final ClusterState currentState, final String indexName) { return ClusterState.builder(currentState) - .metadata(Metadata.builder(currentState.metadata()).remove(indexName)) - .routingTable(RoutingTable.builder(currentState.routingTable()).remove(indexName).build()) + .putProjectMetadata(ProjectMetadata.builder(currentState.metadata().getProject(projectId)).remove(indexName)) + .putRoutingTable(projectId, RoutingTable.builder(currentState.routingTable(projectId)).remove(indexName).build()) .build(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java index 8a49db652374e..a2d81e0203f15 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; @@ -301,6 +302,7 @@ public static ClusterState startInitializingShardsAndReroute( * * @return the cluster state after completing the reroute. */ + @Deprecated(forRemoval = true) public static ClusterState startInitializingShardsAndReroute( AllocationService allocationService, ClusterState clusterState, @@ -313,6 +315,24 @@ public static ClusterState startInitializingShardsAndReroute( ); } + /** + * Mark all initializing shards for the given index as started, then perform a reroute (which may start some other shards initializing). + * + * @return the cluster state after completing the reroute. + */ + public static ClusterState startInitializingShardsAndReroute( + AllocationService allocationService, + ClusterState clusterState, + ProjectId projectId, + String index + ) { + return startShardsAndReroute( + allocationService, + clusterState, + clusterState.routingTable(projectId).index(index).shardsWithState(INITIALIZING) + ); + } + /** * Mark the given shards as started, then perform a reroute (which may start some other shards initializing). * diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java index 2ef255b46f868..7cdb01af8b965 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java @@ -109,6 +109,16 @@ public static ClusterService createClusterService(ThreadPool threadPool) { return createClusterService(threadPool, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); } + public static ClusterService createClusterService(ThreadPool threadPool, ProjectId projectId) { + return createClusterService( + threadPool, + DiscoveryNodeUtils.create("node", "node"), + Settings.EMPTY, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + projectId + ); + } + public static ClusterService createClusterService(ThreadPool threadPool, DiscoveryNode localNode) { return createClusterService(threadPool, localNode, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); } From a2982ea4c188b750d10fe3295332f9b3c096c6b8 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 22 Jul 2025 13:43:16 +1000 Subject: [PATCH 3/7] update transport recovery action and unmute --- .../indices/recovery/TransportRecoveryAction.java | 12 ++++++++---- .../build.gradle | 5 +---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java index 4dbbb317cd1f8..142493a738cfe 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardsIterator; import org.elasticsearch.cluster.service.ClusterService; @@ -43,6 +44,7 @@ public class TransportRecoveryAction extends TransportBroadcastByNodeAction { private final IndicesService indicesService; + private final ProjectResolver projectResolver; @Inject public TransportRecoveryAction( @@ -50,7 +52,8 @@ public TransportRecoveryAction( TransportService transportService, IndicesService indicesService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + ProjectResolver projectResolver ) { super( RecoveryAction.NAME, @@ -62,6 +65,7 @@ public TransportRecoveryAction( transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); this.indicesService = indicesService; + this.projectResolver = projectResolver; } @Override @@ -110,16 +114,16 @@ protected void shardOperation(RecoveryRequest request, ShardRouting shardRouting @Override protected ShardsIterator shards(ClusterState state, RecoveryRequest request, String[] concreteIndices) { - return state.routingTable().allShardsIncludingRelocationTargets(concreteIndices); + return state.routingTable(projectResolver.getProjectId()).allShardsIncludingRelocationTargets(concreteIndices); } @Override protected ClusterBlockException checkGlobalBlock(ClusterState state, RecoveryRequest request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + return state.blocks().globalBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_READ); } @Override protected ClusterBlockException checkRequestBlock(ClusterState state, RecoveryRequest request, String[] concreteIndices) { - return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); + return state.blocks().indicesBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_READ, concreteIndices); } } diff --git a/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle b/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle index 1992998c9a85d..52964b061e66b 100644 --- a/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle +++ b/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle @@ -46,19 +46,16 @@ restResources { tasks.named("yamlRestTest").configure { ArrayList blacklist = [ /* These tests don't work on multi-project yet - we need to go through each of them and make them work */ - '^cat.recovery/*/*', '^cluster.desired_balance/10_basic/*', '^cluster.stats/10_basic/snapshot stats reported in get cluster stats', '^data_stream/40_supported_apis/Verify shard stores api', // uses _shard_stores API '^indices.get_alias/10_basic/Get alias against closed indices', // Does NOT work with security enabled, see also core-rest-tests-with-security - '^indices.recovery/*/*', +// '^indices.recovery/*/*', '^indices.resolve_cluster/*/*', '^indices.resolve_cluster/*/*/*', '^indices.shard_stores/*/*', '^migration/*/*', - '^snapshot.restore/*/*', '^synonyms/*/*', - '^tsdb/30_snapshot/*', '^update_by_query/80_scripting/Update all docs with one deletion and one noop using a stored script', // scripting is not project aware yet // The following tests are muted because the functionality that they are testing is not available in a multi-project setup From 0011bcfec92f5a4ba0dfabb5ce55128ac4d07628 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 22 Jul 2025 15:31:08 +1000 Subject: [PATCH 4/7] fix project metadata serialization --- .../repositories/blobstore/BlobStoreRepository.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index dea02908d215f..43cb1a7d3caec 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -134,6 +134,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.LeakTracker; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -377,7 +378,10 @@ public static String getRepositoryDataBlobName(long repositoryGeneration) { METADATA_NAME_FORMAT, (repoName, parser) -> ProjectMetadata.Builder.fromXContent(parser), projectMetadata -> ChunkedToXContent.wrapAsToXContent( - params -> Iterators.concat(Iterators.single((builder, ignored) -> builder.field("id", projectMetadata.id()))) + params -> Iterators.concat( + Iterators.single((ToXContent) (builder, p) -> builder.field("id", projectMetadata.id())), + projectMetadata.toXContentChunked(params) + ) ) ); From f96a3bc86962fb0cc3d1b64cf68bb0a5f936b31e Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 22 Jul 2025 15:36:52 +1000 Subject: [PATCH 5/7] unmute --- .../core-rest-tests-with-multiple-projects/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle b/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle index 52964b061e66b..61a43a4b4e5d1 100644 --- a/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle +++ b/x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle @@ -50,7 +50,6 @@ tasks.named("yamlRestTest").configure { '^cluster.stats/10_basic/snapshot stats reported in get cluster stats', '^data_stream/40_supported_apis/Verify shard stores api', // uses _shard_stores API '^indices.get_alias/10_basic/Get alias against closed indices', // Does NOT work with security enabled, see also core-rest-tests-with-security -// '^indices.recovery/*/*', '^indices.resolve_cluster/*/*', '^indices.resolve_cluster/*/*/*', '^indices.shard_stores/*/*', From 415fdf7bae67681dda76aab06a11ac2da5a1c27b Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 22 Jul 2025 16:45:21 +1000 Subject: [PATCH 6/7] fix tests --- .../elasticsearch/datastreams/DataStreamsSnapshotsIT.java | 5 +++-- .../java/org/elasticsearch/snapshots/RestoreService.java | 5 ----- .../SearchableSnapshotsRepositoryIntegTests.java | 4 +++- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java index bdac4809834cd..3aa46d8475569 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java @@ -47,6 +47,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; import org.elasticsearch.snapshots.RestoreInfo; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInProgressException; import org.elasticsearch.snapshots.SnapshotInfo; @@ -1394,7 +1395,7 @@ public void testWarningHeaderOnRestoreWithoutTemplates() throws Exception { .get(); RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); - SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId(); + Snapshot snapshot = createSnapshotResponse.getSnapshotInfo().snapshot(); assertEquals(RestStatus.OK, status); assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices()); @@ -1423,7 +1424,7 @@ public void testWarningHeaderOnRestoreWithoutTemplates() throws Exception { client, request, "Snapshot [" - + snapshotId + + snapshot + "] contains data stream [" + datastreamName + "] but custer does not have a matching index " diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 4c0253b7a3826..6ee8d2667d948 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -65,7 +65,6 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; @@ -2002,10 +2001,6 @@ private static void ensureSearchableSnapshotsRestorable( if (Objects.equals(snapshotUuid, otherSnapshotUuid) == false) { continue; // other index is backed by a different snapshot, skip } - @FixForMultiProject( - description = "Repo UUID might be null and repo name is not sufficiently unique. " - + "But searchable snapshots may not be supported in multi-project. See also ES-12138" - ) final String otherRepositoryUuid = otherSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY); final String otherRepositoryName = otherSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY); if (matchRepository(repositoryUuid, repositoryName, otherRepositoryUuid, otherRepositoryName) == false) { diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java index d21e8d1fd939f..40741c1e1cf91 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsRepositoryIntegTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.set.Sets; @@ -27,6 +28,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; +import static org.elasticsearch.repositories.ProjectRepo.projectRepoString; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -374,7 +376,7 @@ public void testRestoreSearchableSnapshotIndexWithDifferentSettingsConflicts() t exception.getMessage(), containsString( "cannot change value of [index.store.snapshot.delete_searchable_snapshot] when restoring searchable snapshot [" - + repository + + projectRepoString(ProjectId.DEFAULT, repository) + ':' + snapshotOfMountedIndices + "] as index [mounted-" From 6667aa489b2015a00ee9bd8d6eab8ebae5775f88 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 25 Jul 2025 11:54:18 +1000 Subject: [PATCH 7/7] tighten assertion --- .../java/org/elasticsearch/snapshots/RestoreService.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 6ee8d2667d948..23e9f23a3e46b 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -1628,10 +1628,13 @@ private void ensureSnapshotNotDeleted(ClusterState currentState) { private void applyGlobalStateRestore(ClusterState currentState, Metadata.Builder mdBuilder, ProjectId projectId) { final var projectBuilder = mdBuilder.getProject(projectId); if (metadata.persistentSettings() != null) { - assert deserializeProjectMetadata == false || metadata.persistentSettings().isEmpty() + assert (deserializeProjectMetadata == false && ProjectId.DEFAULT.equals(projectId)) + || metadata.persistentSettings().isEmpty() : "Inconsistent deserializeProjectMetadata [" + deserializeProjectMetadata - + "] and cluster level persistent settings " + + "], project [" + + projectId + + "], and cluster level persistent settings " + metadata.persistentSettings(); Settings settings = metadata.persistentSettings(); if (request.skipOperatorOnlyState()) {