From 488f2be4f3c711665d37a6184c510dd22da6d716 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Tue, 23 Jun 2026 11:58:32 +0800 Subject: [PATCH 1/5] Fix snapshot loader to keep tsfile and mods on the same data dir When IoTConsensus snapshot fragments land in different receive folders, share fileTarget across dirs so companion mods follow their tsfile. Add unit and cluster IT coverage for delete visibility after region migrate. --- .../env/cluster/config/MppCommonConfig.java | 18 ++ .../env/cluster/config/MppDataNodeConfig.java | 6 + .../cluster/config/MppSharedCommonConfig.java | 20 ++ .../it/env/cluster/node/DataNodeWrapper.java | 2 - .../env/remote/config/RemoteCommonConfig.java | 12 ++ .../remote/config/RemoteDataNodeConfig.java | 5 + .../apache/iotdb/itbase/env/CommonConfig.java | 6 + .../iotdb/itbase/env/DataNodeConfig.java | 2 + ...gionMigrateWithDeletionMultiDataDirIT.java | 180 ++++++++++++++++++ .../dataregion/snapshot/SnapshotLoader.java | 23 ++- .../snapshot/IoTDBSnapshotTest.java | 80 ++++++++ 11 files changed, 345 insertions(+), 9 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateWithDeletionMultiDataDirIT.java diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java index 4dd7f8571627d..9e8e42c4ffdbc 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java @@ -584,6 +584,24 @@ public CommonConfig setPipeAutoSplitFullEnabled(boolean pipeAutoSplitFullEnabled return this; } + @Override + public CommonConfig setMinFolderOccupiedSpaceCacheRefreshIntervalMs( + long minFolderOccupiedSpaceCacheRefreshIntervalMs) { + setProperty( + "min_folder_occupied_space_cache_refresh_interval_ms", + String.valueOf(minFolderOccupiedSpaceCacheRefreshIntervalMs)); + return this; + } + + @Override + public CommonConfig setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( + int minFolderOccupiedSpaceCacheRefreshSelectionThreshold) { + setProperty( + "min_folder_occupied_space_cache_refresh_selection_threshold", + String.valueOf(minFolderOccupiedSpaceCacheRefreshSelectionThreshold)); + return this; + } + @Override public CommonConfig setQueryMemoryProportion(String queryMemoryProportion) { setProperty("chunk_timeseriesmeta_free_memory_proportion", queryMemoryProportion); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java index 5e418072a7d97..4caef4cc1f9f7 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java @@ -143,4 +143,10 @@ public DataNodeConfig setQueryCostStatWindow(int queryCostStatWindow) { setProperty("query_cost_stat_window", String.valueOf(queryCostStatWindow)); return this; } + + @Override + public DataNodeConfig setDnDataDirs(String dnDataDirs) { + setProperty("dn_data_dirs", dnDataDirs); + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java index 8e0398de30973..58e4ffe7c2d03 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java @@ -603,6 +603,26 @@ public CommonConfig setPipeAutoSplitFullEnabled(boolean pipeAutoSplitFullEnabled return this; } + @Override + public CommonConfig setMinFolderOccupiedSpaceCacheRefreshIntervalMs( + long minFolderOccupiedSpaceCacheRefreshIntervalMs) { + dnConfig.setMinFolderOccupiedSpaceCacheRefreshIntervalMs( + minFolderOccupiedSpaceCacheRefreshIntervalMs); + cnConfig.setMinFolderOccupiedSpaceCacheRefreshIntervalMs( + minFolderOccupiedSpaceCacheRefreshIntervalMs); + return this; + } + + @Override + public CommonConfig setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( + int minFolderOccupiedSpaceCacheRefreshSelectionThreshold) { + dnConfig.setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( + minFolderOccupiedSpaceCacheRefreshSelectionThreshold); + cnConfig.setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( + minFolderOccupiedSpaceCacheRefreshSelectionThreshold); + return this; + } + @Override public CommonConfig setQueryMemoryProportion(String queryMemoryProportion) { dnConfig.setQueryMemoryProportion(queryMemoryProportion); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index dac6cf3fcc3cc..d205044bd5290 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -44,7 +44,6 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_PROPERTIES; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_CONNECTION_TIMEOUT_MS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_CONSENSUS_DIR; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_DATA_DIRS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_DATA_REGION_CONSENSUS_PORT; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_JOIN_CLUSTER_RETRY_INTERVAL_MS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_METRIC_INTERNAL_REPORTER_TYPE; @@ -125,7 +124,6 @@ public DataNodeWrapper( immutableNodeProperties.setProperty(IoTDBConstant.DN_SEED_CONFIG_NODE, seedConfigNode); immutableNodeProperties.setProperty(DN_SYSTEM_DIR, MppBaseConfig.NULL_VALUE); - immutableNodeProperties.setProperty(DN_DATA_DIRS, MppBaseConfig.NULL_VALUE); immutableNodeProperties.setProperty(DN_CONSENSUS_DIR, MppBaseConfig.NULL_VALUE); immutableNodeProperties.setProperty(DN_WAL_DIRS, MppBaseConfig.NULL_VALUE); immutableNodeProperties.setProperty(DN_TRACING_DIR, MppBaseConfig.NULL_VALUE); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java index a2d10c01009da..bd301501a39ef 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java @@ -427,6 +427,18 @@ public CommonConfig setPipeAutoSplitFullEnabled(boolean pipeAutoSplitFullEnabled return this; } + @Override + public CommonConfig setMinFolderOccupiedSpaceCacheRefreshIntervalMs( + long minFolderOccupiedSpaceCacheRefreshIntervalMs) { + return this; + } + + @Override + public CommonConfig setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( + int minFolderOccupiedSpaceCacheRefreshSelectionThreshold) { + return this; + } + @Override public CommonConfig setQueryMemoryProportion(String queryMemoryProportion) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java index bba4c964f9578..aa73d962dbab1 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java @@ -98,4 +98,9 @@ public DataNodeConfig setDataNodeMemoryProportion(String dataNodeMemoryProportio public DataNodeConfig setQueryCostStatWindow(int queryCostStatWindow) { return this; } + + @Override + public DataNodeConfig setDnDataDirs(String dnDataDirs) { + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index 5b51a6a8cf916..61ebf6278ff80 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -185,6 +185,12 @@ CommonConfig setPipeConnectorRequestSliceThresholdBytes( CommonConfig setPipeAutoSplitFullEnabled(boolean pipeAutoSplitFullEnabled); + CommonConfig setMinFolderOccupiedSpaceCacheRefreshIntervalMs( + long minFolderOccupiedSpaceCacheRefreshIntervalMs); + + CommonConfig setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( + int minFolderOccupiedSpaceCacheRefreshSelectionThreshold); + CommonConfig setQueryMemoryProportion(String queryMemoryProportion); CommonConfig setDataNodeMemoryProportion(String dataNodeMemoryProportion); diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java index d57015b13964e..01a6114c20679 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java @@ -53,4 +53,6 @@ DataNodeConfig setLoadTsFileAnalyzeSchemaMemorySizeInBytes( DataNodeConfig setDataNodeMemoryProportion(String dataNodeMemoryProportion); DataNodeConfig setQueryCostStatWindow(int queryCostStatWindow); + + DataNodeConfig setDnDataDirs(String dnDataDirs); } diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateWithDeletionMultiDataDirIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateWithDeletionMultiDataDirIT.java new file mode 100644 index 0000000000000..43b2d1bc992a9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateWithDeletionMultiDataDirIT.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1; + +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.apache.tsfile.utils.Pair; +import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getAllDataNodes; +import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getDataRegionMapWithLeader; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBRegionMigrateWithDeletionMultiDataDirIT { + + private static final String MULTI_DATA_DIRS = + "data/datanode/data/disk0,data/datanode/data/disk1,data/datanode/data/disk2"; + + @Before + public void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDataReplicationFactor(2) + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + EnvFactory.getEnv().getConfig().getDataNodeConfig().setDnDataDirs(MULTI_DATA_DIRS); + EnvFactory.getEnv().initClusterEnvironment(1, 3); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testRegionMigratePreservesDeletionWithMultiDataDirs() throws Exception { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE test"); + statement.execute("USE test"); + statement.execute("CREATE TABLE t1 (s1 INT64 FIELD)"); + statement.execute("INSERT INTO t1 (time, s1) VALUES (100, 100), (200, 200), (300, 300)"); + statement.execute("FLUSH"); + statement.execute("DELETE FROM t1 WHERE time <= 200"); + statement.execute("FLUSH"); + + Map>> dataRegionMapWithLeader = + getDataRegionMapWithLeader(statement); + int dataRegionIdForTest = + dataRegionMapWithLeader.keySet().stream().max(Integer::compareTo).orElseThrow(); + assertDeletionVisibleOnAllReplicas(statement, dataRegionIdForTest, 1); + + Pair> leaderAndNodes = dataRegionMapWithLeader.get(dataRegionIdForTest); + Set allDataNodes = getAllDataNodes(statement); + int leaderId = leaderAndNodes.getLeft(); + int followerId = + leaderAndNodes.getRight().stream().filter(id -> id != leaderId).findFirst().orElseThrow(); + int destDataNodeId = + allDataNodes.stream() + .filter(id -> id != leaderId && id != followerId) + .findFirst() + .orElseThrow(); + + statement.execute( + String.format( + "migrate region %d from %d to %d", dataRegionIdForTest, leaderId, destDataNodeId)); + + final int finalDestDataNodeId = destDataNodeId; + Awaitility.await() + .atMost(10, TimeUnit.MINUTES) + .pollDelay(1, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try (ResultSet showRegions = statement.executeQuery("SHOW REGIONS")) { + boolean migrated = false; + while (showRegions.next()) { + if (showRegions.getInt("RegionId") == dataRegionIdForTest + && showRegions.getInt("DataNodeId") == finalDestDataNodeId) { + migrated = true; + break; + } + } + Assert.assertTrue(migrated); + } + }); + + assertDeletionVisibleOnAllReplicas(statement, dataRegionIdForTest, 1); + } + } + + private void assertDeletionVisibleOnAllReplicas( + Statement statement, int dataRegionId, int expectedCount) throws Exception { + Set replicaDataNodeIds = getReplicaDataNodeIds(statement, dataRegionId); + for (int dataNodeId : replicaDataNodeIds) { + DataNodeWrapper dataNodeWrapper = + EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeId).orElseThrow(); + Awaitility.await() + .atMost(2, TimeUnit.MINUTES) + .pollDelay(500, TimeUnit.MILLISECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted(() -> assertDeletionVisibleOnReplica(dataNodeWrapper, expectedCount)); + } + } + + private void assertDeletionVisibleOnReplica(DataNodeWrapper dataNodeWrapper, int expectedCount) + throws Exception { + try (Connection connection = + EnvFactory.getEnv() + .getConnection( + dataNodeWrapper, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + BaseEnv.TABLE_SQL_DIALECT); + Statement dataNodeStatement = connection.createStatement()) { + dataNodeStatement.execute("USE test"); + try (ResultSet countResultSet = dataNodeStatement.executeQuery("SELECT COUNT(s1) FROM t1")) { + Assert.assertTrue(countResultSet.next()); + Assert.assertEquals(expectedCount, countResultSet.getLong(1)); + } + try (ResultSet deletedRangeResultSet = + dataNodeStatement.executeQuery("SELECT s1 FROM t1 WHERE time <= 200")) { + Assert.assertFalse(deletedRangeResultSet.next()); + } + } + } + + private Set getReplicaDataNodeIds(Statement statement, int dataRegionId) + throws Exception { + Set replicaDataNodeIds = new HashSet<>(); + try (ResultSet showRegions = statement.executeQuery("SHOW REGIONS")) { + while (showRegions.next()) { + if ("DataRegion".equals(showRegions.getString("Type")) + && showRegions.getInt("RegionId") == dataRegionId) { + replicaDataNodeIds.add(showRegions.getInt("DataNodeId")); + } + } + } + Assert.assertFalse(replicaDataNodeIds.isEmpty()); + return replicaDataNodeIds; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java index 17136cd24fbb4..588b0e21956f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java @@ -146,9 +146,14 @@ private DataRegion loadSnapshotFromMultipleDirs() { try { deleteAllFilesInDataDirs(); LOGGER.info(StorageEngineMessages.REMOVE_ALL_DATA_FILES_IN_ORIGINAL_DIR); + // IoTConsensus may spread the fragments of one snapshot across several receive folders. + // The fileTarget map must be shared across all of them so that a tsfile and its companion + // files (resource, exclusive mods, etc.) are relinked to the same data dir even when their + // fragments were received on different disks. + Map fileTarget = new HashMap<>(); for (String path : snapshotPaths) { File snapshotDir = new File(path); - createLinksFromSnapshotDirToDataDirWithoutLog(snapshotDir); + createLinksFromSnapshotDirToDataDirWithoutLog(snapshotDir, fileTarget); loadCompressionRatio(snapshotDir); } return loadSnapshot(); @@ -170,7 +175,7 @@ private DataRegion loadSnapshotWithoutLog() { } LOGGER.info(StorageEngineMessages.MOVING_SNAPSHOT_FILE_TO_DATA_DIRS); File snapshotDir = new File(snapshotPath); - createLinksFromSnapshotDirToDataDirWithoutLog(snapshotDir); + createLinksFromSnapshotDirToDataDirWithoutLog(snapshotDir, new HashMap<>()); loadCompressionRatio(snapshotDir); return loadSnapshot(); } catch (IOException | DiskSpaceInsufficientException e) { @@ -294,7 +299,8 @@ private void deleteAllFilesInDataDirs() throws IOException { } } - private void createLinksFromSnapshotDirToDataDirWithoutLog(File sourceDir) + private void createLinksFromSnapshotDirToDataDirWithoutLog( + File sourceDir, Map fileTarget) throws IOException, DiskSpaceInsufficientException { if (!sourceDir.exists()) { throw new IOException( @@ -340,7 +346,7 @@ private void createLinksFromSnapshotDirToDataDirWithoutLog(File sourceDir) + dataRegionId + File.separator + timePartitionFolder.getName(); - createLinksFromSnapshotToSourceDir(targetSuffix, files, folderManager); + createLinksFromSnapshotToSourceDir(targetSuffix, files, folderManager, fileTarget); } } @@ -359,7 +365,7 @@ private void createLinksFromSnapshotDirToDataDirWithoutLog(File sourceDir) + dataRegionId + File.separator + timePartitionFolder.getName(); - createLinksFromSnapshotToSourceDir(targetSuffix, files, folderManager); + createLinksFromSnapshotToSourceDir(targetSuffix, files, folderManager, fileTarget); } } } @@ -406,8 +412,11 @@ private File createLinksFromSnapshotToSourceDir( } private void createLinksFromSnapshotToSourceDir( - String targetSuffix, File[] files, FolderManager folderManager) throws IOException { - Map fileTarget = new HashMap<>(); + String targetSuffix, + File[] files, + FolderManager folderManager, + Map fileTarget) + throws IOException { for (File file : files) { checkTsFileResourceExists(file); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java index d042d2a2ae8fa..101f5c325d41a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java @@ -258,6 +258,51 @@ public void testLoadSnapshotSpreadAcrossReceiveFolders() loadSnapshotSpreadAcrossReceiveFolders(false); } + /** + * When IoTConsensus spreads a tsfile and its exclusive mods across different receive folders, the + * loader must still relink them to the same data dir. Otherwise the mods file is not found next + * to the tsfile and deletion markers are silently ignored. + */ + @Test + public void testLoadSnapshotKeepsTsFileAndModsOnSameDataDirWhenFragmentsAreSpread() + throws IOException, WriteProcessException { + String[][] originDataDirs = IoTDBDescriptor.getInstance().getConfig().getTierDataDirs(); + IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(testDataDirs); + TierManager.getInstance().resetFolders(); + String recvBase0 = "target" + File.separator + "recv-snapshot-mods-0"; + String recvBase1 = "target" + File.separator + "recv-snapshot-mods-1"; + File recvFolder0 = new File(recvBase0, SNAPSHOT_DIR_NAME); + File recvFolder1 = new File(recvBase1, SNAPSHOT_DIR_NAME); + try { + Assert.assertTrue(recvFolder0.mkdirs()); + Assert.assertTrue(recvFolder1.mkdirs()); + + writeSnapshotFragmentWithExclusiveModsSpread(recvFolder0.getAbsolutePath(), 0, recvFolder1); + + DataRegion dataRegion = + new SnapshotLoader( + Arrays.asList(recvFolder0.getAbsolutePath(), recvFolder1.getAbsolutePath()), + testSgName, + "0") + .loadSnapshotForStateMachine(); + + Assert.assertNotNull(dataRegion); + TsFileResource resource = dataRegion.getTsFileManager().getTsFileList(true).get(0); + File tsFile = resource.getTsFile(); + File modsFile = + org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile + .getExclusiveMods(tsFile); + Assert.assertTrue(modsFile.exists()); + Assert.assertEquals( + tsFile.getParentFile().getAbsolutePath(), modsFile.getParentFile().getAbsolutePath()); + } finally { + FileUtils.recursivelyDeleteFolder(recvBase0); + FileUtils.recursivelyDeleteFolder(recvBase1); + IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(originDataDirs); + TierManager.getInstance().resetFolders(); + } + } + /** * The fragments of one snapshot are disjoint across the receive folders, so the order in which * the folders are relinked must not change the loaded data. This loads the same spread snapshot @@ -343,6 +388,41 @@ private void writeSnapshotFragment(String recvSnapshotDir, int i) resource.serialize(); } + private void writeSnapshotFragmentWithExclusiveModsSpread( + String tsFileRecvSnapshotDir, int i, File modsRecvFolder) + throws IOException, WriteProcessException { + writeSnapshotFragment(tsFileRecvSnapshotDir, i); + String tsFileName = String.format("%d-%d-0-0.tsfile", i + 1, i + 1); + File tsFile = + new File( + tsFileRecvSnapshotDir + + File.separator + + "sequence" + + File.separator + + testSgName + + File.separator + + "0" + + File.separator + + "0" + + File.separator + + tsFileName); + File sourceMods = + org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile.getExclusiveMods( + tsFile); + Assert.assertTrue(sourceMods.exists() || sourceMods.createNewFile()); + + File targetModsDir = + new File( + modsRecvFolder, + "sequence" + File.separator + testSgName + File.separator + "0" + File.separator + "0"); + Assert.assertTrue(targetModsDir.exists() || targetModsDir.mkdirs()); + Files.copy( + sourceMods.toPath(), + new File(targetModsDir, sourceMods.getName()).toPath(), + java.nio.file.StandardCopyOption.REPLACE_EXISTING); + Files.delete(sourceMods.toPath()); + } + @Ignore("Need manual execution to specify different disks") @Test public void testLoadSnapshotNoHardLink() From 0dbedb63c15b997a0c30afb2d1640dd397e6498e Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Tue, 23 Jun 2026 12:03:19 +0800 Subject: [PATCH 2/5] Revert unrelated occupied space cache IT config helpers These setters were only needed for an uncommitted manual IT and are not used by the snapshot loader fix. --- .../env/cluster/config/MppCommonConfig.java | 18 ----------------- .../cluster/config/MppSharedCommonConfig.java | 20 ------------------- .../env/remote/config/RemoteCommonConfig.java | 12 ----------- .../apache/iotdb/itbase/env/CommonConfig.java | 6 ------ 4 files changed, 56 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java index 9e8e42c4ffdbc..4dd7f8571627d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java @@ -584,24 +584,6 @@ public CommonConfig setPipeAutoSplitFullEnabled(boolean pipeAutoSplitFullEnabled return this; } - @Override - public CommonConfig setMinFolderOccupiedSpaceCacheRefreshIntervalMs( - long minFolderOccupiedSpaceCacheRefreshIntervalMs) { - setProperty( - "min_folder_occupied_space_cache_refresh_interval_ms", - String.valueOf(minFolderOccupiedSpaceCacheRefreshIntervalMs)); - return this; - } - - @Override - public CommonConfig setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( - int minFolderOccupiedSpaceCacheRefreshSelectionThreshold) { - setProperty( - "min_folder_occupied_space_cache_refresh_selection_threshold", - String.valueOf(minFolderOccupiedSpaceCacheRefreshSelectionThreshold)); - return this; - } - @Override public CommonConfig setQueryMemoryProportion(String queryMemoryProportion) { setProperty("chunk_timeseriesmeta_free_memory_proportion", queryMemoryProportion); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java index 58e4ffe7c2d03..8e0398de30973 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java @@ -603,26 +603,6 @@ public CommonConfig setPipeAutoSplitFullEnabled(boolean pipeAutoSplitFullEnabled return this; } - @Override - public CommonConfig setMinFolderOccupiedSpaceCacheRefreshIntervalMs( - long minFolderOccupiedSpaceCacheRefreshIntervalMs) { - dnConfig.setMinFolderOccupiedSpaceCacheRefreshIntervalMs( - minFolderOccupiedSpaceCacheRefreshIntervalMs); - cnConfig.setMinFolderOccupiedSpaceCacheRefreshIntervalMs( - minFolderOccupiedSpaceCacheRefreshIntervalMs); - return this; - } - - @Override - public CommonConfig setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( - int minFolderOccupiedSpaceCacheRefreshSelectionThreshold) { - dnConfig.setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( - minFolderOccupiedSpaceCacheRefreshSelectionThreshold); - cnConfig.setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( - minFolderOccupiedSpaceCacheRefreshSelectionThreshold); - return this; - } - @Override public CommonConfig setQueryMemoryProportion(String queryMemoryProportion) { dnConfig.setQueryMemoryProportion(queryMemoryProportion); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java index bd301501a39ef..a2d10c01009da 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java @@ -427,18 +427,6 @@ public CommonConfig setPipeAutoSplitFullEnabled(boolean pipeAutoSplitFullEnabled return this; } - @Override - public CommonConfig setMinFolderOccupiedSpaceCacheRefreshIntervalMs( - long minFolderOccupiedSpaceCacheRefreshIntervalMs) { - return this; - } - - @Override - public CommonConfig setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( - int minFolderOccupiedSpaceCacheRefreshSelectionThreshold) { - return this; - } - @Override public CommonConfig setQueryMemoryProportion(String queryMemoryProportion) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index 61ebf6278ff80..5b51a6a8cf916 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -185,12 +185,6 @@ CommonConfig setPipeConnectorRequestSliceThresholdBytes( CommonConfig setPipeAutoSplitFullEnabled(boolean pipeAutoSplitFullEnabled); - CommonConfig setMinFolderOccupiedSpaceCacheRefreshIntervalMs( - long minFolderOccupiedSpaceCacheRefreshIntervalMs); - - CommonConfig setMinFolderOccupiedSpaceCacheRefreshSelectionThreshold( - int minFolderOccupiedSpaceCacheRefreshSelectionThreshold); - CommonConfig setQueryMemoryProportion(String queryMemoryProportion); CommonConfig setDataNodeMemoryProportion(String dataNodeMemoryProportion); From 2b2d9d42cf48cb361d5de465e16ab746cc831548 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Tue, 23 Jun 2026 12:23:35 +0800 Subject: [PATCH 3/5] Skip missing recv paths when selecting snapshot receive folder --- .../MinFolderOccupiedSpaceFirstStrategy.java | 14 ++++++++-- ...rOccupiedSpaceFirstStrategyRealFsTest.java | 28 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java index 285b8b1d5b07a..e3f360f471294 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.utils.TestOnly; import java.io.IOException; +import java.io.UncheckedIOException; /** * Selects the folder with the least occupied space. @@ -117,8 +118,8 @@ private void refreshOccupiedSpace() { continue; } try { - cachedOccupiedSpace[i] = JVMCommonUtils.getOccupiedSpace(folder); - } catch (IOException e) { + cachedOccupiedSpace[i] = computeOccupiedSpace(folder); + } catch (IOException | UncheckedIOException e) { LOGGER.error(UtilMessages.CANNOT_CALCULATE_OCCUPIED_SPACE, folder, e); cachedOccupiedSpace[i] = Long.MAX_VALUE; } @@ -127,6 +128,15 @@ private void refreshOccupiedSpace() { lastRefreshTimeMs = System.currentTimeMillis(); } + /** + * Computes the occupied space of a single folder. Extracted as a seam over the static {@link + * JVMCommonUtils#getOccupiedSpace} so the failure modes of the underlying {@code Files.walk} can + * be simulated deterministically in tests. + */ + protected long computeOccupiedSpace(String folder) throws IOException { + return JVMCommonUtils.getOccupiedSpace(folder); + } + @TestOnly public void setRefreshIntervalMs(long refreshIntervalMs) { this.refreshIntervalMs = refreshIntervalMs; diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java index 6f4797233c563..18e48edf342be 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java @@ -30,7 +30,9 @@ import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -39,6 +41,7 @@ import java.util.stream.Stream; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; /** * Integration test that drives {@link MinFolderOccupiedSpaceFirstStrategy} and {@link @@ -124,4 +127,29 @@ public void cachesOccupiedSpaceAndRefreshesAgainstRealFiles() // and correctly avoids the now-largest folder 0, picking the least occupied folder 1. assertEquals(1, strategy.nextFolderIndex()); } + + @Test + public void deprioritizesFolderWhenWalkThrowsUncheckedIOException() + throws DiskSpaceInsufficientException { + String failingFolder = folders.get(0); + + MinFolderOccupiedSpaceFirstStrategy strategy = + new MinFolderOccupiedSpaceFirstStrategy() { + @Override + protected long computeOccupiedSpace(String folder) throws IOException { + if (folder.equals(failingFolder)) { + // Mirror what Files.walk does when a directory disappears mid-traversal. + throw new UncheckedIOException(new NoSuchFileException(folder)); + } + return super.computeOccupiedSpace(folder); + } + }; + strategy.setFolders(folders); + + // Selection must not propagate the UncheckedIOException; it must skip the failing folder 0 and + // fall back to a healthy one (folder 1, ties broken by index since folders 1 and 2 are empty). + int selected = strategy.nextFolderIndex(); + assertNotEquals(0, selected); + assertEquals(1, selected); + } } From a55b902f8f17048c132de834f8c36e8627c73258 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Tue, 23 Jun 2026 12:32:40 +0800 Subject: [PATCH 4/5] Inline getOccupiedSpace call in MinFolderOccupiedSpaceFirstStrategy --- .../MinFolderOccupiedSpaceFirstStrategy.java | 11 +------- ...rOccupiedSpaceFirstStrategyRealFsTest.java | 28 ------------------- 2 files changed, 1 insertion(+), 38 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java index e3f360f471294..6a893d47aa1a2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java @@ -118,7 +118,7 @@ private void refreshOccupiedSpace() { continue; } try { - cachedOccupiedSpace[i] = computeOccupiedSpace(folder); + cachedOccupiedSpace[i] = JVMCommonUtils.getOccupiedSpace(folder); } catch (IOException | UncheckedIOException e) { LOGGER.error(UtilMessages.CANNOT_CALCULATE_OCCUPIED_SPACE, folder, e); cachedOccupiedSpace[i] = Long.MAX_VALUE; @@ -128,15 +128,6 @@ private void refreshOccupiedSpace() { lastRefreshTimeMs = System.currentTimeMillis(); } - /** - * Computes the occupied space of a single folder. Extracted as a seam over the static {@link - * JVMCommonUtils#getOccupiedSpace} so the failure modes of the underlying {@code Files.walk} can - * be simulated deterministically in tests. - */ - protected long computeOccupiedSpace(String folder) throws IOException { - return JVMCommonUtils.getOccupiedSpace(folder); - } - @TestOnly public void setRefreshIntervalMs(long refreshIntervalMs) { this.refreshIntervalMs = refreshIntervalMs; diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java index 18e48edf342be..6f4797233c563 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java @@ -30,9 +30,7 @@ import java.io.File; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -41,7 +39,6 @@ import java.util.stream.Stream; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; /** * Integration test that drives {@link MinFolderOccupiedSpaceFirstStrategy} and {@link @@ -127,29 +124,4 @@ public void cachesOccupiedSpaceAndRefreshesAgainstRealFiles() // and correctly avoids the now-largest folder 0, picking the least occupied folder 1. assertEquals(1, strategy.nextFolderIndex()); } - - @Test - public void deprioritizesFolderWhenWalkThrowsUncheckedIOException() - throws DiskSpaceInsufficientException { - String failingFolder = folders.get(0); - - MinFolderOccupiedSpaceFirstStrategy strategy = - new MinFolderOccupiedSpaceFirstStrategy() { - @Override - protected long computeOccupiedSpace(String folder) throws IOException { - if (folder.equals(failingFolder)) { - // Mirror what Files.walk does when a directory disappears mid-traversal. - throw new UncheckedIOException(new NoSuchFileException(folder)); - } - return super.computeOccupiedSpace(folder); - } - }; - strategy.setFolders(folders); - - // Selection must not propagate the UncheckedIOException; it must skip the failing folder 0 and - // fall back to a healthy one (folder 1, ties broken by index since folders 1 and 2 are empty). - int selected = strategy.nextFolderIndex(); - assertNotEquals(0, selected); - assertEquals(1, selected); - } } From 76344bb8698608d90f5489cc1ef8035e4be6016a Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Tue, 23 Jun 2026 16:05:53 +0800 Subject: [PATCH 5/5] Fix IoTDBSnapshotTest to match 4-arg createLinksFromSnapshotToSourceDir signature The test reflectively invoked the old 3-arg createLinksFromSnapshotToSourceDir(String, File[], FolderManager), but the production method now takes a Map fileTarget as a 4th argument, causing NoSuchMethodException. Look up the current signature, pass a fileTarget map, and assert the shared fileKey is recorded exactly once at one of the data dirs. --- .../dataregion/snapshot/IoTDBSnapshotTest.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java index 101f5c325d41a..ba7c9310945aa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java @@ -52,7 +52,9 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.apache.iotdb.consensus.iot.IoTConsensusServerImpl.SNAPSHOT_DIR_NAME; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; @@ -591,12 +593,23 @@ public void testFileTargetRecordedWhenHardLinkSuccess() throws Exception { Method method = SnapshotLoader.class.getDeclaredMethod( - "createLinksFromSnapshotToSourceDir", String.class, File[].class, FolderManager.class); + "createLinksFromSnapshotToSourceDir", + String.class, + File[].class, + FolderManager.class, + Map.class); method.setAccessible(true); SnapshotLoader loader = new SnapshotLoader("dummy", "root.testsg", "0"); - method.invoke(loader, targetSuffix, files, folderManager); + // Tracks fileKey -> chosen data dir, so files sharing a fileKey land in the same dir. + Map fileTarget = new HashMap<>(); + method.invoke(loader, targetSuffix, files, folderManager, fileTarget); + + // The shared fileKey must be recorded exactly once, pointing at one of the data dirs. + String fileKey = tsFile.getName().split("\\.")[0]; + Assert.assertEquals(1, fileTarget.size()); + Assert.assertTrue(Arrays.asList(dataDirs).contains(fileTarget.get(fileKey))); // verify: only ONE dir contains all three files int hitDirCount = 0;