From c0d26455966d4461df876eba1de30a23aefdce6d Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Thu, 22 Oct 2026 18:11:36 +0000 Subject: [PATCH 1/2] Support external tablet files outside Accumulo volumes Add an external tablet file type and persist its marker in tablet metadata while preserving compatibility with existing file references. Modified GC to track external candidates, retain files with live tablet or scan references, and delete them when unreferenced. Updated Volume replacement and disk usage for external paths. Added unit and integration coverage for metadata, GC, scans, compactions, and volume replacement. This change is related to #6570. Follow-on changes will modify bulk import to use this feature. --- .../apache/accumulo/core/gc/Reference.java | 5 + .../accumulo/core/gc/ReferenceFile.java | 38 +++- .../core/metadata/ExternalTabletFile.java | 48 +++++ .../core/metadata/ReferencedTabletFile.java | 79 ++++++- .../metadata/ScanServerRefTabletFile.java | 23 ++- .../core/metadata/StoredTabletFile.java | 73 +++++-- .../core/metadata/StoredTabletFileTest.java | 32 +++ .../schema/ReferencedTabletFileTest.java | 20 ++ .../apache/accumulo/server/fs/VolumeUtil.java | 3 + .../accumulo/server/gc/GcVolumeUtil.java | 16 ++ .../server/metadata/RootGcCandidates.java | 39 +++- .../server/metadata/ServerAmpleImpl.java | 18 +- .../accumulo/server/util/TableDiskUsage.java | 9 +- .../server/metadata/RootGcCandidatesTest.java | 55 +++++ .../server/util/TableDiskUsageTest.java | 21 ++ .../java/org/apache/accumulo/gc/GCRun.java | 38 +++- .../gc/GarbageCollectionAlgorithm.java | 59 ++++-- .../accumulo/gc/GarbageCollectionTest.java | 49 +++++ .../gc/SimpleGarbageCollectorTest.java | 15 ++ .../org/apache/accumulo/test/VolumeIT.java | 83 ++++++++ .../compaction/ExternalCompaction_1_IT.java | 99 +++++++++ .../test/functional/GarbageCollectorIT.java | 192 ++++++++++++++++++ 22 files changed, 928 insertions(+), 86 deletions(-) create mode 100644 core/src/main/java/org/apache/accumulo/core/metadata/ExternalTabletFile.java create mode 100644 server/base/src/test/java/org/apache/accumulo/server/metadata/RootGcCandidatesTest.java diff --git a/core/src/main/java/org/apache/accumulo/core/gc/Reference.java b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java index a6ee689ea6b..2915bfff6c2 100644 --- a/core/src/main/java/org/apache/accumulo/core/gc/Reference.java +++ b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java @@ -36,6 +36,11 @@ public interface Reference { */ boolean isScan(); + /** + * Returns true when this reference points to a file outside Accumulo's defined Volumes. + */ + boolean isExternal(); + /** * Get the {@link TableId} of the reference. */ diff --git a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java index 492e67c2e4b..5a4ed5dba0d 100644 --- a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java +++ b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java @@ -18,9 +18,10 @@ */ package org.apache.accumulo.core.gc; -import java.util.Objects; +import static java.util.Objects.requireNonNull; import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.metadata.ReferencedTabletFile; import org.apache.accumulo.core.metadata.ScanServerRefTabletFile; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.schema.MetadataSchema; @@ -35,20 +36,33 @@ public class ReferenceFile implements Reference, Comparable { public final TableId tableId; // 2a public final boolean isScan; public final boolean isDirectory; + public final boolean isExternal; // the exact path from the file reference string that is stored in the metadata protected final String metadataPath; protected ReferenceFile(TableId tableId, String metadataPath, boolean isScan, boolean isDirectory) { - this.tableId = Objects.requireNonNull(tableId); - this.metadataPath = Objects.requireNonNull(metadataPath); + this(tableId, metadataPath, isScan, isDirectory, false); + } + + protected ReferenceFile(TableId tableId, String metadataPath, boolean isScan, boolean isDirectory, + boolean isExternal) { + this.tableId = requireNonNull(tableId); + this.metadataPath = requireNonNull(metadataPath); this.isScan = isScan; this.isDirectory = isDirectory; + this.isExternal = isExternal; } public static ReferenceFile forFile(TableId tableId, StoredTabletFile tabletFile) { - return new ReferenceFile(tableId, tabletFile.getMetadataPath(), false, false); + return new ReferenceFile(tableId, tabletFile.getMetadataPath(), false, false, + tabletFile.isExternal()); + } + + public static ReferenceFile forFile(TableId tableId, ReferencedTabletFile tabletFile) { + return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), false, false, + tabletFile.isExternal()); } public static ReferenceFile forFile(TableId tableId, Path metadataPathPath) { @@ -56,11 +70,18 @@ public static ReferenceFile forFile(TableId tableId, Path metadataPathPath) { } public static ReferenceFile forScan(TableId tableId, ScanServerRefTabletFile tabletFile) { - return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true, false); + return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true, false, + tabletFile.isExternal()); } public static ReferenceFile forScan(TableId tableId, StoredTabletFile tabletFile) { - return new ReferenceFile(tableId, tabletFile.getMetadataPath(), true, false); + return new ReferenceFile(tableId, tabletFile.getMetadataPath(), true, false, + tabletFile.isExternal()); + } + + public static ReferenceFile forScan(TableId tableId, ReferencedTabletFile tabletFile) { + return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true, false, + tabletFile.isExternal()); } public static ReferenceFile forScan(TableId tableId, Path metadataPathPath) { @@ -82,6 +103,11 @@ public boolean isScan() { return isScan; } + @Override + public boolean isExternal() { + return isExternal; + } + @Override public TableId getTableId() { return tableId; diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/ExternalTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/ExternalTabletFile.java new file mode 100644 index 00000000000..6458cab166f --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/metadata/ExternalTabletFile.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.metadata; + +import static java.util.Objects.requireNonNull; + +import org.apache.accumulo.core.data.Range; +import org.apache.hadoop.fs.Path; + +/** + * A reference to a tablet file whose path is not in Accumulo's conventional tables directory. An + * external file is still owned by Accumulo and is eligible for garbage collection when no longer + * referenced. + */ +public class ExternalTabletFile extends ReferencedTabletFile { + + public ExternalTabletFile(Path path) { + this(path, new Range()); + } + + public ExternalTabletFile(Path path, Range range) { + super(requireNonNull(path), range, true); + } + + public static ExternalTabletFile of(Path path) { + return new ExternalTabletFile(path); + } + + public static ExternalTabletFile of(Path path, Range range) { + return new ExternalTabletFile(path, range); + } +} diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java index 2bcae4be9d5..32f3df8c4b8 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.core.metadata; +import static java.util.Objects.requireNonNull; import static org.apache.accumulo.core.Constants.HDFS_TABLES_DIR; import java.net.URI; @@ -154,6 +155,7 @@ public static FileParts parsePath(Path filePath) { } private final FileParts parts; + private final boolean external; private static final Logger log = LoggerFactory.getLogger(ReferencedTabletFile.class); private static final String HDFS_TABLES_DIR_NAME = HDFS_TABLES_DIR.substring(1); @@ -171,26 +173,48 @@ public ReferencedTabletFile(Path metaPath) { * qualify an absolute path or create a new file. */ public ReferencedTabletFile(Path metaPath, Range range) { - super(Objects.requireNonNull(metaPath), range); - log.trace("Parsing TabletFile from {}", metaPath); - parts = parsePath(metaPath); + this(metaPath, range, false); + } + + protected ReferencedTabletFile(Path metaPath, Range range, boolean external) { + super(requireNonNull(metaPath), range); + this.external = external; + if (external) { + Preconditions.checkArgument(metaPath.toUri().getScheme() != null, + constructErrorMsg(metaPath)); + ValidationUtil.validateFileName(metaPath.getName()); + Preconditions.checkArgument(!isVolumePath(metaPath), + "External tablet file path must not use the Accumulo table layout: %s", metaPath); + parts = null; + } else { + log.trace("Parsing TabletFile from {}", metaPath); + parts = parsePath(metaPath); + } } public String getVolume() { + Preconditions.checkState(!external, "External tablet files do not have an Accumulo volume"); return parts.getVolume(); } public TableId getTableId() { + Preconditions.checkState(!external, + "External tablet files do not have a path-derived table ID"); return parts.getTableId(); } public String getTabletDir() { + Preconditions.checkState(!external, "External tablet files do not have a tablet directory"); return parts.getTabletDir(); } @Override public String getFileName() { - return parts.getFileName(); + return external ? path.getName() : parts.getFileName(); + } + + public boolean isExternal() { + return external; } /** @@ -198,14 +222,15 @@ public String getFileName() { * metadata. */ public String getNormalizedPathStr() { - return parts.getNormalizedPath(); + return external ? path.toString() : parts.getNormalizedPath(); } /** * New file was written to metadata so return a StoredTabletFile */ public StoredTabletFile insert() { - return StoredTabletFile.of(getPath(), getRange()); + return external ? StoredTabletFile.ofExternal(getPath(), getRange()) + : StoredTabletFile.of(getPath(), getRange()); } @Override @@ -216,7 +241,7 @@ public int compareTo(ReferencedTabletFile o) { @Override public boolean equals(Object obj) { if (obj instanceof ReferencedTabletFile that) { - return parts.getNormalizedPath().equals(that.parts.getNormalizedPath()) + return getNormalizedPathStr().equals(that.getNormalizedPathStr()) && external == that.external && range.equals(that.range); } return false; @@ -224,12 +249,48 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash(parts.getNormalizedPath(), range); + return Objects.hash(getNormalizedPathStr(), external, range); } @Override public String toString() { - return parts.getNormalizedPath(); + return getNormalizedPathStr(); + } + + /** + * Determines whether a path uses Accumulo's conventional + * {@code tables///} layout. + */ + public static boolean isVolumePath(Path path) { + try { + parsePath(path); + return true; + } catch (RuntimeException e) { + return false; + } + } + + /** + * Determines whether a path is a conventional Accumulo tablet directory. + */ + public static boolean isTabletDirPath(Path path) { + if (path == null || path.toUri().getScheme() == null) { + return false; + } + Path tableIdPath = path.getParent(); + if (tableIdPath == null) { + return false; + } + Path tablesPath = tableIdPath.getParent(); + if (tablesPath == null || !HDFS_TABLES_DIR_NAME.equals(tablesPath.getName())) { + return false; + } + try { + TableId.of(tableIdPath.getName()); + return true; + } catch (RuntimeException e) { + return false; + } } public static ReferencedTabletFile of(final Path path) { diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefTabletFile.java index c289c38a47e..a8ac7927e1e 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefTabletFile.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefTabletFile.java @@ -18,12 +18,15 @@ */ package org.apache.accumulo.core.metadata; +import static java.util.Objects.requireNonNull; + import java.net.URI; import java.util.Objects; import java.util.UUID; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -35,21 +38,23 @@ public class ScanServerRefTabletFile extends ReferencedTabletFile { private final Text uuid; public ScanServerRefTabletFile(UUID serverLockUUID, String serverAddress, String file) { - super(new Path(URI.create(file))); - this.serverAddress = new Text(serverAddress); - uuid = new Text(serverLockUUID.toString()); + this(new Path(URI.create(file)), new Text(serverAddress), + new Text(requireNonNull(serverLockUUID).toString())); } public ScanServerRefTabletFile(String file, String serverAddress, UUID serverLockUUID) { - super(new Path(URI.create(file))); - this.serverAddress = new Text(serverAddress); - this.uuid = new Text(serverLockUUID.toString()); + this(new Path(URI.create(file)), new Text(serverAddress), + new Text(requireNonNull(serverLockUUID).toString())); } public ScanServerRefTabletFile(Key k) { - super(new Path(URI.create(k.getColumnQualifier().toString()))); - serverAddress = k.getColumnFamily(); - uuid = k.getRow(); + this(new Path(URI.create(k.getColumnQualifier().toString())), k.getColumnFamily(), k.getRow()); + } + + private ScanServerRefTabletFile(Path path, Text serverAddress, Text uuid) { + super(path, new Range(), !ReferencedTabletFile.isVolumePath(path)); + this.serverAddress = serverAddress; + this.uuid = uuid; } public Mutation putMutation() { diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java index b1f80ccd75a..2184e43bbee 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.core.metadata; +import static java.util.Objects.requireNonNull; import static org.apache.accumulo.core.util.RowRangeUtil.requireKeyExtentDataRange; import java.io.ByteArrayOutputStream; @@ -76,10 +77,12 @@ private StoredTabletFile(TabletFileCq fileCq) { } private StoredTabletFile(String metadataEntry, TabletFileCq fileCq) { - super(Objects.requireNonNull(fileCq).path, fileCq.range); - this.metadataEntry = Objects.requireNonNull(metadataEntry); + super(requireNonNull(fileCq).path, fileCq.range); + this.metadataEntry = requireNonNull(metadataEntry); this.metadataEntryPath = fileCq.path.toString(); - this.referencedTabletFile = ReferencedTabletFile.of(getPath(), fileCq.range); + this.referencedTabletFile = + fileCq.type == TabletFileType.EXTERNAL ? ExternalTabletFile.of(getPath(), fileCq.range) + : ReferencedTabletFile.of(getPath(), fileCq.range); } /** @@ -113,6 +116,10 @@ public TableId getTableId() { return referencedTabletFile.getTableId(); } + public boolean isExternal() { + return referencedTabletFile.isExternal(); + } + @Override public String getFileName() { return referencedTabletFile.getFileName(); @@ -154,7 +161,7 @@ public String toString() { } public static StoredTabletFile of(final Text metadataEntry) { - return new StoredTabletFile(Objects.requireNonNull(metadataEntry).toString()); + return new StoredTabletFile(requireNonNull(metadataEntry).toString()); } public static StoredTabletFile of(final String metadataEntry) { @@ -162,11 +169,26 @@ public static StoredTabletFile of(final String metadataEntry) { } public static StoredTabletFile of(final URI path, Range range) { - return of(new Path(Objects.requireNonNull(path)), range); + return of(new Path(requireNonNull(path)), range); } public static StoredTabletFile of(final Path path, Range range) { - return new StoredTabletFile(new TabletFileCq(Objects.requireNonNull(path), range)); + return new StoredTabletFile(new TabletFileCq(requireNonNull(path), range)); + } + + /** + * Create a stored file reference whose path does not use Accumulo's conventional table layout. + */ + public static StoredTabletFile ofExternal(final Path path) { + return ofExternal(path, new Range()); + } + + /** + * Create a stored file reference whose path does not use Accumulo's conventional table layout. + */ + public static StoredTabletFile ofExternal(final Path path, Range range) { + return new StoredTabletFile( + new TabletFileCq(requireNonNull(path), range, TabletFileType.EXTERNAL)); } public static StoredTabletFile of(final URI path) { @@ -181,14 +203,13 @@ public static StoredTabletFile of(final Path path) { private static TabletFileCq deserialize(String json) { final TabletFileCqMetadataGson metadata = - gson.fromJson(Objects.requireNonNull(json), TabletFileCqMetadataGson.class); + gson.fromJson(requireNonNull(json), TabletFileCqMetadataGson.class); // Check each field and provide better error messages if null as all fields should be set - Objects.requireNonNull(metadata.path, "Serialized StoredTabletFile path must not be null"); - Objects.requireNonNull(metadata.startRow, + requireNonNull(metadata.path, "Serialized StoredTabletFile path must not be null"); + requireNonNull(metadata.startRow, "Serialized StoredTabletFile range startRow must not be null"); - Objects.requireNonNull(metadata.endRow, - "Serialized StoredTabletFile range endRow must not be null"); + requireNonNull(metadata.endRow, "Serialized StoredTabletFile range endRow must not be null"); // Recreate the exact Range that was originally stored in Metadata. Stored ranges are originally // constructed with inclusive/exclusive for the start and end key inclusivity settings. @@ -203,7 +224,8 @@ private static TabletFileCq deserialize(String json) { // start row and false for end row regardless of what the startRowInclusive and endRowInclusive // flags are set to. return new TabletFileCq(new Path(URI.create(metadata.path)), - new Range(decodeRow(metadata.startRow), true, decodeRow(metadata.endRow), false)); + new Range(decodeRow(metadata.startRow), true, decodeRow(metadata.endRow), false), + metadata.type == null ? TabletFileType.STANDARD : metadata.type); } public static String serialize(String path) { @@ -211,17 +233,25 @@ public static String serialize(String path) { } public static String serialize(String path, Range range) { + return serialize(path, range, TabletFileType.STANDARD); + } + + private static String serialize(String path, Range range, TabletFileType type) { requireKeyExtentDataRange(range); final TabletFileCqMetadataGson metadata = new TabletFileCqMetadataGson(); - metadata.path = Objects.requireNonNull(path); + metadata.path = requireNonNull(path); metadata.startRow = encodeRow(range.getStartKey()); metadata.endRow = encodeRow(range.getEndKey()); + if (type != TabletFileType.STANDARD) { + metadata.type = type; + } return gson.toJson(metadata); } private static String serialize(TabletFileCq tabletFileCq) { - return serialize(Objects.requireNonNull(tabletFileCq).path.toString(), tabletFileCq.range); + requireNonNull(tabletFileCq); + return serialize(tabletFileCq.path.toString(), tabletFileCq.range, tabletFileCq.type); } /** @@ -277,16 +307,27 @@ public static boolean fileNeedsConversion(@NonNull final String candidate) { private static class TabletFileCq { public final Path path; public final Range range; + public final TabletFileType type; public TabletFileCq(Path path, Range range) { - this.path = Objects.requireNonNull(path); - this.range = Objects.requireNonNull(range); + this(path, range, TabletFileType.STANDARD); } + + public TabletFileCq(Path path, Range range, TabletFileType type) { + this.path = requireNonNull(path); + this.range = requireNonNull(range); + this.type = requireNonNull(type); + } + } + + private enum TabletFileType { + STANDARD, EXTERNAL } private static class TabletFileCqMetadataGson { private String path; private byte[] startRow; private byte[] endRow; + private TabletFileType type; } } diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java index 998c9eab112..ca67959cf10 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java @@ -18,9 +18,12 @@ */ package org.apache.accumulo.core.metadata; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.Test; public class StoredTabletFileTest { @@ -37,4 +40,33 @@ public void fileConversionTest() { assertFalse(StoredTabletFile.fileNeedsConversion(s31)); assertFalse(StoredTabletFile.fileNeedsConversion(s31_untrimmed)); } + + @Test + public void externalFileRoundTrip() { + Path path = new Path("hdfs://external-nn/imports/F0001.rf"); + StoredTabletFile stored = StoredTabletFile.ofExternal(path); + + assertTrue(stored.isExternal()); + assertTrue(stored.getTabletFile() instanceof ExternalTabletFile); + assertEquals(path, stored.getPath()); + assertTrue(stored.getMetadata().contains("\"type\":\"EXTERNAL\"")); + + StoredTabletFile roundTripped = StoredTabletFile.of(stored.getMetadata()); + assertTrue(roundTripped.isExternal()); + assertEquals(stored.getMetadata(), roundTripped.getMetadata()); + assertEquals(stored.getPath(), roundTripped.getPath()); + assertEquals(stored.getRange(), roundTripped.getRange()); + assertThrows(IllegalStateException.class, roundTripped::getTableId); + } + + @Test + public void insertingExternalFilePreservesMarker() { + ExternalTabletFile external = + ExternalTabletFile.of(new Path("file:///external/files/F0002.rf")); + + StoredTabletFile stored = external.insert(); + + assertTrue(stored.isExternal()); + assertTrue(StoredTabletFile.of(stored.getMetadata()).isExternal()); + } } diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java index 04c38f9f9b8..4dcf46ce32f 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java @@ -26,6 +26,7 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.metadata.ExternalTabletFile; import org.apache.accumulo.core.metadata.ReferencedTabletFile; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.hadoop.fs.Path; @@ -105,6 +106,25 @@ public void testFullPathWithVolume() { test(metadataEntry, volume, id, dir, filename); } + @Test + public void testExternalTabletFile() { + Path path = new Path("hdfs://external-nn/imports/F0001.rf"); + ExternalTabletFile external = ExternalTabletFile.of(path); + + assertTrue(external.isExternal()); + assertEquals(path, external.getPath()); + assertEquals("F0001.rf", external.getFileName()); + assertEquals(path.toString(), external.getNormalizedPathStr()); + assertThrows(IllegalStateException.class, external::getTableId); + assertThrows(IllegalStateException.class, external::getTabletDir); + assertThrows(IllegalStateException.class, external::getVolume); + + assertThrows(IllegalArgumentException.class, + () -> ExternalTabletFile.of(new Path("hdfs://nn/accumulo/tables/2a/t-1/F0001.rf"))); + assertThrows(IllegalArgumentException.class, + () -> ExternalTabletFile.of(new Path("/external/F0001.rf"))); + } + @Test public void testNormalizePath() { String uglyVolume = "hdfs://nn.somewhere.com:86753/accumulo/blah/.././/bad/bad2/../.././/////"; diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java index e41237b087b..4c9721c30b9 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java @@ -141,6 +141,9 @@ public static void volumeReplacementEvaluation(final Map replacements } for (Entry entry : tm.getFilesMap().entrySet()) { + if (entry.getKey().isExternal()) { + continue; + } log.trace("Evaluating file {} for replacement.", entry.getKey().getPath()); String metaPath = entry.getKey().getMetadata(); Path switchedPath = switchVolume(entry.getKey().getPath(), FileType.TABLE, replacements); diff --git a/server/base/src/main/java/org/apache/accumulo/server/gc/GcVolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/gc/GcVolumeUtil.java index 3f73b85ef4d..0d508d294da 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/gc/GcVolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/gc/GcVolumeUtil.java @@ -28,6 +28,22 @@ public class GcVolumeUtil { // AGCAV : Accumulo Garbage Collector All Volumes static final String ALL_VOLUMES_PREFIX = "agcav:/"; + public static final String EXTERNAL_CANDIDATE_PREFIX = "@external@"; + + public static String encodeExternalCandidatePath(String path) { + return EXTERNAL_CANDIDATE_PREFIX + path; + } + + public static boolean isExternalCandidatePath(String path) { + return path.startsWith(EXTERNAL_CANDIDATE_PREFIX); + } + + public static String decodeExternalCandidatePath(String path) { + if (!isExternalCandidatePath(path)) { + return path; + } + return path.substring(EXTERNAL_CANDIDATE_PREFIX.length()); + } public static Collection expandAllVolumesUri(VolumeManager fs, Path path) { if (path.toString().startsWith(ALL_VOLUMES_PREFIX)) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java index 7a2a17d08fd..4b79bccf3b2 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.gc.GcCandidate; import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.server.gc.GcVolumeUtil; import org.apache.hadoop.fs.Path; public class RootGcCandidates { @@ -77,18 +78,25 @@ public RootGcCandidates(String jsonString) { } public void add(Stream refs) { - refs.forEach(ref -> data.candidates - .computeIfAbsent(ref.getPath().getParent().toString(), k -> new TreeSet<>()) - .add(ref.getFileName())); + refs.forEach(ref -> { + String fileName = ref.getFileName(); + if (ref.isExternal()) { + fileName = GcVolumeUtil.EXTERNAL_CANDIDATE_PREFIX + fileName; + } + data.candidates.computeIfAbsent(ref.getPath().getParent().toString(), k -> new TreeSet<>()) + .add(fileName); + }); } public void remove(Stream refs) { - refs.map(GcCandidate::getPath).map(Path::new).forEach(path -> { - data.candidates.computeIfPresent(path.getParent().toString(), (key, values) -> { - values.remove(path.getName()); - return values.isEmpty() ? null : values; - }); - }); + refs.map(GcCandidate::getPath).map(GcVolumeUtil::decodeExternalCandidatePath).map(Path::new) + .forEach(path -> { + data.candidates.computeIfPresent(path.getParent().toString(), (key, values) -> { + values.remove(path.getName()); + values.remove(GcVolumeUtil.EXTERNAL_CANDIDATE_PREFIX + path.getName()); + return values.isEmpty() ? null : values; + }); + }); } public Stream sortedStream() { @@ -96,8 +104,17 @@ public Stream sortedStream() { return data.candidates.entrySet().stream().flatMap(entry -> { String parent = entry.getKey(); SortedSet names = entry.getValue(); - return names.stream() - .map(name -> new GcCandidate(parent + Path.SEPARATOR + name, uidGen.nextLong())); + return names.stream().map(name -> { + String candidatePath; + if (name.startsWith(GcVolumeUtil.EXTERNAL_CANDIDATE_PREFIX)) { + String actualPath = parent + Path.SEPARATOR + + name.substring(GcVolumeUtil.EXTERNAL_CANDIDATE_PREFIX.length()); + candidatePath = GcVolumeUtil.encodeExternalCandidatePath(actualPath); + } else { + candidatePath = parent + Path.SEPARATOR + name; + } + return new GcCandidate(candidatePath, uidGen.nextLong()); + }); }).sorted(); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java index 16f63126e4c..6dff8815d9b 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java @@ -52,6 +52,8 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.gc.GcVolumeUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -151,7 +153,9 @@ public void putGcFileAndDirCandidates(TableId tableId, Collection if (DataLevel.of(tableId) == DataLevel.ROOT) { // Directories are unexpected for the root tablet, so convert to stored tablet file mutateRootGcCandidates(rgcc -> rgcc.add(candidates.stream() - .map(reference -> StoredTabletFile.of(URI.create(reference.getMetadataPath()))))); + .map(reference -> reference.isExternal() + ? StoredTabletFile.ofExternal(new Path(reference.getMetadataPath())) + : StoredTabletFile.of(URI.create(reference.getMetadataPath()))))); return; } @@ -267,11 +271,19 @@ public Iterator getGcCandidates(DataLevel level) { @Override public Mutation createDeleteMutation(ReferenceFile tabletFilePathToRemove) { - return createDelMutation(ValidationUtil.validate(tabletFilePathToRemove).getMetadataPath()); + String path = ValidationUtil.validate(tabletFilePathToRemove).getMetadataPath(); + if (tabletFilePathToRemove.isExternal()) { + path = GcVolumeUtil.encodeExternalCandidatePath(path); + } + return createDelMutation(path); } public Mutation createDeleteMutation(StoredTabletFile pathToRemove) { - return createDelMutation(pathToRemove.getMetadataPath()); + String path = pathToRemove.getMetadataPath(); + if (pathToRemove.isExternal()) { + path = GcVolumeUtil.encodeExternalCandidatePath(path); + } + return createDelMutation(path); } private Mutation createDelMutation(String path) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java index b0aebbb2288..5de181aeb77 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java @@ -205,24 +205,25 @@ public static Map,Long> getDiskUsage(Set tableIds, // get the table referenced by the file which may not be the same as the current // table we are scanning if the file is shared between multiple tables - final TableId fileTableRef = stf.getTableId(); + final TableId fileTableRef = stf.isExternal() ? tableId : stf.getTableId(); + final String fileKey = stf.isExternal() ? stf.getMetadataPath() : stf.getFileName(); // if this is a ref to a different table than the one we are scanning then we need // to make sure the table is also linked for this shared file if the table is // part of the set of tables we are running du on so we can track shared usages if (!fileTableRef.equals(tableId) && tableIds.contains(fileTableRef)) { // link the table and the shared file for computing shared sizes - tdu.linkFileAndTable(fileTableRef, stf.getFileName()); + tdu.linkFileAndTable(fileTableRef, fileKey); } // link the file to the table we are scanning for - tdu.linkFileAndTable(tableId, stf.getFileName()); + tdu.linkFileAndTable(tableId, fileKey); // add the file size for the table if not already seen for this scan if (allFiles.add(stf)) { // This tracks the file size for individual files for computing shared file statistics // later - tdu.addFileSize(stf.getFileName(), dataFileValue.getSize()); + tdu.addFileSize(fileKey, dataFileValue.getSize()); } } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/metadata/RootGcCandidatesTest.java b/server/base/src/test/java/org/apache/accumulo/server/metadata/RootGcCandidatesTest.java new file mode 100644 index 00000000000..9897ef99c48 --- /dev/null +++ b/server/base/src/test/java/org/apache/accumulo/server/metadata/RootGcCandidatesTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.metadata; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Set; +import java.util.stream.Stream; + +import org.apache.accumulo.core.gc.GcCandidate; +import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.server.gc.GcVolumeUtil; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.Test; + +public class RootGcCandidatesTest { + + @Test + public void externalCandidateRoundTrip() { + StoredTabletFile external = StoredTabletFile.ofExternal(new Path("file:///external/F.rf")); + StoredTabletFile standard = + StoredTabletFile.of(new Path("file:///accumulo/tables/!0/t-0/F.rf")); + + RootGcCandidates candidates = new RootGcCandidates(); + candidates.add(Stream.of(external, standard)); + + RootGcCandidates reloaded = new RootGcCandidates(candidates.toJson()); + Set paths = reloaded.sortedStream().map(GcCandidate::getPath) + .collect(java.util.stream.Collectors.toSet()); + assertEquals(Set.of(GcVolumeUtil.encodeExternalCandidatePath(external.getPath().toString()), + standard.getPath().toString()), paths); + + String externalCandidate = + GcVolumeUtil.encodeExternalCandidatePath(external.getPath().toString()); + reloaded.remove(Stream.of(new GcCandidate(externalCandidate, 1))); + assertEquals(Set.of(standard.getPath().toString()), reloaded.sortedStream() + .map(GcCandidate::getPath).collect(java.util.stream.Collectors.toSet())); + } +} diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java index cab48d722fe..58c0c20d654 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java @@ -43,6 +43,7 @@ import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; +import org.apache.hadoop.fs.Path; import org.easymock.EasyMock; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -132,6 +133,26 @@ public void testMultipleVolumes() throws Exception { EasyMock.verify(client, mockTabletsMetadata); } + @Test + public void testExternalFile() throws Exception { + final ClientContext client = EasyMock.createMock(ClientContext.class); + EasyMock.expect(client.createTableIdToQualifiedNameMap()).andReturn(tableIdToNameMap); + final TabletsMetadata mockTabletsMetadata = mockTabletsMetadata(client, tableId1); + + StoredTabletFile external = + StoredTabletFile.ofExternal(new Path("hdfs://external/imports/F.rf")); + TabletMetadata tablet = TabletMetadata.builder(new KeyExtent(tableId1, null, null)) + .putFile(external, new DataFileValue(1024, 1)).build(); + mockTabletsMetadataIter(mockTabletsMetadata, List.of(tablet).iterator()); + EasyMock.replay(client, mockTabletsMetadata); + + Map,Long> result = TableDiskUsage.getDiskUsage(tableSet(tableId1), client); + + assertEquals(1024, getTotalUsage(result, tableId1)); + assertEquals(1024, result.get(tableNameSet(tableId1))); + EasyMock.verify(client, mockTabletsMetadata); + } + @Test public void testMetadataTable() throws Exception { final ClientContext client = EasyMock.createMock(ClientContext.class); diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java index f13cff3bd1e..f8a3926f8a5 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java @@ -56,6 +56,7 @@ import org.apache.accumulo.core.gc.Reference; import org.apache.accumulo.core.gc.ReferenceFile; import org.apache.accumulo.core.manager.state.tables.TableState; +import org.apache.accumulo.core.metadata.ReferencedTabletFile; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.SystemTables; @@ -304,8 +305,14 @@ public void deleteConfirmedCandidates(SortedMap confirmedDel try { Path fullPath; - Path switchedDelete = VolumeUtil.switchVolume(new Path(delete.getPath()), - VolumeManager.FileType.TABLE, replacements); + boolean externalCandidate = GcVolumeUtil.isExternalCandidatePath(delete.getPath()); + Path deletePath = new Path(GcVolumeUtil.decodeExternalCandidatePath(delete.getPath())); + Path switchedDelete = + (!externalCandidate && (ReferencedTabletFile.isVolumePath(deletePath) + || ReferencedTabletFile.isTabletDirPath(deletePath))) + ? VolumeUtil.switchVolume(deletePath, VolumeManager.FileType.TABLE, + replacements) + : null; if (switchedDelete != null) { // actually replacing the volumes in the metadata table would be tricky because the // entries would be different rows. So it could not be @@ -318,7 +325,7 @@ public void deleteConfirmedCandidates(SortedMap confirmedDel log.debug("Volume replaced {} -> {}", delete.getPath(), switchedDelete); fullPath = ValidationUtil.validate(switchedDelete); } else { - fullPath = new Path(ValidationUtil.validate(delete.getPath())); + fullPath = ValidationUtil.validate(deletePath); } for (Path pathToDel : GcVolumeUtil.expandAllVolumesUri(fs, fullPath)) { @@ -339,10 +346,18 @@ public void deleteConfirmedCandidates(SortedMap confirmedDel // this failure, we still want to remove the metadata entry removeFlag = true; errors.increment(); - String[] parts = pathToDel.toString().split(Constants.HDFS_TABLES_DIR)[1].split("/"); - if (parts.length > 2) { - TableId tableId = TableId.of(parts[1]); - String tabletDir = parts[2]; + if (ReferencedTabletFile.isVolumePath(pathToDel) + || ReferencedTabletFile.isTabletDirPath(pathToDel)) { + TableId tableId; + String tabletDir; + if (ReferencedTabletFile.isTabletDirPath(pathToDel)) { + tableId = TableId.of(pathToDel.getParent().getName()); + tabletDir = pathToDel.getName(); + } else { + var parts = ReferencedTabletFile.parsePath(pathToDel); + tableId = parts.getTableId(); + tabletDir = parts.getTabletDir(); + } TableState tableState = context.getTableManager().getTableState(tableId); if (tableState != TableState.UNKNOWN && tableState != TableState.DELETING) { // clone directories don't always exist @@ -350,9 +365,11 @@ public void deleteConfirmedCandidates(SortedMap confirmedDel log.debug("{} File doesn't exist: {}", fileActionPrefix, pathToDel); } } - } else { + } else if (!deletePath.equals(pathToDel)) { log.warn("{} Delete failed due to invalid file path format: {}", fileActionPrefix, delete.getPath()); + } else { + log.debug("{} File does not exist: {}", fileActionPrefix, pathToDel); } } } @@ -442,6 +459,11 @@ static void minimizeDeletes(SortedMap confirmedDeletes, progressCount++; Map.Entry entry = cdIter.next(); String relPath = entry.getKey(); + if (GcVolumeUtil.isExternalCandidatePath(entry.getValue().getPath())) { + lastDirRel = null; + lastDirAbs = null; + continue; + } Path absPath = new Path(entry.getValue().getPath()); if (SimpleGarbageCollector.isDir(relPath)) { diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java index 462d593cf4e..efc57bdb54b 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java @@ -42,6 +42,8 @@ import org.apache.accumulo.core.metadata.schema.Ample.GcCandidateType; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily; import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.server.gc.GcVolumeUtil; +import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,7 +57,7 @@ public class GarbageCollectionAlgorithm { private static final Logger log = LoggerFactory.getLogger(GarbageCollectionAlgorithm.class); /** - * This method takes a file or directory path and returns a relative path in 1 of 2 forms: + * This method takes a file or directory path and returns a relative path in one of 2 forms: * * 
    *      1- For files: table-id/tablet-directory/filename.rf
@@ -124,11 +126,16 @@ private SortedMap makeRelative(Collection candi
 
     for (GcCandidate candidate : candidates) {
       String relPath;
-      try {
-        relPath = makeRelative(candidate.getPath(), 0);
-      } catch (IllegalArgumentException iae) {
-        log.warn("Ignoring invalid deletion candidate {}", candidate);
-        continue;
+      String path = candidate.getPath();
+      if (GcVolumeUtil.isExternalCandidatePath(path)) {
+        relPath = makeExternalRelative(GcVolumeUtil.decodeExternalCandidatePath(path));
+      } else {
+        try {
+          relPath = makeRelative(path, 0);
+        } catch (IllegalArgumentException iae) {
+          log.warn("Ignoring invalid deletion candidate {}", candidate);
+          continue;
+        }
       }
       ret.put(relPath, candidate);
     }
@@ -136,6 +143,11 @@ private SortedMap makeRelative(Collection candi
     return ret;
   }
 
+  private String makeExternalRelative(String path) {
+    String normalized = new Path(path).toUri().normalize().toString();
+    return "external:" + normalized;
+  }
+
   private void removeCandidatesInUse(GarbageCollectionEnvironment gce,
       SortedMap candidateMap) throws InterruptedException {
 
@@ -161,15 +173,20 @@ private void removeCandidatesInUse(GarbageCollectionEnvironment gce,
           }
         } else {
           String reference = ref.getMetadataPath();
-          if (reference.startsWith("/")) {
-            log.debug("Candidate {} has a relative path, prepend tableId {}", reference,
-                ref.getTableId());
-            reference = "/" + ref.getTableId() + ref.getMetadataPath();
-          } else if (!reference.contains(":") && !reference.startsWith("../")) {
-            throw new RuntimeException("Bad file reference " + reference);
-          }
+          String relativePath;
+          if (ref.isExternal()) {
+            relativePath = makeExternalRelative(reference);
+          } else {
+            if (reference.startsWith("/")) {
+              log.debug("Candidate {} has a relative path, prepend tableId {}", reference,
+                  ref.getTableId());
+              reference = "/" + ref.getTableId() + ref.getMetadataPath();
+            } else if (!reference.contains(":") && !reference.startsWith("../")) {
+              throw new RuntimeException("Bad file reference " + reference);
+            }
 
-          String relativePath = makeRelative(reference, 3);
+            relativePath = makeRelative(reference, 3);
+          }
 
           // WARNING: This line is EXTREMELY IMPORTANT.
           // You MUST REMOVE candidates that are still in use
@@ -183,12 +200,14 @@ private void removeCandidatesInUse(GarbageCollectionEnvironment gce,
             }
           }
 
-          String dir = relativePath.substring(0, relativePath.lastIndexOf('/'));
-          GcCandidate gcT = candidateMap.remove(dir);
-          if (gcT != null) {
-            log.debug("Directory Candidate was still in use by file ref: {}", relativePath);
-            // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created
-            // once.
+          if (!ref.isExternal()) {
+            String dir = relativePath.substring(0, relativePath.lastIndexOf('/'));
+            GcCandidate gcT = candidateMap.remove(dir);
+            if (gcT != null) {
+              log.debug("Directory Candidate was still in use by file ref: {}", relativePath);
+              // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created
+              // once.
+            }
           }
         }
       });
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
index 72cc61e474b..3f56de80bd4 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
@@ -42,9 +42,11 @@
 import org.apache.accumulo.core.gc.Reference;
 import org.apache.accumulo.core.gc.ReferenceFile;
 import org.apache.accumulo.core.manager.state.tables.TableState;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.SystemTables;
 import org.apache.accumulo.core.metadata.schema.Ample;
 import org.apache.accumulo.core.metadata.schema.Ample.GcCandidateType;
+import org.apache.accumulo.server.gc.GcVolumeUtil;
 import org.apache.hadoop.fs.Path;
 import org.junit.jupiter.api.Test;
 
@@ -78,6 +80,10 @@ public GcCandidate addCandidate(String s) {
       return candidate;
     }
 
+    public GcCandidate addExternalCandidate(String path) {
+      return addCandidate(GcVolumeUtil.encodeExternalCandidatePath(path));
+    }
+
     public Long getTimestamp() {
       return timestamp;
     }
@@ -144,6 +150,13 @@ public void addFileReference(String tableId, String endRow, String file) {
       tableIds.add(tid);
     }
 
+    public void addExternalFileReference(String tableId, String endRow, String file) {
+      TableId tid = TableId.of(tableId);
+      references.put(tableId + ":" + endRow + ":" + file,
+          ReferenceFile.forFile(tid, StoredTabletFile.ofExternal(new Path(file))));
+      tableIds.add(tid);
+    }
+
     public void removeFileReference(String tableId, String endRow, String file) {
       references.remove(tableId + ":" + endRow + ":" + file);
       removeLastTableIdRef(TableId.of(tableId));
@@ -167,6 +180,13 @@ public void addScanReference(String tableId, String endRow, String scan) {
       tableIds.add(tid);
     }
 
+    public void addExternalScanReference(String tableId, String endRow, String scan) {
+      TableId tid = TableId.of(tableId);
+      references.put(tableId + ":" + endRow + ":scan:" + scan,
+          ReferenceFile.forScan(tid, StoredTabletFile.ofExternal(new Path(scan))));
+      tableIds.add(tid);
+    }
+
     public void removeScanReference(String tableId, String endRow, String scan) {
       references.remove(tableId + ":" + endRow + ":scan:" + scan);
       removeLastTableIdRef(TableId.of(tableId));
@@ -261,6 +281,35 @@ public void minimalDelete() throws Exception {
     assertCandidateRemoved(gce, GcCandidateType.INUSE, candOne, candTwo, candFour);
   }
 
+  @Test
+  public void externalFileReferencesPreventDeletion() throws Exception {
+    TestGCE gce = new TestGCE();
+    String path = "hdfs://external-nn/imports/F0001.rf";
+    var candidate = gce.addExternalCandidate(path);
+    gce.addExternalFileReference("4", null, path);
+    gce.addExternalScanReference("4", null, path);
+
+    GarbageCollectionAlgorithm gca = new GarbageCollectionAlgorithm();
+    gca.collect(gce);
+
+    assertFileDeleted(gce);
+    assertCandidateRemoved(gce, GcCandidateType.INUSE, candidate);
+
+    candidate = gce.addExternalCandidate(path);
+    gce.removeFileReference("4", null, path);
+    gca.collect(gce);
+
+    assertFileDeleted(gce);
+    assertTrue(gce.candidates.contains(candidate));
+    assertNoCandidatesRemoved(gce);
+
+    gce.removeScanReference("4", null, path);
+    gca.collect(gce);
+
+    assertFileDeleted(gce, candidate);
+    assertNoCandidatesRemoved(gce);
+  }
+
   @Test
   public void testBasic() throws Exception {
     TestGCE gce = new TestGCE();
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
index 9cc2e7f8b22..d64d3257bfd 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
@@ -51,6 +51,7 @@
 import org.apache.accumulo.server.ServerContext;
 import org.apache.accumulo.server.fs.VolumeManager;
 import org.apache.accumulo.server.gc.AllVolumesDirectory;
+import org.apache.accumulo.server.gc.GcVolumeUtil;
 import org.apache.accumulo.server.security.SystemCredentials;
 import org.apache.hadoop.fs.Path;
 import org.junit.jupiter.api.BeforeEach;
@@ -204,4 +205,18 @@ public void testMinimizeDeletes() {
             new GcCandidate("hdfs://nn1/accumulo/tables/5b/t-0004/F0001.rf", 9L)),
         processedDeletes);
   }
+
+  @Test
+  public void testMinimizeDeletesExternalCandidate() {
+    String path = "hdfs://external-nn/imports/F0001.rf";
+    String candidatePath = GcVolumeUtil.encodeExternalCandidatePath(path);
+    TreeMap candidates = new TreeMap<>();
+    candidates.put("external:" + path, new GcCandidate(candidatePath, 1L));
+
+    GCRun.minimizeDeletes(candidates, new ArrayList<>(), createMock(VolumeManager.class), log,
+        Duration.ofMinutes(1));
+
+    assertEquals(1, candidates.size());
+    assertEquals(candidatePath, candidates.get("external:" + path).getPath());
+  }
 }
diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
index 32afdfe5605..47ea5b42b9e 100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
@@ -22,6 +22,8 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -42,17 +44,25 @@
 import org.apache.accumulo.core.data.InstanceId;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.ExternalTabletFile;
 import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.SystemTables;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.server.init.Initialize;
 import org.apache.accumulo.server.util.adminCommand.StopAll;
+import org.apache.accumulo.test.util.Wait;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.junit.jupiter.api.Test;
 
@@ -209,6 +219,79 @@ public void testRemoveVolumes() throws Exception {
     }
   }
 
+  @Test
+  public void testExternalFileSurvivesVolumeReplacement() throws Exception {
+    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
+      String tableName = getUniqueNames(1)[0];
+      verifyVolumesUsed(client, tableName, false, v1, v2);
+      TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));
+
+      TabletMetadata tablet;
+      StoredTabletFile originalFile;
+      DataFileValue fileValue;
+      try (TabletsMetadata tablets = getServerContext().getAmple().readTablets().forTable(tableId)
+          .fetch(ColumnType.FILES).build()) {
+        tablet = tablets.iterator().next();
+        var firstFile = tablet.getFilesMap().entrySet().iterator().next();
+        originalFile = firstFile.getKey();
+        fileValue = firstFile.getValue();
+      }
+
+      Path externalPath =
+          new Path(new Path(v1.getParent(), "external-files"), getUniqueNames(1)[0] + ".rf");
+      FileSystem fs = externalPath.getFileSystem(getServerContext().getHadoopConf());
+      assertTrue(fs.mkdirs(externalPath.getParent()) || fs.exists(externalPath.getParent()));
+      IOUtils.copyBytes(fs.open(originalFile.getPath()), fs.create(externalPath),
+          getServerContext().getHadoopConf(), true);
+
+      ExternalTabletFile externalFile =
+          ExternalTabletFile.of(externalPath, originalFile.getRange());
+      getServerContext().getAmple().mutateTablet(tablet.getExtent()).deleteFile(originalFile)
+          .putFile(externalFile, fileValue).mutate();
+
+      assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor());
+      cluster.stop();
+
+      java.nio.file.Path v1f = java.nio.file.Path.of(v1.toUri());
+      java.nio.file.Path v8f = java.nio.file.Path.of(v1.getParent().toUri()).resolve("v8");
+      Files.move(v1f, v8f, StandardCopyOption.ATOMIC_MOVE);
+      Path v8 = new Path(v8f.toUri());
+
+      java.nio.file.Path v2f = java.nio.file.Path.of(v2.toUri());
+      java.nio.file.Path v9f = java.nio.file.Path.of(v2.getParent().toUri()).resolve("v9");
+      Files.move(v2f, v9f, StandardCopyOption.ATOMIC_MOVE);
+      Path v9 = new Path(v9f.toUri());
+
+      updateConfig(config -> {
+        config.setProperty(Property.INSTANCE_VOLUMES.getKey(), v8 + "," + v9);
+        config.setProperty(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey(),
+            v1 + " " + v8 + "," + v2 + " " + v9);
+      });
+      cluster.start();
+
+      Wait.waitFor(() -> {
+        boolean externalFound = false;
+        try (TabletsMetadata tablets = getServerContext().getAmple().readTablets().forTable(tableId)
+            .fetch(ColumnType.FILES).build()) {
+          for (TabletMetadata currentTablet : tablets) {
+            for (StoredTabletFile file : currentTablet.getFiles()) {
+              if (file.isExternal()) {
+                externalFound |= file.getPath().equals(externalPath);
+              } else if (file.getMetadataPath().startsWith(v1.toString())
+                  || file.getMetadataPath().startsWith(v2.toString())) {
+                return false;
+              }
+            }
+          }
+        }
+        return externalFound;
+      }, 30000, 250, "External reference was lost or volume paths were not replaced");
+
+      verifyData(expected, client.createScanner(tableName, Authorizations.EMPTY));
+      assertTrue(fs.exists(externalPath));
+    }
+  }
+
   @Test
   public void testDirtyReplaceVolumes() throws Exception {
     try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java
index 5bec14ad872..c8ce6014b4c 100644
--- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java
+++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java
@@ -98,13 +98,17 @@
 import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.lock.ServiceLock;
+import org.apache.accumulo.core.metadata.ExternalTabletFile;
 import org.apache.accumulo.core.metadata.ReferencedTabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.SystemTables;
 import org.apache.accumulo.core.metadata.schema.CompactionMetadata;
 import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
 import org.apache.accumulo.core.spi.compaction.CompactionKind;
 import org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher;
 import org.apache.accumulo.manager.tableOps.AbstractFateOperation;
@@ -118,7 +122,9 @@
 import org.apache.accumulo.test.harness.SharedMiniClusterBase;
 import org.apache.accumulo.test.util.Wait;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -526,6 +532,99 @@ public void testCompactionAndCompactorDies() throws Exception {
 
   }
 
+  @Test
+  public void testExternalCompactionWithMissingExternalInput() throws Exception {
+    String tableName = getUniqueNames(1)[0];
+    try (AccumuloClient client =
+        Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
+      createTable(client, tableName, "cs1");
+      writeData(client, tableName);
+
+      TableId tableId = getCluster().getServerContext().getTableId(tableName);
+      TabletMetadata tablet;
+      try (TabletsMetadata tablets = getCluster().getServerContext().getAmple().readTablets()
+          .forTable(tableId).fetch(ColumnType.FILES).build()) {
+        tablet = tablets.iterator().next();
+      }
+
+      var fileEntry = tablet.getFilesMap().entrySet().iterator().next();
+      StoredTabletFile original = fileEntry.getKey();
+      DataFileValue fileValue = fileEntry.getValue();
+      FileSystem fs = getCluster().getFileSystem();
+      Path externalDir = new Path(new Path(fs.getHomeDirectory(), "external-compaction-inputs"),
+          getUniqueNames(1)[0]);
+      assertTrue(fs.mkdirs(externalDir) || fs.exists(externalDir));
+      Path externalPath = new Path(externalDir, "input.rf");
+      IOUtils.copyBytes(fs.open(original.getPath()), fs.create(externalPath),
+          getCluster().getServerContext().getHadoopConf(), true);
+
+      ExternalTabletFile externalFile = ExternalTabletFile.of(externalPath, original.getRange());
+      StoredTabletFile storedExternal = externalFile.insert();
+      getCluster().getServerContext().getAmple().mutateTablet(tablet.getExtent())
+          .deleteFile(original).putFile(externalFile, fileValue).mutate();
+
+      // The external file disappears after the tablet metadata has recorded the reference.
+      assertTrue(fs.delete(externalPath, false));
+
+      getCluster().getClusterControl().stopAllServers(ServerType.COMPACTOR);
+      getCluster().getClusterControl().start(ServerType.COMPACTOR, null, 1,
+          ExternalDoNothingCompactor.class);
+      boolean compactionSubmitted = false;
+      try {
+        compact(client, tableName, 2, GROUP1, false);
+        compactionSubmitted = true;
+        Set ecids = ExternalCompactionTestUtils
+            .waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tableId);
+        assertFalse(ecids.isEmpty());
+
+        boolean externalInputFound = false;
+        try (TabletsMetadata tablets = getCluster().getServerContext().getAmple().readTablets()
+            .forTable(tableId).fetch(ColumnType.ECOMP).build()) {
+          for (TabletMetadata currentTablet : tablets) {
+            for (CompactionMetadata metadata : currentTablet.getExternalCompactions().values()) {
+              if (metadata.getJobFiles().contains(storedExternal)) {
+                externalInputFound = true;
+              }
+            }
+          }
+        }
+        assertTrue(externalInputFound,
+            "External compaction metadata should retain its missing external input reference");
+
+        assertThrows(RuntimeException.class, () -> {
+          try (Scanner scanner = client.createScanner(tableName)) {
+            scanner.iterator().hasNext();
+          }
+        }, "A scan should fail when the external compaction input is missing");
+
+        client.tableOperations().cancelCompaction(tableName);
+        compactionSubmitted = false;
+        ExternalCompactionTestUtils
+            .confirmCompactionsNoLongerRunning(getCluster().getServerContext(), ecids);
+        Wait.waitFor(() -> {
+          try (TabletsMetadata tablets = getCluster().getServerContext().getAmple().readTablets()
+              .forTable(tableId).fetch(ColumnType.ECOMP).build()) {
+            for (TabletMetadata currentTablet : tablets) {
+              if (!currentTablet.getExternalCompactions().isEmpty()) {
+                return false;
+              }
+            }
+            return true;
+          }
+        });
+      } finally {
+        try {
+          if (compactionSubmitted) {
+            client.tableOperations().cancelCompaction(tableName);
+          }
+        } finally {
+          getCluster().getClusterControl().stopAllServers(ServerType.COMPACTOR);
+          getCluster().getClusterControl().start(ServerType.COMPACTOR);
+        }
+      }
+    }
+  }
+
   @Test
   public void testManytablets() throws Exception {
     String table1 = this.getUniqueNames(1)[0];
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index 69f1a482e7f..f2744d51039 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -23,6 +23,7 @@
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
@@ -55,12 +56,17 @@
 import org.apache.accumulo.core.lock.ServiceLock;
 import org.apache.accumulo.core.lock.ServiceLockData;
 import org.apache.accumulo.core.lock.ServiceLockData.ThriftService;
+import org.apache.accumulo.core.metadata.ExternalTabletFile;
 import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.SystemTables;
 import org.apache.accumulo.core.metadata.schema.Ample;
 import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection.SkewedKeyValue;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.TablePermission;
 import org.apache.accumulo.gc.SimpleGarbageCollector;
@@ -70,12 +76,17 @@
 import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
 import org.apache.accumulo.miniclusterImpl.ProcessNotFoundException;
 import org.apache.accumulo.miniclusterImpl.ProcessReference;
+import org.apache.accumulo.server.gc.GcVolumeUtil;
 import org.apache.accumulo.test.TestIngest;
 import org.apache.accumulo.test.VerifyIngest;
 import org.apache.accumulo.test.VerifyIngest.VerifyParams;
+import org.apache.accumulo.test.util.Wait;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NoNodeException;
@@ -171,6 +182,187 @@ public void gcTest() throws Exception {
     }
   }
 
+  @Test
+  public void externalTabletFileIsRetainedWhileReferencedAndDeletedAfterward() throws Exception {
+    killMacGc();
+    String table = getUniqueNames(1)[0];
+
+    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
+      client.tableOperations().create(table);
+      try (BatchWriter writer = client.createBatchWriter(table)) {
+        Mutation mutation = new Mutation("external-row");
+        mutation.put("cf", "cq", "value");
+        writer.addMutation(mutation);
+      }
+      client.tableOperations().flush(table, null, null, true);
+
+      TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(table));
+      Ample ample = getServerContext().getAmple();
+      TabletMetadata tablet;
+      try (TabletsMetadata tablets =
+          ample.readTablets().forTable(tableId).fetch(ColumnType.FILES).build()) {
+        tablet = Iterators.getOnlyElement(tablets.iterator());
+      }
+
+      var fileEntry = tablet.getFilesMap().entrySet().iterator().next();
+      StoredTabletFile original = fileEntry.getKey();
+      DataFileValue fileValue = fileEntry.getValue();
+      FileSystem fs = cluster.getFileSystem();
+      Path externalPath = new Path(new Path(fs.getHomeDirectory(), "external-tablet-files"),
+          getUniqueNames(1)[0] + ".rf");
+      assertTrue(fs.mkdirs(externalPath.getParent()) || fs.exists(externalPath.getParent()));
+      IOUtils.copyBytes(fs.open(original.getPath()), fs.create(externalPath),
+          getServerContext().getHadoopConf(), true);
+
+      ExternalTabletFile external = ExternalTabletFile.of(externalPath, original.getRange());
+      StoredTabletFile storedExternal = external.insert();
+      ample.mutateTablet(tablet.getExtent()).deleteFile(original).putFile(external, fileValue)
+          .mutate();
+
+      try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
+        var entry = Iterators.getOnlyElement(scanner.iterator());
+        assertEquals("external-row", entry.getKey().getRow().toString());
+        assertEquals("value", entry.getValue().toString());
+      }
+
+      ProcessInfo gc = cluster.exec(SimpleGarbageCollector.class);
+      try {
+        DataLevel level = DataLevel.of(tableId);
+        String externalCandidatePath =
+            GcVolumeUtil.encodeExternalCandidatePath(externalPath.toString());
+
+        ample.putGcCandidates(tableId, List.of(storedExternal));
+        Wait.waitFor(() -> !hasGcCandidate(ample, level, externalCandidatePath), 30000, 250,
+            "GC did not process the in-use external file candidate");
+        assertTrue(fs.exists(externalPath), "GC deleted an external file with a live reference");
+
+        ample.mutateTablet(tablet.getExtent()).deleteFile(storedExternal).mutate();
+        ample.putGcCandidates(tableId, List.of(storedExternal));
+        Wait.waitFor(() -> !fs.exists(externalPath), 30000, 250,
+            "GC did not delete the unreferenced external file");
+      } finally {
+        gc.getProcess().destroy();
+      }
+    }
+  }
+
+  @Test
+  public void scansAndGcHandleMissingAndInaccessibleExternalFiles() throws Exception {
+    killMacGc();
+    String table = getUniqueNames(1)[0];
+
+    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
+      client.tableOperations().create(table);
+      try (BatchWriter writer = client.createBatchWriter(table)) {
+        Mutation mutation = new Mutation("external-row");
+        mutation.put("cf", "cq", "value");
+        writer.addMutation(mutation);
+      }
+      client.tableOperations().flush(table, null, null, true);
+
+      TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(table));
+      Ample ample = getServerContext().getAmple();
+      TabletMetadata tablet;
+      try (TabletsMetadata tablets =
+          ample.readTablets().forTable(tableId).fetch(ColumnType.FILES).build()) {
+        tablet = Iterators.getOnlyElement(tablets.iterator());
+      }
+
+      var fileEntry = tablet.getFilesMap().entrySet().iterator().next();
+      StoredTabletFile original = fileEntry.getKey();
+      DataFileValue fileValue = fileEntry.getValue();
+      FileSystem fs = cluster.getFileSystem();
+      Path externalDir =
+          new Path(new Path(fs.getHomeDirectory(), "external-tablet-files"), getUniqueNames(1)[0]);
+      assertTrue(fs.mkdirs(externalDir) || fs.exists(externalDir));
+
+      Path missingPath = new Path(externalDir, "missing.rf");
+      IOUtils.copyBytes(fs.open(original.getPath()), fs.create(missingPath),
+          getServerContext().getHadoopConf(), true);
+      ExternalTabletFile missingExternal = ExternalTabletFile.of(missingPath, original.getRange());
+      StoredTabletFile storedMissing = missingExternal.insert();
+      ample.mutateTablet(tablet.getExtent()).deleteFile(original)
+          .putFile(missingExternal, fileValue).mutate();
+
+      assertTrue(fs.delete(missingPath, false));
+      assertThrows(RuntimeException.class, () -> {
+        try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
+          Iterators.size(scanner.iterator());
+        }
+      }, "A scan should fail when a referenced external file no longer exists");
+
+      ProcessInfo gc = cluster.exec(SimpleGarbageCollector.class);
+      try {
+        DataLevel level = DataLevel.of(tableId);
+        String missingCandidatePath =
+            GcVolumeUtil.encodeExternalCandidatePath(missingPath.toString());
+        ample.putGcCandidates(tableId, List.of(storedMissing));
+        Wait.waitFor(() -> !hasGcCandidate(ample, level, missingCandidatePath), 30000, 250,
+            "GC did not process a missing external file that is still referenced");
+
+        Path unreadablePath = new Path(externalDir, "unreadable.rf");
+        IOUtils.copyBytes(fs.open(original.getPath()), fs.create(unreadablePath),
+            getServerContext().getHadoopConf(), true);
+        ExternalTabletFile unreadableExternal =
+            ExternalTabletFile.of(unreadablePath, original.getRange());
+        StoredTabletFile storedUnreadable = unreadableExternal.insert();
+        ample.mutateTablet(tablet.getExtent()).deleteFile(storedMissing)
+            .putFile(unreadableExternal, fileValue).mutate();
+
+        FsPermission originalFilePermission = fs.getFileStatus(unreadablePath).getPermission();
+        fs.setPermission(unreadablePath, FsPermission.valueOf("----------"));
+        try {
+          assertThrows(RuntimeException.class, () -> {
+            try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
+              Iterators.size(scanner.iterator());
+            }
+          }, "A scan should fail when a referenced external file becomes unreadable");
+        } finally {
+          fs.setPermission(unreadablePath, originalFilePermission);
+        }
+
+        try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
+          var entry = Iterators.getOnlyElement(scanner.iterator());
+          assertEquals("external-row", entry.getKey().getRow().toString());
+          assertEquals("value", entry.getValue().toString());
+        }
+
+        ample.mutateTablet(tablet.getExtent()).deleteFile(storedUnreadable).mutate();
+        String unreadableCandidatePath =
+            GcVolumeUtil.encodeExternalCandidatePath(unreadablePath.toString());
+        FsPermission originalDirectoryPermission = fs.getFileStatus(externalDir).getPermission();
+        try {
+          // Removing write permission after the metadata reference was recorded prevents GC from
+          // deleting the file after its tablet reference is removed.
+          fs.setPermission(externalDir, FsPermission.valueOf("dr-x------"));
+          ample.putGcCandidates(tableId, List.of(storedUnreadable));
+          Thread.sleep(SECONDS.toMillis(5));
+          assertTrue(fs.exists(unreadablePath),
+              "GC should retain an external file when permissions prevent deletion");
+          assertTrue(hasGcCandidate(ample, level, unreadableCandidatePath),
+              "GC should retain the candidate when it cannot delete the external file");
+        } finally {
+          fs.setPermission(externalDir, originalDirectoryPermission);
+        }
+
+        Wait.waitFor(() -> !fs.exists(unreadablePath), 30000, 250,
+            "GC did not retry deleting an external file after permissions were restored");
+      } finally {
+        gc.getProcess().destroy();
+      }
+    }
+  }
+
+  private boolean hasGcCandidate(Ample ample, DataLevel level, String path) {
+    Iterator candidates = ample.getGcCandidates(level);
+    while (candidates.hasNext()) {
+      if (path.equals(candidates.next().getPath())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   @Test
   public void gcLotsOfCandidatesIT() throws Exception {
     killMacGc();

From 4f9f814b67ef1e467fb0c4325c5af85c35924a09 Mon Sep 17 00:00:00 2001
From: Dave Marion 
Date: Thu, 22 Oct 2026 20:18:34 +0000
Subject: [PATCH 2/2] Fix some other issues

---
 .../core/metadata/StoredTabletFile.java       |  7 +++
 .../core/metadata/StoredTabletFileTest.java   | 15 ++++++
 .../server/util/MetadataTableUtil.java        | 31 ++++++-----
 .../manager/tableOps/merge/DeleteRows.java    |  2 +-
 .../manager/tableOps/merge/MergeTablets.java  |  4 +-
 .../tableOps/merge/MergeTabletsTest.java      | 14 +++++
 .../test/functional/GarbageCollectorIT.java   | 52 +++++++++++++++++++
 7 files changed, 108 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
index 2184e43bbee..4572ebdb665 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
@@ -191,6 +191,13 @@ public static StoredTabletFile ofExternal(final Path path, Range range) {
         new TabletFileCq(requireNonNull(path), range, TabletFileType.EXTERNAL));
   }
 
+  /**
+   * Create a new stored reference with the given range, preserving whether this file is external.
+   */
+  public StoredTabletFile withRange(Range range) {
+    return isExternal() ? ofExternal(getPath(), range) : of(getPath(), range);
+  }
+
   public static StoredTabletFile of(final URI path) {
     return of(path, new Range());
   }
diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java
index ca67959cf10..180e5d4004e 100644
--- a/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java
@@ -23,6 +23,7 @@
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import org.apache.accumulo.core.data.Range;
 import org.apache.hadoop.fs.Path;
 import org.junit.jupiter.api.Test;
 
@@ -69,4 +70,18 @@ public void insertingExternalFilePreservesMarker() {
     assertTrue(stored.isExternal());
     assertTrue(StoredTabletFile.of(stored.getMetadata()).isExternal());
   }
+
+  @Test
+  public void changingRangePreservesExternalMarker() {
+    Path path = new Path("hdfs://external-nn/imports/F0003.rf");
+    StoredTabletFile external = StoredTabletFile.ofExternal(path);
+    Range range = new Range("m", false, "z", true);
+
+    StoredTabletFile ranged = external.withRange(range);
+
+    assertTrue(ranged.isExternal());
+    assertEquals(path, ranged.getPath());
+    assertEquals(range, ranged.getRange());
+    assertTrue(StoredTabletFile.of(ranged.getMetadata()).isExternal());
+  }
 }
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index b73eac09149..a34186f07d2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -110,30 +110,33 @@ public static void deleteTable(TableId tableId, boolean insertDeletes, ServerCon
       Ample ample = context.getAmple();
       ms.setRange(new KeyExtent(tableId, null, null).toMetaRange());
 
-      // insert deletes before deleting data from metadata... this makes the code fault-tolerant
+      // Insert delete candidates before deleting data from metadata. This makes the code fault-
+      // tolerant. External files always need file candidates because deleting the table directory
+      // cannot remove them, even when table deletion optimization skips the usual GC candidates.
+      ms.fetchColumnFamily(DataFileColumnFamily.NAME);
       if (insertDeletes) {
-
-        ms.fetchColumnFamily(DataFileColumnFamily.NAME);
         ServerColumnFamily.DIRECTORY_COLUMN.fetch(ms);
+      }
 
-        for (Entry cell : ms) {
-          Key key = cell.getKey();
+      for (Entry cell : ms) {
+        Key key = cell.getKey();
 
-          if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
-            StoredTabletFile stf = new StoredTabletFile(key.getColumnQualifierData().toString());
+        if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+          StoredTabletFile stf = new StoredTabletFile(key.getColumnQualifierData().toString());
+          if (insertDeletes || stf.isExternal()) {
             bw.addMutation(ample.createDeleteMutation(ReferenceFile.forFile(tableId, stf)));
           }
+        }
 
-          if (ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
-            var uri = new AllVolumesDirectory(tableId, cell.getValue().toString());
-            bw.addMutation(ample.createDeleteMutation(uri));
-          }
+        if (insertDeletes && ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
+          var uri = new AllVolumesDirectory(tableId, cell.getValue().toString());
+          bw.addMutation(ample.createDeleteMutation(uri));
         }
+      }
 
-        bw.flush();
+      bw.flush();
 
-        ms.clearColumns();
-      }
+      ms.clearColumns();
 
       for (Entry cell : ms) {
         Key key = cell.getKey();
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java
index 33b1d43d263..ea737d5f911 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java
@@ -138,7 +138,7 @@ private Optional deleteTabletFiles(Ample ample, FateId fateId) {
               // If the existing file is disjoint then later we will delete if the file is not part
               // of the newFiles set which means it is disjoint with all ranges
               if (fenced != null) {
-                final StoredTabletFile newFile = StoredTabletFile.of(existing.getPath(), fenced);
+                final StoredTabletFile newFile = existing.withRange(fenced);
                 log.trace("{} Adding new file {} with range {}", fateId, newFile.getMetadataPath(),
                     newFile.getRange());
 
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
index de3484c4fa4..5967d785087 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
@@ -291,11 +291,11 @@ static void validateTablet(TabletMetadata tabletMeta, FateId fateId, TabletOpera
   /**
    * Fence a file to a tablets data range.
    */
-  private static StoredTabletFile fenceFile(KeyExtent extent, StoredTabletFile file) {
+  static StoredTabletFile fenceFile(KeyExtent extent, StoredTabletFile file) {
     Range fenced = extent.toDataRange();
     // Clip range if exists
     fenced = file.hasRange() ? file.getRange().clip(fenced) : fenced;
-    return StoredTabletFile.of(file.getPath(), fenced);
+    return file.withRange(fenced);
   }
 
   @Override
diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java
index 002ffbeea80..b221fb3e282 100644
--- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java
+++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java
@@ -71,6 +71,7 @@
 import org.apache.accumulo.core.fate.FateInstanceType;
 import org.apache.accumulo.core.gc.ReferenceFile;
 import org.apache.accumulo.core.lock.ServiceLock;
+import org.apache.accumulo.core.metadata.ExternalTabletFile;
 import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.SuspendingTServer;
 import org.apache.accumulo.core.metadata.TServerInstance;
@@ -94,6 +95,7 @@
 import org.apache.accumulo.server.gc.AllVolumesDirectory;
 import org.apache.accumulo.server.metadata.ConditionalTabletMutatorImpl;
 import org.apache.accumulo.server.metadata.ConditionalTabletsMutatorImpl;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.easymock.EasyMock;
 import org.junit.jupiter.api.Test;
@@ -132,6 +134,18 @@ public void checkColumns() {
     }
   }
 
+  @Test
+  public void fenceExternalFilePreservesExternalMarker() {
+    Path path = new Path("hdfs://external-nn/imports/F0001.rf");
+    StoredTabletFile external = ExternalTabletFile.of(path).insert();
+
+    StoredTabletFile fenced = MergeTablets.fenceFile(ke3, external);
+
+    assertTrue(fenced.isExternal());
+    assertEquals(path, fenced.getPath());
+    assertEquals(ke3.toDataRange(), fenced.getRange());
+  }
+
   @Test
   public void testManyColumns() throws Exception {
 
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index f2744d51039..efd02563a26 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -246,6 +246,58 @@ public void externalTabletFileIsRetainedWhileReferencedAndDeletedAfterward() thr
     }
   }
 
+  @Test
+  public void deletingTableWithExternalFileQueuesItForGc() throws Exception {
+    killMacGc();
+    String table = getUniqueNames(1)[0];
+
+    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
+      client.tableOperations().create(table);
+      try (BatchWriter writer = client.createBatchWriter(table)) {
+        Mutation mutation = new Mutation("external-row");
+        mutation.put("cf", "cq", "value");
+        writer.addMutation(mutation);
+      }
+      client.tableOperations().flush(table, null, null, true);
+
+      TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(table));
+      Ample ample = getServerContext().getAmple();
+      TabletMetadata tablet;
+      try (TabletsMetadata tablets = ample.readTablets().forTable(tableId)
+          .fetch(ColumnType.FILES, ColumnType.PREV_ROW).build()) {
+        tablet = Iterators.getOnlyElement(tablets.iterator());
+      }
+
+      StoredTabletFile original = tablet.getFiles().iterator().next();
+      FileSystem fs = cluster.getFileSystem();
+      Path externalPath = new Path(new Path(fs.getHomeDirectory(), "external-tablet-files"),
+          getUniqueNames(1)[0] + ".rf");
+      assertTrue(fs.mkdirs(externalPath.getParent()) || fs.exists(externalPath.getParent()));
+      IOUtils.copyBytes(fs.open(original.getPath()), fs.create(externalPath),
+          getServerContext().getHadoopConf(), true);
+
+      StoredTabletFile storedExternal =
+          ExternalTabletFile.of(externalPath, original.getRange()).insert();
+      ample.mutateTablet(tablet.getExtent()).deleteFile(original)
+          .putFile(storedExternal, tablet.getFilesMap().get(original)).mutate();
+
+      String candidatePath = GcVolumeUtil.encodeExternalCandidatePath(externalPath.toString());
+      client.tableOperations().delete(table);
+
+      DataLevel level = DataLevel.of(tableId);
+      assertTrue(hasGcCandidate(ample, level, candidatePath),
+          "Table deletion did not enqueue an external file candidate");
+
+      ProcessInfo gc = cluster.exec(SimpleGarbageCollector.class);
+      try {
+        Wait.waitFor(() -> !fs.exists(externalPath), 30000, 250,
+            "GC did not delete the external file after its table was deleted");
+      } finally {
+        gc.getProcess().destroy();
+      }
+    }
+  }
+
   @Test
   public void scansAndGcHandleMissingAndInaccessibleExternalFiles() throws Exception {
     killMacGc();