diff --git a/core/src/main/java/org/apache/accumulo/core/gc/Reference.java b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java index a6ee689ea6b..2915bfff6c2 100644 --- a/core/src/main/java/org/apache/accumulo/core/gc/Reference.java +++ b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java @@ -36,6 +36,11 @@ public interface Reference { */ boolean isScan(); + /** + * Returns true when this reference points to a file outside Accumulo's defined Volumes. + */ + boolean isExternal(); + /** * Get the {@link TableId} of the reference. */ diff --git a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java index 492e67c2e4b..5a4ed5dba0d 100644 --- a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java +++ b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java @@ -18,9 +18,10 @@ */ package org.apache.accumulo.core.gc; -import java.util.Objects; +import static java.util.Objects.requireNonNull; import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.metadata.ReferencedTabletFile; import org.apache.accumulo.core.metadata.ScanServerRefTabletFile; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.schema.MetadataSchema; @@ -35,20 +36,33 @@ public class ReferenceFile implements Reference, Comparable { public final TableId tableId; // 2a public final boolean isScan; public final boolean isDirectory; + public final boolean isExternal; // the exact path from the file reference string that is stored in the metadata protected final String metadataPath; protected ReferenceFile(TableId tableId, String metadataPath, boolean isScan, boolean isDirectory) { - this.tableId = Objects.requireNonNull(tableId); - this.metadataPath = Objects.requireNonNull(metadataPath); + this(tableId, metadataPath, isScan, isDirectory, false); + } + + protected ReferenceFile(TableId tableId, String metadataPath, boolean isScan, boolean isDirectory, + boolean isExternal) { + this.tableId = requireNonNull(tableId); + this.metadataPath = requireNonNull(metadataPath); this.isScan = isScan; this.isDirectory = isDirectory; + this.isExternal = isExternal; } public static ReferenceFile forFile(TableId tableId, StoredTabletFile tabletFile) { - return new ReferenceFile(tableId, tabletFile.getMetadataPath(), false, false); + return new ReferenceFile(tableId, tabletFile.getMetadataPath(), false, false, + tabletFile.isExternal()); + } + + public static ReferenceFile forFile(TableId tableId, ReferencedTabletFile tabletFile) { + return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), false, false, + tabletFile.isExternal()); } public static ReferenceFile forFile(TableId tableId, Path metadataPathPath) { @@ -56,11 +70,18 @@ public static ReferenceFile forFile(TableId tableId, Path metadataPathPath) { } public static ReferenceFile forScan(TableId tableId, ScanServerRefTabletFile tabletFile) { - return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true, false); + return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true, false, + tabletFile.isExternal()); } public static ReferenceFile forScan(TableId tableId, StoredTabletFile tabletFile) { - return new ReferenceFile(tableId, tabletFile.getMetadataPath(), true, false); + return new ReferenceFile(tableId, tabletFile.getMetadataPath(), true, false, + tabletFile.isExternal()); + } + + public static ReferenceFile forScan(TableId tableId, ReferencedTabletFile tabletFile) { + return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true, false, + tabletFile.isExternal()); } public static ReferenceFile forScan(TableId tableId, Path metadataPathPath) { @@ -82,6 +103,11 @@ public boolean isScan() { return isScan; } + @Override + public boolean isExternal() { + return isExternal; + } + @Override public TableId getTableId() { return tableId; diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/ExternalTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/ExternalTabletFile.java new file mode 100644 index 00000000000..6458cab166f --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/metadata/ExternalTabletFile.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.metadata; + +import static java.util.Objects.requireNonNull; + +import org.apache.accumulo.core.data.Range; +import org.apache.hadoop.fs.Path; + +/** + * A reference to a tablet file whose path is not in Accumulo's conventional tables directory. An + * external file is still owned by Accumulo and is eligible for garbage collection when no longer + * referenced. + */ +public class ExternalTabletFile extends ReferencedTabletFile { + + public ExternalTabletFile(Path path) { + this(path, new Range()); + } + + public ExternalTabletFile(Path path, Range range) { + super(requireNonNull(path), range, true); + } + + public static ExternalTabletFile of(Path path) { + return new ExternalTabletFile(path); + } + + public static ExternalTabletFile of(Path path, Range range) { + return new ExternalTabletFile(path, range); + } +} diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java index 2bcae4be9d5..32f3df8c4b8 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.core.metadata; +import static java.util.Objects.requireNonNull; import static org.apache.accumulo.core.Constants.HDFS_TABLES_DIR; import java.net.URI; @@ -154,6 +155,7 @@ public static FileParts parsePath(Path filePath) { } private final FileParts parts; + private final boolean external; private static final Logger log = LoggerFactory.getLogger(ReferencedTabletFile.class); private static final String HDFS_TABLES_DIR_NAME = HDFS_TABLES_DIR.substring(1); @@ -171,26 +173,48 @@ public ReferencedTabletFile(Path metaPath) { * qualify an absolute path or create a new file. */ public ReferencedTabletFile(Path metaPath, Range range) { - super(Objects.requireNonNull(metaPath), range); - log.trace("Parsing TabletFile from {}", metaPath); - parts = parsePath(metaPath); + this(metaPath, range, false); + } + + protected ReferencedTabletFile(Path metaPath, Range range, boolean external) { + super(requireNonNull(metaPath), range); + this.external = external; + if (external) { + Preconditions.checkArgument(metaPath.toUri().getScheme() != null, + constructErrorMsg(metaPath)); + ValidationUtil.validateFileName(metaPath.getName()); + Preconditions.checkArgument(!isVolumePath(metaPath), + "External tablet file path must not use the Accumulo table layout: %s", metaPath); + parts = null; + } else { + log.trace("Parsing TabletFile from {}", metaPath); + parts = parsePath(metaPath); + } } public String getVolume() { + Preconditions.checkState(!external, "External tablet files do not have an Accumulo volume"); return parts.getVolume(); } public TableId getTableId() { + Preconditions.checkState(!external, + "External tablet files do not have a path-derived table ID"); return parts.getTableId(); } public String getTabletDir() { + Preconditions.checkState(!external, "External tablet files do not have a tablet directory"); return parts.getTabletDir(); } @Override public String getFileName() { - return parts.getFileName(); + return external ? path.getName() : parts.getFileName(); + } + + public boolean isExternal() { + return external; } /** @@ -198,14 +222,15 @@ public String getFileName() { * metadata. */ public String getNormalizedPathStr() { - return parts.getNormalizedPath(); + return external ? path.toString() : parts.getNormalizedPath(); } /** * New file was written to metadata so return a StoredTabletFile */ public StoredTabletFile insert() { - return StoredTabletFile.of(getPath(), getRange()); + return external ? StoredTabletFile.ofExternal(getPath(), getRange()) + : StoredTabletFile.of(getPath(), getRange()); } @Override @@ -216,7 +241,7 @@ public int compareTo(ReferencedTabletFile o) { @Override public boolean equals(Object obj) { if (obj instanceof ReferencedTabletFile that) { - return parts.getNormalizedPath().equals(that.parts.getNormalizedPath()) + return getNormalizedPathStr().equals(that.getNormalizedPathStr()) && external == that.external && range.equals(that.range); } return false; @@ -224,12 +249,48 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash(parts.getNormalizedPath(), range); + return Objects.hash(getNormalizedPathStr(), external, range); } @Override public String toString() { - return parts.getNormalizedPath(); + return getNormalizedPathStr(); + } + + /** + * Determines whether a path uses Accumulo's conventional + * {@code tables///} layout. + */ + public static boolean isVolumePath(Path path) { + try { + parsePath(path); + return true; + } catch (RuntimeException e) { + return false; + } + } + + /** + * Determines whether a path is a conventional Accumulo tablet directory. + */ + public static boolean isTabletDirPath(Path path) { + if (path == null || path.toUri().getScheme() == null) { + return false; + } + Path tableIdPath = path.getParent(); + if (tableIdPath == null) { + return false; + } + Path tablesPath = tableIdPath.getParent(); + if (tablesPath == null || !HDFS_TABLES_DIR_NAME.equals(tablesPath.getName())) { + return false; + } + try { + TableId.of(tableIdPath.getName()); + return true; + } catch (RuntimeException e) { + return false; + } } public static ReferencedTabletFile of(final Path path) { diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefTabletFile.java index c289c38a47e..a8ac7927e1e 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefTabletFile.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefTabletFile.java @@ -18,12 +18,15 @@ */ package org.apache.accumulo.core.metadata; +import static java.util.Objects.requireNonNull; + import java.net.URI; import java.util.Objects; import java.util.UUID; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -35,21 +38,23 @@ public class ScanServerRefTabletFile extends ReferencedTabletFile { private final Text uuid; public ScanServerRefTabletFile(UUID serverLockUUID, String serverAddress, String file) { - super(new Path(URI.create(file))); - this.serverAddress = new Text(serverAddress); - uuid = new Text(serverLockUUID.toString()); + this(new Path(URI.create(file)), new Text(serverAddress), + new Text(requireNonNull(serverLockUUID).toString())); } public ScanServerRefTabletFile(String file, String serverAddress, UUID serverLockUUID) { - super(new Path(URI.create(file))); - this.serverAddress = new Text(serverAddress); - this.uuid = new Text(serverLockUUID.toString()); + this(new Path(URI.create(file)), new Text(serverAddress), + new Text(requireNonNull(serverLockUUID).toString())); } public ScanServerRefTabletFile(Key k) { - super(new Path(URI.create(k.getColumnQualifier().toString()))); - serverAddress = k.getColumnFamily(); - uuid = k.getRow(); + this(new Path(URI.create(k.getColumnQualifier().toString())), k.getColumnFamily(), k.getRow()); + } + + private ScanServerRefTabletFile(Path path, Text serverAddress, Text uuid) { + super(path, new Range(), !ReferencedTabletFile.isVolumePath(path)); + this.serverAddress = serverAddress; + this.uuid = uuid; } public Mutation putMutation() { diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java index b1f80ccd75a..4572ebdb665 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.core.metadata; +import static java.util.Objects.requireNonNull; import static org.apache.accumulo.core.util.RowRangeUtil.requireKeyExtentDataRange; import java.io.ByteArrayOutputStream; @@ -76,10 +77,12 @@ private StoredTabletFile(TabletFileCq fileCq) { } private StoredTabletFile(String metadataEntry, TabletFileCq fileCq) { - super(Objects.requireNonNull(fileCq).path, fileCq.range); - this.metadataEntry = Objects.requireNonNull(metadataEntry); + super(requireNonNull(fileCq).path, fileCq.range); + this.metadataEntry = requireNonNull(metadataEntry); this.metadataEntryPath = fileCq.path.toString(); - this.referencedTabletFile = ReferencedTabletFile.of(getPath(), fileCq.range); + this.referencedTabletFile = + fileCq.type == TabletFileType.EXTERNAL ? ExternalTabletFile.of(getPath(), fileCq.range) + : ReferencedTabletFile.of(getPath(), fileCq.range); } /** @@ -113,6 +116,10 @@ public TableId getTableId() { return referencedTabletFile.getTableId(); } + public boolean isExternal() { + return referencedTabletFile.isExternal(); + } + @Override public String getFileName() { return referencedTabletFile.getFileName(); @@ -154,7 +161,7 @@ public String toString() { } public static StoredTabletFile of(final Text metadataEntry) { - return new StoredTabletFile(Objects.requireNonNull(metadataEntry).toString()); + return new StoredTabletFile(requireNonNull(metadataEntry).toString()); } public static StoredTabletFile of(final String metadataEntry) { @@ -162,11 +169,33 @@ public static StoredTabletFile of(final String metadataEntry) { } public static StoredTabletFile of(final URI path, Range range) { - return of(new Path(Objects.requireNonNull(path)), range); + return of(new Path(requireNonNull(path)), range); } public static StoredTabletFile of(final Path path, Range range) { - return new StoredTabletFile(new TabletFileCq(Objects.requireNonNull(path), range)); + return new StoredTabletFile(new TabletFileCq(requireNonNull(path), range)); + } + + /** + * Create a stored file reference whose path does not use Accumulo's conventional table layout. + */ + public static StoredTabletFile ofExternal(final Path path) { + return ofExternal(path, new Range()); + } + + /** + * Create a stored file reference whose path does not use Accumulo's conventional table layout. + */ + public static StoredTabletFile ofExternal(final Path path, Range range) { + return new StoredTabletFile( + new TabletFileCq(requireNonNull(path), range, TabletFileType.EXTERNAL)); + } + + /** + * Create a new stored reference with the given range, preserving whether this file is external. + */ + public StoredTabletFile withRange(Range range) { + return isExternal() ? ofExternal(getPath(), range) : of(getPath(), range); } public static StoredTabletFile of(final URI path) { @@ -181,14 +210,13 @@ public static StoredTabletFile of(final Path path) { private static TabletFileCq deserialize(String json) { final TabletFileCqMetadataGson metadata = - gson.fromJson(Objects.requireNonNull(json), TabletFileCqMetadataGson.class); + gson.fromJson(requireNonNull(json), TabletFileCqMetadataGson.class); // Check each field and provide better error messages if null as all fields should be set - Objects.requireNonNull(metadata.path, "Serialized StoredTabletFile path must not be null"); - Objects.requireNonNull(metadata.startRow, + requireNonNull(metadata.path, "Serialized StoredTabletFile path must not be null"); + requireNonNull(metadata.startRow, "Serialized StoredTabletFile range startRow must not be null"); - Objects.requireNonNull(metadata.endRow, - "Serialized StoredTabletFile range endRow must not be null"); + requireNonNull(metadata.endRow, "Serialized StoredTabletFile range endRow must not be null"); // Recreate the exact Range that was originally stored in Metadata. Stored ranges are originally // constructed with inclusive/exclusive for the start and end key inclusivity settings. @@ -203,7 +231,8 @@ private static TabletFileCq deserialize(String json) { // start row and false for end row regardless of what the startRowInclusive and endRowInclusive // flags are set to. return new TabletFileCq(new Path(URI.create(metadata.path)), - new Range(decodeRow(metadata.startRow), true, decodeRow(metadata.endRow), false)); + new Range(decodeRow(metadata.startRow), true, decodeRow(metadata.endRow), false), + metadata.type == null ? TabletFileType.STANDARD : metadata.type); } public static String serialize(String path) { @@ -211,17 +240,25 @@ public static String serialize(String path) { } public static String serialize(String path, Range range) { + return serialize(path, range, TabletFileType.STANDARD); + } + + private static String serialize(String path, Range range, TabletFileType type) { requireKeyExtentDataRange(range); final TabletFileCqMetadataGson metadata = new TabletFileCqMetadataGson(); - metadata.path = Objects.requireNonNull(path); + metadata.path = requireNonNull(path); metadata.startRow = encodeRow(range.getStartKey()); metadata.endRow = encodeRow(range.getEndKey()); + if (type != TabletFileType.STANDARD) { + metadata.type = type; + } return gson.toJson(metadata); } private static String serialize(TabletFileCq tabletFileCq) { - return serialize(Objects.requireNonNull(tabletFileCq).path.toString(), tabletFileCq.range); + requireNonNull(tabletFileCq); + return serialize(tabletFileCq.path.toString(), tabletFileCq.range, tabletFileCq.type); } /** @@ -277,16 +314,27 @@ public static boolean fileNeedsConversion(@NonNull final String candidate) { private static class TabletFileCq { public final Path path; public final Range range; + public final TabletFileType type; public TabletFileCq(Path path, Range range) { - this.path = Objects.requireNonNull(path); - this.range = Objects.requireNonNull(range); + this(path, range, TabletFileType.STANDARD); + } + + public TabletFileCq(Path path, Range range, TabletFileType type) { + this.path = requireNonNull(path); + this.range = requireNonNull(range); + this.type = requireNonNull(type); } } + private enum TabletFileType { + STANDARD, EXTERNAL + } + private static class TabletFileCqMetadataGson { private String path; private byte[] startRow; private byte[] endRow; + private TabletFileType type; } } diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java index 998c9eab112..180e5d4004e 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java @@ -18,9 +18,13 @@ */ package org.apache.accumulo.core.metadata; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.accumulo.core.data.Range; +import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.Test; public class StoredTabletFileTest { @@ -37,4 +41,47 @@ public void fileConversionTest() { assertFalse(StoredTabletFile.fileNeedsConversion(s31)); assertFalse(StoredTabletFile.fileNeedsConversion(s31_untrimmed)); } + + @Test + public void externalFileRoundTrip() { + Path path = new Path("hdfs://external-nn/imports/F0001.rf"); + StoredTabletFile stored = StoredTabletFile.ofExternal(path); + + assertTrue(stored.isExternal()); + assertTrue(stored.getTabletFile() instanceof ExternalTabletFile); + assertEquals(path, stored.getPath()); + assertTrue(stored.getMetadata().contains("\"type\":\"EXTERNAL\"")); + + StoredTabletFile roundTripped = StoredTabletFile.of(stored.getMetadata()); + assertTrue(roundTripped.isExternal()); + assertEquals(stored.getMetadata(), roundTripped.getMetadata()); + assertEquals(stored.getPath(), roundTripped.getPath()); + assertEquals(stored.getRange(), roundTripped.getRange()); + assertThrows(IllegalStateException.class, roundTripped::getTableId); + } + + @Test + public void insertingExternalFilePreservesMarker() { + ExternalTabletFile external = + ExternalTabletFile.of(new Path("file:///external/files/F0002.rf")); + + StoredTabletFile stored = external.insert(); + + assertTrue(stored.isExternal()); + assertTrue(StoredTabletFile.of(stored.getMetadata()).isExternal()); + } + + @Test + public void changingRangePreservesExternalMarker() { + Path path = new Path("hdfs://external-nn/imports/F0003.rf"); + StoredTabletFile external = StoredTabletFile.ofExternal(path); + Range range = new Range("m", false, "z", true); + + StoredTabletFile ranged = external.withRange(range); + + assertTrue(ranged.isExternal()); + assertEquals(path, ranged.getPath()); + assertEquals(range, ranged.getRange()); + assertTrue(StoredTabletFile.of(ranged.getMetadata()).isExternal()); + } } diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java index 04c38f9f9b8..4dcf46ce32f 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java @@ -26,6 +26,7 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.metadata.ExternalTabletFile; import org.apache.accumulo.core.metadata.ReferencedTabletFile; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.hadoop.fs.Path; @@ -105,6 +106,25 @@ public void testFullPathWithVolume() { test(metadataEntry, volume, id, dir, filename); } + @Test + public void testExternalTabletFile() { + Path path = new Path("hdfs://external-nn/imports/F0001.rf"); + ExternalTabletFile external = ExternalTabletFile.of(path); + + assertTrue(external.isExternal()); + assertEquals(path, external.getPath()); + assertEquals("F0001.rf", external.getFileName()); + assertEquals(path.toString(), external.getNormalizedPathStr()); + assertThrows(IllegalStateException.class, external::getTableId); + assertThrows(IllegalStateException.class, external::getTabletDir); + assertThrows(IllegalStateException.class, external::getVolume); + + assertThrows(IllegalArgumentException.class, + () -> ExternalTabletFile.of(new Path("hdfs://nn/accumulo/tables/2a/t-1/F0001.rf"))); + assertThrows(IllegalArgumentException.class, + () -> ExternalTabletFile.of(new Path("/external/F0001.rf"))); + } + @Test public void testNormalizePath() { String uglyVolume = "hdfs://nn.somewhere.com:86753/accumulo/blah/.././/bad/bad2/../.././/////"; diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java index e41237b087b..4c9721c30b9 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java @@ -141,6 +141,9 @@ public static void volumeReplacementEvaluation(final Map replacements } for (Entry entry : tm.getFilesMap().entrySet()) { + if (entry.getKey().isExternal()) { + continue; + } log.trace("Evaluating file {} for replacement.", entry.getKey().getPath()); String metaPath = entry.getKey().getMetadata(); Path switchedPath = switchVolume(entry.getKey().getPath(), FileType.TABLE, replacements); diff --git a/server/base/src/main/java/org/apache/accumulo/server/gc/GcVolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/gc/GcVolumeUtil.java index 3f73b85ef4d..0d508d294da 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/gc/GcVolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/gc/GcVolumeUtil.java @@ -28,6 +28,22 @@ public class GcVolumeUtil { // AGCAV : Accumulo Garbage Collector All Volumes static final String ALL_VOLUMES_PREFIX = "agcav:/"; + public static final String EXTERNAL_CANDIDATE_PREFIX = "@external@"; + + public static String encodeExternalCandidatePath(String path) { + return EXTERNAL_CANDIDATE_PREFIX + path; + } + + public static boolean isExternalCandidatePath(String path) { + return path.startsWith(EXTERNAL_CANDIDATE_PREFIX); + } + + public static String decodeExternalCandidatePath(String path) { + if (!isExternalCandidatePath(path)) { + return path; + } + return path.substring(EXTERNAL_CANDIDATE_PREFIX.length()); + } public static Collection expandAllVolumesUri(VolumeManager fs, Path path) { if (path.toString().startsWith(ALL_VOLUMES_PREFIX)) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java index 7a2a17d08fd..4b79bccf3b2 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.gc.GcCandidate; import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.server.gc.GcVolumeUtil; import org.apache.hadoop.fs.Path; public class RootGcCandidates { @@ -77,18 +78,25 @@ public RootGcCandidates(String jsonString) { } public void add(Stream refs) { - refs.forEach(ref -> data.candidates - .computeIfAbsent(ref.getPath().getParent().toString(), k -> new TreeSet<>()) - .add(ref.getFileName())); + refs.forEach(ref -> { + String fileName = ref.getFileName(); + if (ref.isExternal()) { + fileName = GcVolumeUtil.EXTERNAL_CANDIDATE_PREFIX + fileName; + } + data.candidates.computeIfAbsent(ref.getPath().getParent().toString(), k -> new TreeSet<>()) + .add(fileName); + }); } public void remove(Stream refs) { - refs.map(GcCandidate::getPath).map(Path::new).forEach(path -> { - data.candidates.computeIfPresent(path.getParent().toString(), (key, values) -> { - values.remove(path.getName()); - return values.isEmpty() ? null : values; - }); - }); + refs.map(GcCandidate::getPath).map(GcVolumeUtil::decodeExternalCandidatePath).map(Path::new) + .forEach(path -> { + data.candidates.computeIfPresent(path.getParent().toString(), (key, values) -> { + values.remove(path.getName()); + values.remove(GcVolumeUtil.EXTERNAL_CANDIDATE_PREFIX + path.getName()); + return values.isEmpty() ? null : values; + }); + }); } public Stream sortedStream() { @@ -96,8 +104,17 @@ public Stream sortedStream() { return data.candidates.entrySet().stream().flatMap(entry -> { String parent = entry.getKey(); SortedSet names = entry.getValue(); - return names.stream() - .map(name -> new GcCandidate(parent + Path.SEPARATOR + name, uidGen.nextLong())); + return names.stream().map(name -> { + String candidatePath; + if (name.startsWith(GcVolumeUtil.EXTERNAL_CANDIDATE_PREFIX)) { + String actualPath = parent + Path.SEPARATOR + + name.substring(GcVolumeUtil.EXTERNAL_CANDIDATE_PREFIX.length()); + candidatePath = GcVolumeUtil.encodeExternalCandidatePath(actualPath); + } else { + candidatePath = parent + Path.SEPARATOR + name; + } + return new GcCandidate(candidatePath, uidGen.nextLong()); + }); }).sorted(); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java index 16f63126e4c..6dff8815d9b 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java @@ -52,6 +52,8 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.gc.GcVolumeUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -151,7 +153,9 @@ public void putGcFileAndDirCandidates(TableId tableId, Collection if (DataLevel.of(tableId) == DataLevel.ROOT) { // Directories are unexpected for the root tablet, so convert to stored tablet file mutateRootGcCandidates(rgcc -> rgcc.add(candidates.stream() - .map(reference -> StoredTabletFile.of(URI.create(reference.getMetadataPath()))))); + .map(reference -> reference.isExternal() + ? StoredTabletFile.ofExternal(new Path(reference.getMetadataPath())) + : StoredTabletFile.of(URI.create(reference.getMetadataPath()))))); return; } @@ -267,11 +271,19 @@ public Iterator getGcCandidates(DataLevel level) { @Override public Mutation createDeleteMutation(ReferenceFile tabletFilePathToRemove) { - return createDelMutation(ValidationUtil.validate(tabletFilePathToRemove).getMetadataPath()); + String path = ValidationUtil.validate(tabletFilePathToRemove).getMetadataPath(); + if (tabletFilePathToRemove.isExternal()) { + path = GcVolumeUtil.encodeExternalCandidatePath(path); + } + return createDelMutation(path); } public Mutation createDeleteMutation(StoredTabletFile pathToRemove) { - return createDelMutation(pathToRemove.getMetadataPath()); + String path = pathToRemove.getMetadataPath(); + if (pathToRemove.isExternal()) { + path = GcVolumeUtil.encodeExternalCandidatePath(path); + } + return createDelMutation(path); } private Mutation createDelMutation(String path) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index b73eac09149..a34186f07d2 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -110,30 +110,33 @@ public static void deleteTable(TableId tableId, boolean insertDeletes, ServerCon Ample ample = context.getAmple(); ms.setRange(new KeyExtent(tableId, null, null).toMetaRange()); - // insert deletes before deleting data from metadata... this makes the code fault-tolerant + // Insert delete candidates before deleting data from metadata. This makes the code fault- + // tolerant. External files always need file candidates because deleting the table directory + // cannot remove them, even when table deletion optimization skips the usual GC candidates. + ms.fetchColumnFamily(DataFileColumnFamily.NAME); if (insertDeletes) { - - ms.fetchColumnFamily(DataFileColumnFamily.NAME); ServerColumnFamily.DIRECTORY_COLUMN.fetch(ms); + } - for (Entry cell : ms) { - Key key = cell.getKey(); + for (Entry cell : ms) { + Key key = cell.getKey(); - if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { - StoredTabletFile stf = new StoredTabletFile(key.getColumnQualifierData().toString()); + if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { + StoredTabletFile stf = new StoredTabletFile(key.getColumnQualifierData().toString()); + if (insertDeletes || stf.isExternal()) { bw.addMutation(ample.createDeleteMutation(ReferenceFile.forFile(tableId, stf))); } + } - if (ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) { - var uri = new AllVolumesDirectory(tableId, cell.getValue().toString()); - bw.addMutation(ample.createDeleteMutation(uri)); - } + if (insertDeletes && ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) { + var uri = new AllVolumesDirectory(tableId, cell.getValue().toString()); + bw.addMutation(ample.createDeleteMutation(uri)); } + } - bw.flush(); + bw.flush(); - ms.clearColumns(); - } + ms.clearColumns(); for (Entry cell : ms) { Key key = cell.getKey(); diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java index b0aebbb2288..5de181aeb77 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java @@ -205,24 +205,25 @@ public static Map,Long> getDiskUsage(Set tableIds, // get the table referenced by the file which may not be the same as the current // table we are scanning if the file is shared between multiple tables - final TableId fileTableRef = stf.getTableId(); + final TableId fileTableRef = stf.isExternal() ? tableId : stf.getTableId(); + final String fileKey = stf.isExternal() ? stf.getMetadataPath() : stf.getFileName(); // if this is a ref to a different table than the one we are scanning then we need // to make sure the table is also linked for this shared file if the table is // part of the set of tables we are running du on so we can track shared usages if (!fileTableRef.equals(tableId) && tableIds.contains(fileTableRef)) { // link the table and the shared file for computing shared sizes - tdu.linkFileAndTable(fileTableRef, stf.getFileName()); + tdu.linkFileAndTable(fileTableRef, fileKey); } // link the file to the table we are scanning for - tdu.linkFileAndTable(tableId, stf.getFileName()); + tdu.linkFileAndTable(tableId, fileKey); // add the file size for the table if not already seen for this scan if (allFiles.add(stf)) { // This tracks the file size for individual files for computing shared file statistics // later - tdu.addFileSize(stf.getFileName(), dataFileValue.getSize()); + tdu.addFileSize(fileKey, dataFileValue.getSize()); } } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/metadata/RootGcCandidatesTest.java b/server/base/src/test/java/org/apache/accumulo/server/metadata/RootGcCandidatesTest.java new file mode 100644 index 00000000000..9897ef99c48 --- /dev/null +++ b/server/base/src/test/java/org/apache/accumulo/server/metadata/RootGcCandidatesTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.metadata; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Set; +import java.util.stream.Stream; + +import org.apache.accumulo.core.gc.GcCandidate; +import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.server.gc.GcVolumeUtil; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.Test; + +public class RootGcCandidatesTest { + + @Test + public void externalCandidateRoundTrip() { + StoredTabletFile external = StoredTabletFile.ofExternal(new Path("file:///external/F.rf")); + StoredTabletFile standard = + StoredTabletFile.of(new Path("file:///accumulo/tables/!0/t-0/F.rf")); + + RootGcCandidates candidates = new RootGcCandidates(); + candidates.add(Stream.of(external, standard)); + + RootGcCandidates reloaded = new RootGcCandidates(candidates.toJson()); + Set paths = reloaded.sortedStream().map(GcCandidate::getPath) + .collect(java.util.stream.Collectors.toSet()); + assertEquals(Set.of(GcVolumeUtil.encodeExternalCandidatePath(external.getPath().toString()), + standard.getPath().toString()), paths); + + String externalCandidate = + GcVolumeUtil.encodeExternalCandidatePath(external.getPath().toString()); + reloaded.remove(Stream.of(new GcCandidate(externalCandidate, 1))); + assertEquals(Set.of(standard.getPath().toString()), reloaded.sortedStream() + .map(GcCandidate::getPath).collect(java.util.stream.Collectors.toSet())); + } +} diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java index cab48d722fe..58c0c20d654 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java @@ -43,6 +43,7 @@ import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; +import org.apache.hadoop.fs.Path; import org.easymock.EasyMock; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -132,6 +133,26 @@ public void testMultipleVolumes() throws Exception { EasyMock.verify(client, mockTabletsMetadata); } + @Test + public void testExternalFile() throws Exception { + final ClientContext client = EasyMock.createMock(ClientContext.class); + EasyMock.expect(client.createTableIdToQualifiedNameMap()).andReturn(tableIdToNameMap); + final TabletsMetadata mockTabletsMetadata = mockTabletsMetadata(client, tableId1); + + StoredTabletFile external = + StoredTabletFile.ofExternal(new Path("hdfs://external/imports/F.rf")); + TabletMetadata tablet = TabletMetadata.builder(new KeyExtent(tableId1, null, null)) + .putFile(external, new DataFileValue(1024, 1)).build(); + mockTabletsMetadataIter(mockTabletsMetadata, List.of(tablet).iterator()); + EasyMock.replay(client, mockTabletsMetadata); + + Map,Long> result = TableDiskUsage.getDiskUsage(tableSet(tableId1), client); + + assertEquals(1024, getTotalUsage(result, tableId1)); + assertEquals(1024, result.get(tableNameSet(tableId1))); + EasyMock.verify(client, mockTabletsMetadata); + } + @Test public void testMetadataTable() throws Exception { final ClientContext client = EasyMock.createMock(ClientContext.class); diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java index f13cff3bd1e..f8a3926f8a5 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java @@ -56,6 +56,7 @@ import org.apache.accumulo.core.gc.Reference; import org.apache.accumulo.core.gc.ReferenceFile; import org.apache.accumulo.core.manager.state.tables.TableState; +import org.apache.accumulo.core.metadata.ReferencedTabletFile; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.SystemTables; @@ -304,8 +305,14 @@ public void deleteConfirmedCandidates(SortedMap confirmedDel try { Path fullPath; - Path switchedDelete = VolumeUtil.switchVolume(new Path(delete.getPath()), - VolumeManager.FileType.TABLE, replacements); + boolean externalCandidate = GcVolumeUtil.isExternalCandidatePath(delete.getPath()); + Path deletePath = new Path(GcVolumeUtil.decodeExternalCandidatePath(delete.getPath())); + Path switchedDelete = + (!externalCandidate && (ReferencedTabletFile.isVolumePath(deletePath) + || ReferencedTabletFile.isTabletDirPath(deletePath))) + ? VolumeUtil.switchVolume(deletePath, VolumeManager.FileType.TABLE, + replacements) + : null; if (switchedDelete != null) { // actually replacing the volumes in the metadata table would be tricky because the // entries would be different rows. So it could not be @@ -318,7 +325,7 @@ public void deleteConfirmedCandidates(SortedMap confirmedDel log.debug("Volume replaced {} -> {}", delete.getPath(), switchedDelete); fullPath = ValidationUtil.validate(switchedDelete); } else { - fullPath = new Path(ValidationUtil.validate(delete.getPath())); + fullPath = ValidationUtil.validate(deletePath); } for (Path pathToDel : GcVolumeUtil.expandAllVolumesUri(fs, fullPath)) { @@ -339,10 +346,18 @@ public void deleteConfirmedCandidates(SortedMap confirmedDel // this failure, we still want to remove the metadata entry removeFlag = true; errors.increment(); - String[] parts = pathToDel.toString().split(Constants.HDFS_TABLES_DIR)[1].split("/"); - if (parts.length > 2) { - TableId tableId = TableId.of(parts[1]); - String tabletDir = parts[2]; + if (ReferencedTabletFile.isVolumePath(pathToDel) + || ReferencedTabletFile.isTabletDirPath(pathToDel)) { + TableId tableId; + String tabletDir; + if (ReferencedTabletFile.isTabletDirPath(pathToDel)) { + tableId = TableId.of(pathToDel.getParent().getName()); + tabletDir = pathToDel.getName(); + } else { + var parts = ReferencedTabletFile.parsePath(pathToDel); + tableId = parts.getTableId(); + tabletDir = parts.getTabletDir(); + } TableState tableState = context.getTableManager().getTableState(tableId); if (tableState != TableState.UNKNOWN && tableState != TableState.DELETING) { // clone directories don't always exist @@ -350,9 +365,11 @@ public void deleteConfirmedCandidates(SortedMap confirmedDel log.debug("{} File doesn't exist: {}", fileActionPrefix, pathToDel); } } - } else { + } else if (!deletePath.equals(pathToDel)) { log.warn("{} Delete failed due to invalid file path format: {}", fileActionPrefix, delete.getPath()); + } else { + log.debug("{} File does not exist: {}", fileActionPrefix, pathToDel); } } } @@ -442,6 +459,11 @@ static void minimizeDeletes(SortedMap confirmedDeletes, progressCount++; Map.Entry entry = cdIter.next(); String relPath = entry.getKey(); + if (GcVolumeUtil.isExternalCandidatePath(entry.getValue().getPath())) { + lastDirRel = null; + lastDirAbs = null; + continue; + } Path absPath = new Path(entry.getValue().getPath()); if (SimpleGarbageCollector.isDir(relPath)) { diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java index 462d593cf4e..efc57bdb54b 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java @@ -42,6 +42,8 @@ import org.apache.accumulo.core.metadata.schema.Ample.GcCandidateType; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily; import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.server.gc.GcVolumeUtil; +import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,7 +57,7 @@ public class GarbageCollectionAlgorithm { private static final Logger log = LoggerFactory.getLogger(GarbageCollectionAlgorithm.class); /** - * This method takes a file or directory path and returns a relative path in 1 of 2 forms: + * This method takes a file or directory path and returns a relative path in one of 2 forms: * * 
    *      1- For files: table-id/tablet-directory/filename.rf
@@ -124,11 +126,16 @@ private SortedMap makeRelative(Collection candi
 
     for (GcCandidate candidate : candidates) {
       String relPath;
-      try {
-        relPath = makeRelative(candidate.getPath(), 0);
-      } catch (IllegalArgumentException iae) {
-        log.warn("Ignoring invalid deletion candidate {}", candidate);
-        continue;
+      String path = candidate.getPath();
+      if (GcVolumeUtil.isExternalCandidatePath(path)) {
+        relPath = makeExternalRelative(GcVolumeUtil.decodeExternalCandidatePath(path));
+      } else {
+        try {
+          relPath = makeRelative(path, 0);
+        } catch (IllegalArgumentException iae) {
+          log.warn("Ignoring invalid deletion candidate {}", candidate);
+          continue;
+        }
       }
       ret.put(relPath, candidate);
     }
@@ -136,6 +143,11 @@ private SortedMap makeRelative(Collection candi
     return ret;
   }
 
+  private String makeExternalRelative(String path) {
+    String normalized = new Path(path).toUri().normalize().toString();
+    return "external:" + normalized;
+  }
+
   private void removeCandidatesInUse(GarbageCollectionEnvironment gce,
       SortedMap candidateMap) throws InterruptedException {
 
@@ -161,15 +173,20 @@ private void removeCandidatesInUse(GarbageCollectionEnvironment gce,
           }
         } else {
           String reference = ref.getMetadataPath();
-          if (reference.startsWith("/")) {
-            log.debug("Candidate {} has a relative path, prepend tableId {}", reference,
-                ref.getTableId());
-            reference = "/" + ref.getTableId() + ref.getMetadataPath();
-          } else if (!reference.contains(":") && !reference.startsWith("../")) {
-            throw new RuntimeException("Bad file reference " + reference);
-          }
+          String relativePath;
+          if (ref.isExternal()) {
+            relativePath = makeExternalRelative(reference);
+          } else {
+            if (reference.startsWith("/")) {
+              log.debug("Candidate {} has a relative path, prepend tableId {}", reference,
+                  ref.getTableId());
+              reference = "/" + ref.getTableId() + ref.getMetadataPath();
+            } else if (!reference.contains(":") && !reference.startsWith("../")) {
+              throw new RuntimeException("Bad file reference " + reference);
+            }
 
-          String relativePath = makeRelative(reference, 3);
+            relativePath = makeRelative(reference, 3);
+          }
 
           // WARNING: This line is EXTREMELY IMPORTANT.
           // You MUST REMOVE candidates that are still in use
@@ -183,12 +200,14 @@ private void removeCandidatesInUse(GarbageCollectionEnvironment gce,
             }
           }
 
-          String dir = relativePath.substring(0, relativePath.lastIndexOf('/'));
-          GcCandidate gcT = candidateMap.remove(dir);
-          if (gcT != null) {
-            log.debug("Directory Candidate was still in use by file ref: {}", relativePath);
-            // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created
-            // once.
+          if (!ref.isExternal()) {
+            String dir = relativePath.substring(0, relativePath.lastIndexOf('/'));
+            GcCandidate gcT = candidateMap.remove(dir);
+            if (gcT != null) {
+              log.debug("Directory Candidate was still in use by file ref: {}", relativePath);
+              // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created
+              // once.
+            }
           }
         }
       });
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
index 72cc61e474b..3f56de80bd4 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
@@ -42,9 +42,11 @@
 import org.apache.accumulo.core.gc.Reference;
 import org.apache.accumulo.core.gc.ReferenceFile;
 import org.apache.accumulo.core.manager.state.tables.TableState;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.SystemTables;
 import org.apache.accumulo.core.metadata.schema.Ample;
 import org.apache.accumulo.core.metadata.schema.Ample.GcCandidateType;
+import org.apache.accumulo.server.gc.GcVolumeUtil;
 import org.apache.hadoop.fs.Path;
 import org.junit.jupiter.api.Test;
 
@@ -78,6 +80,10 @@ public GcCandidate addCandidate(String s) {
       return candidate;
     }
 
+    public GcCandidate addExternalCandidate(String path) {
+      return addCandidate(GcVolumeUtil.encodeExternalCandidatePath(path));
+    }
+
     public Long getTimestamp() {
       return timestamp;
     }
@@ -144,6 +150,13 @@ public void addFileReference(String tableId, String endRow, String file) {
       tableIds.add(tid);
     }
 
+    public void addExternalFileReference(String tableId, String endRow, String file) {
+      TableId tid = TableId.of(tableId);
+      references.put(tableId + ":" + endRow + ":" + file,
+          ReferenceFile.forFile(tid, StoredTabletFile.ofExternal(new Path(file))));
+      tableIds.add(tid);
+    }
+
     public void removeFileReference(String tableId, String endRow, String file) {
       references.remove(tableId + ":" + endRow + ":" + file);
       removeLastTableIdRef(TableId.of(tableId));
@@ -167,6 +180,13 @@ public void addScanReference(String tableId, String endRow, String scan) {
       tableIds.add(tid);
     }
 
+    public void addExternalScanReference(String tableId, String endRow, String scan) {
+      TableId tid = TableId.of(tableId);
+      references.put(tableId + ":" + endRow + ":scan:" + scan,
+          ReferenceFile.forScan(tid, StoredTabletFile.ofExternal(new Path(scan))));
+      tableIds.add(tid);
+    }
+
     public void removeScanReference(String tableId, String endRow, String scan) {
       references.remove(tableId + ":" + endRow + ":scan:" + scan);
       removeLastTableIdRef(TableId.of(tableId));
@@ -261,6 +281,35 @@ public void minimalDelete() throws Exception {
     assertCandidateRemoved(gce, GcCandidateType.INUSE, candOne, candTwo, candFour);
   }
 
+  @Test
+  public void externalFileReferencesPreventDeletion() throws Exception {
+    TestGCE gce = new TestGCE();
+    String path = "hdfs://external-nn/imports/F0001.rf";
+    var candidate = gce.addExternalCandidate(path);
+    gce.addExternalFileReference("4", null, path);
+    gce.addExternalScanReference("4", null, path);
+
+    GarbageCollectionAlgorithm gca = new GarbageCollectionAlgorithm();
+    gca.collect(gce);
+
+    assertFileDeleted(gce);
+    assertCandidateRemoved(gce, GcCandidateType.INUSE, candidate);
+
+    candidate = gce.addExternalCandidate(path);
+    gce.removeFileReference("4", null, path);
+    gca.collect(gce);
+
+    assertFileDeleted(gce);
+    assertTrue(gce.candidates.contains(candidate));
+    assertNoCandidatesRemoved(gce);
+
+    gce.removeScanReference("4", null, path);
+    gca.collect(gce);
+
+    assertFileDeleted(gce, candidate);
+    assertNoCandidatesRemoved(gce);
+  }
+
   @Test
   public void testBasic() throws Exception {
     TestGCE gce = new TestGCE();
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
index 9cc2e7f8b22..d64d3257bfd 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/SimpleGarbageCollectorTest.java
@@ -51,6 +51,7 @@
 import org.apache.accumulo.server.ServerContext;
 import org.apache.accumulo.server.fs.VolumeManager;
 import org.apache.accumulo.server.gc.AllVolumesDirectory;
+import org.apache.accumulo.server.gc.GcVolumeUtil;
 import org.apache.accumulo.server.security.SystemCredentials;
 import org.apache.hadoop.fs.Path;
 import org.junit.jupiter.api.BeforeEach;
@@ -204,4 +205,18 @@ public void testMinimizeDeletes() {
             new GcCandidate("hdfs://nn1/accumulo/tables/5b/t-0004/F0001.rf", 9L)),
         processedDeletes);
   }
+
+  @Test
+  public void testMinimizeDeletesExternalCandidate() {
+    String path = "hdfs://external-nn/imports/F0001.rf";
+    String candidatePath = GcVolumeUtil.encodeExternalCandidatePath(path);
+    TreeMap candidates = new TreeMap<>();
+    candidates.put("external:" + path, new GcCandidate(candidatePath, 1L));
+
+    GCRun.minimizeDeletes(candidates, new ArrayList<>(), createMock(VolumeManager.class), log,
+        Duration.ofMinutes(1));
+
+    assertEquals(1, candidates.size());
+    assertEquals(candidatePath, candidates.get("external:" + path).getPath());
+  }
 }
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java
index 33b1d43d263..ea737d5f911 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java
@@ -138,7 +138,7 @@ private Optional deleteTabletFiles(Ample ample, FateId fateId) {
               // If the existing file is disjoint then later we will delete if the file is not part
               // of the newFiles set which means it is disjoint with all ranges
               if (fenced != null) {
-                final StoredTabletFile newFile = StoredTabletFile.of(existing.getPath(), fenced);
+                final StoredTabletFile newFile = existing.withRange(fenced);
                 log.trace("{} Adding new file {} with range {}", fateId, newFile.getMetadataPath(),
                     newFile.getRange());
 
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
index de3484c4fa4..5967d785087 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
@@ -291,11 +291,11 @@ static void validateTablet(TabletMetadata tabletMeta, FateId fateId, TabletOpera
   /**
    * Fence a file to a tablets data range.
    */
-  private static StoredTabletFile fenceFile(KeyExtent extent, StoredTabletFile file) {
+  static StoredTabletFile fenceFile(KeyExtent extent, StoredTabletFile file) {
     Range fenced = extent.toDataRange();
     // Clip range if exists
     fenced = file.hasRange() ? file.getRange().clip(fenced) : fenced;
-    return StoredTabletFile.of(file.getPath(), fenced);
+    return file.withRange(fenced);
   }
 
   @Override
diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java
index 002ffbeea80..b221fb3e282 100644
--- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java
+++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java
@@ -71,6 +71,7 @@
 import org.apache.accumulo.core.fate.FateInstanceType;
 import org.apache.accumulo.core.gc.ReferenceFile;
 import org.apache.accumulo.core.lock.ServiceLock;
+import org.apache.accumulo.core.metadata.ExternalTabletFile;
 import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.SuspendingTServer;
 import org.apache.accumulo.core.metadata.TServerInstance;
@@ -94,6 +95,7 @@
 import org.apache.accumulo.server.gc.AllVolumesDirectory;
 import org.apache.accumulo.server.metadata.ConditionalTabletMutatorImpl;
 import org.apache.accumulo.server.metadata.ConditionalTabletsMutatorImpl;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.easymock.EasyMock;
 import org.junit.jupiter.api.Test;
@@ -132,6 +134,18 @@ public void checkColumns() {
     }
   }
 
+  @Test
+  public void fenceExternalFilePreservesExternalMarker() {
+    Path path = new Path("hdfs://external-nn/imports/F0001.rf");
+    StoredTabletFile external = ExternalTabletFile.of(path).insert();
+
+    StoredTabletFile fenced = MergeTablets.fenceFile(ke3, external);
+
+    assertTrue(fenced.isExternal());
+    assertEquals(path, fenced.getPath());
+    assertEquals(ke3.toDataRange(), fenced.getRange());
+  }
+
   @Test
   public void testManyColumns() throws Exception {
 
diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
index 32afdfe5605..47ea5b42b9e 100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
@@ -22,6 +22,8 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -42,17 +44,25 @@
 import org.apache.accumulo.core.data.InstanceId;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.ExternalTabletFile;
 import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.SystemTables;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.server.init.Initialize;
 import org.apache.accumulo.server.util.adminCommand.StopAll;
+import org.apache.accumulo.test.util.Wait;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.junit.jupiter.api.Test;
 
@@ -209,6 +219,79 @@ public void testRemoveVolumes() throws Exception {
     }
   }
 
+  @Test
+  public void testExternalFileSurvivesVolumeReplacement() throws Exception {
+    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
+      String tableName = getUniqueNames(1)[0];
+      verifyVolumesUsed(client, tableName, false, v1, v2);
+      TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName));
+
+      TabletMetadata tablet;
+      StoredTabletFile originalFile;
+      DataFileValue fileValue;
+      try (TabletsMetadata tablets = getServerContext().getAmple().readTablets().forTable(tableId)
+          .fetch(ColumnType.FILES).build()) {
+        tablet = tablets.iterator().next();
+        var firstFile = tablet.getFilesMap().entrySet().iterator().next();
+        originalFile = firstFile.getKey();
+        fileValue = firstFile.getValue();
+      }
+
+      Path externalPath =
+          new Path(new Path(v1.getParent(), "external-files"), getUniqueNames(1)[0] + ".rf");
+      FileSystem fs = externalPath.getFileSystem(getServerContext().getHadoopConf());
+      assertTrue(fs.mkdirs(externalPath.getParent()) || fs.exists(externalPath.getParent()));
+      IOUtils.copyBytes(fs.open(originalFile.getPath()), fs.create(externalPath),
+          getServerContext().getHadoopConf(), true);
+
+      ExternalTabletFile externalFile =
+          ExternalTabletFile.of(externalPath, originalFile.getRange());
+      getServerContext().getAmple().mutateTablet(tablet.getExtent()).deleteFile(originalFile)
+          .putFile(externalFile, fileValue).mutate();
+
+      assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor());
+      cluster.stop();
+
+      java.nio.file.Path v1f = java.nio.file.Path.of(v1.toUri());
+      java.nio.file.Path v8f = java.nio.file.Path.of(v1.getParent().toUri()).resolve("v8");
+      Files.move(v1f, v8f, StandardCopyOption.ATOMIC_MOVE);
+      Path v8 = new Path(v8f.toUri());
+
+      java.nio.file.Path v2f = java.nio.file.Path.of(v2.toUri());
+      java.nio.file.Path v9f = java.nio.file.Path.of(v2.getParent().toUri()).resolve("v9");
+      Files.move(v2f, v9f, StandardCopyOption.ATOMIC_MOVE);
+      Path v9 = new Path(v9f.toUri());
+
+      updateConfig(config -> {
+        config.setProperty(Property.INSTANCE_VOLUMES.getKey(), v8 + "," + v9);
+        config.setProperty(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey(),
+            v1 + " " + v8 + "," + v2 + " " + v9);
+      });
+      cluster.start();
+
+      Wait.waitFor(() -> {
+        boolean externalFound = false;
+        try (TabletsMetadata tablets = getServerContext().getAmple().readTablets().forTable(tableId)
+            .fetch(ColumnType.FILES).build()) {
+          for (TabletMetadata currentTablet : tablets) {
+            for (StoredTabletFile file : currentTablet.getFiles()) {
+              if (file.isExternal()) {
+                externalFound |= file.getPath().equals(externalPath);
+              } else if (file.getMetadataPath().startsWith(v1.toString())
+                  || file.getMetadataPath().startsWith(v2.toString())) {
+                return false;
+              }
+            }
+          }
+        }
+        return externalFound;
+      }, 30000, 250, "External reference was lost or volume paths were not replaced");
+
+      verifyData(expected, client.createScanner(tableName, Authorizations.EMPTY));
+      assertTrue(fs.exists(externalPath));
+    }
+  }
+
   @Test
   public void testDirtyReplaceVolumes() throws Exception {
     try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java
index 5bec14ad872..c8ce6014b4c 100644
--- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java
+++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java
@@ -98,13 +98,17 @@
 import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.lock.ServiceLock;
+import org.apache.accumulo.core.metadata.ExternalTabletFile;
 import org.apache.accumulo.core.metadata.ReferencedTabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.SystemTables;
 import org.apache.accumulo.core.metadata.schema.CompactionMetadata;
 import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
 import org.apache.accumulo.core.spi.compaction.CompactionKind;
 import org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher;
 import org.apache.accumulo.manager.tableOps.AbstractFateOperation;
@@ -118,7 +122,9 @@
 import org.apache.accumulo.test.harness.SharedMiniClusterBase;
 import org.apache.accumulo.test.util.Wait;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -526,6 +532,99 @@ public void testCompactionAndCompactorDies() throws Exception {
 
   }
 
+  @Test
+  public void testExternalCompactionWithMissingExternalInput() throws Exception {
+    String tableName = getUniqueNames(1)[0];
+    try (AccumuloClient client =
+        Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
+      createTable(client, tableName, "cs1");
+      writeData(client, tableName);
+
+      TableId tableId = getCluster().getServerContext().getTableId(tableName);
+      TabletMetadata tablet;
+      try (TabletsMetadata tablets = getCluster().getServerContext().getAmple().readTablets()
+          .forTable(tableId).fetch(ColumnType.FILES).build()) {
+        tablet = tablets.iterator().next();
+      }
+
+      var fileEntry = tablet.getFilesMap().entrySet().iterator().next();
+      StoredTabletFile original = fileEntry.getKey();
+      DataFileValue fileValue = fileEntry.getValue();
+      FileSystem fs = getCluster().getFileSystem();
+      Path externalDir = new Path(new Path(fs.getHomeDirectory(), "external-compaction-inputs"),
+          getUniqueNames(1)[0]);
+      assertTrue(fs.mkdirs(externalDir) || fs.exists(externalDir));
+      Path externalPath = new Path(externalDir, "input.rf");
+      IOUtils.copyBytes(fs.open(original.getPath()), fs.create(externalPath),
+          getCluster().getServerContext().getHadoopConf(), true);
+
+      ExternalTabletFile externalFile = ExternalTabletFile.of(externalPath, original.getRange());
+      StoredTabletFile storedExternal = externalFile.insert();
+      getCluster().getServerContext().getAmple().mutateTablet(tablet.getExtent())
+          .deleteFile(original).putFile(externalFile, fileValue).mutate();
+
+      // The external file disappears after the tablet metadata has recorded the reference.
+      assertTrue(fs.delete(externalPath, false));
+
+      getCluster().getClusterControl().stopAllServers(ServerType.COMPACTOR);
+      getCluster().getClusterControl().start(ServerType.COMPACTOR, null, 1,
+          ExternalDoNothingCompactor.class);
+      boolean compactionSubmitted = false;
+      try {
+        compact(client, tableName, 2, GROUP1, false);
+        compactionSubmitted = true;
+        Set ecids = ExternalCompactionTestUtils
+            .waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tableId);
+        assertFalse(ecids.isEmpty());
+
+        boolean externalInputFound = false;
+        try (TabletsMetadata tablets = getCluster().getServerContext().getAmple().readTablets()
+            .forTable(tableId).fetch(ColumnType.ECOMP).build()) {
+          for (TabletMetadata currentTablet : tablets) {
+            for (CompactionMetadata metadata : currentTablet.getExternalCompactions().values()) {
+              if (metadata.getJobFiles().contains(storedExternal)) {
+                externalInputFound = true;
+              }
+            }
+          }
+        }
+        assertTrue(externalInputFound,
+            "External compaction metadata should retain its missing external input reference");
+
+        assertThrows(RuntimeException.class, () -> {
+          try (Scanner scanner = client.createScanner(tableName)) {
+            scanner.iterator().hasNext();
+          }
+        }, "A scan should fail when the external compaction input is missing");
+
+        client.tableOperations().cancelCompaction(tableName);
+        compactionSubmitted = false;
+        ExternalCompactionTestUtils
+            .confirmCompactionsNoLongerRunning(getCluster().getServerContext(), ecids);
+        Wait.waitFor(() -> {
+          try (TabletsMetadata tablets = getCluster().getServerContext().getAmple().readTablets()
+              .forTable(tableId).fetch(ColumnType.ECOMP).build()) {
+            for (TabletMetadata currentTablet : tablets) {
+              if (!currentTablet.getExternalCompactions().isEmpty()) {
+                return false;
+              }
+            }
+            return true;
+          }
+        });
+      } finally {
+        try {
+          if (compactionSubmitted) {
+            client.tableOperations().cancelCompaction(tableName);
+          }
+        } finally {
+          getCluster().getClusterControl().stopAllServers(ServerType.COMPACTOR);
+          getCluster().getClusterControl().start(ServerType.COMPACTOR);
+        }
+      }
+    }
+  }
+
   @Test
   public void testManytablets() throws Exception {
     String table1 = this.getUniqueNames(1)[0];
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index 69f1a482e7f..efd02563a26 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -23,6 +23,7 @@
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
@@ -55,12 +56,17 @@
 import org.apache.accumulo.core.lock.ServiceLock;
 import org.apache.accumulo.core.lock.ServiceLockData;
 import org.apache.accumulo.core.lock.ServiceLockData.ThriftService;
+import org.apache.accumulo.core.metadata.ExternalTabletFile;
 import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.SystemTables;
 import org.apache.accumulo.core.metadata.schema.Ample;
 import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection.SkewedKeyValue;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.TablePermission;
 import org.apache.accumulo.gc.SimpleGarbageCollector;
@@ -70,12 +76,17 @@
 import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
 import org.apache.accumulo.miniclusterImpl.ProcessNotFoundException;
 import org.apache.accumulo.miniclusterImpl.ProcessReference;
+import org.apache.accumulo.server.gc.GcVolumeUtil;
 import org.apache.accumulo.test.TestIngest;
 import org.apache.accumulo.test.VerifyIngest;
 import org.apache.accumulo.test.VerifyIngest.VerifyParams;
+import org.apache.accumulo.test.util.Wait;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NoNodeException;
@@ -171,6 +182,239 @@ public void gcTest() throws Exception {
     }
   }
 
+  @Test
+  public void externalTabletFileIsRetainedWhileReferencedAndDeletedAfterward() throws Exception {
+    killMacGc();
+    String table = getUniqueNames(1)[0];
+
+    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
+      client.tableOperations().create(table);
+      try (BatchWriter writer = client.createBatchWriter(table)) {
+        Mutation mutation = new Mutation("external-row");
+        mutation.put("cf", "cq", "value");
+        writer.addMutation(mutation);
+      }
+      client.tableOperations().flush(table, null, null, true);
+
+      TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(table));
+      Ample ample = getServerContext().getAmple();
+      TabletMetadata tablet;
+      try (TabletsMetadata tablets =
+          ample.readTablets().forTable(tableId).fetch(ColumnType.FILES).build()) {
+        tablet = Iterators.getOnlyElement(tablets.iterator());
+      }
+
+      var fileEntry = tablet.getFilesMap().entrySet().iterator().next();
+      StoredTabletFile original = fileEntry.getKey();
+      DataFileValue fileValue = fileEntry.getValue();
+      FileSystem fs = cluster.getFileSystem();
+      Path externalPath = new Path(new Path(fs.getHomeDirectory(), "external-tablet-files"),
+          getUniqueNames(1)[0] + ".rf");
+      assertTrue(fs.mkdirs(externalPath.getParent()) || fs.exists(externalPath.getParent()));
+      IOUtils.copyBytes(fs.open(original.getPath()), fs.create(externalPath),
+          getServerContext().getHadoopConf(), true);
+
+      ExternalTabletFile external = ExternalTabletFile.of(externalPath, original.getRange());
+      StoredTabletFile storedExternal = external.insert();
+      ample.mutateTablet(tablet.getExtent()).deleteFile(original).putFile(external, fileValue)
+          .mutate();
+
+      try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
+        var entry = Iterators.getOnlyElement(scanner.iterator());
+        assertEquals("external-row", entry.getKey().getRow().toString());
+        assertEquals("value", entry.getValue().toString());
+      }
+
+      ProcessInfo gc = cluster.exec(SimpleGarbageCollector.class);
+      try {
+        DataLevel level = DataLevel.of(tableId);
+        String externalCandidatePath =
+            GcVolumeUtil.encodeExternalCandidatePath(externalPath.toString());
+
+        ample.putGcCandidates(tableId, List.of(storedExternal));
+        Wait.waitFor(() -> !hasGcCandidate(ample, level, externalCandidatePath), 30000, 250,
+            "GC did not process the in-use external file candidate");
+        assertTrue(fs.exists(externalPath), "GC deleted an external file with a live reference");
+
+        ample.mutateTablet(tablet.getExtent()).deleteFile(storedExternal).mutate();
+        ample.putGcCandidates(tableId, List.of(storedExternal));
+        Wait.waitFor(() -> !fs.exists(externalPath), 30000, 250,
+            "GC did not delete the unreferenced external file");
+      } finally {
+        gc.getProcess().destroy();
+      }
+    }
+  }
+
+  @Test
+  public void deletingTableWithExternalFileQueuesItForGc() throws Exception {
+    killMacGc();
+    String table = getUniqueNames(1)[0];
+
+    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
+      client.tableOperations().create(table);
+      try (BatchWriter writer = client.createBatchWriter(table)) {
+        Mutation mutation = new Mutation("external-row");
+        mutation.put("cf", "cq", "value");
+        writer.addMutation(mutation);
+      }
+      client.tableOperations().flush(table, null, null, true);
+
+      TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(table));
+      Ample ample = getServerContext().getAmple();
+      TabletMetadata tablet;
+      try (TabletsMetadata tablets = ample.readTablets().forTable(tableId)
+          .fetch(ColumnType.FILES, ColumnType.PREV_ROW).build()) {
+        tablet = Iterators.getOnlyElement(tablets.iterator());
+      }
+
+      StoredTabletFile original = tablet.getFiles().iterator().next();
+      FileSystem fs = cluster.getFileSystem();
+      Path externalPath = new Path(new Path(fs.getHomeDirectory(), "external-tablet-files"),
+          getUniqueNames(1)[0] + ".rf");
+      assertTrue(fs.mkdirs(externalPath.getParent()) || fs.exists(externalPath.getParent()));
+      IOUtils.copyBytes(fs.open(original.getPath()), fs.create(externalPath),
+          getServerContext().getHadoopConf(), true);
+
+      StoredTabletFile storedExternal =
+          ExternalTabletFile.of(externalPath, original.getRange()).insert();
+      ample.mutateTablet(tablet.getExtent()).deleteFile(original)
+          .putFile(storedExternal, tablet.getFilesMap().get(original)).mutate();
+
+      String candidatePath = GcVolumeUtil.encodeExternalCandidatePath(externalPath.toString());
+      client.tableOperations().delete(table);
+
+      DataLevel level = DataLevel.of(tableId);
+      assertTrue(hasGcCandidate(ample, level, candidatePath),
+          "Table deletion did not enqueue an external file candidate");
+
+      ProcessInfo gc = cluster.exec(SimpleGarbageCollector.class);
+      try {
+        Wait.waitFor(() -> !fs.exists(externalPath), 30000, 250,
+            "GC did not delete the external file after its table was deleted");
+      } finally {
+        gc.getProcess().destroy();
+      }
+    }
+  }
+
+  @Test
+  public void scansAndGcHandleMissingAndInaccessibleExternalFiles() throws Exception {
+    killMacGc();
+    String table = getUniqueNames(1)[0];
+
+    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
+      client.tableOperations().create(table);
+      try (BatchWriter writer = client.createBatchWriter(table)) {
+        Mutation mutation = new Mutation("external-row");
+        mutation.put("cf", "cq", "value");
+        writer.addMutation(mutation);
+      }
+      client.tableOperations().flush(table, null, null, true);
+
+      TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(table));
+      Ample ample = getServerContext().getAmple();
+      TabletMetadata tablet;
+      try (TabletsMetadata tablets =
+          ample.readTablets().forTable(tableId).fetch(ColumnType.FILES).build()) {
+        tablet = Iterators.getOnlyElement(tablets.iterator());
+      }
+
+      var fileEntry = tablet.getFilesMap().entrySet().iterator().next();
+      StoredTabletFile original = fileEntry.getKey();
+      DataFileValue fileValue = fileEntry.getValue();
+      FileSystem fs = cluster.getFileSystem();
+      Path externalDir =
+          new Path(new Path(fs.getHomeDirectory(), "external-tablet-files"), getUniqueNames(1)[0]);
+      assertTrue(fs.mkdirs(externalDir) || fs.exists(externalDir));
+
+      Path missingPath = new Path(externalDir, "missing.rf");
+      IOUtils.copyBytes(fs.open(original.getPath()), fs.create(missingPath),
+          getServerContext().getHadoopConf(), true);
+      ExternalTabletFile missingExternal = ExternalTabletFile.of(missingPath, original.getRange());
+      StoredTabletFile storedMissing = missingExternal.insert();
+      ample.mutateTablet(tablet.getExtent()).deleteFile(original)
+          .putFile(missingExternal, fileValue).mutate();
+
+      assertTrue(fs.delete(missingPath, false));
+      assertThrows(RuntimeException.class, () -> {
+        try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
+          Iterators.size(scanner.iterator());
+        }
+      }, "A scan should fail when a referenced external file no longer exists");
+
+      ProcessInfo gc = cluster.exec(SimpleGarbageCollector.class);
+      try {
+        DataLevel level = DataLevel.of(tableId);
+        String missingCandidatePath =
+            GcVolumeUtil.encodeExternalCandidatePath(missingPath.toString());
+        ample.putGcCandidates(tableId, List.of(storedMissing));
+        Wait.waitFor(() -> !hasGcCandidate(ample, level, missingCandidatePath), 30000, 250,
+            "GC did not process a missing external file that is still referenced");
+
+        Path unreadablePath = new Path(externalDir, "unreadable.rf");
+        IOUtils.copyBytes(fs.open(original.getPath()), fs.create(unreadablePath),
+            getServerContext().getHadoopConf(), true);
+        ExternalTabletFile unreadableExternal =
+            ExternalTabletFile.of(unreadablePath, original.getRange());
+        StoredTabletFile storedUnreadable = unreadableExternal.insert();
+        ample.mutateTablet(tablet.getExtent()).deleteFile(storedMissing)
+            .putFile(unreadableExternal, fileValue).mutate();
+
+        FsPermission originalFilePermission = fs.getFileStatus(unreadablePath).getPermission();
+        fs.setPermission(unreadablePath, FsPermission.valueOf("----------"));
+        try {
+          assertThrows(RuntimeException.class, () -> {
+            try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
+              Iterators.size(scanner.iterator());
+            }
+          }, "A scan should fail when a referenced external file becomes unreadable");
+        } finally {
+          fs.setPermission(unreadablePath, originalFilePermission);
+        }
+
+        try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
+          var entry = Iterators.getOnlyElement(scanner.iterator());
+          assertEquals("external-row", entry.getKey().getRow().toString());
+          assertEquals("value", entry.getValue().toString());
+        }
+
+        ample.mutateTablet(tablet.getExtent()).deleteFile(storedUnreadable).mutate();
+        String unreadableCandidatePath =
+            GcVolumeUtil.encodeExternalCandidatePath(unreadablePath.toString());
+        FsPermission originalDirectoryPermission = fs.getFileStatus(externalDir).getPermission();
+        try {
+          // Removing write permission after the metadata reference was recorded prevents GC from
+          // deleting the file after its tablet reference is removed.
+          fs.setPermission(externalDir, FsPermission.valueOf("dr-x------"));
+          ample.putGcCandidates(tableId, List.of(storedUnreadable));
+          Thread.sleep(SECONDS.toMillis(5));
+          assertTrue(fs.exists(unreadablePath),
+              "GC should retain an external file when permissions prevent deletion");
+          assertTrue(hasGcCandidate(ample, level, unreadableCandidatePath),
+              "GC should retain the candidate when it cannot delete the external file");
+        } finally {
+          fs.setPermission(externalDir, originalDirectoryPermission);
+        }
+
+        Wait.waitFor(() -> !fs.exists(unreadablePath), 30000, 250,
+            "GC did not retry deleting an external file after permissions were restored");
+      } finally {
+        gc.getProcess().destroy();
+      }
+    }
+  }
+
+  private boolean hasGcCandidate(Ample ample, DataLevel level, String path) {
+    Iterator candidates = ample.getGcCandidates(level);
+    while (candidates.hasNext()) {
+      if (path.equals(candidates.next().getPath())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   @Test
   public void gcLotsOfCandidatesIT() throws Exception {
     killMacGc();